From 4a18f4325362fb8f832c76531c50f0c309bcbabf Mon Sep 17 00:00:00 2001 From: Jennifer Taylor Date: Mon, 31 May 2021 18:15:10 +0000 Subject: [PATCH] Check in environment flag to use experimental mypyc compiler instead of cython for some modules. --- setup.py | 233 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 137 insertions(+), 96 deletions(-) diff --git a/setup.py b/setup.py index 2701c71..c81f41d 100644 --- a/setup.py +++ b/setup.py @@ -4,6 +4,10 @@ from setuptools import Command, setup def extensions(): + # None of these are required for operating any of the utilities found in this repo. + # They are all present for speed. If you cannot compile arbitrary code or cython, + # run setup.py with "PURE_PYTHON=1" environment variable defined to skip compiling + # extensions. Note that the pure python code will run slower. if 'PURE_PYTHON' in os.environ: # We've been asked not to compile extensions. return [] @@ -11,103 +15,144 @@ def extensions(): from setuptools import Extension from Cython.Build import cythonize - return [ - *cythonize( + cython_only_code = [ + # Alternative, orders of magnitude faster, memory-unsafe version of + # LZ77 which drastically speeds up packet processing time. + Extension( + "bemani.protocol.lz77cpp", [ - # Hot code for anything constructing or parsing a remote game packet. - Extension( - "bemani.protocol.binary", - [ - "bemani/protocol/binary.py", - ] - ), - # Even though we have a C++ implementation of this, some of the code - # is still used as a wrapper to the C++ implementation and it is very - # hot code (almost every packet touches this). - Extension( - "bemani.protocol.lz77", - [ - "bemani/protocol/lz77.py", - ] - ), - # Alternative, orders of magnitude faster, memory-unsafe version of - # LZ77 which drastically speeds up packet processing time. - Extension( - "bemani.protocol.lz77cpp", - [ - "bemani/protocol/lz77cpp.cxx", - ], - language="c++", - extra_compile_args=["-std=c++14"], - extra_link_args=["-std=c++14"], - ), - # Every single backend service uses this class for construction and - # parsing, so compiling this makes sense. - Extension( - "bemani.protocol.node", - [ - "bemani/protocol/node.py", - ] - ), - # This is the top-level protocol marshall which gets touched at least - # once per packet, so its worth it to squeeze more speed out of this. - Extension( - "bemani.protocol.protocol", - [ - "bemani/protocol/protocol.py", - ] - ), - # This is used to implement a convenient way of parsing/creating binary - # data and it is memory-safe accessses of bytes so it is necessarily - # a bottleneck. - Extension( - "bemani.protocol.stream", - [ - "bemani/protocol/stream.py", - ] - ), - # This gets used less frequently (only on the oldest games) but it is - # still worth it to get a bit of a speed boost by compiling. - Extension( - "bemani.protocol.xml", - [ - "bemani/protocol/xml.py", - ] - ), - # This is a memory-unsafe, orders of magnitude faster threaded implementation - # of the pure python blend code which takes rendering rough animations down - # from over an hour to around a minute. - Extension( - "bemani.format.afp.blend.blendcpp", - [ - "bemani/format/afp/blend/blendcpp.pyx", - "bemani/format/afp/blend/blendcppimpl.cxx", - ], - language="c++", - extra_compile_args=["-std=c++14"], - extra_link_args=["-std=c++14"], - ), - # These types include operations such as matrix math and color conversion so - # it is worth it to speed this up when rendering animations. - Extension( - "bemani.format.afp.types.generic", - [ - "bemani/format/afp/types/generic.py", - ] - ), - # DXT is slow enough that it might be worth it to write a C++ implementation of - # this at some point, but for now we squeeze a bit of speed out of this by compiling. - Extension( - "bemani.format.dxt", - [ - "bemani/format/dxt.py", - ] - ), + "bemani/protocol/lz77cpp.cxx", ], - language_level=3, + language="c++", + extra_compile_args=["-std=c++14"], + extra_link_args=["-std=c++14"], + ), + # This is a memory-unsafe, orders of magnitude faster threaded implementation + # of the pure python blend code which takes rendering rough animations down + # from over an hour to around a minute. + Extension( + "bemani.format.afp.blend.blendcpp", + [ + "bemani/format/afp/blend/blendcpp.pyx", + "bemani/format/afp/blend/blendcppimpl.cxx", + ], + language="c++", + extra_compile_args=["-std=c++14"], + extra_link_args=["-std=c++14"], ), ] + if 'EXPERIMENTAL_MYPYC_COMPILER' in os.environ: + from mypyc.build import mypycify + + return [ + *mypycify( + [ + # List of modules that works as compiled mypyc code. + "bemani/protocol", + "bemani/common", + ], + ), + *cythonize( + [ + # Always include code that must be compiled with cython. + *cython_only_code, + # The format module is not ready for mypyc compilation yet, there are some bugs in + # the compiler that prevent us from using it. + Extension( + "bemani.format.dxt", + [ + "bemani/format/dxt.py", + ] + ), + # The types module is not ready for mypyc compilation yet, there are some bugs in + # the compiler that prevent us from using it. + Extension( + "bemani.format.afp.types.generic", + [ + "bemani/format/afp/types/generic.py", + ] + ), + ], + language_level=3, + ), + ] + else: + return [ + *cythonize( + [ + # Always include code that must be compiled with cython. + *cython_only_code, + # Hot code for anything constructing or parsing a remote game packet. + Extension( + "bemani.protocol.binary", + [ + "bemani/protocol/binary.py", + ] + ), + # Even though we have a C++ implementation of this, some of the code + # is still used as a wrapper to the C++ implementation and it is very + # hot code (almost every packet touches this). + Extension( + "bemani.protocol.lz77", + [ + "bemani/protocol/lz77.py", + ] + ), + # Every single backend service uses this class for construction and + # parsing, so compiling this makes sense. + Extension( + "bemani.protocol.node", + [ + "bemani/protocol/node.py", + ] + ), + # This is the top-level protocol marshall which gets touched at least + # once per packet, so its worth it to squeeze more speed out of this. + Extension( + "bemani.protocol.protocol", + [ + "bemani/protocol/protocol.py", + ] + ), + # This is used to implement a convenient way of parsing/creating binary + # data and it is memory-safe accessses of bytes so it is necessarily + # a bottleneck. + Extension( + "bemani.protocol.stream", + [ + "bemani/protocol/stream.py", + ] + ), + # This gets used less frequently (only on the oldest games) but it is + # still worth it to get a bit of a speed boost by compiling. + Extension( + "bemani.protocol.xml", + [ + "bemani/protocol/xml.py", + ] + ), + # These types include operations such as matrix math and color conversion so + # it is worth it to speed this up when rendering animations. + Extension( + "bemani.format.afp.types.generic", + [ + "bemani/format/afp/types/generic.py", + ] + ), + # DXT is slow enough that it might be worth it to write a C++ implementation of + # this at some point, but for now we squeeze a bit of speed out of this by compiling. + Extension( + "bemani.format.dxt", + [ + "bemani/format/dxt.py", + ] + ), + ], + language_level=3, + ), + ] + class CleanExtCommand(Command): description = 'Clean all compiled python extensions from the current directory.' @@ -204,10 +249,6 @@ setup( install_requires=[ req for req in open('requirements.txt').read().split('\n') if len(req) > 0 ], - # None of these are required for operating any of the utilities found in this repo. - # They are all present for speed. If you cannot compile arbitrary code or cython, - # remove the ext_modules line and run setuptools again. Everything should work, but - # it will run slower. ext_modules=extensions(), cmdclass={ 'clean_ext': CleanExtCommand,