Cool_Tools/bemaniutils
1
0
Fork 0
mirror of synced 2024-11-27 23:50:47 +01:00
Code Releases Activity

Check in environment flag to use experimental mypyc compiler instead of cython for some modules.

Browse Source
This commit is contained in:
Jennifer Taylor 2021-05-31 18:15:10 +00:00
parent b1b8b67fe3
commit 4a18f43253
1 changed files with 137 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

233
setup.py
View File

@ -4,6 +4,10 @@ from setuptools import Command, setup
def extensions():
# None of these are required for operating any of the utilities found in this repo.
# They are all present for speed. If you cannot compile arbitrary code or cython,
# run setup.py with "PURE_PYTHON=1" environment variable defined to skip compiling
# extensions. Note that the pure python code will run slower.
if 'PURE_PYTHON' in os.environ:
# We've been asked not to compile extensions.
return []
@ -11,103 +15,144 @@ def extensions():
from setuptools import Extension
from Cython.Build import cythonize
return [
*cythonize(
cython_only_code = [
# Alternative, orders of magnitude faster, memory-unsafe version of
# LZ77 which drastically speeds up packet processing time.
Extension(
"bemani.protocol.lz77cpp",
[
# Hot code for anything constructing or parsing a remote game packet.
Extension(
"bemani.protocol.binary",
[
"bemani/protocol/binary.py",
]
),
# Even though we have a C++ implementation of this, some of the code
# is still used as a wrapper to the C++ implementation and it is very
# hot code (almost every packet touches this).
Extension(
"bemani.protocol.lz77",
[
"bemani/protocol/lz77.py",
]
),
# Alternative, orders of magnitude faster, memory-unsafe version of
# LZ77 which drastically speeds up packet processing time.
Extension(
"bemani.protocol.lz77cpp",
[
"bemani/protocol/lz77cpp.cxx",
],
language="c++",
extra_compile_args=["-std=c++14"],
extra_link_args=["-std=c++14"],
),
# Every single backend service uses this class for construction and
# parsing, so compiling this makes sense.
Extension(
"bemani.protocol.node",
[
"bemani/protocol/node.py",
]
),
# This is the top-level protocol marshall which gets touched at least
# once per packet, so its worth it to squeeze more speed out of this.
Extension(
"bemani.protocol.protocol",
[
"bemani/protocol/protocol.py",
]
),
# This is used to implement a convenient way of parsing/creating binary
# data and it is memory-safe accessses of bytes so it is necessarily
# a bottleneck.
Extension(
"bemani.protocol.stream",
[
"bemani/protocol/stream.py",
]
),
# This gets used less frequently (only on the oldest games) but it is
# still worth it to get a bit of a speed boost by compiling.
Extension(
"bemani.protocol.xml",
[
"bemani/protocol/xml.py",
]
),
# This is a memory-unsafe, orders of magnitude faster threaded implementation
# of the pure python blend code which takes rendering rough animations down
# from over an hour to around a minute.
Extension(
"bemani.format.afp.blend.blendcpp",
[
"bemani/format/afp/blend/blendcpp.pyx",
"bemani/format/afp/blend/blendcppimpl.cxx",
],
language="c++",
extra_compile_args=["-std=c++14"],
extra_link_args=["-std=c++14"],
),
# These types include operations such as matrix math and color conversion so
# it is worth it to speed this up when rendering animations.
Extension(
"bemani.format.afp.types.generic",
[
"bemani/format/afp/types/generic.py",
]
),
# DXT is slow enough that it might be worth it to write a C++ implementation of
# this at some point, but for now we squeeze a bit of speed out of this by compiling.
Extension(
"bemani.format.dxt",
[
"bemani/format/dxt.py",
]
),
"bemani/protocol/lz77cpp.cxx",
],
language_level=3,
language="c++",
extra_compile_args=["-std=c++14"],
extra_link_args=["-std=c++14"],
),
# This is a memory-unsafe, orders of magnitude faster threaded implementation
# of the pure python blend code which takes rendering rough animations down
# from over an hour to around a minute.
Extension(
"bemani.format.afp.blend.blendcpp",
[
"bemani/format/afp/blend/blendcpp.pyx",
"bemani/format/afp/blend/blendcppimpl.cxx",
],
language="c++",
extra_compile_args=["-std=c++14"],
extra_link_args=["-std=c++14"],
),
]
if 'EXPERIMENTAL_MYPYC_COMPILER' in os.environ:
from mypyc.build import mypycify
return [
*mypycify(
[
# List of modules that works as compiled mypyc code.
"bemani/protocol",
"bemani/common",
],
),
*cythonize(
[
# Always include code that must be compiled with cython.
*cython_only_code,
# The format module is not ready for mypyc compilation yet, there are some bugs in
# the compiler that prevent us from using it.
Extension(
"bemani.format.dxt",
[
"bemani/format/dxt.py",
]
),
# The types module is not ready for mypyc compilation yet, there are some bugs in
# the compiler that prevent us from using it.
Extension(
"bemani.format.afp.types.generic",
[
"bemani/format/afp/types/generic.py",
]
),
],
language_level=3,
),
]
else:
return [
*cythonize(
[
# Always include code that must be compiled with cython.
*cython_only_code,
# Hot code for anything constructing or parsing a remote game packet.
Extension(
"bemani.protocol.binary",
[
"bemani/protocol/binary.py",
]
),
# Even though we have a C++ implementation of this, some of the code
# is still used as a wrapper to the C++ implementation and it is very
# hot code (almost every packet touches this).
Extension(
"bemani.protocol.lz77",
[
"bemani/protocol/lz77.py",
]
),
# Every single backend service uses this class for construction and
# parsing, so compiling this makes sense.
Extension(
"bemani.protocol.node",
[
"bemani/protocol/node.py",
]
),
# This is the top-level protocol marshall which gets touched at least
# once per packet, so its worth it to squeeze more speed out of this.
Extension(
"bemani.protocol.protocol",
[
"bemani/protocol/protocol.py",
]
),
# This is used to implement a convenient way of parsing/creating binary
# data and it is memory-safe accessses of bytes so it is necessarily
# a bottleneck.
Extension(
"bemani.protocol.stream",
[
"bemani/protocol/stream.py",
]
),
# This gets used less frequently (only on the oldest games) but it is
# still worth it to get a bit of a speed boost by compiling.
Extension(
"bemani.protocol.xml",
[
"bemani/protocol/xml.py",
]
),
# These types include operations such as matrix math and color conversion so
# it is worth it to speed this up when rendering animations.
Extension(
"bemani.format.afp.types.generic",
[
"bemani/format/afp/types/generic.py",
]
),
# DXT is slow enough that it might be worth it to write a C++ implementation of
# this at some point, but for now we squeeze a bit of speed out of this by compiling.
Extension(
"bemani.format.dxt",
[
"bemani/format/dxt.py",
]
),
],
language_level=3,
),
]
class CleanExtCommand(Command):
description = 'Clean all compiled python extensions from the current directory.'
@ -204,10 +249,6 @@ setup(
install_requires=[
req for req in open('requirements.txt').read().split('\n') if len(req) > 0
],
# None of these are required for operating any of the utilities found in this repo.
# They are all present for speed. If you cannot compile arbitrary code or cython,
# remove the ext_modules line and run setuptools again. Everything should work, but
# it will run slower.
ext_modules=extensions(),
cmdclass={
'clean_ext': CleanExtCommand,