Update to stricter type checking, get rid of a bunch of unnecessary ignore statements.

bemani/api/app.py

@@ -2,7 +2,7 @@ import copy
 import json
 import traceback
 from typing import Any, Callable, Dict
-from flask import Flask, abort, request, Response  # type: ignore
+from flask import Flask, abort, request, Response
 from functools import wraps
 
 from bemani.api.exceptions import APIException

bemani/common/aes.py

@@ -26,7 +26,7 @@ class AESCipher:
 
     def encrypt(self, raw: str) -> str:
         raw = self.__pad(raw)
-        random = Random.new()  # type: ignore
+        random = Random.new()
         iv = random.read(AES.block_size)
         cipher = AES.new(self.__key, AES.MODE_CBC, iv)
         return base64.b64encode(iv + cipher.encrypt(raw.encode('utf-8')), altchars=b"._").decode('utf-8')

bemani/common/time.py

@@ -33,7 +33,7 @@ class Time:
             now.year,
             now.month,
             now.day,
-            tzinfo=tz.tzutc()  # type: ignore
+            tzinfo=tz.tzutc()
         )
         end_of_day = beginning_of_day + datetime.timedelta(days=1)
         return calendar.timegm(end_of_day.timetuple())
@@ -48,7 +48,7 @@ class Time:
             now.year,
             now.month,
             now.day,
-            tzinfo=tz.tzutc()  # type: ignore
+            tzinfo=tz.tzutc()
         )
         return calendar.timegm(beginning_of_day.timetuple())
 
@@ -58,7 +58,7 @@ class Time:
         Returns the unix timestamp for the end of this week in UTC timezone.
         """
         now = datetime.datetime.utcnow().date()
-        this_week = now - datetime.timedelta(days=now.timetuple().tm_wday)  # type: ignore
+        this_week = now - datetime.timedelta(days=now.timetuple().tm_wday)
         next_week = this_week + datetime.timedelta(days=7)
         return calendar.timegm(next_week.timetuple())
 
@@ -68,7 +68,7 @@ class Time:
         Returns the unix timestamp for the beginning of this week in UTC timezone.
         """
         now = datetime.datetime.utcnow().date()
-        this_week = now - datetime.timedelta(days=now.timetuple().tm_wday)  # type: ignore
+        this_week = now - datetime.timedelta(days=now.timetuple().tm_wday)
         return calendar.timegm(this_week.timetuple())
 
     @staticmethod
@@ -117,7 +117,7 @@ class Time:
             date = datetime.datetime.utcnow().date()
         else:
             date = datetime.datetime.utcfromtimestamp(timestamp).date()
-        week = date - datetime.timedelta(days=date.timetuple().tm_wday)  # type: ignore
+        week = date - datetime.timedelta(days=date.timetuple().tm_wday)
         return (week - datetime.date(1970, 1, 1)).days
 
     @staticmethod
@@ -131,7 +131,7 @@ class Time:
             date = datetime.datetime.utcnow().date().timetuple()
         else:
             date = datetime.datetime.utcfromtimestamp(timestamp).date().timetuple()
-        return [date.tm_year, date.tm_yday]  # type: ignore
+        return [date.tm_year, date.tm_yday]
 
     @staticmethod
     def days_into_week(timestamp: Optional[int]=None) -> int:
@@ -144,7 +144,7 @@ class Time:
             date = datetime.datetime.utcnow().date().timetuple()
         else:
             date = datetime.datetime.utcfromtimestamp(timestamp).date().timetuple()
-        return date.tm_wday  # type: ignore
+        return date.tm_wday
 
     @staticmethod
     def timestamp_from_date(year: int, month: int=1, day: int=1) -> int:
@@ -164,7 +164,7 @@ class Time:
             year,
             month,
             day,
-            tzinfo=tz.tzutc()  # type: ignore
+            tzinfo=tz.tzutc()
         )
         return calendar.timegm(date.timetuple())
 

bemani/data/data.py

@@ -6,7 +6,7 @@ from alembic.migration import MigrationContext  # type: ignore
 from alembic.autogenerate import compare_metadata  # type: ignore
 from sqlalchemy import create_engine  # type: ignore
 from sqlalchemy.orm import scoped_session  # type: ignore
-from sqlalchemy.orm import sessionmaker  # type: ignore
+from sqlalchemy.orm import sessionmaker
 from sqlalchemy.engine import Engine  # type: ignore
 from sqlalchemy.sql import text  # type: ignore
 from sqlalchemy.exc import ProgrammingError  # type: ignore
@@ -125,7 +125,7 @@ class Data:
 
     @classmethod
     def create_engine(cls, config: Dict[str, Any]) -> Engine:
-        return create_engine(  # type: ignore
+        return create_engine(
             Data.sqlalchemy_url(config),
             pool_recycle=3600,
         )
@@ -161,7 +161,7 @@ class Data:
             # Cowardly refused to do anything, we should be using the upgrade path instead.
             raise DBCreateException('Tables already created, use upgrade to upgrade schema!')
 
-        metadata.create_all(  # type: ignore
+        metadata.create_all(
             self.__config['database']['engine'].connect(),
             checkfirst=True,
         )

bemani/data/mysql/api.py

@@ -12,7 +12,7 @@ from bemani.data.types import Client, Server
 Table for storing registered clients to a data exchange API, as added
 by an admin.
 """
-client = Table(  # type: ignore
+client = Table(
     'client',
     metadata,
     Column('id', Integer, nullable=False, primary_key=True),
@@ -26,7 +26,7 @@ client = Table(  # type: ignore
 Table for storing remote servers to a data exchange API, as added
 by an admin.
 """
-server = Table(  # type: ignore
+server = Table(
     'server',
     metadata,
     Column('id', Integer, nullable=False, primary_key=True),

bemani/data/mysql/base.py

@@ -10,13 +10,13 @@ from sqlalchemy.sql import text  # type: ignore
 from sqlalchemy.types import String, Integer  # type: ignore
 from sqlalchemy import Table, Column, MetaData  # type: ignore
 
-metadata = MetaData()  # type: ignore
+metadata = MetaData()
 
 """
 Table for storing session IDs, so a session ID can be used to look up an arbitrary ID.
 This is currently used for user logins, user and arcade PASELI sessions.
 """
-session = Table(  # type: ignore
+session = Table(
     'session',
     metadata,
     Column('id', Integer, nullable=False),
@@ -74,7 +74,7 @@ class BaseData:
             ]:
                 if write_statement in sql.lower() and not safe_write_operation:
                     raise Exception('Read-only mode is active!')
-        return self.__conn.execute(  # type: ignore
+        return self.__conn.execute(
             text(sql),
             params if params is not None else {},
         )

bemani/data/mysql/game.py

@@ -12,7 +12,7 @@ Table for storing game settings that span multiple versions of the same
 game, such as play statistics. This table intentionally doesn't have a
 key on game version, just game string and userid.
 """
-game_settings = Table(  # type: ignore
+game_settings = Table(
     'game_settings',
     metadata,
     Column('game', String(32), nullable=False),
@@ -25,7 +25,7 @@ game_settings = Table(  # type: ignore
 """
 Table for storing shop items that are server-side verified.
 """
-catalog = Table(  # type: ignore
+catalog = Table(
     'catalog',
     metadata,
     Column('game', String(32), nullable=False),
@@ -42,7 +42,7 @@ Table for storing series achievements that span multiple versions of the same
 game, such as course scores. This table intentionally doesn't have a
 key on game version, just game string and userid.
 """
-series_achievement = Table(  # type: ignore
+series_achievement = Table(
     'series_achievement',
     metadata,
     Column('game', String(32), nullable=False),
@@ -58,7 +58,7 @@ series_achievement = Table(  # type: ignore
 Table for storing time-based game settings that aren't tied to a user
 account, such as dailies, weeklies, etc.
 """
-time_sensitive_settings = Table(  # type: ignore
+time_sensitive_settings = Table(
     'time_sensitive_settings',
     metadata,
     Column('game', String(32), nullable=False),

bemani/data/mysql/lobby.py

@@ -14,7 +14,7 @@ Table for storing logistical information about a player who's session is
 live. Mostly, this is used to store IP addresses and such for players that
 could potentially match.
 """
-playsession = Table(  # type: ignore
+playsession = Table(
     'playsession',
     metadata,
     Column('id', Integer, nullable=False, primary_key=True),
@@ -30,7 +30,7 @@ playsession = Table(  # type: ignore
 """
 Table for storing open lobbies for matching between games.
 """
-lobby = Table(  # type: ignore
+lobby = Table(
     'lobby',
     metadata,
     Column('id', Integer, nullable=False, primary_key=True),

bemani/data/mysql/machine.py

@@ -13,7 +13,7 @@ with PCBID enforcement to ensure machines not authorized on the network are deni
 a connection. It is also used for settings such as port forwarding and which arcade
 a machine belongs to for the purpose of PASELI balance.
 """
-machine = Table(  # type: ignore
+machine = Table(
     'machine',
     metadata,
     Column('id', Integer, nullable=False, primary_key=True),
@@ -32,7 +32,7 @@ machine = Table(  # type: ignore
 Table for storing an arcade, to which zero or more machines may belong. This allows
 an arcade to override some global settings such as PASELI enabled and infinite.
 """
-arcade = Table(  # type: ignore
+arcade = Table(
     'arcade',
     metadata,
     Column('id', Integer, nullable=False, primary_key=True),
@@ -46,7 +46,7 @@ arcade = Table(  # type: ignore
 """
 Table for storing arcade ownership. This allows for more than one owner to own an arcade.
 """
-arcade_owner = Table(  # type: ignore
+arcade_owner = Table(
     'arcade_owner',
     metadata,
     Column('userid', BigInteger(unsigned=True), nullable=False),
@@ -60,7 +60,7 @@ Table for storing arcade settings for a particular game/version. This allows the
 owner to change settings related to a particular game, such as the events active or the
 shop ranking courses.
 """
-arcade_settings = Table(  # type: ignore
+arcade_settings = Table(
     'arcade_settings',
     metadata,
     Column('arcadeid', Integer, nullable=False),

bemani/data/mysql/music.py

@@ -19,7 +19,7 @@ Note that this is NOT keyed by game song id and chart, but by an internal musici
 managed by the music table. This is so we can support keeping the same score across
 multiple games, even if the game changes the ID it refers to the song by.
 """
-score = Table(  # type: ignore
+score = Table(
     'score',
     metadata,
     Column('id', Integer, nullable=False, primary_key=True),
@@ -39,7 +39,7 @@ Table for storing score history for a particular game. Every entry that is store
 or updated in score will be written into this table as well, for looking up history
 over time.
 """
-score_history = Table(  # type: ignore
+score_history = Table(
     'score_history',
     metadata,
     Column('id', Integer, nullable=False, primary_key=True),
@@ -63,7 +63,7 @@ NOTE that it is expected to see the same songid/chart present multiple times as
 as the game version changes. In this way, a song which is in multiple versions of
 the game can be found when playing each version.
 """
-music = Table(  # type: ignore
+music = Table(
     'music',
     metadata,
     Column('id', Integer, nullable=False, index=True),

bemani/data/mysql/network.py

@@ -11,7 +11,7 @@ from bemani.data.types import News, Event, UserID, ArcadeID
 Table for storing network news, as edited by an admin. This is displayed
 on the front page of the frontend of the network.
 """
-news = Table(  # type: ignore
+news = Table(
     'news',
     metadata,
     Column('id', Integer, nullable=False, primary_key=True),
@@ -25,7 +25,7 @@ news = Table(  # type: ignore
 Table for storing scheduled work history, so that individual game code
 can determine if it should run scheduled work or not.
 """
-scheduled_work = Table(  # type: ignore
+scheduled_work = Table(
     'scheduled_work',
     metadata,
     Column('game', String(32), nullable=False),
@@ -43,7 +43,7 @@ Table for storing audit entries, such as crashes, PCBID denials, daily
 song selection, etc. Anything that could be inspected later to verify
 correct operation of the network.
 """
-audit = Table(  # type: ignore
+audit = Table(
     'audit',
     metadata,
     Column('id', Integer, nullable=False, primary_key=True),

bemani/data/mysql/user.py

@@ -19,7 +19,7 @@ and password are optional as a user does not need to create a web login
 to use the network. However, an active user account is required
 before creating a web login.
 """
-user = Table(  # type: ignore
+user = Table(
     'user',
     metadata,
     Column('id', Integer, nullable=False, primary_key=True),
@@ -37,7 +37,7 @@ or more cards associated with them. When a new card is used in a game
 a new user will be created to associate with a card, but it can later
 be unlinked.
 """
-card = Table(  # type: ignore
+card = Table(
     'card',
     metadata,
     Column('id', String(16), nullable=False, unique=True),
@@ -49,7 +49,7 @@ card = Table(  # type: ignore
 Table representing an extid for a user across a game series. Each game
 series on the network gets its own extid (8 digit number) for each user.
 """
-extid = Table(  # type: ignore
+extid = Table(
     'extid',
     metadata,
     Column('game', String(32), nullable=False),
@@ -68,7 +68,7 @@ will be generated for the user.
 Note that a user might have an extid/refid for a game without a profile,
 but a user cannot have a profile without an extid/refid.
 """
-refid = Table(  # type: ignore
+refid = Table(
     'refid',
     metadata,
     Column('game', String(32), nullable=False),
@@ -82,7 +82,7 @@ refid = Table(  # type: ignore
 """
 Table for storing JSON profile blobs, indexed by refid.
 """
-profile = Table(  # type: ignore
+profile = Table(
     'profile',
     metadata,
     Column('refid', String(16), nullable=False, unique=True),
@@ -96,7 +96,7 @@ with a unique ID and type. Games are free to store a JSON blob for each
 achievement. Examples would be tran medals, event unlocks, items earned,
 etc.
 """
-achievement = Table(  # type: ignore
+achievement = Table(
     'achievement',
     metadata,
     Column('refid', String(16), nullable=False),
@@ -115,7 +115,7 @@ you earn it. Games are free to store a JSON blob for each achievement and
 the blob does not need to be equal across different instances of the same
 achievement for the same user. Examples would be calorie earnings for DDR.
 """
-time_based_achievement = Table(  # type: ignore
+time_based_achievement = Table(
     'time_based_achievement',
     metadata,
     Column('refid', String(16), nullable=False),
@@ -131,7 +131,7 @@ time_based_achievement = Table(  # type: ignore
 Table for storing a user's PASELI balance, given an arcade. There is no global
 balance on this network.
 """
-balance = Table(  # type: ignore
+balance = Table(
     'balance',
     metadata,
     Column('userid', BigInteger(unsigned=True), nullable=False),
@@ -146,7 +146,7 @@ Table for storing links between two users in a game/version, whatever that
 may be. Typically used for rivals.
 etc.
 """
-link = Table(  # type: ignore
+link = Table(
     'link',
     metadata,
     Column('game', String(32), nullable=False),

bemani/utils/api.py

@@ -1,5 +1,5 @@
 import argparse
-import yaml  # type: ignore
+import yaml
 
 from bemani.data import Data
 from bemani.api import app, config  # noqa: F401
@@ -8,7 +8,7 @@ from bemani.api import app, config  # noqa: F401
 def load_config(filename: str) -> None:
     global config
 
-    config.update(yaml.safe_load(open(filename)))  # type: ignore
+    config.update(yaml.safe_load(open(filename)))
     config['database']['engine'] = Data.create_engine(config)
 
 
@@ -23,7 +23,7 @@ def main() -> None:
     load_config(args.config)
 
     if args.profile:
-        from werkzeug.contrib.profiler import ProfilerMiddleware  # type: ignore
+        from werkzeug.contrib.profiler import ProfilerMiddleware
         app.wsgi_app = ProfilerMiddleware(app.wsgi_app, profile_dir='.')  # type: ignore
 
     # Run the app

bemani/utils/dbutils.py

@@ -1,7 +1,7 @@
 import argparse
 import getpass
 import sys
-import yaml  # type: ignore
+import yaml
 from typing import Any, Dict, Optional
 
 from bemani.data import Data, DBCreateException
@@ -96,7 +96,7 @@ def main() -> None:
     parser.add_argument("-c", "--config", help="Core configuration. Defaults to server.yaml", type=str, default="server.yaml")
     args = parser.parse_args()
 
-    config = yaml.safe_load(open(args.config))  # type: ignore
+    config = yaml.safe_load(open(args.config))
     config['database']['engine'] = Data.create_engine(config)
     try:
         if args.operation == "create":

bemani/utils/frontend.py

@@ -1,5 +1,5 @@
 import argparse
-import yaml  # type: ignore
+import yaml
 
 from bemani.backend.iidx import IIDXFactory
 from bemani.backend.popn import PopnMusicFactory
@@ -76,7 +76,7 @@ def register_games() -> None:
 def load_config(filename: str) -> None:
     global config
 
-    config.update(yaml.safe_load(open(filename)))  # type: ignore
+    config.update(yaml.safe_load(open(filename)))
     config['database']['engine'] = Data.create_engine(config)
     app.secret_key = config['secret_key']
 
@@ -98,7 +98,7 @@ def main() -> None:
     register_games()
 
     if args.profile:
-        from werkzeug.contrib.profiler import ProfilerMiddleware  # type: ignore
+        from werkzeug.contrib.profiler import ProfilerMiddleware
         app.wsgi_app = ProfilerMiddleware(app.wsgi_app, profile_dir='.')  # type: ignore
 
     # Run the app

bemani/utils/proxy.py

@@ -1,7 +1,7 @@
 import argparse
 import requests
 import socket
-import yaml  # type: ignore
+import yaml
 from flask import Flask, Response, request
 from typing import Any, Dict, Optional
 import urllib.parse as urlparse
@@ -180,7 +180,7 @@ def receive_request(path: str) -> Response:
         data=req_binary,
     ).prepare()
     sess = requests.Session()
-    r = sess.send(prep_req, timeout=config['timeout'])  # type: ignore
+    r = sess.send(prep_req, timeout=config['timeout'])
 
     if r.status_code != 200:
         # Failed on remote side
@@ -233,7 +233,7 @@ def receive_request(path: str) -> Response:
 def load_proxy_config(filename: str) -> None:
     global config
 
-    config_data = yaml.safe_load(open(filename))  # type: ignore
+    config_data = yaml.safe_load(open(filename))
     if 'pcbid' in config_data and config_data['pcbid'] is not None:
         for pcbid in config_data['pcbid']:
             remote_name = config_data['pcbid'][pcbid]
@@ -244,7 +244,7 @@ def load_proxy_config(filename: str) -> None:
 def load_config(filename: str) -> None:
     global config
 
-    config_data = yaml.safe_load(open(filename))  # type: ignore
+    config_data = yaml.safe_load(open(filename))
     config.update({
         'local_host': config_data['local']['host'],
         'local_port': config_data['local']['port'],

bemani/utils/read.py

@@ -1,6 +1,6 @@
 # vim: set fileencoding=utf-8
 
-import csv  # type: ignore
+import csv
 import argparse
 import copy
 import io
@@ -9,7 +9,7 @@ import json
 import os
 import pefile  # type: ignore
 import struct
-import yaml  # type: ignore
+import yaml
 import xml.etree.ElementTree as ET
 from sqlalchemy import create_engine  # type: ignore
 from sqlalchemy.engine import CursorResult  # type: ignore
@@ -62,7 +62,7 @@ class ImportBase:
         self.no_combine = no_combine
         self.__config = config
         self.__url = f"mysql://{config['database']['user']}:{config['database']['password']}@{config['database']['address']}/{config['database']['database']}?charset=utf8mb4"
-        self.__engine = create_engine(self.__url)  # type: ignore
+        self.__engine = create_engine(self.__url)
         self.__sessionmanager = sessionmaker(self.__engine)
         self.__conn = self.__engine.connect()
         self.__session = self.__sessionmanager(bind=self.__conn)
@@ -1438,7 +1438,7 @@ class ImportJubeat(ImportBase):
             raise Exception("Unsupported Jubeat version, expected one of the following: all")
 
         with open(tsvfile, newline='') as tsvhandle:
-            jubeatreader = csv.reader(tsvhandle, delimiter='\t', quotechar='"')  # type: ignore
+            jubeatreader = csv.reader(tsvhandle, delimiter='\t', quotechar='"')
             for row in jubeatreader:
                 songid = int(row[0])
                 name = row[1]
@@ -1787,7 +1787,7 @@ class ImportIIDX(ImportBase):
             raise Exception("Unsupported IIDX version, expected one of the following: all")
 
         with open(tsvfile, newline='') as tsvhandle:
-            iidxreader = csv.reader(tsvhandle, delimiter='\t', quotechar='"')  # type: ignore
+            iidxreader = csv.reader(tsvhandle, delimiter='\t', quotechar='"')
             for row in iidxreader:
                 songid = int(row[0])
                 name = row[1]
@@ -2586,7 +2586,7 @@ class ImportSDVX(ImportBase):
             csvdata = csvhandle.read().decode('shift_jisx0213')
 
         csvstr = io.StringIO(csvdata)
-        appealreader = csv.reader(csvstr, delimiter=',', quotechar='"')  # type: ignore
+        appealreader = csv.reader(csvstr, delimiter=',', quotechar='"')
         for row in appealreader:
             appealids = []
             if self.version == VersionConstants.SDVX_INFINITE_INFECTION:
@@ -3436,7 +3436,7 @@ if __name__ == "__main__":
         raise Exception("Cannot specify both a remote server and a local file to read from!")
 
     # Load the config so we can talk to the server
-    config = yaml.safe_load(open(args.config))  # type: ignore
+    config = yaml.safe_load(open(args.config))
 
     if args.series == GameConstants.POPN_MUSIC:
         popn = ImportPopn(config, args.version, args.no_combine, args.update)

bemani/utils/scheduler.py

@@ -1,5 +1,5 @@
 import argparse
-import yaml  # type: ignore
+import yaml
 from typing import Any, Dict, List
 
 from bemani.backend.popn import PopnMusicFactory
@@ -55,11 +55,11 @@ def run_scheduled_work(config: Dict[str, Any]) -> None:
 
     # First, run any backend scheduled work
     for factory in enabled_factories:
-        factory.run_scheduled_work(data, config)  # type: ignore
+        factory.run_scheduled_work(data, config)
 
     # Now, warm the caches for the frontend
     for cache in enabled_caches:
-        cache.preload(data, config)  # type: ignore
+        cache.preload(data, config)
 
     # Now, possibly delete old log entries
     keep_duration = config.get('event_log_duration', 0)
@@ -75,7 +75,7 @@ if __name__ == '__main__':
     args = parser.parse_args()
 
     # Set up global configuration
-    config = yaml.safe_load(open(args.config))  # type: ignore
+    config = yaml.safe_load(open(args.config))
     config['database']['engine'] = Data.create_engine(config)
 
     # Run out of band work

bemani/utils/services.py

@@ -1,7 +1,7 @@
 import argparse
 import copy
 import traceback
-import yaml  # type: ignore
+import yaml
 from typing import Any, Dict
 from flask import Flask, request, redirect, Response, make_response
 
@@ -128,7 +128,7 @@ def receive_request(path: str) -> Response:
 def load_config(filename: str) -> None:
     global config
 
-    config.update(yaml.safe_load(open(filename)))  # type: ignore
+    config.update(yaml.safe_load(open(filename)))
     config['database']['engine'] = Data.create_engine(config)
 
 
@@ -168,7 +168,7 @@ if __name__ == '__main__':
     register_games()
 
     if args.profile:
-        from werkzeug.contrib.profiler import ProfilerMiddleware  # type: ignore
+        from werkzeug.contrib.profiler import ProfilerMiddleware
         app.wsgi_app = ProfilerMiddleware(app.wsgi_app, profile_dir='.')  # type: ignore
 
     # Run the app

bemani/utils/trafficgen.py

@@ -1,7 +1,7 @@
 import argparse
 import sys
 from typing import Any, Dict, Optional
-import yaml  # type: ignore
+import yaml
 
 from bemani.client import ClientProtocol, BaseClient
 from bemani.client.iidx import (
@@ -463,7 +463,7 @@ def mainloop(address: str, port: int, configfile: str, action: str, game: str, c
             print(f'Unknown game {game}')
             sys.exit(2)
 
-        config = yaml.safe_load(open(configfile))  # type: ignore
+        config = yaml.safe_load(open(configfile))
 
         print(f'Emulating {games[game]["name"]}')
         emu = get_client(

verifytyping

@@ -34,6 +34,10 @@ done
 MYPYPATH=$(python -c "import os; print(os.path.realpath('.'))") mypy \
     "${cmdline[@]}" \
     --warn-redundant-casts \
+    --warn-unused-ignores \
     --disallow-untyped-calls \
     --disallow-untyped-defs \
+    --disallow-subclassing-any \
+    --disallow-incomplete-defs \
+    --no-implicit-optional \
     --no-strict-optional