From 5011d5d8d146ec12e2177bb60a42856cea3708eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Sat, 29 Jan 2022 13:53:17 +0100 Subject: [PATCH] DISPATCH-2321 Add mypy check for the Python code --- python/qpid_dispatch/management/error.py | 12 ++-- python/qpid_dispatch_internal/dispatch.pyi | 68 ++++++++++++++++++++ tests/TCP_echo_server.py | 5 +- tests/http2_server.py | 21 ++++-- tests/http2_slow_q2_server.py | 43 ++++++++----- tests/hyperh2_server.py | 43 ++++++++----- tests/run_system_tests.py | 2 - tests/system_test.py | 6 +- tests/system_tests_edge_router.py | 4 +- tests/system_tests_priority.py | 2 +- tests/system_tests_sasl_plain.py | 4 -- tests/system_tests_ssl.py | 10 +-- tests/system_tests_topology.py | 2 +- tests/system_tests_topology_addition.py | 2 +- tests/system_tests_topology_disposition.py | 2 +- tests/system_tests_websockets.py | 2 +- tests/test_command.py | 4 +- tests/tox.ini.in | 74 +++++++++++++++++++++- 18 files changed, 231 insertions(+), 75 deletions(-) create mode 100644 python/qpid_dispatch_internal/dispatch.pyi diff --git a/python/qpid_dispatch/management/error.py b/python/qpid_dispatch/management/error.py index 49b43f765..95ee1f298 100644 --- a/python/qpid_dispatch/management/error.py +++ b/python/qpid_dispatch/management/error.py @@ -114,27 +114,27 @@ def __init__(self, description): ManagementError.__init__(self, status, descript return Error -class BadRequestStatus(_error_class(BAD_REQUEST)): +class BadRequestStatus(_error_class(BAD_REQUEST)): # type: ignore[misc] # Unsupported dynamic base class "_error_class" pass -class UnauthorizedStatus(_error_class(UNAUTHORIZED)): +class UnauthorizedStatus(_error_class(UNAUTHORIZED)): # type: ignore[misc] pass -class ForbiddenStatus(_error_class(FORBIDDEN)): +class ForbiddenStatus(_error_class(FORBIDDEN)): # type: ignore[misc] pass -class NotFoundStatus(_error_class(NOT_FOUND)): +class NotFoundStatus(_error_class(NOT_FOUND)): # type: ignore[misc] pass -class InternalServerErrorStatus(_error_class(INTERNAL_SERVER_ERROR)): +class InternalServerErrorStatus(_error_class(INTERNAL_SERVER_ERROR)): # type: ignore[misc] pass -class NotImplementedStatus(_error_class(NOT_IMPLEMENTED)): +class NotImplementedStatus(_error_class(NOT_IMPLEMENTED)): # type: ignore[misc] pass diff --git a/python/qpid_dispatch_internal/dispatch.pyi b/python/qpid_dispatch_internal/dispatch.pyi new file mode 100644 index 000000000..9fdbc4668 --- /dev/null +++ b/python/qpid_dispatch_internal/dispatch.pyi @@ -0,0 +1,68 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License +# + +"""Type stubs for objects implemented in the C extension module""" + +import ctypes +from typing import List + + +class QdDll(ctypes.PyDLL): + def __init__(self, handle): + ... + + def _prototype(self, f, restype, argtypes, check=True): + ... + + def function(self, fname, restype, argtypes, check=True): + ... + + +FORBIDDEN: List[str] + +LOG_TRACE: int +LOG_DEBUG: int +LOG_INFO: int +LOG_NOTICE: int +LOG_WARNING: int +LOG_ERROR: int +LOG_CRITICAL: int +LOG_STACK_LIMIT: int + +TREATMENT_MULTICAST_FLOOD: int +TREATMENT_MULTICAST_ONCE: int +TREATMENT_ANYCAST_CLOSEST: int +TREATMENT_ANYCAST_BALANCED: int +TREATMENT_LINK_BALANCED: int + + +class LogAdapter: + def __init__(self, mod_name): + ... + + def log(self, level, text): + ... + + +class IoAdapter: + def __init__(self, handler, address, global_address=False): + ... + + def send(self, address, properties, application_properties, body, correlation_id=None): + ... diff --git a/tests/TCP_echo_server.py b/tests/TCP_echo_server.py index db15c024c..2da12d137 100755 --- a/tests/TCP_echo_server.py +++ b/tests/TCP_echo_server.py @@ -83,7 +83,7 @@ def split_chunk_for_display(raw_bytes): class TcpEchoServer: def __init__(self, prefix="ECHO_SERVER", port: Union[str, int] = "0", echo_count=0, timeout=0.0, logger=None, - conn_stall=0.0, close_on_conn=False, close_on_data=False): + conn_stall=0.0, close_on_conn=False, close_on_data=False) -> None: """ Start echo server in separate thread @@ -92,9 +92,8 @@ def __init__(self, prefix="ECHO_SERVER", port: Union[str, int] = "0", echo_count :param echo_count: exit after echoing this many bytes :param timeout: exit after this many seconds :param logger: Logger() object - :return: """ - self.sock = None + self.sock: socket.socket self.prefix = prefix self.port = int(port) self.echo_count = echo_count diff --git a/tests/http2_server.py b/tests/http2_server.py index 7cad01092..e9eadc782 100644 --- a/tests/http2_server.py +++ b/tests/http2_server.py @@ -22,14 +22,14 @@ from quart import Quart, request try: - from quart.static import send_file + from quart.static import send_file # type: ignore[attr-defined] except ImportError: - from quart.helpers import send_file + from quart.helpers import send_file # type: ignore[attr-defined, no-redef] # mypy#1153 try: - from quart.exceptions import HTTPStatusException + from quart.exceptions import HTTPStatusException # type: ignore[attr-defined] except ImportError: - from werkzeug.exceptions import InternalServerError as HTTPStatusException + from werkzeug.exceptions import InternalServerError as HTTPStatusException # type: ignore[no-redef] # mypy#1153 app = Quart(__name__) @@ -117,5 +117,14 @@ async def process_upload_data(): return "Success!" -#app.run(port=5000, certfile='cert.pem', keyfile='key.pem') -app.run(port=os.getenv('SERVER_LISTEN_PORT')) +def main(): + port = os.getenv('SERVER_LISTEN_PORT') + if port is None: + raise RuntimeError("Environment variable `SERVER_LISTEN_PORT` is not set.") + + # app.run(port=5000, certfile='cert.pem', keyfile='key.pem') + app.run(port=int(port)) + + +if __name__ == '__main__': + main() diff --git a/tests/http2_slow_q2_server.py b/tests/http2_slow_q2_server.py index 82ef2a6c3..e2c6a3ed1 100644 --- a/tests/http2_slow_q2_server.py +++ b/tests/http2_slow_q2_server.py @@ -88,20 +88,29 @@ def handle(sock): sock.sendall(data_to_send) -signal.signal(signal.SIGHUP, receive_signal) -signal.signal(signal.SIGINT, receive_signal) -signal.signal(signal.SIGQUIT, receive_signal) -signal.signal(signal.SIGILL, receive_signal) -signal.signal(signal.SIGTERM, receive_signal) - -sock = socket.socket() -sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) -sock.bind(('0.0.0.0', int(os.getenv('SERVER_LISTEN_PORT')))) -sock.listen(5) - -while True: - # The accept method blocks until someone attempts to connect to our TCP - # port: when they do, it returns a tuple: the first element is a new - # socket object, the second element is a tuple of the address the new - # connection is from - handle(sock.accept()[0]) +def main(): + signal.signal(signal.SIGHUP, receive_signal) + signal.signal(signal.SIGINT, receive_signal) + signal.signal(signal.SIGQUIT, receive_signal) + signal.signal(signal.SIGILL, receive_signal) + signal.signal(signal.SIGTERM, receive_signal) + + port = os.getenv('SERVER_LISTEN_PORT') + if port is None: + raise RuntimeError("Environment variable `SERVER_LISTEN_PORT` is not set.") + + sock = socket.socket() + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(('0.0.0.0', int(port))) + sock.listen(5) + + while True: + # The accept method blocks until someone attempts to connect to our TCP + # port: when they do, it returns a tuple: the first element is a new + # socket object, the second element is a tuple of the address the new + # connection is from + handle(sock.accept()[0]) + + +if __name__ == '__main__': + main() diff --git a/tests/hyperh2_server.py b/tests/hyperh2_server.py index 13e579803..2784e450d 100644 --- a/tests/hyperh2_server.py +++ b/tests/hyperh2_server.py @@ -88,20 +88,29 @@ def handle(sock): sock.sendall(data_to_send) -signal.signal(signal.SIGHUP, receive_signal) -signal.signal(signal.SIGINT, receive_signal) -signal.signal(signal.SIGQUIT, receive_signal) -signal.signal(signal.SIGILL, receive_signal) -signal.signal(signal.SIGTERM, receive_signal) - -sock = socket.socket() -sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) -sock.bind(('0.0.0.0', int(os.getenv('SERVER_LISTEN_PORT')))) -sock.listen(5) - -while True: - # The accept method blocks until someone attempts to connect to our TCP - # port: when they do, it returns a tuple: the first element is a new - # socket object, the second element is a tuple of the address the new - # connection is from - handle(sock.accept()[0]) +def main(): + signal.signal(signal.SIGHUP, receive_signal) + signal.signal(signal.SIGINT, receive_signal) + signal.signal(signal.SIGQUIT, receive_signal) + signal.signal(signal.SIGILL, receive_signal) + signal.signal(signal.SIGTERM, receive_signal) + + port = os.getenv('SERVER_LISTEN_PORT') + if port is None: + raise RuntimeError("Environment variable `SERVER_LISTEN_PORT` is not set.") + + sock = socket.socket() + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(('0.0.0.0', int(port))) + sock.listen(5) + + while True: + # The accept method blocks until someone attempts to connect to our TCP + # port: when they do, it returns a tuple: the first element is a new + # socket object, the second element is a tuple of the address the new + # connection is from + handle(sock.accept()[0]) + + +if __name__ == '__main__': + main() diff --git a/tests/run_system_tests.py b/tests/run_system_tests.py index 51f82e1c0..cad6b5e85 100644 --- a/tests/run_system_tests.py +++ b/tests/run_system_tests.py @@ -42,5 +42,3 @@ all_tests.addTest(tests) result = unittest.TextTestRunner(verbosity=2).run(all_tests) sys.exit(not result.wasSuccessful()) - -sys.argv = ['unittest', '-v'] + tests diff --git a/tests/system_test.py b/tests/system_test.py index 75cdd26e4..833fe471c 100755 --- a/tests/system_test.py +++ b/tests/system_test.py @@ -28,6 +28,8 @@ - Sundry other tools. """ +from typing import Callable + import errno import sys import time @@ -126,7 +128,7 @@ def retry_delay(deadline, delay, max_delay): TIMEOUT = float(os.environ.get("QPID_SYSTEM_TEST_TIMEOUT", 60)) -def retry(function, timeout=TIMEOUT, delay=.001, max_delay=1): +def retry(function: Callable[[], bool], timeout: float = TIMEOUT, delay: float = .001, max_delay: float = 1): """Call function until it returns a true value or timeout expires. Double the delay for each retry up to max_delay. Returns what function returns or None if timeout expires. @@ -382,7 +384,7 @@ def wait_ports(self, **retry_kwargs): class Qdrouterd(Process): """Run a Qpid Dispatch Router Daemon""" - class Config(list, Config): + class Config(list, Config): # type: ignore[misc] # Cannot resolve name "Config" (possible cyclic definition) # mypy#10958 """ A router configuration. diff --git a/tests/system_tests_edge_router.py b/tests/system_tests_edge_router.py index 10c1fb91b..a50e79eff 100644 --- a/tests/system_tests_edge_router.py +++ b/tests/system_tests_edge_router.py @@ -2029,6 +2029,7 @@ def __init__(self, receiver1_host, receiver2_host, receiver3_host, def on_released(self, event): self.n_released += 1 + self.send_test_message() def timeout(self): if self.dup_msg: @@ -2144,9 +2145,6 @@ def on_message(self, event): self.receiver3_conn.close() self.sender_conn.close() - def on_released(self, event): - self.send_test_message() - def run(self): Container(self).run() diff --git a/tests/system_tests_priority.py b/tests/system_tests_priority.py index 70e42f991..d6e50631b 100644 --- a/tests/system_tests_priority.py +++ b/tests/system_tests_priority.py @@ -18,7 +18,7 @@ # -from proton import Message, Timeout +from proton import Message from proton.handlers import MessagingHandler from proton.reactor import Container diff --git a/tests/system_tests_sasl_plain.py b/tests/system_tests_sasl_plain.py index 474d7b17c..e635e978d 100644 --- a/tests/system_tests_sasl_plain.py +++ b/tests/system_tests_sasl_plain.py @@ -671,10 +671,6 @@ def setUpClass(cls): cls.routers[1].wait_router_connected('QDR.X') - @staticmethod - def ssl_file(name): - return os.path.join(DIR, 'ssl_certs', name) - def common_asserts(self, results): search = "QDR.X" found = False diff --git a/tests/system_tests_ssl.py b/tests/system_tests_ssl.py index 196a91042..2d55b0b16 100644 --- a/tests/system_tests_ssl.py +++ b/tests/system_tests_ssl.py @@ -117,6 +117,7 @@ class RouterTestSslClient(RouterTestSslBase): p = Popen(['openssl', 'version'], stdout=PIPE, universal_newlines=True) openssl_out = p.communicate()[0] m = re.search(r'[0-9]+\.[0-9]+\.[0-9]+', openssl_out) + assert m is not None OPENSSL_OUT_VER = m.group(0) OPENSSL_VER_1_1_GT = StrictVersion(OPENSSL_OUT_VER) >= StrictVersion('1.1') print("OpenSSL Version found = %s" % OPENSSL_OUT_VER) @@ -132,13 +133,8 @@ class RouterTestSslClient(RouterTestSslBase): OPENSSL_ALLOW_TLSV1_3 = False # Test if OpenSSL has TLSv1_3 - OPENSSL_HAS_TLSV1_3 = False - if OPENSSL_VER_1_1_GT: - try: - _ = ssl.TLSVersion.TLSv1_3 - OPENSSL_HAS_TLSV1_3 = True - except AttributeError: - pass + # (see https://mypy.readthedocs.io/en/stable/common_issues.html#python-version-and-system-platform-checks for mypy considerations) + OPENSSL_HAS_TLSV1_3 = OPENSSL_VER_1_1_GT and sys.version_info >= (3, 7) and ssl.HAS_TLSv1_3 # Test if Proton supports TLSv1_3 try: diff --git a/tests/system_tests_topology.py b/tests/system_tests_topology.py index c10122b97..c5dcf0866 100644 --- a/tests/system_tests_topology.py +++ b/tests/system_tests_topology.py @@ -19,7 +19,7 @@ import time -from proton import Message, Timeout +from proton import Message from proton.handlers import MessagingHandler from proton.reactor import Container diff --git a/tests/system_tests_topology_addition.py b/tests/system_tests_topology_addition.py index 5972425fd..60a20a7c6 100644 --- a/tests/system_tests_topology_addition.py +++ b/tests/system_tests_topology_addition.py @@ -19,7 +19,7 @@ import unittest -from proton import Message, Timeout +from proton import Message from proton.handlers import MessagingHandler from proton.reactor import Container diff --git a/tests/system_tests_topology_disposition.py b/tests/system_tests_topology_disposition.py index 2f7aaddc7..25e8046c8 100644 --- a/tests/system_tests_topology_disposition.py +++ b/tests/system_tests_topology_disposition.py @@ -24,7 +24,7 @@ from subprocess import PIPE, STDOUT import proton -from proton import Message, Timeout +from proton import Message from proton.handlers import MessagingHandler from proton.reactor import Container from qpid_dispatch_internal.compat import UNICODE diff --git a/tests/system_tests_websockets.py b/tests/system_tests_websockets.py index d9b585479..9f331fd68 100644 --- a/tests/system_tests_websockets.py +++ b/tests/system_tests_websockets.py @@ -23,7 +23,7 @@ try: import websockets except ImportError: - websockets = None + websockets = None # type: ignore[assignment] # expression has type "None", variable has type Module from system_test import Qdrouterd from system_test import main_module, TestCase, Process diff --git a/tests/test_command.py b/tests/test_command.py index 099542601..8b071d990 100644 --- a/tests/test_command.py +++ b/tests/test_command.py @@ -34,9 +34,9 @@ def mock_error(self, message): raise ValueError(message) -argparse.ArgumentParser.error = mock_error +argparse.ArgumentParser.error = mock_error # type: ignore[assignment] # Cannot assign to a method -# Since BusManager file is definded in tools/qdmanage.in -> tools/qdmanage +# Since BusManager file is defined in tools/qdmanage.in -> tools/qdmanage # otherwise it could be just imported diff --git a/tests/tox.ini.in b/tests/tox.ini.in index 61c6d4bdc..0fc9ef47e 100644 --- a/tests/tox.ini.in +++ b/tests/tox.ini.in @@ -35,7 +35,7 @@ commands = ${CMAKE_SOURCE_DIR}/tools/qdmanage # TODO(pylint#5648): crash while parsing system_test.py - pylint --rcfile ${CMAKE_BINARY_DIR}/tests/tox.ini \ + pylint --jobs 2 --rcfile ${CMAKE_BINARY_DIR}/tests/tox.ini \ --ignore=system_test.py \ ${CMAKE_SOURCE_DIR}/python \ ${CMAKE_SOURCE_DIR}/docs \ @@ -44,11 +44,19 @@ commands = ${CMAKE_SOURCE_DIR}/tools/qdstat \ ${CMAKE_SOURCE_DIR}/tools/qdmanage + mypy --config-file ${CMAKE_BINARY_DIR}/tests/tox.ini \ + ${CMAKE_SOURCE_DIR}/python \ + ${CMAKE_SOURCE_DIR}/docs \ + ${CMAKE_SOURCE_DIR}/tests \ + ${CMAKE_SOURCE_DIR}/tools \ + ${CMAKE_BINARY_DIR}/python/qpid_dispatch_site.py + deps = hacking==4.1.0 # hacking 4.1.0 requires flake8<3.9.0 and >=3.8.0 flake8==3.8.4 pylint==2.12.2 + mypy==0.910 [testenv:py36] basepython = python3.6 @@ -205,3 +213,67 @@ disable = useless-else-on-loop, useless-super-delegation, wrong-import-position, + +[mypy] +warn_redundant_casts = True +warn_unused_ignores = False + +# mypy cannot handle overridden attributes +# https://github.com/python/mypy/issues/7505 +allow_untyped_globals = True + +# https://mypy.readthedocs.io/en/stable/error_codes.html#displaying-error-codes +show_error_codes = True + +# this would print lots and lots of errors +# check_untyped_defs = True + +# ignore missing stub files for dependencies + +#[mypy-_ssl] +#ignore_missing_imports = True + +[mypy-proton.*] +ignore_missing_imports = True + +[mypy-cproton] +ignore_missing_imports = True + +[mypy-qpidtoollibs] +ignore_missing_imports = True + +[mypy-qpid_messaging] +ignore_missing_imports = True + +[mypy-pyprof2calltree] +ignore_missing_imports = True + +[mypy-quart.*] +ignore_missing_imports = True + +[mypy-werkzeug.*] +ignore_missing_imports = True + +[mypy-selectors] +ignore_missing_imports = True + +[mypy-h2.*] +ignore_missing_imports = True + +[mypy-google.protobuf] +ignore_missing_imports = True + +[mypy-grpc] +ignore_missing_imports = True + +[mypy-grpcio] +ignore_missing_imports = True + +[mypy-protobuf] +ignore_missing_imports = True + +[mypy-websockets] +ignore_missing_imports = True + +[mypy-pytest] +ignore_missing_imports = True