From a2d79e16a56f42f01b8b072eadbc4a54cce9e9a4 Mon Sep 17 00:00:00 2001 From: dhoomakethu Date: Mon, 3 Dec 2018 15:31:45 +0530 Subject: [PATCH 01/26] #357 Support registration of custom requests --- CHANGELOG.rst | 1 + examples/common/asynchronous_server.py | 6 +- examples/common/custom_message.py | 46 +++++-- .../common/custom_message_async_clients.py | 124 ++++++++++++++++++ examples/common/custom_synchronous_server.py | 116 ++++++++++++++++ pymodbus/client/sync.py | 8 ++ pymodbus/datastore/context.py | 11 ++ pymodbus/exceptions.py | 62 +++++---- pymodbus/factory.py | 124 ++++++++++++------ pymodbus/interfaces.py | 116 ++++++++-------- pymodbus/server/async.py | 31 +++-- pymodbus/server/sync.py | 26 +++- test/test_client_sync.py | 10 ++ test/test_factory.py | 20 ++- test/test_interfaces.py | 5 +- test/test_server_context.py | 8 ++ 16 files changed, 569 insertions(+), 145 deletions(-) create mode 100755 examples/common/custom_message_async_clients.py create mode 100755 examples/common/custom_synchronous_server.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4b61313ff..0c2496117 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,7 @@ Version 2.1.1 ----------------------------------------------------------- * Provide an option to disable inter char timeouts with Modbus RTU. +* Add support to register custom requests in clients and server instances. Version 2.1.0 ----------------------------------------------------------- diff --git a/examples/common/asynchronous_server.py b/examples/common/asynchronous_server.py index 2186698cc..1f28de565 100755 --- a/examples/common/asynchronous_server.py +++ b/examples/common/asynchronous_server.py @@ -20,6 +20,7 @@ from pymodbus.transaction import (ModbusRtuFramer, ModbusAsciiFramer, ModbusBinaryFramer) +from custom_message import CustomModbusRequest # --------------------------------------------------------------------------- # # configure the service logging @@ -92,6 +93,8 @@ def run_async_server(): co=ModbusSequentialDataBlock(0, [17]*100), hr=ModbusSequentialDataBlock(0, [17]*100), ir=ModbusSequentialDataBlock(0, [17]*100)) + store.register(CustomModbusRequest.function_code, 'cm', + ModbusSequentialDataBlock(0, [17] * 100)) context = ModbusServerContext(slaves=store, single=True) # ----------------------------------------------------------------------- # @@ -113,7 +116,8 @@ def run_async_server(): # TCP Server - StartTcpServer(context, identity=identity, address=("localhost", 5020)) + StartTcpServer(context, identity=identity, address=("localhost", 5020), + custom_functions=[CustomModbusRequest]) # TCP Server with deferred reactor run diff --git a/examples/common/custom_message.py b/examples/common/custom_message.py index 1ff87b3ae..bf339c139 100755 --- a/examples/common/custom_message.py +++ b/examples/common/custom_message.py @@ -20,6 +20,7 @@ from pymodbus.pdu import ModbusRequest, ModbusResponse, ModbusExceptions from pymodbus.client.sync import ModbusTcpClient as ModbusClient from pymodbus.bit_read_message import ReadCoilsRequest +from pymodbus.compat import int2byte, byte2int # --------------------------------------------------------------------------- # # configure the client logging # --------------------------------------------------------------------------- # @@ -40,15 +41,41 @@ class CustomModbusResponse(ModbusResponse): - pass + function_code = 55 + _rtu_byte_count_pos = 2 + + def __init__(self, values=None, **kwargs): + ModbusResponse.__init__(self, **kwargs) + self.values = values or [] + + def encode(self): + """ Encodes response pdu + + :returns: The encoded packet message + """ + result = int2byte(len(self.values) * 2) + for register in self.values: + result += struct.pack('>H', register) + return result + + def decode(self, data): + """ Decodes response pdu + + :param data: The packet data to decode + """ + byte_count = byte2int(data[0]) + self.values = [] + for i in range(1, byte_count + 1, 2): + self.values.append(struct.unpack('>H', data[i:i + 2])[0]) class CustomModbusRequest(ModbusRequest): - function_code = 1 + function_code = 55 + _rtu_frame_size = 8 - def __init__(self, address): - ModbusRequest.__init__(self) + def __init__(self, address=None, **kwargs): + ModbusRequest.__init__(self, **kwargs) self.address = address self.count = 16 @@ -74,12 +101,12 @@ def execute(self, context): class Read16CoilsRequest(ReadCoilsRequest): - def __init__(self, address): + def __init__(self, address, **kwargs): """ Initializes a new instance :param address: The address to start reading from """ - ReadCoilsRequest.__init__(self, address, 16) + ReadCoilsRequest.__init__(self, address, 16, **kwargs) # --------------------------------------------------------------------------- # # execute the request with your client @@ -90,7 +117,8 @@ def __init__(self, address): if __name__ == "__main__": - with ModbusClient('127.0.0.1') as client: - request = CustomModbusRequest(0) + with ModbusClient(host='localhost', port=5020) as client: + client.register(CustomModbusResponse) + request = CustomModbusRequest(1, unit=1) result = client.execute(request) - print(result) + print(result.values) diff --git a/examples/common/custom_message_async_clients.py b/examples/common/custom_message_async_clients.py new file mode 100755 index 000000000..bf339c139 --- /dev/null +++ b/examples/common/custom_message_async_clients.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +""" +Pymodbus Synchronous Client Examples +-------------------------------------------------------------------------- + +The following is an example of how to use the synchronous modbus client +implementation from pymodbus. + +It should be noted that the client can also be used with +the guard construct that is available in python 2.5 and up:: + + with ModbusClient('127.0.0.1') as client: + result = client.read_coils(1,10) + print result +""" +import struct +# --------------------------------------------------------------------------- # +# import the various server implementations +# --------------------------------------------------------------------------- # +from pymodbus.pdu import ModbusRequest, ModbusResponse, ModbusExceptions +from pymodbus.client.sync import ModbusTcpClient as ModbusClient +from pymodbus.bit_read_message import ReadCoilsRequest +from pymodbus.compat import int2byte, byte2int +# --------------------------------------------------------------------------- # +# configure the client logging +# --------------------------------------------------------------------------- # +import logging +logging.basicConfig() +log = logging.getLogger() +log.setLevel(logging.DEBUG) + +# --------------------------------------------------------------------------- # +# create your custom message +# --------------------------------------------------------------------------- # +# The following is simply a read coil request that always reads 16 coils. +# Since the function code is already registered with the decoder factory, +# this will be decoded as a read coil response. If you implement a new +# method that is not currently implemented, you must register the request +# and response with a ClientDecoder factory. +# --------------------------------------------------------------------------- # + + +class CustomModbusResponse(ModbusResponse): + function_code = 55 + _rtu_byte_count_pos = 2 + + def __init__(self, values=None, **kwargs): + ModbusResponse.__init__(self, **kwargs) + self.values = values or [] + + def encode(self): + """ Encodes response pdu + + :returns: The encoded packet message + """ + result = int2byte(len(self.values) * 2) + for register in self.values: + result += struct.pack('>H', register) + return result + + def decode(self, data): + """ Decodes response pdu + + :param data: The packet data to decode + """ + byte_count = byte2int(data[0]) + self.values = [] + for i in range(1, byte_count + 1, 2): + self.values.append(struct.unpack('>H', data[i:i + 2])[0]) + + +class CustomModbusRequest(ModbusRequest): + + function_code = 55 + _rtu_frame_size = 8 + + def __init__(self, address=None, **kwargs): + ModbusRequest.__init__(self, **kwargs) + self.address = address + self.count = 16 + + def encode(self): + return struct.pack('>HH', self.address, self.count) + + def decode(self, data): + self.address, self.count = struct.unpack('>HH', data) + + def execute(self, context): + if not (1 <= self.count <= 0x7d0): + return self.doException(ModbusExceptions.IllegalValue) + if not context.validate(self.function_code, self.address, self.count): + return self.doException(ModbusExceptions.IllegalAddress) + values = context.getValues(self.function_code, self.address, + self.count) + return CustomModbusResponse(values) + +# --------------------------------------------------------------------------- # +# This could also have been defined as +# --------------------------------------------------------------------------- # + + +class Read16CoilsRequest(ReadCoilsRequest): + + def __init__(self, address, **kwargs): + """ Initializes a new instance + + :param address: The address to start reading from + """ + ReadCoilsRequest.__init__(self, address, 16, **kwargs) + +# --------------------------------------------------------------------------- # +# execute the request with your client +# --------------------------------------------------------------------------- # +# using the with context, the client will automatically be connected +# and closed when it leaves the current scope. +# --------------------------------------------------------------------------- # + + +if __name__ == "__main__": + with ModbusClient(host='localhost', port=5020) as client: + client.register(CustomModbusResponse) + request = CustomModbusRequest(1, unit=1) + result = client.execute(request) + print(result.values) diff --git a/examples/common/custom_synchronous_server.py b/examples/common/custom_synchronous_server.py new file mode 100755 index 000000000..191b6fac0 --- /dev/null +++ b/examples/common/custom_synchronous_server.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +""" +Pymodbus Synchronous Server Example with Custom functions +-------------------------------------------------------------------------- + +Implements a custom function code not in standard modbus function code list +and its response which otherwise would throw `IllegalFunction (0x1)` error. + +Steps: +1. Create CustomModbusRequest class derived from ModbusRequest + ```class CustomModbusRequest(ModbusRequest): + function_code = 75 # Value less than 0x80) + _rtu_frame_size = # Required only For RTU support + + def __init__(custom_arg=None, **kwargs) + # Make sure the arguments has default values, will error out + # while decoding otherwise + ModbusRequest.__init__(self, **kwargs) + self.custom_request_arg = custom_arg + + def encode(self): + # Implement encoding logic + + def decode(self, data): + # implement decoding logic + + def execute(self, context): + # Implement execute logic + ... + return CustomModbusResponse(..) + + ``` +2. Create CustomModbusResponse class derived from ModbusResponse + ```class CustomModbusResponse(ModbusResponse): + function_code = 75 # Value less than 0x80) + _rtu_byte_count_pos = # Required only For RTU support + + def __init__(self, custom_args=None, **kwargs): + # Make sure the arguments has default values, will error out + # while decoding otherwise + ModbusResponse.__init__(self, **kwargs) + self.custom_reponse_values = values + + def encode(self): + # Implement encoding logic + def decode(self, data): + # Implement decoding logic + ``` +3. Register with ModbusSlaveContext, + if your request has to access some values from the data-store. + ```store = ModbusSlaveContext(...) + store.register(CustomModbusRequest.function_code, 'dummy_context_name') + ``` +4. Pass CustomModbusRequest class as argument to StartServer + ``` + StartTcpServer(..., custom_functions=[CustomModbusRequest]..) + ``` + +""" +# --------------------------------------------------------------------------- # +# import the various server implementations +# --------------------------------------------------------------------------- # +from pymodbus.server.sync import StartTcpServer + +from pymodbus.device import ModbusDeviceIdentification +from pymodbus.datastore import ModbusSequentialDataBlock +from pymodbus.datastore import ModbusSlaveContext, ModbusServerContext +from custom_message import CustomModbusRequest + +# --------------------------------------------------------------------------- # +# configure the service logging +# --------------------------------------------------------------------------- # +import logging + +FORMAT = ('%(asctime)-15s %(threadName)-15s' + ' %(levelname)-8s %(module)-15s:%(lineno)-8s %(message)s') +logging.basicConfig(format=FORMAT) +log = logging.getLogger() +log.setLevel(logging.DEBUG) + + +def run_server(): + store = ModbusSlaveContext( + di=ModbusSequentialDataBlock(0, [17] * 100), + co=ModbusSequentialDataBlock(0, [17] * 100), + hr=ModbusSequentialDataBlock(0, [17] * 100), + ir=ModbusSequentialDataBlock(0, [17] * 100)) + + store.register(CustomModbusRequest.function_code, 'cm', + ModbusSequentialDataBlock(0, [17] * 100)) + context = ModbusServerContext(slaves=store, single=True) + + # ----------------------------------------------------------------------- # + # initialize the server information + # ----------------------------------------------------------------------- # + # If you don't set this or any fields, they are defaulted to empty strings. + # ----------------------------------------------------------------------- # + identity = ModbusDeviceIdentification() + identity.VendorName = 'Pymodbus' + identity.ProductCode = 'PM' + identity.VendorUrl = 'http://github.com/riptideio/pymodbus/' + identity.ProductName = 'Pymodbus Server' + identity.ModelName = 'Pymodbus Server' + identity.MajorMinorRevision = '2.1.0' + + # ----------------------------------------------------------------------- # + # run the server you want + # ----------------------------------------------------------------------- # + # Tcp: + StartTcpServer(context, identity=identity, address=("localhost", 5020), + custom_functions=[CustomModbusRequest]) + + +if __name__ == "__main__": + run_server() + diff --git a/pymodbus/client/sync.py b/pymodbus/client/sync.py index 01ca0fcbe..93c54b80c 100644 --- a/pymodbus/client/sync.py +++ b/pymodbus/client/sync.py @@ -157,6 +157,14 @@ def _dump(self, data, direction): self._logger.debug(hexlify_packets(data)) self._logger.exception(e) + def register(self, function): + """ + Registers a function and sub function class with the decoder + :param function: Custom function class to register + :return: + """ + self.framer.decoder.register(function) + def __str__(self): """ Builds a string representation of the connection diff --git a/pymodbus/datastore/context.py b/pymodbus/datastore/context.py index a99cbc204..a440546d2 100644 --- a/pymodbus/datastore/context.py +++ b/pymodbus/datastore/context.py @@ -88,6 +88,17 @@ def setValues(self, fx, address, values): _logger.debug("setValues[%d] %d:%d" % (fx, address, len(values))) self.store[self.decode(fx)].setValues(address, values) + def register(self, fc, fx, datablock=None): + """ + Registers a datablock with the slave context + :param fc: function code (int) + :param fx: string representation of function code (e.g 'cf' ) + :param datablock: datablock to associate with this function code + :return: + """ + self.store[fx] = datablock or ModbusSequentialDataBlock.create() + self._IModbusSlaveContext__fx_mapper[fc] = fx + class ModbusServerContext(object): ''' This represents a master collection of slave contexts. diff --git a/pymodbus/exceptions.py b/pymodbus/exceptions.py index 9c6a6844a..7918f6479 100644 --- a/pymodbus/exceptions.py +++ b/pymodbus/exceptions.py @@ -1,18 +1,18 @@ -''' +""" Pymodbus Exceptions -------------------- Custom exceptions to be used in the Modbus code. -''' +""" class ModbusException(Exception): - ''' Base modbus exception ''' + """ Base modbus exception """ def __init__(self, string): - ''' Initialize the exception + """ Initialize the exception :param string: The message to append to the error - ''' + """ self.string = string def __str__(self): @@ -24,61 +24,61 @@ def isError(self): class ModbusIOException(ModbusException): - ''' Error resulting from data i/o ''' + """ Error resulting from data i/o """ def __init__(self, string="", function_code=None): - ''' Initialize the exception + """ Initialize the exception :param string: The message to append to the error - ''' + """ self.fcode = function_code self.message = "[Input/Output] %s" % string ModbusException.__init__(self, self.message) class ParameterException(ModbusException): - ''' Error resulting from invalid parameter ''' + """ Error resulting from invalid parameter """ def __init__(self, string=""): - ''' Initialize the exception + """ Initialize the exception :param string: The message to append to the error - ''' + """ message = "[Invalid Parameter] %s" % string ModbusException.__init__(self, message) class NoSuchSlaveException(ModbusException): - ''' Error resulting from making a request to a slave - that does not exist ''' + """ Error resulting from making a request to a slave + that does not exist """ def __init__(self, string=""): - ''' Initialize the exception + """ Initialize the exception :param string: The message to append to the error - ''' + """ message = "[No Such Slave] %s" % string ModbusException.__init__(self, message) class NotImplementedException(ModbusException): - ''' Error resulting from not implemented function ''' + """ Error resulting from not implemented function """ def __init__(self, string=""): - ''' Initialize the exception + """ Initialize the exception :param string: The message to append to the error - ''' + """ message = "[Not Implemented] %s" % string ModbusException.__init__(self, message) class ConnectionException(ModbusException): - ''' Error resulting from a bad connection ''' + """ Error resulting from a bad connection """ def __init__(self, string=""): - ''' Initialize the exception + """ Initialize the exception :param string: The message to append to the error - ''' + """ message = "[Connection] %s" % string ModbusException.__init__(self, message) @@ -89,20 +89,30 @@ class InvalidMessageReceivedException(ModbusException): """ def __init__(self, string=""): - ''' Initialize the exception + """ Initialize the exception :param string: The message to append to the error - ''' + """ message = "[Invalid Message] %s" % string ModbusException.__init__(self, message) -#---------------------------------------------------------------------------# +class MessageRegisterException(ModbusException): + """ + Error resulting from failing to register a custom message request/response + """ + def __init__(self, string=""): + message = '[Error registering message] %s' % string + ModbusException.__init__(self, message) + + +# --------------------------------------------------------------------------- # # Exported symbols -#---------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # __all__ = [ "ModbusException", "ModbusIOException", "ParameterException", "NotImplementedException", "ConnectionException", "NoSuchSlaveException", - "InvalidMessageReceivedException" + "InvalidMessageReceivedException", + "MessageRegisterException" ] diff --git a/pymodbus/factory.py b/pymodbus/factory.py index 37f3eb491..6f54ad78e 100644 --- a/pymodbus/factory.py +++ b/pymodbus/factory.py @@ -13,9 +13,10 @@ from pymodbus.pdu import IllegalFunctionRequest from pymodbus.pdu import ExceptionResponse +from pymodbus.pdu import ModbusRequest, ModbusResponse from pymodbus.pdu import ModbusExceptions as ecode from pymodbus.interfaces import IModbusDecoder -from pymodbus.exceptions import ModbusException +from pymodbus.exceptions import ModbusException, MessageRegisterException from pymodbus.bit_read_message import * from pymodbus.bit_write_message import * from pymodbus.diag_message import * @@ -26,21 +27,22 @@ from pymodbus.register_write_message import * from pymodbus.compat import byte2int -#---------------------------------------------------------------------------# + +# --------------------------------------------------------------------------- # # Logging -#---------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # import logging _logger = logging.getLogger(__name__) -#---------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # # Server Decoder -#---------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # class ServerDecoder(IModbusDecoder): - ''' Request Message Factory (Server) + """ Request Message Factory (Server) To add more implemented functions, simply add them to the list - ''' + """ __function_table = [ ReadHoldingRegistersRequest, ReadDiscreteInputsRequest, @@ -51,19 +53,15 @@ class ServerDecoder(IModbusDecoder): WriteSingleRegisterRequest, WriteSingleCoilRequest, ReadWriteMultipleRegistersRequest, - DiagnosticStatusRequest, - ReadExceptionStatusRequest, GetCommEventCounterRequest, GetCommEventLogRequest, ReportSlaveIdRequest, - ReadFileRecordRequest, WriteFileRecordRequest, MaskWriteRegisterRequest, ReadFifoQueueRequest, - ReadDeviceInformationRequest, ] __sub_function_table = [ @@ -84,13 +82,12 @@ class ServerDecoder(IModbusDecoder): ReturnIopOverrunCountRequest, ClearOverrunCountRequest, GetClearModbusPlusRequest, - ReadDeviceInformationRequest, ] def __init__(self): - ''' Initializes the client lookup tables - ''' + """ Initializes the client lookup tables + """ functions = set(f.function_code for f in self.__function_table) self.__lookup = dict([(f.function_code, f) for f in self.__function_table]) self.__sub_lookup = dict((f, {}) for f in functions) @@ -98,11 +95,11 @@ def __init__(self): self.__sub_lookup[f.function_code][f.sub_function_code] = f def decode(self, message): - ''' Wrapper to decode a request packet + """ Wrapper to decode a request packet :param message: The raw modbus request packet :return: The decoded modbus message or None if error - ''' + """ try: return self._helper(message) except ModbusException as er: @@ -110,22 +107,22 @@ def decode(self, message): return None def lookupPduClass(self, function_code): - ''' Use `function_code` to determine the class of the PDU. + """ Use `function_code` to determine the class of the PDU. :param function_code: The function code specified in a frame. :returns: The class of the PDU that has a matching `function_code`. - ''' + """ return self.__lookup.get(function_code, ExceptionResponse) def _helper(self, data): - ''' + """ This factory is used to generate the correct request object from a valid request packet. This decodes from a list of the currently implemented request types. :param data: The request packet to decode :returns: The decoded request or illegal function request object - ''' + """ function_code = byte2int(data[0]) _logger.debug("Factory Request[%d]" % function_code) request = self.__lookup.get(function_code, lambda: None)() @@ -139,16 +136,36 @@ def _helper(self, data): if subtype: request.__class__ = subtype return request - - -#---------------------------------------------------------------------------# + + def register(self, function=None): + """ + Registers a function and sub function class with the decoder + :param function: Custom function class to register + :return: + """ + if function and not issubclass(function, ModbusRequest): + raise MessageRegisterException("'{}' is Not a valid Modbus Message" + ". Class needs to be derived from " + "`pymodbus.pdu.ModbusRequest` " + "".format( + function.__class__.__name__ + )) + self.__lookup[function.function_code] = function + if hasattr(function, "sub_function_code"): + if function.function_code not in self.__sub_lookup: + self.__sub_lookup[function.function_code] = dict() + self.__sub_lookup[function.function_code][ + function.sub_function_code] = function + + +# --------------------------------------------------------------------------- # # Client Decoder -#---------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # class ClientDecoder(IModbusDecoder): - ''' Response Message Factory (Client) + """ Response Message Factory (Client) To add more implemented functions, simply add them to the list - ''' + """ __function_table = [ ReadHoldingRegistersResponse, ReadDiscreteInputsResponse, @@ -159,19 +176,15 @@ class ClientDecoder(IModbusDecoder): WriteSingleRegisterResponse, WriteSingleCoilResponse, ReadWriteMultipleRegistersResponse, - DiagnosticStatusResponse, - ReadExceptionStatusResponse, GetCommEventCounterResponse, GetCommEventLogResponse, ReportSlaveIdResponse, - ReadFileRecordResponse, WriteFileRecordResponse, MaskWriteRegisterResponse, ReadFifoQueueResponse, - ReadDeviceInformationResponse, ] __sub_function_table = [ @@ -192,33 +205,33 @@ class ClientDecoder(IModbusDecoder): ReturnIopOverrunCountResponse, ClearOverrunCountResponse, GetClearModbusPlusResponse, - ReadDeviceInformationResponse, ] def __init__(self): - ''' Initializes the client lookup tables - ''' + """ Initializes the client lookup tables + """ functions = set(f.function_code for f in self.__function_table) - self.__lookup = dict([(f.function_code, f) for f in self.__function_table]) + self.__lookup = dict([(f.function_code, f) + for f in self.__function_table]) self.__sub_lookup = dict((f, {}) for f in functions) for f in self.__sub_function_table: self.__sub_lookup[f.function_code][f.sub_function_code] = f def lookupPduClass(self, function_code): - ''' Use `function_code` to determine the class of the PDU. + """ Use `function_code` to determine the class of the PDU. :param function_code: The function code specified in a frame. :returns: The class of the PDU that has a matching `function_code`. - ''' + """ return self.__lookup.get(function_code, ExceptionResponse) def decode(self, message): - ''' Wrapper to decode a response packet + """ Wrapper to decode a response packet :param message: The raw packet to decode :return: The decoded modbus message or None if error - ''' + """ try: return self._helper(message) except ModbusException as er: @@ -229,14 +242,14 @@ def decode(self, message): return None def _helper(self, data): - ''' + """ This factory is used to generate the correct response object from a valid response packet. This decodes from a list of the currently implemented request types. :param data: The response packet to decode :returns: The decoded request or an exception response object - ''' + """ fc_string = function_code = byte2int(data[0]) if function_code in self.__lookup: fc_string = "%s: %s" % ( @@ -259,7 +272,34 @@ def _helper(self, data): return response -#---------------------------------------------------------------------------# + def register(self, function=None, sub_function=None, force=False): + """ + Registers a function and sub function class with the decoder + :param function: Custom function class to register + :param sub_function: Custom sub function class to register + :param force: Force update the existing class + :return: + """ + if function and not issubclass(function, ModbusResponse): + raise MessageRegisterException("'{}' is Not a valid Modbus Message" + ". Class needs to be derived from " + "`pymodbus.pdu.ModbusResponse` " + "".format( + function.__class__.__name__ + )) + self.__lookup[function.function_code] = function + if hasattr(function, "sub_function_code"): + if function.function_code not in self.__sub_lookup: + self.__sub_lookup[function.function_code] = dict() + self.__sub_lookup[function.function_code][ + function.sub_function_code] = function + + +# --------------------------------------------------------------------------- # # Exported symbols -#---------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # + + __all__ = ['ServerDecoder', 'ClientDecoder'] + + diff --git a/pymodbus/interfaces.py b/pymodbus/interfaces.py index 4d5843018..d32e9978a 100644 --- a/pymodbus/interfaces.py +++ b/pymodbus/interfaces.py @@ -1,128 +1,138 @@ -''' +""" Pymodbus Interfaces --------------------- A collection of base classes that are used throughout the pymodbus library. -''' -from pymodbus.exceptions import NotImplementedException +""" +from pymodbus.exceptions import (NotImplementedException, + MessageRegisterException) -#---------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # # Generic -#---------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # class Singleton(object): - ''' + """ Singleton base class http://mail.python.org/pipermail/python-list/2007-July/450681.html - ''' + """ def __new__(cls, *args, **kwargs): - ''' Create a new instance - ''' + """ Create a new instance + """ if '_inst' not in vars(cls): cls._inst = object.__new__(cls) return cls._inst -#---------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # # Project Specific -#---------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # class IModbusDecoder(object): - ''' Modbus Decoder Base Class + """ Modbus Decoder Base Class This interface must be implemented by a modbus message decoder factory. These factories are responsible for abstracting away converting a raw packet into a request / response message object. - ''' + """ def decode(self, message): - ''' Wrapper to decode a given packet + """ Wrapper to decode a given packet :param message: The raw modbus request packet :return: The decoded modbus message or None if error - ''' + """ raise NotImplementedException( "Method not implemented by derived class") def lookupPduClass(self, function_code): - ''' Use `function_code` to determine the class of the PDU. + """ Use `function_code` to determine the class of the PDU. :param function_code: The function code specified in a frame. :returns: The class of the PDU that has a matching `function_code`. - ''' + """ + raise NotImplementedException( + "Method not implemented by derived class") + + def register(self, function=None): + """ + Registers a function and sub function class with the decoder + :param function: Custom function class to register + :return: + """ raise NotImplementedException( "Method not implemented by derived class") class IModbusFramer(object): - ''' + """ A framer strategy interface. The idea is that we abstract away all the detail about how to detect if a current message frame exists, decoding it, sending it, etc so that we can plug in a new Framer object (tcp, rtu, ascii). - ''' + """ def checkFrame(self): - ''' Check and decode the next frame + """ Check and decode the next frame :returns: True if we successful, False otherwise - ''' + """ raise NotImplementedException( "Method not implemented by derived class") def advanceFrame(self): - ''' Skip over the current framed message + """ Skip over the current framed message This allows us to skip over the current message after we have processed it or determined that it contains an error. It also has to reset the current frame header handle - ''' + """ raise NotImplementedException( "Method not implemented by derived class") def addToFrame(self, message): - ''' Add the next message to the frame buffer + """ Add the next message to the frame buffer This should be used before the decoding while loop to add the received data to the buffer handle. :param message: The most recent packet - ''' + """ raise NotImplementedException( "Method not implemented by derived class") def isFrameReady(self): - ''' Check if we should continue decode logic + """ Check if we should continue decode logic This is meant to be used in a while loop in the decoding phase to let the decoder know that there is still data in the buffer. :returns: True if ready, False otherwise - ''' + """ raise NotImplementedException( "Method not implemented by derived class") def getFrame(self): - ''' Get the next frame from the buffer + """ Get the next frame from the buffer :returns: The frame data or '' - ''' + """ raise NotImplementedException( "Method not implemented by derived class") def populateResult(self, result): - ''' Populates the modbus result with current frame header + """ Populates the modbus result with current frame header We basically copy the data back over from the current header to the result header. This may not be needed for serial messages. :param result: The response packet - ''' + """ raise NotImplementedException( "Method not implemented by derived class") def processIncomingPacket(self, data, callback): - ''' The new packet processing pattern + """ The new packet processing pattern This takes in a new request packet, adds it to the current packet stream, and performs framing on it. That is, checks @@ -135,25 +145,25 @@ def processIncomingPacket(self, data, callback): :param data: The new packet data :param callback: The function to send results to - ''' + """ raise NotImplementedException( "Method not implemented by derived class") def buildPacket(self, message): - ''' Creates a ready to send modbus packet + """ Creates a ready to send modbus packet The raw packet is built off of a fully populated modbus request / response message. :param message: The request/response to send :returns: The built packet - ''' + """ raise NotImplementedException( "Method not implemented by derived class") class IModbusSlaveContext(object): - ''' + """ Interface for a modbus slave data context Derived classes must implemented the following methods: @@ -161,76 +171,76 @@ class IModbusSlaveContext(object): validate(self, fx, address, count=1) getValues(self, fx, address, count=1) setValues(self, fx, address, values) - ''' + """ __fx_mapper = {2: 'd', 4: 'i'} __fx_mapper.update([(i, 'h') for i in [3, 6, 16, 22, 23]]) __fx_mapper.update([(i, 'c') for i in [1, 5, 15]]) def decode(self, fx): - ''' Converts the function code to the datastore to + """ Converts the function code to the datastore to :param fx: The function we are working with :returns: one of [d(iscretes),i(inputs),h(oliding),c(oils) - ''' + """ return self.__fx_mapper[fx] def reset(self): - ''' Resets all the datastores to their default values - ''' + """ Resets all the datastores to their default values + """ raise NotImplementedException("Context Reset") def validate(self, fx, address, count=1): - ''' Validates the request to make sure it is in range + """ Validates the request to make sure it is in range :param fx: The function we are working with :param address: The starting address :param count: The number of values to test :returns: True if the request in within range, False otherwise - ''' + """ raise NotImplementedException("validate context values") def getValues(self, fx, address, count=1): - ''' Get `count` values from datastore + """ Get `count` values from datastore :param fx: The function we are working with :param address: The starting address :param count: The number of values to retrieve :returns: The requested values from a:a+c - ''' + """ raise NotImplementedException("get context values") def setValues(self, fx, address, values): - ''' Sets the datastore with the supplied values + """ Sets the datastore with the supplied values :param fx: The function we are working with :param address: The starting address :param values: The new values to be set - ''' + """ raise NotImplementedException("set context values") class IPayloadBuilder(object): - ''' + """ This is an interface to a class that can build a payload for a modbus register write command. It should abstract the codec for encoding data to the required format (bcd, binary, char, etc). - ''' + """ def build(self): - ''' Return the payload buffer as a list + """ Return the payload buffer as a list This list is two bytes per element and can thus be treated as a list of registers. :returns: The payload buffer as a list - ''' + """ raise NotImplementedException("set context values") -#---------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # # Exported symbols -#---------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # __all__ = [ 'Singleton', 'IModbusDecoder', 'IModbusFramer', 'IModbusSlaveContext', diff --git a/pymodbus/server/async.py b/pymodbus/server/async.py index d58027a0d..14150cd32 100644 --- a/pymodbus/server/async.py +++ b/pymodbus/server/async.py @@ -154,7 +154,8 @@ def __init__(self, store, framer=None, identity=None, **kwargs): a missing slave """ framer = framer or ModbusSocketFramer - self.framer = framer(decoder=ServerDecoder()) + self.decoder = ServerDecoder() + self.framer = framer(self.decoder) self.store = store or ModbusServerContext() self.control = ModbusControlBlock() self.access = ModbusAccessControl() @@ -230,7 +231,8 @@ def _is_main_thread(): def StartTcpServer(context, identity=None, address=None, - console=False, defer_reactor_run=False, **kwargs): + console=False, defer_reactor_run=False, custom_functions=[], + **kwargs): """ Helper method to start the Modbus Async TCP server @@ -242,12 +244,16 @@ def StartTcpServer(context, identity=None, address=None, to a missing slave :param defer_reactor_run: True/False defer running reactor.run() as part \ of starting server, to be explictly started by the user + :param custom_functions: An optional list of custom function classes + supported by server instance. """ from twisted.internet import reactor address = address or ("", Defaults.Port) framer = kwargs.pop("framer", ModbusSocketFramer) factory = ModbusServerFactory(context, framer, identity, **kwargs) + for f in custom_functions: + factory.decoder.register(f) if console: InstallManagementConsole({'factory': factory}) @@ -258,7 +264,7 @@ def StartTcpServer(context, identity=None, address=None, def StartUdpServer(context, identity=None, address=None, - defer_reactor_run=False, **kwargs): + defer_reactor_run=False, custom_functions=[], **kwargs): """ Helper method to start the Modbus Async Udp server @@ -269,12 +275,16 @@ def StartUdpServer(context, identity=None, address=None, to a missing slave :param defer_reactor_run: True/False defer running reactor.run() as part \ of starting server, to be explictly started by the user + :param custom_functions: An optional list of custom function classes + supported by server instance. """ from twisted.internet import reactor address = address or ("", Defaults.Port) framer = kwargs.pop("framer", ModbusSocketFramer) - server = ModbusUdpProtocol(context, framer, identity, **kwargs) + server = ModbusUdpProtocol(context, framer, identity, **kwargs) + for f in custom_functions: + server.decoder.register(f) _logger.info("Starting Modbus UDP Server on %s:%s" % address) reactor.listenUDP(address[1], server, interface=address[0]) @@ -282,10 +292,8 @@ def StartUdpServer(context, identity=None, address=None, reactor.run(installSignalHandlers=_is_main_thread()) -def StartSerialServer(context, identity=None, - framer=ModbusAsciiFramer, - defer_reactor_run=False, - **kwargs): +def StartSerialServer(context, identity=None, framer=ModbusAsciiFramer, + defer_reactor_run=False, custom_functions=[], **kwargs): """ Helper method to start the Modbus Async Serial server @@ -299,6 +307,9 @@ def StartSerialServer(context, identity=None, missing slave :param defer_reactor_run: True/False defer running reactor.run() as part \ of starting server, to be explictly started by the user + :param custom_functions: An optional list of custom function classes + supported by server instance. + """ from twisted.internet import reactor from twisted.internet.serialport import SerialPort @@ -309,6 +320,10 @@ def StartSerialServer(context, identity=None, _logger.info("Starting Modbus Serial Server on %s" % port) factory = ModbusServerFactory(context, framer, identity, **kwargs) + for f in custom_functions: + factory.decoder.register(f) + if console: + InstallManagementConsole({'factory': factory}) if console: InstallManagementConsole({'factory': factory}) diff --git a/pymodbus/server/sync.py b/pymodbus/server/sync.py index 937a89294..01d65685c 100644 --- a/pymodbus/server/sync.py +++ b/pymodbus/server/sync.py @@ -83,6 +83,7 @@ def handle(self): """ raise NotImplementedException("Method not implemented" " by derived class") + def send(self, message): """ Send a request (string) to the network @@ -91,6 +92,7 @@ def send(self, message): raise NotImplementedException("Method not implemented " "by derived class") + class ModbusSingleRequestHandler(ModbusBaseRequestHandler): """ Implements the modbus server protocol @@ -505,40 +507,54 @@ def server_close(self): # --------------------------------------------------------------------------- # # Creation Factories # --------------------------------------------------------------------------- # -def StartTcpServer(context=None, identity=None, address=None, **kwargs): +def StartTcpServer(context=None, identity=None, address=None, + custom_functions=[], **kwargs): """ A factory to start and run a tcp modbus server :param context: The ModbusServerContext datastore :param identity: An optional identify structure :param address: An optional (interface, port) to bind to. + :param custom_functions: An optional list of custom function classes + supported by server instance. :param ignore_missing_slaves: True to not send errors on a request to a - missing slave + missing slave """ framer = kwargs.pop("framer", ModbusSocketFramer) server = ModbusTcpServer(context, framer, identity, address, **kwargs) + + for f in custom_functions: + server.decoder.register(f) server.serve_forever() -def StartUdpServer(context=None, identity=None, address=None, **kwargs): +def StartUdpServer(context=None, identity=None, address=None, + custom_functions=[], **kwargs): """ A factory to start and run a udp modbus server :param context: The ModbusServerContext datastore :param identity: An optional identify structure :param address: An optional (interface, port) to bind to. + :param custom_functions: An optional list of custom function classes + supported by server instance. :param framer: The framer to operate with (default ModbusSocketFramer) :param ignore_missing_slaves: True to not send errors on a request to a missing slave """ framer = kwargs.pop('framer', ModbusSocketFramer) server = ModbusUdpServer(context, framer, identity, address, **kwargs) + for f in custom_functions: + server.decoder.register(f) server.serve_forever() -def StartSerialServer(context=None, identity=None, **kwargs): +def StartSerialServer(context=None, identity=None, custom_functions=[], + **kwargs): """ A factory to start and run a serial modbus server :param context: The ModbusServerContext datastore :param identity: An optional identify structure + :param custom_functions: An optional list of custom function classes + supported by server instance. :param framer: The framer to operate with (default ModbusAsciiFramer) :param port: The serial port to attach to :param stopbits: The number of stop bits to use @@ -551,6 +567,8 @@ def StartSerialServer(context=None, identity=None, **kwargs): """ framer = kwargs.pop('framer', ModbusAsciiFramer) server = ModbusSerialServer(context, framer, identity, **kwargs) + for f in custom_functions: + server.decoder.register(f) server.serve_forever() # --------------------------------------------------------------------------- # diff --git a/test/test_client_sync.py b/test/test_client_sync.py index 5e5c62d7f..9c9496283 100644 --- a/test/test_client_sync.py +++ b/test/test_client_sync.py @@ -241,6 +241,14 @@ def testSerialClientRpr(self): client.host, client.port, client.timeout ) self.assertEqual(repr(client), rep) + + def testTcpClientRegister(self): + class CustomeRequest: + function_code = 79 + client = ModbusTcpClient() + client.framer = Mock() + client.register(CustomeRequest) + assert client.framer.decoder.register.called_once_with(CustomeRequest) # -----------------------------------------------------------------------# # Test Serial Client # -----------------------------------------------------------------------# @@ -352,6 +360,8 @@ def testSerialClientRepr(self): client.method, client.timeout ) self.assertEqual(repr(client), rep) + + # ---------------------------------------------------------------------------# # Main # ---------------------------------------------------------------------------# diff --git a/test/test_factory.py b/test/test_factory.py index f5aec7f3d..e0cc6cbb4 100644 --- a/test/test_factory.py +++ b/test/test_factory.py @@ -1,7 +1,8 @@ #!/usr/bin/env python import unittest from pymodbus.factory import ServerDecoder, ClientDecoder -from pymodbus.exceptions import ModbusException +from pymodbus.exceptions import ModbusException, MessageRegisterException +from pymodbus.pdu import ModbusResponse, ModbusRequest def _raise_exception(_): raise ModbusException('something') @@ -142,6 +143,23 @@ def testServerFactoryFails(self): actual = self.server.decode(None) self.assertEqual(actual, None) + def testServerRegisterCustomRequest(self): + class CustomRequest(ModbusRequest): + function_code = 0xff + self.server.register(CustomRequest) + assert self.client.lookupPduClass(CustomRequest.function_code) + CustomRequest.sub_function_code = 0xff + self.server.register(CustomRequest) + assert self.server.lookupPduClass(CustomRequest.function_code) + + def testClientRegisterCustomResponse(self): + class CustomResponse(ModbusResponse): + function_code = 0xff + self.client.register(CustomResponse) + assert self.client.lookupPduClass(CustomResponse.function_code) + CustomResponse.sub_function_code = 0xff + self.client.register(CustomResponse) + assert self.client.lookupPduClass(CustomResponse.function_code) #---------------------------------------------------------------------------# # I don't actually know what is supposed to be returned here, I assume that # since the high bit is set, it will simply echo the resulting message diff --git a/test/test_interfaces.py b/test/test_interfaces.py index 722d27a52..f20c67a73 100644 --- a/test/test_interfaces.py +++ b/test/test_interfaces.py @@ -30,7 +30,10 @@ def testModbusDecoderInterface(self): x = None instance = IModbusDecoder() self.assertRaises(NotImplementedException, lambda: instance.decode(x)) - self.assertRaises(NotImplementedException, lambda: instance.lookupPduClass(x)) + self.assertRaises(NotImplementedException, + lambda: instance.lookupPduClass(x)) + self.assertRaises(NotImplementedException, + lambda: instance.register(x)) def testModbusFramerInterface(self): ''' Test that the base class isn't implemented ''' diff --git a/test/test_server_context.py b/test/test_server_context.py index 1b0e1beec..3781c901c 100644 --- a/test/test_server_context.py +++ b/test/test_server_context.py @@ -48,6 +48,14 @@ def testSingleContextSet(self): actual = self.context[0x00] self.assertEqual(slave, actual) + def testSingleContestRegister(self): + db = [1, 2, 3] + slave = ModbusSlaveContext() + slave.register(0xff, 'custom_request', db) + assert slave.store["custom_request"] == db + assert slave.decode(0xff) == 'custom_request' + + class ModbusServerMultipleContextTest(unittest.TestCase): ''' This is the unittest for the pymodbus.datastore.ModbusServerContext using multiple slave contexts. From f07dcee454f1601295590c3fb268a1e61ef34f4a Mon Sep 17 00:00:00 2001 From: Michael Muhlbaier Date: Thu, 10 Jan 2019 06:09:34 -0500 Subject: [PATCH 02/26] #368 Fixes write to broadcast address When writing to broadcast address (unit_id=0) there should be no response according to the Modbus spec. This fix changes expected_response_length to 0 when writing to unit_id=0. This will break any existing code that is improperly using unit_id 0 for a slave address. --- pymodbus/transaction.py | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/pymodbus/transaction.py b/pymodbus/transaction.py index d869f8413..ddfa06f4d 100644 --- a/pymodbus/transaction.py +++ b/pymodbus/transaction.py @@ -126,7 +126,10 @@ def execute(self, request): response_pdu_size = response_pdu_size * 2 if response_pdu_size: expected_response_length = self._calculate_response_length(response_pdu_size) - if request.unit_id in self._no_response_devices: + if request.unit_id == 0: + full = True + expected_response_length = 0 + elif request.unit_id in self._no_response_devices: full = True else: full = False @@ -161,21 +164,24 @@ def execute(self, request): # Remove entry self._no_response_devices.remove(request.unit_id) break - addTransaction = partial(self.addTransaction, - tid=request.transaction_id) - self.client.framer.processIncomingPacket(response, - addTransaction, - request.unit_id) - response = self.getTransaction(request.transaction_id) - if not response: - if len(self.transactions): - response = self.getTransaction(tid=0) - else: - last_exception = last_exception or ( - "No Response received from the remote unit" - "/Unable to decode response") - response = ModbusIOException(last_exception, - request.function_code) + if expected_response_length > 0: + addTransaction = partial(self.addTransaction, + tid=request.transaction_id) + self.client.framer.processIncomingPacket(response, + addTransaction, + request.unit_id) + response = self.getTransaction(request.transaction_id) + if not response: + if len(self.transactions): + response = self.getTransaction(tid=0) + else: + last_exception = last_exception or ( + "No Response received from the remote unit" + "/Unable to decode response") + response = ModbusIOException(last_exception, + request.function_code) + else: + _logger.debug("No response expected when sending to broadcast address 0") if hasattr(self.client, "state"): _logger.debug("Changing transaction state from " "'PROCESSING REPLY' to " From 2e169b61b9149f1855618031e0dc40c8d16a4773 Mon Sep 17 00:00:00 2001 From: dhoomakethu Date: Mon, 14 Jan 2019 09:12:42 +0530 Subject: [PATCH 03/26] Bump version to 2.2.0 Fix #366 Update failures in sql context Update Changelog Fix major minor version in example codes --- CHANGELOG.rst | 12 +++- examples/common/asynchronous_server.py | 2 +- examples/common/callback_server.py | 2 +- examples/common/custom_datablock.py | 2 +- examples/common/dbstore_update_server.py | 6 +- examples/common/modbus_payload_server.py | 2 +- examples/common/synchronous_server.py | 2 +- examples/common/updating_server.py | 2 +- .../contrib/deviceinfo_showcase_server.py | 2 +- examples/gui/bottle/frontend.py | 2 +- pymodbus/datastore/database/sql_datastore.py | 72 ++++++++++--------- pymodbus/version.py | 2 +- test/test_datastore.py | 8 ++- 13 files changed, 67 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3fad37cf5..713c9ccc0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,16 @@ -Version 2.1.1 +Version 2.2.0 ----------------------------------------------------------- +**NOTE: Supports python 3.7, async client is now moved to pymodbus/client/asychronous** +``` +from pymodbus.client.asynchronous import ModbusTcpClient +``` + +* Support Python 3.7 +* Fix `AttributeError` when setting `interCharTimeout` for serial clients. * Provide an option to disable inter char timeouts with Modbus RTU. +* Fix SQLDbcontext always returning InvalidAddress error. +* Fix SQLDbcontext update failure +* Fix Binary payload example for endianess. Version 2.1.0 ----------------------------------------------------------- diff --git a/examples/common/asynchronous_server.py b/examples/common/asynchronous_server.py index f961c62d9..79a791133 100755 --- a/examples/common/asynchronous_server.py +++ b/examples/common/asynchronous_server.py @@ -105,7 +105,7 @@ def run_async_server(): identity.VendorUrl = 'http://github.com/bashwork/pymodbus/' identity.ProductName = 'Pymodbus Server' identity.ModelName = 'Pymodbus Server' - identity.MajorMinorRevision = '1.5' + identity.MajorMinorRevision = '2.2.0' # ----------------------------------------------------------------------- # # run the server you want diff --git a/examples/common/callback_server.py b/examples/common/callback_server.py index b04110e30..d7f3a7bc4 100755 --- a/examples/common/callback_server.py +++ b/examples/common/callback_server.py @@ -132,7 +132,7 @@ def run_callback_server(): identity.VendorUrl = 'http://github.com/bashwork/pymodbus/' identity.ProductName = 'pymodbus Server' identity.ModelName = 'pymodbus Server' - identity.MajorMinorRevision = '1.0' + identity.MajorMinorRevision = '2.2.0' # ----------------------------------------------------------------------- # # run the server you want diff --git a/examples/common/custom_datablock.py b/examples/common/custom_datablock.py index db0d715bb..00b55fad0 100755 --- a/examples/common/custom_datablock.py +++ b/examples/common/custom_datablock.py @@ -68,7 +68,7 @@ def run_custom_db_server(): identity.VendorUrl = 'http://github.com/bashwork/pymodbus/' identity.ProductName = 'pymodbus Server' identity.ModelName = 'pymodbus Server' - identity.MajorMinorRevision = '1.0' + identity.MajorMinorRevision = '2.2.0' # ----------------------------------------------------------------------- # # run the server you want diff --git a/examples/common/dbstore_update_server.py b/examples/common/dbstore_update_server.py index 5de74c31d..ba37520ac 100644 --- a/examples/common/dbstore_update_server.py +++ b/examples/common/dbstore_update_server.py @@ -61,7 +61,8 @@ def updating_writer(a): rand_addr = random.randint(0, 65000) log.debug("Writing to datastore: {}, {}".format(rand_addr, rand_value)) # import pdb; pdb.set_trace() - context[slave_id].setValues(writefunction, rand_addr, [rand_value]) + context[slave_id].setValues(writefunction, rand_addr, [rand_value], + update=False) values = context[slave_id].getValues(readfunction, rand_addr, count) log.debug("Values from datastore: " + str(values)) @@ -85,7 +86,7 @@ def run_dbstore_update_server(): identity.VendorUrl = 'http://github.com/bashwork/pymodbus/' identity.ProductName = 'pymodbus Server' identity.ModelName = 'pymodbus Server' - identity.MajorMinorRevision = '1.0' + identity.MajorMinorRevision = '2.2.0' # ----------------------------------------------------------------------- # # run the server you want @@ -93,6 +94,7 @@ def run_dbstore_update_server(): time = 5 # 5 seconds delay loop = LoopingCall(f=updating_writer, a=(context,)) loop.start(time, now=False) # initially delay by time + loop.stop() StartTcpServer(context, identity=identity, address=("", 5020)) diff --git a/examples/common/modbus_payload_server.py b/examples/common/modbus_payload_server.py index b2eb58e78..d9d48d241 100755 --- a/examples/common/modbus_payload_server.py +++ b/examples/common/modbus_payload_server.py @@ -78,7 +78,7 @@ def run_payload_server(): identity.VendorUrl = 'http://github.com/bashwork/pymodbus/' identity.ProductName = 'Pymodbus Server' identity.ModelName = 'Pymodbus Server' - identity.MajorMinorRevision = '1.5' + identity.MajorMinorRevision = '2.2.0' # ----------------------------------------------------------------------- # # run the server you want # ----------------------------------------------------------------------- # diff --git a/examples/common/synchronous_server.py b/examples/common/synchronous_server.py index 233adaa0d..d3e53b23a 100755 --- a/examples/common/synchronous_server.py +++ b/examples/common/synchronous_server.py @@ -105,7 +105,7 @@ def run_server(): identity.VendorUrl = 'http://github.com/riptideio/pymodbus/' identity.ProductName = 'Pymodbus Server' identity.ModelName = 'Pymodbus Server' - identity.MajorMinorRevision = '1.5' + identity.MajorMinorRevision = '2.2.0' # ----------------------------------------------------------------------- # # run the server you want diff --git a/examples/common/updating_server.py b/examples/common/updating_server.py index 99fc33b8c..1393712a9 100755 --- a/examples/common/updating_server.py +++ b/examples/common/updating_server.py @@ -78,7 +78,7 @@ def run_updating_server(): identity.VendorUrl = 'http://github.com/bashwork/pymodbus/' identity.ProductName = 'pymodbus Server' identity.ModelName = 'pymodbus Server' - identity.MajorMinorRevision = '1.0' + identity.MajorMinorRevision = '2.2.0' # ----------------------------------------------------------------------- # # run the server you want diff --git a/examples/contrib/deviceinfo_showcase_server.py b/examples/contrib/deviceinfo_showcase_server.py index 96b003586..53bc753a0 100755 --- a/examples/contrib/deviceinfo_showcase_server.py +++ b/examples/contrib/deviceinfo_showcase_server.py @@ -55,7 +55,7 @@ def run_server(): identity.VendorUrl = 'http://github.com/riptideio/pymodbus/' identity.ProductName = 'Pymodbus Server' identity.ModelName = 'Pymodbus Server' - identity.MajorMinorRevision = '1.5' + identity.MajorMinorRevision = '2.2.0' # ----------------------------------------------------------------------- # # Add an example which is long enough to force the ReadDeviceInformation diff --git a/examples/gui/bottle/frontend.py b/examples/gui/bottle/frontend.py index a09fb0d15..c3e7c10c0 100644 --- a/examples/gui/bottle/frontend.py +++ b/examples/gui/bottle/frontend.py @@ -277,7 +277,7 @@ def RunDebugModbusFrontend(server, port=8080): identity.VendorUrl = 'http://github.com/bashwork/pymodbus/' identity.ProductName = 'Pymodbus Server' identity.ModelName = 'Pymodbus Server' - identity.MajorMinorRevision = '1.0' + identity.MajorMinorRevision = '2.2.0' # ------------------------------------------------------------ # initialize the datastore diff --git a/pymodbus/datastore/database/sql_datastore.py b/pymodbus/datastore/database/sql_datastore.py index 2ad6d3c55..769dab750 100644 --- a/pymodbus/datastore/database/sql_datastore.py +++ b/pymodbus/datastore/database/sql_datastore.py @@ -7,87 +7,91 @@ from pymodbus.exceptions import NotImplementedException from pymodbus.interfaces import IModbusSlaveContext -#---------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # # Logging -#---------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # import logging _logger = logging.getLogger(__name__) -#---------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # # Context -#---------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # class SqlSlaveContext(IModbusSlaveContext): - ''' + """ This creates a modbus data model with each data access stored in its own personal block - ''' + """ def __init__(self, *args, **kwargs): - ''' Initializes the datastores + """ Initializes the datastores :param kwargs: Each element is a ModbusDataBlock - ''' + """ self.table = kwargs.get('table', 'pymodbus') self.database = kwargs.get('database', 'sqlite:///pymodbus.db') self._db_create(self.table, self.database) def __str__(self): - ''' Returns a string representation of the context + """ Returns a string representation of the context :returns: A string representation of the context - ''' + """ return "Modbus Slave Context" def reset(self): - ''' Resets all the datastores to their default values ''' + """ Resets all the datastores to their default values """ self._metadata.drop_all() self._db_create(self.table, self.database) def validate(self, fx, address, count=1): - ''' Validates the request to make sure it is in range + """ Validates the request to make sure it is in range :param fx: The function we are working with :param address: The starting address :param count: The number of values to test :returns: True if the request in within range, False otherwise - ''' + """ address = address + 1 # section 4.4 of specification _logger.debug("validate[%d] %d:%d" % (fx, address, count)) return self._validate(self.decode(fx), address, count) def getValues(self, fx, address, count=1): - ''' Get `count` values from datastore + """ Get `count` values from datastore :param fx: The function we are working with :param address: The starting address :param count: The number of values to retrieve :returns: The requested values from a:a+c - ''' + """ address = address + 1 # section 4.4 of specification _logger.debug("get-values[%d] %d:%d" % (fx, address, count)) return self._get(self.decode(fx), address, count) - def setValues(self, fx, address, values): - ''' Sets the datastore with the supplied values + def setValues(self, fx, address, values, update=True): + """ Sets the datastore with the supplied values :param fx: The function we are working with :param address: The starting address :param values: The new values to be set - ''' + :param update: Update existing register in the db + """ address = address + 1 # section 4.4 of specification _logger.debug("set-values[%d] %d:%d" % (fx, address, len(values))) - self._set(self.decode(fx), address, values) + if update: + self._update(self.decode(fx), address, values) + else: + self._set(self.decode(fx), address, values) - #--------------------------------------------------------------------------# + # ----------------------------------------------------------------------- # # Sqlite Helper Methods - #--------------------------------------------------------------------------# + # ----------------------------------------------------------------------- # def _db_create(self, table, database): - ''' A helper method to initialize the database and handles + """ A helper method to initialize the database and handles :param table: The table name to create :param database: The database uri to use - ''' + """ self._engine = sqlalchemy.create_engine(database, echo=False) self._metadata = sqlalchemy.MetaData(self._engine) self._table = sqlalchemy.Table(table, self._metadata, @@ -99,12 +103,12 @@ def _db_create(self, table, database): self._connection = self._engine.connect() def _get(self, type, offset, count): - ''' + """ :param type: The key prefix to use :param offset: The address offset to start at :param count: The number of bits to read :returns: The resulting values - ''' + """ query = self._table.select(and_( self._table.c.type == type, self._table.c.index >= offset, @@ -115,13 +119,13 @@ def _get(self, type, offset, count): return [row.value for row in result] def _build_set(self, type, offset, values, prefix=''): - ''' A helper method to generate the sql update context + """ A helper method to generate the sql update context :param type: The key prefix to use :param offset: The address offset to start at :param values: The values to set :param prefix: Prefix fields index and type, defaults to empty string - ''' + """ result = [] for index, value in enumerate(values): result.append({ @@ -136,12 +140,12 @@ def _check(self, type, offset, values): return False if len(result) > 0 else True def _set(self, type, offset, values): - ''' + """ :param key: The type prefix to use :param offset: The address offset to start at :param values: The values to set - ''' + """ if self._check(type, offset, values): context = self._build_set(type, offset, values) query = self._table.insert() @@ -151,14 +155,14 @@ def _set(self, type, offset, values): return False def _update(self, type, offset, values): - ''' + """ :param type: The type prefix to use :param offset: The address offset to start at :param values: The values to set - ''' + """ context = self._build_set(type, offset, values, prefix='x_') - query = self._table.update().values(name='value') + query = self._table.update().values(value='value') query = query.where(and_( self._table.c.type == bindparam('x_type'), self._table.c.index == bindparam('x_index'))) @@ -166,12 +170,12 @@ def _update(self, type, offset, values): return result.rowcount == len(values) def _validate(self, type, offset, count): - ''' + """ :param key: The key prefix to use :param offset: The address offset to start at :param count: The number of bits to read :returns: The result of the validation - ''' + """ query = self._table.select(and_( self._table.c.type == type, self._table.c.index >= offset, diff --git a/pymodbus/version.py b/pymodbus/version.py index 1e8057b9b..51da88745 100644 --- a/pymodbus/version.py +++ b/pymodbus/version.py @@ -41,7 +41,7 @@ def __str__(self): return '[%s, version %s]' % (self.package, self.short()) -version = Version('pymodbus', 2, 1, 1) +version = Version('pymodbus', 2, 2, 0) version.__name__ = 'pymodbus' # fix epydoc error diff --git a/test/test_datastore.py b/test/test_datastore.py index 55f2166ff..cd9c44d3a 100644 --- a/test/test_datastore.py +++ b/test/test_datastore.py @@ -330,7 +330,8 @@ def testSetValues(self): self.slave._set = MagicMock() for key, value in self.function_map.items(): - self.slave.setValues(key, self.mock_addr, self.mock_values) + self.slave.setValues(key, self.mock_addr, + self.mock_values, update=False) self.slave._set.assert_called_with( value, self.mock_addr + 1, self.mock_values ) @@ -365,8 +366,9 @@ def testUpdateFailure(self): self.slave._update(self.mock_type, self.mock_offset, self.mock_values) ) -#---------------------------------------------------------------------------# + +# --------------------------------------------------------------------------- # # Main -#---------------------------------------------------------------------------# +# --------------------------------------------------------------------------- # if __name__ == "__main__": unittest.main() From 826240b54e881835639329858ddbede5961104de Mon Sep 17 00:00:00 2001 From: dhoomakethu Date: Mon, 14 Jan 2019 11:05:02 +0530 Subject: [PATCH 04/26] Fix #371 pymodbus repl on python3 --- pymodbus/repl/helper.py | 2 +- pymodbus/repl/main.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymodbus/repl/helper.py b/pymodbus/repl/helper.py index eff1023c7..7f255a7e6 100644 --- a/pymodbus/repl/helper.py +++ b/pymodbus/repl/helper.py @@ -20,7 +20,7 @@ if IS_PYTHON2 or PYTHON_VERSION < (3, 3): argspec = inspect.getargspec else: - predicate = inspect.ismethod + predicate = inspect.isfunction argspec = inspect.signature diff --git a/pymodbus/repl/main.py b/pymodbus/repl/main.py index d0501322c..cd13f29d1 100644 --- a/pymodbus/repl/main.py +++ b/pymodbus/repl/main.py @@ -41,7 +41,7 @@ \/ \/ \/ \/ \/ \/|__| v{} - {} ---------------------------------------------------------------------------- -""".format("1.0.0", version) +""".format("1.1.0", version) log = None From 6e72e44fa6e3fadde3be603a535e3332ec6930a2 Mon Sep 17 00:00:00 2001 From: dhoomakethu Date: Tue, 15 Jan 2019 14:00:29 +0530 Subject: [PATCH 05/26] 1. Fix tornado async serial client `TypeError` while processing incoming packet. 2. Fix asyncio examples. 3. Minor update in factory.py, now server logs prints received request instead of only function cod --- examples/common/async_asyncio_serial_client.py | 5 +---- examples/common/async_tornado_client_serial.py | 5 ++++- pymodbus/client/asynchronous/tornado/__init__.py | 2 ++ pymodbus/factory.py | 9 ++++++++- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/examples/common/async_asyncio_serial_client.py b/examples/common/async_asyncio_serial_client.py index 95d7a44d0..47f0f5bf9 100755 --- a/examples/common/async_asyncio_serial_client.py +++ b/examples/common/async_asyncio_serial_client.py @@ -49,9 +49,6 @@ async def start_async_test(client): # which defaults to `0x00` # ----------------------------------------------------------------------- # try: - log.debug("Reading Coils") - rr = client.read_coils(1, 1, unit=UNIT) - # ----------------------------------------------------------------------- # # example requests # ----------------------------------------------------------------------- # @@ -137,7 +134,7 @@ async def start_async_test(client): # socat -d -d PTY,link=/tmp/ptyp0,raw,echo=0,ispeed=9600 PTY, # link=/tmp/ttyp0,raw,echo=0,ospeed=9600 loop, client = ModbusClient(schedulers.ASYNC_IO, port='/tmp/ptyp0', - baudrate=9600, timeout=2, method="rtu") + baudrate=9600, method="rtu") loop.run_until_complete(start_async_test(client.protocol)) loop.close() diff --git a/examples/common/async_tornado_client_serial.py b/examples/common/async_tornado_client_serial.py index 74ee932e8..a170fb0f1 100755 --- a/examples/common/async_tornado_client_serial.py +++ b/examples/common/async_tornado_client_serial.py @@ -24,7 +24,10 @@ # configure the client logging # ---------------------------------------------------------------------------# import logging -logging.basicConfig() + +FORMAT = ('%(asctime)-15s %(threadName)-15s' + ' %(levelname)-8s %(module)-15s:%(lineno)-8s %(message)s') +logging.basicConfig(format=FORMAT) log = logging.getLogger() log.setLevel(logging.DEBUG) diff --git a/pymodbus/client/asynchronous/tornado/__init__.py b/pymodbus/client/asynchronous/tornado/__init__.py index 780bf5f07..e0af49e0d 100644 --- a/pymodbus/client/asynchronous/tornado/__init__.py +++ b/pymodbus/client/asynchronous/tornado/__init__.py @@ -175,9 +175,11 @@ def callback(*args): data = self.stream.connection.read(waiting) LOGGER.debug( "recv: " + " ".join([hex(byte2int(x)) for x in data])) + unit = self.framer.decode_data(data).get("uid", 0) self.framer.processIncomingPacket( data, self._handle_response, + unit, tid=request.transaction_id ) break diff --git a/pymodbus/factory.py b/pymodbus/factory.py index 6f54ad78e..ad2dff1ae 100644 --- a/pymodbus/factory.py +++ b/pymodbus/factory.py @@ -124,10 +124,17 @@ def _helper(self, data): :returns: The decoded request or illegal function request object """ function_code = byte2int(data[0]) - _logger.debug("Factory Request[%d]" % function_code) request = self.__lookup.get(function_code, lambda: None)() if not request: + _logger.debug("Factory Request[%d]" % function_code) request = IllegalFunctionRequest(function_code) + else: + fc_string = "%s: %s" % ( + str(self.__lookup[function_code]).split('.')[-1].rstrip( + "'>"), + function_code + ) + _logger.debug("Factory Request[%s]" % fc_string) request.decode(data[1:]) if hasattr(request, 'sub_function_code'): From 18fe036d1f5ad3ad6d2408e0b395d067da001eba Mon Sep 17 00:00:00 2001 From: Marek Lewandowski Date: Sun, 6 Jan 2019 19:31:32 +0100 Subject: [PATCH 06/26] [fix v3] poprawa sprawdzania timeout --- pymodbus/client/sync.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pymodbus/client/sync.py b/pymodbus/client/sync.py index 93c54b80c..164c176bb 100644 --- a/pymodbus/client/sync.py +++ b/pymodbus/client/sync.py @@ -263,21 +263,22 @@ def _recv(self, size): recv_size = size data = b'' - begin = time.time() + time_ = time.time() + end = time_ + timeout while recv_size > 0: - ready = select.select([self.socket], [], [], timeout) + ready = select.select([self.socket], [], [], end - time_) if ready[0]: data += self.socket.recv(recv_size) + time_ = time.time() # If size isn't specified continue to read until timeout expires. if size: recv_size = size - len(data) # Timeout is reduced also if some data has been received in order - # to avoid infinite loops when there isn't an expected response size - # and the slave sends noisy data continuosly. - timeout -= time.time() - begin - if timeout <= 0: + # to avoid infinite loops when there isn't an expected response + # size and the slave sends noisy data continuosly. + if time_ > end: break return data From 964a565c9cf50959596bb6329d6ceeaf688c3c89 Mon Sep 17 00:00:00 2001 From: dhoomakethu Date: Wed, 16 Jan 2019 17:46:34 +0530 Subject: [PATCH 07/26] Release candidate for pymodbus 2.2.0 --- CHANGELOG.rst | 13 ++++++++++++- pymodbus/version.py | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8f0c2bff7..4f05214bd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,10 +8,21 @@ from pymodbus.client.asynchronous import ModbusTcpClient * Support Python 3.7 * Fix `AttributeError` when setting `interCharTimeout` for serial clients. * Provide an option to disable inter char timeouts with Modbus RTU. +* Add support to register custom requests in clients and server instances. +* Fix read timeout calculation in ModbusTCP. * Fix SQLDbcontext always returning InvalidAddress error. * Fix SQLDbcontext update failure * Fix Binary payload example for endianess. -* Add support to register custom requests in clients and server instances. +* Fix tornado async serial client `TypeError` while processing incoming packet. +* Fix asyncio examples. +* Minor update in factory.py, now server logs prints received request instead of only function code +``` +# Now +DEBUG:pymodbus.factory:Factory Request[ReadInputRegistersRequest: 4] +# Before +DEBUG:pymodbus.factory:Factory Request[4] + +``` Version 2.1.0 diff --git a/pymodbus/version.py b/pymodbus/version.py index 51da88745..c7589341c 100644 --- a/pymodbus/version.py +++ b/pymodbus/version.py @@ -41,7 +41,7 @@ def __str__(self): return '[%s, version %s]' % (self.package, self.short()) -version = Version('pymodbus', 2, 2, 0) +version = Version('pymodbus', 2, 2, 0, "rc1") version.__name__ = 'pymodbus' # fix epydoc error From 6960d9c0787b624d2e8534a3e7ffcf966c3b9a07 Mon Sep 17 00:00:00 2001 From: dhoomakethu Date: Sat, 26 Jan 2019 10:27:50 +0530 Subject: [PATCH 08/26] Fix #377 when invalid port is supplied and minor updates in logging --- CHANGELOG.rst | 1 + pymodbus/client/sync.py | 8 ++++---- pymodbus/datastore/context.py | 6 ++++-- pymodbus/server/sync.py | 7 ++++--- pymodbus/version.py | 2 +- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4f05214bd..4fef73516 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,6 +15,7 @@ from pymodbus.client.asynchronous import ModbusTcpClient * Fix Binary payload example for endianess. * Fix tornado async serial client `TypeError` while processing incoming packet. * Fix asyncio examples. +* Improved logging in Modbus Server . * Minor update in factory.py, now server logs prints received request instead of only function code ``` # Now diff --git a/pymodbus/client/sync.py b/pymodbus/client/sync.py index 164c176bb..0ed9f6c01 100644 --- a/pymodbus/client/sync.py +++ b/pymodbus/client/sync.py @@ -481,13 +481,13 @@ def connect(self): stopbits=self.stopbits, baudrate=self.baudrate, parity=self.parity) + if self.method == "rtu": + if self._strict: + self.socket.interCharTimeout = self.inter_char_timeout + self.last_frame_end = None except serial.SerialException as msg: _logger.error(msg) self.close() - if self.method == "rtu": - if self._strict: - self.socket.interCharTimeout = self.inter_char_timeout - self.last_frame_end = None return self.socket is not None def close(self): diff --git a/pymodbus/datastore/context.py b/pymodbus/datastore/context.py index a440546d2..ab87ffbef 100644 --- a/pymodbus/datastore/context.py +++ b/pymodbus/datastore/context.py @@ -60,7 +60,8 @@ def validate(self, fx, address, count=1): ''' if not self.zero_mode: address = address + 1 - _logger.debug("validate[%d] %d:%d" % (fx, address, count)) + _logger.debug("validate: fc-[%d] address-%d: count-%d" % (fx, address, + count)) return self.store[self.decode(fx)].validate(address, count) def getValues(self, fx, address, count=1): @@ -73,7 +74,8 @@ def getValues(self, fx, address, count=1): ''' if not self.zero_mode: address = address + 1 - _logger.debug("getValues[%d] %d:%d" % (fx, address, count)) + _logger.debug("getValues fc-[%d] address-%d: count-%d" % (fx, address, + count)) return self.store[self.decode(fx)].getValues(address, count) def setValues(self, fx, address, values): diff --git a/pymodbus/server/sync.py b/pymodbus/server/sync.py index 01d65685c..6fea466fa 100644 --- a/pymodbus/server/sync.py +++ b/pymodbus/server/sync.py @@ -125,7 +125,7 @@ def send(self, message): # self.server.control.Counter.BusMessage += 1 pdu = self.framer.buildPacket(message) if _logger.isEnabledFor(logging.DEBUG): - _logger.debug('send: %s' % b2a_hex(pdu)) + _logger.debug('send: [%s]- %s' % (message, b2a_hex(pdu))) return self.request.send(pdu) @@ -203,7 +203,7 @@ def send(self, message): # self.server.control.Counter.BusMessage += 1 pdu = self.framer.buildPacket(message) if _logger.isEnabledFor(logging.DEBUG): - _logger.debug('send: %s' % b2a_hex(pdu)) + _logger.debug('send: [%s]- %s' % (message, b2a_hex(pdu))) return self.request.send(pdu) @@ -226,6 +226,7 @@ def handle(self): data, self.socket = self.request if not data: self.running = False + data = b'' if _logger.isEnabledFor(logging.DEBUG): _logger.debug('Handling data: ' + hexlify_packets(data)) # if not self.server.control.ListenOnly: @@ -258,7 +259,7 @@ def send(self, message): #self.server.control.Counter.BusMessage += 1 pdu = self.framer.buildPacket(message) if _logger.isEnabledFor(logging.DEBUG): - _logger.debug('send: %s' % b2a_hex(pdu)) + _logger.debug('send: [%s]- %s' % (message, b2a_hex(pdu))) return self.socket.sendto(pdu, self.client_address) diff --git a/pymodbus/version.py b/pymodbus/version.py index c7589341c..3e81d7af5 100644 --- a/pymodbus/version.py +++ b/pymodbus/version.py @@ -41,7 +41,7 @@ def __str__(self): return '[%s, version %s]' % (self.package, self.short()) -version = Version('pymodbus', 2, 2, 0, "rc1") +version = Version('pymodbus', 2, 2, 0, "rc2") version.__name__ = 'pymodbus' # fix epydoc error From 60aca50b9312863aa6d46c7f58bfdf3c5633f93c Mon Sep 17 00:00:00 2001 From: Michael Muhlbaier Date: Thu, 31 Jan 2019 08:55:35 -0500 Subject: [PATCH 09/26] #368 adds broadcast support for sync client and server Adds broadcast_enable parameter to client and server, default value is False. When true it will treat unit_id 0 as broadcast and execute requests on all server slave contexts and not send a response and on the client side will send the request and not try to receive a response. --- pymodbus/client/sync.py | 1 + pymodbus/constants.py | 10 ++++ pymodbus/server/sync.py | 45 ++++++++++++--- pymodbus/transaction.py | 125 +++++++++++++++++++++------------------- 4 files changed, 112 insertions(+), 69 deletions(-) diff --git a/pymodbus/client/sync.py b/pymodbus/client/sync.py index bb07caa52..26a6fadfc 100644 --- a/pymodbus/client/sync.py +++ b/pymodbus/client/sync.py @@ -45,6 +45,7 @@ def __init__(self, framer, **kwargs): self.transaction = FifoTransactionManager(self, **kwargs) self._debug = False self._debugfd = None + self.broadcast_enable = kwargs.get('broadcast_enable', Defaults.broadcast_enable) # ----------------------------------------------------------------------- # # Client interface diff --git a/pymodbus/constants.py b/pymodbus/constants.py index 5763c77d7..c05f0b555 100644 --- a/pymodbus/constants.py +++ b/pymodbus/constants.py @@ -88,6 +88,15 @@ class Defaults(Singleton): should be returned or simply ignored. This is useful for the case of a serial server emulater where a request to a non-existant slave on a bus will never respond. The client in this case will simply timeout. + + .. attribute:: broadcast_enable + + When False unit_id 0 will be treated as any other unit_id. When True and + the unit_id is 0 the server will execute all requests on all server + contexts and not respond and the client will skip trying to receive a + response. Default value False does not conform to Modbus spec but maintains + legacy behavior for existing pymodbus users. + ''' Port = 502 Retries = 3 @@ -104,6 +113,7 @@ class Defaults(Singleton): ZeroMode = False IgnoreMissingSlaves = False ReadSize = 1024 + broadcast_enable = False class ModbusStatus(Singleton): ''' diff --git a/pymodbus/server/sync.py b/pymodbus/server/sync.py index 937a89294..866b0cb09 100644 --- a/pymodbus/server/sync.py +++ b/pymodbus/server/sync.py @@ -59,8 +59,15 @@ def execute(self, request): :param request: The decoded request message """ try: - context = self.server.context[request.unit_id] - response = request.execute(context) + if self.server.broadcast_enable and request.unit_id == 0: + broadcast = True + # if broadcasting then execute on all slave contexts, note response will be ignored + for unit_id in self.server.context.slaves(): + response = request.execute(self.server.context[unit_id]) + else: + broadcast = False + context = self.server.context[request.unit_id] + response = request.execute(context) except NoSuchSlaveException as ex: _logger.debug("requested slave does " "not exist: %s" % request.unit_id ) @@ -71,9 +78,11 @@ def execute(self, request): _logger.debug("Datastore unable to fulfill request: " "%s; %s", ex, traceback.format_exc()) response = request.doException(merror.SlaveFailure) - response.transaction_id = request.transaction_id - response.unit_id = request.unit_id - self.send(response) + # no response when broadcasting + if not broadcast: + response.transaction_id = request.transaction_id + response.unit_id = request.unit_id + self.send(response) # ----------------------------------------------------------------------- # # Base class implementations @@ -105,6 +114,12 @@ def handle(self): data = self.request.recv(1024) if data: units = self.server.context.slaves() + if not isinstance(units, (list, tuple)): + units = [units] + # if broadcast is enabled make sure to process requests to address 0 + if self.server.broadcast_enable: + if 0 not in units: + units.append(0) single = self.server.context.single self.framer.processIncomingPacket(data, self.execute, units, single=single) @@ -288,8 +303,10 @@ def __init__(self, context, framer=None, identity=None, ModbusConnectedRequestHandler :param allow_reuse_address: Whether the server will allow the reuse of an address. - :param ignore_missing_slaves: True to not send errors on a request - to a missing slave + :param ignore_missing_slaves: True to not send errors on a request + to a missing slave + :param broadcast_enable: True to treat unit_id 0 as broadcast address, + False to treat 0 as any other unit_id """ self.threads = [] self.allow_reuse_address = allow_reuse_address @@ -301,6 +318,8 @@ def __init__(self, context, framer=None, identity=None, self.handler = handler or ModbusConnectedRequestHandler self.ignore_missing_slaves = kwargs.get('ignore_missing_slaves', Defaults.IgnoreMissingSlaves) + self.broadcast_enable = kwargs.get('broadcast_enable', + Defaults.broadcast_enable) if isinstance(identity, ModbusDeviceIdentification): self.control.Identity.update(identity) @@ -358,7 +377,9 @@ def __init__(self, context, framer=None, identity=None, address=None, :param handler: A handler for each client session; default is ModbusDisonnectedRequestHandler :param ignore_missing_slaves: True to not send errors on a request - to a missing slave + to a missing slave + :param broadcast_enable: True to treat unit_id 0 as broadcast address, + False to treat 0 as any other unit_id """ self.threads = [] self.decoder = ServerDecoder() @@ -369,6 +390,8 @@ def __init__(self, context, framer=None, identity=None, address=None, self.handler = handler or ModbusDisconnectedRequestHandler self.ignore_missing_slaves = kwargs.get('ignore_missing_slaves', Defaults.IgnoreMissingSlaves) + self.broadcast_enable = kwargs.get('broadcast_enable', + Defaults.broadcast_enable) if isinstance(identity, ModbusDeviceIdentification): self.control.Identity.update(identity) @@ -423,7 +446,9 @@ def __init__(self, context, framer=None, identity=None, **kwargs): :param baudrate: The baud rate to use for the serial device :param timeout: The timeout to use for the serial device :param ignore_missing_slaves: True to not send errors on a request - to a missing slave + to a missing slave + :param broadcast_enable: True to treat unit_id 0 as broadcast address, + False to treat 0 as any other unit_id """ self.threads = [] self.decoder = ServerDecoder() @@ -442,6 +467,8 @@ def __init__(self, context, framer=None, identity=None, **kwargs): self.timeout = kwargs.get('timeout', Defaults.Timeout) self.ignore_missing_slaves = kwargs.get('ignore_missing_slaves', Defaults.IgnoreMissingSlaves) + self.broadcast_enable = kwargs.get('broadcast_enable', + Defaults.broadcast_enable) self.socket = None if self._connect(): self.is_running = True diff --git a/pymodbus/transaction.py b/pymodbus/transaction.py index ddfa06f4d..17be1507b 100644 --- a/pymodbus/transaction.py +++ b/pymodbus/transaction.py @@ -118,53 +118,53 @@ def execute(self, request): _logger.debug("Clearing current Frame : - {}".format(_buffer)) self.client.framer.resetFrame() - expected_response_length = None - if not isinstance(self.client.framer, ModbusSocketFramer): - if hasattr(request, "get_response_pdu_size"): - response_pdu_size = request.get_response_pdu_size() - if isinstance(self.client.framer, ModbusAsciiFramer): - response_pdu_size = response_pdu_size * 2 - if response_pdu_size: - expected_response_length = self._calculate_response_length(response_pdu_size) - if request.unit_id == 0: - full = True - expected_response_length = 0 - elif request.unit_id in self._no_response_devices: - full = True + if request.unit_id == 0 and self.client.broadcast_enable: + response, last_exception = self._transact(request, None) + response = b'Broadcast write sent - no response expected' else: - full = False - c_str = str(self.client) - if "modbusudpclient" in c_str.lower().strip(): - full = True - if not expected_response_length: - expected_response_length = Defaults.ReadSize - response, last_exception = self._transact(request, - expected_response_length, - full=full - ) - if not response and ( - request.unit_id not in self._no_response_devices): - self._no_response_devices.append(request.unit_id) - elif request.unit_id in self._no_response_devices and response: - self._no_response_devices.remove(request.unit_id) - if not response and self.retry_on_empty and retries: - while retries > 0: - if hasattr(self.client, "state"): - _logger.debug("RESETTING Transaction state to " - "'IDLE' for retry") - self.client.state = ModbusTransactionState.IDLE - _logger.debug("Retry on empty - {}".format(retries)) - response, last_exception = self._transact( - request, - expected_response_length - ) - if not response: - retries -= 1 - continue - # Remove entry + expected_response_length = None + if not isinstance(self.client.framer, ModbusSocketFramer): + if hasattr(request, "get_response_pdu_size"): + response_pdu_size = request.get_response_pdu_size() + if isinstance(self.client.framer, ModbusAsciiFramer): + response_pdu_size = response_pdu_size * 2 + if response_pdu_size: + expected_response_length = self._calculate_response_length(response_pdu_size) + if request.unit_id in self._no_response_devices: + full = True + else: + full = False + c_str = str(self.client) + if "modbusudpclient" in c_str.lower().strip(): + full = True + if not expected_response_length: + expected_response_length = Defaults.ReadSize + response, last_exception = self._transact(request, + expected_response_length, + full=full + ) + if not response and ( + request.unit_id not in self._no_response_devices): + self._no_response_devices.append(request.unit_id) + elif request.unit_id in self._no_response_devices and response: self._no_response_devices.remove(request.unit_id) - break - if expected_response_length > 0: + if not response and self.retry_on_empty and retries: + while retries > 0: + if hasattr(self.client, "state"): + _logger.debug("RESETTING Transaction state to " + "'IDLE' for retry") + self.client.state = ModbusTransactionState.IDLE + _logger.debug("Retry on empty - {}".format(retries)) + response, last_exception = self._transact( + request, + expected_response_length + ) + if not response: + retries -= 1 + continue + # Remove entry + self._no_response_devices.remove(request.unit_id) + break addTransaction = partial(self.addTransaction, tid=request.transaction_id) self.client.framer.processIncomingPacket(response, @@ -180,14 +180,12 @@ def execute(self, request): "/Unable to decode response") response = ModbusIOException(last_exception, request.function_code) - else: - _logger.debug("No response expected when sending to broadcast address 0") - if hasattr(self.client, "state"): - _logger.debug("Changing transaction state from " - "'PROCESSING REPLY' to " - "'TRANSACTION_COMPLETE'") - self.client.state = ( - ModbusTransactionState.TRANSACTION_COMPLETE) + if hasattr(self.client, "state"): + _logger.debug("Changing transaction state from " + "'PROCESSING REPLY' to " + "'TRANSACTION_COMPLETE'") + self.client.state = ( + ModbusTransactionState.TRANSACTION_COMPLETE) return response except ModbusIOException as ex: # Handle decode errors in processIncomingPacket method @@ -211,13 +209,20 @@ def _transact(self, packet, response_length, full=False): if _logger.isEnabledFor(logging.DEBUG): _logger.debug("SEND: " + hexlify_packets(packet)) size = self._send(packet) - if size: - _logger.debug("Changing transaction state from 'SENDING' " - "to 'WAITING FOR REPLY'") - self.client.state = ModbusTransactionState.WAITING_FOR_REPLY - result = self._recv(response_length, full) - if _logger.isEnabledFor(logging.DEBUG): - _logger.debug("RECV: " + hexlify_packets(result)) + if response_length is not None: + if size: + _logger.debug("Changing transaction state from 'SENDING' " + "to 'WAITING FOR REPLY'") + self.client.state = ModbusTransactionState.WAITING_FOR_REPLY + result = self._recv(response_length, full) + if _logger.isEnabledFor(logging.DEBUG): + _logger.debug("RECV: " + hexlify_packets(result)) + else: + if size: + _logger.debug("Changing transaction state from 'SENDING' " + "to 'TRANSACTION_COMPLETE'") + self.client.state = ModbusTransactionState.TRANSACTION_COMPLETE + result = b'' except (socket.error, ModbusIOException, InvalidMessageReceivedException) as msg: self.client.close() From e07e01e1a25e0f7d4156e613c6d787e8d031f336 Mon Sep 17 00:00:00 2001 From: Michael Muhlbaier Date: Thu, 31 Jan 2019 09:20:18 -0500 Subject: [PATCH 10/26] #368 Fixes minor bug in broadcast support code --- pymodbus/server/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymodbus/server/sync.py b/pymodbus/server/sync.py index 866b0cb09..a93df7a3e 100644 --- a/pymodbus/server/sync.py +++ b/pymodbus/server/sync.py @@ -58,6 +58,7 @@ def execute(self, request): :param request: The decoded request message """ + broadcast = False try: if self.server.broadcast_enable and request.unit_id == 0: broadcast = True @@ -65,7 +66,6 @@ def execute(self, request): for unit_id in self.server.context.slaves(): response = request.execute(self.server.context[unit_id]) else: - broadcast = False context = self.server.context[request.unit_id] response = request.execute(context) except NoSuchSlaveException as ex: From 5030514e851ef910632d6153de50570986c70d7c Mon Sep 17 00:00:00 2001 From: JStrbg Date: Wed, 16 Jan 2019 13:55:53 +0100 Subject: [PATCH 11/26] Fixed erronous CRC handling If the CRC recieved is not correct in my case my slave got caught in a deadlock, not taking any new requests. This addition fixed that. --- pymodbus/framer/rtu_framer.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pymodbus/framer/rtu_framer.py b/pymodbus/framer/rtu_framer.py index 8579a57a4..f2e4eb0df 100644 --- a/pymodbus/framer/rtu_framer.py +++ b/pymodbus/framer/rtu_framer.py @@ -91,7 +91,12 @@ def checkFrame(self): data = self._buffer[:frame_size - 2] crc = self._buffer[frame_size - 2:frame_size] crc_val = (byte2int(crc[0]) << 8) + byte2int(crc[1]) - return checkCRC(data, crc_val) + if checkCRC(data, crc_val): + return True + else: + _logger.debug("CRC invalid, discarding header!!") + self.resetFrame() + return False except (IndexError, KeyError, struct.error): return False From e5c26153ed34f7a1150d0d757e489542fc90d8a2 Mon Sep 17 00:00:00 2001 From: dhoomakethu Date: Mon, 11 Feb 2019 09:43:30 +0530 Subject: [PATCH 12/26] Update Changelog --- CHANGELOG.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4fef73516..e76039699 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,9 +14,12 @@ from pymodbus.client.asynchronous import ModbusTcpClient * Fix SQLDbcontext update failure * Fix Binary payload example for endianess. * Fix tornado async serial client `TypeError` while processing incoming packet. +* Fix erroneous CRC handling in Modbus RTU framer. +* Support broadcasting in Modbus Client and Servers (sync). * Fix asyncio examples. * Improved logging in Modbus Server . * Minor update in factory.py, now server logs prints received request instead of only function code + ``` # Now DEBUG:pymodbus.factory:Factory Request[ReadInputRegistersRequest: 4] From f66f464d911cc672225a62a5c89b34c817f48473 Mon Sep 17 00:00:00 2001 From: dhoomakethu Date: Mon, 11 Feb 2019 12:59:09 +0530 Subject: [PATCH 13/26] Fix test coverage --- .coveragerc | 3 ++- pymodbus/framer/ascii_framer.py | 6 ------ test/test_framers.py | 29 +++++++++++++++++++++++++++-- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/.coveragerc b/.coveragerc index 894602930..dbdb75230 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,3 +1,4 @@ [run] omit = - pymodbus/repl/* \ No newline at end of file + pymodbus/repl/* + pymodbus/internal/* \ No newline at end of file diff --git a/pymodbus/framer/ascii_framer.py b/pymodbus/framer/ascii_framer.py index f98bdca7b..a88f6499f 100644 --- a/pymodbus/framer/ascii_framer.py +++ b/pymodbus/framer/ascii_framer.py @@ -1,16 +1,10 @@ import struct -import socket from binascii import b2a_hex, a2b_hex from pymodbus.exceptions import ModbusIOException from pymodbus.utilities import checkLRC, computeLRC from pymodbus.framer import ModbusFramer, FRAME_HEADER, BYTE_ORDER -# Python 2 compatibility. -try: - TimeoutError -except NameError: - TimeoutError = socket.timeout ASCII_FRAME_HEADER = BYTE_ORDER + FRAME_HEADER diff --git a/test/test_framers.py b/test/test_framers.py index 4423bb9c3..c11c72f37 100644 --- a/test/test_framers.py +++ b/test/test_framers.py @@ -12,18 +12,22 @@ else: # Python 2 from mock import Mock + @pytest.fixture def rtu_framer(): return ModbusRtuFramer(ClientDecoder()) + @pytest.fixture def ascii_framer(): return ModbusAsciiFramer(ClientDecoder()) + @pytest.fixture def binary_framer(): return ModbusBinaryFramer(ClientDecoder()) + @pytest.mark.parametrize("framer", [ModbusRtuFramer, ModbusAsciiFramer, ModbusBinaryFramer, @@ -116,9 +120,12 @@ def test_populate_result(rtu_framer): assert result.unit_id == 255 -def test_process_incoming_packet(rtu_framer): +@pytest.mark.parametrize('framer', [ascii_framer, rtu_framer, binary_framer]) +def test_process_incoming_packet(framer): def cb(res): return res + # data = b'' + # framer.processIncomingPacket(data, cb, unit=1, single=False) def test_build_packet(rtu_framer): @@ -160,4 +167,22 @@ def cb(res): def test_get_raw_frame(rtu_framer): rtu_framer._buffer = b'\x00\x01\x00\x01\x00\n\xec\x1c' - assert rtu_framer.getRawFrame() == rtu_framer._buffer \ No newline at end of file + assert rtu_framer.getRawFrame() == rtu_framer._buffer + + +def test_validate_unit_id(rtu_framer): + rtu_framer.populateHeader( b'\x00\x01\x00\x01\x00\n\xec\x1c') + assert rtu_framer._validate_unit_id([0], False) + assert rtu_framer._validate_unit_id([1], True) + + +@pytest.mark.parametrize('data', [b':010100010001FC\r\n', + b'']) +def test_decode_ascii_data(ascii_framer, data): + data = ascii_framer.decode_data(data) + assert isinstance(data, dict) + if data: + assert data.get("unit") == 1 + assert data.get("fcode") == 1 + else: + assert not data \ No newline at end of file From 765042155c8a0baf57f0a356fda171ac7cd22d53 Mon Sep 17 00:00:00 2001 From: dhoomakethu Date: Sat, 16 Feb 2019 13:46:55 +0530 Subject: [PATCH 14/26] Fix #387 Transactions failing on 2.2.0rc2. --- CHANGELOG.rst | 1 + pymodbus/server/sync.py | 13 +++++++++--- pymodbus/transaction.py | 45 +++++++++++++++++++++++------------------ pymodbus/version.py | 2 +- 4 files changed, 37 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e76039699..ccdeb179b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -18,6 +18,7 @@ from pymodbus.client.asynchronous import ModbusTcpClient * Support broadcasting in Modbus Client and Servers (sync). * Fix asyncio examples. * Improved logging in Modbus Server . +* Fix regression introduced in 2.2.0rc2 (Modbus sync client transaction failing) * Minor update in factory.py, now server logs prints received request instead of only function code ``` diff --git a/pymodbus/server/sync.py b/pymodbus/server/sync.py index d51d4cdc4..9492265f7 100644 --- a/pymodbus/server/sync.py +++ b/pymodbus/server/sync.py @@ -181,14 +181,21 @@ def handle(self): reset_frame = False while self.running: try: + units = self.server.context.slaves() data = self.request.recv(1024) if not data: self.running = False + else: + if not isinstance(units, (list, tuple)): + units = [units] + # if broadcast is enabled make sure to + # process requests to address 0 + if self.server.broadcast_enable: + if 0 not in units: + units.append(0) + if _logger.isEnabledFor(logging.DEBUG): _logger.debug('Handling data: ' + hexlify_packets(data)) - # if not self.server.control.ListenOnly: - - units = self.server.context.slaves() single = self.server.context.single self.framer.processIncomingPacket(data, self.execute, units, single=single) diff --git a/pymodbus/transaction.py b/pymodbus/transaction.py index 17be1507b..579fcd86d 100644 --- a/pymodbus/transaction.py +++ b/pymodbus/transaction.py @@ -112,14 +112,17 @@ def execute(self, request): ) retries = self.retries request.transaction_id = self.getNextTID() - _logger.debug("Running transaction %d" % request.transaction_id) + _logger.debug("Running transaction " + "{}".format(request.transaction_id)) _buffer = hexlify_packets(self.client.framer._buffer) if _buffer: - _logger.debug("Clearing current Frame : - {}".format(_buffer)) + _logger.debug("Clearing current Frame " + ": - {}".format(_buffer)) self.client.framer.resetFrame() - - if request.unit_id == 0 and self.client.broadcast_enable: - response, last_exception = self._transact(request, None) + broadcast = (self.client.broadcast_enable + and request.unit_id == 0) + if broadcast: + self._transact(request, None, broadcast=True) response = b'Broadcast write sent - no response expected' else: expected_response_length = None @@ -139,10 +142,12 @@ def execute(self, request): full = True if not expected_response_length: expected_response_length = Defaults.ReadSize - response, last_exception = self._transact(request, - expected_response_length, - full=full - ) + response, last_exception = self._transact( + request, + expected_response_length, + full=full, + broadcast=broadcast + ) if not response and ( request.unit_id not in self._no_response_devices): self._no_response_devices.append(request.unit_id) @@ -193,7 +198,7 @@ def execute(self, request): self.client.state = ModbusTransactionState.TRANSACTION_COMPLETE return ex - def _transact(self, packet, response_length, full=False): + def _transact(self, packet, response_length, full=False, broadcast=False): """ Does a Write and Read transaction :param packet: packet to be sent @@ -209,20 +214,20 @@ def _transact(self, packet, response_length, full=False): if _logger.isEnabledFor(logging.DEBUG): _logger.debug("SEND: " + hexlify_packets(packet)) size = self._send(packet) - if response_length is not None: - if size: - _logger.debug("Changing transaction state from 'SENDING' " - "to 'WAITING FOR REPLY'") - self.client.state = ModbusTransactionState.WAITING_FOR_REPLY - result = self._recv(response_length, full) - if _logger.isEnabledFor(logging.DEBUG): - _logger.debug("RECV: " + hexlify_packets(result)) - else: + if broadcast: if size: _logger.debug("Changing transaction state from 'SENDING' " "to 'TRANSACTION_COMPLETE'") self.client.state = ModbusTransactionState.TRANSACTION_COMPLETE - result = b'' + return b'', None + if size: + _logger.debug("Changing transaction state from 'SENDING' " + "to 'WAITING FOR REPLY'") + self.client.state = ModbusTransactionState.WAITING_FOR_REPLY + result = self._recv(response_length, full) + if _logger.isEnabledFor(logging.DEBUG): + _logger.debug("RECV: " + hexlify_packets(result)) + except (socket.error, ModbusIOException, InvalidMessageReceivedException) as msg: self.client.close() diff --git a/pymodbus/version.py b/pymodbus/version.py index 3e81d7af5..13a4dff55 100644 --- a/pymodbus/version.py +++ b/pymodbus/version.py @@ -41,7 +41,7 @@ def __str__(self): return '[%s, version %s]' % (self.package, self.short()) -version = Version('pymodbus', 2, 2, 0, "rc2") +version = Version('pymodbus', 2, 2, 0, "rc3") version.__name__ = 'pymodbus' # fix epydoc error From 6233706a22aa0200686a48611d5c552f17b4569d Mon Sep 17 00:00:00 2001 From: Ryan Parry-Jones Date: Wed, 12 Dec 2018 18:10:01 +1100 Subject: [PATCH 15/26] Task Cancellation and CRC Errors Alternate solution for #356 and #360. Changes the RTU to make the transaction ID as the unit ID instead of an ever incrementing number. Previously this transaction ID was always 0 on the receiving end but was the unique transaction ID on sending. As such the FIFO buffer made the most sense. By tying it to the unit ID, we can recover from failure modes such as: - - Asyncio task cancellations (eg. timeouts) #360 - Skipped responses from slaves. (hangs on master #360) - CRC Errors #356 - Busy response --- pymodbus/client/sync.py | 5 +---- pymodbus/framer/rtu_framer.py | 5 +++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pymodbus/client/sync.py b/pymodbus/client/sync.py index c8e34b616..04d7778e3 100644 --- a/pymodbus/client/sync.py +++ b/pymodbus/client/sync.py @@ -39,10 +39,7 @@ def __init__(self, framer, **kwargs): :param framer: The modbus framer implementation to use """ self.framer = framer - if isinstance(self.framer, ModbusSocketFramer): - self.transaction = DictTransactionManager(self, **kwargs) - else: - self.transaction = FifoTransactionManager(self, **kwargs) + self.transaction = DictTransactionManager(self, **kwargs) self._debug = False self._debugfd = None self.broadcast_enable = kwargs.get('broadcast_enable', Defaults.broadcast_enable) diff --git a/pymodbus/framer/rtu_framer.py b/pymodbus/framer/rtu_framer.py index f2e4eb0df..c2d74d0d4 100644 --- a/pymodbus/framer/rtu_framer.py +++ b/pymodbus/framer/rtu_framer.py @@ -191,6 +191,7 @@ def populateResult(self, result): :param result: The response packet """ result.unit_id = self._header['uid'] + result.transaction_id = self._header['uid'] # ----------------------------------------------------------------------- # # Public Member Functions @@ -227,6 +228,9 @@ def processIncomingPacket(self, data, callback, unit, **kwargs): _logger.debug("Not a valid unit id - {}, " "ignoring!!".format(self._header['uid'])) self.resetFrame() + else: + _logger.debug("Frame check failed, ignoring!!") + self.resetFrame() else: _logger.debug("Frame - [{}] not ready".format(data)) @@ -241,6 +245,7 @@ def buildPacket(self, message): message.unit_id, message.function_code) + data packet += struct.pack(">H", computeCRC(packet)) + message.transaction_id = message.unit_id # Ensure that transaction is actually the unit id for serial comms return packet def sendPacket(self, message): From 89d3909aff79e3f3ff31a6a077090e9e477cafb0 Mon Sep 17 00:00:00 2001 From: dhoomakethu Date: Wed, 6 Mar 2019 09:59:21 +0530 Subject: [PATCH 16/26] Cherry pick commit from PR #367 , Update changelog , bump version to 2.2.0rc4 --- CHANGELOG.rst | 1 + pymodbus/version.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ccdeb179b..bbe59f86e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,7 @@ from pymodbus.client.asynchronous import ModbusTcpClient ``` * Support Python 3.7 +* Fix to task cancellations and CRC errors for async serial clients. * Fix `AttributeError` when setting `interCharTimeout` for serial clients. * Provide an option to disable inter char timeouts with Modbus RTU. * Add support to register custom requests in clients and server instances. diff --git a/pymodbus/version.py b/pymodbus/version.py index 13a4dff55..ef10764b6 100644 --- a/pymodbus/version.py +++ b/pymodbus/version.py @@ -41,7 +41,7 @@ def __str__(self): return '[%s, version %s]' % (self.package, self.short()) -version = Version('pymodbus', 2, 2, 0, "rc3") +version = Version('pymodbus', 2, 2, 0, "rc4") version.__name__ = 'pymodbus' # fix epydoc error From a36ccbf32201d926aea05f7ef7429a08d5ec872f Mon Sep 17 00:00:00 2001 From: Memet Bilgin Date: Sun, 24 Mar 2019 20:50:02 +1100 Subject: [PATCH 17/26] native asyncio implementation of ModbusTcpServer and ModbusUdpServer --- pymodbus/server/asyncio.py | 580 +++++++++++++++++++++++++++++++++++++ 1 file changed, 580 insertions(+) create mode 100755 pymodbus/server/asyncio.py diff --git a/pymodbus/server/asyncio.py b/pymodbus/server/asyncio.py new file mode 100755 index 000000000..ef802dd20 --- /dev/null +++ b/pymodbus/server/asyncio.py @@ -0,0 +1,580 @@ +""" +Implementation of a Threaded Modbus Server +------------------------------------------ + +""" +from binascii import b2a_hex +import serial +import socket +import traceback + +import asyncio +from pymodbus.constants import Defaults +from pymodbus.utilities import hexlify_packets +from pymodbus.factory import ServerDecoder +from pymodbus.datastore import ModbusServerContext +from pymodbus.device import ModbusControlBlock +from pymodbus.device import ModbusDeviceIdentification +from pymodbus.transaction import * +from pymodbus.exceptions import NotImplementedException, NoSuchSlaveException +from pymodbus.pdu import ModbusExceptions as merror +from pymodbus.compat import socketserver, byte2int + +# --------------------------------------------------------------------------- # +# Logging +# --------------------------------------------------------------------------- # +import logging +_logger = logging.getLogger(__name__) +_logger.setLevel(logging.DEBUG) + + +# --------------------------------------------------------------------------- # +# Protocol Handlers +# --------------------------------------------------------------------------- # + +class ModbusBaseRequestHandler(asyncio.BaseProtocol): + """ Implements modbus slave wire protocol + This uses the asyncio.Protocol to implement the client handler. + + When a connection is established, the asyncio.Protocol.connection_made + callback is called. This callback will setup the connection and + create and schedule an asyncio.Task and assign it to running_task. + + running_task will be canceled upon connection_lost event. + """ + def __init__(self, owner): + self.server = owner + self.running = False + self.receive_queue = asyncio.Queue() + self.handler_task = None # coroutine to be run on asyncio loop + + def connection_made(self, transport): + """ + asyncio.BaseProtocol callback for socket establish + + For streamed protocols (TCP) this will also correspond to an + entire conversation; however for datagram protocols (UDP) this + corresponds to the socket being opened + """ + try: + _logger.debug("Socket [%s:%s] opened" % transport.get_extra_info('sockname')) + self.transport = transport + self.running = True + self.framer = self.server.framer(self.server.decoder, client=None) + + # schedule the connection handler on the event loop + self.handler_task = asyncio.create_task(self.handle()) + except Exception as ex: + _logger.debug("Datastore unable to fulfill request: " + "%s; %s", ex, traceback.format_exc()) + + def connection_lost(self, exc): + """ + asyncio.BaseProtocol callback for socket tear down + + For streamed protocols any break in the network connection will + be reported here; for datagram protocols, only a teardown of the + socket itself will result in this call. + """ + try: + exc_ = self.handler_task.exception() # this will contain any pending exceptions + self.handler_task.cancel() + _logger.debug("Socket [%s] closed" % transport.get_extra_info('sockname')) + + if exc is not None: + __logger.debug("Client Disconnection [%s:%s] due to %s" % (*self.client_address, exc)) + else: + _logger.debug("Client Disconnected [%s:%s]" % self.client_address) + self.server.active_connections.pop(self.client_address) + self.running = False + + except Exception as ex: + _logger.debug("Datastore unable to fulfill request: " + "%s; %s", ex, traceback.format_exc()) + + async def handle(self): + """Asyncio coroutine which represents a single conversation between + the modbus slave and master + + Once the client connection is established, the data chunks will be + fed to this coroutine via the asyncio.Queue object which is fed by + the ModbusBaseRequestHandler class's callback Future. + + This function will execute without blocking in the while-loop and + yield to the asyncio event loop when the is exhausted. + As a result, multiple clients can be interleaved without any + interference between them. + + To respond to Modbus...Server.server_close() (which clears each + handler's self.running), derive from this class to provide an + alternative handler that awakens from time to time when no input is + available and checks self.running. + Use Modbus...Server( handler=... ) keyword to supply the alternative + request handler class. + + """ + reset_frame = False + while self.running: + try: + units = self.server.context.slaves() + data = await self._recv_() + if isinstance(data, tuple): + data, *rest = data # rest carries possible contextual information + else: + rest = (None,) # empty tuple + + if not data: + self.running = False + # data = b'' # is this required? Once the running + # flag is unset, the whole thing comes down anyways + else: + if not isinstance(units, (list, tuple)): + units = [units] + # if broadcast is enabled make sure to + # process requests to address 0 + if self.server.broadcast_enable: + if 0 not in units: + units.append(0) + + if _logger.isEnabledFor(logging.DEBUG): + _logger.debug('Handling data: ' + hexlify_packets(data)) + + single = self.server.context.single + self.framer.processIncomingPacket(data, lambda x: self.execute(x, *rest) , + units, single=single) + + except asyncio.TimeoutError as msg: + if _logger.isEnabledFor(logging.DEBUG): + _logger.debug("Socket timeout occurred %s", msg) + reset_frame = True + except asyncio.InvalidStateError as e: + _logger.error("Socket error occurred %s" % e) + self.running = False + finally: + if reset_frame: + self.framer.resetFrame() + reset_frame = False + + def execute(self, request, *rest): + """ The callback to call with the resulting message + + :param request: The decoded request message + """ + broadcast = False + try: + print(request) + if self.server.broadcast_enable and request.unit_id == 0: + broadcast = True + # if broadcasting then execute on all slave contexts, note response will be ignored + for unit_id in self.server.context.slaves(): + response = request.execute(self.server.context[unit_id]) + else: + context = self.server.context[request.unit_id] + response = request.execute(context) + except NoSuchSlaveException as ex: + _logger.debug("requested slave does " + "not exist: %s" % request.unit_id ) + if self.server.ignore_missing_slaves: + return # the client will simply timeout waiting for a response + response = request.doException(merror.GatewayNoResponse) + except Exception as ex: + _logger.debug("Datastore unable to fulfill request: " + "%s; %s", ex, traceback.format_exc()) + response = request.doException(merror.SlaveFailure) + # no response when broadcasting + if not broadcast: + response.transaction_id = request.transaction_id + response.unit_id = request.unit_id + self.send(response, *rest) + + + def send(self, message, *rest): + if message.should_respond: + # self.server.control.Counter.BusMessage += 1 + pdu = self.framer.buildPacket(message) + if _logger.isEnabledFor(logging.DEBUG): + _logger.debug('send: [%s]- %s' % (message, b2a_hex(pdu))) + print(rest) + if rest == (None,): + self._send_(pdu) + else: + self._send_(pdu, *rest) + + # ----------------------------------------------------------------------- # + # Derived class implementations + # ----------------------------------------------------------------------- # + + def _send_(self, data): + """ Send a request (string) to the network + + :param message: The unencoded modbus response + """ + raise NotImplementedException("Method not implemented " + "by derived class") + async def _recv_(self): + """ Receive data from the network + + :return: + """ + raise NotImplementedException("Method not implemented " + "by derived class") + + +class ModbusConnectedRequestHandler(ModbusBaseRequestHandler,asyncio.Protocol): + """ Implements the modbus server protocol + + This uses asyncio.Protocol to implement + the client handler for a connected protocol (TCP). + """ + + def connection_made(self, transport): + """ asyncio.BaseProtocol: Called when a connection is made. """ + super().connection_made(transport) + + self.client_address = transport.get_extra_info('peername') + self.server.active_connections[self.client_address] = self + _logger.debug("TCP client connection established [%s:%s]" % self.client_address) + + def connection_lost(self, exc): + """ asyncio.BaseProtocol: Called when the connection is lost or closed.""" + _logger.debug("TCP client disconnected [%s:%s]" % self.client_address) + self.server.active_connections.pop(self.client_address) + + + def data_received(self,data): + """ + asyncio.Protocol: (TCP) Called when some data is received. + data is a non-empty bytes object containing the incoming data. + """ + asyncio.create_task(self.receive_queue.put(data)) + + async def _recv_(self): + return await self.receive_queue.get() + + def _send_(self, data): + """ tcp send """ + self.transport.write(data) + + +class ModbusDisconnectedRequestHandler(ModbusBaseRequestHandler, asyncio.DatagramProtocol): + """ Implements the modbus server protocol + + This uses the socketserver.BaseRequestHandler to implement + the client handler for a disconnected protocol (UDP). The + only difference is that we have to specify who to send the + resulting packet data to. + """ + def __init__(self,owner): + super().__init__(owner) + self.server.on_connection_terminated = asyncio.get_event_loop().create_future() + + def connection_lost(self,exc): + super().connection_lost(exc) + self.server.on_connection_terminated.set_result(True) + + def datagram_received(self,data, addr): + """ + asyncio.DatagramProtocol: Called when a datagram is received. + data is a bytes object containing the incoming data. addr + is the address of the peer sending the data; the exact + format depends on the transport. + """ + asyncio.create_task(self.receive_queue.put((data, addr))) + + def error_received(self,exc): + """ + asyncio.DatagramProtocol: Called when a previous send + or receive operation raises an OSError. exc is the + OSError instance. + + This method is called in rare conditions, + when the transport (e.g. UDP) detects that a datagram could + not be delivered to its recipient. In many conditions + though, undeliverable datagrams will be silently dropped. + """ + _logger.error("datagram connection error [%s]" % exc) + + async def _recv_(self): + return await self.receive_queue.get() + + def _send_(self, data, addr): + self.transport.sendto(data, addr=addr) + +# --------------------------------------------------------------------------- # +# Server Implementations +# --------------------------------------------------------------------------- # +class ModbusTcpServer: + """ + A modbus threaded tcp socket server + + We inherit and overload the socket server so that we + can control the client threads as well as have a single + server context instance. + """ + + def __init__(self, + context, + framer=None, + identity=None, + address=None, + handler=None, + allow_reuse_address=False, + allow_reuse_port=False, + defer_start=False, + backlog=20, + loop=None, + **kwargs): + """ Overloaded initializer for the socket server + + If the identify structure is not passed in, the ModbusControlBlock + uses its own empty structure. + + :param context: The ModbusServerContext datastore + :param framer: The framer strategy to use + :param identity: An optional identify structure + :param address: An optional (interface, port) to bind to. + :param handler: A handler for each client session; default is + ModbusConnectedRequestHandler. The handler class + receives connection create/teardown events + :param allow_reuse_address: Whether the server will allow the + reuse of an address. + :param allow_reuse_port: Whether the server will allow the + reuse of a port. + :param backlog: is the maximum number of queued connections + passed to listen(). Defaults to 20, increase if many + connections are being made and broken to your Modbus slave + :param loop: optional asyncio event loop to run in. Will default to + asyncio.get_event_loop() supplied value if None. + :param ignore_missing_slaves: True to not send errors on a request + to a missing slave + :param broadcast_enable: True to treat unit_id 0 as broadcast address, + False to treat 0 as any other unit_id + """ + self.active_connections = {} + self.loop = loop or asyncio.get_event_loop() + self.allow_reuse_address = allow_reuse_address + self.decoder = ServerDecoder() + self.framer = framer or ModbusSocketFramer + self.context = context or ModbusServerContext() + self.control = ModbusControlBlock() + self.address = address or ("", Defaults.Port) + self.handler = handler or ModbusConnectedRequestHandler + self.handler.server = self + self.ignore_missing_slaves = kwargs.get('ignore_missing_slaves', + Defaults.IgnoreMissingSlaves) + self.broadcast_enable = kwargs.get('broadcast_enable', + Defaults.broadcast_enable) + + if isinstance(identity, ModbusDeviceIdentification): + self.control.Identity.update(identity) + + self.server = None + self.server_factory = self.loop.create_server(lambda : self.handler(self), + *self.address, + reuse_address=allow_reuse_address, + reuse_port=allow_reuse_port, + backlog=backlog, + start_serving=not defer_start) + + async def serve_forever(self): + if self.server is None: + self.server = await self.server_factory + + await self.server.serve_forever() + + def server_close(self): + self.server.close() + + +class ModbusUdpServer: + """ + A modbus threaded udp socket server + + We inherit and overload the socket server so that we + can control the client threads as well as have a single + server context instance. + """ + + def __init__(self, context, framer=None, identity=None, address=None, + handler=None, allow_reuse_address=False, + allow_reuse_port=False, + defer_start=False, + backlog=20, + loop=None, + **kwargs): + """ Overloaded initializer for the socket server + + If the identify structure is not passed in, the ModbusControlBlock + uses its own empty structure. + + :param context: The ModbusServerContext datastore + :param framer: The framer strategy to use + :param identity: An optional identify structure + :param address: An optional (interface, port) to bind to. + :param handler: A handler for each client session; default is + ModbusDisonnectedRequestHandler + :param ignore_missing_slaves: True to not send errors on a request + to a missing slave + :param broadcast_enable: True to treat unit_id 0 as broadcast address, + False to treat 0 as any other unit_id + """ + self.loop = loop or asyncio.get_event_loop() + self.decoder = ServerDecoder() + self.framer = framer or ModbusSocketFramer + self.context = context or ModbusServerContext() + self.control = ModbusControlBlock() + self.address = address or ("", Defaults.Port) + self.handler = handler or ModbusDisconnectedRequestHandler + self.ignore_missing_slaves = kwargs.get('ignore_missing_slaves', + Defaults.IgnoreMissingSlaves) + self.broadcast_enable = kwargs.get('broadcast_enable', + Defaults.broadcast_enable) + + if isinstance(identity, ModbusDeviceIdentification): + self.control.Identity.update(identity) + + self.protocol = None + self.endpoint = None + self.on_connection_terminated = None + self.server_factory = self.loop.create_datagram_endpoint(lambda: self.handler(self), + local_addr=self.address, + reuse_address=allow_reuse_address, + reuse_port=allow_reuse_port, + allow_broadcast=True) + + async def serve_forever(self): + if self.protocol is None: + self.protocol, self.endpoint = await self.server_factory + + await self.on_connection_terminated + + def server_close(self): + self.endpoint.close() + + + +class ModbusSerialServer(object): + """ + A modbus threaded serial socket server + + We inherit and overload the socket server so that we + can control the client threads as well as have a single + server context instance. + """ + + handler = None + + def __init__(self, context, framer=None, identity=None, **kwargs): + """ Overloaded initializer for the socket server + + If the identify structure is not passed in, the ModbusControlBlock + uses its own empty structure. + + :param context: The ModbusServerContext datastore + :param framer: The framer strategy to use + :param identity: An optional identify structure + :param port: The serial port to attach to + :param stopbits: The number of stop bits to use + :param bytesize: The bytesize of the serial messages + :param parity: Which kind of parity to use + :param baudrate: The baud rate to use for the serial device + :param timeout: The timeout to use for the serial device + :param ignore_missing_slaves: True to not send errors on a request + to a missing slave + :param broadcast_enable: True to treat unit_id 0 as broadcast address, + False to treat 0 as any other unit_id + """ + raise NotImplementedException + +# --------------------------------------------------------------------------- # +# Creation Factories +# --------------------------------------------------------------------------- # +async def StartTcpServer(context=None, identity=None, address=None, + custom_functions=[], defer_start=True, **kwargs): + """ A factory to start and run a tcp modbus server + + :param context: The ModbusServerContext datastore + :param identity: An optional identify structure + :param address: An optional (interface, port) to bind to. + :param custom_functions: An optional list of custom function classes + supported by server instance. + :param defer_start: if set, a coroutine which can be started and stopped + will be returned. Otherwise, the server will be immediately spun + up without the ability to shut it off from within the asyncio loop + :param ignore_missing_slaves: True to not send errors on a request to a + missing slave + :return: an initialized but inactive server object coroutine + """ + framer = kwargs.pop("framer", ModbusSocketFramer) + server = ModbusTcpServer(context, framer, identity, address, **kwargs) + + for f in custom_functions: + server.decoder.register(f) + + if not defer_start: + await server.serve_forever() + + return server + + + + +async def StartUdpServer(context=None, identity=None, address=None, + custom_functions=[], defer_start=True, **kwargs): + """ A factory to start and run a udp modbus server + + :param context: The ModbusServerContext datastore + :param identity: An optional identify structure + :param address: An optional (interface, port) to bind to. + :param custom_functions: An optional list of custom function classes + supported by server instance. + :param framer: The framer to operate with (default ModbusSocketFramer) + :param ignore_missing_slaves: True to not send errors on a request + to a missing slave + """ + framer = kwargs.pop('framer', ModbusSocketFramer) + server = ModbusUdpServer(context, framer, identity, address, **kwargs) + + for f in custom_functions: + server.decoder.register(f) + + if not defer_start: + await server.serve_forever() + + return server + + + +def StartSerialServer(context=None, identity=None, custom_functions=[], + **kwargs): + """ A factory to start and run a serial modbus server + + :param context: The ModbusServerContext datastore + :param identity: An optional identify structure + :param custom_functions: An optional list of custom function classes + supported by server instance. + :param framer: The framer to operate with (default ModbusAsciiFramer) + :param port: The serial port to attach to + :param stopbits: The number of stop bits to use + :param bytesize: The bytesize of the serial messages + :param parity: Which kind of parity to use + :param baudrate: The baud rate to use for the serial device + :param timeout: The timeout to use for the serial device + :param ignore_missing_slaves: True to not send errors on a request to a + missing slave + """ + framer = kwargs.pop('framer', ModbusAsciiFramer) + server = ModbusSerialServer(context, framer, identity, **kwargs) + for f in custom_functions: + server.decoder.register(f) + server.serve_forever() + +# --------------------------------------------------------------------------- # +# Exported symbols +# --------------------------------------------------------------------------- # + + +__all__ = [ + "StartTcpServer", "StartUdpServer", "StartSerialServer" +] + From a952ff240202c8c52c8db21d670c39ab2a542d5e Mon Sep 17 00:00:00 2001 From: Memet Bilgin Date: Sun, 24 Mar 2019 21:01:21 +1100 Subject: [PATCH 18/26] preliminary asyncio server examples --- examples/common/asyncio_server.py | 158 ++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100755 examples/common/asyncio_server.py diff --git a/examples/common/asyncio_server.py b/examples/common/asyncio_server.py new file mode 100755 index 000000000..e55fce6b5 --- /dev/null +++ b/examples/common/asyncio_server.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python +""" +Pymodbus Asyncio Server Example +-------------------------------------------------------------------------- + +The asyncio server is implemented in pure python without any third +party libraries (unless you need to use the serial protocols which require +asyncio-pyserial). This is helpful in constrained or old environments where using +twisted is just not feasible. What follows is an example of its use: +""" +# --------------------------------------------------------------------------- # +# import the various server implementations +# --------------------------------------------------------------------------- # +from pymodbus.server.asyncio import StartTcpServer +from pymodbus.server.asyncio import StartUdpServer +from pymodbus.server.asyncio import StartSerialServer + +from pymodbus.device import ModbusDeviceIdentification +from pymodbus.datastore import ModbusSequentialDataBlock, ModbusSparseDataBlock +from pymodbus.datastore import ModbusSlaveContext, ModbusServerContext + +from pymodbus.transaction import ModbusRtuFramer, ModbusBinaryFramer +# --------------------------------------------------------------------------- # +# configure the service logging +# --------------------------------------------------------------------------- # +import logging +FORMAT = ('%(asctime)-15s %(threadName)-15s' + ' %(levelname)-8s %(module)-15s:%(lineno)-8s %(message)s') +logging.basicConfig(format=FORMAT) +log = logging.getLogger() +log.setLevel(logging.DEBUG) + + +async def run_server(): + # ----------------------------------------------------------------------- # + # initialize your data store + # ----------------------------------------------------------------------- # + # The datastores only respond to the addresses that they are initialized to + # Therefore, if you initialize a DataBlock to addresses of 0x00 to 0xFF, a + # request to 0x100 will respond with an invalid address exception. This is + # because many devices exhibit this kind of behavior (but not all):: + # + # block = ModbusSequentialDataBlock(0x00, [0]*0xff) + # + # Continuing, you can choose to use a sequential or a sparse DataBlock in + # your data context. The difference is that the sequential has no gaps in + # the data while the sparse can. Once again, there are devices that exhibit + # both forms of behavior:: + # + # block = ModbusSparseDataBlock({0x00: 0, 0x05: 1}) + # block = ModbusSequentialDataBlock(0x00, [0]*5) + # + # Alternately, you can use the factory methods to initialize the DataBlocks + # or simply do not pass them to have them initialized to 0x00 on the full + # address range:: + # + # store = ModbusSlaveContext(di = ModbusSequentialDataBlock.create()) + # store = ModbusSlaveContext() + # + # Finally, you are allowed to use the same DataBlock reference for every + # table or you may use a separate DataBlock for each table. + # This depends if you would like functions to be able to access and modify + # the same data or not:: + # + # block = ModbusSequentialDataBlock(0x00, [0]*0xff) + # store = ModbusSlaveContext(di=block, co=block, hr=block, ir=block) + # + # The server then makes use of a server context that allows the server to + # respond with different slave contexts for different unit ids. By default + # it will return the same context for every unit id supplied (broadcast + # mode). + # However, this can be overloaded by setting the single flag to False and + # then supplying a dictionary of unit id to context mapping:: + # + # slaves = { + # 0x01: ModbusSlaveContext(...), + # 0x02: ModbusSlaveContext(...), + # 0x03: ModbusSlaveContext(...), + # } + # context = ModbusServerContext(slaves=slaves, single=False) + # + # The slave context can also be initialized in zero_mode which means that a + # request to address(0-7) will map to the address (0-7). The default is + # False which is based on section 4.4 of the specification, so address(0-7) + # will map to (1-8):: + # + # store = ModbusSlaveContext(..., zero_mode=True) + # ----------------------------------------------------------------------- # + store = ModbusSlaveContext( + di=ModbusSequentialDataBlock(0, [17]*100), + co=ModbusSequentialDataBlock(0, [17]*100), + hr=ModbusSequentialDataBlock(0, [17]*100), + ir=ModbusSequentialDataBlock(0, [17]*100)) + + context = ModbusServerContext(slaves=store, single=True) + + # ----------------------------------------------------------------------- # + # initialize the server information + # ----------------------------------------------------------------------- # + # If you don't set this or any fields, they are defaulted to empty strings. + # ----------------------------------------------------------------------- # + identity = ModbusDeviceIdentification() + identity.VendorName = 'Pymodbus' + identity.ProductCode = 'PM' + identity.VendorUrl = 'http://github.com/riptideio/pymodbus/' + identity.ProductName = 'Pymodbus Server' + identity.ModelName = 'Pymodbus Server' + identity.MajorMinorRevision = '2.2.0' + + # ----------------------------------------------------------------------- # + # run the server you want + # ----------------------------------------------------------------------- # + # Tcp: + # immediately start serving: + server = await StartTcpServer(context, identity=identity, address=("0.0.0.0", 5020), + allow_reuse_address=True) + + # deferred start: + # server = await StartTcpServer(context, identity=identity, address=("0.0.0.0", 5020), + # allow_reuse_address=True, defer_start=True) + + # asyncio.get_event_loop().call_later(20, lambda : server.) + # await server.serve_forever() + + + + + # TCP with different framer + # StartTcpServer(context, identity=identity, + # framer=ModbusRtuFramer, address=("0.0.0.0", 5020)) + + # Udp: + # server = await StartUdpServer(context, identity=identity, address=("0.0.0.0", 5020), + # allow_reuse_address=True, defer_start=True) + # + # await server.serve_forever() + + + # !!! SERIAL SERVER NOT IMPLEMENTED !!! + # Ascii: + # StartSerialServer(context, identity=identity, + # port='/dev/ttyp0', timeout=1) + + # RTU: + # StartSerialServer(context, framer=ModbusRtuFramer, identity=identity, + # port='/dev/ttyp0', timeout=.005, baudrate=9600) + + # Binary + # StartSerialServer(context, + # identity=identity, + # framer=ModbusBinaryFramer, + # port='/dev/ttyp0', + # timeout=1) + + +if __name__ == "__main__": + asyncio.run(run_server()) + From 50c511791c3601cc5eb2f8fb081f01db10f5d656 Mon Sep 17 00:00:00 2001 From: Memet Bilgin Date: Thu, 2 May 2019 15:21:36 +1000 Subject: [PATCH 19/26] move serial module dependency into class instantiation --- pymodbus/client/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymodbus/client/sync.py b/pymodbus/client/sync.py index 04d7778e3..d413279e9 100644 --- a/pymodbus/client/sync.py +++ b/pymodbus/client/sync.py @@ -1,6 +1,5 @@ import socket import select -import serial import time import sys from functools import partial @@ -425,6 +424,7 @@ def __init__(self, method='ascii', **kwargs): :param strict: Use Inter char timeout for baudrates <= 19200 (adhere to modbus standards) """ + import serial self.method = method self.socket = None BaseModbusClient.__init__(self, self.__implementation(method, self), From dcd5074f49c255cc24412697c8e023ff1ddfc1fa Mon Sep 17 00:00:00 2001 From: Memet Bilgin Date: Mon, 13 May 2019 14:23:32 +1000 Subject: [PATCH 20/26] unittests for asyncio based server implementation --- pymodbus/server/asyncio.py | 178 +++++++----- test/test_server_asyncio.py | 544 ++++++++++++++++++++++++++++++++++++ 2 files changed, 658 insertions(+), 64 deletions(-) create mode 100755 test/test_server_asyncio.py diff --git a/pymodbus/server/asyncio.py b/pymodbus/server/asyncio.py index ef802dd20..d93189c46 100755 --- a/pymodbus/server/asyncio.py +++ b/pymodbus/server/asyncio.py @@ -64,7 +64,7 @@ def connection_made(self, transport): # schedule the connection handler on the event loop self.handler_task = asyncio.create_task(self.handle()) - except Exception as ex: + except Exception as ex: # pragma: no cover _logger.debug("Datastore unable to fulfill request: " "%s; %s", ex, traceback.format_exc()) @@ -77,18 +77,19 @@ def connection_lost(self, exc): socket itself will result in this call. """ try: - exc_ = self.handler_task.exception() # this will contain any pending exceptions self.handler_task.cancel() - _logger.debug("Socket [%s] closed" % transport.get_extra_info('sockname')) - if exc is not None: + if exc is None: + if hasattr(self, "client_address"): # TCP connection + _logger.debug("Disconnected from client [%s:%s]" % self.client_address) + else: + _logger.debug("Disconnected from client [%s]" % self.transport.get_extra_info("peername")) + else: # pragma: no cover __logger.debug("Client Disconnection [%s:%s] due to %s" % (*self.client_address, exc)) - else: - _logger.debug("Client Disconnected [%s:%s]" % self.client_address) - self.server.active_connections.pop(self.client_address) + self.running = False - except Exception as ex: + except Exception as ex: # pragma: no cover _logger.debug("Datastore unable to fulfill request: " "%s; %s", ex, traceback.format_exc()) @@ -100,69 +101,78 @@ async def handle(self): fed to this coroutine via the asyncio.Queue object which is fed by the ModbusBaseRequestHandler class's callback Future. + This callback future gets data from either asyncio.DatagramProtocol.datagram_received + or from asyncio.BaseProtocol.data_received. + This function will execute without blocking in the while-loop and - yield to the asyncio event loop when the is exhausted. + yield to the asyncio event loop when the frame is exhausted. As a result, multiple clients can be interleaved without any interference between them. - To respond to Modbus...Server.server_close() (which clears each - handler's self.running), derive from this class to provide an - alternative handler that awakens from time to time when no input is - available and checks self.running. - Use Modbus...Server( handler=... ) keyword to supply the alternative - request handler class. + For ModbusConnectedRequestHandler, each connection will be given an + instance of the handle() coroutine and this instance will be put in the + active_connections dict. Calling server_close will individually cancel + each running handle() task. + + For ModbusDisconnectedRequestHandler, a single handle() coroutine will + be started and maintained. Calling server_close will cancel that task. """ reset_frame = False while self.running: try: units = self.server.context.slaves() - data = await self._recv_() + data = await self._recv_() # this is an asyncio.Queue await, it will never fail if isinstance(data, tuple): - data, *rest = data # rest carries possible contextual information + data, *addr = data # addr is populated when talking over UDP else: - rest = (None,) # empty tuple + addr = (None,) # empty tuple - if not data: - self.running = False - # data = b'' # is this required? Once the running - # flag is unset, the whole thing comes down anyways - else: - if not isinstance(units, (list, tuple)): - units = [units] - # if broadcast is enabled make sure to - # process requests to address 0 - if self.server.broadcast_enable: - if 0 not in units: - units.append(0) + if not isinstance(units, (list, tuple)): + units = [units] + # if broadcast is enabled make sure to + # process requests to address 0 + if self.server.broadcast_enable: # pragma: no cover + if 0 not in units: + units.append(0) if _logger.isEnabledFor(logging.DEBUG): _logger.debug('Handling data: ' + hexlify_packets(data)) single = self.server.context.single - self.framer.processIncomingPacket(data, lambda x: self.execute(x, *rest) , - units, single=single) - - except asyncio.TimeoutError as msg: - if _logger.isEnabledFor(logging.DEBUG): - _logger.debug("Socket timeout occurred %s", msg) - reset_frame = True - except asyncio.InvalidStateError as e: - _logger.error("Socket error occurred %s" % e) - self.running = False + self.framer.processIncomingPacket(data=data, + callback=lambda x: self.execute(x, *addr), + unit=units, + single=single) + + except asyncio.CancelledError: + # catch and ignore cancelation errors + if isinstance(self, ModbusConnectedRequestHandler): + _logger.debug("Handler for stream [%s:%s] has been canceled" % self.client_address) + else: + _logger.debug("Handler for UDP socket [%s] has been canceled" % self.protocol._sock.getsockname()[1]) + + except Exception as e: + # force TCP socket termination as processIncomingPacket should handle applicaiton layer errors + # for UDP sockets, simply reset the frame + if isinstance(self, ModbusConnectedRequestHandler): + _logger.info("Unknown exception '%s' on stream [%s:%s] forcing disconnect" % (e, *self.client_address)) + self.transport.close() + else: + _logger.error("Unknown error occurred %s" % e) + reset_frame = True # graceful recovery finally: if reset_frame: self.framer.resetFrame() reset_frame = False - def execute(self, request, *rest): + def execute(self, request, *addr): """ The callback to call with the resulting message :param request: The decoded request message """ broadcast = False try: - print(request) if self.server.broadcast_enable and request.unit_id == 0: broadcast = True # if broadcasting then execute on all slave contexts, note response will be ignored @@ -185,33 +195,32 @@ def execute(self, request, *rest): if not broadcast: response.transaction_id = request.transaction_id response.unit_id = request.unit_id - self.send(response, *rest) + self.send(response, *addr) - def send(self, message, *rest): + def send(self, message, *addr): if message.should_respond: # self.server.control.Counter.BusMessage += 1 pdu = self.framer.buildPacket(message) if _logger.isEnabledFor(logging.DEBUG): _logger.debug('send: [%s]- %s' % (message, b2a_hex(pdu))) - print(rest) - if rest == (None,): + if addr == (None,): self._send_(pdu) else: - self._send_(pdu, *rest) + self._send_(pdu, *addr) # ----------------------------------------------------------------------- # # Derived class implementations # ----------------------------------------------------------------------- # - def _send_(self, data): + def _send_(self, data): # pragma: no cover """ Send a request (string) to the network :param message: The unencoded modbus response """ raise NotImplementedException("Method not implemented " "by derived class") - async def _recv_(self): + async def _recv_(self): # pragma: no cover """ Receive data from the network :return: @@ -237,8 +246,10 @@ def connection_made(self, transport): def connection_lost(self, exc): """ asyncio.BaseProtocol: Called when the connection is lost or closed.""" + super().connection_lost(exc) _logger.debug("TCP client disconnected [%s:%s]" % self.client_address) - self.server.active_connections.pop(self.client_address) + if self.client_address in self.server.active_connections: + self.server.active_connections.pop(self.client_address) def data_received(self,data): @@ -246,7 +257,7 @@ def data_received(self,data): asyncio.Protocol: (TCP) Called when some data is received. data is a non-empty bytes object containing the incoming data. """ - asyncio.create_task(self.receive_queue.put(data)) + self.receive_queue.put_nowait(data) async def _recv_(self): return await self.receive_queue.get() @@ -279,9 +290,9 @@ def datagram_received(self,data, addr): is the address of the peer sending the data; the exact format depends on the transport. """ - asyncio.create_task(self.receive_queue.put((data, addr))) + self.receive_queue.put_nowait((data, addr)) - def error_received(self,exc): + def error_received(self,exc): # pragma: no cover """ asyncio.DatagramProtocol: Called when a previous send or receive operation raises an OSError. exc is the @@ -300,6 +311,19 @@ async def _recv_(self): def _send_(self, data, addr): self.transport.sendto(data, addr=addr) +class ModbusServerFactory: + """ + Builder class for a modbus server + + This also holds the server datastore so that it is persisted between connections + """ + + def __init__(self, store, framer=None, identity=None, **kwargs): + import warnings + warnings.warn("deprecated API for asyncio. ServerFactory's are a twisted construct and don't have an equivalent in asyncio", + DeprecationWarning) + + # --------------------------------------------------------------------------- # # Server Implementations # --------------------------------------------------------------------------- # @@ -368,7 +392,8 @@ def __init__(self, if isinstance(identity, ModbusDeviceIdentification): self.control.Identity.update(identity) - self.server = None + self.serving = self.loop.create_future() # asyncio future that will be done once server has started + self.server = None # constructors cannot be declared async, so we have to defer the initialization of the server self.server_factory = self.loop.create_server(lambda : self.handler(self), *self.address, reuse_address=allow_reuse_address, @@ -379,10 +404,16 @@ def __init__(self, async def serve_forever(self): if self.server is None: self.server = await self.server_factory - - await self.server.serve_forever() + self.serving.set_result(True) + await self.server.serve_forever() + else: + raise RuntimeError("Can't call serve_forever on an already running server object") def server_close(self): + for k,v in self.active_connections.items(): + _logger.warning(f"aborting active session {k}") + v.handler_task.cancel() + self.active_connections = {} self.server.close() @@ -436,6 +467,8 @@ def __init__(self, context, framer=None, identity=None, address=None, self.protocol = None self.endpoint = None self.on_connection_terminated = None + self.stop_serving = self.loop.create_future() + self.serving = self.loop.create_future() # asyncio future that will be done once server has started self.server_factory = self.loop.create_datagram_endpoint(lambda: self.handler(self), local_addr=self.address, reuse_address=allow_reuse_address, @@ -445,11 +478,17 @@ def __init__(self, context, framer=None, identity=None, address=None, async def serve_forever(self): if self.protocol is None: self.protocol, self.endpoint = await self.server_factory - - await self.on_connection_terminated + self.serving.set_result(True) + await self.stop_serving + else: + raise RuntimeError("Can't call serve_forever on an already running server object") def server_close(self): - self.endpoint.close() + self.stop_serving.set_result(True) + if self.endpoint.handler_task is not None: + self.endpoint.handler_task.cancel() + + self.protocol.close() @@ -464,7 +503,7 @@ class ModbusSerialServer(object): handler = None - def __init__(self, context, framer=None, identity=None, **kwargs): + def __init__(self, context, framer=None, identity=None, **kwargs): # pragma: no cover """ Overloaded initializer for the socket server If the identify structure is not passed in, the ModbusControlBlock @@ -509,7 +548,7 @@ async def StartTcpServer(context=None, identity=None, address=None, server = ModbusTcpServer(context, framer, identity, address, **kwargs) for f in custom_functions: - server.decoder.register(f) + server.decoder.register(f) # pragma: no cover if not defer_start: await server.serve_forever() @@ -536,17 +575,17 @@ async def StartUdpServer(context=None, identity=None, address=None, server = ModbusUdpServer(context, framer, identity, address, **kwargs) for f in custom_functions: - server.decoder.register(f) + server.decoder.register(f) # pragma: no cover if not defer_start: - await server.serve_forever() + await server.serve_forever() # pragma: no cover return server def StartSerialServer(context=None, identity=None, custom_functions=[], - **kwargs): + **kwargs):# pragma: no cover """ A factory to start and run a serial modbus server :param context: The ModbusServerContext datastore @@ -563,12 +602,23 @@ def StartSerialServer(context=None, identity=None, custom_functions=[], :param ignore_missing_slaves: True to not send errors on a request to a missing slave """ + raise NotImplementedException framer = kwargs.pop('framer', ModbusAsciiFramer) server = ModbusSerialServer(context, framer, identity, **kwargs) for f in custom_functions: server.decoder.register(f) server.serve_forever() +def StopServer(): + """ + Helper method to stop Async Server + """ + import warnings + warnings.warn("deprecated API for asyncio. Call server_close() on server object returned by StartXxxServer", + DeprecationWarning) + + + # --------------------------------------------------------------------------- # # Exported symbols # --------------------------------------------------------------------------- # diff --git a/test/test_server_asyncio.py b/test/test_server_asyncio.py new file mode 100755 index 000000000..8b86af329 --- /dev/null +++ b/test/test_server_asyncio.py @@ -0,0 +1,544 @@ +#!/usr/bin/env python +from pymodbus.compat import IS_PYTHON3 +import asynctest +import asyncio +import logging +_logger = logging.getLogger() +if IS_PYTHON3: # Python 3 + from asynctest.mock import patch, Mock, MagicMock +else: + assert(False, "asyncio is not available pre-python 3.6") + +from pymodbus.device import ModbusDeviceIdentification +from pymodbus.factory import ServerDecoder +from pymodbus.server.asynchronous import ModbusTcpProtocol, ModbusUdpProtocol +from pymodbus.server.asyncio import StartTcpServer, StartUdpServer, StartSerialServer, StopServer, ModbusServerFactory +from pymodbus.server.asyncio import ModbusConnectedRequestHandler, ModbusBaseRequestHandler +from pymodbus.datastore import ModbusSequentialDataBlock +from pymodbus.datastore import ModbusSlaveContext, ModbusServerContext +from pymodbus.compat import byte2int +from pymodbus.transaction import ModbusSocketFramer +from pymodbus.exceptions import NoSuchSlaveException, ModbusIOException + +import sys +#---------------------------------------------------------------------------# +# Fixture +#---------------------------------------------------------------------------# +import platform +from distutils.version import LooseVersion + +IS_DARWIN = platform.system().lower() == "darwin" +OSX_SIERRA = LooseVersion("10.12") +if IS_DARWIN: + IS_HIGH_SIERRA_OR_ABOVE = LooseVersion(platform.mac_ver()[0]) + SERIAL_PORT = '/dev/ptyp0' if not IS_HIGH_SIERRA_OR_ABOVE else '/dev/ttyp0' +else: + IS_HIGH_SIERRA_OR_ABOVE = False + SERIAL_PORT = "/dev/ptmx" + + +class AsyncioServerTest(asynctest.TestCase): + ''' + This is the unittest for the pymodbus.server.asyncio module + + The scope of this unit test is the life-cycle management of the network + connections and server objects. + + This unittest suite does not attempt to test any of the underlying protocol details + ''' + + #-----------------------------------------------------------------------# + # Setup/TearDown + #-----------------------------------------------------------------------# + def setUp(self): + ''' + Initialize the test environment by setting up a dummy store and context + ''' + self.store = ModbusSlaveContext( di=ModbusSequentialDataBlock(0, [17]*100), + co=ModbusSequentialDataBlock(0, [17]*100), + hr=ModbusSequentialDataBlock(0, [17]*100), + ir=ModbusSequentialDataBlock(0, [17]*100)) + self.context = ModbusServerContext(slaves=self.store, single=True) + + def tearDown(self): + ''' Cleans up the test environment ''' + pass + + #-----------------------------------------------------------------------# + # Test ModbusConnectedRequestHandler + #-----------------------------------------------------------------------# + async def testStartTcpServer(self): + ''' Test that the modbus tcp asyncio server starts correctly ''' + identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) + self.loop = asynctest.Mock(self.loop) + server = await StartTcpServer(context=self.context,loop=self.loop,identity=identity) + self.assertEqual(server.control.Identity.VendorName, 'VendorName') + self.loop.create_server.assert_called_once() + + async def testTcpServerServeNoDefer(self): + ''' Test StartTcpServer without deferred start (immediate execution of server) ''' + with patch('asyncio.base_events.Server.serve_forever', new_callable=asynctest.CoroutineMock) as serve: + server = await StartTcpServer(context=self.context,address=("127.0.0.1", 0), loop=self.loop, defer_start=False) + serve.assert_awaited() + + async def testTcpServerServeForever(self): + ''' Test StartTcpServer serve_forever() method ''' + with patch('asyncio.base_events.Server.serve_forever', new_callable=asynctest.CoroutineMock) as serve: + server = await StartTcpServer(context=self.context,address=("127.0.0.1", 0), loop=self.loop) + await server.serve_forever() + serve.assert_awaited() + + async def testTcpServerServeForeverTwice(self): + ''' Call on serve_forever() twice should result in a runtime error ''' + server = await StartTcpServer(context=self.context,address=("127.0.0.1", 0), loop=self.loop) + server_task = asyncio.create_task(server.serve_forever()) + await server.serving + with self.assertRaises(RuntimeError): + await server.serve_forever() + server.server_close() + + async def testTcpServerReceiveData(self): + ''' Test data sent on socket is received by internals - doesn't not process data ''' + data = b'\x01\x00\x00\x00\x00\x06\x01\x03\x00\x00\x00\x19' + server = await StartTcpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) + server_task = asyncio.create_task(server.serve_forever()) + await server.serving + with patch('pymodbus.transaction.ModbusSocketFramer.processIncomingPacket', new_callable=Mock) as process: + # process = server.framer.processIncomingPacket = Mock() + connected = self.loop.create_future() + random_port = server.server.sockets[0].getsockname()[1] # get the random server port + + class BasicClient(asyncio.BaseProtocol): + def connection_made(self, transport): + self.transport = transport + self.transport.write(data) + connected.set_result(True) + + def eof_received(self): + pass + + transport, protocol = await self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) + await asyncio.sleep(0.1) # this may be better done by making an internal hook in the actual implementation + # if this unit test fails on a machine, see if increasing the sleep time makes a difference, if it does + # blame author for a fix + + process.assert_called_once() + self.assertTrue( process.call_args[1]["data"] == data ) + server.server_close() + + async def testTcpServerRoundtrip(self): + ''' Test sending and receiving data on tcp socket ''' + data = b"\x01\x00\x00\x00\x00\x06\x01\x03\x00\x00\x00\x01" # unit 1, read register + expected_response = b'\x01\x00\x00\x00\x00\x05\x01\x03\x02\x00\x11' # value of 17 as per context + server = await StartTcpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) + server_task = asyncio.create_task(server.serve_forever()) + await server.serving + + random_port = server.server.sockets[0].getsockname()[1] # get the random server port + + connected, done = self.loop.create_future(),self.loop.create_future() + received_value = None + + class BasicClient(asyncio.BaseProtocol): + def connection_made(self, transport): + self.transport = transport + self.transport.write(data) + connected.set_result(True) + + def data_received(self, data): + nonlocal received_value, done + received_value = data + done.set_result(True) + + def eof_received(self): + pass + + transport, protocol = await self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) + await asyncio.wait_for(done, timeout=0.1) + + self.assertEqual(received_value, expected_response) + + transport.close() + await asyncio.sleep(0) + server.server_close() + + async def testTcpServerConnectionLost(self): + ''' Test tcp stream interruption ''' + data = b"\x01\x00\x00\x00\x00\x06\x01\x01\x00\x00\x00\x01" + server = await StartTcpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) + server_task = asyncio.create_task(server.serve_forever()) + await server.serving + + random_port = server.server.sockets[0].getsockname()[1] # get the random server port + + step1 = self.loop.create_future() + done = self.loop.create_future() + received_value = None + + class BasicClient(asyncio.BaseProtocol): + def connection_made(self, transport): + self.transport = transport + step1.set_result(True) + + transport, protocol = await self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) + await step1 + + self.assertTrue( len(server.active_connections) == 1 ) + + protocol.transport.close() # close isn't synchronous and there's no notification that it's done + # so we have to wait a bit + await asyncio.sleep(0.1) + self.assertTrue( len(server.active_connections) == 0 ) + server.server_close() + + async def testTcpServerCloseActiveConnection(self): + ''' Test server_close() while there are active TCP connections ''' + data = b"\x01\x00\x00\x00\x00\x06\x01\x01\x00\x00\x00\x01" + server = await StartTcpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) + server_task = asyncio.create_task(server.serve_forever()) + await server.serving + + random_port = server.server.sockets[0].getsockname()[1] # get the random server port + + step1 = self.loop.create_future() + done = self.loop.create_future() + received_value = None + + class BasicClient(asyncio.BaseProtocol): + def connection_made(self, transport): + self.transport = transport + step1.set_result(True) + + transport, protocol = await self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) + await step1 + + server.server_close() + + # close isn't synchronous and there's no notification that it's done + # so we have to wait a bit + await asyncio.sleep(0.0) + self.assertTrue( len(server.active_connections) == 0 ) + + async def testTcpServerException(self): + ''' Sending garbage data on a TCP socket should drop the connection ''' + garbage = b'\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF' + server = await StartTcpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) + server_task = asyncio.create_task(server.serve_forever()) + await server.serving + with patch('pymodbus.transaction.ModbusSocketFramer.processIncomingPacket', + new_callable=lambda : Mock(side_effect=Exception)) as process: + connect, receive, eof = self.loop.create_future(),self.loop.create_future(),self.loop.create_future() + received_data = None + random_port = server.server.sockets[0].getsockname()[1] # get the random server port + + class BasicClient(asyncio.BaseProtocol): + def connection_made(self, transport): + _logger.debug("Client connected") + self.transport = transport + transport.write(garbage) + connect.set_result(True) + + def data_received(self, data): + _logger.debug("Client received data") + receive.set_result(True) + received_data = data + + def eof_received(self): + _logger.debug("Client stream eof") + eof.set_result(True) + + transport, protocol = await self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) + await asyncio.wait_for(connect, timeout=0.1) + await asyncio.wait_for(eof, timeout=0.1) + # neither of these should timeout if the test is successful + server.server_close() + + async def testTcpServerNoSlave(self): + ''' Test unknown slave unit exception ''' + context = ModbusServerContext(slaves={0x01: self.store, 0x02: self.store }, single=False) + data = b"\x01\x00\x00\x00\x00\x06\x05\x03\x00\x00\x00\x01" # get slave 5 function 3 (holding register) + server = await StartTcpServer(context=context,address=("127.0.0.1", 0),loop=self.loop) + server_task = asyncio.create_task(server.serve_forever()) + await server.serving + connect, receive, eof = self.loop.create_future(),self.loop.create_future(),self.loop.create_future() + received_data = None + random_port = server.server.sockets[0].getsockname()[1] # get the random server port + + class BasicClient(asyncio.BaseProtocol): + def connection_made(self, transport): + _logger.debug("Client connected") + self.transport = transport + transport.write(data) + connect.set_result(True) + + def data_received(self, data): + _logger.debug("Client received data") + receive.set_result(True) + received_data = data + + def eof_received(self): + _logger.debug("Client stream eof") + eof.set_result(True) + + transport, protocol = await self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) + await asyncio.wait_for(connect, timeout=0.1) + self.assertFalse(eof.done()) + server.server_close() + + + # async def testTcpServerGarbage(self): + # ''' Test sending garbage data on a TCP socket should drop the connection ''' + # garbage = b'\x01\x02\x03\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF' + # server = await StartTcpServer(address=("127.0.0.1", 0),loop=self.loop) + # server_task = asyncio.create_task(server.serve_forever()) + # await server.serving + # connect, receive, eof = self.loop.create_future(),self.loop.create_future(),self.loop.create_future() + # received_data = None + # random_port = server.server.sockets[0].getsockname()[1] # get the random server port + # + # class BasicClient(asyncio.BaseProtocol): + # def connection_made(self, transport): + # _logger.debug("Client connected") + # self.transport = transport + # transport.write(garbage) + # connect.set_result(True) + # + # def data_received(self, data): + # _logger.debug("Client received data") + # receive.set_result(True) + # received_data = data + # + # def eof_received(self): + # _logger.debug("Client stream eof") + # eof.set_result(True) + # + # transport, protocol = await self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) + # await asyncio.wait_for(connect, timeout=0.1) + # await asyncio.wait_for(eof, timeout=0.1) + # self.assertFalse(receive.done()) + # + # server.server_close() + + #-----------------------------------------------------------------------# + # Test ModbusUdpProtocol + #-----------------------------------------------------------------------# + + async def testStartUdpServer(self): + ''' Test that the modbus udp asyncio server starts correctly ''' + identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) + self.loop = asynctest.Mock(self.loop) + server = await StartUdpServer(context=self.context,loop=self.loop,identity=identity) + self.assertEqual(server.control.Identity.VendorName, 'VendorName') + self.loop.create_datagram_endpoint.assert_called_once() + + # async def testUdpServerServeNoDefer(self): + # ''' Test StartUdpServer without deferred start - NOT IMPLEMENTED - this test is hard to do without additional + # internal plumbing added to the implementation ''' + # asyncio.base_events.Server.serve_forever = asynctest.CoroutineMock() + # server = await StartUdpServer(address=("127.0.0.1", 0), loop=self.loop, defer_start=False) + # server.server.serve_forever.assert_awaited() + + async def testUdpServerServeForeverStart(self): + ''' Test StartUdpServer serve_forever() method ''' + with patch('asyncio.base_events.Server.serve_forever', new_callable=asynctest.CoroutineMock) as serve: + server = await StartTcpServer(context=self.context,address=("127.0.0.1", 0), loop=self.loop) + await server.serve_forever() + serve.assert_awaited() + + async def testUdpServerServeForeverClose(self): + ''' Test StartUdpServer serve_forever() method ''' + server = await StartUdpServer(context=self.context,address=("127.0.0.1", 0), loop=self.loop) + server_task = asyncio.create_task(server.serve_forever()) + await server.serving + + self.assertTrue(asyncio.isfuture(server.on_connection_terminated)) + self.assertFalse(server.on_connection_terminated.done()) + + server.server_close() + self.assertTrue(server.protocol.is_closing()) + + + async def testUdpServerServeForeverTwice(self): + ''' Call on serve_forever() twice should result in a runtime error ''' + identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) + server = await StartUdpServer(context=self.context,address=("127.0.0.1", 0), + loop=self.loop,identity=identity) + server_task = asyncio.create_task(server.serve_forever()) + await server.serving + with self.assertRaises(RuntimeError): + await server.serve_forever() + server.server_close() + + async def testUdpServerReceiveData(self): + ''' Test that the sending data on datagram socket gets data pushed to framer ''' + server = await StartUdpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) + server_task = asyncio.create_task(server.serve_forever()) + await server.serving + with patch('pymodbus.transaction.ModbusSocketFramer.processIncomingPacket',new_callable=Mock) as process: + + server.endpoint.datagram_received(data=b"12345", addr=("127.0.0.1", 12345)) + await asyncio.sleep(0.1) + process.seal() + + process.assert_called_once() + self.assertTrue( process.call_args[1]["data"] == b"12345" ) + + server.server_close() + + async def testUdpServerSendData(self): + ''' Test that the modbus udp asyncio server correctly sends data outbound ''' + identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) + data = b'x\01\x00\x00\x00\x00\x06\x01\x03\x00\x00\x00\x19' + server = await StartUdpServer(context=self.context,address=("127.0.0.1", 0)) + server_task = asyncio.create_task(server.serve_forever()) + await server.serving + random_port = server.protocol._sock.getsockname()[1] + received = server.endpoint.datagram_received = Mock(wraps=server.endpoint.datagram_received) + done = self.loop.create_future() + received_value = None + + class BasicClient(asyncio.DatagramProtocol): + def connection_made(self, transport): + self.transport = transport + self.transport.sendto(data) + + def datagram_received(self, data, addr): + nonlocal received_value, done + print("received") + received_value = data + done.set_result(True) + self.transport.close() + + transport, protocol = await self.loop.create_datagram_endpoint( BasicClient, + remote_addr=('127.0.0.1', random_port)) + + await asyncio.sleep(0.1) + + received.assert_called_once() + self.assertEqual(received.call_args[0][0], data) + + server.server_close() + + self.assertTrue(server.protocol.is_closing()) + await asyncio.sleep(0.1) + + async def testUdpServerRoundtrip(self): + ''' Test sending and receiving data on udp socket''' + data = b"\x01\x00\x00\x00\x00\x06\x01\x03\x00\x00\x00\x01" # unit 1, read register + expected_response = b'\x01\x00\x00\x00\x00\x05\x01\x03\x02\x00\x11' # value of 17 as per context + server = await StartUdpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) + server_task = asyncio.create_task(server.serve_forever()) + await server.serving + + random_port = server.protocol._sock.getsockname()[1] + + connected, done = self.loop.create_future(),self.loop.create_future() + received_value = None + + class BasicClient(asyncio.DatagramProtocol): + def connection_made(self, transport): + self.transport = transport + self.transport.sendto(data) + + def datagram_received(self, data, addr): + nonlocal received_value, done + print("received") + received_value = data + done.set_result(True) + + transport, protocol = await self.loop.create_datagram_endpoint( BasicClient, + remote_addr=('127.0.0.1', random_port)) + await asyncio.wait_for(done, timeout=0.1) + + self.assertEqual(received_value, expected_response) + + transport.close() + await asyncio.sleep(0) + server.server_close() + + + async def testUdpServerException(self): + ''' Test sending garbage data on a TCP socket should drop the connection ''' + garbage = b'\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF' + server = await StartUdpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) + server_task = asyncio.create_task(server.serve_forever()) + await server.serving + with patch('pymodbus.transaction.ModbusSocketFramer.processIncomingPacket', + new_callable=lambda: Mock(side_effect=Exception)) as process: + connect, receive, eof = self.loop.create_future(),self.loop.create_future(),self.loop.create_future() + received_data = None + random_port = server.protocol._sock.getsockname()[1] # get the random server port + + class BasicClient(asyncio.DatagramProtocol): + def connection_made(self, transport): + _logger.debug("Client connected") + self.transport = transport + transport.sendto(garbage) + connect.set_result(True) + + def datagram_received(self, data, addr): + nonlocal receive + _logger.debug("Client received data") + receive.set_result(True) + received_data = data + + transport, protocol = await self.loop.create_datagram_endpoint(BasicClient, + remote_addr=('127.0.0.1', random_port)) + await asyncio.wait_for(connect, timeout=0.1) + self.assertFalse(receive.done()) + self.assertFalse(server.protocol._sock._closed) + server.server_close() + + + # async def testUdpServerGarbage(self): + # ''' Test sending garbage data on a socket following by good data - this should reset + # the framer but not drop the connection ''' + # garbage = b'\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF' + # server = await StartUdpServer(address=("127.0.0.1", 0),loop=self.loop) + # server_task = asyncio.create_task(server.serve_forever()) + # await server.serving + # connect, receive, eof = self.loop.create_future(),self.loop.create_future(),self.loop.create_future() + # received_data = None + # random_port = server.protocol._sock.getsockname()[1] # get the random server port + # + # class BasicClient(asyncio.DatagramProtocol): + # def connection_made(self, transport): + # _logger.debug("Client connected") + # self.transport = transport + # transport.sendto(garbage) + # connect.set_result(True) + # + # def datagram_received(self, data, addr): + # nonlocal receive + # _logger.debug("Client received data") + # receive.set_result(True) + # received_data = data + # + # transport, protocol = await self.loop.create_datagram_endpoint(BasicClient, + # remote_addr=('127.0.0.1', random_port)) + # await connect + # self.assertFalse( receive.done() ) + # self.assertFalse(server.protocol._sock._closed) + # + # server.server_close() + + + + # -----------------------------------------------------------------------# + # Test ModbusServerFactory + # -----------------------------------------------------------------------# + def testModbusServerFactory(self): + ''' Test the base class for all the clients ''' + with self.assertWarns(DeprecationWarning): + factory = ModbusServerFactory(store=None) + + def testStopServer(self): + with self.assertWarns(DeprecationWarning): + StopServer() + + +# --------------------------------------------------------------------------- # +# Main +# --------------------------------------------------------------------------- # +if __name__ == "__main__": + asynctest.main() \ No newline at end of file From 5e314f9d646389693c76fce13f3230f414d0231f Mon Sep 17 00:00:00 2001 From: Memet Bilgin Date: Mon, 13 May 2019 20:37:46 +1000 Subject: [PATCH 21/26] induce exception in execute method by mock patching the request object's execute method --- test/test_server_asyncio.py | 139 +++++++++++++++++++----------------- 1 file changed, 72 insertions(+), 67 deletions(-) diff --git a/test/test_server_asyncio.py b/test/test_server_asyncio.py index 8b86af329..7e40afc6e 100755 --- a/test/test_server_asyncio.py +++ b/test/test_server_asyncio.py @@ -286,38 +286,78 @@ def eof_received(self): server.server_close() - # async def testTcpServerGarbage(self): - # ''' Test sending garbage data on a TCP socket should drop the connection ''' - # garbage = b'\x01\x02\x03\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF' - # server = await StartTcpServer(address=("127.0.0.1", 0),loop=self.loop) - # server_task = asyncio.create_task(server.serve_forever()) - # await server.serving - # connect, receive, eof = self.loop.create_future(),self.loop.create_future(),self.loop.create_future() - # received_data = None - # random_port = server.server.sockets[0].getsockname()[1] # get the random server port - # - # class BasicClient(asyncio.BaseProtocol): - # def connection_made(self, transport): - # _logger.debug("Client connected") - # self.transport = transport - # transport.write(garbage) - # connect.set_result(True) - # - # def data_received(self, data): - # _logger.debug("Client received data") - # receive.set_result(True) - # received_data = data - # - # def eof_received(self): - # _logger.debug("Client stream eof") - # eof.set_result(True) - # - # transport, protocol = await self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) - # await asyncio.wait_for(connect, timeout=0.1) - # await asyncio.wait_for(eof, timeout=0.1) - # self.assertFalse(receive.done()) - # - # server.server_close() + async def testTcpServerModbusError(self): + ''' Test sending garbage data on a TCP socket should drop the connection ''' + data = b"\x01\x00\x00\x00\x00\x06\x01\x03\x00\x00\x00\x01" # get slave 5 function 3 (holding register) + server = await StartTcpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) + server_task = asyncio.create_task(server.serve_forever()) + await server.serving + with patch("pymodbus.register_read_message.ReadHoldingRegistersRequest.execute", + side_effect=NoSuchSlaveException): + connect, receive, eof = self.loop.create_future(),self.loop.create_future(),self.loop.create_future() + received_data = None + random_port = server.server.sockets[0].getsockname()[1] # get the random server port + + class BasicClient(asyncio.BaseProtocol): + def connection_made(self, transport): + _logger.debug("Client connected") + self.transport = transport + transport.write(data) + connect.set_result(True) + + def data_received(self, data): + _logger.debug("Client received data") + receive.set_result(True) + received_data = data + + def eof_received(self): + _logger.debug("Client stream eof") + eof.set_result(True) + + transport, protocol = await self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) + await asyncio.wait_for(connect, timeout=0.1) + await asyncio.wait_for(receive, timeout=0.1) + self.assertFalse(eof.done()) + transport.close() + server.server_close() + + async def testTcpServerInternalException(self): + ''' Test sending garbage data on a TCP socket should drop the connection ''' + data = b"\x01\x00\x00\x00\x00\x06\x01\x03\x00\x00\x00\x01" # get slave 5 function 3 (holding register) + server = await StartTcpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) + server_task = asyncio.create_task(server.serve_forever()) + await server.serving + with patch("pymodbus.register_read_message.ReadHoldingRegistersRequest.execute", + side_effect=Exception): + connect, receive, eof = self.loop.create_future(),self.loop.create_future(),self.loop.create_future() + received_data = None + random_port = server.server.sockets[0].getsockname()[1] # get the random server port + + class BasicClient(asyncio.BaseProtocol): + def connection_made(self, transport): + _logger.debug("Client connected") + self.transport = transport + transport.write(data) + connect.set_result(True) + + def data_received(self, data): + _logger.debug("Client received data") + receive.set_result(True) + received_data = data + + def eof_received(self): + _logger.debug("Client stream eof") + eof.set_result(True) + + transport, protocol = await self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) + await asyncio.wait_for(connect, timeout=0.1) + await asyncio.wait_for(receive, timeout=0.1) + self.assertFalse(eof.done()) + + transport.close() + server.server_close() + + #-----------------------------------------------------------------------# # Test ModbusUdpProtocol @@ -489,41 +529,6 @@ def datagram_received(self, data, addr): self.assertFalse(server.protocol._sock._closed) server.server_close() - - # async def testUdpServerGarbage(self): - # ''' Test sending garbage data on a socket following by good data - this should reset - # the framer but not drop the connection ''' - # garbage = b'\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF' - # server = await StartUdpServer(address=("127.0.0.1", 0),loop=self.loop) - # server_task = asyncio.create_task(server.serve_forever()) - # await server.serving - # connect, receive, eof = self.loop.create_future(),self.loop.create_future(),self.loop.create_future() - # received_data = None - # random_port = server.protocol._sock.getsockname()[1] # get the random server port - # - # class BasicClient(asyncio.DatagramProtocol): - # def connection_made(self, transport): - # _logger.debug("Client connected") - # self.transport = transport - # transport.sendto(garbage) - # connect.set_result(True) - # - # def datagram_received(self, data, addr): - # nonlocal receive - # _logger.debug("Client received data") - # receive.set_result(True) - # received_data = data - # - # transport, protocol = await self.loop.create_datagram_endpoint(BasicClient, - # remote_addr=('127.0.0.1', random_port)) - # await connect - # self.assertFalse( receive.done() ) - # self.assertFalse(server.protocol._sock._closed) - # - # server.server_close() - - - # -----------------------------------------------------------------------# # Test ModbusServerFactory # -----------------------------------------------------------------------# From b89364f07f1dd71a315ade863e0c5f27322cc2a6 Mon Sep 17 00:00:00 2001 From: Memet Bilgin Date: Fri, 17 May 2019 09:53:17 +1000 Subject: [PATCH 22/26] move serial module dependency into class instantiation --- pymodbus/server/asyncio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymodbus/server/asyncio.py b/pymodbus/server/asyncio.py index d93189c46..4ea4d218a 100755 --- a/pymodbus/server/asyncio.py +++ b/pymodbus/server/asyncio.py @@ -4,7 +4,6 @@ """ from binascii import b2a_hex -import serial import socket import traceback @@ -603,6 +602,7 @@ def StartSerialServer(context=None, identity=None, custom_functions=[], missing slave """ raise NotImplementedException + import serial framer = kwargs.pop('framer', ModbusAsciiFramer) server = ModbusSerialServer(context, framer, identity, **kwargs) for f in custom_functions: From c5c2d2d7afc8625a691a1d98ef593e1103f5da6d Mon Sep 17 00:00:00 2001 From: Memet Bilgin Date: Sun, 19 May 2019 21:30:21 +1000 Subject: [PATCH 23/26] added asynctest depency to requirements-tests.txt --- requirements-tests.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-tests.txt b/requirements-tests.txt index 515eba29e..a5de1c06d 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -16,3 +16,4 @@ verboselogs >= 1.5 tornado==4.5.3 Twisted>=17.1.0 zope.interface>=4.4.0 +asynctest>=0.13.0 From 985d0cf1ba00504a4642bee30dfe7c16d885c12c Mon Sep 17 00:00:00 2001 From: Memet Bilgin Date: Sun, 19 May 2019 21:36:33 +1000 Subject: [PATCH 24/26] add unittest skip condition for unsupported targets, remove failing assertion from unsupported targets, use lower asynctest version --- requirements-tests.txt | 2 +- test/test_server_asyncio.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index a5de1c06d..2ca42aa2d 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -16,4 +16,4 @@ verboselogs >= 1.5 tornado==4.5.3 Twisted>=17.1.0 zope.interface>=4.4.0 -asynctest>=0.13.0 +asynctest>=0.10.0 diff --git a/test/test_server_asyncio.py b/test/test_server_asyncio.py index 7e40afc6e..396afe34c 100755 --- a/test/test_server_asyncio.py +++ b/test/test_server_asyncio.py @@ -1,13 +1,12 @@ #!/usr/bin/env python from pymodbus.compat import IS_PYTHON3 +import pytest import asynctest import asyncio import logging _logger = logging.getLogger() if IS_PYTHON3: # Python 3 from asynctest.mock import patch, Mock, MagicMock -else: - assert(False, "asyncio is not available pre-python 3.6") from pymodbus.device import ModbusDeviceIdentification from pymodbus.factory import ServerDecoder @@ -36,7 +35,7 @@ IS_HIGH_SIERRA_OR_ABOVE = False SERIAL_PORT = "/dev/ptmx" - +@pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") class AsyncioServerTest(asynctest.TestCase): ''' This is the unittest for the pymodbus.server.asyncio module From 9dd23bef4d2fe4d6a076054f589afbfc1f113c15 Mon Sep 17 00:00:00 2001 From: Memet Bilgin Date: Mon, 20 May 2019 23:25:59 +1000 Subject: [PATCH 25/26] remove logger setLevel call since doing so may override library consumers' already set log level --- pymodbus/server/asyncio.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pymodbus/server/asyncio.py b/pymodbus/server/asyncio.py index 4ea4d218a..0652c3372 100755 --- a/pymodbus/server/asyncio.py +++ b/pymodbus/server/asyncio.py @@ -24,7 +24,6 @@ # --------------------------------------------------------------------------- # import logging _logger = logging.getLogger(__name__) -_logger.setLevel(logging.DEBUG) # --------------------------------------------------------------------------- # From 6b7972317775460415ff1eb6e0c2c3253a50b93c Mon Sep 17 00:00:00 2001 From: Memet Bilgin Date: Tue, 21 May 2019 22:36:56 +1000 Subject: [PATCH 26/26] remove async def/await keywords from unittest so that the ast can be loaded in py2 even if the test is to be skipped --- test/test_server_asyncio.py | 205 +++++++++++++++++++----------------- 1 file changed, 111 insertions(+), 94 deletions(-) diff --git a/test/test_server_asyncio.py b/test/test_server_asyncio.py index 396afe34c..26927a2ab 100755 --- a/test/test_server_asyncio.py +++ b/test/test_server_asyncio.py @@ -66,42 +66,47 @@ def tearDown(self): #-----------------------------------------------------------------------# # Test ModbusConnectedRequestHandler #-----------------------------------------------------------------------# - async def testStartTcpServer(self): + @asyncio.coroutine + def testStartTcpServer(self): ''' Test that the modbus tcp asyncio server starts correctly ''' identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) self.loop = asynctest.Mock(self.loop) - server = await StartTcpServer(context=self.context,loop=self.loop,identity=identity) + server = yield from StartTcpServer(context=self.context,loop=self.loop,identity=identity) self.assertEqual(server.control.Identity.VendorName, 'VendorName') self.loop.create_server.assert_called_once() - async def testTcpServerServeNoDefer(self): + @asyncio.coroutine + def testTcpServerServeNoDefer(self): ''' Test StartTcpServer without deferred start (immediate execution of server) ''' with patch('asyncio.base_events.Server.serve_forever', new_callable=asynctest.CoroutineMock) as serve: - server = await StartTcpServer(context=self.context,address=("127.0.0.1", 0), loop=self.loop, defer_start=False) + server = yield from StartTcpServer(context=self.context,address=("127.0.0.1", 0), loop=self.loop, defer_start=False) serve.assert_awaited() - async def testTcpServerServeForever(self): + @asyncio.coroutine + def testTcpServerServeForever(self): ''' Test StartTcpServer serve_forever() method ''' with patch('asyncio.base_events.Server.serve_forever', new_callable=asynctest.CoroutineMock) as serve: - server = await StartTcpServer(context=self.context,address=("127.0.0.1", 0), loop=self.loop) - await server.serve_forever() + server = yield from StartTcpServer(context=self.context,address=("127.0.0.1", 0), loop=self.loop) + yield from server.serve_forever() serve.assert_awaited() - async def testTcpServerServeForeverTwice(self): + @asyncio.coroutine + def testTcpServerServeForeverTwice(self): ''' Call on serve_forever() twice should result in a runtime error ''' - server = await StartTcpServer(context=self.context,address=("127.0.0.1", 0), loop=self.loop) + server = yield from StartTcpServer(context=self.context,address=("127.0.0.1", 0), loop=self.loop) server_task = asyncio.create_task(server.serve_forever()) - await server.serving + yield from server.serving with self.assertRaises(RuntimeError): - await server.serve_forever() + yield from server.serve_forever() server.server_close() - async def testTcpServerReceiveData(self): + @asyncio.coroutine + def testTcpServerReceiveData(self): ''' Test data sent on socket is received by internals - doesn't not process data ''' data = b'\x01\x00\x00\x00\x00\x06\x01\x03\x00\x00\x00\x19' - server = await StartTcpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) + server = yield from StartTcpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) server_task = asyncio.create_task(server.serve_forever()) - await server.serving + yield from server.serving with patch('pymodbus.transaction.ModbusSocketFramer.processIncomingPacket', new_callable=Mock) as process: # process = server.framer.processIncomingPacket = Mock() connected = self.loop.create_future() @@ -116,8 +121,8 @@ def connection_made(self, transport): def eof_received(self): pass - transport, protocol = await self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) - await asyncio.sleep(0.1) # this may be better done by making an internal hook in the actual implementation + transport, protocol = yield from self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) + yield from asyncio.sleep(0.1) # this may be better done by making an internal hook in the actual implementation # if this unit test fails on a machine, see if increasing the sleep time makes a difference, if it does # blame author for a fix @@ -125,13 +130,14 @@ def eof_received(self): self.assertTrue( process.call_args[1]["data"] == data ) server.server_close() - async def testTcpServerRoundtrip(self): + @asyncio.coroutine + def testTcpServerRoundtrip(self): ''' Test sending and receiving data on tcp socket ''' data = b"\x01\x00\x00\x00\x00\x06\x01\x03\x00\x00\x00\x01" # unit 1, read register expected_response = b'\x01\x00\x00\x00\x00\x05\x01\x03\x02\x00\x11' # value of 17 as per context - server = await StartTcpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) + server = yield from StartTcpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) server_task = asyncio.create_task(server.serve_forever()) - await server.serving + yield from server.serving random_port = server.server.sockets[0].getsockname()[1] # get the random server port @@ -152,21 +158,22 @@ def data_received(self, data): def eof_received(self): pass - transport, protocol = await self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) - await asyncio.wait_for(done, timeout=0.1) + transport, protocol = yield from self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) + yield from asyncio.wait_for(done, timeout=0.1) self.assertEqual(received_value, expected_response) transport.close() - await asyncio.sleep(0) + yield from asyncio.sleep(0) server.server_close() - async def testTcpServerConnectionLost(self): + @asyncio.coroutine + def testTcpServerConnectionLost(self): ''' Test tcp stream interruption ''' data = b"\x01\x00\x00\x00\x00\x06\x01\x01\x00\x00\x00\x01" - server = await StartTcpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) + server = yield from StartTcpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) server_task = asyncio.create_task(server.serve_forever()) - await server.serving + yield from server.serving random_port = server.server.sockets[0].getsockname()[1] # get the random server port @@ -179,23 +186,24 @@ def connection_made(self, transport): self.transport = transport step1.set_result(True) - transport, protocol = await self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) - await step1 + transport, protocol = yield from self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) + yield from step1 self.assertTrue( len(server.active_connections) == 1 ) protocol.transport.close() # close isn't synchronous and there's no notification that it's done # so we have to wait a bit - await asyncio.sleep(0.1) + yield from asyncio.sleep(0.1) self.assertTrue( len(server.active_connections) == 0 ) server.server_close() - async def testTcpServerCloseActiveConnection(self): + @asyncio.coroutine + def testTcpServerCloseActiveConnection(self): ''' Test server_close() while there are active TCP connections ''' data = b"\x01\x00\x00\x00\x00\x06\x01\x01\x00\x00\x00\x01" - server = await StartTcpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) + server = yield from StartTcpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) server_task = asyncio.create_task(server.serve_forever()) - await server.serving + yield from server.serving random_port = server.server.sockets[0].getsockname()[1] # get the random server port @@ -208,22 +216,23 @@ def connection_made(self, transport): self.transport = transport step1.set_result(True) - transport, protocol = await self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) - await step1 + transport, protocol = yield from self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) + yield from step1 server.server_close() # close isn't synchronous and there's no notification that it's done # so we have to wait a bit - await asyncio.sleep(0.0) + yield from asyncio.sleep(0.0) self.assertTrue( len(server.active_connections) == 0 ) - async def testTcpServerException(self): + @asyncio.coroutine + def testTcpServerException(self): ''' Sending garbage data on a TCP socket should drop the connection ''' garbage = b'\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF' - server = await StartTcpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) + server = yield from StartTcpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) server_task = asyncio.create_task(server.serve_forever()) - await server.serving + yield from server.serving with patch('pymodbus.transaction.ModbusSocketFramer.processIncomingPacket', new_callable=lambda : Mock(side_effect=Exception)) as process: connect, receive, eof = self.loop.create_future(),self.loop.create_future(),self.loop.create_future() @@ -246,19 +255,20 @@ def eof_received(self): _logger.debug("Client stream eof") eof.set_result(True) - transport, protocol = await self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) - await asyncio.wait_for(connect, timeout=0.1) - await asyncio.wait_for(eof, timeout=0.1) + transport, protocol = yield from self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) + yield from asyncio.wait_for(connect, timeout=0.1) + yield from asyncio.wait_for(eof, timeout=0.1) # neither of these should timeout if the test is successful server.server_close() - async def testTcpServerNoSlave(self): + @asyncio.coroutine + def testTcpServerNoSlave(self): ''' Test unknown slave unit exception ''' context = ModbusServerContext(slaves={0x01: self.store, 0x02: self.store }, single=False) data = b"\x01\x00\x00\x00\x00\x06\x05\x03\x00\x00\x00\x01" # get slave 5 function 3 (holding register) - server = await StartTcpServer(context=context,address=("127.0.0.1", 0),loop=self.loop) + server = yield from StartTcpServer(context=context,address=("127.0.0.1", 0),loop=self.loop) server_task = asyncio.create_task(server.serve_forever()) - await server.serving + yield from server.serving connect, receive, eof = self.loop.create_future(),self.loop.create_future(),self.loop.create_future() received_data = None random_port = server.server.sockets[0].getsockname()[1] # get the random server port @@ -279,18 +289,18 @@ def eof_received(self): _logger.debug("Client stream eof") eof.set_result(True) - transport, protocol = await self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) - await asyncio.wait_for(connect, timeout=0.1) + transport, protocol = yield from self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) + yield from asyncio.wait_for(connect, timeout=0.1) self.assertFalse(eof.done()) server.server_close() - - async def testTcpServerModbusError(self): + @asyncio.coroutine + def testTcpServerModbusError(self): ''' Test sending garbage data on a TCP socket should drop the connection ''' data = b"\x01\x00\x00\x00\x00\x06\x01\x03\x00\x00\x00\x01" # get slave 5 function 3 (holding register) - server = await StartTcpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) + server = yield from StartTcpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) server_task = asyncio.create_task(server.serve_forever()) - await server.serving + yield from server.serving with patch("pymodbus.register_read_message.ReadHoldingRegistersRequest.execute", side_effect=NoSuchSlaveException): connect, receive, eof = self.loop.create_future(),self.loop.create_future(),self.loop.create_future() @@ -313,19 +323,20 @@ def eof_received(self): _logger.debug("Client stream eof") eof.set_result(True) - transport, protocol = await self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) - await asyncio.wait_for(connect, timeout=0.1) - await asyncio.wait_for(receive, timeout=0.1) + transport, protocol = yield from self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) + yield from asyncio.wait_for(connect, timeout=0.1) + yield from asyncio.wait_for(receive, timeout=0.1) self.assertFalse(eof.done()) transport.close() server.server_close() - async def testTcpServerInternalException(self): + @asyncio.coroutine + def testTcpServerInternalException(self): ''' Test sending garbage data on a TCP socket should drop the connection ''' data = b"\x01\x00\x00\x00\x00\x06\x01\x03\x00\x00\x00\x01" # get slave 5 function 3 (holding register) - server = await StartTcpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) + server = yield from StartTcpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) server_task = asyncio.create_task(server.serve_forever()) - await server.serving + yield from server.serving with patch("pymodbus.register_read_message.ReadHoldingRegistersRequest.execute", side_effect=Exception): connect, receive, eof = self.loop.create_future(),self.loop.create_future(),self.loop.create_future() @@ -348,9 +359,9 @@ def eof_received(self): _logger.debug("Client stream eof") eof.set_result(True) - transport, protocol = await self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) - await asyncio.wait_for(connect, timeout=0.1) - await asyncio.wait_for(receive, timeout=0.1) + transport, protocol = yield from self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) + yield from asyncio.wait_for(connect, timeout=0.1) + yield from asyncio.wait_for(receive, timeout=0.1) self.assertFalse(eof.done()) transport.close() @@ -362,11 +373,12 @@ def eof_received(self): # Test ModbusUdpProtocol #-----------------------------------------------------------------------# - async def testStartUdpServer(self): + @asyncio.coroutine + def testStartUdpServer(self): ''' Test that the modbus udp asyncio server starts correctly ''' identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) self.loop = asynctest.Mock(self.loop) - server = await StartUdpServer(context=self.context,loop=self.loop,identity=identity) + server = yield from StartUdpServer(context=self.context,loop=self.loop,identity=identity) self.assertEqual(server.control.Identity.VendorName, 'VendorName') self.loop.create_datagram_endpoint.assert_called_once() @@ -374,21 +386,23 @@ async def testStartUdpServer(self): # ''' Test StartUdpServer without deferred start - NOT IMPLEMENTED - this test is hard to do without additional # internal plumbing added to the implementation ''' # asyncio.base_events.Server.serve_forever = asynctest.CoroutineMock() - # server = await StartUdpServer(address=("127.0.0.1", 0), loop=self.loop, defer_start=False) + # server = yield from StartUdpServer(address=("127.0.0.1", 0), loop=self.loop, defer_start=False) # server.server.serve_forever.assert_awaited() - async def testUdpServerServeForeverStart(self): + @asyncio.coroutine + def testUdpServerServeForeverStart(self): ''' Test StartUdpServer serve_forever() method ''' with patch('asyncio.base_events.Server.serve_forever', new_callable=asynctest.CoroutineMock) as serve: - server = await StartTcpServer(context=self.context,address=("127.0.0.1", 0), loop=self.loop) - await server.serve_forever() + server = yield from StartTcpServer(context=self.context,address=("127.0.0.1", 0), loop=self.loop) + yield from server.serve_forever() serve.assert_awaited() - async def testUdpServerServeForeverClose(self): + @asyncio.coroutine + def testUdpServerServeForeverClose(self): ''' Test StartUdpServer serve_forever() method ''' - server = await StartUdpServer(context=self.context,address=("127.0.0.1", 0), loop=self.loop) + server = yield from StartUdpServer(context=self.context,address=("127.0.0.1", 0), loop=self.loop) server_task = asyncio.create_task(server.serve_forever()) - await server.serving + yield from server.serving self.assertTrue(asyncio.isfuture(server.on_connection_terminated)) self.assertFalse(server.on_connection_terminated.done()) @@ -396,27 +410,28 @@ async def testUdpServerServeForeverClose(self): server.server_close() self.assertTrue(server.protocol.is_closing()) - - async def testUdpServerServeForeverTwice(self): + @asyncio.coroutine + def testUdpServerServeForeverTwice(self): ''' Call on serve_forever() twice should result in a runtime error ''' identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) - server = await StartUdpServer(context=self.context,address=("127.0.0.1", 0), + server = yield from StartUdpServer(context=self.context,address=("127.0.0.1", 0), loop=self.loop,identity=identity) server_task = asyncio.create_task(server.serve_forever()) - await server.serving + yield from server.serving with self.assertRaises(RuntimeError): - await server.serve_forever() + yield from server.serve_forever() server.server_close() - async def testUdpServerReceiveData(self): + @asyncio.coroutine + def testUdpServerReceiveData(self): ''' Test that the sending data on datagram socket gets data pushed to framer ''' - server = await StartUdpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) + server = yield from StartUdpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) server_task = asyncio.create_task(server.serve_forever()) - await server.serving + yield from server.serving with patch('pymodbus.transaction.ModbusSocketFramer.processIncomingPacket',new_callable=Mock) as process: server.endpoint.datagram_received(data=b"12345", addr=("127.0.0.1", 12345)) - await asyncio.sleep(0.1) + yield from asyncio.sleep(0.1) process.seal() process.assert_called_once() @@ -424,13 +439,14 @@ async def testUdpServerReceiveData(self): server.server_close() - async def testUdpServerSendData(self): + @asyncio.coroutine + def testUdpServerSendData(self): ''' Test that the modbus udp asyncio server correctly sends data outbound ''' identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) data = b'x\01\x00\x00\x00\x00\x06\x01\x03\x00\x00\x00\x19' - server = await StartUdpServer(context=self.context,address=("127.0.0.1", 0)) + server = yield from StartUdpServer(context=self.context,address=("127.0.0.1", 0)) server_task = asyncio.create_task(server.serve_forever()) - await server.serving + yield from server.serving random_port = server.protocol._sock.getsockname()[1] received = server.endpoint.datagram_received = Mock(wraps=server.endpoint.datagram_received) done = self.loop.create_future() @@ -448,10 +464,10 @@ def datagram_received(self, data, addr): done.set_result(True) self.transport.close() - transport, protocol = await self.loop.create_datagram_endpoint( BasicClient, + transport, protocol = yield from self.loop.create_datagram_endpoint( BasicClient, remote_addr=('127.0.0.1', random_port)) - await asyncio.sleep(0.1) + yield from asyncio.sleep(0.1) received.assert_called_once() self.assertEqual(received.call_args[0][0], data) @@ -459,15 +475,16 @@ def datagram_received(self, data, addr): server.server_close() self.assertTrue(server.protocol.is_closing()) - await asyncio.sleep(0.1) + yield from asyncio.sleep(0.1) - async def testUdpServerRoundtrip(self): + @asyncio.coroutine + def testUdpServerRoundtrip(self): ''' Test sending and receiving data on udp socket''' data = b"\x01\x00\x00\x00\x00\x06\x01\x03\x00\x00\x00\x01" # unit 1, read register expected_response = b'\x01\x00\x00\x00\x00\x05\x01\x03\x02\x00\x11' # value of 17 as per context - server = await StartUdpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) + server = yield from StartUdpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) server_task = asyncio.create_task(server.serve_forever()) - await server.serving + yield from server.serving random_port = server.protocol._sock.getsockname()[1] @@ -485,23 +502,23 @@ def datagram_received(self, data, addr): received_value = data done.set_result(True) - transport, protocol = await self.loop.create_datagram_endpoint( BasicClient, + transport, protocol = yield from self.loop.create_datagram_endpoint( BasicClient, remote_addr=('127.0.0.1', random_port)) - await asyncio.wait_for(done, timeout=0.1) + yield from asyncio.wait_for(done, timeout=0.1) self.assertEqual(received_value, expected_response) transport.close() - await asyncio.sleep(0) + yield from asyncio.sleep(0) server.server_close() - - async def testUdpServerException(self): + @asyncio.coroutine + def testUdpServerException(self): ''' Test sending garbage data on a TCP socket should drop the connection ''' garbage = b'\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF' - server = await StartUdpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) + server = yield from StartUdpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) server_task = asyncio.create_task(server.serve_forever()) - await server.serving + yield from server.serving with patch('pymodbus.transaction.ModbusSocketFramer.processIncomingPacket', new_callable=lambda: Mock(side_effect=Exception)) as process: connect, receive, eof = self.loop.create_future(),self.loop.create_future(),self.loop.create_future() @@ -521,9 +538,9 @@ def datagram_received(self, data, addr): receive.set_result(True) received_data = data - transport, protocol = await self.loop.create_datagram_endpoint(BasicClient, + transport, protocol = yield from self.loop.create_datagram_endpoint(BasicClient, remote_addr=('127.0.0.1', random_port)) - await asyncio.wait_for(connect, timeout=0.1) + yield from asyncio.wait_for(connect, timeout=0.1) self.assertFalse(receive.done()) self.assertFalse(server.protocol._sock._closed) server.server_close()