diff --git a/.coveragerc b/.coveragerc index 472afaae6..01ac4162e 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,4 +2,5 @@ omit = pymodbus/repl/* pymodbus/internal/* - pymodbus/server/asyncio.py \ No newline at end of file + pymodbus/server/asyncio.py + pymodbus/server/reactive/* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..3b35144d9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,307 @@ +name: CI + +on: + push: + branches: + - dev + - master + tags: + - v* + pull_request: + branches: + - "*" + schedule: + # Daily at 05:14 + - cron: '14 5 * * *' + +jobs: + test: + # Should match JOB_NAME below + name: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }} + runs-on: ${{ matrix.os.runs-on }} + container: ${{ matrix.os.container[matrix.python.docker] }} + # present runtime seems to be about 1 minute 30 seconds + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + task: + - name: Test + tox: test + coverage: true + os: + - name: Linux + runs-on: ubuntu-latest + matrix: linux + container: + 2.7: docker://python:2.7-buster + 3.6: docker://python:3.6-buster + 3.7: docker://python:3.7-buster + 3.8: docker://python:3.8-buster + 3.9: docker://python:3.9-buster + pypy2: docker://pypy:2-jessie + pypy3: docker://pypy:3-stretch + - name: macOS + runs-on: macos-latest + matrix: macos + - name: Windows + runs-on: windows-latest + matrix: windows + openssl: + x86: win32 + x64: win64 + python: + - name: CPython 2.7 + tox: py27 + action: 2.7 + docker: 2.7 + matrix: 2.7 + implementation: cpython + - name: PyPy 2.7 + tox: pypy27 + action: pypy-2.7 + docker: pypy2.7 + matrix: 2.7 + implementation: pypy + openssl_msvc_version: 2019 + - name: CPython 3.6 + tox: py36 + action: 3.6 + docker: 3.6 + matrix: 3.6 + implementation: cpython + - name: CPython 3.7 + tox: py37 + action: 3.7 + docker: 3.7 + matrix: 3.7 + implementation: cpython + - name: CPython 3.8 + tox: py38 + action: 3.8 + docker: 3.8 + matrix: 3.8 + implementation: cpython + - name: CPython 3.9 + tox: py39 + action: 3.9 + docker: 3.9 + matrix: 3.9 + implementation: cpython + - name: PyPy 3.6 + tox: pypy36 + action: pypy-3.6 + docker: pypy3.6 + matrix: 3.6 + implementation: pypy + openssl_msvc_version: 2019 + - name: PyPy 3.7 + tox: pypy37 + action: pypy-3.7 + docker: pypy3.7 + matrix: 3.7 + implementation: pypy + openssl_msvc_version: 2019 + arch: + - name: x86 + action: x86 + matrix: x86 + - name: x64 + action: x64 + matrix: x64 + exclude: + - os: + matrix: linux + arch: + matrix: x86 + - os: + matrix: macos + arch: + matrix: x86 + - os: + matrix: windows + python: + implementation: pypy + arch: + matrix: x64 + env: + # Should match name above + JOB_NAME: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up ${{ matrix.python.name }} (if CPython) + if: ${{ job.container == '' && matrix.python.implementation == 'cpython'}} + uses: actions/setup-python@v2 + with: + python-version: '${{ matrix.python.action }}.0-alpha - ${{ matrix.python.action }}.X' + architecture: '${{ matrix.arch.action }}' + - name: Set up ${{ matrix.python.name }} (if PyPy) + if: ${{ job.container == '' && matrix.python.implementation == 'pypy'}} + uses: actions/setup-python@v2 + with: + python-version: '${{ matrix.python.action }}' + architecture: '${{ matrix.arch.action }}' + - name: Install + run: | + pip install --upgrade pip setuptools wheel + pip install --upgrade tox + - uses: twisted/python-info-action@v1.0.1 + - name: Add PyPy Externals + if: ${{ matrix.os.matrix == 'windows' && matrix.python.implementation == 'pypy'}} + env: + PYPY_EXTERNALS_PATH: ${{ github.workspace }}/pypy_externals + shell: bash + run: | + echo $PYPY_EXTERNALS_PATH + mkdir --parents $(dirname $PYPY_EXTERNALS_PATH) + hg clone https://foss.heptapod.net/pypy/externals/ $PYPY_EXTERNALS_PATH + dir $PYPY_EXTERNALS_PATH + cd $PYPY_EXTERNALS_PATH && hg update win32_14x + echo "INCLUDE=$PYPY_EXTERNALS_PATH/include;$INCLUDE" >> $GITHUB_ENV + echo "LIB=$PYPY_EXTERNALS_PATH/lib;$LIB" >> $GITHUB_ENV +# echo "CL=${{ matrix.PYTHON.CL_FLAGS }}" >> $GITHUB_ENV + - name: Add Brew + if: ${{ matrix.os.matrix == 'macos' && matrix.python.implementation == 'pypy'}} + shell: bash + run: | + brew install openssl@1.1 rust + echo "LDFLAGS=-L$(brew --prefix openssl@1.1)/lib" >> $GITHUB_ENV + echo "CFLAGS=-I$(brew --prefix openssl@1.1)/include" >> $GITHUB_ENV + - name: rustup + if: ${{ matrix.os.matrix == 'windows' && matrix.python.implementation == 'pypy'}} + shell: bash + run: | + rustup target add i686-pc-windows-msvc + - name: Test + env: + # When compiling Cryptography for PyPy on Windows there is a cleanup + # failure. This is CI, it doesn't matter. + PIP_NO_CLEAN: 1 + run: | + tox -vv -e ${{ matrix.python.tox }} + - name: Coverage Processing + if: matrix.task.coverage + run: | + mkdir coverage_reports + cp .coverage "coverage_reports/.coverage.${{ env.JOB_NAME }}" + cp coverage.xml "coverage_reports/coverage.${{ env.JOB_NAME }}.xml" + - name: Upload Coverage + if: matrix.task.coverage + uses: actions/upload-artifact@v2 + with: + name: coverage + path: coverage_reports/* + check: + # Should match JOB_NAME below + name: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }} + runs-on: ${{ matrix.os.runs-on }} + container: ${{ matrix.os.container[matrix.python.docker] }} + strategy: + fail-fast: false + matrix: + task: + - name: flake8 + tox: flake8 + continue_on_error: true + - name: Docs + tox: docs + os: + - name: Linux + runs-on: ubuntu-latest + matrix: linux + container: + 3.8: docker://python:3.8-buster + python: + - name: CPython 3.8 + tox: py38 + action: 3.8 + docker: 3.8 + implementation: cpython + arch: + - name: x64 + action: x64 + matrix: x64 + env: + # Should match name above + JOB_NAME: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Install + run: | + pip install --upgrade pip setuptools wheel + pip install --upgrade tox + - uses: twisted/python-info-action@v1.0.1 + - name: Test + continue-on-error: ${{ matrix.task.continue_on_error == true }} + run: | + tox -vv -e ${{ matrix.task.tox }} + coverage: + # Should match JOB_NAME below + name: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }} + runs-on: ${{ matrix.os.runs-on }} + if: always() + needs: + - test + container: ${{ matrix.os.container[matrix.python.docker] }} + strategy: + fail-fast: false + matrix: + task: + - name: Coverage + tox: combined-coverage + download_coverage: true + os: + - name: Linux + runs-on: ubuntu-latest + matrix: linux + container: + 3.8: docker://python:3.8-buster + python: + - name: CPython 3.8 + tox: py38 + action: 3.8 + docker: 3.8 + implementation: cpython + arch: + - name: x64 + action: x64 + matrix: x64 + env: + # Should match name above + JOB_NAME: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Install + run: | + pip install --upgrade pip setuptools wheel + pip install --upgrade tox + pip install --upgrade six + - uses: twisted/python-info-action@v1.0.1 + - name: Download Coverage + if: matrix.task.download_coverage + uses: actions/download-artifact@v2 + with: + name: coverage + path: coverage_reports + - name: Test + continue-on-error: ${{ matrix.task.continue_on_error == true }} + run: | + tox -vv -e ${{ matrix.task.tox }} + all: + name: All + runs-on: ubuntu-latest + needs: + - check + - coverage + - test + steps: + - name: This + shell: python + run: | + import this diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 000000000..f8d233527 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,30 @@ +name: Mark stale issues and pull requests + +on: + schedule: + - cron: "30 1 * * *" + +jobs: + stale: + + runs-on: ubuntu-latest + + steps: + - uses: actions/stale@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.' + stale-pr-message: 'This PR is stale because it has been open 45 days with no activity. Remove stale label or comment or this will be closed in 10 days.' + close-issue-message: 'This issue was closed because it has been stalled for 5 days with no activity.' + close-pr-message: 'This PR was closed because it has been stalled for 10 days with no activity.' + days-before-issue-stale: 30 + days-before-pr-stale: 45 + days-before-issue-close: 5 + days-before-pr-close: 10 + stale-issue-label: 'no-issue-activity' + exempt-issue-labels: 'Bug,Enhancements,Investigating,in progress,Documentation Update Required,3.x' + stale-pr-label: 'no-pr-activity' + exempt-pr-labels: 'IN REVIEW,Reviewing,Draft,in progress,3.x,2.5.0' + remove-stale-when-updated: true +# only-labels: "More Information Required, Not an Issue, question, Won't Do" + diff --git a/.gitignore b/.gitignore index 426321af3..c738736e8 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,5 @@ test/__pycache__/ /doc/_build/ .pytest_cache/ **/.pymodhis +/build/ +/dist/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ae4bc08e5..000000000 --- a/.travis.yml +++ /dev/null @@ -1,35 +0,0 @@ -sudo: false -language: python -matrix: - include: - - os: linux - python: "2.7" - - os: linux - python: "3.6" - - os: linux - python: "3.7" - - os: linux - python: "3.8" - - os: osx - osx_image: xcode8.3 - language: generic -before_install: - - if [ $TRAVIS_OS_NAME = osx ]; then brew update; fi - - if [ $TRAVIS_OS_NAME = osx ]; then brew install openssl; fi -# - if [$TRAVIS_OS_NAME = osx ]; then python -c "import fcntl; fcntl.fcntl(1, fcntl.F_SETFL, 0)"; fi - -install: -# - scripts/travis.sh pip install pip-accel - - if [ $TRAVIS_OS_NAME = osx ]; then scripts/travis.sh pip install -U "\"setuptools<45"\"; else pip install -U setuptools --upgrade ; fi - - scripts/travis.sh pip install coveralls - - scripts/travis.sh pip install --requirement=requirements-checks.txt - - scripts/travis.sh pip install --requirement=requirements-tests.txt - - scripts/travis.sh LC_ALL=C pip install . -script: -# - scripts/travis.sh make check - - scripts/travis.sh make test -after_success: - - scripts/travis.sh coveralls -branches: - except: - - /^[0-9]/ diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 37f0f5eee..9e93d58ba 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,52 @@ +version 2.5.1 +---------------------------------------------------------- +* Bug fix TCP Repl server. +* Support multiple UID's with REPL server. +* Support serial for URL (sync serial client) + +version 2.5.0 +---------------------------------------------------------- +* Support response types `stray` and `empty` in repl server. +* Minor updates in asyncio server. +* Update reactive server to send stray response of given length. +* Transaction manager updates on retries for empty and invalid packets. +* Test fixes for asyncio client and transaction manager. +* Fix sync client and processing of incomplete frames with rtu framers +* Support synchronous diagnostic client (TCP) +* Server updates (REPL and async) +* Handle Memory leak in sync servers due to socketserver memory leak + +version 2.5.0rc3 +---------------------------------------------------------- +* Minor fix in documentations +* Travis fix for Mac OSX +* Disable unnecessary deprecation warning while using async clients. +* Use Github actions for builds in favor of travis. + + +version 2.5.0rc2 +---------------------------------------------------------- +* Documentation updates +* Disable `strict` mode by default. +* Fix `ReportSlaveIdRequest` request +* Sparse datablock initialization updates. + +version 2.5.0rc1 +---------------------------------------------------------- +* Support REPL for modbus server (only python3 and asyncio) +* Fix REPL client for write requests +* Fix examples + * Asyncio server + * Asynchronous server (with custom datablock) + * Fix version info for servers +* Fix and enhancements to Tornado clients (seril and tcp) +* Fix and enhancements to Asyncio client and server +* Update Install instructions +* Synchronous client retry on empty and error enhancments +* Add new modbus state `RETRYING` +* Support runtime response manipulations for Servers +* Bug fixes with logging module in servers +* Asyncio modbus serial server support Version 2.4.0 ---------------------------------------------------------- diff --git a/Makefile b/Makefile index 92edfa795..954cb9470 100644 --- a/Makefile +++ b/Makefile @@ -42,11 +42,20 @@ check: install test: install @pip install --upgrade --quiet --requirement=requirements-tests.txt ifeq ($(PYVER),3.6) + $(info Running tests on $(PYVER)) + @pip install --upgrade pip --quiet @pytest --cov=pymodbus/ --cov-report term-missing test/test_server_asyncio.py test + @coverage report --fail-under=85 -i +else ifeq ($(PYVER),2.7) + $(info Running tests on $(PYVER)) + @pip install pip==20.3.4 --quiet + @pytest --cov-config=.coveragerc --cov=pymodbus/ --cov-report term-missing --ignore test/test_server_asyncio.py --ignore test/test_client_async_asyncio.py test @coverage report --fail-under=90 -i else - @pytest --cov=pymodbus/ --cov-report term-missing - @coverage report --fail-under=90 -i + $(info Running tests on $(PYVER)) + @pip install --upgrade pip --quiet + @pytest --cov=pymodbus/ --cov-report term-missing test + @coverage report --fail-under=85 -i endif tox: install diff --git a/README.rst b/README.rst index eb88622e7..cb1643d4a 100644 --- a/README.rst +++ b/README.rst @@ -7,7 +7,7 @@ PyModbus - A Python Modbus Stack .. image:: https://badges.gitter.im/Join%20Chat.svg :target: https://gitter.im/pymodbus_dev/Lobby .. image:: https://readthedocs.org/projects/pymodbus/badge/?version=latest - :target: http://pymodbus.readthedocs.io/en/async/?badge=latest + :target: http://pymodbus.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status .. image:: http://pepy.tech/badge/pymodbus :target: http://pepy.tech/project/pymodbus diff --git a/doc/INSTALL b/doc/INSTALL index 49a55b87f..0c04786a5 100644 --- a/doc/INSTALL +++ b/doc/INSTALL @@ -1,8 +1,8 @@ Requirements ------------- -* Python 2.3 or later. -* Python Twisted +* Python 2.7 or later. +* Python Twisted, Tornado or asyncio (For async client and server) * Pyserial On Windows pywin32 is recommended (this is built in to ActivePython, @@ -35,7 +35,7 @@ much easier to run with the nose package. With that installed, you can use either of the following:: python setup.py test - nosetests + pytest Building Documentation diff --git a/doc/conf.py b/doc/conf.py index 294b41b79..4f0dc4122 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -47,7 +47,7 @@ # ones. #extensions = ['sphinx.ext.autodoc', 'm2r', 'recommonmark'] -extensions = ['sphinx.ext.autodoc', 'm2r'] +extensions = ['sphinx.ext.autodoc', 'm2r2'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/doc/source/library/REPL.md b/doc/source/library/REPL.md index 48a426993..9f4cd1818 100644 --- a/doc/source/library/REPL.md +++ b/doc/source/library/REPL.md @@ -200,7 +200,7 @@ result.raw Return raw result dict. ``` -Every command has auto suggetion on the arguments supported , supply arg and value are to be supplied in `arg=val` format. +Every command has auto suggestion on the arguments supported, arg and value are to be supplied in `arg=val` format. ``` > client.read_holding_registers count=4 address=9 unit=1 diff --git a/examples/common/README.rst b/examples/common/README.rst index 777ef82ac..0f9726399 100644 --- a/examples/common/README.rst +++ b/examples/common/README.rst @@ -92,11 +92,13 @@ the tools/nullmodem/linux directory:: sudo ./run +The third method is Generic Unix method below. + ------------------------------------------------------------ Windows ------------------------------------------------------------ -For Windows, simply use the com2com application that is in +For Windows, simply use the com0com application that is in the directory tools/nullmodem/windows. Instructions are included in the Readme.txt. diff --git a/examples/common/asynchronous_server.py b/examples/common/asynchronous_server.py index 15e9b70c2..4f3895cbc 100755 --- a/examples/common/asynchronous_server.py +++ b/examples/common/asynchronous_server.py @@ -9,7 +9,8 @@ """ # --------------------------------------------------------------------------- # # import the various server implementations -# --------------------------------------------------------------------------- # +# --------------------------------------------------------------------------- # +from pymodbus.version import version from pymodbus.server.asynchronous import StartTcpServer from pymodbus.server.asynchronous import StartUdpServer from pymodbus.server.asynchronous import StartSerialServer @@ -105,10 +106,10 @@ def run_async_server(): identity = ModbusDeviceIdentification() identity.VendorName = 'Pymodbus' identity.ProductCode = 'PM' - identity.VendorUrl = 'http://github.com/bashwork/pymodbus/' + identity.VendorUrl = 'http://github.com/riptideio/pymodbus/' identity.ProductName = 'Pymodbus Server' identity.ModelName = 'Pymodbus Server' - identity.MajorMinorRevision = '2.3.0' + identity.MajorMinorRevision = version.short() # ----------------------------------------------------------------------- # # run the server you want diff --git a/examples/common/asyncio_server.py b/examples/common/asyncio_server.py index be34dad3d..ad2c7c5cf 100755 --- a/examples/common/asyncio_server.py +++ b/examples/common/asyncio_server.py @@ -12,6 +12,7 @@ # import the various server implementations # --------------------------------------------------------------------------- # import asyncio +from pymodbus.version import version from pymodbus.server.async_io import StartTcpServer from pymodbus.server.async_io import StartTlsServer from pymodbus.server.async_io import StartUdpServer @@ -107,22 +108,22 @@ async def run_server(): identity.VendorUrl = 'http://github.com/riptideio/pymodbus/' identity.ProductName = 'Pymodbus Server' identity.ModelName = 'Pymodbus Server' - identity.MajorMinorRevision = '2.3.0' + identity.MajorMinorRevision = version.short() # ----------------------------------------------------------------------- # # run the server you want # ----------------------------------------------------------------------- # # Tcp: # immediately start serving: - await StartTcpServer(context, identity=identity, address=("0.0.0.0", 5020), allow_reuse_address=True, - defer_start=False) + # await StartTcpServer(context, identity=identity, address=("0.0.0.0", 5020), allow_reuse_address=True, + # defer_start=False) # deferred start: - # server = await StartTcpServer(context, identity=identity, address=("0.0.0.0", 5020), - # allow_reuse_address=True, defer_start=True) - # - # asyncio.get_event_loop().call_later(20, lambda : server.serve_forever) - # await server.serve_forever() + server = await StartTcpServer(context, identity=identity, address=("0.0.0.0", 5020), + allow_reuse_address=True, defer_start=True) + + asyncio.get_event_loop().call_later(20, lambda: server.serve_forever) + await server.serve_forever() # TCP with different framer # StartTcpServer(context, identity=identity, diff --git a/examples/common/callback_server.py b/examples/common/callback_server.py index 325fbca56..60e65ba96 100755 --- a/examples/common/callback_server.py +++ b/examples/common/callback_server.py @@ -10,6 +10,7 @@ # --------------------------------------------------------------------------- # # import the modbus libraries we need # --------------------------------------------------------------------------- # +from pymodbus.version import version from pymodbus.server.asynchronous import StartTcpServer from pymodbus.device import ModbusDeviceIdentification from pymodbus.datastore import ModbusSparseDataBlock @@ -129,10 +130,10 @@ def run_callback_server(): identity = ModbusDeviceIdentification() identity.VendorName = 'pymodbus' identity.ProductCode = 'PM' - identity.VendorUrl = 'http://github.com/bashwork/pymodbus/' + identity.VendorUrl = 'http://github.com/riptideio/pymodbus/' identity.ProductName = 'pymodbus Server' identity.ModelName = 'pymodbus Server' - identity.MajorMinorRevision = '2.3.0' + identity.MajorMinorRevision = version.short() # ----------------------------------------------------------------------- # # run the server you want diff --git a/examples/common/custom_datablock.py b/examples/common/custom_datablock.py index 350a76abe..f59a4e2fb 100755 --- a/examples/common/custom_datablock.py +++ b/examples/common/custom_datablock.py @@ -10,6 +10,7 @@ # import the modbus libraries we need # --------------------------------------------------------------------------- # from __future__ import print_function +from pymodbus.version import version from pymodbus.server.asynchronous import StartTcpServer from pymodbus.device import ModbusDeviceIdentification from pymodbus.datastore import ModbusSparseDataBlock @@ -65,10 +66,10 @@ def run_custom_db_server(): identity = ModbusDeviceIdentification() identity.VendorName = 'pymodbus' identity.ProductCode = 'PM' - identity.VendorUrl = 'http://github.com/bashwork/pymodbus/' + identity.VendorUrl = 'http://github.com/riptideio/pymodbus/' identity.ProductName = 'pymodbus Server' identity.ModelName = 'pymodbus Server' - identity.MajorMinorRevision = '2.3.0' + identity.MajorMinorRevision = version.short() # ----------------------------------------------------------------------- # # run the server you want diff --git a/examples/common/custom_synchronous_server.py b/examples/common/custom_synchronous_server.py index 66f6f1b3c..78a271392 100755 --- a/examples/common/custom_synchronous_server.py +++ b/examples/common/custom_synchronous_server.py @@ -60,8 +60,8 @@ def decode(self, data): # --------------------------------------------------------------------------- # # import the various server implementations # --------------------------------------------------------------------------- # +from pymodbus.version import version from pymodbus.server.sync import StartTcpServer - from pymodbus.device import ModbusDeviceIdentification from pymodbus.datastore import ModbusSequentialDataBlock from pymodbus.datastore import ModbusSlaveContext, ModbusServerContext @@ -101,7 +101,7 @@ def run_server(): identity.VendorUrl = 'http://github.com/riptideio/pymodbus/' identity.ProductName = 'Pymodbus Server' identity.ModelName = 'Pymodbus Server' - identity.MajorMinorRevision = '2.3.0' + identity.MajorMinorRevision = version.short() # ----------------------------------------------------------------------- # # run the server you want diff --git a/examples/common/dbstore_update_server.py b/examples/common/dbstore_update_server.py index ef467de0a..525375b8a 100644 --- a/examples/common/dbstore_update_server.py +++ b/examples/common/dbstore_update_server.py @@ -16,6 +16,7 @@ # --------------------------------------------------------------------------- # # import the modbus libraries we need # --------------------------------------------------------------------------- # +from pymodbus.version import version from pymodbus.server.asynchronous import StartTcpServer from pymodbus.device import ModbusDeviceIdentification from pymodbus.datastore import ModbusSequentialDataBlock @@ -83,10 +84,10 @@ def run_dbstore_update_server(): identity = ModbusDeviceIdentification() identity.VendorName = 'pymodbus' identity.ProductCode = 'PM' - identity.VendorUrl = 'http://github.com/bashwork/pymodbus/' + identity.VendorUrl = 'http://github.com/riptideio/pymodbus/' identity.ProductName = 'pymodbus Server' identity.ModelName = 'pymodbus Server' - identity.MajorMinorRevision = '2.3.0' + identity.MajorMinorRevision = version.short() # ----------------------------------------------------------------------- # # run the server you want diff --git a/examples/common/modbus_payload.py b/examples/common/modbus_payload.py index ea31e78fe..a9204f3b4 100755 --- a/examples/common/modbus_payload.py +++ b/examples/common/modbus_payload.py @@ -3,7 +3,7 @@ Pymodbus Payload Building/Decoding Example -------------------------------------------------------------------------- -# Run modbus-payload-server.py or synchronous-server.py to check the behavior +# Run modbus_payload_server.py or synchronous_server.py to check the behavior """ from pymodbus.constants import Endian from pymodbus.payload import BinaryPayloadDecoder diff --git a/examples/common/modbus_payload_server.py b/examples/common/modbus_payload_server.py index 2fac2209a..6d8c5b25d 100755 --- a/examples/common/modbus_payload_server.py +++ b/examples/common/modbus_payload_server.py @@ -9,8 +9,8 @@ # --------------------------------------------------------------------------- # # import the various server implementations # --------------------------------------------------------------------------- # +from pymodbus.version import version from pymodbus.server.sync import StartTcpServer - from pymodbus.device import ModbusDeviceIdentification from pymodbus.datastore import ModbusSequentialDataBlock from pymodbus.datastore import ModbusSlaveContext, ModbusServerContext @@ -77,10 +77,10 @@ def run_payload_server(): identity = ModbusDeviceIdentification() identity.VendorName = 'Pymodbus' identity.ProductCode = 'PM' - identity.VendorUrl = 'http://github.com/bashwork/pymodbus/' + identity.VendorUrl = 'http://github.com/riptideio/pymodbus/' identity.ProductName = 'Pymodbus Server' identity.ModelName = 'Pymodbus Server' - identity.MajorMinorRevision = '2.3.0' + identity.MajorMinorRevision = version.short() # ----------------------------------------------------------------------- # # run the server you want # ----------------------------------------------------------------------- # diff --git a/examples/common/synchronous_server.py b/examples/common/synchronous_server.py index d2bfaf2a6..4266fac23 100755 --- a/examples/common/synchronous_server.py +++ b/examples/common/synchronous_server.py @@ -11,6 +11,7 @@ # --------------------------------------------------------------------------- # # import the various server implementations # --------------------------------------------------------------------------- # +from pymodbus.version import version from pymodbus.server.sync import StartTcpServer from pymodbus.server.sync import StartTlsServer from pymodbus.server.sync import StartUdpServer @@ -106,7 +107,7 @@ def run_server(): identity.VendorUrl = 'http://github.com/riptideio/pymodbus/' identity.ProductName = 'Pymodbus Server' identity.ModelName = 'Pymodbus Server' - identity.MajorMinorRevision = '2.3.0' + identity.MajorMinorRevision = version.short() # ----------------------------------------------------------------------- # # run the server you want diff --git a/examples/common/updating_server.py b/examples/common/updating_server.py index b5b04faa3..1894e159b 100755 --- a/examples/common/updating_server.py +++ b/examples/common/updating_server.py @@ -15,6 +15,7 @@ # --------------------------------------------------------------------------- # # import the modbus libraries we need # --------------------------------------------------------------------------- # +from pymodbus.version import version from pymodbus.server.asynchronous import StartTcpServer from pymodbus.device import ModbusDeviceIdentification from pymodbus.datastore import ModbusSequentialDataBlock @@ -75,10 +76,10 @@ def run_updating_server(): identity = ModbusDeviceIdentification() identity.VendorName = 'pymodbus' identity.ProductCode = 'PM' - identity.VendorUrl = 'http://github.com/bashwork/pymodbus/' + identity.VendorUrl = 'http://github.com/riptideio/pymodbus/' identity.ProductName = 'pymodbus Server' identity.ModelName = 'pymodbus Server' - identity.MajorMinorRevision = '2.3.0' + identity.MajorMinorRevision = version.short() # ----------------------------------------------------------------------- # # run the server you want diff --git a/examples/contrib/deviceinfo_showcase_server.py b/examples/contrib/deviceinfo_showcase_server.py index 983bb7111..28a9d0431 100755 --- a/examples/contrib/deviceinfo_showcase_server.py +++ b/examples/contrib/deviceinfo_showcase_server.py @@ -10,7 +10,8 @@ """ # --------------------------------------------------------------------------- # # import the various server implementations -# --------------------------------------------------------------------------- # +# --------------------------------------------------------------------------- # +from pymodbus.version import version from pymodbus.server.sync import StartTcpServer from pymodbus.server.sync import StartUdpServer from pymodbus.server.sync import StartSerialServer @@ -55,7 +56,7 @@ def run_server(): identity.VendorUrl = 'http://github.com/riptideio/pymodbus/' identity.ProductName = 'Pymodbus Server' identity.ModelName = 'Pymodbus Server' - identity.MajorMinorRevision = '2.3.0' + identity.MajorMinorRevision = version.short() # ----------------------------------------------------------------------- # # Add an example which is long enough to force the ReadDeviceInformation diff --git a/examples/contrib/message_parser.py b/examples/contrib/message_parser.py index 73d109931..f7539df32 100755 --- a/examples/contrib/message_parser.py +++ b/examples/contrib/message_parser.py @@ -135,7 +135,7 @@ def get_options(): parser.add_option("-a", "--ascii", help="The indicates that the message is ascii", - action="store_true", dest="ascii", default=True) + action="store_true", dest="ascii", default=False) parser.add_option("-b", "--binary", help="The indicates that the message is binary", @@ -148,6 +148,9 @@ def get_options(): parser.add_option("-t", "--transaction", help="If the incoming message is in hexadecimal format", action="store_true", dest="transaction", default=False) + parser.add_option("--framer", + help="Framer to use", dest="framer", default=None, + ) (opt, arg) = parser.parse_args() @@ -195,7 +198,7 @@ def main(): if option.debug: try: - modbus_log.setLevel(logging.DEBUG) + log.setLevel(logging.DEBUG) logging.basicConfig() except Exception as e: print("Logging is not supported on this system- {}".format(e)) @@ -205,7 +208,7 @@ def main(): 'rtu': ModbusRtuFramer, 'binary': ModbusBinaryFramer, 'ascii': ModbusAsciiFramer, - }.get(option.parser, ModbusSocketFramer) + }.get(option.framer or option.parser, ModbusSocketFramer) decoder = Decoder(framer, option.ascii) for message in get_messages(option): diff --git a/examples/gui/bottle/frontend.py b/examples/gui/bottle/frontend.py index 3e79e0b46..929c76252 100644 --- a/examples/gui/bottle/frontend.py +++ b/examples/gui/bottle/frontend.py @@ -6,6 +6,7 @@ This can be hosted using any wsgi adapter. """ from __future__ import print_function +from pymodbus.version import version import json, inspect from bottle import route, request, Bottle from bottle import static_file @@ -274,10 +275,10 @@ def RunDebugModbusFrontend(server, port=8080): identity = ModbusDeviceIdentification() identity.VendorName = 'Pymodbus' identity.ProductCode = 'PM' - identity.VendorUrl = 'http://github.com/bashwork/pymodbus/' + identity.VendorUrl = 'http://github.com/riptideio/pymodbus/' identity.ProductName = 'Pymodbus Server' identity.ModelName = 'Pymodbus Server' - identity.MajorMinorRevision = '2.3.0' + identity.MajorMinorRevision = version.short() # ------------------------------------------------------------ # initialize the datastore diff --git a/pymodbus/bit_read_message.py b/pymodbus/bit_read_message.py index d8624fa16..01434920d 100644 --- a/pymodbus/bit_read_message.py +++ b/pymodbus/bit_read_message.py @@ -118,7 +118,7 @@ def __str__(self): :returns: A string representation of the instance ''' - return "ReadBitResponse(%d)" % len(self.bits) + return "%s(%d)" % (self.__class__.__name__, len(self.bits)) class ReadCoilsRequest(ReadBitsRequestBase): diff --git a/pymodbus/client/asynchronous/__init__.py b/pymodbus/client/asynchronous/__init__.py index c339353d5..b7d084de3 100644 --- a/pymodbus/client/asynchronous/__init__.py +++ b/pymodbus/client/asynchronous/__init__.py @@ -37,8 +37,7 @@ if installed: # Import deprecated async client only if twisted is installed #338 from pymodbus.client.asynchronous.deprecated.asynchronous import * -else: import logging logger = logging.getLogger(__name__) - logger.warning("Not Importing deprecated clients. " - "Dependency Twisted is not Installed") + logger.warning("Importing deprecated clients. " + "Dependency Twisted is Installed") diff --git a/pymodbus/client/asynchronous/tornado/__init__.py b/pymodbus/client/asynchronous/tornado/__init__.py index 29e0e4db2..8eb61f4ef 100644 --- a/pymodbus/client/asynchronous/tornado/__init__.py +++ b/pymodbus/client/asynchronous/tornado/__init__.py @@ -315,7 +315,7 @@ def __init__(self, *args, **kwargs): self.silent_interval = 3.5 * self._t0 self.silent_interval = round(self.silent_interval, 6) self.last_frame_end = 0.0 - super().__init__(*args, **kwargs) + super(AsyncModbusSerialClient, self).__init__(*args, **kwargs) def get_socket(self): """ @@ -459,7 +459,8 @@ def sleep(timeout): LOGGER.info( "Cleanup recv buffer before send: " + hexlify_packets(result)) except OSError as e: - self.transaction.getTransaction(request.transaction_id).set_exception(ModbusIOException(e)) + self.transaction.getTransaction( + message.transaction_id).set_exception(ModbusIOException(e)) return start = time.time() diff --git a/pymodbus/client/sync.py b/pymodbus/client/sync.py index 8b0b832b6..3d1fb6d6f 100644 --- a/pymodbus/client/sync.py +++ b/pymodbus/client/sync.py @@ -72,8 +72,9 @@ def is_socket_open(self): ) def send(self, request): - _logger.debug("New Transaction state 'SENDING'") - self.state = ModbusTransactionState.SENDING + if self.state != ModbusTransactionState.RETRYING: + _logger.debug("New Transaction state 'SENDING'") + self.state = ModbusTransactionState.SENDING return self._send(request) def _send(self, request): @@ -154,8 +155,8 @@ def _dump(self, data, direction): try: fd.write(hexlify_packets(data)) except Exception as e: - self._logger.debug(hexlify_packets(data)) - self._logger.exception(e) + _logger.debug(hexlify_packets(data)) + _logger.exception(e) def register(self, function): """ @@ -204,12 +205,15 @@ def connect(self): :returns: True if connection succeeded, False otherwise """ - if self.socket: return True + if self.socket: + return True try: self.socket = socket.create_connection( (self.host, self.port), timeout=self.timeout, source_address=self.source_address) + _logger.debug("Connection to Modbus server established. " + "Socket {}".format(self.socket.getsockname())) except socket.error as msg: _logger.error('Connection to (%s, %s) ' 'failed: %s' % (self.host, self.port, msg)) @@ -223,6 +227,16 @@ def close(self): self.socket.close() self.socket = None + def _check_read_buffer(self, recv_size=None): + time_ = time.time() + end = time_ + self.timeout + data = None + data_length = 0 + ready = select.select([self.socket], [], [], end - time_) + if ready[0]: + data = self.socket.recv(1024) + return data + def _send(self, request): """ Sends data on the underlying socket @@ -231,6 +245,11 @@ def _send(self, request): """ if not self.socket: raise ConnectionException(self.__str__()) + if self.state == ModbusTransactionState.RETRYING: + data = self._check_read_buffer() + if data: + return data + if request: return self.socket.send(request) return 0 @@ -239,7 +258,12 @@ def _recv(self, size): """ Reads data from the underlying descriptor :param size: The number of bytes to read - :return: The bytes read + :return: The bytes read if the peer sent a response, or a zero-length + response if no data packets were received from the client at + all. + :raises: ConnectionException if the socket is not initialized, or the + peer either has closed the connection before this method is + invoked or closes it before sending any data before timeout. """ if not self.socket: raise ConnectionException(self.__str__()) @@ -256,9 +280,9 @@ def _recv(self, size): timeout = self.timeout - # If size isn't specified read 1 byte at a time. + # If size isn't specified read up to 4096 bytes at a time. if size is None: - recv_size = 1 + recv_size = 4096 else: recv_size = size @@ -270,6 +294,9 @@ def _recv(self, size): ready = select.select([self.socket], [], [], end - time_) if ready[0]: recv_data = self.socket.recv(recv_size) + if recv_data == b'': + return self._handle_abrupt_socket_close( + size, data, time.time() - time_) data.append(recv_data) data_length += len(recv_data) time_ = time.time() @@ -286,6 +313,35 @@ def _recv(self, size): return b"".join(data) + def _handle_abrupt_socket_close(self, size, data, duration): + """ Handle unexpected socket close by remote end + + Intended to be invoked after determining that the remote end + has unexpectedly closed the connection, to clean up and handle + the situation appropriately. + + :param size: The number of bytes that was attempted to read + :param data: The actual data returned + :param duration: Duration from the read was first attempted + until it was determined that the remote closed the + socket + :return: The more than zero bytes read from the remote end + :raises: ConnectionException If the remote end didn't send any + data at all before closing the connection. + """ + self.close() + readsize = ("read of %s bytes" % size if size + else "unbounded read") + msg = ("%s: Connection unexpectedly closed " + "%.6f seconds into %s" % (self, duration, readsize)) + if data: + result = b"".join(data) + msg += " after returning %s bytes" % len(result) + _logger.warning(msg) + return result + msg += " without response from unit before it closed connection" + raise ConnectionException(msg) + def is_socket_open(self): return True if self.socket is not None else False @@ -494,7 +550,9 @@ def _recv(self, size): return self.socket.recvfrom(size)[0] def is_socket_open(self): - return True if self.socket is not None else False + if self.socket: + return True + return self.connect() def __str__(self): """ Builds a string representation of the connection @@ -552,7 +610,7 @@ def __init__(self, method='ascii', **kwargs): self.parity = kwargs.get('parity', Defaults.Parity) self.baudrate = kwargs.get('baudrate', Defaults.Baudrate) self.timeout = kwargs.get('timeout', Defaults.Timeout) - self._strict = kwargs.get("strict", True) + self._strict = kwargs.get("strict", False) self.last_frame_end = None self.handle_local_echo = kwargs.get("handle_local_echo", False) if self.method == "rtu": @@ -591,7 +649,7 @@ def connect(self): if self.socket: return True try: - self.socket = serial.Serial(port=self.port, + self.socket = serial.serial_for_url(self.port, timeout=self.timeout, bytesize=self.bytesize, stopbits=self.stopbits, @@ -641,12 +699,19 @@ def _send(self, request): waitingbytes = self._in_waiting() if waitingbytes: result = self.socket.read(waitingbytes) + if self.state == ModbusTransactionState.RETRYING: + _logger.debug("Sending available data in recv " + "buffer {}".format( + hexlify_packets(result))) + return result if _logger.isEnabledFor(logging.WARNING): _logger.warning("Cleanup recv buffer before " "send: " + hexlify_packets(result)) except NotImplementedError: pass - + if self.state != ModbusTransactionState.SENDING: + _logger.debug("New Transaction state 'SENDING'") + self.state = ModbusTransactionState.SENDING size = self.socket.write(request) return size return 0 diff --git a/pymodbus/client/sync_diag.py b/pymodbus/client/sync_diag.py new file mode 100644 index 000000000..e2e41291f --- /dev/null +++ b/pymodbus/client/sync_diag.py @@ -0,0 +1,167 @@ +import socket +import logging +import time + +from pymodbus.constants import Defaults +from pymodbus.client.sync import ModbusTcpClient +from pymodbus.transaction import ModbusSocketFramer +from pymodbus.exceptions import ConnectionException + +_logger = logging.getLogger(__name__) + +LOG_MSGS = { + 'conn_msg': 'Connecting to modbus device %s', + 'connfail_msg': 'Connection to (%s, %s) failed: %s', + 'discon_msg': 'Disconnecting from modbus device %s', + 'timelimit_read_msg': + 'Modbus device read took %.4f seconds, ' + 'returned %s bytes in timelimit read', + 'timeout_msg': + 'Modbus device timeout after %.4f seconds, ' + 'returned %s bytes %s', + 'delay_msg': + 'Modbus device read took %.4f seconds, ' + 'returned %s bytes of %s expected', + 'read_msg': + 'Modbus device read took %.4f seconds, ' + 'returned %s bytes of %s expected', + 'unexpected_dc_msg': '%s %s'} + + +class ModbusTcpDiagClient(ModbusTcpClient): + """ + Variant of pymodbus.client.sync.ModbusTcpClient with additional + logging to diagnose network issues. + + The following events are logged: + + +---------+-----------------------------------------------------------------+ + | Level | Events | + +=========+=================================================================+ + | ERROR | Failure to connect to modbus unit; unexpected disconnect by | + | | modbus unit | + +---------+-----------------------------------------------------------------+ + | WARNING | Timeout on normal read; read took longer than warn_delay_limit | + +---------+-----------------------------------------------------------------+ + | INFO | Connection attempt to modbus unit; disconnection from modbus | + | | unit; each time limited read | + +---------+-----------------------------------------------------------------+ + | DEBUG | Normal read with timing information | + +---------+-----------------------------------------------------------------+ + + Reads are differentiated between "normal", which reads a specified number of + bytes, and "time limited", which reads all data for a duration equal to the + timeout period configured for this instance. + """ + + # pylint: disable=no-member + + def __init__(self, host='127.0.0.1', port=Defaults.Port, + framer=ModbusSocketFramer, **kwargs): + """ Initialize a client instance + + The keys of LOG_MSGS can be used in kwargs to customize the messages. + + :param host: The host to connect to (default 127.0.0.1) + :param port: The modbus port to connect to (default 502) + :param source_address: The source address tuple to bind to (default ('', 0)) + :param timeout: The timeout to use for this socket (default Defaults.Timeout) + :param warn_delay_limit: Log reads that take longer than this as warning. + Default True sets it to half of "timeout". None never logs these as + warning, 0 logs everything as warning. + :param framer: The modbus framer to use (default ModbusSocketFramer) + + .. note:: The host argument will accept ipv4 and ipv6 hosts + """ + self.warn_delay_limit = kwargs.get('warn_delay_limit', True) + super(ModbusTcpDiagClient, self).__init__(host, port, framer, **kwargs) + if self.warn_delay_limit is True: + self.warn_delay_limit = self.timeout / 2 + + # Set logging messages, defaulting to LOG_MSGS + for (k, v) in LOG_MSGS.items(): + self.__dict__[k] = kwargs.get(k, v) + + def connect(self): + """ Connect to the modbus tcp server + + :returns: True if connection succeeded, False otherwise + """ + if self.socket: + return True + try: + _logger.info(self.conn_msg, self) + self.socket = socket.create_connection( + (self.host, self.port), + timeout=self.timeout, + source_address=self.source_address) + except socket.error as msg: + _logger.error(self.connfail_msg, self.host, self.port, msg) + self.close() + return self.socket is not None + + def close(self): + """ Closes the underlying socket connection + """ + if self.socket: + _logger.info(self.discon_msg, self) + self.socket.close() + self.socket = None + + def _recv(self, size): + try: + start = time.time() + + result = super(ModbusTcpDiagClient, self)._recv(size) + + delay = time.time() - start + if self.warn_delay_limit is not None and delay >= self.warn_delay_limit: + self._log_delayed_response(len(result), size, delay) + elif not size: + _logger.debug(self.timelimit_read_msg, delay, len(result)) + else: + _logger.debug(self.read_msg, delay, len(result), size) + + return result + except ConnectionException as ex: + # Only log actual network errors, "if not self.socket" then it's a internal code issue + if 'Connection unexpectedly closed' in ex.string: + _logger.error(self.unexpected_dc_msg, self, ex) + raise ex + + def _log_delayed_response(self, result_len, size, delay): + if not size and result_len > 0: + _logger.info(self.timelimit_read_msg, delay, result_len) + elif (result_len == 0 or (size and result_len < size)) and delay >= self.timeout: + read_type = ("of %i expected" % size) if size else "in timelimit read" + _logger.warning(self.timeout_msg, delay, result_len, read_type) + else: + _logger.warning(self.delay_msg, delay, result_len, size) + + def __str__(self): + """ Builds a string representation of the connection + + :returns: The string representation + """ + return "ModbusTcpDiagClient(%s:%s)" % (self.host, self.port) + + +def get_client(): + """ Returns an appropriate client based on logging level + + This will be ModbusTcpDiagClient by default, or the parent class + if the log level is such that the diagnostic client will not log + anything. + + :returns: ModbusTcpClient or a child class thereof + """ + return ModbusTcpDiagClient if _logger.isEnabledFor(logging.ERROR) else ModbusTcpClient + + +# --------------------------------------------------------------------------- # +# Exported symbols +# --------------------------------------------------------------------------- # + +__all__ = [ + "ModbusTcpDiagClient", "get_client" +] diff --git a/pymodbus/compat.py b/pymodbus/compat.py index 48920ba74..33e37dbbe 100644 --- a/pymodbus/compat.py +++ b/pymodbus/compat.py @@ -51,7 +51,9 @@ # module renames # ----------------------------------------------------------------------- # import socketserver - + # #609 monkey patch for socket server memory leaks + # Refer https://bugs.python.org/issue37193 + socketserver.ThreadingMixIn.daemon_threads = True # ----------------------------------------------------------------------- # # decorators # ----------------------------------------------------------------------- # diff --git a/pymodbus/datastore/store.py b/pymodbus/datastore/store.py index 2d99f1a11..13acadd56 100644 --- a/pymodbus/datastore/store.py +++ b/pymodbus/datastore/store.py @@ -190,33 +190,59 @@ def setValues(self, address, values): class ModbusSparseDataBlock(BaseModbusDataBlock): - ''' Creates a sparse modbus datastore ''' + """ + Creates a sparse modbus datastore - def __init__(self, values): - ''' Initializes the datastore + E.g Usage. + sparse = ModbusSparseDataBlock({10: [3, 5, 6, 8], 30: 1, 40: [0]*20}) + + This would create a datablock with 3 blocks starting at + offset 10 with length 4 , 30 with length 1 and 40 with length 20 + + sparse = ModbusSparseDataBlock([10]*100) + Creates a sparse datablock of length 100 starting at offset 0 and default value of 10 - Using the input values we create the default - datastore value and the starting address + sparse = ModbusSparseDataBlock() --> Create Empty datablock + sparse.setValues(0, [10]*10) --> Add block 1 at offset 0 with length 10 (default value 10) + sparse.setValues(30, [20]*5) --> Add block 2 at offset 30 with length 5 (default value 20) + + if mutable is set to True during initialization, the datablock can not be altered with + setValues (new datablocks can not be added) + """ + + def __init__(self, values=None, mutable=True): + """ + Initializes a sparse datastore. Will only answer to addresses + registered, either initially here, or later via setValues() :param values: Either a list or a dictionary of values - ''' - if isinstance(values, dict): - self.values = values - elif hasattr(values, '__iter__'): - self.values = dict(enumerate(values)) - else: raise ParameterException( - "Values for datastore must be a list or dictionary") - self.default_value = get_next(itervalues(self.values)).__class__() - self.address = get_next(iterkeys(self.values)) + :param mutable: The data-block can be altered later with setValues(i.e add more blocks) + + If values are list , This is as good as sequential datablock. + Values as dictionary should be in {offset: } format, if values + is a list, a sparse datablock is created starting at offset with the length of values. + If values is a integer, then the value is set for the corresponding offset. + + """ + self.values = {} + self._process_values(values) + self.mutable = mutable + self.default_value = self.values.copy() + self.address = get_next(iterkeys(self.values), None) @classmethod - def create(klass): - ''' Factory method to create a datastore with the - full address space initialized to 0x00 + def create(klass, values=None): + ''' Factory method to create sparse datastore. + Use setValues to initialize registers. + :param values: Either a list or a dictionary of values :returns: An initialized datastore ''' - return klass([0x00] * 65536) + return klass(values) + + def reset(self): + ''' Reset the store to the intially provided defaults''' + self.values = self.default_value.copy() def validate(self, address, count=1): ''' Checks to see if the request is in range @@ -239,17 +265,49 @@ def getValues(self, address, count=1): ''' return [self.values[i] for i in range(address, address + count)] - def setValues(self, address, values): + def _process_values(self, values): + def _process_as_dict(values): + for idx, val in iteritems(values): + if isinstance(val, (list, tuple)): + for i, v in enumerate(val): + self.values[idx + i] = v + else: + self.values[idx] = int(val) + if isinstance(values, dict): + _process_as_dict(values) + return + if hasattr(values, '__iter__'): + values = dict(enumerate(values)) + elif values is None: + values = {} # Must make a new dict here per instance + else: + raise ParameterException("Values for datastore must " + "be a list or dictionary") + _process_as_dict(values) + + def setValues(self, address, values, use_as_default=False): ''' Sets the requested values of the datastore :param address: The starting address :param values: The new values to be set + :param use_as_default: Use the values as default ''' if isinstance(values, dict): - for idx, val in iteritems(values): - self.values[idx] = val + new_offsets = list(set(list(values.keys())) - set(list(self.values.keys()))) + if new_offsets and not self.mutable: + raise ParameterException("Offsets {} not " + "in range".format(new_offsets)) + self._process_values(values) else: if not isinstance(values, list): values = [values] for idx, val in enumerate(values): + if address+idx not in self.values and not self.mutable: + raise ParameterException("Offset {} not " + "in range".format(address+idx)) self.values[address + idx] = val + if not self.address: + self.address = get_next(iterkeys(self.values), None) + if use_as_default: + for idx, val in iteritems(self.values): + self.default_value[idx] = val diff --git a/pymodbus/exceptions.py b/pymodbus/exceptions.py index 651666d8b..0a4b0f6dc 100644 --- a/pymodbus/exceptions.py +++ b/pymodbus/exceptions.py @@ -105,6 +105,7 @@ def __init__(self, string=""): message = '[Error registering message] %s' % string ModbusException.__init__(self, message) + class TimeOutException(ModbusException): """ Error resulting from modbus response timeout """ diff --git a/pymodbus/framer/rtu_framer.py b/pymodbus/framer/rtu_framer.py index c5fe5a616..6c246bae1 100644 --- a/pymodbus/framer/rtu_framer.py +++ b/pymodbus/framer/rtu_framer.py @@ -60,7 +60,7 @@ def __init__(self, decoder, client=None): :param decoder: The decoder factory implementation to use """ self._buffer = b'' - self._header = {'uid': 0x00, 'len': 0, 'crc': '0000'} + self._header = {'uid': 0x00, 'len': 0, 'crc': b'\x00\x00'} self._hsize = 0x01 self._end = b'\x0d\x0a' self._min_frame_size = 4 @@ -89,14 +89,9 @@ def checkFrame(self): self.populateHeader() frame_size = self._header['len'] data = self._buffer[:frame_size - 2] - crc = self._buffer[frame_size - 2:frame_size] + crc = self._header['crc'] crc_val = (byte2int(crc[0]) << 8) + byte2int(crc[1]) - if checkCRC(data, crc_val): - return True - else: - _logger.debug("CRC invalid, discarding header!!") - self.resetFrame() - return False + return checkCRC(data, crc_val) except (IndexError, KeyError, struct.error): return False @@ -107,13 +102,10 @@ def advanceFrame(self): it or determined that it contains an error. It also has to reset the current frame header handle """ - try: - self._buffer = self._buffer[self._header['len']:] - except KeyError: - # Error response, no header len found - self.resetFrame() + + self._buffer = self._buffer[self._header['len']:] _logger.debug("Frame advanced, resetting header!!") - self._header = {} + self._header = {'uid': 0x00, 'len': 0, 'crc': b'\x00\x00'} def resetFrame(self): """ @@ -127,7 +119,7 @@ def resetFrame(self): _logger.debug("Resetting frame - Current Frame in " "buffer - {}".format(hexlify_packets(self._buffer))) self._buffer = b'' - self._header = {} + self._header = {'uid': 0x00, 'len': 0, 'crc': b'\x00\x00'} def isFrameReady(self): """ @@ -137,31 +129,38 @@ def isFrameReady(self): :returns: True if ready, False otherwise """ - if len(self._buffer) > self._hsize: - if not self._header: - self.populateHeader() + if len(self._buffer) <= self._hsize: + return False - return self._header and len(self._buffer) >= self._header['len'] - else: + try: + # Frame is ready only if populateHeader() successfully populates crc field which finishes RTU frame + # Otherwise, if buffer is not yet long enough, populateHeader() raises IndexError + self.populateHeader() + except IndexError: return False + return True + def populateHeader(self, data=None): """ Try to set the headers `uid`, `len` and `crc`. This method examines `self._buffer` and writes meta - information into `self._header`. It calculates only the - values for headers that are not already in the dictionary. + information into `self._header`. Beware that this method will raise an IndexError if `self._buffer` is not yet long enough. """ - data = data if data else self._buffer + data = data if data is not None else self._buffer self._header['uid'] = byte2int(data[0]) func_code = byte2int(data[1]) pdu_class = self.decoder.lookupPduClass(func_code) size = pdu_class.calculateRtuFrameSize(data) self._header['len'] = size + + if len(data) < size: + # crc yet not available + raise IndexError self._header['crc'] = data[size - 2:size] def addToFrame(self, message): @@ -282,6 +281,11 @@ def sendPacket(self, message): # Recovering from last error ?? time.sleep(self.client.silent_interval) self.client.state = ModbusTransactionState.IDLE + elif self.client.state == ModbusTransactionState.RETRYING: + # Simple lets settle down!!! + # To check for higher baudrates + time.sleep(self.client.timeout) + break else: if time.time() > timeout: _logger.debug("Spent more time than the read time out, " diff --git a/pymodbus/interfaces.py b/pymodbus/interfaces.py index d32e9978a..49e6939cd 100644 --- a/pymodbus/interfaces.py +++ b/pymodbus/interfaces.py @@ -180,7 +180,7 @@ def decode(self, fx): """ Converts the function code to the datastore to :param fx: The function we are working with - :returns: one of [d(iscretes),i(inputs),h(oliding),c(oils) + :returns: one of [d(iscretes),i(nputs),h(olding),c(oils) """ return self.__fx_mapper[fx] diff --git a/pymodbus/other_message.py b/pymodbus/other_message.py index c2774fe8d..31e6734bb 100644 --- a/pymodbus/other_message.py +++ b/pymodbus/other_message.py @@ -366,7 +366,7 @@ def execute(self, context=None): ''' reportSlaveIdData = None if context: - reportSlaveIdData = context.reportSlaveIdData + reportSlaveIdData = getattr(context, 'reportSlaveIdData', None) if not reportSlaveIdData: information = DeviceInformationFactory.get(_MCB) identifier = "-".join(information.values()).encode() diff --git a/pymodbus/register_read_message.py b/pymodbus/register_read_message.py index 0a202bb10..1c406ead3 100644 --- a/pymodbus/register_read_message.py +++ b/pymodbus/register_read_message.py @@ -102,7 +102,7 @@ def __str__(self): :returns: A string representation of the instance ''' - return "ReadRegisterResponse (%d)" % len(self.registers) + return "%s (%d)" % (self.__class__.__name__, len(self.registers)) class ReadHoldingRegistersRequest(ReadRegistersRequestBase): diff --git a/pymodbus/repl/client/__init__.py b/pymodbus/repl/client/__init__.py new file mode 100644 index 000000000..bc4e39484 --- /dev/null +++ b/pymodbus/repl/client/__init__.py @@ -0,0 +1,4 @@ +""" +Copyright (c) 2020 by RiptideIO +All rights reserved. +""" diff --git a/pymodbus/repl/completer.py b/pymodbus/repl/client/completer.py similarity index 97% rename from pymodbus/repl/completer.py rename to pymodbus/repl/client/completer.py index 391c245a1..426c5b29e 100644 --- a/pymodbus/repl/completer.py +++ b/pymodbus/repl/client/completer.py @@ -9,7 +9,7 @@ from prompt_toolkit.styles import Style from prompt_toolkit.filters import Condition from prompt_toolkit.application.current import get_app -from pymodbus.repl.helper import get_commands +from pymodbus.repl.client.helper import get_commands from pymodbus.compat import string_types @@ -33,7 +33,7 @@ class CmdCompleter(Completer): Completer for Pymodbus REPL. """ - def __init__(self, client, commands=None, ignore_case=True): + def __init__(self, client=None, commands=None, ignore_case=True): """ :param client: Modbus Client diff --git a/pymodbus/repl/helper.py b/pymodbus/repl/client/helper.py similarity index 99% rename from pymodbus/repl/helper.py rename to pymodbus/repl/client/helper.py index 38a29e9df..eb7ede644 100644 --- a/pymodbus/repl/helper.py +++ b/pymodbus/repl/client/helper.py @@ -160,7 +160,7 @@ def get_meta(self, cmd): def __str__(self): if self.doc: - return "Command {0:>50}{:<20}".format(self.name, self.doc) + return "Command {:>50}{:<20}".format(self.name, self.doc) return "Command {}".format(self.name) diff --git a/pymodbus/repl/main.py b/pymodbus/repl/client/main.py similarity index 90% rename from pymodbus/repl/main.py rename to pymodbus/repl/client/main.py index 65e6efa77..bdfdea9e7 100644 --- a/pymodbus/repl/main.py +++ b/pymodbus/repl/client/main.py @@ -26,8 +26,8 @@ from prompt_toolkit.history import FileHistory from prompt_toolkit.auto_suggest import AutoSuggestFromHistory from pymodbus.version import version -from pymodbus.repl.completer import CmdCompleter, has_selected_completion -from pymodbus.repl.helper import Result, CLIENT_ATTRIBUTES +from pymodbus.repl.client.completer import CmdCompleter, has_selected_completion +from pymodbus.repl.client.helper import Result, CLIENT_ATTRIBUTES click.disable_unicode_literals_warning = True @@ -41,9 +41,9 @@ \/ \/ \/ \/ \/ \/|__| v{} - {} ---------------------------------------------------------------------------- -""".format("1.2.0", version) -log = None +""".format("1.3.0", version) +log = None style = Style.from_dict({ 'completion-menu.completion': 'bg:#008888 #ffffff', @@ -169,7 +169,7 @@ def _process_args(args, string=True): complete_while_typing=True, bottom_toolbar=bottom_toolbar, key_bindings=kb, - history=FileHistory('.pymodhis'), + history=FileHistory('../.pymodhis'), auto_suggest=AutoSuggestFromHistory()) click.secho("{}".format(TITLE), fg='green') result = None @@ -226,9 +226,16 @@ def _process_args(args, string=True): @click.group('pymodbus-repl') @click.version_option(version, message=TITLE) @click.option("--verbose", is_flag=True, default=False, help="Verbose logs") -@click.option("--broadcast-support", is_flag=True, default=False, help="Support broadcast messages") +@click.option("--broadcast-support", is_flag=True, default=False, + help="Support broadcast messages") +@click.option("--retry-on-empty", is_flag=True, default=False, + help="Retry on empty response") +@click.option("--retry-on-error", is_flag=True, default=False, + help="Retry on error response") +@click.option("--retries", default=3, help="Retry count") @click.pass_context -def main(ctx, verbose, broadcast_support): +def main(ctx, verbose, broadcast_support, retry_on_empty, + retry_on_error, retries): if verbose: global log import logging @@ -237,13 +244,19 @@ def main(ctx, verbose, broadcast_support): log = logging.getLogger('pymodbus') logging.basicConfig(format=format) log.setLevel(logging.DEBUG) - ctx.obj = {"broadcast": broadcast_support} + ctx.obj = { + "broadcast": broadcast_support, + "retry_on_empty": retry_on_empty, + "retry_on_invalid": retry_on_error, + "retries": retries + } @main.command("tcp") @click.pass_context @click.option( "--host", + default='localhost', help="Modbus TCP IP " ) @click.option( @@ -259,9 +272,9 @@ def main(ctx, verbose, broadcast_support): help="Override the default packet framer tcp|rtu", ) def tcp(ctx, host, port, framer): - from pymodbus.repl.client import ModbusTcpClient - broadcast = ctx.obj.get("broadcast") - kwargs = dict(host=host, port=port, broadcast_enable=broadcast) + from pymodbus.repl.client.mclient import ModbusTcpClient + kwargs = dict(host=host, port=port) + kwargs.update(**ctx.obj) if framer == 'rtu': from pymodbus.framer.rtu_framer import ModbusRtuFramer kwargs['framer'] = ModbusRtuFramer @@ -349,7 +362,7 @@ def tcp(ctx, host, port, framer): ) def serial(ctx, method, port, baudrate, bytesize, parity, stopbits, xonxoff, rtscts, dsrdtr, timeout, write_timeout): - from pymodbus.repl.client import ModbusSerialClient + from pymodbus.repl.client.mclient import ModbusSerialClient client = ModbusSerialClient(method=method, port=port, baudrate=baudrate, @@ -360,7 +373,8 @@ def serial(ctx, method, port, baudrate, bytesize, parity, stopbits, xonxoff, rtscts=rtscts, dsrdtr=dsrdtr, timeout=timeout, - write_timeout=write_timeout) + write_timeout=write_timeout, + **ctx.obj) cli(client) diff --git a/pymodbus/repl/client.py b/pymodbus/repl/client/mclient.py similarity index 97% rename from pymodbus/repl/client.py rename to pymodbus/repl/client/mclient.py index c219387cb..6c53230e2 100644 --- a/pymodbus/repl/client.py +++ b/pymodbus/repl/client/mclient.py @@ -5,7 +5,7 @@ """ from __future__ import absolute_import, unicode_literals - +import functools from pymodbus.pdu import ModbusExceptions, ExceptionResponse from pymodbus.exceptions import ModbusIOException from pymodbus.client.sync import ModbusSerialClient as _ModbusSerialClient @@ -35,7 +35,23 @@ GetClearModbusPlusRequest) +def make_response_dict(resp): + rd = { + 'function_code': resp.function_code, + 'address': resp.address + } + if hasattr(resp, "value"): + rd['value'] = resp.value + elif hasattr(resp, 'values'): + rd['values'] = resp.values + elif hasattr(resp, 'count'): + rd['count'] = resp.count + + return rd + + def handle_brodcast(func): + @functools.wraps(func) def _wrapper(*args, **kwargs): self = args[0] resp = func(*args, **kwargs) @@ -44,11 +60,7 @@ def _wrapper(*args, **kwargs): 'broadcasted': True } if not resp.isError(): - return { - 'function_code': resp.function_code, - 'address': resp.address, - 'count': resp.count - } + return make_response_dict(resp) else: return ExtendedRequestSupport._process_exception(resp, **kwargs) return _wrapper @@ -143,7 +155,7 @@ def write_coils(self, address, values, **kwargs): Write `value` to coil at `address`. :param address: coil offset to write to - :param value: list of bit values to write (comma seperated) + :param values: list of bit values to write (comma seperated) :param unit: The slave unit this request is targeting :return: """ @@ -171,7 +183,7 @@ def write_registers(self, address, values, **kwargs): Write list of `values` to registers starting at `address`. :param address: register offset to write to - :param value: list of register value to write (comma seperated) + :param values: list of register value to write (comma seperated) :param unit: The slave unit this request is targeting :return: """ diff --git a/pymodbus/repl/server/__init__.py b/pymodbus/repl/server/__init__.py new file mode 100644 index 000000000..bc4e39484 --- /dev/null +++ b/pymodbus/repl/server/__init__.py @@ -0,0 +1,4 @@ +""" +Copyright (c) 2020 by RiptideIO +All rights reserved. +""" diff --git a/pymodbus/repl/server/cli.py b/pymodbus/repl/server/cli.py new file mode 100644 index 000000000..ebe442544 --- /dev/null +++ b/pymodbus/repl/server/cli.py @@ -0,0 +1,205 @@ +""" +Copyright (c) 2020 by RiptideIO +All rights reserved. +""" +import json +import click +import shutil +import logging + +from prompt_toolkit.shortcuts import clear +from prompt_toolkit.shortcuts.progress_bar import formatters +from prompt_toolkit.styles import Style + +from prompt_toolkit import PromptSession, print_formatted_text +from prompt_toolkit.patch_stdout import patch_stdout +from prompt_toolkit.completion import NestedCompleter +from prompt_toolkit.formatted_text import HTML + + +logger = logging.getLogger(__name__) + +TITLE = """ +__________ .______. _________ +\______ \___.__. _____ ____ __| _/\_ |__ __ __ ______ / _____/ ______________ __ ___________ + | ___< | |/ \ / _ \ / __ | | __ \| | \/ ___/ \_____ \_/ __ \_ __ \ \/ // __ \_ __ \\ + | | \___ | Y Y ( <_> ) /_/ | | \_\ \ | /\___ \ / \ ___/| | \/\ /\ ___/| | \/ + |____| / ____|__|_| /\____/\____ | |___ /____//____ > /_______ /\___ >__| \_/ \___ >__| + \/ \/ \/ \/ \/ \/ \/ \/""" + +SMALL_TITLE = "Pymodbus server..." +BOTTOM_TOOLBAR = HTML('(MODBUS SERVER) Type "help" ' + 'for list of available commands') +COMMAND_ARGS = ["response_type", "error_code", "delay_by", + "clear_after", "data_len"] +RESPONSE_TYPES = ["normal", "error", "delayed", "empty", "stray"] +COMMANDS = { + "manipulator": { + "response_type": None, + "error_code": None, + "delay_by": None, + "clear_after": None + }, + "exit": None, + "help": None, + "clear": None +} +USAGE = "manipulator response_type=|normal|error|delayed|empty|stray \n" \ + "\tAdditional parameters\n" \ + "\t\terror_code=<int> \n\t\tdelay_by=<in seconds> \n\t\t" \ + "clear_after=<clear after n messages int>" \ + "\n\t\tdata_len=<length of stray data (int)>\n" \ + "\n\tExample usage: \n\t" \ + "1. Send error response 3 for 4 requests\n\t" \ + " manipulator response_type=error error_code=3 clear_after=4\n\t" \ + "2. Delay outgoing response by 5 seconds indefinitely\n\t" \ + " manipulator response_type=delayed delay_by=5\n\t" \ + "3. Send empty response\n\t" \ + " manipulator response_type=empty\n\t" \ + "4. Send stray response of lenght 12 and revert to normal after 2 responses\n\t" \ + " manipulator response_type=stray data_len=11 clear_after=2\n\t" \ + "5. To disable response manipulation\n\t" \ + " manipulator response_type=normal" +COMMAND_HELPS = { + "manipulator": "Manipulate response from server.\nUsage: {}".format(USAGE), + "clear": "Clears screen" + +} + + +STYLE = Style.from_dict({"": "cyan"}) +CUSTOM_FORMATTERS = [ + formatters.Label(suffix=": "), + formatters.Bar(start="|", end="|", sym_a="#", sym_b="#", sym_c="-"), + formatters.Text(" "), + formatters.Text(" "), + formatters.TimeElapsed(), + formatters.Text(" "), + ] + + +def info(message): + click.secho(str(message), fg="green") + + +def warning(message): + click.secho(str(message), fg="yellow") + + +def error(message): + click.secho(str(message), fg="red") + + +def get_terminal_width(): + return shutil.get_terminal_size()[0] + + +def print_help(): + print_formatted_text(HTML("Available commands:")) + for cmd, hlp in sorted(COMMAND_HELPS.items()): + print_formatted_text( + HTML("{:45s}{:100s}".format(cmd, hlp)) + ) + + +async def interactive_shell(server): + """ + CLI interactive shell + """ + col = get_terminal_width() + max_len = max([len(t) for t in TITLE.split("\n")]) + if col > max_len: + info(TITLE) + else: + print_formatted_text(HTML(''.format(SMALL_TITLE))) + info("") + completer = NestedCompleter.from_nested_dict(COMMANDS) + session = PromptSession("SERVER > ", + completer=completer, + bottom_toolbar=BOTTOM_TOOLBAR) + + # Run echo loop. Read text from stdin, and reply it back. + while True: + try: + invalid_command = False + result = await session.prompt_async() + if result == "exit": + await server.web_app.shutdown() + break + if result == "help": + print_help() + continue + if result == "clear": + clear() + continue + command = result.split() + if command: + if command[0] not in COMMANDS: + invalid_command = True + if invalid_command: + warning("Invalid command or invalid usage of command - {}".format(command)) + continue + if len(command) == 1: + warning("Usage: '{}'".format(USAGE)) + else: + args = command[1:] + skip_next = False + val_dict = {} + for index, arg in enumerate(args): + if skip_next: + skip_next = False + continue + if "=" in arg: + arg, value = arg.split("=") + else: + if arg in COMMAND_ARGS: + try: + value = args[index+1] + skip_next = True + except IndexError: + error("Missing value " + "for argument - {}".format(arg)) + warning("Usage: '{}'".format(USAGE)) + break + valid = True + if arg == "response_type": + if value not in RESPONSE_TYPES: + warning("Invalid response " + "type request - {}".format(value)) + warning("Choose from {}".format(RESPONSE_TYPES)) + valid = False + elif arg in ["error_code", "delay_by", + "clear_after", "data_len"]: + try: + value = int(value) + except ValueError: + warning("Expected integer " + "value for {}".format(arg)) + valid = False + + if valid: + val_dict[arg] = value + if val_dict: + server.update_manipulator_config(val_dict) + # server.manipulator_config = val_dict + # result = await run_command(tester, *command) + + except (EOFError, KeyboardInterrupt): + return + + +async def main(server): + with patch_stdout(): + try: + await interactive_shell(server) + finally: + pass + warning("Bye Bye!!!") + + +async def run_repl(server): + await main(server) + + diff --git a/pymodbus/repl/server/main.py b/pymodbus/repl/server/main.py new file mode 100644 index 000000000..8d300f1f4 --- /dev/null +++ b/pymodbus/repl/server/main.py @@ -0,0 +1,117 @@ +""" +Copyright (c) 2020 by RiptideIO +All rights reserved. +""" +import asyncio +import json +import click +from pymodbus.compat import IS_PYTHON3, PYTHON_VERSION +from pymodbus.framer.socket_framer import ModbusSocketFramer +from pymodbus.server.reactive.main import ( + ReactiveServer, DEFAULT_FRAMER, DEFUALT_HANDLERS) +from pymodbus.server.reactive.default_config import DEFUALT_CONFIG +from pymodbus.repl.server.cli import run_repl + +if IS_PYTHON3 and PYTHON_VERSION > (3, 7): + CANCELLED_ERROR = asyncio.exceptions.CancelledError +else: + CANCELLED_ERROR = asyncio.CancelledError + + +@click.group("ReactiveModbusServer") +@click.option("--host", default="localhost", help="Host address") +@click.option("--web-port", default=8080, help="Web app port") +@click.option("--broadcast-support", is_flag=True, + default=False, help="Support broadcast messages") +@click.option("--repl/--no-repl", is_flag=True, + default=True, help="Enable/Disable repl for server") +@click.option("--verbose", is_flag=True, + help="Run with debug logs enabled for pymodbus") +@click.pass_context +def server(ctx, host, web_port, broadcast_support, repl, verbose): + global logger + import logging + FORMAT = ('%(asctime)-15s %(threadName)-15s' + ' %(levelname)-8s %(module)-15s:%(lineno)-8s %(message)s') + pymodbus_logger = logging.getLogger("pymodbus") + logging.basicConfig(format=FORMAT) + logger = logging.getLogger(__name__) + if verbose: + pymodbus_logger.setLevel(logging.DEBUG) + logger.setLevel(logging.DEBUG) + else: + pymodbus_logger.setLevel(logging.ERROR) + logger.setLevel(logging.ERROR) + + ctx.obj = {"repl": repl, "host": host, "web_port": web_port, + "broadcast": broadcast_support} + + +@server.command("run") +@click.option("--modbus-server", default="tcp", + type=click.Choice(["tcp", "serial", "tls", "udp"], + case_sensitive=False), + help="Modbus server") +@click.option("--modbus-framer", default="socket", + type=click.Choice(["socket", "rtu", "tls", "ascii", "binary"], + case_sensitive=False), + help="Modbus framer to use") +@click.option("--modbus-port", default="5020", help="Modbus port") +@click.option("--modbus-unit-id", default=[1], type=int, + multiple=True, help="Modbus unit id") +@click.option("--modbus-config", type=click.Path(exists=True), + help="Path to additional modbus server config") +@click.option("-r", "--randomize", default=0, help="Randomize every `r` reads." + " 0=never, 1=always, " + "2=every-second-read, " + "and so on. " + "Applicable IR and DI.") +@click.pass_context +def run(ctx, modbus_server, modbus_framer, modbus_port, modbus_unit_id, + modbus_config, randomize): + """ + Run Reactive Modbus server exposing REST endpoint + for response manipulation. + """ + if not IS_PYTHON3: + click.secho("Pymodbus Server REPL not supported on python2", fg="read") + exit(1) + repl = ctx.obj.pop("repl") + web_app_config = ctx.obj + loop = asyncio.get_event_loop() + framer = DEFAULT_FRAMER.get(modbus_framer, ModbusSocketFramer) + if modbus_config: + with open(modbus_config) as f: + modbus_config = json.load(f) + else: + modbus_config = DEFUALT_CONFIG + modbus_config = modbus_config.get(modbus_server, {}) + if modbus_server != "serial": + modbus_port = int(modbus_port) + handler = modbus_config.pop("handler", "ModbusConnectedRequestHandler") + else: + handler = modbus_config.pop("handler", "ModbusSingleRequestHandler") + handler = DEFUALT_HANDLERS.get(handler.strip()) + + modbus_config["handler"] = handler + modbus_config["randomize"] = randomize + app = ReactiveServer.factory(modbus_server, framer, + modbus_port=modbus_port, + unit=modbus_unit_id, + loop=loop, + **web_app_config, **modbus_config) + try: + if repl: + loop.run_until_complete(app.run_async()) + + loop.run_until_complete(run_repl(app)) + loop.run_forever() + else: + app.run() + + except CANCELLED_ERROR: + print("Done!!!!!") + + +if __name__ == '__main__': + server() diff --git a/pymodbus/server/async_io.py b/pymodbus/server/async_io.py index 690332843..a85b37a78 100755 --- a/pymodbus/server/async_io.py +++ b/pymodbus/server/async_io.py @@ -4,7 +4,8 @@ """ from binascii import b2a_hex -import socket +import serial +from serial_asyncio import create_serial_connection import ssl import traceback @@ -48,6 +49,19 @@ def __init__(self, owner): self.receive_queue = asyncio.Queue() self.handler_task = None # coroutine to be run on asyncio loop + def _log_exception(self): + if isinstance(self, ModbusConnectedRequestHandler): + _logger.error( + "Handler for stream [%s:%s] has " + "been canceled" % self.client_address[:2]) + elif isinstance(self, ModbusSingleRequestHandler): + _logger.error( + "Handler for serial port has been cancelled") + else: + sock_name = self.protocol._sock.getsockname() + _logger.error("Handler for UDP socket [%s] has " + "been canceled" % sock_name[1]) + def connection_made(self, transport): """ asyncio.BaseProtocol callback for socket establish @@ -57,7 +71,17 @@ def connection_made(self, transport): corresponds to the socket being opened """ try: - _logger.debug("Socket [%s:%s] opened" % transport.get_extra_info('sockname')) + sockname = transport.get_extra_info('sockname') + if sockname is not None: + _logger.debug( + "Socket [%s:%s] opened" % transport.get_extra_info( + 'sockname')[:2]) + else: + if hasattr(transport, 'serial'): + _logger.debug( + "Serial connection opened on port: {}".format( + transport.serial.port) + ) self.transport = transport self.running = True self.framer = self.server.framer(self.server.decoder, client=None) @@ -68,7 +92,7 @@ def connection_made(self, transport): else: self.handler_task = asyncio.ensure_future(self.handle()) except Exception as ex: # pragma: no cover - _logger.debug("Datastore unable to fulfill request: " + _logger.error("Datastore unable to fulfill request: " "%s; %s", ex, traceback.format_exc()) def connection_lost(self, exc): @@ -81,20 +105,18 @@ def connection_lost(self, exc): """ try: self.handler_task.cancel() - if exc is None: - if hasattr(self, "client_address"): # TCP connection - _logger.debug("Disconnected from client [%s:%s]" % self.client_address) - else: - _logger.debug("Disconnected from client [%s]" % self.transport.get_extra_info("peername")) + self._log_exception() else: # pragma: no cover - _logger.debug("Client Disconnection [%s:%s] due to %s" % (*self.client_address, exc)) + if hasattr(self, "client_address"): # TCP connection + _logger.debug("Client Disconnection {} due " + "to {}".format(*self.client_address, exc)) self.running = False except Exception as ex: # pragma: no cover - _logger.debug("Datastore unable to fulfill request: " - "%s; %s", ex, traceback.format_exc()) + _logger.error("Datastore unable to fulfill request: " + "%s; %s", ex, traceback.format_exc()) async def handle(self): """Asyncio coroutine which represents a single conversation between @@ -104,8 +126,9 @@ async def handle(self): fed to this coroutine via the asyncio.Queue object which is fed by the ModbusBaseRequestHandler class's callback Future. - This callback future gets data from either asyncio.DatagramProtocol.datagram_received - or from asyncio.BaseProtocol.data_received. + This callback future gets data from either + asyncio.DatagramProtocol.datagram_received or + from asyncio.BaseProtocol.data_received. This function will execute without blocking in the while-loop and yield to the asyncio event loop when the frame is exhausted. @@ -125,11 +148,13 @@ async def handle(self): while self.running: try: units = self.server.context.slaves() - data = await self._recv_() # this is an asyncio.Queue await, it will never fail + # this is an asyncio.Queue await, it will never fail + data = await self._recv_() if isinstance(data, tuple): - data, *addr = data # addr is populated when talking over UDP + # addr is populated when talking over UDP + data, *addr = data else: - addr = (None,) # empty tuple + addr = (None,) # empty tuple if not isinstance(units, (list, tuple)): units = [units] @@ -143,23 +168,21 @@ async def handle(self): _logger.debug('Handling data: ' + hexlify_packets(data)) single = self.server.context.single - self.framer.processIncomingPacket(data=data, - callback=lambda x: self.execute(x, *addr), - unit=units, - single=single) + self.framer.processIncomingPacket( + data=data, callback=lambda x: self.execute(x, *addr), + unit=units, single=single) except asyncio.CancelledError: # catch and ignore cancelation errors - if isinstance(self, ModbusConnectedRequestHandler): - _logger.debug("Handler for stream [%s:%s] has been canceled" % self.client_address) - else: - _logger.debug("Handler for UDP socket [%s] has been canceled" % self.protocol._sock.getsockname()[1]) - + self._log_exception() except Exception as e: - # force TCP socket termination as processIncomingPacket should handle applicaiton layer errors + # force TCP socket termination as processIncomingPacket + # should handle applicaiton layer errors # for UDP sockets, simply reset the frame if isinstance(self, ModbusConnectedRequestHandler): - _logger.info("Unknown exception '%s' on stream [%s:%s] forcing disconnect" % (e, *self.client_address)) + client_addr = self.client_address[:2] + _logger.error("Unknown exception '{}' on stream {} " + "forcing disconnect".format(e, client_addr)) self.transport.close() else: _logger.error("Unknown error occurred %s" % e) @@ -178,39 +201,49 @@ def execute(self, request, *addr): try: if self.server.broadcast_enable and request.unit_id == 0: broadcast = True - # if broadcasting then execute on all slave contexts, note response will be ignored + # if broadcasting then execute on all slave contexts, + # note response will be ignored for unit_id in self.server.context.slaves(): response = request.execute(self.server.context[unit_id]) else: context = self.server.context[request.unit_id] response = request.execute(context) except NoSuchSlaveException as ex: - _logger.debug("requested slave does " - "not exist: %s" % request.unit_id ) + _logger.error("requested slave does " + "not exist: %s" % request.unit_id) if self.server.ignore_missing_slaves: return # the client will simply timeout waiting for a response response = request.doException(merror.GatewayNoResponse) except Exception as ex: - _logger.debug("Datastore unable to fulfill request: " + _logger.error("Datastore unable to fulfill request: " "%s; %s", ex, traceback.format_exc()) response = request.doException(merror.SlaveFailure) # no response when broadcasting if not broadcast: response.transaction_id = request.transaction_id response.unit_id = request.unit_id - self.send(response, *addr) + skip_encoding = False + if self.server.response_manipulator: + response, skip_encoding = self.server.response_manipulator(response) + self.send(response, *addr, skip_encoding=skip_encoding) - - def send(self, message, *addr): - if message.should_respond: - # self.server.control.Counter.BusMessage += 1 - pdu = self.framer.buildPacket(message) + def send(self, message, *addr, **kwargs): + def __send(msg, *addr): if _logger.isEnabledFor(logging.DEBUG): - _logger.debug('send: [%s]- %s' % (message, b2a_hex(pdu))) + _logger.debug('send: [%s]- %s' % (message, b2a_hex(msg))) if addr == (None,): - self._send_(pdu) + self._send_(msg) else: - self._send_(pdu, *addr) + self._send_(msg, *addr) + skip_encoding = kwargs.get("skip_encoding", False) + if skip_encoding: + __send(message, *addr) + elif message.should_respond: + # self.server.control.Counter.BusMessage += 1 + pdu = self.framer.buildPacket(message) + __send(pdu, *addr) + else: + _logger.debug("Skipping sending response!!") # ----------------------------------------------------------------------- # # Derived class implementations @@ -223,6 +256,7 @@ def _send_(self, data): # pragma: no cover """ raise NotImplementedException("Method not implemented " "by derived class") + async def _recv_(self): # pragma: no cover """ Receive data from the network @@ -245,17 +279,20 @@ def connection_made(self, transport): self.client_address = transport.get_extra_info('peername') self.server.active_connections[self.client_address] = self - _logger.debug("TCP client connection established [%s:%s]" % self.client_address) + _logger.debug("TCP client connection established " + "[%s:%s]" % self.client_address[:2]) def connection_lost(self, exc): - """ asyncio.BaseProtocol: Called when the connection is lost or closed.""" + """ + asyncio.BaseProtocol: Called when the connection is lost or closed. + """ super().connection_lost(exc) - _logger.debug("TCP client disconnected [%s:%s]" % self.client_address) + client_addr = self.client_address[:2] + _logger.debug("TCP client disconnected [%s:%s]" % client_addr) if self.client_address in self.server.active_connections: self.server.active_connections.pop(self.client_address) - - def data_received(self,data): + def data_received(self, data): """ asyncio.Protocol: (TCP) Called when some data is received. data is a non-empty bytes object containing the incoming data. @@ -270,7 +307,8 @@ def _send_(self, data): self.transport.write(data) -class ModbusDisconnectedRequestHandler(ModbusBaseRequestHandler, asyncio.DatagramProtocol): +class ModbusDisconnectedRequestHandler(ModbusBaseRequestHandler, + asyncio.DatagramProtocol): """ Implements the modbus server protocol This uses the socketserver.BaseRequestHandler to implement @@ -280,7 +318,8 @@ class ModbusDisconnectedRequestHandler(ModbusBaseRequestHandler, asyncio.Datagra """ def __init__(self,owner): super().__init__(owner) - self.server.on_connection_terminated = asyncio.get_event_loop().create_future() + _future = asyncio.get_event_loop().create_future() + self.server.on_connection_terminated = _future def connection_lost(self,exc): super().connection_lost(exc) @@ -314,6 +353,7 @@ async def _recv_(self): def _send_(self, data, addr): self.transport.sendto(data, addr=addr) + class ModbusServerFactory: """ Builder class for a modbus server @@ -323,13 +363,42 @@ class ModbusServerFactory: def __init__(self, store, framer=None, identity=None, **kwargs): import warnings - warnings.warn("deprecated API for asyncio. ServerFactory's are a twisted construct and don't have an equivalent in asyncio", + warnings.warn("deprecated API for asyncio. ServerFactory's are a " + "twisted construct and don't have an equivalent in " + "asyncio", DeprecationWarning) +class ModbusSingleRequestHandler(ModbusBaseRequestHandler, asyncio.Protocol): + """ Implements the modbus server protocol + This uses asyncio.Protocol to implement + the client handler for a serial connection. + """ + def connection_made(self, transport): + super().connection_made(transport) + + _logger.debug("Serial connection established") + + def connection_lost(self, exc): + super().connection_lost(exc) + _logger.debug("Serial conection lost") + if hasattr(self.server, 'on_connection_lost'): + self.server.on_connection_lost() + + def data_received(self, data): + self.receive_queue.put_nowait(data) + + async def _recv_(self): + return await self.receive_queue.get() + + def _send_(self, data): + if self.transport is not None: + self.transport.write(data) + # --------------------------------------------------------------------------- # # Server Implementations # --------------------------------------------------------------------------- # + class ModbusTcpServer: """ A modbus threaded tcp socket server @@ -376,6 +445,8 @@ def __init__(self, to a missing slave :param broadcast_enable: True to treat unit_id 0 as broadcast address, False to treat 0 as any other unit_id + :param response_manipulator: Callback method for manipulating the + response """ self.active_connections = {} self.loop = loop or asyncio.get_event_loop() @@ -391,26 +462,33 @@ def __init__(self, Defaults.IgnoreMissingSlaves) self.broadcast_enable = kwargs.get('broadcast_enable', Defaults.broadcast_enable) - + self.response_manipulator = kwargs.get("response_manipulator", None) if isinstance(identity, ModbusDeviceIdentification): self.control.Identity.update(identity) - self.serving = self.loop.create_future() # asyncio future that will be done once server has started - self.server = None # constructors cannot be declared async, so we have to defer the initialization of the server + # asyncio future that will be done once server has started + self.serving = self.loop.create_future() + # constructors cannot be declared async, so we have to + # defer the initialization of the server + self.server = None if PYTHON_VERSION >= (3, 7): # start_serving is new in version 3.7 - self.server_factory = self.loop.create_server(lambda : self.handler(self), - *self.address, - reuse_address=allow_reuse_address, - reuse_port=allow_reuse_port, - backlog=backlog, - start_serving=not defer_start) + self.server_factory = self.loop.create_server( + lambda: self.handler(self), + *self.address, + reuse_address=allow_reuse_address, + reuse_port=allow_reuse_port, + backlog=backlog, + start_serving=not defer_start + ) else: - self.server_factory = self.loop.create_server(lambda : self.handler(self), - *self.address, - reuse_address=allow_reuse_address, - reuse_port=allow_reuse_port, - backlog=backlog) + self.server_factory = self.loop.create_server( + lambda: self.handler(self), + *self.address, + reuse_address=allow_reuse_address, + reuse_port=allow_reuse_port, + backlog=backlog + ) async def serve_forever(self): if self.server is None: @@ -418,10 +496,11 @@ async def serve_forever(self): self.serving.set_result(True) await self.server.serve_forever() else: - raise RuntimeError("Can't call serve_forever on an already running server object") + raise RuntimeError("Can't call serve_forever on " + "an already running server object") def server_close(self): - for k,v in self.active_connections.items(): + for k, v in self.active_connections.items(): _logger.warning("aborting active session {}".format(k)) v.handler_task.cancel() self.active_connections = {} @@ -481,6 +560,8 @@ def __init__(self, to a missing slave :param broadcast_enable: True to treat unit_id 0 as broadcast address, False to treat 0 as any other unit_id + :param response_manipulator: Callback method for + manipulating the response """ self.active_connections = {} self.loop = loop or asyncio.get_event_loop() @@ -496,6 +577,7 @@ def __init__(self, Defaults.IgnoreMissingSlaves) self.broadcast_enable = kwargs.get('broadcast_enable', Defaults.broadcast_enable) + self.response_manipulator = kwargs.get("response_manipulator", None) if isinstance(identity, ModbusDeviceIdentification): self.control.Identity.update(identity) @@ -512,26 +594,31 @@ def __init__(self, self.sslctx.options |= ssl.OP_NO_SSLv2 self.sslctx.verify_mode = ssl.CERT_OPTIONAL self.sslctx.check_hostname = False - - self.serving = self.loop.create_future() # asyncio future that will be done once server has started - self.server = None # constructors cannot be declared async, so we have to defer the initialization of the server + # asyncio future that will be done once server has started + self.serving = self.loop.create_future() + # constructors cannot be declared async, so we have to + # defer the initialization of the server + self.server = None if PYTHON_VERSION >= (3, 7): # start_serving is new in version 3.7 - self.server_factory = self.loop.create_server(lambda : self.handler(self), - *self.address, - ssl=self.sslctx, - reuse_address=allow_reuse_address, - reuse_port=allow_reuse_port, - backlog=backlog, - start_serving=not defer_start) + self.server_factory = self.loop.create_server( + lambda: self.handler(self), + *self.address, + ssl=self.sslctx, + reuse_address=allow_reuse_address, + reuse_port=allow_reuse_port, + backlog=backlog, + start_serving=not defer_start + ) else: - self.server_factory = self.loop.create_server(lambda : self.handler(self), - *self.address, - ssl=self.sslctx, - reuse_address=allow_reuse_address, - reuse_port=allow_reuse_port, - backlog=backlog) - + self.server_factory = self.loop.create_server( + lambda: self.handler(self), + *self.address, + ssl=self.sslctx, + reuse_address=allow_reuse_address, + reuse_port=allow_reuse_port, + backlog=backlog + ) class ModbusUdpServer: @@ -565,6 +652,8 @@ def __init__(self, context, framer=None, identity=None, address=None, to a missing slave :param broadcast_enable: True to treat unit_id 0 as broadcast address, False to treat 0 as any other unit_id + :param response_manipulator: Callback method for + manipulating the response """ self.loop = loop or asyncio.get_event_loop() self.decoder = ServerDecoder() @@ -577,6 +666,7 @@ def __init__(self, context, framer=None, identity=None, address=None, Defaults.IgnoreMissingSlaves) self.broadcast_enable = kwargs.get('broadcast_enable', Defaults.broadcast_enable) + self.response_manipulator = kwargs.get("response_manipulator", None) if isinstance(identity, ModbusDeviceIdentification): self.control.Identity.update(identity) @@ -585,12 +675,15 @@ def __init__(self, context, framer=None, identity=None, address=None, self.endpoint = None self.on_connection_terminated = None self.stop_serving = self.loop.create_future() - self.serving = self.loop.create_future() # asyncio future that will be done once server has started - self.server_factory = self.loop.create_datagram_endpoint(lambda: self.handler(self), - local_addr=self.address, - reuse_address=allow_reuse_address, - reuse_port=allow_reuse_port, - allow_broadcast=True) + # asyncio future that will be done once server has started + self.serving = self.loop.create_future() + self.server_factory = self.loop.create_datagram_endpoint( + lambda: self.handler(self), + local_addr=self.address, + reuse_address=allow_reuse_address, + reuse_port=allow_reuse_port, + allow_broadcast=True + ) async def serve_forever(self): if self.protocol is None: @@ -598,7 +691,8 @@ async def serve_forever(self): self.serving.set_result(True) await self.stop_serving else: - raise RuntimeError("Can't call serve_forever on an already running server object") + raise RuntimeError("Can't call serve_forever on an " + "already running server object") def server_close(self): self.stop_serving.set_result(True) @@ -608,11 +702,9 @@ def server_close(self): self.protocol.close() - class ModbusSerialServer(object): """ A modbus threaded serial socket server - We inherit and overload the socket server so that we can control the client threads as well as have a single server context instance. @@ -620,15 +712,12 @@ class ModbusSerialServer(object): handler = None - def __init__(self, context, framer=None, identity=None, **kwargs): # pragma: no cover + def __init__(self, context, framer=None, **kwargs): # pragma: no cover """ Overloaded initializer for the socket server - If the identify structure is not passed in, the ModbusControlBlock uses its own empty structure. - :param context: The ModbusServerContext datastore :param framer: The framer strategy to use - :param identity: An optional identify structure :param port: The serial port to attach to :param stopbits: The number of stop bits to use :param bytesize: The bytesize of the serial messages @@ -639,8 +728,88 @@ def __init__(self, context, framer=None, identity=None, **kwargs): # pragma: no to a missing slave :param broadcast_enable: True to treat unit_id 0 as broadcast address, False to treat 0 as any other unit_id + :param autoreonnect: True to enable automatic reconnection, + False otherwise + :param reconnect_delay: reconnect delay in seconds + :param response_manipulator: Callback method for + manipulating the response """ - raise NotImplementedException + self.device = kwargs.get('port', 0) + self.stopbits = kwargs.get('stopbits', Defaults.Stopbits) + self.bytesize = kwargs.get('bytesize', Defaults.Bytesize) + self.parity = kwargs.get('parity', Defaults.Parity) + self.baudrate = kwargs.get('baudrate', Defaults.Baudrate) + self.timeout = kwargs.get('timeout', Defaults.Timeout) + self.ignore_missing_slaves = kwargs.get('ignore_missing_slaves', + Defaults.IgnoreMissingSlaves) + self.broadcast_enable = kwargs.get('broadcast_enable', + Defaults.broadcast_enable) + self.auto_reconnect = kwargs.get('auto_reconnect', False) + self.reconnect_delay = kwargs.get('reconnect_delay', 2) + self.reconnecting_task = None + + self.handler = kwargs.get("handler") or ModbusSingleRequestHandler + self.framer = framer or ModbusRtuFramer + self.decoder = ServerDecoder() + self.context = context or ModbusServerContext() + self.response_manipulator = kwargs.get("response_manipulator", None) + self.protocol = None + self.transport = None + + async def start(self): + await self._connect() + + def _protocol_factory(self): + return self.handler(self) + + async def _delayed_connect(self): + await asyncio.sleep(self.reconnect_delay) + await self._connect() + + async def _connect(self): + if self.reconnecting_task is not None: + self.reconnecting_task = None + + try: + self.transport, self.protocol = await create_serial_connection( + asyncio.get_event_loop(), + self._protocol_factory, + self.device, + baudrate=self.baudrate, + bytesize=self.bytesize, + parity=self.parity, + stopbits=self.stopbits, + timeout=self.timeout + ) + except serial.serialutil.SerialException as e: + _logger.debug("Failed to open serial port: {}".format(self.device)) + if not self.auto_reconnect: + raise e + + self._check_reconnect() + + except Exception as e: + _logger.debug("Exception while create - {}".format(e)) + + def on_connection_lost(self): + if self.transport is not None: + self.transport.close() + self.transport = None + self.protocol = None + + self._check_reconnect() + + def _check_reconnect(self): + _logger.debug("checkking autoreconnect {} {}".format( + self.auto_reconnect, self.reconnecting_task)) + if self.auto_reconnect and (self.reconnecting_task is None): + _logger.debug("Scheduling serial connection reconnect") + loop = asyncio.get_event_loop() + self.reconnecting_task = loop.create_task(self._delayed_connect()) + + async def serve_forever(self): + while True: + await asyncio.sleep(360) # --------------------------------------------------------------------------- # @@ -666,7 +835,7 @@ async def StartTcpServer(context=None, identity=None, address=None, server = ModbusTcpServer(context, framer, identity, address, **kwargs) for f in custom_functions: - server.decoder.register(f) # pragma: no cover + server.decoder.register(f) # pragma: no cover if not defer_start: await server.serve_forever() @@ -674,9 +843,12 @@ async def StartTcpServer(context=None, identity=None, address=None, return server -async def StartTlsServer(context=None, identity=None, address=None, sslctx=None, - certfile=None, keyfile=None, allow_reuse_address=False, - allow_reuse_port=False, custom_functions=[], +async def StartTlsServer(context=None, identity=None, address=None, + sslctx=None, + certfile=None, keyfile=None, + allow_reuse_address=False, + allow_reuse_port=False, + custom_functions=[], defer_start=True, **kwargs): """ A factory to start and run a tls modbus server @@ -714,7 +886,7 @@ async def StartTlsServer(context=None, identity=None, address=None, sslctx=None, async def StartUdpServer(context=None, identity=None, address=None, - custom_functions=[], defer_start=True, **kwargs): + custom_functions=[], defer_start=True, **kwargs): """ A factory to start and run a udp modbus server :param context: The ModbusServerContext datastore @@ -738,9 +910,8 @@ async def StartUdpServer(context=None, identity=None, address=None, return server - -def StartSerialServer(context=None, identity=None, custom_functions=[], - **kwargs):# pragma: no cover +async def StartSerialServer(context=None, identity=None, + custom_functions=[], **kwargs): # pragma: no cover """ A factory to start and run a serial modbus server :param context: The ModbusServerContext datastore @@ -757,24 +928,24 @@ def StartSerialServer(context=None, identity=None, custom_functions=[], :param ignore_missing_slaves: True to not send errors on a request to a missing slave """ - raise NotImplementedException - import serial framer = kwargs.pop('framer', ModbusAsciiFramer) server = ModbusSerialServer(context, framer, identity, **kwargs) for f in custom_functions: server.decoder.register(f) - server.serve_forever() + await server.start() + await server.serve_forever() + def StopServer(): """ Helper method to stop Async Server """ import warnings - warnings.warn("deprecated API for asyncio. Call server_close() on server object returned by StartXxxServer", + warnings.warn("deprecated API for asyncio. Call server_close() on " + "server object returned by StartXxxServer", DeprecationWarning) - # --------------------------------------------------------------------------- # # Exported symbols # --------------------------------------------------------------------------- # @@ -785,4 +956,3 @@ def StopServer(): "StartTcpServer", "StartTlsServer", "StartUdpServer", "StartSerialServer" ] - diff --git a/pymodbus/server/reactive/__init__.py b/pymodbus/server/reactive/__init__.py new file mode 100644 index 000000000..bc4e39484 --- /dev/null +++ b/pymodbus/server/reactive/__init__.py @@ -0,0 +1,4 @@ +""" +Copyright (c) 2020 by RiptideIO +All rights reserved. +""" diff --git a/pymodbus/server/reactive/default_config.json b/pymodbus/server/reactive/default_config.json new file mode 100644 index 000000000..6f076676a --- /dev/null +++ b/pymodbus/server/reactive/default_config.json @@ -0,0 +1,32 @@ +{ + "tcp": { + "handler": "ModbusConnectedRequestHandler", + "allow_reuse_address": true, + "allow_reuse_port": true, + "backlog": 20, + "ignore_missing_slaves": false + }, + "rtu": { + "handler": "ModbusSingleRequestHandler", + "stopbits": 1, + "bytesize": 8, + "parity": "N", + "baudrate": 9600, + "timeout": 3, + "auto_reconnect": false, + "reconnect_delay": 2 + }, + "tls": { + "handler": "ModbusConnectedRequestHandler", + "certfile": null, + "keyfile": null, + "allow_reuse_address": true, + "allow_reuse_port": true, + "backlog": 20, + "ignore_missing_slaves": false + }, + "udp": { + "handler": "ModbusDisonnectedRequestHandler", + "ignore_missing_slaves": false + } +} \ No newline at end of file diff --git a/pymodbus/server/reactive/default_config.py b/pymodbus/server/reactive/default_config.py new file mode 100644 index 000000000..ad274e8d2 --- /dev/null +++ b/pymodbus/server/reactive/default_config.py @@ -0,0 +1,37 @@ +""" +Copyright (c) 2020 by RiptideIO +All rights reserved. +""" + +DEFUALT_CONFIG = { + "tcp": { + "handler": "ModbusConnectedRequestHandler", + "allow_reuse_address": True, + "allow_reuse_port": True, + "backlog": 20, + "ignore_missing_slaves": False + }, + "serial": { + "handler": "ModbusSingleRequestHandler", + "stopbits": 1, + "bytesize": 8, + "parity": "N", + "baudrate": 9600, + "timeout": 3, + "auto_reconnect": False, + "reconnect_delay": 2 + }, + "tls": { + "handler": "ModbusConnectedRequestHandler", + "certfile": None, + "keyfile": None, + "allow_reuse_address": True, + "allow_reuse_port": True, + "backlog": 20, + "ignore_missing_slaves": False + }, + "udp": { + "handler": "ModbusDisonnectedRequestHandler", + "ignore_missing_slaves": False + } +} diff --git a/pymodbus/server/reactive/main.py b/pymodbus/server/reactive/main.py new file mode 100644 index 000000000..cf1422574 --- /dev/null +++ b/pymodbus/server/reactive/main.py @@ -0,0 +1,372 @@ +""" +Copyright (c) 2020 by RiptideIO +All rights reserved. +""" +import os +import asyncio +import time +import random +import logging +from pymodbus.version import version as pymodbus_version +from pymodbus.compat import IS_PYTHON3, PYTHON_VERSION +from pymodbus.pdu import ExceptionResponse, ModbusExceptions +from pymodbus.datastore.store import (ModbusSparseDataBlock, + ModbusSequentialDataBlock) +from pymodbus.datastore import ModbusSlaveContext, ModbusServerContext +from pymodbus.device import ModbusDeviceIdentification + +if not IS_PYTHON3 or PYTHON_VERSION < (3, 6): + print(f"You are running {PYTHON_VERSION}." + "Reactive server requires python3.6 or above") + exit() + + +try: + from aiohttp import web +except ImportError as e: + print("Reactive server requires aiohttp. " + "Please install with 'pip install aiohttp' and try again.") + exit(1) + +from pymodbus.server.async_io import (ModbusTcpServer, + ModbusTlsServer, + ModbusSerialServer, + ModbusUdpServer, + ModbusSingleRequestHandler, + ModbusConnectedRequestHandler, + ModbusDisconnectedRequestHandler) +from pymodbus.transaction import (ModbusRtuFramer, + ModbusSocketFramer, + ModbusTlsFramer, + ModbusAsciiFramer, + ModbusBinaryFramer) +logger = logging.getLogger(__name__) + +SERVER_MAPPER = { + "tcp": ModbusTcpServer, + "serial": ModbusSerialServer, + "udp": ModbusUdpServer, + "tls": ModbusTlsServer +} + +DEFAULT_FRAMER = { + "tcp": ModbusSocketFramer, + "rtu": ModbusRtuFramer, + "tls": ModbusTlsFramer, + "udp": ModbusSocketFramer, + "ascii": ModbusAsciiFramer, + "binary": ModbusBinaryFramer +} + +DEFAULT_MANIPULATOR = { + "response_type": "normal", # normal, error, delayed, empty + "delay_by": 0, + "error_code": ModbusExceptions.IllegalAddress, + "clear_after": 5 # request count + +} +DEFUALT_HANDLERS = { + "ModbusSingleRequestHandler": ModbusSingleRequestHandler, + "ModbusConnectedRequestHandler": ModbusConnectedRequestHandler, + "ModbusDisconnectedRequestHandler": ModbusDisconnectedRequestHandler +} +DEFAULT_MODBUS_MAP = {"start_offset": 0, + "count": 100, + "value": 0, "sparse": False} +DEFAULT_DATA_BLOCK = { + "co": DEFAULT_MODBUS_MAP, + "di": DEFAULT_MODBUS_MAP, + "ir": DEFAULT_MODBUS_MAP, + "hr": DEFAULT_MODBUS_MAP + +} + +HINT = """ +Reactive Modbus Server started. +{} + +=========================================================================== +Example Usage: +curl -X POST http://{}:{} -d '{{"response_type": "error", "error_code": 4}}' +=========================================================================== +""" + + +class ReactiveServer: + """ + Modbus Asynchronous Server which can manipulate the response dynamically. + Useful for testing + """ + def __init__(self, host, port, modbus_server, loop=None): + self._web_app = web.Application() + self._runner = web.AppRunner(self._web_app) + self._host = host + self._port = int(port) + self._modbus_server = modbus_server + self._loop = loop + self._add_routes() + self._counter = 0 + self._modbus_server.response_manipulator = self.manipulate_response + self._manipulator_config = dict(**DEFAULT_MANIPULATOR) + self._web_app.on_startup.append(self.start_modbus_server) + self._web_app.on_shutdown.append(self.stop_modbus_server) + + @property + def web_app(self): + return self._web_app + + @property + def manipulator_config(self): + return self._manipulator_config + + @manipulator_config.setter + def manipulator_config(self, value): + if isinstance(value, dict): + self._manipulator_config.update(**value) + + def _add_routes(self): + self._web_app.add_routes([ + web.post('/', self._response_manipulator)]) + + async def start_modbus_server(self, app): + """ + Start Modbus server as asyncio task after startup + :param app: Webapp + :return: + """ + try: + if hasattr(asyncio, "create_task"): + if isinstance(self._modbus_server, ModbusSerialServer): + app["modbus_serial_server"] = asyncio.create_task( + self._modbus_server.start()) + app["modbus_server"] = asyncio.create_task( + self._modbus_server.serve_forever()) + else: + if isinstance(self._modbus_server, ModbusSerialServer): + app["modbus_serial_server"] = asyncio.ensure_future( + self._modbus_server.start() + ) + app["modbus_server"] = asyncio.ensure_future( + self._modbus_server.serve_forever()) + + logger.info("Modbus server started") + except Exception as e: + logger.error("Error starting modbus server") + logger.error(e) + + async def stop_modbus_server(self, app): + """ + Stop modbus server + :param app: Webapp + :return: + """ + logger.info("Stopping modbus server") + if isinstance(self._modbus_server, ModbusSerialServer): + app["modbus_serial_server"].cancel() + app["modbus_server"].cancel() + await app["modbus_server"] + logger.info("Modbus server Stopped") + + async def _response_manipulator(self, request): + """ + POST request Handler for response manipulation end point + Payload is a dict with following fields + :response_type : One among (normal, delayed, error, empty, stray) + :error_code: Modbus error code for error response + :delay_by: Delay sending response by seconds + + :param request: + :return: + """ + data = await request.json() + self._manipulator_config.update(data) + return web.json_response(data=data) + + def update_manipulator_config(self, config): + """ + Updates manipulator config. Resets previous counters + :param config: Manipulator config (dict) + :return: + """ + self._counter = 0 + self._manipulator_config = config + + def manipulate_response(self, response): + """ + Manipulates the actual response according to the required error state. + :param response: Modbus response object + :return: Modbus response + """ + skip_encoding = False + if not self._manipulator_config: + return response + else: + clear_after = self._manipulator_config.get("clear_after") + if clear_after and self._counter > clear_after: + logger.info("Resetting manipulator" + " after {} responses".format(clear_after)) + self.update_manipulator_config(dict(DEFAULT_MANIPULATOR)) + return response + response_type = self._manipulator_config.get("response_type") + if response_type == "error": + error_code = self._manipulator_config.get("error_code") + logger.warning( + "Sending error response for all incoming requests") + err_response = ExceptionResponse(response.function_code, error_code) + err_response.transaction_id = response.transaction_id + err_response.unit_id = response.unit_id + response = err_response + self._counter += 1 + elif response_type == "delayed": + delay_by = self._manipulator_config.get("delay_by") + logger.warning( + "Delaying response by {}s for " + "all incoming requests".format(delay_by)) + time.sleep(delay_by) + self._counter += 1 + elif response_type == "empty": + logger.warning("Sending empty response") + self._counter += 1 + response.should_respond = False + elif response_type == "stray": + data_len = self._manipulator_config.get("data_len", 10) + if data_len <= 0: + logger.warning(f"Invalid data_len {data_len}. " + f"Using default lenght 10") + data_len = 10 + response = os.urandom(data_len) + self._counter += 1 + skip_encoding = True + return response, skip_encoding + + def run(self): + """ + Run Web app + :return: + """ + def _info(message): + msg = HINT.format(message, self._host, self._port) + print(msg) + # print(message) + web.run_app(self._web_app, host=self._host, port=self._port, + print=_info) + + async def run_async(self): + """ + Run Web app + :return: + """ + try: + await self._runner.setup() + site = web.TCPSite(self._runner, self._host, self._port) + await site.start() + except Exception as e: + logger.error(e) + + @classmethod + def create_identity(cls, vendor="Pymodbus", product_code="PM", + vendor_url='http://github.com/riptideio/pymodbus/', + product_name="Pymodbus Server", + model_name="Reactive Server", + version=pymodbus_version.short()): + """ + Create modbus identity + :param vendor: + :param product_code: + :param vendor_url: + :param product_name: + :param model_name: + :param version: + :return: ModbusIdentity object + """ + identity = ModbusDeviceIdentification() + identity.VendorName = vendor + identity.ProductCode = product_code + identity.VendorUrl = vendor_url + identity.ProductName = product_name + identity.ModelName = model_name + identity.MajorMinorRevision = version + + return identity + + @classmethod + def create_context(cls, data_block=None, unit=1, + single=False): + """ + Create Modbus context. + :param data_block: Datablock (dict) Refer DEFAULT_DATA_BLOCK + :param unit: Unit id for the slave + :param single: To run as a single slave + :return: ModbusServerContext object + """ + block = dict() + data_block = data_block or DEFAULT_DATA_BLOCK + for modbus_entity, block_desc in data_block.items(): + start_address = block_desc.get("start_address", 0) + default_count = block_desc.get("count", 0) + default_value = block_desc.get("value", 0) + default_values = [default_value]*default_count + sparse = block_desc.get("sparse", False) + db = ModbusSequentialDataBlock if not sparse else ModbusSparseDataBlock + if sparse: + address_map = block_desc.get("address_map") + if not address_map: + address_map = random.sample( + range(start_address+1, default_count), default_count-1) + address_map.insert(0, 0) + block[modbus_entity] = {add: val for add in sorted(address_map) for val in default_values} + else: + block[modbus_entity] =db(start_address, default_values) + + slave_context = ModbusSlaveContext(**block, zero_mode=True) + if not single: + slaves = {} + for i in unit: + slaves[i] = slave_context + else: + slaves = slave_context + server_context = ModbusServerContext(slaves, single=single) + return server_context + + @classmethod + def factory(cls, server, framer=None, context=None, unit=1, single=False, + host="localhost", modbus_port=5020, web_port=8080, + data_block=DEFAULT_DATA_BLOCK, identity=None, loop=None, **kwargs): + """ + Factory to create ReactiveModbusServer + :param server: Modbus server type (tcp, rtu, tls, udp) + :param framer: Modbus framer (ModbusSocketFramer, ModbusRTUFramer, ModbusTLSFramer) + :param context: Modbus server context to use + :param unit: Modbus unit id + :param single: Run in single mode + :param host: Host address to use for both web app and modbus server (default localhost) + :param modbus_port: Modbus port for TCP and UDP server(default: 5020) + :param web_port: Web App port (default: 8080) + :param data_block: Datablock (refer DEFAULT_DATA_BLOCK) + :param identity: Modbus identity object + :param loop: Asyncio loop to use + :param kwargs: Other server specific keyword arguments, refer corresponding servers documentation + :return: ReactiveServer object + """ + if server.lower() not in SERVER_MAPPER: + logger.error(f"Invalid server {server}", server) + exit(1) + server = SERVER_MAPPER.get(server) + if not framer: + framer = DEFAULT_FRAMER.get(server) + if not context: + context = cls.create_context(data_block=data_block, + unit=unit, single=single) + if not identity: + identity = cls.create_identity() + if server == ModbusSerialServer: + kwargs["port"] = modbus_port + server = server(context, framer=framer, identity=identity, + **kwargs) + else: + server = server(context, framer=framer, identity=identity, + address=(host, modbus_port), defer_start=False, + **kwargs) + return ReactiveServer(host, web_port, server, loop) + +# __END__ diff --git a/pymodbus/server/sync.py b/pymodbus/server/sync.py index f7b22454f..041069aab 100644 --- a/pymodbus/server/sync.py +++ b/pymodbus/server/sync.py @@ -327,16 +327,17 @@ def __init__(self, context, framer=None, identity=None, self.control = ModbusControlBlock() self.address = address or ("", Defaults.Port) self.handler = handler or ModbusConnectedRequestHandler - self.ignore_missing_slaves = kwargs.get('ignore_missing_slaves', + self.ignore_missing_slaves = kwargs.pop('ignore_missing_slaves', Defaults.IgnoreMissingSlaves) - self.broadcast_enable = kwargs.get('broadcast_enable', + self.broadcast_enable = kwargs.pop('broadcast_enable', Defaults.broadcast_enable) if isinstance(identity, ModbusDeviceIdentification): self.control.Identity.update(identity) socketserver.ThreadingTCPServer.__init__(self, self.address, - self.handler) + self.handler, + **kwargs) def process_request(self, request, client): """ Callback for connecting a new client thread @@ -456,16 +457,16 @@ def __init__(self, context, framer=None, identity=None, address=None, self.control = ModbusControlBlock() self.address = address or ("", Defaults.Port) self.handler = handler or ModbusDisconnectedRequestHandler - self.ignore_missing_slaves = kwargs.get('ignore_missing_slaves', + self.ignore_missing_slaves = kwargs.pop('ignore_missing_slaves', Defaults.IgnoreMissingSlaves) - self.broadcast_enable = kwargs.get('broadcast_enable', + self.broadcast_enable = kwargs.pop('broadcast_enable', Defaults.broadcast_enable) if isinstance(identity, ModbusDeviceIdentification): self.control.Identity.update(identity) socketserver.ThreadingUDPServer.__init__(self, - self.address, self.handler) + self.address, self.handler, **kwargs) # self._BaseServer__shutdown_request = True def process_request(self, request, client): @@ -581,7 +582,10 @@ def serve_forever(self): if not self.handler: self._build_handler() while self.is_running: - self.handler.handle() + if hasattr(self.handler, "response_manipulator"): + self.handler.response_manipulator() + else: + self.handler.handle() else: _logger.error("Error opening serial port , " "Unable to start server!!") diff --git a/pymodbus/transaction.py b/pymodbus/transaction.py index 0da18a607..38169a829 100644 --- a/pymodbus/transaction.py +++ b/pymodbus/transaction.py @@ -64,8 +64,10 @@ def __init__(self, client, **kwargs): self.tid = Defaults.TransactionId self.client = client self.backoff = kwargs.get('backoff', Defaults.Backoff) or 0.3 - self.retry_on_empty = kwargs.get('retry_on_empty', Defaults.RetryOnEmpty) - self.retry_on_invalid = kwargs.get('retry_on_invalid', Defaults.RetryOnInvalid) + self.retry_on_empty = kwargs.get('retry_on_empty', + Defaults.RetryOnEmpty) + self.retry_on_invalid = kwargs.get('retry_on_invalid', + Defaults.RetryOnInvalid) self.retries = kwargs.get('retries', Defaults.Retries) or 1 self._transaction_lock = RLock() self._no_response_devices = [] @@ -108,6 +110,25 @@ def _calculate_exception_length(self): return None + def _validate_response(self, request, response, exp_resp_len): + """ + Validate Incoming response against request + :param request: Request sent + :param response: Response received + :param exp_resp_len: Expected response length + :return: New transactions state + """ + if not response: + return False + + mbap = self.client.framer.decode_data(response) + if mbap.get('unit') != request.unit_id or mbap.get('fcode') != request.function_code: + return False + + if 'length' in mbap and exp_resp_len: + return mbap.get('length') == exp_resp_len + return True + def execute(self, request): """ Starts the producer to send the next request to consumer.write(Frame(request)) @@ -149,42 +170,38 @@ def execute(self, request): full = True if not expected_response_length: expected_response_length = Defaults.ReadSize - retries += 1 + response, last_exception = self._transact( + request, + expected_response_length, + full=full, + broadcast=broadcast + ) while retries > 0: - response, last_exception = self._transact( - request, - expected_response_length, - full=full, - broadcast=broadcast + valid_response = self._validate_response( + request, response, expected_response_length ) - if not response and ( - request.unit_id not in self._no_response_devices): - self._no_response_devices.append(request.unit_id) - elif request.unit_id in self._no_response_devices and response: - self._no_response_devices.remove(request.unit_id) - if not response and self.retry_on_empty: - _logger.debug("Retry on empty - {}".format(retries)) - elif not response: - break - if not self.retry_on_invalid: + if valid_response: + if request.unit_id in self._no_response_devices and response: + self._no_response_devices.remove(request.unit_id) + _logger.debug("Got response!!!") break - mbap = self.client.framer.decode_data(response) - if (mbap.get('unit') == request.unit_id): - break - if ('length' in mbap and expected_response_length and - mbap.get('length') == expected_response_length): - break - _logger.debug("Retry on invalid - {}".format(retries)) - if hasattr(self.client, "state"): - _logger.debug("RESETTING Transaction state to 'IDLE' for retry") - self.client.state = ModbusTransactionState.IDLE - if self.backoff: - delay = 2 ** (self.retries - retries) * self.backoff - time.sleep(delay) - _logger.debug("Sleeping {}".format(delay)) - full = False - broadcast = False - retries -= 1 + else: + if not response: + if request.unit_id not in self._no_response_devices: + self._no_response_devices.append(request.unit_id) + if self.retry_on_empty: + response, last_exception = self._retry_transaction(retries, "empty", request, expected_response_length, full=full) + retries -= 1 + else: + # No response received and retries not enabled + break + else: + if self.retry_on_invalid: + response, last_exception = self._retry_transaction(retries, "invalid", request, expected_response_length, full=full) + retries -= 1 + else: + break + # full = False addTransaction = partial(self.addTransaction, tid=request.transaction_id) self.client.framer.processIncomingPacket(response, @@ -206,14 +223,35 @@ def execute(self, request): "'TRANSACTION_COMPLETE'") self.client.state = ( ModbusTransactionState.TRANSACTION_COMPLETE) + self.client.close() return response except ModbusIOException as ex: # Handle decode errors in processIncomingPacket method _logger.exception(ex) + self.client.close() self.client.state = ModbusTransactionState.TRANSACTION_COMPLETE return ex - def _transact(self, packet, response_length, full=False, broadcast=False): + def _retry_transaction(self, retries, reason, + packet, response_length, full=False): + _logger.debug("Retry on {} response - {}".format(reason, retries)) + _logger.debug("Changing transaction state from " + "'WAITING_FOR_REPLY' to 'RETRYING'") + self.client.state = ModbusTransactionState.RETRYING + if self.backoff: + delay = 2 ** (self.retries - retries) * self.backoff + time.sleep(delay) + _logger.debug("Sleeping {}".format(delay)) + self.client.connect() + in_waiting = self.client._in_waiting() + if in_waiting: + if response_length == in_waiting: + result = self._recv(response_length, full) + return result, None + return self._transact(packet, response_length, full=full) + + def _transact(self, packet, response_length, + full=False, broadcast=False): """ Does a Write and Read transaction :param packet: packet to be sent @@ -229,6 +267,11 @@ def _transact(self, packet, response_length, full=False, broadcast=False): if _logger.isEnabledFor(logging.DEBUG): _logger.debug("SEND: " + hexlify_packets(packet)) size = self._send(packet) + if isinstance(size, bytes) and self.client.state == ModbusTransactionState.RETRYING: + _logger.debug("Changing transaction state from " + "'RETRYING' to 'PROCESSING REPLY'") + self.client.state = ModbusTransactionState.PROCESSING_REPLY + return size, None if broadcast: if size: _logger.debug("Changing transaction state from 'SENDING' " @@ -244,6 +287,7 @@ def _transact(self, packet, response_length, full=False, broadcast=False): if local_echo_packet != packet: return b'', "Wrong local echo" result = self._recv(response_length, full) + # result2 = self._recv(response_length, full) if _logger.isEnabledFor(logging.DEBUG): _logger.debug("RECV: " + hexlify_packets(result)) @@ -255,7 +299,7 @@ def _transact(self, packet, response_length, full=False, broadcast=False): result = b'' return result, last_exception - def _send(self, packet): + def _send(self, packet, retrying=False): return self.client.framer.sendPacket(packet) def _recv(self, expected_response_length, full): @@ -275,9 +319,10 @@ def _recv(self, expected_response_length, full): read_min = self.client.framer.recvPacket(min_size) if len(read_min) != min_size: + msg_start = "Incomplete message" if read_min else "No response" raise InvalidMessageReceivedException( - "Incomplete message received, expected at least %d bytes " - "(%d received)" % (min_size, len(read_min)) + "%s received, expected at least %d bytes " + "(%d received)" % (msg_start, min_size, len(read_min)) ) if read_min: if isinstance(self.client.framer, ModbusSocketFramer): @@ -312,9 +357,14 @@ def _recv(self, expected_response_length, full): result = read_min + result actual = len(result) if total is not None and actual != total: - _logger.debug("Incomplete message received, " + msg_start = "Incomplete message" if actual else "No response" + _logger.debug("{} received, " "Expected {} bytes Recieved " - "{} bytes !!!!".format(total, actual)) + "{} bytes !!!!".format(msg_start, total, actual)) + elif actual == 0: + # If actual == 0 and total is not None then the above + # should be triggered, so total must be None here + _logger.debug("No response received to unbounded read !!!!") if self.client.state != ModbusTransactionState.PROCESSING_REPLY: _logger.debug("Changing transaction state from " "'WAITING FOR REPLY' to 'PROCESSING REPLY'") @@ -324,7 +374,7 @@ def _recv(self, expected_response_length, full): def addTransaction(self, request, tid=None): """ Adds a transaction to the handler - This holds the requets in case it needs to be resent. + This holds the request in case it needs to be resent. After being sent, the request is removed. :param request: The request to hold on to diff --git a/pymodbus/utilities.py b/pymodbus/utilities.py index 6d38ca3dd..9e31d0974 100644 --- a/pymodbus/utilities.py +++ b/pymodbus/utilities.py @@ -20,6 +20,8 @@ class ModbusTransactionState(object): PROCESSING_REPLY = 4 PROCESSING_ERROR = 5 TRANSACTION_COMPLETE = 6 + RETRYING = 7 + NO_RESPONSE_STATE = 8 @classmethod def to_string(cls, state): @@ -30,7 +32,8 @@ def to_string(cls, state): ModbusTransactionState.WAITING_TURNAROUND_DELAY: "WAITING_TURNAROUND_DELAY", ModbusTransactionState.PROCESSING_REPLY: "PROCESSING_REPLY", ModbusTransactionState.PROCESSING_ERROR: "PROCESSING_ERROR", - ModbusTransactionState.TRANSACTION_COMPLETE: "TRANSACTION_COMPLETE" + ModbusTransactionState.TRANSACTION_COMPLETE: "TRANSACTION_COMPLETE", + ModbusTransactionState.RETRYING: "RETRYING TRANSACTION", } return states.get(state, None) @@ -242,7 +245,10 @@ def hexlify_packets(packet): """ if not packet: return '' - return " ".join([hex(byte2int(x)) for x in packet]) + if IS_PYTHON3: + return " ".join([hex(byte2int(x)) for x in packet]) + else: + return u" ".join([hex(byte2int(x)) for x in packet]) # --------------------------------------------------------------------------- # # Exported symbols # --------------------------------------------------------------------------- # diff --git a/pymodbus/version.py b/pymodbus/version.py index 2fc273abd..1e556a81d 100644 --- a/pymodbus/version.py +++ b/pymodbus/version.py @@ -41,7 +41,7 @@ def __str__(self): return '[%s, version %s]' % (self.package, self.short()) -version = Version('pymodbus', 2, 4, 0) +version = Version('pymodbus', 2, 5, 1) version.__name__ = 'pymodbus' # fix epydoc error diff --git a/requirements-coverage.txt b/requirements-coverage.txt new file mode 100644 index 000000000..9808587c9 --- /dev/null +++ b/requirements-coverage.txt @@ -0,0 +1 @@ +coverage >= 4.2 diff --git a/requirements-docs.txt b/requirements-docs.txt index f1165c9c6..fc6cdda47 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -9,9 +9,9 @@ recommonmark>=0.4.0 Sphinx>=1.6.5 sphinx-rtd-theme>=0.2.4 SQLAlchemy>=1.1.15 # Required to parse some files -tornado>=4.5.3 # Required to parse some files +tornado==4.5.3 # Required to parse some files Twisted>=17.1.0 # Required to parse some files prompt_toolkit>=2.0.4 click>=7.0 -m2r>=0.2.0 +m2r2>=0.2.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index 02a4ff6b5..72e49e63b 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,6 +1,6 @@ bcrypt>=3.1.6 capturer >= 2.2 -coverage >= 4.2 +-r requirements-coverage.txt cryptography>= 2.3 mock >= 1.0.1 pyserial-asyncio>=0.4.0;python_version>="3.4" @@ -15,6 +15,11 @@ sqlalchemy>=1.1.15 #wsgiref>=0.1.2 verboselogs >= 1.5 tornado==4.5.3 -Twisted>=20.3.0 +# using platform_python_implementation rather than +# implementation_name for Python 2 support +Twisted[conch,serial]>=20.3.0; platform_python_implementation != "PyPy" or sys_platform != "win32" +# pywin32 isn't supported on pypy +# https://github.com/mhammond/pywin32/issues/1289 +Twisted[conch]>=20.3.0; platform_python_implementation == "PyPy" and sys_platform == "win32" zope.interface>=4.4.0 asynctest>=0.10.0 diff --git a/scripts/travis.sh b/scripts/travis.sh deleted file mode 100755 index 8f4338270..000000000 --- a/scripts/travis.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -e -set -x -if [ "$TRAVIS_OS_NAME" = osx ]; then - VIRTUAL_ENV="$HOME/.virtualenvs/python2.7" - if [ ! -x "$VIRTUAL_ENV/bin/python" ]; then - virtualenv "$VIRTUAL_ENV" - fi - source "$VIRTUAL_ENV/bin/activate" -fi - -eval "$@" diff --git a/setup.py b/setup.py index 8b60584b4..8575b2b5b 100644 --- a/setup.py +++ b/setup.py @@ -23,9 +23,15 @@ try: from setup_commands import command_classes except ImportError: - command_classes={} + command_classes = {} from pymodbus import __version__, __author__, __maintainer__ +from pymodbus.utilities import IS_PYTHON3 +CONSOLE_SCRIPTS = [ + 'pymodbus.console=pymodbus.repl.client.main:main' + ] +if IS_PYTHON3: + CONSOLE_SCRIPTS.append('pymodbus.server=pymodbus.repl.server.main:server') with open('requirements.txt') as reqs: install_requires = [ line for line in reqs.read().split('\n') @@ -47,6 +53,13 @@ """, classifiers=[ 'Development Status :: 4 - Beta', + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", 'Environment :: Console', 'Environment :: X11 Applications :: GTK', 'Framework :: Twisted', @@ -71,6 +84,7 @@ platforms=['Linux', 'Mac OS X', 'Win'], include_package_data=True, zip_safe=True, + python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*', install_requires=install_requires, extras_require={ 'quality': [ @@ -83,24 +97,36 @@ 'sphinx_rtd_theme', 'humanfriendly'], 'twisted': [ - 'twisted >= 20.3.0', - 'pyasn1 >= 0.1.4', + # using platform_python_implementation rather than + # implementation_name for Python 2 support + 'Twisted[conch,serial]>=20.3.0; platform_python_implementation != "PyPy" or sys_platform != "win32"', + # pywin32 isn't supported on pypy + # https://github.com/mhammond/pywin32/issues/1289 + 'Twisted[conch]>=20.3.0; platform_python_implementation == "PyPy" and sys_platform == "win32"', ], 'tornado': [ 'tornado == 4.5.3' ], + 'trio': [ 'trio ~= 0.17.0', 'async_generator ~= 1.10', ], - 'repl': [ + 'repl:python_version <= "2.7"': [ 'click>=7.0', 'prompt-toolkit==2.0.4', - 'pygments==2.2.0' + 'pygments>=2.2.0' + ], + 'repl:python_version >= "3.6"': [ + 'click>=7.0', + 'prompt-toolkit>=3.0.8', + 'pygments>=2.2.0', + 'aiohttp>=3.7.3', + 'pyserial-asyncio>=0.5' ] }, entry_points={ - 'console_scripts': ['pymodbus.console=pymodbus.repl.main:main'], + 'console_scripts': CONSOLE_SCRIPTS, }, test_suite='nose.collector', cmdclass=command_classes, diff --git a/test/conftest.py b/test/conftest.py index 748fd4b7a..932e8124c 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,3 +1,8 @@ -from pymodbus.compat import IS_PYTHON3, PYTHON_VERSION -if not IS_PYTHON3 or IS_PYTHON3 and PYTHON_VERSION.minor < 7: - collect_ignore = ["test_server_asyncio.py"] +from pymodbus.compat import PYTHON_VERSION +if PYTHON_VERSION < (3,): + # These files use syntax introduced between Python 2 and our lowest + # supported Python 3 version. We just won't run these tests in Python 2. + collect_ignore = [ + "test_client_async_asyncio.py", + "test_server_asyncio.py", + ] diff --git a/test/test_bit_read_messages.py b/test/test_bit_read_messages.py index 03507121d..75b678f09 100644 --- a/test/test_bit_read_messages.py +++ b/test/test_bit_read_messages.py @@ -45,7 +45,7 @@ def testReadBitBaseClassMethods(self): msg = "ReadBitRequest(1,1)" self.assertEqual(msg, str(handle)) handle = ReadBitsResponseBase([1,1]) - msg = "ReadBitResponse(2)" + msg = "ReadBitsResponseBase(2)" self.assertEqual(msg, str(handle)) def testBitReadBaseRequestEncoding(self): diff --git a/test/test_bit_write_messages.py b/test/test_bit_write_messages.py index 8807963a8..18459f553 100644 --- a/test/test_bit_write_messages.py +++ b/test/test_bit_write_messages.py @@ -60,6 +60,8 @@ def testWriteMultipleCoilsRequest(self): self.assertEqual(request.byte_count, 1) self.assertEqual(request.address, 1) self.assertEqual(request.values, [True]*5) + self.assertEqual(request.get_response_pdu_size(), 5) + def testInvalidWriteMultipleCoilsRequest(self): request = WriteMultipleCoilsRequest(1, None) diff --git a/test/test_client_async.py b/test/test_client_async.py index 06d0acd43..54002ac2d 100644 --- a/test/test_client_async.py +++ b/test/test_client_async.py @@ -1,4 +1,6 @@ #!/usr/bin/env python +import contextlib +import sys import unittest import pytest from pymodbus.compat import IS_PYTHON3, PYTHON_VERSION @@ -31,13 +33,18 @@ import ssl IS_DARWIN = platform.system().lower() == "darwin" +IS_WINDOWS = platform.system().lower() == "windows" OSX_SIERRA = LooseVersion("10.12") if IS_DARWIN: IS_HIGH_SIERRA_OR_ABOVE = LooseVersion(platform.mac_ver()[0]) SERIAL_PORT = '/dev/ttyp0' if not IS_HIGH_SIERRA_OR_ABOVE else '/dev/ptyp0' else: IS_HIGH_SIERRA_OR_ABOVE = False - SERIAL_PORT = "/dev/ptmx" + if IS_WINDOWS: + # the use is mocked out + SERIAL_PORT = "" + else: + SERIAL_PORT = "/dev/ptmx" # ---------------------------------------------------------------------------# # Fixture @@ -48,6 +55,15 @@ def mock_asyncio_gather(coro): return coro +@contextlib.contextmanager +def maybe_manage(condition, manager): + if condition: + with manager as value: + yield value + else: + yield None + + class TestAsynchronousClient(object): """ This is the unittest for the pymodbus.client.asynchronous module @@ -198,6 +214,10 @@ def testUdpAsycioClient(self, mock_gather, mock_event_loop): # Test Serial client # -----------------------------------------------------------------------# + @pytest.mark.skipif( + sys.platform == 'win32' and platform.python_implementation() == 'PyPy', + reason='Twisted serial requires pywin32 which is not compatible with PyPy', + ) @pytest.mark.parametrize("method, framer", [("rtu", ModbusRtuFramer), ("socket", ModbusSocketFramer), ("binary", ModbusBinaryFramer), @@ -208,30 +228,30 @@ def testSerialTwistedClient(self, method, framer): with patch("serial.Serial") as mock_sp: from twisted.internet import reactor from twisted.internet.serialport import SerialPort + with maybe_manage(sys.platform == 'win32', patch.object(SerialPort, "_finishPortSetup")): + with patch('twisted.internet.reactor') as mock_reactor: - with patch('twisted.internet.reactor') as mock_reactor: - - protocol, client = AsyncModbusSerialClient(schedulers.REACTOR, - method=method, - port=SERIAL_PORT, - proto_cls=ModbusSerClientProtocol) + protocol, client = AsyncModbusSerialClient(schedulers.REACTOR, + method=method, + port=SERIAL_PORT, + proto_cls=ModbusSerClientProtocol) - assert (isinstance(client, SerialPort)) - assert (isinstance(client.protocol, ModbusSerClientProtocol)) - assert (0 == len(list(client.protocol.transaction))) - assert (isinstance(client.protocol.framer, framer)) - assert (client.protocol._connected) + assert (isinstance(client, SerialPort)) + assert (isinstance(client.protocol, ModbusSerClientProtocol)) + assert (0 == len(list(client.protocol.transaction))) + assert (isinstance(client.protocol.framer, framer)) + assert (client.protocol._connected) - def handle_failure(failure): - assert (isinstance(failure.exception(), ConnectionException)) + def handle_failure(failure): + assert (isinstance(failure.exception(), ConnectionException)) - d = client.protocol._buildResponse(0x00) - d.addCallback(handle_failure) + d = client.protocol._buildResponse(0x00) + d.addCallback(handle_failure) - assert (client.protocol._connected) - client.protocol.close() - protocol.stop() - assert (not client.protocol._connected) + assert (client.protocol._connected) + client.protocol.close() + protocol.stop() + assert (not client.protocol._connected) @pytest.mark.parametrize("method, framer", [("rtu", ModbusRtuFramer), ("socket", ModbusSocketFramer), @@ -239,24 +259,26 @@ def handle_failure(failure): ("ascii", ModbusAsciiFramer)]) def testSerialTornadoClient(self, method, framer): """ Test the serial tornado client client initialize """ - protocol, future = AsyncModbusSerialClient(schedulers.IO_LOOP, method=method, port=SERIAL_PORT) - client = future.result() - assert(isinstance(client, AsyncTornadoModbusSerialClient)) - assert(0 == len(list(client.transaction))) - assert(isinstance(client.framer, framer)) - assert(client.port == SERIAL_PORT) - assert(client._connected) - - def handle_failure(failure): - assert(isinstance(failure.exception(), ConnectionException)) - - d = client._build_response(0x00) - d.add_done_callback(handle_failure) - - assert(client._connected) - client.close() - protocol.stop() - assert(not client._connected) + from serial import Serial + with maybe_manage(sys.platform in ('darwin', 'win32'), patch.object(Serial, "open")): + protocol, future = AsyncModbusSerialClient(schedulers.IO_LOOP, method=method, port=SERIAL_PORT) + client = future.result() + assert(isinstance(client, AsyncTornadoModbusSerialClient)) + assert(0 == len(list(client.transaction))) + assert(isinstance(client.framer, framer)) + assert(client.port == SERIAL_PORT) + assert(client._connected) + + def handle_failure(failure): + assert(isinstance(failure.exception(), ConnectionException)) + + d = client._build_response(0x00) + d.add_done_callback(handle_failure) + + assert(client._connected) + client.close() + protocol.stop() + assert(not client._connected) @pytest.mark.skipif(IS_PYTHON3 , reason="requires python2.7") def testSerialAsyncioClientPython2(self): diff --git a/test/test_client_async_asyncio.py b/test/test_client_async_asyncio.py index 64c73ae27..a666ebbca 100644 --- a/test/test_client_async_asyncio.py +++ b/test/test_client_async_asyncio.py @@ -3,6 +3,7 @@ if IS_PYTHON3 and PYTHON_VERSION >= (3, 4): from unittest import mock from pymodbus.client.asynchronous.async_io import ( + BaseModbusAsyncClientProtocol, ReconnectingAsyncioModbusTcpClient, ModbusClientProtocol, ModbusUdpClientProtocol) from test.asyncio_test_helper import return_as_coroutine, run_coroutine @@ -10,7 +11,7 @@ from pymodbus.exceptions import ConnectionException from pymodbus.transaction import ModbusSocketFramer from pymodbus.bit_read_message import ReadCoilsRequest, ReadCoilsResponse - protocols = [ModbusUdpClientProtocol, ModbusClientProtocol] + protocols = [BaseModbusAsyncClientProtocol, ModbusUdpClientProtocol, ModbusClientProtocol] else: import mock protocols = [None, None] @@ -18,6 +19,12 @@ @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") class TestAsyncioClient(object): + def test_base_modbus_async_client_protocol(self): + protocol = BaseModbusAsyncClientProtocol() + assert protocol.factory is None + assert protocol.transport is None + assert not protocol._connected + def test_protocol_connection_state_propagation_to_factory(self): protocol = ModbusClientProtocol() assert protocol.factory is None @@ -28,7 +35,8 @@ def test_protocol_connection_state_propagation_to_factory(self): protocol.connection_made(mock.sentinel.TRANSPORT) assert protocol.transport is mock.sentinel.TRANSPORT - protocol.factory.protocol_made_connection.assert_called_once_with(protocol) + protocol.factory.protocol_made_connection.assert_called_once_with( + protocol) assert protocol.factory.protocol_lost_connection.call_count == 0 protocol.factory.reset_mock() @@ -36,7 +44,19 @@ def test_protocol_connection_state_propagation_to_factory(self): protocol.connection_lost(mock.sentinel.REASON) assert protocol.transport is None assert protocol.factory.protocol_made_connection.call_count == 0 - protocol.factory.protocol_lost_connection.assert_called_once_with(protocol) + protocol.factory.protocol_lost_connection.assert_called_once_with( + protocol) + protocol.raise_future = mock.MagicMock() + request = mock.MagicMock() + protocol.transaction.addTransaction(request, 1) + protocol.connection_lost(mock.sentinel.REASON) + if PYTHON_VERSION.major == 3 and PYTHON_VERSION.minor >= 8: + call_args = protocol.raise_future.call_args.args + else: + call_args = protocol.raise_future.call_args[0] + protocol.raise_future.assert_called_once() + assert call_args[0] == request + assert isinstance(call_args[1], ConnectionException) def test_factory_initialization_state(self): mock_protocol_class = mock.MagicMock() @@ -116,15 +136,18 @@ def test_factory_protocol_lost_connection(self, mock_async): assert not client.connected assert client.protocol is None - @mock.patch('pymodbus.client.asynchronous.async_io.asyncio.ensure_future') - def test_factory_start_success(self, mock_async): + # @mock.patch('pymodbus.client.asynchronous.async_io.asyncio.ensure_future') + @pytest.mark.asyncio + async def test_factory_start_success(self): mock_protocol_class = mock.MagicMock() - mock_loop = mock.MagicMock() - client = ReconnectingAsyncioModbusTcpClient(protocol_class=mock_protocol_class, loop=mock_loop) + # mock_loop = mock.MagicMock() + client = ReconnectingAsyncioModbusTcpClient(protocol_class=mock_protocol_class) + # client = ReconnectingAsyncioModbusTcpClient(protocol_class=mock_protocol_class, loop=mock_loop) - run_coroutine(client.start(mock.sentinel.HOST, mock.sentinel.PORT)) - mock_loop.create_connection.assert_called_once_with(mock.ANY, mock.sentinel.HOST, mock.sentinel.PORT) - assert mock_async.call_count == 0 + await client.start(mock.sentinel.HOST, mock.sentinel.PORT) + # run_coroutine(client.start(mock.sentinel.HOST, mock.sentinel.PORT)) + # mock_loop.create_connection.assert_called_once_with(mock.ANY, mock.sentinel.HOST, mock.sentinel.PORT) + # assert mock_async.call_count == 0 @mock.patch('pymodbus.client.asynchronous.async_io.asyncio.ensure_future') def test_factory_start_failing_and_retried(self, mock_async): @@ -227,27 +250,34 @@ def testClientProtocolDataReceived(self, protocol): # setup existing request d = protocol._buildResponse(0x00) - if isinstance(protocol, ModbusClientProtocol): - protocol.data_received(data) - else: + if isinstance(protocol, ModbusUdpClientProtocol): protocol.datagram_received(data, None) + else: + protocol.data_received(data) result = d.result() assert isinstance(result, ReadCoilsResponse) - @pytest.mark.skip("To fix") + # @pytest.mark.skip("To fix") + @pytest.mark.asyncio @pytest.mark.parametrize("protocol", protocols) - def testClientProtocolExecute(self, protocol): + async def testClientProtocolExecute(self, protocol): ''' Test the client protocol execute method ''' + import asyncio framer = ModbusSocketFramer(None) protocol = protocol(framer=framer) + protocol.create_future = mock.MagicMock() + fut = asyncio.Future() + fut.set_result(fut) + protocol.create_future.return_value = fut transport = mock.MagicMock() protocol.connection_made(transport) protocol.transport.write = mock.Mock() request = ReadCoilsRequest(1, 1) - d = protocol.execute(request) + d = await protocol.execute(request) tid = request.transaction_id - assert d == protocol.transaction.getTransaction(tid) + f = protocol.transaction.getTransaction(tid) + assert d == f @pytest.mark.parametrize("protocol", protocols) def testClientProtocolHandleResponse(self, protocol): @@ -257,7 +287,9 @@ def testClientProtocolHandleResponse(self, protocol): protocol.connection_made(transport=transport) reply = ReadCoilsRequest(1, 1) reply.transaction_id = 0x00 - + # if isinstance(protocol.create_future, mock.MagicMock): + # import asyncio + # protocol.create_future.return_value = asyncio.Future() # handle skipped cases protocol._handleResponse(None) protocol._handleResponse(reply) @@ -272,6 +304,9 @@ def testClientProtocolHandleResponse(self, protocol): def testClientProtocolBuildResponse(self, protocol): ''' Test the udp client protocol builds responses ''' protocol = protocol() + # if isinstance(protocol.create_future, mock.MagicMock): + # import asyncio + # protocol.create_future.return_value = asyncio.Future() assert not len(list(protocol.transaction)) d = protocol._buildResponse(0x00) diff --git a/test/test_client_sync.py b/test/test_client_sync.py old mode 100644 new mode 100755 index 1014a9382..30456a58d --- a/test/test_client_sync.py +++ b/test/test_client_sync.py @@ -1,5 +1,7 @@ #!/usr/bin/env python import unittest +from itertools import count +from io import StringIO from pymodbus.compat import IS_PYTHON3 if IS_PYTHON3: # Python 3 @@ -9,6 +11,9 @@ import socket import serial import ssl +import sys + +import pytest from pymodbus.client.sync import ModbusTcpClient, ModbusUdpClient from pymodbus.client.sync import ModbusSerialClient, BaseModbusClient @@ -17,6 +22,8 @@ from pymodbus.exceptions import ParameterException from pymodbus.transaction import ModbusAsciiFramer, ModbusRtuFramer from pymodbus.transaction import ModbusBinaryFramer +from pymodbus.transaction import ModbusSocketFramer +from pymodbus.utilities import hexlify_packets # ---------------------------------------------------------------------------# @@ -43,6 +50,15 @@ def setblocking(self, flag): return None def in_waiting(self): return None +inet_pton_skipif = pytest.mark.skipif( + sys.platform == "win32" and sys.version_info < (3, 4), + reason=( + "Uses socket.inet_pton() which wasn't available on Windows until" + " 3.4.", + ) +) + + # ---------------------------------------------------------------------------# # Fixture @@ -62,8 +78,8 @@ def testBaseModbusClient(self): client = BaseModbusClient(None) client.transaction = None self.assertRaises(NotImplementedException, lambda: client.connect()) - self.assertRaises(NotImplementedException, lambda: client._send(None)) - self.assertRaises(NotImplementedException, lambda: client._recv(None)) + self.assertRaises(NotImplementedException, lambda: client.send(None)) + self.assertRaises(NotImplementedException, lambda: client.recv(None)) self.assertRaises(NotImplementedException, lambda: client.__enter__()) self.assertRaises(NotImplementedException, lambda: client.execute()) self.assertRaises(NotImplementedException, lambda: client.is_socket_open()) @@ -71,6 +87,20 @@ def testBaseModbusClient(self): client.close() client.__exit__(0, 0, 0) + # Test information methods + client.last_frame_end = 2 + client.silent_interval = 2 + self.assertEqual(4, client.idle_time()) + client.last_frame_end = None + self.assertEqual(0, client.idle_time()) + + # Test debug/trace/_dump methods + self.assertEqual(False, client.debug_enabled()) + writable = StringIO() + client.trace(writable) + client._dump(b'\x00\x01\x02', None) + self.assertEqual(hexlify_packets(b'\x00\x01\x02'), writable.getvalue()) + # a successful execute client.connect = lambda: True client.transaction = Mock(**{'execute.return_value': True}) @@ -110,6 +140,7 @@ def testBasicSyncUdpClient(self): self.assertEqual("ModbusUdpClient(127.0.0.1:502)", str(client)) + @inet_pton_skipif def testUdpClientAddressFamily(self): ''' Test the Udp client get address family method''' client = ModbusUdpClient() @@ -117,6 +148,7 @@ def testUdpClientAddressFamily(self): client._get_address_family('127.0.0.1')) self.assertEqual(socket.AF_INET6, client._get_address_family('::1')) + @inet_pton_skipif def testUdpClientConnect(self): ''' Test the Udp client connection method''' with patch.object(socket, 'socket') as mock_method: @@ -133,6 +165,12 @@ def settimeout(self, *a, **kwa): client = ModbusUdpClient() self.assertFalse(client.connect()) + @inet_pton_skipif + def testUdpClientIsSocketOpen(self): + ''' Test the udp client is_socket_open method''' + client = ModbusUdpClient() + self.assertTrue(client.is_socket_open()) + def testUdpClientSend(self): ''' Test the udp client send method''' client = ModbusUdpClient() @@ -192,8 +230,10 @@ def testBasicSyncTcpClient(self, mock_select): def testTcpClientConnect(self): ''' Test the tcp client connection method''' with patch.object(socket, 'create_connection') as mock_method: - mock_method.return_value = object() + _socket = MagicMock() + mock_method.return_value = _socket client = ModbusTcpClient() + _socket.getsockname.return_value = ('dmmy', 1234) self.assertTrue(client.connect()) with patch.object(socket, 'create_connection') as mock_method: @@ -201,6 +241,11 @@ def testTcpClientConnect(self): client = ModbusTcpClient() self.assertFalse(client.connect()) + def testTcpClientIsSocketOpen(self): + ''' Test the tcp client is_socket_open method''' + client = ModbusTcpClient() + self.assertFalse(client.is_socket_open()) + def testTcpClientSend(self): ''' Test the tcp client send method''' client = ModbusTcpClient() @@ -210,11 +255,13 @@ def testTcpClientSend(self): self.assertEqual(0, client._send(None)) self.assertEqual(4, client._send('1234')) + @patch('pymodbus.client.sync.time') @patch('pymodbus.client.sync.select') - def testTcpClientRecv(self, mock_select): + def testTcpClientRecv(self, mock_select, mock_time): ''' Test the tcp client receive method''' mock_select.select.return_value = [True] + mock_time.time.side_effect = count() client = ModbusTcpClient() self.assertRaises(ConnectionException, lambda: client._recv(1024)) @@ -225,7 +272,7 @@ def testTcpClientRecv(self, mock_select): mock_socket = MagicMock() mock_socket.recv.side_effect = iter([b'\x00', b'\x01', b'\x02']) client.socket = mock_socket - client.timeout = 1 + client.timeout = 3 self.assertEqual(b'\x00\x01\x02', client._recv(3)) mock_socket.recv.side_effect = iter([b'\x00', b'\x01', b'\x02']) self.assertEqual(b'\x00\x01', client._recv(2)) @@ -235,7 +282,16 @@ def testTcpClientRecv(self, mock_select): mock_select.select.return_value = [True] self.assertIn(b'\x00', client._recv(None)) - def testSerialClientRpr(self): + mock_socket = MagicMock() + mock_socket.recv.return_value = b'' + client.socket = mock_socket + self.assertRaises(ConnectionException, lambda: client._recv(1024)) + + mock_socket.recv.side_effect = iter([b'\x00', b'\x01', b'\x02', b'']) + client.socket = mock_socket + self.assertEqual(b'\x00\x01\x02', client._recv(1024)) + + def testTcpClientRpr(self): client = ModbusTcpClient() rep = "<{} at {} socket={}, ipaddr={}, port={}, timeout={}>".format( client.__class__.__name__, hex(id(client)), client.socket, @@ -307,19 +363,25 @@ def testTlsClientSend(self): self.assertEqual(0, client._send(None)) self.assertEqual(4, client._send('1234')) - def testTlsClientRecv(self): + @patch('pymodbus.client.sync.time') + def testTlsClientRecv(self, mock_time): ''' Test the tls client receive method''' client = ModbusTlsClient() self.assertRaises(ConnectionException, lambda: client._recv(1024)) + mock_time.time.side_effect = count() + client.socket = mockSocket() self.assertEqual(b'', client._recv(0)) self.assertEqual(b'\x00' * 4, client._recv(4)) + client.timeout = 2 + self.assertIn(b'\x00', client._recv(None)) + mock_socket = MagicMock() mock_socket.recv.side_effect = iter([b'\x00', b'\x01', b'\x02']) client.socket = mock_socket - client.timeout = 1 + client.timeout = 3 self.assertEqual(b'\x00\x01\x02', client._recv(3)) mock_socket.recv.side_effect = iter([b'\x00', b'\x01', b'\x02']) self.assertEqual(b'\x00\x01', client._recv(2)) @@ -354,6 +416,8 @@ def testSyncSerialClientInstantiation(self): ModbusRtuFramer)) self.assertTrue(isinstance(ModbusSerialClient(method='binary').framer, ModbusBinaryFramer)) + self.assertTrue(isinstance(ModbusSerialClient(method='socket').framer, + ModbusSocketFramer)) self.assertRaises(ParameterException, lambda: ModbusSerialClient(method='something')) @@ -384,6 +448,12 @@ def testBasicSyncSerialClient(self, mock_serial): self.assertTrue(client.connect()) client.close() + # rtu connect/disconnect + rtu_client = ModbusSerialClient(method='rtu', strict=True) + self.assertTrue(rtu_client.connect()) + self.assertEqual(rtu_client.socket.interCharTimeout, rtu_client.inter_char_timeout) + rtu_client.close() + # already closed socket client.socket = False client.close() @@ -393,7 +463,7 @@ def testBasicSyncSerialClient(self, mock_serial): def testSerialClientConnect(self): ''' Test the serial client connection method''' with patch.object(serial, 'Serial') as mock_method: - mock_method.return_value = object() + mock_method.return_value = MagicMock() client = ModbusSerialClient() self.assertTrue(client.connect()) @@ -402,6 +472,14 @@ def testSerialClientConnect(self): client = ModbusSerialClient() self.assertFalse(client.connect()) + @patch("serial.Serial") + def testSerialClientIsSocketOpen(self, mock_serial): + ''' Test the serial client is_socket_open method''' + client = ModbusSerialClient() + self.assertFalse(client.is_socket_open()) + client.socket = mock_serial + self.assertTrue(client.is_socket_open()) + @patch("serial.Serial") def testSerialClientSend(self, mock_serial): ''' Test the serial client send method''' @@ -444,6 +522,8 @@ def testSerialClientRecv(self): self.assertEqual(b'', client._recv(None)) client.socket.timeout = 0 self.assertEqual(b'', client._recv(0)) + client.timeout = None + self.assertEqual(b'', client._recv(None)) def testSerialClientRepr(self): client = ModbusSerialClient() diff --git a/test/test_client_sync_diag.py b/test/test_client_sync_diag.py new file mode 100755 index 000000000..c32d283e7 --- /dev/null +++ b/test/test_client_sync_diag.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +import unittest +from itertools import count +from pymodbus.compat import IS_PYTHON3 + +if IS_PYTHON3: # Python 3 + from unittest.mock import patch, Mock, MagicMock +else: # Python 2 + from mock import patch, Mock, MagicMock +import socket + +from pymodbus.client.sync_diag import ModbusTcpDiagClient, get_client +from pymodbus.exceptions import ConnectionException, NotImplementedException +from pymodbus.exceptions import ParameterException +from test.test_client_sync import mockSocket + + +# ---------------------------------------------------------------------------# +# Fixture +# ---------------------------------------------------------------------------# +class SynchronousDiagnosticClientTest(unittest.TestCase): + ''' + This is the unittest for the pymodbus.client.sync_diag module. It is + a copy of parts of the test for the TCP class in the pymodbus.client.sync + module, as it should operate identically and only log some additional + lines. + ''' + + # -----------------------------------------------------------------------# + # Test TCP Diagnostic Client + # -----------------------------------------------------------------------# + + def testSyncTcpDiagClientInstantiation(self): + client = get_client() + self.assertNotEqual(client, None) + + def testBasicSyncTcpDiagClient(self): + ''' Test the basic methods for the tcp sync diag client''' + + # connect/disconnect + client = ModbusTcpDiagClient() + client.socket = mockSocket() + self.assertTrue(client.connect()) + client.close() + + def testTcpDiagClientConnect(self): + ''' Test the tcp sync diag client connection method''' + with patch.object(socket, 'create_connection') as mock_method: + mock_method.return_value = object() + client = ModbusTcpDiagClient() + self.assertTrue(client.connect()) + + with patch.object(socket, 'create_connection') as mock_method: + mock_method.side_effect = socket.error() + client = ModbusTcpDiagClient() + self.assertFalse(client.connect()) + + @patch('pymodbus.client.sync.time') + @patch('pymodbus.client.sync_diag.time') + @patch('pymodbus.client.sync.select') + def testTcpDiagClientRecv(self, mock_select, mock_diag_time, mock_time): + ''' Test the tcp sync diag client receive method''' + + mock_select.select.return_value = [True] + mock_time.time.side_effect = count() + mock_diag_time.time.side_effect = count() + client = ModbusTcpDiagClient() + self.assertRaises(ConnectionException, lambda: client._recv(1024)) + + client.socket = mockSocket() + # Test logging of non-delayed responses + self.assertIn(b'\x00', client._recv(None)) + self.assertEqual(b'\x00', client._recv(1)) + + # Fool diagnostic logger into thinking we're running late, + # test logging of delayed responses + mock_diag_time.time.side_effect = count(step=3) + self.assertEqual(b'', client._recv(0)) + self.assertEqual(b'\x00' * 4, client._recv(4)) + + mock_socket = MagicMock() + mock_socket.recv.side_effect = iter([b'\x00', b'\x01', b'\x02']) + client.socket = mock_socket + client.timeout = 3 + self.assertEqual(b'\x00\x01\x02', client._recv(3)) + mock_socket.recv.side_effect = iter([b'\x00', b'\x01', b'\x02']) + self.assertEqual(b'\x00\x01', client._recv(2)) + mock_select.select.return_value = [False] + self.assertEqual(b'', client._recv(2)) + client.socket = mockSocket() + mock_select.select.return_value = [True] + self.assertIn(b'\x00', client._recv(None)) + + mock_socket = MagicMock() + mock_socket.recv.return_value = b'' + client.socket = mock_socket + self.assertRaises(ConnectionException, lambda: client._recv(1024)) + + mock_socket.recv.side_effect = iter([b'\x00', b'\x01', b'\x02', b'']) + client.socket = mock_socket + self.assertEqual(b'\x00\x01\x02', client._recv(1024)) + + def testTcpDiagClientRpr(self): + client = ModbusTcpDiagClient() + rep = "<{} at {} socket={}, ipaddr={}, port={}, timeout={}>".format( + client.__class__.__name__, hex(id(client)), client.socket, + client.host, client.port, client.timeout + ) + self.assertEqual(repr(client), rep) + + +# ---------------------------------------------------------------------------# +# Main +# ---------------------------------------------------------------------------# +if __name__ == "__main__": + unittest.main() diff --git a/test/test_datastore.py b/test/test_datastore.py index cd9c44d3a..03763b10a 100644 --- a/test/test_datastore.py +++ b/test/test_datastore.py @@ -98,9 +98,44 @@ def testModbusSparseDataBlock(self): block.setValues(0x00, dict(enumerate([False]*10))) self.assertEqual(block.getValues(0x00, 10), [False]*10) + block = ModbusSparseDataBlock({3: [10, 11, 12], 10: 1, 15: [0] * 4}) + self.assertEqual(block.values, {3: 10, 4: 11, 5: 12, 10: 1, + 15:0 , 16:0, 17:0, 18:0 }) + self.assertEqual(block.default_value, {3: 10, 4: 11, 5: 12, 10: 1, + 15:0 , 16:0, 17:0, 18:0 }) + self.assertEqual(block.mutable, True) + block.setValues(3, [20, 21, 22, 23], use_as_default=True) + self.assertEqual(block.getValues(3, 4), [20, 21, 22, 23]) + self.assertEqual(block.default_value, {3: 20, 4: 21, 5: 22, 6:23, 10: 1, + 15:0 , 16:0, 17:0, 18:0 }) + # check when values is a dict, address is ignored + block.setValues(0, {5: 32, 7: 43}) + self.assertEqual(block.getValues(5, 3), [32, 23, 43]) + + # assert value is empty dict when initialized without params + block = ModbusSparseDataBlock() + self.assertEqual(block.values, {}) + + # mark block as unmutable and see if parameter exeception + # is raised for invalid offset writes + block = ModbusSparseDataBlock({1: 100}, mutable=False) + self.assertRaises(ParameterException, block.setValues, 0, 1) + self.assertRaises(ParameterException, block.setValues, 0, {2: 100}) + self.assertRaises(ParameterException, block.setValues, 0, [1] * 10) + + # Reset datablock + block = ModbusSparseDataBlock({3: [10, 11, 12], 10: 1, 15: [0] * 4}) + block.setValues(0, {3: [20, 21, 22], 10: 11, 15: [10] * 4}) + self.assertEqual(block.values, {3: 20, 4: 21, 5: 22, 10: 11, + 15: 10 ,16:10, 17:10, 18:10 }) + block.reset() + self.assertEqual(block.values, {3: 10, 4: 11, 5: 12, 10: 1, + 15: 0, 16: 0, 17: 0, 18: 0}) + + def testModbusSparseDataBlockFactory(self): ''' Test the sparse data block store factory ''' - block = ModbusSparseDataBlock.create() + block = ModbusSparseDataBlock.create([0x00]*65536) self.assertEqual(block.getValues(0x00, 65536), [False]*65536) def testModbusSparseDataBlockOther(self): @@ -109,6 +144,7 @@ def testModbusSparseDataBlockOther(self): self.assertRaises(ParameterException, lambda: ModbusSparseDataBlock(True)) + def testModbusSlaveContext(self): ''' Test a modbus slave context ''' store = { diff --git a/test/test_framers.py b/test/test_framers.py index 520409fc6..d70dac3b9 100644 --- a/test/test_framers.py +++ b/test/test_framers.py @@ -8,9 +8,9 @@ from pymodbus.exceptions import ModbusIOException from pymodbus.compat import IS_PYTHON3 if IS_PYTHON3: - from unittest.mock import Mock + from unittest.mock import Mock, patch else: # Python 2 - from mock import Mock + from mock import Mock, patch @pytest.fixture @@ -44,7 +44,7 @@ def test_framer_initialization(framer): assert framer._start == b':' assert framer._end == b"\r\n" elif isinstance(framer, ModbusRtuFramer): - assert framer._header == {'uid': 0x00, 'len': 0, 'crc': '0000'} + assert framer._header == {'uid': 0x00, 'len': 0, 'crc': b'\x00\x00'} assert framer._hsize == 0x01 assert framer._end == b'\x0d\x0a' assert framer._min_frame_size == 4 @@ -64,47 +64,78 @@ def test_decode_data(rtu_framer, data): assert decoded == expected -@pytest.mark.parametrize("data", [(b'', False), - (b'\x02\x01\x01\x00Q\xcc', True)]) +@pytest.mark.parametrize("data", [ + (b'', False), + (b'\x02\x01\x01\x00Q\xcc', True), + (b'\x11\x03\x06\xAE\x41\x56\x52\x43\x40\x49\xAD', True), # valid frame + (b'\x11\x03\x06\xAE\x41\x56\x52\x43\x40\x49\xAC', False), # invalid frame CRC +]) def test_check_frame(rtu_framer, data): data, expected = data rtu_framer._buffer = data assert expected == rtu_framer.checkFrame() -@pytest.mark.parametrize("data", [b'', b'abcd']) -def test_advance_framer(rtu_framer, data): - rtu_framer._buffer = data +@pytest.mark.parametrize("data", [ + (b'', {'uid': 0x00, 'len': 0, 'crc': b'\x00\x00'}, b''), + (b'abcd', {'uid': 0x00, 'len': 2, 'crc': b'\x00\x00'}, b'cd'), + (b'\x11\x03\x06\xAE\x41\x56\x52\x43\x40\x49\xAD\x12\x03', # real case, frame size is 11 + {'uid': 0x00, 'len': 11, 'crc': b'\x00\x00'}, b'\x12\x03'), +]) +def test_rtu_advance_framer(rtu_framer, data): + before_buf, before_header, after_buf = data + + rtu_framer._buffer = before_buf + rtu_framer._header = before_header rtu_framer.advanceFrame() - assert rtu_framer._header == {} - assert rtu_framer._buffer == data + assert rtu_framer._header == {'uid': 0x00, 'len': 0, 'crc': b'\x00\x00'} + assert rtu_framer._buffer == after_buf @pytest.mark.parametrize("data", [b'', b'abcd']) -def test_reset_framer(rtu_framer, data): +def test_rtu_reset_framer(rtu_framer, data): rtu_framer._buffer = data rtu_framer.resetFrame() - assert rtu_framer._header == {} + assert rtu_framer._header == {'uid': 0x00, 'len': 0, 'crc': b'\x00\x00'} assert rtu_framer._buffer == b'' @pytest.mark.parametrize("data", [ (b'', False), + (b'\x11', False), + (b'\x11\x03', False), (b'\x11\x03\x06', False), (b'\x11\x03\x06\xAE\x41\x56\x52\x43\x40\x49', False), (b'\x11\x03\x06\xAE\x41\x56\x52\x43\x40\x49\xAD', True), - (b'\x11\x03\x06\xAE\x41\x56\x52\x43\x40\x49\xAD\xAB\xCD', True) + (b'\x11\x03\x06\xAE\x41\x56\x52\x43\x40\x49\xAD\xAB\xCD', True), ]) def test_is_frame_ready(rtu_framer, data): data, expected = data rtu_framer._buffer = data - rtu_framer.advanceFrame() + # rtu_framer.advanceFrame() assert rtu_framer.isFrameReady() == expected -def test_populate_header(rtu_framer): - rtu_framer.populateHeader(b'abcd') - assert rtu_framer._header == {'crc': b'd', 'uid': 97, 'len': 5} +@pytest.mark.parametrize("data", [ + b'', + b'\x11', + b'\x11\x03', + b'\x11\x03\x06', + b'\x11\x03\x06\xAE\x41\x56\x52\x43\x40\x43', +]) +def test_rtu_populate_header_fail(rtu_framer, data): + with pytest.raises(IndexError): + rtu_framer.populateHeader(data) + + +@pytest.mark.parametrize("data", [ + (b'\x11\x03\x06\xAE\x41\x56\x52\x43\x40\x49\xAD', {'crc': b'\x49\xAD', 'uid': 17, 'len': 11}), + (b'\x11\x03\x06\xAE\x41\x56\x52\x43\x40\x49\xAD\x11\x03', {'crc': b'\x49\xAD', 'uid': 17, 'len': 11}) +]) +def test_rtu_populate_header(rtu_framer, data): + buffer, expected = data + rtu_framer.populateHeader(buffer) + assert rtu_framer._header == expected def test_add_to_frame(rtu_framer): @@ -126,12 +157,26 @@ def test_populate_result(rtu_framer): assert result.unit_id == 255 -@pytest.mark.parametrize('framer', [ascii_framer, rtu_framer, binary_framer]) -def test_process_incoming_packet(framer): - def cb(res): - return res - # data = b'' - # framer.processIncomingPacket(data, cb, unit=1, single=False) +@pytest.mark.parametrize("data", [ + (b'\x11', 17, False, False), # not complete frame + (b'\x11\x03', 17, False, False), # not complete frame + (b'\x11\x03\x06', 17, False, False), # not complete frame + (b'\x11\x03\x06\xAE\x41\x56\x52\x43', 17, False, False), # not complete frame + (b'\x11\x03\x06\xAE\x41\x56\x52\x43\x40', 17, False, False), # not complete frame + (b'\x11\x03\x06\xAE\x41\x56\x52\x43\x40\x49', 17, False, False), # not complete frame + (b'\x11\x03\x06\xAE\x41\x56\x52\x43\x40\x49\xAC', 17, True, False), # bad crc + (b'\x11\x03\x06\xAE\x41\x56\x52\x43\x40\x49\xAD', 17, False, True), # good frame + (b'\x11\x03\x06\xAE\x41\x56\x52\x43\x40\x49\xAD', 16, True, False), # incorrect unit id + (b'\x11\x03\x06\xAE\x41\x56\x52\x43\x40\x49\xAD\x11\x03', 17, False, True), # good frame + part of next frame +]) +def test_rtu_process_incoming_packet(rtu_framer, data): + buffer, units, reset_called, process_called = data + + with patch.object(rtu_framer, '_process') as mock_process, \ + patch.object(rtu_framer, 'resetFrame') as mock_reset: + rtu_framer.processIncomingPacket(buffer, Mock(), units) + assert mock_process.call_count == (1 if process_called else 0) + assert mock_reset.call_count == (1 if reset_called else 0) def test_build_packet(rtu_framer): diff --git a/test/test_server_async.py b/test/test_server_async.py index 4fe04add3..c88f6e4db 100644 --- a/test/test_server_async.py +++ b/test/test_server_async.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from pymodbus.compat import IS_PYTHON3 import unittest +import pytest if IS_PYTHON3: # Python 3 from unittest.mock import patch, Mock, MagicMock else: # Python 2 @@ -32,6 +33,11 @@ IS_HIGH_SIERRA_OR_ABOVE = False SERIAL_PORT = "/dev/ptmx" +no_twisted_serial_on_windows_with_pypy = pytest.mark.skipif( + sys.platform == 'win32' and platform.python_implementation() == 'PyPy', + reason='Twisted serial requires pywin32 which is not compatible with PyPy', +) + class AsynchronousServerTest(unittest.TestCase): ''' @@ -188,6 +194,7 @@ def testUdpServerStartup(self): self.assertEqual(mock_reactor.listenUDP.call_count, 1) self.assertEqual(mock_reactor.run.call_count, 1) + @no_twisted_serial_on_windows_with_pypy @patch("twisted.internet.serialport.SerialPort") def testSerialServerStartup(self, mock_sp): ''' Test that the modbus serial asynchronous server starts correctly ''' @@ -195,6 +202,7 @@ def testSerialServerStartup(self, mock_sp): StartSerialServer(context=None, port=SERIAL_PORT) self.assertEqual(mock_reactor.run.call_count, 1) + @no_twisted_serial_on_windows_with_pypy @patch("twisted.internet.serialport.SerialPort") def testStopServerFromMainThread(self, mock_sp): """ @@ -207,6 +215,7 @@ def testStopServerFromMainThread(self, mock_sp): StopServer() self.assertEqual(mock_reactor.stop.call_count, 1) + @no_twisted_serial_on_windows_with_pypy @patch("twisted.internet.serialport.SerialPort") def testStopServerFromThread(self, mock_sp): """ diff --git a/test/test_server_asyncio.py b/test/test_server_asyncio.py index 42cb67ad4..c0b9b647d 100755 --- a/test/test_server_asyncio.py +++ b/test/test_server_asyncio.py @@ -72,6 +72,7 @@ def tearDown(self): # Test ModbusConnectedRequestHandler #-----------------------------------------------------------------------# @asyncio.coroutine + @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") def testStartTcpServer(self): ''' Test that the modbus tcp asyncio server starts correctly ''' identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) @@ -99,6 +100,7 @@ def testTcpServerServeForever(self): serve.assert_awaited() @asyncio.coroutine + @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") def testTcpServerServeForeverTwice(self): ''' Call on serve_forever() twice should result in a runtime error ''' server = yield from StartTcpServer(context=self.context,address=("127.0.0.1", 0), loop=self.loop) @@ -112,6 +114,7 @@ def testTcpServerServeForeverTwice(self): server.server_close() @asyncio.coroutine + @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") def testTcpServerReceiveData(self): ''' Test data sent on socket is received by internals - doesn't not process data ''' data = b'\x01\x00\x00\x00\x00\x06\x01\x03\x00\x00\x00\x19' @@ -146,6 +149,7 @@ def eof_received(self): server.server_close() @asyncio.coroutine + @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") def testTcpServerRoundtrip(self): ''' Test sending and receiving data on tcp socket ''' data = b"\x01\x00\x00\x00\x00\x06\x01\x03\x00\x00\x00\x01" # unit 1, read register @@ -186,6 +190,7 @@ def eof_received(self): server.server_close() @asyncio.coroutine + @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") def testTcpServerConnectionLost(self): ''' Test tcp stream interruption ''' data = b"\x01\x00\x00\x00\x00\x06\x01\x01\x00\x00\x00\x01" @@ -211,7 +216,9 @@ def connection_made(self, transport): transport, protocol = yield from self.loop.create_connection(BasicClient, host='127.0.0.1', port=random_port) yield from step1 - # await asyncio.sleep(1) + # On Windows we seem to need to give this an extra chance to finish, + # otherwise there ends up being an active connection at the assert. + yield from asyncio.sleep(0.0) self.assertTrue(len(server.active_connections) == 1) protocol.transport.close() # close isn't synchronous and there's no notification that it's done @@ -222,6 +229,7 @@ def connection_made(self, transport): server.server_close() @asyncio.coroutine + @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") def testTcpServerCloseActiveConnection(self): ''' Test server_close() while there are active TCP connections ''' data = b"\x01\x00\x00\x00\x00\x06\x01\x01\x00\x00\x00\x01" @@ -246,6 +254,9 @@ def connection_made(self, transport): transport, protocol = yield from self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) yield from step1 + # On Windows we seem to need to give this an extra chance to finish, + # otherwise there ends up being an active connection at the assert. + yield from asyncio.sleep(0.0) server.server_close() # close isn't synchronous and there's no notification that it's done @@ -254,6 +265,7 @@ def connection_made(self, transport): self.assertTrue( len(server.active_connections) == 0 ) @asyncio.coroutine + @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") def testTcpServerNoSlave(self): ''' Test unknown slave unit exception ''' context = ModbusServerContext(slaves={0x01: self.store, 0x02: self.store }, single=False) @@ -290,6 +302,7 @@ def eof_received(self): server.server_close() @asyncio.coroutine + @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") def testTcpServerModbusError(self): ''' Test sending garbage data on a TCP socket should drop the connection ''' data = b"\x01\x00\x00\x00\x00\x06\x01\x03\x00\x00\x00\x01" # get slave 5 function 3 (holding register) @@ -329,6 +342,7 @@ def eof_received(self): server.server_close() @asyncio.coroutine + @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") def testTcpServerInternalException(self): ''' Test sending garbage data on a TCP socket should drop the connection ''' data = b"\x01\x00\x00\x00\x00\x06\x01\x03\x00\x00\x00\x01" # get slave 5 function 3 (holding register) @@ -373,6 +387,7 @@ def eof_received(self): # Test ModbusTlsProtocol #-----------------------------------------------------------------------# @asyncio.coroutine + @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") def testStartTlsServer(self): ''' Test that the modbus tls asyncio server starts correctly ''' with patch.object(ssl.SSLContext, 'load_cert_chain') as mock_method: @@ -404,6 +419,7 @@ def testTlsServerServeForever(self): serve.assert_awaited() @asyncio.coroutine + @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") def testTlsServerServeForeverTwice(self): ''' Call on serve_forever() twice should result in a runtime error ''' with patch.object(ssl.SSLContext, 'load_cert_chain') as mock_method: @@ -423,6 +439,7 @@ def testTlsServerServeForeverTwice(self): #-----------------------------------------------------------------------# @asyncio.coroutine + @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") def testStartUdpServer(self): ''' Test that the modbus udp asyncio server starts correctly ''' identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) @@ -449,6 +466,7 @@ def testUdpServerServeForeverStart(self): serve.assert_awaited() @asyncio.coroutine + @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") def testUdpServerServeForeverClose(self): ''' Test StartUdpServer serve_forever() method ''' server = yield from StartUdpServer(context=self.context,address=("127.0.0.1", 0), loop=self.loop) @@ -465,6 +483,7 @@ def testUdpServerServeForeverClose(self): self.assertTrue(server.protocol.is_closing()) @asyncio.coroutine + @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") def testUdpServerServeForeverTwice(self): ''' Call on serve_forever() twice should result in a runtime error ''' identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) @@ -480,6 +499,7 @@ def testUdpServerServeForeverTwice(self): server.server_close() @asyncio.coroutine + @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") def testUdpServerReceiveData(self): ''' Test that the sending data on datagram socket gets data pushed to framer ''' server = yield from StartUdpServer(context=self.context,address=("127.0.0.1", 0),loop=self.loop) @@ -501,6 +521,7 @@ def testUdpServerReceiveData(self): server.server_close() @asyncio.coroutine + @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") def testUdpServerSendData(self): ''' Test that the modbus udp asyncio server correctly sends data outbound ''' identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) @@ -543,6 +564,7 @@ def datagram_received(self, data, addr): yield from asyncio.sleep(0.1) @asyncio.coroutine + @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") def testUdpServerRoundtrip(self): ''' Test sending and receiving data on udp socket''' data = b"\x01\x00\x00\x00\x00\x06\x01\x03\x00\x00\x00\x01" # unit 1, read register @@ -581,6 +603,7 @@ def datagram_received(self, data, addr): server.server_close() @asyncio.coroutine + @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") def testUdpServerException(self): ''' Test sending garbage data on a TCP socket should drop the connection ''' garbage = b'\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF' @@ -619,16 +642,19 @@ def datagram_received(self, data, addr): # -----------------------------------------------------------------------# # Test ModbusServerFactory # -----------------------------------------------------------------------# + @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") def testModbusServerFactory(self): ''' Test the base class for all the clients ''' with self.assertWarns(DeprecationWarning): factory = ModbusServerFactory(store=None) + @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") def testStopServer(self): with self.assertWarns(DeprecationWarning): StopServer() @asyncio.coroutine + @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") def testTcpServerException(self): ''' Sending garbage data on a TCP socket should drop the connection ''' garbage = b'\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF' @@ -669,6 +695,7 @@ def eof_received(self): @asyncio.coroutine + @pytest.mark.skipif(not IS_PYTHON3, reason="requires python3.4 or above") def testTcpServerException(self): ''' Sending garbage data on a TCP socket should drop the connection ''' garbage = b'\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF' diff --git a/test/test_server_sync.py b/test/test_server_sync.py index 74ba0cfa5..8b37b9ac2 100644 --- a/test/test_server_sync.py +++ b/test/test_server_sync.py @@ -260,13 +260,12 @@ def _callback(a, b): #-----------------------------------------------------------------------# def testTcpServerClose(self): ''' test that the synchronous TCP server closes correctly ''' - with patch.object(socket.socket, 'bind') as mock_socket: - identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) - server = ModbusTcpServer(context=None, identity=identity) - server.threads.append(Mock(**{'running': True})) - server.server_close() - self.assertEqual(server.control.Identity.VendorName, 'VendorName') - self.assertFalse(server.threads[0].running) + identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) + server = ModbusTcpServer(context=None, identity=identity, bind_and_activate=False) + server.threads.append(Mock(**{'running': True})) + server.server_close() + self.assertEqual(server.control.Identity.VendorName, 'VendorName') + self.assertFalse(server.threads[0].running) def testTcpServerProcess(self): ''' test that the synchronous TCP server processes requests ''' @@ -280,30 +279,33 @@ def testTcpServerProcess(self): #-----------------------------------------------------------------------# def testTlsServerInit(self): ''' test that the synchronous TLS server intial correctly ''' - with patch.object(socket.socket, 'bind') as mock_socket: + with patch.object(socketserver.TCPServer, 'server_activate'): with patch.object(ssl.SSLContext, 'load_cert_chain') as mock_method: identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) - server = ModbusTlsServer(context=None, identity=identity) + server = ModbusTlsServer(context=None, identity=identity, + bind_and_activate=False) + server.server_activate() self.assertIsNotNone(server.sslctx) self.assertEqual(type(server.socket), ssl.SSLSocket) server.server_close() sslctx = ssl.create_default_context() server = ModbusTlsServer(context=None, identity=identity, - sslctx=sslctx) + sslctx=sslctx, bind_and_activate=False) + server.server_activate() self.assertEqual(server.sslctx, sslctx) self.assertEqual(type(server.socket), ssl.SSLSocket) server.server_close() def testTlsServerClose(self): ''' test that the synchronous TLS server closes correctly ''' - with patch.object(socket.socket, 'bind') as mock_socket: - with patch.object(ssl.SSLContext, 'load_cert_chain') as mock_method: - identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) - server = ModbusTlsServer(context=None, identity=identity) - server.threads.append(Mock(**{'running': True})) - server.server_close() - self.assertEqual(server.control.Identity.VendorName, 'VendorName') - self.assertFalse(server.threads[0].running) + with patch.object(ssl.SSLContext, 'load_cert_chain') as mock_method: + identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) + server = ModbusTlsServer(context=None, identity=identity, + bind_and_activate=False) + server.threads.append(Mock(**{'running': True})) + server.server_close() + self.assertEqual(server.control.Identity.VendorName, 'VendorName') + self.assertFalse(server.threads[0].running) def testTlsServerProcess(self): ''' test that the synchronous TLS server processes requests ''' @@ -318,13 +320,14 @@ def testTlsServerProcess(self): #-----------------------------------------------------------------------# def testUdpServerClose(self): ''' test that the synchronous UDP server closes correctly ''' - with patch.object(socket.socket, 'bind') as mock_socket: - identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) - server = ModbusUdpServer(context=None, identity=identity) - server.threads.append(Mock(**{'running': True})) - server.server_close() - self.assertEqual(server.control.Identity.VendorName, 'VendorName') - self.assertFalse(server.threads[0].running) + identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) + server = ModbusUdpServer(context=None, identity=identity, + bind_and_activate=False) + server.server_activate() + server.threads.append(Mock(**{'running': True})) + server.server_close() + self.assertEqual(server.control.Identity.VendorName, 'VendorName') + self.assertFalse(server.threads[0].running) def testUdpServerProcess(self): ''' test that the synchronous UDP server processes requests ''' @@ -365,9 +368,9 @@ def testSerialServerServeForever(self): with patch('pymodbus.server.sync.CustomSingleRequestHandler') as mock_handler: server = ModbusSerialServer(None) instance = mock_handler.return_value - instance.handle.side_effect = server.server_close + instance.response_manipulator.side_effect = server.server_close server.serve_forever() - instance.handle.assert_any_call() + instance.response_manipulator.assert_any_call() def testSerialServerClose(self): ''' test that the synchronous serial server closes correctly ''' @@ -383,15 +386,13 @@ def testSerialServerClose(self): def testStartTcpServer(self): ''' Test the tcp server starting factory ''' with patch.object(ModbusTcpServer, 'serve_forever') as mock_server: - with patch.object(socketserver.TCPServer, 'server_bind') as mock_binder: - StartTcpServer() + StartTcpServer(bind_and_activate=False) def testStartTlsServer(self): ''' Test the tls server starting factory ''' with patch.object(ModbusTlsServer, 'serve_forever') as mock_server: - with patch.object(socketserver.TCPServer, 'server_bind') as mock_binder: - with patch.object(ssl.SSLContext, 'load_cert_chain') as mock_method: - StartTlsServer() + with patch.object(ssl.SSLContext, 'load_cert_chain') as mock_method: + StartTlsServer(bind_and_activate=False) def testStartUdpServer(self): ''' Test the udp server starting factory ''' diff --git a/test/test_transaction.py b/test/test_transaction.py old mode 100644 new mode 100755 index a3c469da1..1b34ca638 --- a/test/test_transaction.py +++ b/test/test_transaction.py @@ -1,6 +1,14 @@ #!/usr/bin/env python import pytest import unittest +from itertools import count +from pymodbus.compat import IS_PYTHON3 + +if IS_PYTHON3: # Python 3 + from unittest.mock import patch, Mock, MagicMock +else: # Python 2 + from mock import patch, Mock, MagicMock + from binascii import a2b_hex from pymodbus.pdu import * from pymodbus.transaction import * @@ -82,7 +90,10 @@ def testCalculateExceptionLength(self): self.assertEqual(self._tm._calculate_exception_length(), exception_length) - def testExecute(self): + @patch('pymodbus.transaction.time') + def testExecute(self, mock_time): + mock_time.time.side_effect = count() + client = MagicMock() client.framer = self._ascii client.framer._buffer = b'deadbeef' @@ -92,10 +103,16 @@ def testExecute(self): client.framer.buildPacket.return_value = b'deadbeef' client.framer.sendPacket = MagicMock() client.framer.sendPacket.return_value = len(b'deadbeef') - + client.framer.decode_data = MagicMock() + client.framer.decode_data.return_value = { + "unit": 1, + "fcode": 222, + "length": 27 + } request = MagicMock() request.get_response_pdu_size.return_value = 10 request.unit_id = 1 + request.function_code = 222 tm = ModbusTransactionManager(client) tm._recv = MagicMock(return_value=b'abcdef') self.assertEqual(tm.retries, 3) @@ -103,6 +120,7 @@ def testExecute(self): # tm._transact = MagicMock() # some response # tm._transact.return_value = (b'abcdef', None) + tm.getTransaction = MagicMock() tm.getTransaction.return_value = 'response' response = tm.execute(request) @@ -123,6 +141,15 @@ def testExecute(self): response = tm.execute(request) self.assertIsInstance(response, ModbusIOException) + # wrong handle_local_echo + tm._recv = MagicMock(side_effect=iter([b'abcdef', b'deadbe', b'123456'])) + client.handle_local_echo = True + tm.retry_on_empty = False + tm.retry_on_invalid = False + self.assertEqual(tm.execute(request).message, + '[Input/Output] Wrong local echo') + client.handle_local_echo = False + # retry on invalid response tm.retry_on_invalid = True tm._recv = MagicMock(side_effect=iter([b'', b'abcdef', b'deadbe', b'123456'])) @@ -136,6 +163,14 @@ def testExecute(self): client.framer.processIncomingPacket.side_effect = MagicMock(side_effect=ModbusIOException()) self.assertIsInstance(tm.execute(request), ModbusIOException) + # Broadcast + client.broadcast_enable = True + request.unit_id = 0 + response = tm.execute(request) + self.assertEqual(response, b'Broadcast write sent - ' + b'no response expected') + + # ----------------------------------------------------------------------- # # Dictionary based transaction manager # ----------------------------------------------------------------------- # @@ -455,7 +490,7 @@ def testRTUFramerTransactionReady(self): msg_parts = [b"\x00\x01\x00", b"\x00\x00\x01\xfc\x1b"] self._rtu.addToFrame(msg_parts[0]) - self.assertTrue(self._rtu.isFrameReady()) + self.assertFalse(self._rtu.isFrameReady()) self.assertFalse(self._rtu.checkFrame()) self._rtu.addToFrame(msg_parts[1]) diff --git a/tox.ini b/tox.ini index 909d6a74d..ccd8544ce 100644 --- a/tox.ini +++ b/tox.ini @@ -4,12 +4,42 @@ # directory. [tox] -envlist = py27, py35, py36, py37, pypy +envlist = py{27,py27,36,37,38,39,py36,py37} [testenv] deps = -r requirements-tests.txt -commands = py.test {posargs} -setenv = with_gmp=no +commands = + pytest {posargs:--cov=pymodbus/ --cov-report=term-missing --cov-report=xml} +passenv = + INCLUDE + LIB + PIP_* +setenv = + with_gmp=no + +[testenv:flake8] +deps = -r requirements-checks.txt +commands = + flake8 + +[testenv:docs] +allowlist_externals = + make +deps = -r requirements-docs.txt +commands = + make -C doc/ clean + make -C doc/ html + +[testenv:combined-coverage] +allowlist_externals = + ls +deps = + -r requirements-coverage.txt + -r requirements.txt +commands = + ls -la coverage_reports + coverage combine coverage_reports + coverage report --fail-under=85 --ignore-errors [flake8] exclude = .tox