overhaul with CI/CD, envvar-renames and lots of fixes to make CI happy

.flake8

@@ -0,0 +1,22 @@
+# Generated from:
+# https://github.com/plone/meta/tree/master/config/default
+# See the inline comments on how to expand/tweak this configuration file
+[flake8]
+doctests = 1
+ignore =
+    # black takes care of line length
+    E501,
+    # black takes care of where to break lines
+    W503,
+    # black takes care of spaces within slicing (list[:])
+    E203,
+    # black takes care of spaces after commas
+    E231,
+
+##
+# Add extra configuration options in .meta.toml:
+#  [flake8]
+#  extra_lines = """
+#  _your own configuration lines_
+#  """
+##

.github/workflows/ci.yaml

@@ -0,0 +1,25 @@
+name: Python package CI
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11', '3.12']
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install tox tox-gh-actions
+    - name: Test with tox
+      run: tox

.gitignore

@@ -22,6 +22,7 @@ output.xml
 pip-selfcheck.json
 report.html
 .vscode/
+.tox
 .python-version
 reports/
 # excludes

.pre-commit-config.yaml

@@ -0,0 +1,42 @@
+ci:
+    autofix_prs: false
+    autoupdate_schedule: monthly
+
+repos:
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v3.15.0
+    hooks:
+    -   id: pyupgrade
+-   repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+    -   id: isort
+-   repo: https://github.com/psf/black
+    rev: 23.9.1
+    hooks:
+    -   id: black
+-   repo: https://github.com/PyCQA/flake8
+    rev: 6.1.0
+    hooks:
+    -   id: flake8
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.5.1
+    hooks:
+      - id: mypy
+        additional_dependencies:
+        - "types-requests"
+        - "pytest-stub"
+# -   repo: https://github.com/codespell-project/codespell
+#     rev: v2.2.5
+#     hooks:
+#     -   id: codespell
+#         additional_dependencies:
+#           - tomli
+-   repo: https://github.com/mgedmin/check-manifest
+    rev: "0.49"
+    hooks:
+    -   id: check-manifest
+-   repo: https://github.com/regebro/pyroma
+    rev: "4.2"
+    hooks:
+    -   id: pyroma

CHANGES.rst

@@ -15,6 +15,8 @@ Changelog
   [jensens]
 - Add docker-compose file to start OpensSearch to example directory and move `.env` to example too.
   [jensens]
+- rename `ELASTIC_*` environemnt variables to have an consistent naming scheme, see README for details. [jensens]
+- Add tox, Github Actions, CI and CD. [jensens]
 
 
 

CONTRIBUTORS.rst

@@ -1,5 +1,8 @@
 Contributors
 ============
 
-- Jens W. Klein, [email protected]
-- Katja Süss, Rohberg, @ksuess
+- Peter Holzer - Initiative, idea and testing.
+- Jens W. Klein, [email protected] - Concept & code.
+- Katja Süss, Rohberg, @ksuess - Text analysis code and configuration.
+
+Initial implementation was made possible by `Evangelisch-reformierte Landeskirche des Kantons Zürich <https://zhref.ch/>`_.

README.rst

@@ -41,27 +41,27 @@ Configuration is done via environment variables and JSON files.
 
 Environment variables are:
 
-ELASTICSEARCH_INGEST_SERVER
+INGEST_SERVER
     The URL of the ElasticSearch or OpenSearch server.
 
     Default: localhost:9200
 
-ELASTICSEARCH_INGEST_USE_SSL
-    Wether to use a secure connection or not.
+INGEST_USE_SSL
+    Whether to use a secure connection or not.
 
     Default: 0
 
-OPENSEARCH
-    Wether to use OpenSearch or ElasticSearch.
+INGEST_OPENSEARCH
+    Whether to use OpenSearch or ElasticSearch.
 
     Default: 1
 
-ELASTICSEARCH_INGEST_LOGIN
+INGEST_LOGIN
     Username for the ElasticSearch 8+ or OpenSearch server.
 
     Default: admin
 
-ELASTICSEARCH_INGEST_PASSWORD
+INGEST_PASSWORD
     Password for the ElasticSearch 8+ or OpenSearch server.
 
     Default: admin
@@ -139,8 +139,7 @@ A docker-compose file ``docker-compose.yml`` to start an OpenSearch server is pr
 
 Precondition:
 - Docker and docker-compose are installed.
-- Max virtual memory map needs increase to run this: `sudo sysctl -w vm.max_map_count=262144`
-  (not permanent, `see StackOverflow post <https://stackoverflow.com/questions/66444027/max-virtual-memory-areas-vm-max-map-count-65530-is-too-low-increase-to-at-lea>`_).
+- Max virtual memory map needs increase to run this: `sudo sysctl -w vm.max_map_count=262144` (not permanent, `see StackOverflow post <https://stackoverflow.com/questions/66444027/max-virtual-memory-areas-vm-max-map-count-65530-is-too-low-increase-to-at-lea>`_).
 
 Enter the directory ``examples`` and start the server with ``docker-compose up``.
 Now you have an OpenSearch server running on ``http://localhost:9200`` and an OpenSearch Dashboard running on ``http://localhost:5601`` (user/pass: admin/admin).
@@ -245,39 +244,17 @@ We appreciate any contribution and if a release is needed to be done on pypi, pl
 We also offer commercial support if any training, coaching, integration or adaptions are needed.
 
 
--------------
-Contributions
--------------
-
-Initial implementation was made possible by `Evangelisch-reformierte Landeskirche des Kantons Zürich <https://zhref.ch/>`_.
-
-Idea and testing by Peter Holzer
-
-Concept & code by Jens W. Klein
-
-Text analysis code and configuration by Katja Süss
-
-
-
 ----------------------------
 Installation for development
 ----------------------------
 
 - clone source code repository,
 - enter repository directory
 - recommended: create a virtualenv ``python -mvenv env``
-- development install ``./bin/env/pip install -e .[redis,opensearch]``
+- development install ``./bin/env/pip install -e .[test,redis,opensearch]``
 - load environment configuration ``source examples/.env``.
 
 
-----
-Todo
-----
-
-- query status of a task
-- simple statistics about tasks-count: pending, done, errored
-- celery retry on failure, i.e. restart of ElasticSearch, Plone, ...
-
 -------
 License
 -------

examples/.env

@@ -5,11 +5,11 @@
 # Then `source .env` and start the ingest-service with: celery -A collective.elastic.ingest.celery.app worker -l debug
 export CELERY_BROKER=redis://localhost:6379/0
 
-export ELASTICSEARCH_INGEST_SERVER=localhost:9200
-export ELASTICSEARCH_INGEST_USE_SSL=1
-export OPENSEARCH=1
-export ELASTICSEARCH_INGEST_LOGIN=admin
-export ELASTICSEARCH_INGEST_PASSWORD=admin
+export INGEST_SERVER=localhost:9200
+export INGEST_USE_SSL=1
+export INGEST_OPENSEARCH=1
+export INGEST_LOGIN=admin
+export INGEST_PASSWORD=admin
 
 export PLONE_SERVICE=http://localhost:8080
 export PLONE_PATH=Plone

mypy.ini

@@ -0,0 +1,2 @@
+[mypy]
+mypy_path=./src

pyproject.toml

@@ -0,0 +1,90 @@
+[project]
+name = "collective.elastic.ingest"
+version = "2.0.0dev0"
+description = "Ingestion service queue runner between Plone RestAPI and ElasticSearch or OpenSearch."
+keywords = ["elasticsearch", "opensearch", "plone", "celery", "search", "indexer"]
+readme = "README.rst"
+
+authors = [
+    {name = "Jens Klein", email = "[email protected]"},
+    {name = "Katja Süss"},
+    {name = "Peter Holzer"},
+]
+requires-python = ">=3.8"
+license = { text = "GPL 2.0" }
+classifiers = [
+    "Environment :: Web Environment",
+    "Framework :: Plone",
+    "Framework :: Plone :: Addon",
+    "Framework :: Plone :: 5.2",
+    "Framework :: Plone :: 6.0",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Operating System :: OS Independent",
+    "License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
+    "Development Status :: 5 - Production/Stable",
+]
+dependencies = [
+    "CacheControl",
+    "celery",
+    "requests",
+    "setuptools",
+]
+
+[project.urls]
+PyPI = "https://pypi.python.org/pypi/collective.elastic.ingest"
+Changelog = "https://github.com/collective/collective.elastic.ingest/blob/main/CHANGES.rst"
+Source = "https://github.com/collective/collective.elastic.ingest"
+Issues = "https://github.com/collective/collective.elastic.ingest/issues"
+
+[project.optional-dependencies]
+redis = ["celery[redis]"]
+rabbitmq = ["celery[librabbitmq]"]
+opensearch = ["opensearch-py"]
+elasticsearch = ["elasticsearch>=8.0"]
+test = [
+    "pytest",
+    "requests-mock",
+    "pdbpp",
+]
+
+[build-system]
+requires = ["setuptools>=61"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+minversion = "6.0"
+testpaths = [
+    "src",
+]
+
+[tool.isort]
+profile = "plone"
+
+[tool.black]
+include = "src"
+
+[tool.codespell]
+ignore-words-list = "discreet,"
+skip = './examples/*,./venv/*'
+
+[tool.check-manifest]
+ignore = [
+    ".editorconfig",
+    ".pre-commit-config.yaml",
+    "tox.ini",
+    "mypy.ini",
+    ".flake8",
+    "mx.ini",
+
+]
+
+[zest.releaser]
+create-wheel = true