Initial healthcheck implementation

- Use setuptools-scm for dynamic version
- Run FastAPI in a separate thread to provide liveness & readiness endpoints
- Querying the root returns the name and version information of the package
- Cleaned up dependencies
- Enable more rules on ruff linter
- Refactor project structure
- Remove redundant linting

.dockerignore

@@ -1,5 +1,3 @@
 *
-!app.py
-!endpoints/
-!requirements.txt
-!tests/
+!/parser
+!/requirements.txt

.gitignore

@@ -159,3 +159,6 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 .idea/
+
+# Version file generated by setuptools-scm
+parser/_version.py

.pre-commit-config.yaml

@@ -5,10 +5,6 @@ repos:
     - id: check-yaml
     - id: end-of-file-fixer
     - id: trailing-whitespace
-  - repo: https://github.com/psf/black
-    rev: 23.12.0
-    hooks:
-      - id: black
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     rev: 'v0.1.8'
     hooks:

Dockerfile

@@ -27,4 +27,5 @@ RUN chgrp -R 0 /home/app && \
 
 USER app
 
-CMD ["python", "./app.py"]
+EXPOSE 5000
+CMD ["python", "parser"]

README.md

@@ -1 +1,10 @@
 # Mittaridatapumppu parser
+
+## Development setup
+
+```shell
+python -m venv venv
+. venv/bin/activate
+pip-sync
+pre-commit install
+```

parser/__main__.py

@@ -0,0 +1,4 @@
+import parser
+
+if __name__ == "__main__":
+    parser.main()

app.py → parser/parser.py

@@ -1,9 +1,12 @@
+from contextlib import asynccontextmanager
 import datetime
 import importlib
 import json
 import logging
 import os
 import pathlib
+import uvicorn
+from multiprocessing import Process
 from pprint import pformat
 from zoneinfo import ZoneInfo
 
@@ -15,6 +18,21 @@
 from fvhiot.utils import init_script
 from fvhiot.utils.data import data_unpack, data_pack
 from fvhiot.utils.kafka import get_kafka_producer_by_envs, get_kafka_consumer_by_envs
+from _version import __version__
+from fastapi import FastAPI
+from sentry_asgi import SentryMiddleware
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    logging.info("Starting up.")
+    yield
+
+    logging.info("Shutting down.")
+
+
+app = FastAPI(lifespan=lifespan)
+app.add_middleware(SentryMiddleware)
 
 
 def backup_messages(raw_data_topic: str, msg):
@@ -167,7 +185,8 @@ def create_parsed_data_message(
         )  # create mapping for silly "0", "1", "2" named columns and real data keys
         for k in keys:
             col_name = str(col_cnt)  # "0", "1", "2" and so on
-            columns[col_name] = {"name": k}  # e.g. columns["0] = {"name" : "temp"}
+            # e.g. columns["0] = {"name" : "temp"}
+            columns[col_name] = {"name": k}
             col_map[k] = col_name  # e.g. col_map["temp"] = "0"
             col_cnt += 1
         for item in payload:  # create list of data items
@@ -195,7 +214,7 @@ def create_parsed_data_message(
     return parsed_data
 
 
-def main():
+def parser():
     init_script()
     raw_data_topic = os.getenv("KAFKA_RAW_DATA_TOPIC_NAME")
     parsed_data_topic = os.getenv("KAFKA_PARSED_DATA_TOPIC_NAME")
@@ -217,11 +236,13 @@ def main():
         logging.info(pformat(data))
         # if os.getenv("DEBUG"):
         #    backup_messages(raw_data_topic, msg)
-        data["raw_data_topic"] = raw_data_topic  # TODO: add this value in endpoint
+        # TODO: add this value in endpoint
+        data["raw_data_topic"] = raw_data_topic
         try:
             uplink_obj = get_uplink_obj(data)  # TODO: handle errors here
         except KeyError as err:
-            logging.warning(f"KeyError '{err}', message has no DevEUI_uplink key?")
+            logging.warning(
+                f"KeyError '{err}', message has no DevEUI_uplink key?")
             continue
         except ValidationError as err:
             logging.warning(f"ValidationError '{err}'")
@@ -271,8 +292,31 @@ def main():
             # TODO: send data to spare topic for future reprocessing?
 
 
-if __name__ == "__main__":
-    try:
-        main()
-    except KeyboardInterrupt:
-        print("Bye!")
+@app.get("/liveness")
+def liveness():
+    return "Liveness check completed"
+
+
+@app.get("/readiness")
+def readiness():
+    return "Readiness check completed"
+
+
+@app.get("/")
+def index():
+    return "{name} {version}".format(name=__package__, version=__version__)
+
+
+def run_fastapi():
+    uvicorn.run(app, host="0.0.0.0", port=5000)
+
+
+def main():
+    parser_process = Process(target=parser)
+    fastapi_process = Process(target=run_fastapi)
+
+    parser_process.start()
+    fastapi_process.start()
+
+    parser_process.join()
+    fastapi_process.terminate()

pyproject.toml

@@ -1,42 +1,41 @@
 [build-system]
-requires = ["setuptools"]
+requires = [
+    "setuptools>=60",
+    "setuptools-scm>=8.0"]
 build-backend = "setuptools.build_meta"
 
+[tool.setuptools_scm]
+version_file = "parser/_version.py"
+
 [tool.ruff]
 line-length = 120
-target-version = "py311"
+select = ["E", "F", "B", "Q"]
 
 [project]
 name = "mittaridatapumppu-parser"
-description = ""
+description = """\
+Python based microservice that consumes streaming data and converts it from \
+raw format to something else as defined in corresponding parser modules \
+obtained from device metadata \
+"""
 readme = "README.md"
 requires-python = ">=3.10"
-version = "v0.1.2"
+dynamic = ["version"]
 dependencies = [
   "fvhiot[kafka]@https://github.com/ForumViriumHelsinki/FVHIoT-python/releases/download/v1.0.2/FVHIoT-1.0.2-py3-none-any.whl",
-  "httpx",
-  "kafka-python",
-  "pydantic",
-  "sentry_sdk"
+  "fastapi ~= 0.109.0",
+  "kafka-python ~= 2.0.2",
+  "httpx ~= 0.26.0",
+  "pydantic ~= 2.5.3",
+  "sentry_asgi ~= 0.2.0",
+  "uvicorn ~= 0.27.0"
 ]
 
 [project.optional-dependencies]
 dev = [
-  "autoflake",
-  "autopep8",
-  "black",
-  "flake8",
-  "gunicorn",
-  "httpx",
-  "isort",
-  "pep8-naming",
-  "pre-commit",
-  "pydantic",
-  "pytest",
-  "pytest-asyncio",
-  "pytest-cov",
+  "pre-commit ~= 3.6.0",
 ]
 test = [
-    "pytest",
-    "requests"
-] #pip-compile --resolver=backtracking --extra test pyproject.toml
+  "ruff ~= 0.1.15",
+  "msgpack ~= 1.0.7"
+]