Merge pull request #61 from NebraLtd/marvinmarnold/mega-refactor

refactor: entire build, code, and docs

.github/workflows/python-tests.yml

@@ -1,5 +1,4 @@
-name: Python Flake8
-
+name: Python Unit Tests
 on: [push, pull_request]
 
 jobs:
@@ -13,9 +12,11 @@ jobs:
       uses: actions/setup-python@v1
       with:
         python-version: 3.8
-    - name: Setup flake8 annotations
-      uses: rbialon/flake8-annotations@v1
-    - name: Lint with flake8
+    - name: Run PyTest
       run: |
-        pip install flake8
-        flake8 . --count --max-complexity=10 --statistics
+        sudo apt-get update
+        sudo apt-get install libdbus-1-3 network-manager python3-gi libdbus-1-dev python3-gi-cairo gir1.2-gtk-3.0 dbus libusb-dev libdbus-1-dev libglib2.0-dev libudev-dev libical-dev libreadline-dev libgirepository1.0-dev python3-cairo-dev libcairo2 libcairo2-dev -y
+        pip install -r test-requirements.txt
+        pip install -r requirements.txt
+        export PYTHONPATH=`pwd`
+        pytest --cov=minerconfig --cov=lib --cov-fail-under=1

.gitignore

@@ -14,7 +14,6 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
 lib64/
 parts/
 sdist/

Dockerfile

@@ -1,40 +1,93 @@
-#Nebra Helium Hotspot - BTLE Configuration Software Container
-#(C) Nebra LTD. 2021
-#Licensed under the MIT License.
+# Nebra Helium Hotspot - BTLE Configuration Software Container
+# (C) Nebra LTD. 2021
+# Licensed under the MIT License.
 
-FROM balenalib/raspberry-pi-debian:buster-run
+ARG SYSTEM_TIMEZONE="Europe/London"
 
+####################################################################################################
+################################## Stage: builder ##################################################
+
+# The balenalib/raspberry-pi-debian-python image was tested but missed many dependencies.
+FROM balenalib/raspberry-pi-debian:buster-build-20210705 as builder
+
+# Nebra uses /opt by convention
 WORKDIR /opt/
 
+# Copy python dependencies for `pip install` later
 COPY requirements.txt requirements.txt
 
+# This will be the path that venv uses for installation below
+ENV PATH="/opt/venv/bin:$PATH"
+
+# Install python3-minimal, pip3, wget, venv.
+# Then set venv environment copied from builder.
+# Finally, use pip to install dependencies.
 RUN \
     apt-get update && \
     DEBIAN_FRONTEND="noninteractive" \
-    TZ="Europe/London" \
+    TZ="$SYSTEM_TIMEZONE" \
         apt-get -y install \
             python3-minimal=3.7.3-1 \
-            bluez=5.50-1.2~deb10u1+rpt2 \
-            libdbus-1-3=1.12.20-0+deb10u1 \
             python3-pip=18.1-5+rpt1 \
-            network-manager=1.14.6-2+deb10u1 \
-            python3-gi=3.30.4-1 \
             wget=1.20.1-1.1 \
+            python3-venv=3.7.3-1 \
+            # The remaining dependencies are for PyGObject
+            # https://pygobject.readthedocs.io/en/latest/getting_started.html#ubuntu-logo-ubuntu-debian-logo-debian
+            libgirepository1.0-dev=1.58.3-2 \
+            gcc=4:8.3.0-1+rpi2 \
+            libcairo2-dev=1.16.0-4+rpt1 \
+            pkg-config=0.29-6 \
+            python3-dev=3.7.3-1 \
+            gir1.2-gtk-3.0=3.24.5-1+rpt2 \
             --no-install-recommends && \
-        pip3 install --no-cache-dir -r requirements.txt &&\
-        apt-get purge python3-pip -y &&\
-        apt-get autoremove -y &&\
-        apt-get clean && \
-        rm -rf /var/lib/apt/lists/*
+    # Because the PATH is already updated above, this command creates a new venv AND activates it
+    python3 -m venv /opt/venv && \
+    # Given venv is active, this `pip` refers to the python3 variant
+    pip install --no-cache-dir -r requirements.txt
+
+# No need to cleanup the builder
+
+####################################################################################################
+################################### Stage: runner ##################################################
+
+FROM balenalib/raspberry-pi-debian-python:buster-run-20210705 as runner
+
+# Install bluez, libdbus, network-manager, python3-gi, and venv
+RUN \
+    apt-get update && \
+    DEBIAN_FRONTEND="noninteractive" \
+    TZ="$SYSTEM_TIMEZONE" \
+    apt-get install -y \
+        bluez=5.50-1.2~deb10u1+rpt2 \
+        libdbus-1-3=1.12.20-0+deb10u1 \
+        network-manager=1.14.6-2+deb10u1 \
+        python3-gi=3.30.4-1 \
+        python3-venv=3.7.3-1
 
+# Nebra uses /opt by convention
 WORKDIR /opt/
 
+# Copy the code and starter script
+COPY lib/ lib/
+COPY gatewayconfig/ gatewayconfig/
 COPY start-gateway-config.sh start-gateway-config.sh
+ENV PYTHONPATH="/opt:$PYTHONPATH"
 
-COPY config-python/ config-python/
+# Copy venv from builder and update PATH to activate it
+COPY --from=builder /opt/venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
 
-WORKDIR /opt/config-python/
+# Cleanup
+RUN apt-get autoremove -y && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
-RUN wget -q "https://raw.githubusercontent.com/NebraLtd/helium-hardware-definitions/577ce0ebca4349398480e01842aee4ad2662d2d4/variant_definitions.py"
+# START DEBUGGING
+# Uncomment the lines below to mock parts of the configuration
+# COPY example/ example/
+# ENV ONBOARDING_KEY_FILEPATH=/opt/example/device_keys.txt 
+# ENV ETH0_MAC_ADDRESS_PATH=/opt/example/eth0_mac_address.txt
+# END DEBUGGING
 
+# Run start-gateway-config script
 ENTRYPOINT ["sh", "/opt/start-gateway-config.sh"]

README.md

@@ -1,27 +1,84 @@
-# hm-config
-Helium Miner Config Container
+# hm-config: Helium Miner Config Container
 
 This repository contains the Dockerfile, basic scripts  and additional libraries required for the BTLE Application tool.
+[helium/gateway-config](https://github.com/helium/gateway-config) is the upstream repo that this is built against.
 
-Github then builds the docker containers ready to be pushed to the Nebra Hotspots.
+Directory layout:
 
-The base repository for the Python Application is in the subfolder config-python.
-
-## Wheels
-To resolve arm64 compiling with pip for now the repository includes arm64 wheels for the H3 Python library and the RPi.GPIO python library.
+- `.github/`: Github workflows and other settings.
+- `example/`: Files that are examples of what will be loaded on an actual hotspot. These files are especially useful for testing without a full hotspot.
+- `gatewayconfig/`: The main Python application.
+- `lib/`: Python files copied from other reposittories.
+- `protos/`: Protobuf definitions. Generated protos go to `gatewayconfgi/protos/` by default.
+- `tests/`: Test files.
 
 ## Local development environment
 
-**Note:** Right now the build process fails due to missing wheels.
+Running locally:
+
+```
+PYTHONPATH=./ MINER_KEYS_FILEPATH=./example/onboarding_key.txt ETH0_MAC_ADDRESS_PATH=./example/eth0_mac_address.txt python minerconfig
+```
 
 Because the stack is tightly intertwined with Balena, the easiest way to test the code base on your own Raspberry Pi in your own Balena project.
+The code has been developped and tested with the Raspberry Pi 3 B+. There are a few ways to build this app:
+
+1. Cross-compile locally and deploy to Balena: `balena deploy dev-XXX --build` (preferred method)
+2. Cross-compile locally only: `docker buildx build --platform linux/arm64 .`
+3. ARM build on Balena: `git push balena YourLocalBranch:master` (deprecated)
+4. Build directly on device with [local mode](https://www.balena.io/docs/learn/develop/local-mode/): `balena push local` (over 10 hours)
+
 
+```
+balena deploy hm-diag --build --debug
+```
+
+### Balena setup
 * Create a new Balena project for Raspberry Pi 3 (64 Bit)
 * Download and flash out the disk image provided and boot the device
+* Add the remote Balena repo (`git remote add balena [email protected]:BALENA_USERNAME/BALENA_PROJECT.git`)
+* The following ENV variables must be set: `FREQ`, `SENTRY_CONFIG`, `SENTRY_DIAG`, `SENTRY_PKTFWD`, and `VARIANT`
 * Add the remote Balena repo (`git remote add balena [email protected]:YourUser/YourProject.git`)
 
 You can now push your changes using the following command:
 
 ```
 $ git push balena YourLocalBranch:master
 ```
+
+### Setting up Python on Ubuntu
+
+These are optional instructions to have an Ubuntu environment closely mimic production.
+
+1. Install pyenv: `curl https://pyenv.run | bash`
+2. Install Python 3.7.3 dependency: 
+
+```
+sudo apt-get install -y libffi-dev libssl-dev make build-essential libssl-dev zlib1g-dev libbz2-dev \
+    libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
+    xz-utils tk-dev libffi-dev liblzma-dev python-openssl git \
+     libdbus-glib-1-dev libgirepository1.0-dev python3-gi bluez
+```
+3. Install Python 3.7.3: `pyenv install 3.7.3 && pyenv local 3.7.3`
+4. Check correctly installed: `python -V`
+5. Setup virtualenv: `python3 -m venv venv && source venv/bin/activate`
+6. Install dependencies: `pip install -r requirements.txt`
+
+## Testing
+
+Assuming virtualenv has been activated, execute the following command to run the tests:
+
+```
+pip install -r test-requirements.txt
+pytest
+# Or test and run coverage report, with failure under 50%
+PYTHONPATH=./ pytest --cov=minerconfig --cov=lib
+```
+
+## Generating protobufs
+
+- Install protobuf
+    - Ubuntu: `sudo snap install protobuf` 
+    - Mac: `brew install protobuf`
+- Run `generate-protos.sh`
+    - `cd protos && sh generate-protos.sh`