New features, refactoring and replacement of Flask by Quart

.editorconfig

@@ -0,0 +1,21 @@
+# EditorConfig is awesome: https://EditorConfig.org
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file
+
+[*]
+# Unix-style newlines
+end_of_line = lf
+# Always end with an empty new line
+insert_final_newline = true
+# Set default charset to utf-8
+charset = utf-8
+# Indent with 2 spaces
+indent_style = space
+indent_size = 2
+
+# 4 space indentation
+[*.py]
+indent_style = space
+indent_size = 4

.gitignore

@@ -1 +1,7 @@
 config.ini
+
+# ignore venv
+**/.venv
+
+# ignore python cache
+**/__pycache__

Dockerfile

@@ -1,7 +1,7 @@
 FROM python:alpine
 WORKDIR /usr/src/app
 RUN apk update
-RUN apk add git rsync
+RUN apk add git rsync terraform
 RUN apk add docker
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt

config.ini

@@ -2,6 +2,8 @@
 # ALLOWED_IP_RANGE = 192.168.0.0/24
 # DEBUG = true
 # GIT_SSL_NO_VERIFY = true
+# GIT_SSH_NO_VERIFY = true
+# GIT_PROTOCOL = http
 # LISTEN_PORT = 1706
 
 [rsync]

requirements.txt

@@ -1,2 +1 @@
-Flask>=2.1.0
-waitress>=2.1.0
+quart==0.18.3

runner/__init__.py

@@ -0,0 +1,123 @@
+import logging
+from ipaddress import ip_address, ip_network
+from os import access, environ, X_OK
+from shutil import which
+
+from quart import Quart, request, jsonify
+
+
+def create_app(config):
+    print("Tea Runner")
+    # Configure loglevel
+    if config.getboolean("runner", "DEBUG", fallback="False") == True:
+        logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.DEBUG)
+        logging.info("Debug logging is on")
+    else:
+        logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO)
+
+    app = Quart(__name__)
+
+    # Configure Quart
+    app.runner_config = config
+    app.git_protocol = app.runner_config.get("runner", "GIT_PROTOCOL", fallback="http")
+
+    # Check presence of external programs
+    app.docker = which("docker")
+    try:
+        access(app.docker, X_OK)
+    except:
+        logging.error("docker binary not found or not executable")
+        exit(1)
+
+    app.git = which("git")
+    try:
+        access(app.git, X_OK)
+    except:
+        logging.error("git binary not found or not executable")
+        exit(1)
+
+    app.rsync = which("rsync")
+    try:
+        access(app.rsync, X_OK)
+    except:
+        logging.error("rsync binary not found or not executable")
+        exit(1)
+
+    app.tf_bin = which("terraform")
+    try:
+        access(app.tf_bin, X_OK)
+    except:
+        logging.error("terraform binary not found or not executable")
+        exit(1)
+
+    # Set environment variables
+    if (
+        app.runner_config.getboolean("runner", "GIT_SSL_NO_VERIFY", fallback="False")
+        == True
+    ):
+        environ["GIT_SSL_NO_VERIFY"] = "true"
+    if (
+        app.runner_config.getboolean("runner", "GIT_SSH_NO_VERIFY", fallback="False")
+        == True
+    ):
+        environ[
+            "GIT_SSH_COMMAND"
+        ] = "ssh -o UserKnownHostsFile=test -o StrictHostKeyChecking=no"
+
+    # Log some informations
+    logging.info("git protocol is " + app.git_protocol)
+    logging.info(
+        "Limiting requests to: "
+        + app.runner_config.get("runner", "ALLOWED_IP_RANGE", fallback="<any>")
+    )
+
+    @app.before_request
+    async def check_authorized():
+        """
+        Only respond to requests from ALLOWED_IP_RANGE if it's configured in config.ini
+        """
+        if app.runner_config.has_option("runner", "ALLOWED_IP_RANGE"):
+            allowed_ip_range = ip_network(
+                app.runner_config["runner"]["ALLOWED_IP_RANGE"]
+            )
+            requesting_ip = ip_address(request.remote_addr)
+            if requesting_ip not in allowed_ip_range:
+                logging.info(
+                    "Dropping request from unauthorized host " + request.remote_addr
+                )
+                return jsonify(status="forbidden"), 403
+            else:
+                logging.info("Request from " + request.remote_addr)
+
+    @app.before_request
+    async def check_media_type():
+        """
+        Only respond requests with Content-Type header of application/json
+        """
+        if (
+            not request.headers.get("Content-Type")
+            .lower()
+            .startswith("application/json")
+        ):
+            logging.error(
+                '"Content-Type: application/json" header missing from request made by '
+                + request.remote_addr
+            )
+            return jsonify(status="unsupported media type"), 415
+
+    @app.route("/test", methods=["POST"])
+    async def test():
+        logging.debug("Content-Type: " + request.headers.get("Content-Type"))
+        logging.debug(await request.get_json(force=True))
+        return jsonify(status="success", sender=request.remote_addr)
+
+    # Register Blueprints
+    from runner.docker import docker as docker_bp
+    from runner.rsync import rsync as rsync_bp
+    from runner.terraform import terraform as terraform_bp
+
+    app.register_blueprint(docker_bp, url_prefix="/docker")
+    app.register_blueprint(rsync_bp)
+    app.register_blueprint(terraform_bp, url_prefix="/terraform")
+
+    return app

runner/docker.py

@@ -0,0 +1,38 @@
+import logging
+from os import chdir, getcwd
+from subprocess import run, DEVNULL
+from tempfile import TemporaryDirectory
+
+from quart import Blueprint, current_app, jsonify, request
+
+import runner.utils
+
+docker = Blueprint("docker", __name__)
+
+
+@docker.route("/build", methods=["POST"])
+async def docker_build():
+    body = await request.get_json()
+
+    with TemporaryDirectory() as temp_dir:
+        current_dir = getcwd()
+        if runner.utils.git_clone(
+            body["repository"]["clone_url"]
+            if current_app.git_protocol == "http"
+            else body["repository"]["ssh_url"],
+            temp_dir,
+        ):
+            logging.info("docker build")
+            chdir(temp_dir)
+            result = run(
+                [current_app.docker, "build", "-t", body["repository"]["name"], "."],
+                stdout=None if logging.root.level == logging.DEBUG else DEVNULL,
+                stderr=None if logging.root.level == logging.DEBUG else DEVNULL,
+            )
+            chdir(current_dir)
+            if result.returncode != 0:
+                return jsonify(status="docker build failed"), 500
+        else:
+            return jsonify(status="git clone failed"), 500
+
+    return jsonify(status="success")

runner/rsync.py

@@ -0,0 +1,60 @@
+import logging
+from os import chdir, getcwd, path
+from subprocess import run, DEVNULL
+from tempfile import TemporaryDirectory
+
+from quart import Blueprint, current_app, jsonify, request
+from werkzeug import utils
+
+import runner.utils
+
+rsync = Blueprint("rsync", __name__)
+
+
+@rsync.route("/rsync", methods=["POST"])
+async def route_rsync():
+    body = await request.get_json()
+    dest = request.args.get("dest") or body["repository"]["name"]
+    rsync_root = current_app.runner_config.get("rsync", "RSYNC_ROOT", fallback="")
+    if rsync_root:
+        dest = path.join(rsync_root, utils.secure_filename(dest))
+        logging.debug("rsync dest path updated to " + dest)
+
+    with TemporaryDirectory() as temp_dir:
+        current_dir = getcwd()
+        if runner.utils.git_clone(
+            body["repository"]["clone_url"]
+            if current_app.git_protocol == "http"
+            else body["repository"]["ssh_url"],
+            temp_dir,
+        ):
+            logging.info("rsync " + body["repository"]["name"] + " to " + dest)
+            chdir(temp_dir)
+            if current_app.runner_config.get("rsync", "DELETE", fallback=""):
+                result = run(
+                    [
+                        current_app.rsync,
+                        "-r",
+                        "--exclude=.git",
+                        "--delete-during"
+                        if current_app.runner_config.get("rsync", "DELETE", fallback="")
+                        else "",
+                        ".",
+                        dest,
+                    ],
+                    stdout=None if logging.root.level == logging.DEBUG else DEVNULL,
+                    stderr=None if logging.root.level == logging.DEBUG else DEVNULL,
+                )
+            else:
+                result = run(
+                    [current_app.rsync, "-r", "--exclude=.git", ".", dest],
+                    stdout=None if logging.root.level == logging.DEBUG else DEVNULL,
+                    stderr=None if logging.root.level == logging.DEBUG else DEVNULL,
+                )
+            chdir(current_dir)
+            if result.returncode != 0:
+                return jsonify(status="rsync failed"), 500
+        else:
+            return jsonify(status="git clone failed"), 500
+
+    return jsonify(status="success")

runner/terraform.py

@@ -0,0 +1,81 @@
+import logging
+from os import chdir, getcwd
+from subprocess import run, DEVNULL
+from tempfile import TemporaryDirectory
+
+from quart import Blueprint, current_app, jsonify, request
+
+import runner.utils
+
+terraform = Blueprint("terraform", __name__)
+
+
+@terraform.route("/plan", methods=["POST"])
+async def terraform_plan():
+    body = await request.get_json()
+
+    with TemporaryDirectory() as temp_dir:
+        current_dir = getcwd()
+        if runner.utils.git_clone(
+            body["repository"]["clone_url"]
+            if current_app.git_protocol == "http"
+            else body["repository"]["ssh_url"],
+            temp_dir,
+        ):
+            logging.info("terraform init")
+            chdir(temp_dir)
+            result = run(
+                [current_app.tf_bin, "init", "-no-color"],
+                stdout=None if logging.root.level == logging.DEBUG else DEVNULL,
+                stderr=None if logging.root.level == logging.DEBUG else DEVNULL,
+            )
+            if result.returncode != 0:
+                chdir(current_dir)
+                return jsonify(status="terraform init failed"), 500
+            result = run(
+                [current_app.tf_bin, "plan", "-no-color"], stdout=None, stderr=None
+            )
+            if result.returncode != 0:
+                chdir(current_dir)
+                return jsonify(status="terraform plan failed"), 500
+        else:
+            chdir(current_dir)
+            return jsonify(status="git clone failed"), 500
+        chdir(current_dir)
+    return jsonify(status="success")
+
+
+@terraform.route("/apply", methods=["POST"])
+def terraform_apply():
+    body = request.get_json()
+    with TemporaryDirectory() as temp_dir:
+        current_dir = getcwd()
+        if runner.utils.git_clone(
+            body["repository"]["clone_url"]
+            if current_app.git_protocol == "http"
+            else body["repository"]["ssh_url"],
+            temp_dir,
+        ):
+            logging.info("terraform init")
+            chdir(temp_dir)
+            result = run(
+                [current_app.tf_bin, "init", "-no-color"],
+                stdout=None if logging.root.level == logging.DEBUG else DEVNULL,
+                stderr=None if logging.root.level == logging.DEBUG else DEVNULL,
+            )
+            if result.returncode != 0:
+                chdir(current_dir)
+                return jsonify(status="terraform init failed"), 500
+            result = run(
+                [current_app.tf_bin, "apply", "-auto-approve", "-no-color"],
+                stdout=None if logging.root.level == logging.DEBUG else DEVNULL,
+                stderr=None if logging.root.level == logging.DEBUG else DEVNULL,
+            )
+            if result.returncode != 0:
+                chdir(current_dir)
+                return jsonify(status="terraform apply failed"), 500
+        else:
+            chdir(current_dir)
+            return jsonify(status="git clone failed"), 500
+        chdir(current_dir)
+    return jsonify(status="success")

runner/utils.py

@@ -0,0 +1,29 @@
+import logging
+from os import chdir, getcwd
+from subprocess import run, DEVNULL
+
+from quart import current_app
+
+
+def git_clone(src_url, dest_dir):
+    """
+    Clone a remote git repository into a local directory.
+
+    Args:
+        src_url (string): Url used to clone the repo.
+        dest_dir (string): Path to the local directory.
+
+    Returns:
+       (boolean): True if command returns success.
+    """
+
+    logging.info("git clone " + src_url)
+    current_dir = getcwd()
+    chdir(dest_dir)
+    clone_result = run(
+        [current_app.git, "clone", src_url, "."],
+        stdout=None if logging.root.level == logging.DEBUG else DEVNULL,
+        stderr=None if logging.root.level == logging.DEBUG else DEVNULL,
+    )
+    chdir(current_dir)
+    return clone_result.returncode == 0