From e1ba0249f65f0fe1ba448c7ac6936e88a9b54430 Mon Sep 17 00:00:00 2001 From: Benoit Garcia Date: Sun, 22 Jan 2023 22:22:44 +0100 Subject: [PATCH 01/20] feat(terraform): Add basic support of terraform. --- Dockerfile | 2 +- tea_runner.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ae5e67f..1d16cfd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM python:alpine WORKDIR /usr/src/app RUN apk update -RUN apk add git rsync +RUN apk add git rsync terraform RUN apk add docker COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt diff --git a/tea_runner.py b/tea_runner.py index cc24979..78ba278 100644 --- a/tea_runner.py +++ b/tea_runner.py @@ -35,6 +35,7 @@ GIT_BIN = '/usr/bin/git' RSYNC_BIN = '/usr/bin/rsync' DOCKER_BIN = '/usr/bin/docker' +TF_BIN = '/usr/bin/terraform' print("Tea Runner") @@ -183,6 +184,46 @@ def docker_build(): return jsonify(status='success') +@app.route('/terraform/plan', methods=['POST']) +def terraform_plan(): + body = request.get_json() + + with TemporaryDirectory() as temp_dir: + if git_clone(body['repository']['clone_url'], temp_dir): + logging.info('terraform init') + chdir(temp_dir) + result = run([TF_BIN, 'init', '-no-color'], + stdout=None if args.debug else DEVNULL, stderr=None if args.debug else DEVNULL) + if result.returncode != 0: + return jsonify(status='terraform init failed'), 500 + result = run([TF_BIN, 'plan', '-no-color'], + stdout=None, stderr=None) + if result.returncode != 0: + return jsonify(status='terraform plan failed'), 500 + else: + return jsonify(status='git clone failed'), 500 + + return jsonify(status='success') + +@app.route('/terraform/apply', methods=['POST']) +def terraform_apply(): + body = request.get_json() + with TemporaryDirectory() as temp_dir: + if git_clone(body['repository']['clone_url'], temp_dir): + logging.info('terraform init') + chdir(temp_dir) + result = run([TF_BIN, 'init', '-no-color'], + stdout=None if args.debug else DEVNULL, stderr=None if args.debug else DEVNULL) + if result.returncode != 0: + return jsonify(status='terraform init failed'), 500 + result = run([TF_BIN, 'apply', '-auto-approve', '-no-color'], + stdout=None if args.debug else DEVNULL, stderr=None if args.debug else DEVNULL) + if result.returncode != 0: + return jsonify(status='terraform apply failed'), 500 + else: + return jsonify(status='git clone failed'), 500 + + return jsonify(status='success') if __name__ == '__main__': logging.info('Limiting requests to: ' + config.get('runner', From c0e149909c0d7694747b3632f93ad2058a2d2a0a Mon Sep 17 00:00:00 2001 From: Benoit Garcia Date: Mon, 23 Jan 2023 15:04:50 +0100 Subject: [PATCH 02/20] chore(python): Use black to reformat python code. --- tea_runner.py | 209 ++++++++++++++++++++++++++++---------------------- 1 file changed, 118 insertions(+), 91 deletions(-) diff --git a/tea_runner.py b/tea_runner.py index 78ba278..2f674f1 100644 --- a/tea_runner.py +++ b/tea_runner.py @@ -32,32 +32,31 @@ from argparse import ArgumentParser import logging -GIT_BIN = '/usr/bin/git' -RSYNC_BIN = '/usr/bin/rsync' -DOCKER_BIN = '/usr/bin/docker' -TF_BIN = '/usr/bin/terraform' +GIT_BIN = "/usr/bin/git" +RSYNC_BIN = "/usr/bin/rsync" +DOCKER_BIN = "/usr/bin/docker" +TF_BIN = "/usr/bin/terraform" print("Tea Runner") # Debug is a command-line option, but most configuration comes from config.ini arg_parser = ArgumentParser() -arg_parser.add_argument('-d', '--debug', action='store_true', - help='display debugging output while running') +arg_parser.add_argument( + "-d", "--debug", action="store_true", help="display debugging output while running" +) args = arg_parser.parse_args() config = ConfigParser() -config.read('config.ini') +config.read("config.ini") if args.debug: - config.set('runner', 'DEBUG', "true") + config.set("runner", "DEBUG", "true") -if config.getboolean('runner', 'DEBUG', fallback='False') == True: - logging.basicConfig(format='%(levelname)s: %(message)s', - level=logging.DEBUG) - logging.info('Debug logging is on') +if config.getboolean("runner", "DEBUG", fallback="False") == True: + logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.DEBUG) + logging.info("Debug logging is on") else: - logging.basicConfig( - format='%(levelname)s: %(message)s', level=logging.INFO) + logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO) if not access(GIT_BIN, X_OK): logging.error("git binary not found or not executable") @@ -82,12 +81,15 @@ def git_clone(src_url, dest_dir): (boolean): True if command returns success. """ - logging.info('git clone ' + src_url) - if config.getboolean('runner', 'GIT_SSL_NO_VERIFY', fallback='False') == True: - environ['GIT_SSL_NO_VERIFY'] = 'true' + logging.info("git clone " + src_url) + if config.getboolean("runner", "GIT_SSL_NO_VERIFY", fallback="False") == True: + environ["GIT_SSL_NO_VERIFY"] = "true" chdir(dest_dir) - clone_result = run([GIT_BIN, 'clone', src_url, '.'], - stdout=None if args.debug else DEVNULL, stderr=None if args.debug else DEVNULL) + clone_result = run( + [GIT_BIN, "clone", src_url, "."], + stdout=None if args.debug else DEVNULL, + stderr=None if args.debug else DEVNULL, + ) return clone_result.returncode == 0 @@ -99,15 +101,16 @@ def check_authorized(): """ Only respond to requests from ALLOWED_IP_RANGE if it's configured in config.ini """ - if config.has_option('runner', 'ALLOWED_IP_RANGE'): - allowed_ip_range = ip_network(config['runner']['ALLOWED_IP_RANGE']) + if config.has_option("runner", "ALLOWED_IP_RANGE"): + allowed_ip_range = ip_network(config["runner"]["ALLOWED_IP_RANGE"]) requesting_ip = ip_address(request.remote_addr) if requesting_ip not in allowed_ip_range: logging.info( - 'Dropping request from unauthorized host ' + request.remote_addr) - return jsonify(status='forbidden'), 403 + "Dropping request from unauthorized host " + request.remote_addr + ) + return jsonify(status="forbidden"), 403 else: - logging.info('Request from ' + request.remote_addr) + logging.info("Request from " + request.remote_addr) @app.before_request @@ -115,118 +118,142 @@ def check_media_type(): """ Only respond requests with Content-Type header of application/json """ - if not request.headers.get('Content-Type').lower().startswith('application/json'): + if not request.headers.get("Content-Type").lower().startswith("application/json"): logging.error( - '"Content-Type: application/json" header missing from request made by ' + request.remote_addr) - return jsonify(status='unsupported media type'), 415 + '"Content-Type: application/json" header missing from request made by ' + + request.remote_addr + ) + return jsonify(status="unsupported media type"), 415 -@app.route('/test', methods=['POST']) +@app.route("/test", methods=["POST"]) def test(): - logging.debug('Content-Type: ' + request.headers.get('Content-Type')) + logging.debug("Content-Type: " + request.headers.get("Content-Type")) logging.debug(request.get_json(force=True)) - return jsonify(status='success', sender=request.remote_addr) + return jsonify(status="success", sender=request.remote_addr) -@app.route('/rsync', methods=['POST']) +@app.route("/rsync", methods=["POST"]) def rsync(): body = request.get_json() - dest = request.args.get('dest') or body['repository']['name'] - rsync_root = config.get('rsync', 'RSYNC_ROOT', fallback='') + dest = request.args.get("dest") or body["repository"]["name"] + rsync_root = config.get("rsync", "RSYNC_ROOT", fallback="") if rsync_root: dest = path.join(rsync_root, utils.secure_filename(dest)) - logging.debug('rsync dest path updated to ' + dest) + logging.debug("rsync dest path updated to " + dest) with TemporaryDirectory() as temp_dir: - if git_clone(body['repository']['clone_url'], temp_dir): - logging.info('rsync ' + body['repository']['name'] + ' to ' + dest) + if git_clone(body["repository"]["clone_url"], temp_dir): + logging.info("rsync " + body["repository"]["name"] + " to " + dest) chdir(temp_dir) - if config.get('rsync', 'DELETE', fallback=''): - result = run([RSYNC_BIN, '-r', - '--exclude=.git', - '--delete-during' if config.get( - 'rsync', 'DELETE', fallback='') else '', - '.', - dest], - stdout=None if args.debug else DEVNULL, - stderr=None if args.debug else DEVNULL - ) + if config.get("rsync", "DELETE", fallback=""): + result = run( + [ + RSYNC_BIN, + "-r", + "--exclude=.git", + "--delete-during" + if config.get("rsync", "DELETE", fallback="") + else "", + ".", + dest, + ], + stdout=None if args.debug else DEVNULL, + stderr=None if args.debug else DEVNULL, + ) else: - result = run([RSYNC_BIN, '-r', - '--exclude=.git', - '.', - dest], - stdout=None if args.debug else DEVNULL, - stderr=None if args.debug else DEVNULL - ) + result = run( + [RSYNC_BIN, "-r", "--exclude=.git", ".", dest], + stdout=None if args.debug else DEVNULL, + stderr=None if args.debug else DEVNULL, + ) if result.returncode != 0: - return jsonify(status='rsync failed'), 500 + return jsonify(status="rsync failed"), 500 else: - return jsonify(status='git clone failed'), 500 + return jsonify(status="git clone failed"), 500 - return jsonify(status='success') + return jsonify(status="success") -@app.route('/docker/build', methods=['POST']) +@app.route("/docker/build", methods=["POST"]) def docker_build(): body = request.get_json() with TemporaryDirectory() as temp_dir: - if git_clone(body['repository']['clone_url'], temp_dir): - logging.info('docker build') + if git_clone(body["repository"]["clone_url"], temp_dir): + logging.info("docker build") chdir(temp_dir) - result = run([DOCKER_BIN, 'build', '-t', body['repository']['name'], '.'], - stdout=None if args.debug else DEVNULL, stderr=None if args.debug else DEVNULL) + result = run( + [DOCKER_BIN, "build", "-t", body["repository"]["name"], "."], + stdout=None if args.debug else DEVNULL, + stderr=None if args.debug else DEVNULL, + ) if result.returncode != 0: - return jsonify(status='docker build failed'), 500 + return jsonify(status="docker build failed"), 500 else: - return jsonify(status='git clone failed'), 500 + return jsonify(status="git clone failed"), 500 - return jsonify(status='success') + return jsonify(status="success") -@app.route('/terraform/plan', methods=['POST']) + +@app.route("/terraform/plan", methods=["POST"]) def terraform_plan(): body = request.get_json() with TemporaryDirectory() as temp_dir: - if git_clone(body['repository']['clone_url'], temp_dir): - logging.info('terraform init') + if git_clone(body["repository"]["clone_url"], temp_dir): + logging.info("terraform init") chdir(temp_dir) - result = run([TF_BIN, 'init', '-no-color'], - stdout=None if args.debug else DEVNULL, stderr=None if args.debug else DEVNULL) + result = run( + [TF_BIN, "init", "-no-color"], + stdout=None if args.debug else DEVNULL, + stderr=None if args.debug else DEVNULL, + ) if result.returncode != 0: - return jsonify(status='terraform init failed'), 500 - result = run([TF_BIN, 'plan', '-no-color'], - stdout=None, stderr=None) + return jsonify(status="terraform init failed"), 500 + result = run([TF_BIN, "plan", "-no-color"], stdout=None, stderr=None) if result.returncode != 0: - return jsonify(status='terraform plan failed'), 500 + return jsonify(status="terraform plan failed"), 500 else: - return jsonify(status='git clone failed'), 500 + return jsonify(status="git clone failed"), 500 + + return jsonify(status="success") - return jsonify(status='success') -@app.route('/terraform/apply', methods=['POST']) +@app.route("/terraform/apply", methods=["POST"]) def terraform_apply(): body = request.get_json() with TemporaryDirectory() as temp_dir: - if git_clone(body['repository']['clone_url'], temp_dir): - logging.info('terraform init') + if git_clone(body["repository"]["clone_url"], temp_dir): + logging.info("terraform init") chdir(temp_dir) - result = run([TF_BIN, 'init', '-no-color'], - stdout=None if args.debug else DEVNULL, stderr=None if args.debug else DEVNULL) + result = run( + [TF_BIN, "init", "-no-color"], + stdout=None if args.debug else DEVNULL, + stderr=None if args.debug else DEVNULL, + ) if result.returncode != 0: - return jsonify(status='terraform init failed'), 500 - result = run([TF_BIN, 'apply', '-auto-approve', '-no-color'], - stdout=None if args.debug else DEVNULL, stderr=None if args.debug else DEVNULL) + return jsonify(status="terraform init failed"), 500 + result = run( + [TF_BIN, "apply", "-auto-approve", "-no-color"], + stdout=None if args.debug else DEVNULL, + stderr=None if args.debug else DEVNULL, + ) if result.returncode != 0: - return jsonify(status='terraform apply failed'), 500 + return jsonify(status="terraform apply failed"), 500 else: - return jsonify(status='git clone failed'), 500 + return jsonify(status="git clone failed"), 500 + + return jsonify(status="success") - return jsonify(status='success') -if __name__ == '__main__': - logging.info('Limiting requests to: ' + config.get('runner', - 'ALLOWED_IP_RANGE', fallback='')) - serve(app, host=config.get('runner', 'LISTEN_IP', fallback='0.0.0.0'), - port=config.getint('runner', 'LISTEN_PORT', fallback=1706)) +if __name__ == "__main__": + logging.info( + "Limiting requests to: " + + config.get("runner", "ALLOWED_IP_RANGE", fallback="") + ) + serve( + app, + host=config.get("runner", "LISTEN_IP", fallback="0.0.0.0"), + port=config.getint("runner", "LISTEN_PORT", fallback=1706), + ) From f694253b208f679848d0b8a5269c82f84bd5c466 Mon Sep 17 00:00:00 2001 From: Benoit Garcia Date: Mon, 23 Jan 2023 16:10:56 +0100 Subject: [PATCH 03/20] feat(git): Add support of the SSH protocol to clone repositories. --- config.ini | 2 ++ tea_runner.py | 48 ++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/config.ini b/config.ini index 6d35c62..09f7e8a 100644 --- a/config.ini +++ b/config.ini @@ -2,6 +2,8 @@ # ALLOWED_IP_RANGE = 192.168.0.0/24 # DEBUG = true # GIT_SSL_NO_VERIFY = true +# GIT_SSH_NO_VERIFY = true +# GIT_PROTOCOL = http # LISTEN_PORT = 1706 [rsync] diff --git a/tea_runner.py b/tea_runner.py index 2f674f1..6ecaab1 100644 --- a/tea_runner.py +++ b/tea_runner.py @@ -11,8 +11,12 @@ [runner] ALLOWED_IP_RANGE=xxx.xxx.xxx.xxx/mm # Only respond to requests made from this range of IP addresses. Eg. 192.168.1.0/24 + GIT_PROTOCOL= + # Choose the protocol to use when cloning repositories. Default to http GIT_SSL_NO_VERIFY=true # Ignore certificate host verification errors. Useful for self-signed certs. + GIT_SSH_NO_VERIFY=true + # Ignore certificate host verification errors. LISTEN_IP=xxx.xxx.xxx.xxx # IP address for incoming requests. Defaults to 0.0.0.0 (Any). LISTEN_PORT=xxxx @@ -58,6 +62,9 @@ else: logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO) +git_protocol = config.get("runner", "GIT_PROTOCOL", fallback="http") +logging.info("git protocol is " + git_protocol) + if not access(GIT_BIN, X_OK): logging.error("git binary not found or not executable") exit(1) @@ -69,13 +76,14 @@ exit(1) -def git_clone(src_url, dest_dir): +def git_clone(src_url, dest_dir, protocol): """ Clone a remote git repository into a local directory. Args: - src_url (string): HTTP(S) url used to clone the repo. + src_url (string): Url used to clone the repo. dest_dir (string): Path to the local directory. + protocol (string): Protocol to use to clone the repo. Returns: (boolean): True if command returns success. @@ -85,6 +93,10 @@ def git_clone(src_url, dest_dir): if config.getboolean("runner", "GIT_SSL_NO_VERIFY", fallback="False") == True: environ["GIT_SSL_NO_VERIFY"] = "true" chdir(dest_dir) + if config.getboolean("runner", "GIT_SSH_NO_VERIFY", fallback="False") == True: + environ[ + "GIT_SSH_COMMAND" + ] = "ssh -o UserKnownHostsFile=test -o StrictHostKeyChecking=no" clone_result = run( [GIT_BIN, "clone", src_url, "."], stdout=None if args.debug else DEVNULL, @@ -143,7 +155,13 @@ def rsync(): logging.debug("rsync dest path updated to " + dest) with TemporaryDirectory() as temp_dir: - if git_clone(body["repository"]["clone_url"], temp_dir): + if git_clone( + body["repository"]["clone_url"] + if git_protocol == "http" + else body["repository"]["ssh_url"], + temp_dir, + git_protocol, + ): logging.info("rsync " + body["repository"]["name"] + " to " + dest) chdir(temp_dir) if config.get("rsync", "DELETE", fallback=""): @@ -180,7 +198,13 @@ def docker_build(): body = request.get_json() with TemporaryDirectory() as temp_dir: - if git_clone(body["repository"]["clone_url"], temp_dir): + if git_clone( + body["repository"]["clone_url"] + if git_protocol == "http" + else body["repository"]["ssh_url"], + temp_dir, + git_protocol, + ): logging.info("docker build") chdir(temp_dir) result = run( @@ -201,7 +225,13 @@ def terraform_plan(): body = request.get_json() with TemporaryDirectory() as temp_dir: - if git_clone(body["repository"]["clone_url"], temp_dir): + if git_clone( + body["repository"]["clone_url"] + if git_protocol == "http" + else body["repository"]["ssh_url"], + temp_dir, + git_protocol, + ): logging.info("terraform init") chdir(temp_dir) result = run( @@ -224,7 +254,13 @@ def terraform_plan(): def terraform_apply(): body = request.get_json() with TemporaryDirectory() as temp_dir: - if git_clone(body["repository"]["clone_url"], temp_dir): + if git_clone( + body["repository"]["clone_url"] + if git_protocol == "http" + else body["repository"]["ssh_url"], + temp_dir, + git_protocol, + ): logging.info("terraform init") chdir(temp_dir) result = run( From 72435424e31de69cd65043b75da61af3cfe87781 Mon Sep 17 00:00:00 2001 From: Benoit Garcia Date: Wed, 25 Jan 2023 16:26:53 +0100 Subject: [PATCH 04/20] chore(python): Order import statements as recommanded in --- tea_runner.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tea_runner.py b/tea_runner.py index 6ecaab1..12ad855 100644 --- a/tea_runner.py +++ b/tea_runner.py @@ -23,18 +23,18 @@ # TCP port number used for incoming requests. Defaults to 1706. """ +import logging +from argparse import ArgumentParser +from configparser import ConfigParser from ipaddress import ip_address, ip_network -from os import path +from os import access, chdir, environ, path, X_OK from subprocess import run, DEVNULL from sys import exit -from os import access, X_OK, chdir, environ, path from tempfile import TemporaryDirectory + +from flask import Flask, request, jsonify from waitress import serve from werkzeug import utils -from flask import Flask, request, jsonify -from configparser import ConfigParser -from argparse import ArgumentParser -import logging GIT_BIN = "/usr/bin/git" RSYNC_BIN = "/usr/bin/rsync" From 36739d7343d322621628b783a8a790cd9b5ef1b1 Mon Sep 17 00:00:00 2001 From: Benoit Garcia Date: Mon, 30 Jan 2023 10:33:21 +0100 Subject: [PATCH 05/20] chore(repo): Add venv and cache directories to gitignore. --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 2fa7ce7..337c1bc 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,7 @@ config.ini + +# ignore venv +**/.venv + +# ignore python cache +**/__pycache__ From d5d3dedd76ebbd7dd59c4495ed0aa606613c7837 Mon Sep 17 00:00:00 2001 From: Benoit Garcia Date: Mon, 30 Jan 2023 10:37:02 +0100 Subject: [PATCH 06/20] chore(repo): Add an EditorConfig file. --- .editorconfig | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..146fc71 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,21 @@ +# EditorConfig is awesome: https://EditorConfig.org +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file + +[*] +# Unix-style newlines +end_of_line = lf +# Always end with an empty new line +insert_final_newline = true +# Set default charset to utf-8 +charset = utf-8 +# Indent with 2 spaces +indent_style = space +indent_size = 2 + +# 4 space indentation +[*.py] +indent_style = space +indent_size = 4 From b4fb7ef932abffe2de22c43c301f06163803fe4c Mon Sep 17 00:00:00 2001 From: Benoit Garcia Date: Mon, 30 Jan 2023 11:38:34 +0100 Subject: [PATCH 07/20] chore(git_clone): Remove useless parameter. --- tea_runner.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tea_runner.py b/tea_runner.py index 12ad855..d6ff3b7 100644 --- a/tea_runner.py +++ b/tea_runner.py @@ -76,14 +76,13 @@ exit(1) -def git_clone(src_url, dest_dir, protocol): +def git_clone(src_url, dest_dir): """ Clone a remote git repository into a local directory. Args: src_url (string): Url used to clone the repo. dest_dir (string): Path to the local directory. - protocol (string): Protocol to use to clone the repo. Returns: (boolean): True if command returns success. @@ -160,7 +159,6 @@ def rsync(): if git_protocol == "http" else body["repository"]["ssh_url"], temp_dir, - git_protocol, ): logging.info("rsync " + body["repository"]["name"] + " to " + dest) chdir(temp_dir) @@ -203,7 +201,6 @@ def docker_build(): if git_protocol == "http" else body["repository"]["ssh_url"], temp_dir, - git_protocol, ): logging.info("docker build") chdir(temp_dir) @@ -230,7 +227,6 @@ def terraform_plan(): if git_protocol == "http" else body["repository"]["ssh_url"], temp_dir, - git_protocol, ): logging.info("terraform init") chdir(temp_dir) @@ -259,7 +255,6 @@ def terraform_apply(): if git_protocol == "http" else body["repository"]["ssh_url"], temp_dir, - git_protocol, ): logging.info("terraform init") chdir(temp_dir) From fcd609dfe04fbf81f9f5ac042ea0401d0cf0e221 Mon Sep 17 00:00:00 2001 From: Benoit Garcia Date: Mon, 30 Jan 2023 16:19:28 +0100 Subject: [PATCH 08/20] chore(syntax): Fix syntax using black. --- tea_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tea_runner.py b/tea_runner.py index d6ff3b7..c42c82a 100644 --- a/tea_runner.py +++ b/tea_runner.py @@ -27,7 +27,7 @@ from argparse import ArgumentParser from configparser import ConfigParser from ipaddress import ip_address, ip_network -from os import access, chdir, environ, path, X_OK +from os import access, chdir, environ, path, X_OK from subprocess import run, DEVNULL from sys import exit from tempfile import TemporaryDirectory From 53fe7f1361ec2172a2285eff8fb7a3030656434f Mon Sep 17 00:00:00 2001 From: Benoit Garcia Date: Mon, 30 Jan 2023 11:19:40 +0100 Subject: [PATCH 09/20] refacto: Change method to check presence of external applications. --- tea_runner.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/tea_runner.py b/tea_runner.py index c42c82a..c1a7cb6 100644 --- a/tea_runner.py +++ b/tea_runner.py @@ -28,6 +28,7 @@ from configparser import ConfigParser from ipaddress import ip_address, ip_network from os import access, chdir, environ, path, X_OK +from shutil import which from subprocess import run, DEVNULL from sys import exit from tempfile import TemporaryDirectory @@ -36,10 +37,10 @@ from waitress import serve from werkzeug import utils -GIT_BIN = "/usr/bin/git" -RSYNC_BIN = "/usr/bin/rsync" -DOCKER_BIN = "/usr/bin/docker" -TF_BIN = "/usr/bin/terraform" +GIT_BIN = which("git") +RSYNC_BIN = which("rsync") +DOCKER_BIN = which("docker") +TF_BIN = which("terraform") print("Tea Runner") @@ -65,15 +66,26 @@ git_protocol = config.get("runner", "GIT_PROTOCOL", fallback="http") logging.info("git protocol is " + git_protocol) -if not access(GIT_BIN, X_OK): +try: + access(GIT_BIN, X_OK) +except: logging.error("git binary not found or not executable") exit(1) -if not access(RSYNC_BIN, X_OK): +try: + access(RSYNC_BIN, X_OK) +except: logging.error("rsync binary not found or not executable") exit(1) -if not access(DOCKER_BIN, X_OK): +try: + access(DOCKER_BIN, X_OK) +except: logging.error("docker binary not found or not executable") exit(1) +try: + access(TF_BIN, X_OK) +except: + logging.error("terraform binary not found or not executable") + exit(1) def git_clone(src_url, dest_dir): From 013df29a891100197f4f4b9ea93a3418b937f461 Mon Sep 17 00:00:00 2001 From: Benoit Garcia Date: Mon, 30 Jan 2023 16:16:42 +0100 Subject: [PATCH 10/20] refacto: Move declarations of external programs in the Flask application. --- tea_runner.py | 69 ++++++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/tea_runner.py b/tea_runner.py index c1a7cb6..b93f9db 100644 --- a/tea_runner.py +++ b/tea_runner.py @@ -37,11 +37,6 @@ from waitress import serve from werkzeug import utils -GIT_BIN = which("git") -RSYNC_BIN = which("rsync") -DOCKER_BIN = which("docker") -TF_BIN = which("terraform") - print("Tea Runner") # Debug is a command-line option, but most configuration comes from config.ini @@ -66,27 +61,6 @@ git_protocol = config.get("runner", "GIT_PROTOCOL", fallback="http") logging.info("git protocol is " + git_protocol) -try: - access(GIT_BIN, X_OK) -except: - logging.error("git binary not found or not executable") - exit(1) -try: - access(RSYNC_BIN, X_OK) -except: - logging.error("rsync binary not found or not executable") - exit(1) -try: - access(DOCKER_BIN, X_OK) -except: - logging.error("docker binary not found or not executable") - exit(1) -try: - access(TF_BIN, X_OK) -except: - logging.error("terraform binary not found or not executable") - exit(1) - def git_clone(src_url, dest_dir): """ @@ -109,7 +83,7 @@ def git_clone(src_url, dest_dir): "GIT_SSH_COMMAND" ] = "ssh -o UserKnownHostsFile=test -o StrictHostKeyChecking=no" clone_result = run( - [GIT_BIN, "clone", src_url, "."], + [app.git, "clone", src_url, "."], stdout=None if args.debug else DEVNULL, stderr=None if args.debug else DEVNULL, ) @@ -177,7 +151,7 @@ def rsync(): if config.get("rsync", "DELETE", fallback=""): result = run( [ - RSYNC_BIN, + app.rsync, "-r", "--exclude=.git", "--delete-during" @@ -191,7 +165,7 @@ def rsync(): ) else: result = run( - [RSYNC_BIN, "-r", "--exclude=.git", ".", dest], + [app.rsync, "-r", "--exclude=.git", ".", dest], stdout=None if args.debug else DEVNULL, stderr=None if args.debug else DEVNULL, ) @@ -217,7 +191,7 @@ def docker_build(): logging.info("docker build") chdir(temp_dir) result = run( - [DOCKER_BIN, "build", "-t", body["repository"]["name"], "."], + [app.docker, "build", "-t", body["repository"]["name"], "."], stdout=None if args.debug else DEVNULL, stderr=None if args.debug else DEVNULL, ) @@ -243,13 +217,13 @@ def terraform_plan(): logging.info("terraform init") chdir(temp_dir) result = run( - [TF_BIN, "init", "-no-color"], + [app.tf_bin, "init", "-no-color"], stdout=None if args.debug else DEVNULL, stderr=None if args.debug else DEVNULL, ) if result.returncode != 0: return jsonify(status="terraform init failed"), 500 - result = run([TF_BIN, "plan", "-no-color"], stdout=None, stderr=None) + result = run([app.tf_bin, "plan", "-no-color"], stdout=None, stderr=None) if result.returncode != 0: return jsonify(status="terraform plan failed"), 500 else: @@ -271,14 +245,14 @@ def terraform_apply(): logging.info("terraform init") chdir(temp_dir) result = run( - [TF_BIN, "init", "-no-color"], + [app.tf_bin, "init", "-no-color"], stdout=None if args.debug else DEVNULL, stderr=None if args.debug else DEVNULL, ) if result.returncode != 0: return jsonify(status="terraform init failed"), 500 result = run( - [TF_BIN, "apply", "-auto-approve", "-no-color"], + [app.tf_bin, "apply", "-auto-approve", "-no-color"], stdout=None if args.debug else DEVNULL, stderr=None if args.debug else DEVNULL, ) @@ -295,6 +269,33 @@ def terraform_apply(): "Limiting requests to: " + config.get("runner", "ALLOWED_IP_RANGE", fallback="") ) + + app.git = which("git") + app.rsync = which("rsync") + app.docker = which("docker") + app.tf_bin = which("terraform") + + try: + access(app.git, X_OK) + except: + logging.error("git binary not found or not executable") + exit(1) + try: + access(app.rsync, X_OK) + except: + logging.error("rsync binary not found or not executable") + exit(1) + try: + access(app.docker, X_OK) + except: + logging.error("docker binary not found or not executable") + exit(1) + try: + access(app.tf_bin, X_OK) + except: + logging.error("terraform binary not found or not executable") + exit(1) + serve( app, host=config.get("runner", "LISTEN_IP", fallback="0.0.0.0"), From 05075c1675b487871970d63b505ee6a2112361e7 Mon Sep 17 00:00:00 2001 From: Benoit Garcia Date: Mon, 30 Jan 2023 17:56:48 +0100 Subject: [PATCH 11/20] refacto: Move configuration in the Flask application. --- tea_runner.py | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/tea_runner.py b/tea_runner.py index b93f9db..101ffc0 100644 --- a/tea_runner.py +++ b/tea_runner.py @@ -46,21 +46,6 @@ ) args = arg_parser.parse_args() -config = ConfigParser() -config.read("config.ini") - -if args.debug: - config.set("runner", "DEBUG", "true") - -if config.getboolean("runner", "DEBUG", fallback="False") == True: - logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.DEBUG) - logging.info("Debug logging is on") -else: - logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO) - -git_protocol = config.get("runner", "GIT_PROTOCOL", fallback="http") -logging.info("git protocol is " + git_protocol) - def git_clone(src_url, dest_dir): """ @@ -75,10 +60,10 @@ def git_clone(src_url, dest_dir): """ logging.info("git clone " + src_url) - if config.getboolean("runner", "GIT_SSL_NO_VERIFY", fallback="False") == True: + if app.runner_config.getboolean("runner", "GIT_SSL_NO_VERIFY", fallback="False") == True: environ["GIT_SSL_NO_VERIFY"] = "true" chdir(dest_dir) - if config.getboolean("runner", "GIT_SSH_NO_VERIFY", fallback="False") == True: + if app.runner_config.getboolean("runner", "GIT_SSH_NO_VERIFY", fallback="False") == True: environ[ "GIT_SSH_COMMAND" ] = "ssh -o UserKnownHostsFile=test -o StrictHostKeyChecking=no" @@ -98,8 +83,8 @@ def check_authorized(): """ Only respond to requests from ALLOWED_IP_RANGE if it's configured in config.ini """ - if config.has_option("runner", "ALLOWED_IP_RANGE"): - allowed_ip_range = ip_network(config["runner"]["ALLOWED_IP_RANGE"]) + if app.runner_config.has_option("runner", "ALLOWED_IP_RANGE"): + allowed_ip_range = ip_network(app.runner_config["runner"]["ALLOWED_IP_RANGE"]) requesting_ip = ip_address(request.remote_addr) if requesting_ip not in allowed_ip_range: logging.info( @@ -134,7 +119,7 @@ def test(): def rsync(): body = request.get_json() dest = request.args.get("dest") or body["repository"]["name"] - rsync_root = config.get("rsync", "RSYNC_ROOT", fallback="") + rsync_root = app.runner_config.get("rsync", "RSYNC_ROOT", fallback="") if rsync_root: dest = path.join(rsync_root, utils.secure_filename(dest)) logging.debug("rsync dest path updated to " + dest) @@ -148,14 +133,14 @@ def rsync(): ): logging.info("rsync " + body["repository"]["name"] + " to " + dest) chdir(temp_dir) - if config.get("rsync", "DELETE", fallback=""): + if app.runner_config.get("rsync", "DELETE", fallback=""): result = run( [ app.rsync, "-r", "--exclude=.git", "--delete-during" - if config.get("rsync", "DELETE", fallback="") + if app.runner_config.get("rsync", "DELETE", fallback="") else "", ".", dest, @@ -265,9 +250,24 @@ def terraform_apply(): if __name__ == "__main__": + app.runner_config = ConfigParser() + app.runner_config.read("config.ini") + + if args.debug: + app.runner_config.set("runner", "DEBUG", "true") + + if app.runner_config.getboolean("runner", "DEBUG", fallback="False") == True: + logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.DEBUG) + logging.info("Debug logging is on") + else: + logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO) + + git_protocol = app.runner_config.get("runner", "GIT_PROTOCOL", fallback="http") + logging.info("git protocol is " + git_protocol) + logging.info( "Limiting requests to: " - + config.get("runner", "ALLOWED_IP_RANGE", fallback="") + + app.runner_config.get("runner", "ALLOWED_IP_RANGE", fallback="") ) app.git = which("git") @@ -298,6 +298,6 @@ def terraform_apply(): serve( app, - host=config.get("runner", "LISTEN_IP", fallback="0.0.0.0"), - port=config.getint("runner", "LISTEN_PORT", fallback=1706), + host=app.runner_config.get("runner", "LISTEN_IP", fallback="0.0.0.0"), + port=app.runner_config.getint("runner", "LISTEN_PORT", fallback=1706), ) From b09e07e0c189d63dd9368bfa8a014ce1cfd966ad Mon Sep 17 00:00:00 2001 From: Benoit Garcia Date: Mon, 30 Jan 2023 18:00:44 +0100 Subject: [PATCH 12/20] refacto: Move initialization of git related env variables into the initilization of the Flask app. --- tea_runner.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tea_runner.py b/tea_runner.py index 101ffc0..cf864b6 100644 --- a/tea_runner.py +++ b/tea_runner.py @@ -60,13 +60,7 @@ def git_clone(src_url, dest_dir): """ logging.info("git clone " + src_url) - if app.runner_config.getboolean("runner", "GIT_SSL_NO_VERIFY", fallback="False") == True: - environ["GIT_SSL_NO_VERIFY"] = "true" chdir(dest_dir) - if app.runner_config.getboolean("runner", "GIT_SSH_NO_VERIFY", fallback="False") == True: - environ[ - "GIT_SSH_COMMAND" - ] = "ssh -o UserKnownHostsFile=test -o StrictHostKeyChecking=no" clone_result = run( [app.git, "clone", src_url, "."], stdout=None if args.debug else DEVNULL, @@ -296,6 +290,19 @@ def terraform_apply(): logging.error("terraform binary not found or not executable") exit(1) + if ( + app.runner_config.getboolean("runner", "GIT_SSL_NO_VERIFY", fallback="False") + == True + ): + environ["GIT_SSL_NO_VERIFY"] = "true" + if ( + app.runner_config.getboolean("runner", "GIT_SSH_NO_VERIFY", fallback="False") + == True + ): + environ[ + "GIT_SSH_COMMAND" + ] = "ssh -o UserKnownHostsFile=test -o StrictHostKeyChecking=no" + serve( app, host=app.runner_config.get("runner", "LISTEN_IP", fallback="0.0.0.0"), From f7e1e1a91c02c6dd106dcbbce68bd0234ae8a83f Mon Sep 17 00:00:00 2001 From: Benoit Garcia Date: Mon, 30 Jan 2023 18:15:53 +0100 Subject: [PATCH 13/20] refacto: Use settings of the logging library to determine STDOUT. --- tea_runner.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tea_runner.py b/tea_runner.py index cf864b6..f71ddb0 100644 --- a/tea_runner.py +++ b/tea_runner.py @@ -63,8 +63,8 @@ def git_clone(src_url, dest_dir): chdir(dest_dir) clone_result = run( [app.git, "clone", src_url, "."], - stdout=None if args.debug else DEVNULL, - stderr=None if args.debug else DEVNULL, + stdout=None if logging.root.level == logging.DEBUG else DEVNULL, + stderr=None if logging.root.level == logging.DEBUG else DEVNULL, ) return clone_result.returncode == 0 @@ -139,14 +139,14 @@ def rsync(): ".", dest, ], - stdout=None if args.debug else DEVNULL, - stderr=None if args.debug else DEVNULL, + stdout=None if logging.root.level == logging.DEBUG else DEVNULL, + stderr=None if logging.root.level == logging.DEBUG else DEVNULL, ) else: result = run( [app.rsync, "-r", "--exclude=.git", ".", dest], - stdout=None if args.debug else DEVNULL, - stderr=None if args.debug else DEVNULL, + stdout=None if logging.root.level == logging.DEBUG else DEVNULL, + stderr=None if logging.root.level == logging.DEBUG else DEVNULL, ) if result.returncode != 0: return jsonify(status="rsync failed"), 500 @@ -171,8 +171,8 @@ def docker_build(): chdir(temp_dir) result = run( [app.docker, "build", "-t", body["repository"]["name"], "."], - stdout=None if args.debug else DEVNULL, - stderr=None if args.debug else DEVNULL, + stdout=None if logging.root.level == logging.DEBUG else DEVNULL, + stderr=None if logging.root.level == logging.DEBUG else DEVNULL, ) if result.returncode != 0: return jsonify(status="docker build failed"), 500 @@ -197,8 +197,8 @@ def terraform_plan(): chdir(temp_dir) result = run( [app.tf_bin, "init", "-no-color"], - stdout=None if args.debug else DEVNULL, - stderr=None if args.debug else DEVNULL, + stdout=None if logging.root.level == logging.DEBUG else DEVNULL, + stderr=None if logging.root.level == logging.DEBUG else DEVNULL, ) if result.returncode != 0: return jsonify(status="terraform init failed"), 500 @@ -225,15 +225,15 @@ def terraform_apply(): chdir(temp_dir) result = run( [app.tf_bin, "init", "-no-color"], - stdout=None if args.debug else DEVNULL, - stderr=None if args.debug else DEVNULL, + stdout=None if logging.root.level == logging.DEBUG else DEVNULL, + stderr=None if logging.root.level == logging.DEBUG else DEVNULL, ) if result.returncode != 0: return jsonify(status="terraform init failed"), 500 result = run( [app.tf_bin, "apply", "-auto-approve", "-no-color"], - stdout=None if args.debug else DEVNULL, - stderr=None if args.debug else DEVNULL, + stdout=None if logging.root.level == logging.DEBUG else DEVNULL, + stderr=None if logging.root.level == logging.DEBUG else DEVNULL, ) if result.returncode != 0: return jsonify(status="terraform apply failed"), 500 From 775a6884ec49649feb57f542b7d587401cebd951 Mon Sep 17 00:00:00 2001 From: Benoit Garcia Date: Mon, 30 Jan 2023 18:24:58 +0100 Subject: [PATCH 14/20] refacto: Move functions definitions into a separate file. --- runner/utils.py | 27 +++++++++++++++++++++++++++ tea_runner.py | 32 ++++++-------------------------- 2 files changed, 33 insertions(+), 26 deletions(-) create mode 100644 runner/utils.py diff --git a/runner/utils.py b/runner/utils.py new file mode 100644 index 0000000..7239b3a --- /dev/null +++ b/runner/utils.py @@ -0,0 +1,27 @@ +import logging +from os import chdir +from subprocess import run, DEVNULL + +from flask import current_app + + +def git_clone(src_url, dest_dir): + """ + Clone a remote git repository into a local directory. + + Args: + src_url (string): Url used to clone the repo. + dest_dir (string): Path to the local directory. + + Returns: + (boolean): True if command returns success. + """ + + logging.info("git clone " + src_url) + chdir(dest_dir) + clone_result = run( + [current_app.git, "clone", src_url, "."], + stdout=None if logging.root.level == logging.DEBUG else DEVNULL, + stderr=None if logging.root.level == logging.DEBUG else DEVNULL, + ) + return clone_result.returncode == 0 diff --git a/tea_runner.py b/tea_runner.py index f71ddb0..d7726d7 100644 --- a/tea_runner.py +++ b/tea_runner.py @@ -37,6 +37,8 @@ from waitress import serve from werkzeug import utils +import runner.utils + print("Tea Runner") # Debug is a command-line option, but most configuration comes from config.ini @@ -47,28 +49,6 @@ args = arg_parser.parse_args() -def git_clone(src_url, dest_dir): - """ - Clone a remote git repository into a local directory. - - Args: - src_url (string): Url used to clone the repo. - dest_dir (string): Path to the local directory. - - Returns: - (boolean): True if command returns success. - """ - - logging.info("git clone " + src_url) - chdir(dest_dir) - clone_result = run( - [app.git, "clone", src_url, "."], - stdout=None if logging.root.level == logging.DEBUG else DEVNULL, - stderr=None if logging.root.level == logging.DEBUG else DEVNULL, - ) - return clone_result.returncode == 0 - - app = Flask(__name__) @@ -119,7 +99,7 @@ def rsync(): logging.debug("rsync dest path updated to " + dest) with TemporaryDirectory() as temp_dir: - if git_clone( + if runner.utils.git_clone( body["repository"]["clone_url"] if git_protocol == "http" else body["repository"]["ssh_url"], @@ -161,7 +141,7 @@ def docker_build(): body = request.get_json() with TemporaryDirectory() as temp_dir: - if git_clone( + if runner.utils.git_clone( body["repository"]["clone_url"] if git_protocol == "http" else body["repository"]["ssh_url"], @@ -187,7 +167,7 @@ def terraform_plan(): body = request.get_json() with TemporaryDirectory() as temp_dir: - if git_clone( + if runner.utils.git_clone( body["repository"]["clone_url"] if git_protocol == "http" else body["repository"]["ssh_url"], @@ -215,7 +195,7 @@ def terraform_plan(): def terraform_apply(): body = request.get_json() with TemporaryDirectory() as temp_dir: - if git_clone( + if runner.utils.git_clone( body["repository"]["clone_url"] if git_protocol == "http" else body["repository"]["ssh_url"], From 487eac382f7c89967608a69fd793294450f628d1 Mon Sep 17 00:00:00 2001 From: Benoit Garcia Date: Mon, 30 Jan 2023 19:14:51 +0100 Subject: [PATCH 15/20] refacto: Store git_protocol inside the Flask app. --- tea_runner.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tea_runner.py b/tea_runner.py index d7726d7..e16aebf 100644 --- a/tea_runner.py +++ b/tea_runner.py @@ -101,7 +101,7 @@ def rsync(): with TemporaryDirectory() as temp_dir: if runner.utils.git_clone( body["repository"]["clone_url"] - if git_protocol == "http" + if app.git_protocol == "http" else body["repository"]["ssh_url"], temp_dir, ): @@ -143,7 +143,7 @@ def docker_build(): with TemporaryDirectory() as temp_dir: if runner.utils.git_clone( body["repository"]["clone_url"] - if git_protocol == "http" + if app.git_protocol == "http" else body["repository"]["ssh_url"], temp_dir, ): @@ -169,7 +169,7 @@ def terraform_plan(): with TemporaryDirectory() as temp_dir: if runner.utils.git_clone( body["repository"]["clone_url"] - if git_protocol == "http" + if app.git_protocol == "http" else body["repository"]["ssh_url"], temp_dir, ): @@ -197,7 +197,7 @@ def terraform_apply(): with TemporaryDirectory() as temp_dir: if runner.utils.git_clone( body["repository"]["clone_url"] - if git_protocol == "http" + if app.git_protocol == "http" else body["repository"]["ssh_url"], temp_dir, ): @@ -236,8 +236,8 @@ def terraform_apply(): else: logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO) - git_protocol = app.runner_config.get("runner", "GIT_PROTOCOL", fallback="http") - logging.info("git protocol is " + git_protocol) + app.git_protocol = app.runner_config.get("runner", "GIT_PROTOCOL", fallback="http") + logging.info("git protocol is " + app.git_protocol) logging.info( "Limiting requests to: " From 93547a2d08c971ec75f70d8421bdf8f4725da2fb Mon Sep 17 00:00:00 2001 From: Benoit Garcia Date: Mon, 30 Jan 2023 19:45:37 +0100 Subject: [PATCH 16/20] refacto: Move route definitions into separate files. --- runner/docker.py | 36 +++++++++++ runner/rsync.py | 58 +++++++++++++++++ runner/terraform.py | 73 ++++++++++++++++++++++ tea_runner.py | 147 +++----------------------------------------- 4 files changed, 176 insertions(+), 138 deletions(-) create mode 100644 runner/docker.py create mode 100644 runner/rsync.py create mode 100644 runner/terraform.py diff --git a/runner/docker.py b/runner/docker.py new file mode 100644 index 0000000..3ab0545 --- /dev/null +++ b/runner/docker.py @@ -0,0 +1,36 @@ +import logging +from os import chdir, path +from subprocess import run, DEVNULL +from tempfile import TemporaryDirectory + +from flask import Blueprint, current_app, jsonify, request + +import runner.utils + +docker = Blueprint("docker", __name__) + + +@docker.route("/build", methods=["POST"]) +def docker_build(): + body = request.get_json() + + with TemporaryDirectory() as temp_dir: + if runner.utils.git_clone( + body["repository"]["clone_url"] + if current_app.git_protocol == "http" + else body["repository"]["ssh_url"], + temp_dir, + ): + logging.info("docker build") + chdir(temp_dir) + result = run( + [current_app.docker, "build", "-t", body["repository"]["name"], "."], + stdout=None if logging.root.level == logging.DEBUG else DEVNULL, + stderr=None if logging.root.level == logging.DEBUG else DEVNULL, + ) + if result.returncode != 0: + return jsonify(status="docker build failed"), 500 + else: + return jsonify(status="git clone failed"), 500 + + return jsonify(status="success") diff --git a/runner/rsync.py b/runner/rsync.py new file mode 100644 index 0000000..79eccfa --- /dev/null +++ b/runner/rsync.py @@ -0,0 +1,58 @@ +import logging +from os import chdir, path +from subprocess import run, DEVNULL +from tempfile import TemporaryDirectory + +from flask import Blueprint, current_app, jsonify, request +from werkzeug import utils + +import runner.utils + +rsync = Blueprint("rsync", __name__) + + +@rsync.route("/rsync", methods=["POST"]) +def route_rsync(): + body = request.get_json() + dest = request.args.get("dest") or body["repository"]["name"] + rsync_root = current_app.runner_config.get("rsync", "RSYNC_ROOT", fallback="") + if rsync_root: + dest = path.join(rsync_root, utils.secure_filename(dest)) + logging.debug("rsync dest path updated to " + dest) + + with TemporaryDirectory() as temp_dir: + if runner.utils.git_clone( + body["repository"]["clone_url"] + if current_app.git_protocol == "http" + else body["repository"]["ssh_url"], + temp_dir, + ): + logging.info("rsync " + body["repository"]["name"] + " to " + dest) + chdir(temp_dir) + if current_app.runner_config.get("rsync", "DELETE", fallback=""): + result = run( + [ + current_app.rsync, + "-r", + "--exclude=.git", + "--delete-during" + if current_app.runner_config.get("rsync", "DELETE", fallback="") + else "", + ".", + dest, + ], + stdout=None if logging.root.level == logging.DEBUG else DEVNULL, + stderr=None if logging.root.level == logging.DEBUG else DEVNULL, + ) + else: + result = run( + [current_app.rsync, "-r", "--exclude=.git", ".", dest], + stdout=None if logging.root.level == logging.DEBUG else DEVNULL, + stderr=None if logging.root.level == logging.DEBUG else DEVNULL, + ) + if result.returncode != 0: + return jsonify(status="rsync failed"), 500 + else: + return jsonify(status="git clone failed"), 500 + + return jsonify(status="success") diff --git a/runner/terraform.py b/runner/terraform.py new file mode 100644 index 0000000..f1acccf --- /dev/null +++ b/runner/terraform.py @@ -0,0 +1,73 @@ +import logging +from os import chdir +from subprocess import run, DEVNULL +from tempfile import TemporaryDirectory + +from flask import Blueprint, current_app, jsonify, request + +import runner.utils + +terraform = Blueprint("terraform", __name__) + + +@terraform.route("/plan", methods=["POST"]) +def terraform_plan(): + body = request.get_json() + + with TemporaryDirectory() as temp_dir: + if runner.utils.git_clone( + body["repository"]["clone_url"] + if current_app.git_protocol == "http" + else body["repository"]["ssh_url"], + temp_dir, + ): + logging.info("terraform init") + chdir(temp_dir) + result = run( + [current_app.tf_bin, "init", "-no-color"], + stdout=None if logging.root.level == logging.DEBUG else DEVNULL, + stderr=None if logging.root.level == logging.DEBUG else DEVNULL, + ) + if result.returncode != 0: + return jsonify(status="terraform init failed"), 500 + result = run( + [current_app.tf_bin, "plan", "-no-color"], stdout=None, stderr=None + ) + if result.returncode != 0: + return jsonify(status="terraform plan failed"), 500 + else: + return jsonify(status="git clone failed"), 500 + + return jsonify(status="success") + + +@terraform.route("/apply", methods=["POST"]) +def terraform_apply(): + body = request.get_json() + with TemporaryDirectory() as temp_dir: + if runner.utils.git_clone( + body["repository"]["clone_url"] + if current_app.git_protocol == "http" + else body["repository"]["ssh_url"], + temp_dir, + ): + logging.info("terraform init") + chdir(temp_dir) + result = run( + [current_app.tf_bin, "init", "-no-color"], + stdout=None if logging.root.level == logging.DEBUG else DEVNULL, + stderr=None if logging.root.level == logging.DEBUG else DEVNULL, + ) + if result.returncode != 0: + return jsonify(status="terraform init failed"), 500 + result = run( + [current_app.tf_bin, "apply", "-auto-approve", "-no-color"], + stdout=None if logging.root.level == logging.DEBUG else DEVNULL, + stderr=None if logging.root.level == logging.DEBUG else DEVNULL, + ) + if result.returncode != 0: + return jsonify(status="terraform apply failed"), 500 + else: + return jsonify(status="git clone failed"), 500 + + return jsonify(status="success") diff --git a/tea_runner.py b/tea_runner.py index e16aebf..7ca50af 100644 --- a/tea_runner.py +++ b/tea_runner.py @@ -27,15 +27,12 @@ from argparse import ArgumentParser from configparser import ConfigParser from ipaddress import ip_address, ip_network -from os import access, chdir, environ, path, X_OK +from os import access, environ, X_OK from shutil import which -from subprocess import run, DEVNULL from sys import exit -from tempfile import TemporaryDirectory from flask import Flask, request, jsonify from waitress import serve -from werkzeug import utils import runner.utils @@ -89,140 +86,6 @@ def test(): return jsonify(status="success", sender=request.remote_addr) -@app.route("/rsync", methods=["POST"]) -def rsync(): - body = request.get_json() - dest = request.args.get("dest") or body["repository"]["name"] - rsync_root = app.runner_config.get("rsync", "RSYNC_ROOT", fallback="") - if rsync_root: - dest = path.join(rsync_root, utils.secure_filename(dest)) - logging.debug("rsync dest path updated to " + dest) - - with TemporaryDirectory() as temp_dir: - if runner.utils.git_clone( - body["repository"]["clone_url"] - if app.git_protocol == "http" - else body["repository"]["ssh_url"], - temp_dir, - ): - logging.info("rsync " + body["repository"]["name"] + " to " + dest) - chdir(temp_dir) - if app.runner_config.get("rsync", "DELETE", fallback=""): - result = run( - [ - app.rsync, - "-r", - "--exclude=.git", - "--delete-during" - if app.runner_config.get("rsync", "DELETE", fallback="") - else "", - ".", - dest, - ], - stdout=None if logging.root.level == logging.DEBUG else DEVNULL, - stderr=None if logging.root.level == logging.DEBUG else DEVNULL, - ) - else: - result = run( - [app.rsync, "-r", "--exclude=.git", ".", dest], - stdout=None if logging.root.level == logging.DEBUG else DEVNULL, - stderr=None if logging.root.level == logging.DEBUG else DEVNULL, - ) - if result.returncode != 0: - return jsonify(status="rsync failed"), 500 - else: - return jsonify(status="git clone failed"), 500 - - return jsonify(status="success") - - -@app.route("/docker/build", methods=["POST"]) -def docker_build(): - body = request.get_json() - - with TemporaryDirectory() as temp_dir: - if runner.utils.git_clone( - body["repository"]["clone_url"] - if app.git_protocol == "http" - else body["repository"]["ssh_url"], - temp_dir, - ): - logging.info("docker build") - chdir(temp_dir) - result = run( - [app.docker, "build", "-t", body["repository"]["name"], "."], - stdout=None if logging.root.level == logging.DEBUG else DEVNULL, - stderr=None if logging.root.level == logging.DEBUG else DEVNULL, - ) - if result.returncode != 0: - return jsonify(status="docker build failed"), 500 - else: - return jsonify(status="git clone failed"), 500 - - return jsonify(status="success") - - -@app.route("/terraform/plan", methods=["POST"]) -def terraform_plan(): - body = request.get_json() - - with TemporaryDirectory() as temp_dir: - if runner.utils.git_clone( - body["repository"]["clone_url"] - if app.git_protocol == "http" - else body["repository"]["ssh_url"], - temp_dir, - ): - logging.info("terraform init") - chdir(temp_dir) - result = run( - [app.tf_bin, "init", "-no-color"], - stdout=None if logging.root.level == logging.DEBUG else DEVNULL, - stderr=None if logging.root.level == logging.DEBUG else DEVNULL, - ) - if result.returncode != 0: - return jsonify(status="terraform init failed"), 500 - result = run([app.tf_bin, "plan", "-no-color"], stdout=None, stderr=None) - if result.returncode != 0: - return jsonify(status="terraform plan failed"), 500 - else: - return jsonify(status="git clone failed"), 500 - - return jsonify(status="success") - - -@app.route("/terraform/apply", methods=["POST"]) -def terraform_apply(): - body = request.get_json() - with TemporaryDirectory() as temp_dir: - if runner.utils.git_clone( - body["repository"]["clone_url"] - if app.git_protocol == "http" - else body["repository"]["ssh_url"], - temp_dir, - ): - logging.info("terraform init") - chdir(temp_dir) - result = run( - [app.tf_bin, "init", "-no-color"], - stdout=None if logging.root.level == logging.DEBUG else DEVNULL, - stderr=None if logging.root.level == logging.DEBUG else DEVNULL, - ) - if result.returncode != 0: - return jsonify(status="terraform init failed"), 500 - result = run( - [app.tf_bin, "apply", "-auto-approve", "-no-color"], - stdout=None if logging.root.level == logging.DEBUG else DEVNULL, - stderr=None if logging.root.level == logging.DEBUG else DEVNULL, - ) - if result.returncode != 0: - return jsonify(status="terraform apply failed"), 500 - else: - return jsonify(status="git clone failed"), 500 - - return jsonify(status="success") - - if __name__ == "__main__": app.runner_config = ConfigParser() app.runner_config.read("config.ini") @@ -283,6 +146,14 @@ def terraform_apply(): "GIT_SSH_COMMAND" ] = "ssh -o UserKnownHostsFile=test -o StrictHostKeyChecking=no" + from runner.docker import docker as docker_bp + from runner.rsync import rsync as rsync_bp + from runner.terraform import terraform as terraform_bp + + app.register_blueprint(docker_bp, url_prefix="/docker") + app.register_blueprint(rsync_bp) + app.register_blueprint(terraform_bp, url_prefix="/terraform") + serve( app, host=app.runner_config.get("runner", "LISTEN_IP", fallback="0.0.0.0"), From d013fbfa0ed8d56742f74132db2888fa93e0d7a1 Mon Sep 17 00:00:00 2001 From: Benoit Garcia Date: Tue, 31 Jan 2023 17:53:37 +0100 Subject: [PATCH 17/20] feat: Move from Flask to Quart, removing Waitress in the process. --- requirements.txt | 3 +-- runner/docker.py | 6 +++--- runner/rsync.py | 6 +++--- runner/terraform.py | 6 +++--- runner/utils.py | 2 +- tea_runner.py | 18 +++++++++--------- 6 files changed, 20 insertions(+), 21 deletions(-) diff --git a/requirements.txt b/requirements.txt index 84baa64..1216b2a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ -Flask>=2.1.0 -waitress>=2.1.0 +quart==0.18.3 diff --git a/runner/docker.py b/runner/docker.py index 3ab0545..5579427 100644 --- a/runner/docker.py +++ b/runner/docker.py @@ -3,7 +3,7 @@ from subprocess import run, DEVNULL from tempfile import TemporaryDirectory -from flask import Blueprint, current_app, jsonify, request +from quart import Blueprint, current_app, jsonify, request import runner.utils @@ -11,8 +11,8 @@ @docker.route("/build", methods=["POST"]) -def docker_build(): - body = request.get_json() +async def docker_build(): + body = await request.get_json() with TemporaryDirectory() as temp_dir: if runner.utils.git_clone( diff --git a/runner/rsync.py b/runner/rsync.py index 79eccfa..9337776 100644 --- a/runner/rsync.py +++ b/runner/rsync.py @@ -3,7 +3,7 @@ from subprocess import run, DEVNULL from tempfile import TemporaryDirectory -from flask import Blueprint, current_app, jsonify, request +from quart import Blueprint, current_app, jsonify, request from werkzeug import utils import runner.utils @@ -12,8 +12,8 @@ @rsync.route("/rsync", methods=["POST"]) -def route_rsync(): - body = request.get_json() +async def route_rsync(): + body = await request.get_json() dest = request.args.get("dest") or body["repository"]["name"] rsync_root = current_app.runner_config.get("rsync", "RSYNC_ROOT", fallback="") if rsync_root: diff --git a/runner/terraform.py b/runner/terraform.py index f1acccf..b3cc381 100644 --- a/runner/terraform.py +++ b/runner/terraform.py @@ -3,7 +3,7 @@ from subprocess import run, DEVNULL from tempfile import TemporaryDirectory -from flask import Blueprint, current_app, jsonify, request +from quart import Blueprint, current_app, jsonify, request import runner.utils @@ -11,8 +11,8 @@ @terraform.route("/plan", methods=["POST"]) -def terraform_plan(): - body = request.get_json() +async def terraform_plan(): + body = await request.get_json() with TemporaryDirectory() as temp_dir: if runner.utils.git_clone( diff --git a/runner/utils.py b/runner/utils.py index 7239b3a..327d308 100644 --- a/runner/utils.py +++ b/runner/utils.py @@ -2,7 +2,7 @@ from os import chdir from subprocess import run, DEVNULL -from flask import current_app +from quart import current_app def git_clone(src_url, dest_dir): diff --git a/tea_runner.py b/tea_runner.py index 7ca50af..68b4237 100644 --- a/tea_runner.py +++ b/tea_runner.py @@ -31,8 +31,7 @@ from shutil import which from sys import exit -from flask import Flask, request, jsonify -from waitress import serve +from quart import Quart, request, jsonify import runner.utils @@ -46,11 +45,11 @@ args = arg_parser.parse_args() -app = Flask(__name__) +app = Quart(__name__) @app.before_request -def check_authorized(): +async def check_authorized(): """ Only respond to requests from ALLOWED_IP_RANGE if it's configured in config.ini """ @@ -67,7 +66,7 @@ def check_authorized(): @app.before_request -def check_media_type(): +async def check_media_type(): """ Only respond requests with Content-Type header of application/json """ @@ -80,9 +79,9 @@ def check_media_type(): @app.route("/test", methods=["POST"]) -def test(): +async def test(): logging.debug("Content-Type: " + request.headers.get("Content-Type")) - logging.debug(request.get_json(force=True)) + logging.debug(await request.get_json(force=True)) return jsonify(status="success", sender=request.remote_addr) @@ -154,8 +153,9 @@ def test(): app.register_blueprint(rsync_bp) app.register_blueprint(terraform_bp, url_prefix="/terraform") - serve( - app, + + app.run( host=app.runner_config.get("runner", "LISTEN_IP", fallback="0.0.0.0"), port=app.runner_config.getint("runner", "LISTEN_PORT", fallback=1706), + debug=logging.root.level, ) From dec3077a106284ccf5086e4533062791e4343311 Mon Sep 17 00:00:00 2001 From: Benoit Garcia Date: Tue, 31 Jan 2023 17:56:05 +0100 Subject: [PATCH 18/20] fix: Move from temporary directories when they are destroyed. --- runner/docker.py | 4 +++- runner/rsync.py | 4 +++- runner/terraform.py | 14 +++++++++++--- runner/utils.py | 4 +++- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/runner/docker.py b/runner/docker.py index 5579427..2a913b7 100644 --- a/runner/docker.py +++ b/runner/docker.py @@ -1,5 +1,5 @@ import logging -from os import chdir, path +from os import chdir, getcwd, path from subprocess import run, DEVNULL from tempfile import TemporaryDirectory @@ -15,6 +15,7 @@ async def docker_build(): body = await request.get_json() with TemporaryDirectory() as temp_dir: + current_dir = getcwd() if runner.utils.git_clone( body["repository"]["clone_url"] if current_app.git_protocol == "http" @@ -28,6 +29,7 @@ async def docker_build(): stdout=None if logging.root.level == logging.DEBUG else DEVNULL, stderr=None if logging.root.level == logging.DEBUG else DEVNULL, ) + chdir(current_dir) if result.returncode != 0: return jsonify(status="docker build failed"), 500 else: diff --git a/runner/rsync.py b/runner/rsync.py index 9337776..e657da5 100644 --- a/runner/rsync.py +++ b/runner/rsync.py @@ -1,5 +1,5 @@ import logging -from os import chdir, path +from os import chdir, getcwd, path from subprocess import run, DEVNULL from tempfile import TemporaryDirectory @@ -21,6 +21,7 @@ async def route_rsync(): logging.debug("rsync dest path updated to " + dest) with TemporaryDirectory() as temp_dir: + current_dir = getcwd() if runner.utils.git_clone( body["repository"]["clone_url"] if current_app.git_protocol == "http" @@ -50,6 +51,7 @@ async def route_rsync(): stdout=None if logging.root.level == logging.DEBUG else DEVNULL, stderr=None if logging.root.level == logging.DEBUG else DEVNULL, ) + chdir(current_dir) if result.returncode != 0: return jsonify(status="rsync failed"), 500 else: diff --git a/runner/terraform.py b/runner/terraform.py index b3cc381..daafbaa 100644 --- a/runner/terraform.py +++ b/runner/terraform.py @@ -1,5 +1,5 @@ import logging -from os import chdir +from os import chdir, getcwd from subprocess import run, DEVNULL from tempfile import TemporaryDirectory @@ -15,6 +15,7 @@ async def terraform_plan(): body = await request.get_json() with TemporaryDirectory() as temp_dir: + current_dir = getcwd() if runner.utils.git_clone( body["repository"]["clone_url"] if current_app.git_protocol == "http" @@ -29,15 +30,18 @@ async def terraform_plan(): stderr=None if logging.root.level == logging.DEBUG else DEVNULL, ) if result.returncode != 0: + chdir(current_dir) return jsonify(status="terraform init failed"), 500 result = run( [current_app.tf_bin, "plan", "-no-color"], stdout=None, stderr=None ) if result.returncode != 0: + chdir(current_dir) return jsonify(status="terraform plan failed"), 500 else: + chdir(current_dir) return jsonify(status="git clone failed"), 500 - + chdir(current_dir) return jsonify(status="success") @@ -45,6 +49,7 @@ async def terraform_plan(): def terraform_apply(): body = request.get_json() with TemporaryDirectory() as temp_dir: + current_dir = getcwd() if runner.utils.git_clone( body["repository"]["clone_url"] if current_app.git_protocol == "http" @@ -59,6 +64,7 @@ def terraform_apply(): stderr=None if logging.root.level == logging.DEBUG else DEVNULL, ) if result.returncode != 0: + chdir(current_dir) return jsonify(status="terraform init failed"), 500 result = run( [current_app.tf_bin, "apply", "-auto-approve", "-no-color"], @@ -66,8 +72,10 @@ def terraform_apply(): stderr=None if logging.root.level == logging.DEBUG else DEVNULL, ) if result.returncode != 0: + chdir(current_dir) return jsonify(status="terraform apply failed"), 500 else: + chdir(current_dir) return jsonify(status="git clone failed"), 500 - + chdir(current_dir) return jsonify(status="success") diff --git a/runner/utils.py b/runner/utils.py index 327d308..631bdca 100644 --- a/runner/utils.py +++ b/runner/utils.py @@ -1,5 +1,5 @@ import logging -from os import chdir +from os import chdir, getcwd from subprocess import run, DEVNULL from quart import current_app @@ -18,10 +18,12 @@ def git_clone(src_url, dest_dir): """ logging.info("git clone " + src_url) + current_dir = getcwd() chdir(dest_dir) clone_result = run( [current_app.git, "clone", src_url, "."], stdout=None if logging.root.level == logging.DEBUG else DEVNULL, stderr=None if logging.root.level == logging.DEBUG else DEVNULL, ) + chdir(current_dir) return clone_result.returncode == 0 From 07b0017dec380d9c9f20747a6262ece7f9ec21e3 Mon Sep 17 00:00:00 2001 From: Benoit Garcia Date: Tue, 31 Jan 2023 22:07:42 +0100 Subject: [PATCH 19/20] feat: Use hypercorn to server the `runner` Quart module. --- runner/__init__.py | 123 ++++++++++++++++++++++++++++++++++++++++ tea_runner.py | 136 +++++---------------------------------------- 2 files changed, 138 insertions(+), 121 deletions(-) create mode 100644 runner/__init__.py diff --git a/runner/__init__.py b/runner/__init__.py new file mode 100644 index 0000000..5cb698f --- /dev/null +++ b/runner/__init__.py @@ -0,0 +1,123 @@ +import logging +from ipaddress import ip_address, ip_network +from os import access, environ, X_OK +from shutil import which + +from quart import Quart, request, jsonify + + +def create_app(config): + print("Tea Runner") + # Configure loglevel + if config.getboolean("runner", "DEBUG", fallback="False") == True: + logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.DEBUG) + logging.info("Debug logging is on") + else: + logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO) + + app = Quart(__name__) + + # Configure Quart + app.runner_config = config + app.git_protocol = app.runner_config.get("runner", "GIT_PROTOCOL", fallback="http") + + # Check presence of external programs + app.docker = which("docker") + try: + access(app.docker, X_OK) + except: + logging.error("docker binary not found or not executable") + exit(1) + + app.git = which("git") + try: + access(app.git, X_OK) + except: + logging.error("git binary not found or not executable") + exit(1) + + app.rsync = which("rsync") + try: + access(app.rsync, X_OK) + except: + logging.error("rsync binary not found or not executable") + exit(1) + + app.tf_bin = which("terraform") + try: + access(app.tf_bin, X_OK) + except: + logging.error("terraform binary not found or not executable") + exit(1) + + # Set environment variables + if ( + app.runner_config.getboolean("runner", "GIT_SSL_NO_VERIFY", fallback="False") + == True + ): + environ["GIT_SSL_NO_VERIFY"] = "true" + if ( + app.runner_config.getboolean("runner", "GIT_SSH_NO_VERIFY", fallback="False") + == True + ): + environ[ + "GIT_SSH_COMMAND" + ] = "ssh -o UserKnownHostsFile=test -o StrictHostKeyChecking=no" + + # Log some informations + logging.info("git protocol is " + app.git_protocol) + logging.info( + "Limiting requests to: " + + app.runner_config.get("runner", "ALLOWED_IP_RANGE", fallback="") + ) + + @app.before_request + async def check_authorized(): + """ + Only respond to requests from ALLOWED_IP_RANGE if it's configured in config.ini + """ + if app.runner_config.has_option("runner", "ALLOWED_IP_RANGE"): + allowed_ip_range = ip_network( + app.runner_config["runner"]["ALLOWED_IP_RANGE"] + ) + requesting_ip = ip_address(request.remote_addr) + if requesting_ip not in allowed_ip_range: + logging.info( + "Dropping request from unauthorized host " + request.remote_addr + ) + return jsonify(status="forbidden"), 403 + else: + logging.info("Request from " + request.remote_addr) + + @app.before_request + async def check_media_type(): + """ + Only respond requests with Content-Type header of application/json + """ + if ( + not request.headers.get("Content-Type") + .lower() + .startswith("application/json") + ): + logging.error( + '"Content-Type: application/json" header missing from request made by ' + + request.remote_addr + ) + return jsonify(status="unsupported media type"), 415 + + @app.route("/test", methods=["POST"]) + async def test(): + logging.debug("Content-Type: " + request.headers.get("Content-Type")) + logging.debug(await request.get_json(force=True)) + return jsonify(status="success", sender=request.remote_addr) + + # Register Blueprints + from runner.docker import docker as docker_bp + from runner.rsync import rsync as rsync_bp + from runner.terraform import terraform as terraform_bp + + app.register_blueprint(docker_bp, url_prefix="/docker") + app.register_blueprint(rsync_bp) + app.register_blueprint(terraform_bp, url_prefix="/terraform") + + return app diff --git a/tea_runner.py b/tea_runner.py index 68b4237..6f05f45 100644 --- a/tea_runner.py +++ b/tea_runner.py @@ -23,19 +23,13 @@ # TCP port number used for incoming requests. Defaults to 1706. """ -import logging +import asyncio from argparse import ArgumentParser from configparser import ConfigParser -from ipaddress import ip_address, ip_network -from os import access, environ, X_OK -from shutil import which -from sys import exit -from quart import Quart, request, jsonify +from hypercorn.asyncio import Config, serve -import runner.utils - -print("Tea Runner") +import runner # Debug is a command-line option, but most configuration comes from config.ini arg_parser = ArgumentParser() @@ -44,118 +38,18 @@ ) args = arg_parser.parse_args() +quart_config = ConfigParser() +quart_config.read("config.ini") +hypercorn_config = Config() -app = Quart(__name__) - - -@app.before_request -async def check_authorized(): - """ - Only respond to requests from ALLOWED_IP_RANGE if it's configured in config.ini - """ - if app.runner_config.has_option("runner", "ALLOWED_IP_RANGE"): - allowed_ip_range = ip_network(app.runner_config["runner"]["ALLOWED_IP_RANGE"]) - requesting_ip = ip_address(request.remote_addr) - if requesting_ip not in allowed_ip_range: - logging.info( - "Dropping request from unauthorized host " + request.remote_addr - ) - return jsonify(status="forbidden"), 403 - else: - logging.info("Request from " + request.remote_addr) - - -@app.before_request -async def check_media_type(): - """ - Only respond requests with Content-Type header of application/json - """ - if not request.headers.get("Content-Type").lower().startswith("application/json"): - logging.error( - '"Content-Type: application/json" header missing from request made by ' - + request.remote_addr - ) - return jsonify(status="unsupported media type"), 415 - - -@app.route("/test", methods=["POST"]) -async def test(): - logging.debug("Content-Type: " + request.headers.get("Content-Type")) - logging.debug(await request.get_json(force=True)) - return jsonify(status="success", sender=request.remote_addr) - - -if __name__ == "__main__": - app.runner_config = ConfigParser() - app.runner_config.read("config.ini") - - if args.debug: - app.runner_config.set("runner", "DEBUG", "true") - - if app.runner_config.getboolean("runner", "DEBUG", fallback="False") == True: - logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.DEBUG) - logging.info("Debug logging is on") - else: - logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO) - - app.git_protocol = app.runner_config.get("runner", "GIT_PROTOCOL", fallback="http") - logging.info("git protocol is " + app.git_protocol) - - logging.info( - "Limiting requests to: " - + app.runner_config.get("runner", "ALLOWED_IP_RANGE", fallback="") - ) - - app.git = which("git") - app.rsync = which("rsync") - app.docker = which("docker") - app.tf_bin = which("terraform") - - try: - access(app.git, X_OK) - except: - logging.error("git binary not found or not executable") - exit(1) - try: - access(app.rsync, X_OK) - except: - logging.error("rsync binary not found or not executable") - exit(1) - try: - access(app.docker, X_OK) - except: - logging.error("docker binary not found or not executable") - exit(1) - try: - access(app.tf_bin, X_OK) - except: - logging.error("terraform binary not found or not executable") - exit(1) - - if ( - app.runner_config.getboolean("runner", "GIT_SSL_NO_VERIFY", fallback="False") - == True - ): - environ["GIT_SSL_NO_VERIFY"] = "true" - if ( - app.runner_config.getboolean("runner", "GIT_SSH_NO_VERIFY", fallback="False") - == True - ): - environ[ - "GIT_SSH_COMMAND" - ] = "ssh -o UserKnownHostsFile=test -o StrictHostKeyChecking=no" - - from runner.docker import docker as docker_bp - from runner.rsync import rsync as rsync_bp - from runner.terraform import terraform as terraform_bp - - app.register_blueprint(docker_bp, url_prefix="/docker") - app.register_blueprint(rsync_bp) - app.register_blueprint(terraform_bp, url_prefix="/terraform") +hypercorn_config.bind = ( + quart_config.get("runner", "LISTEN_IP", fallback="0.0.0.0") + + ":" + + str(quart_config.getint("runner", "LISTEN_PORT", fallback=1706)) +) +if args.debug: + quart_config.set("runner", "DEBUG", "true") + hypercorn_config.loglevel = "debug" - app.run( - host=app.runner_config.get("runner", "LISTEN_IP", fallback="0.0.0.0"), - port=app.runner_config.getint("runner", "LISTEN_PORT", fallback=1706), - debug=logging.root.level, - ) +asyncio.run(serve(runner.create_app(quart_config), hypercorn_config)) From dece7541968ffb24d4a9b22fcd9db369506d9b6a Mon Sep 17 00:00:00 2001 From: Benoit Garcia Date: Tue, 31 Jan 2023 22:08:55 +0100 Subject: [PATCH 20/20] chore(style): Remove useless import. --- runner/docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/docker.py b/runner/docker.py index 2a913b7..29239e0 100644 --- a/runner/docker.py +++ b/runner/docker.py @@ -1,5 +1,5 @@ import logging -from os import chdir, getcwd, path +from os import chdir, getcwd from subprocess import run, DEVNULL from tempfile import TemporaryDirectory