From 669a4f769f13c391da7ce1d3f7923b5974463a10 Mon Sep 17 00:00:00 2001 From: Arthur Date: Sun, 31 Oct 2021 11:31:10 -0400 Subject: [PATCH 01/16] Python VENV --- .gitignore | 6 ++++++ .vscode/settings.json | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 0cde044..416c16d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,9 @@ cover/* .coverage */my.cnf *.env +bin/* +lib/* +share/* +man/* +lib64 +pyvenv.cfg diff --git a/.vscode/settings.json b/.vscode/settings.json index 230f4de..5b0298d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -15,5 +15,5 @@ "editor.tabSize": 4, "editor.renderControlCharacters": true, "editor.renderWhitespace": "all", - -} \ No newline at end of file + "python.pythonPath": "bin/python", // Python VENV +} From 2b599ac5ed8f11f730e51ad66ba23b26a7b313ac Mon Sep 17 00:00:00 2001 From: Arthur Date: Sun, 31 Oct 2021 17:00:53 -0400 Subject: [PATCH 02/16] Tab Size Change --- .vscode/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 5b0298d..18fc8c9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,7 +12,7 @@ "files.insertFinalNewline": true, "editor.formatOnPaste": true, "editor.formatOnSave": true, - "editor.tabSize": 4, + "editor.tabSize": 2, "editor.renderControlCharacters": true, "editor.renderWhitespace": "all", "python.pythonPath": "bin/python", // Python VENV From 40a41b43a8ac4851b2c15c4e6be2c8d94668db76 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 2 Nov 2021 22:11:08 -0400 Subject: [PATCH 03/16] Kuberentes Prevent Running as Root --- .github/workflows/tests.yml | 75 ++++++++++++++++----------------- .gitlab-ci.yml | 15 +++---- AnalyticsforSpotify/settings.py | 7 +-- docker/000-default.conf | 19 ++++----- docker/ports.conf | 12 ++++++ docker/startup.sh | 4 +- dockerfile | 24 ++++++++--- kubernetes/configmap.yaml | 4 +- kubernetes/deployment.yaml | 19 ++++++--- kubernetes/service.yaml | 4 +- kubernetes/traefik.yaml | 2 +- setup.py | 7 ++- 12 files changed, 107 insertions(+), 85 deletions(-) create mode 100644 docker/ports.conf diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 51b4cd4..82c76ff 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,48 +9,45 @@ on: jobs: build: - runs-on: ubuntu-latest strategy: matrix: python-version: [3.9] steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - sudo apt-get install mysql-server - sudo apt-get install python3-pip libmysqlclient-dev - python -m pip install --upgrade pip - python -m pip install flake8 - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Setup Database - run: | - sudo /etc/init.d/mysql start - sudo mysql -uroot -proot -Bse "CREATE USER 'spotifyTest'@'localhost' IDENTIFIED BY 'spotifyTest';" - sudo mysql -uroot -proot -Bse "GRANT ALL PRIVILEGES ON *.* TO 'spotifyTest'@'localhost';" - sudo mysql -uroot -proot -Bse "flush privileges;" - sudo mysql -uroot -proot -Bse "create database spotifyTest;" - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with Django - run: | - echo "[client] - host = localhost - database = spotifyTest - user = spotifyTest - password = spotifyTest - default-character-set = utf8"> AnalyticsforSpotify/my.cnf - python3 manage.py test - env: - TEST: test - - name: Coveralls Python - uses: AndreMiras/coveralls-python-action@master + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + sudo apt-get install mysql-server + sudo apt-get install python3-pip libmysqlclient-dev + python -m pip install --upgrade pip + python -m pip install flake8 + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Setup Database + run: | + sudo /etc/init.d/mysql start + sudo mysql -uroot -proot -Bse "CREATE USER 'spotifyTest'@'localhost' IDENTIFIED BY 'spotifyTest';" + sudo mysql -uroot -proot -Bse "GRANT ALL PRIVILEGES ON *.* TO 'spotifyTest'@'localhost';" + sudo mysql -uroot -proot -Bse "flush privileges;" + sudo mysql -uroot -proot -Bse "create database spotifyTest;" + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with Django + run: | + export HOST=localhost + export DATABASE=spotifyTest + export DB_USER=spotifyTest + export DB_PASSWORD=spotifyTest + python3 manage.py test + env: + TEST: test + - name: Coveralls Python + uses: AndreMiras/coveralls-python-action@master diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a1a128c..60ed5a5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,13 +22,10 @@ PythonTests: MARIADB_ROOT_PASSWORD: root MARIADB_DATABASE: spotify before_script: - - | - echo "[client] - host = mariadb - database = spotify - user = root - password = root - default-character-set = utf8"> AnalyticsforSpotify/my.cnf + - export HOST=mariadb + - export DATABASE=spotify + - export DB_USER=root + - export DB_PASSWORD=root - apt-get update - apt-get install -y libmariadb-dev gcc - python -m pip install --upgrade pip @@ -113,8 +110,8 @@ DeployKubernetes: - sed -i "s,,${REDIRECT_URL},g" kubernetes/configmap.yaml - sed -i "s,,${HOST},g" kubernetes/configmap.yaml - sed -i "s,,${DATABASE},g" kubernetes/configmap.yaml - - sed -i "s,,${USER},g" kubernetes/configmap.yaml - - sed -i "s,,${PASSWORD},g" kubernetes/configmap.yaml + - sed -i "s,,${DB_USER},g" kubernetes/configmap.yaml + - sed -i "s,,${DB_PASSWORD},g" kubernetes/configmap.yaml - kubectl apply -f kubernetes/configmap.yaml - kubectl apply -f kubernetes/service.yaml diff --git a/AnalyticsforSpotify/settings.py b/AnalyticsforSpotify/settings.py index d2a79e4..837266f 100644 --- a/AnalyticsforSpotify/settings.py +++ b/AnalyticsforSpotify/settings.py @@ -86,9 +86,10 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.mysql', - 'OPTIONS': { - 'read_default_file': BASE_DIR + '/AnalyticsforSpotify/my.cnf', - }, + 'NAME': os.environ.get('DATABASE'), + 'USER': os.environ.get('DB_USER'), + 'PASSWORD': os.environ.get('DB_PASSWORD'), + 'HOST': os.environ.get('HOST'), } } WSGI_APPLICATION = 'AnalyticsforSpotify.wsgi.application' diff --git a/docker/000-default.conf b/docker/000-default.conf index 775ca86..d473516 100644 --- a/docker/000-default.conf +++ b/docker/000-default.conf @@ -1,21 +1,20 @@ - - ErrorLog ${APACHE_LOG_DIR}/error.log - ErrorLog /proc/self/fd/1 - CustomLog ${APACHE_LOG_DIR}/access.log combined + + ErrorLog /proc/self/fd/1 + CustomLog /proc/self/fd/1 combined CustomLog /proc/self/fd/1 customLog - Alias /spotify /home/root/analytics-for-spotify/webFrontend - + Alias /spotify /home/www/analytics-for-spotify/webFrontend + Require all granted - + Require all granted - WSGIDaemonProcess AnalyticsforSpotify python-path=//home/root/analytics-for-spotify/ - WSGIScriptAlias / /home/root/analytics-for-spotify/AnalyticsforSpotify/wsgi.py process-group=AnalyticsforSpotify application-group=%{GLOBAL} + WSGIDaemonProcess AnalyticsforSpotify python-path=//home/www/analytics-for-spotify/ + WSGIScriptAlias / /home/www/analytics-for-spotify/AnalyticsforSpotify/wsgi.py process-group=AnalyticsforSpotify application-group=%{GLOBAL} WSGIProcessGroup AnalyticsforSpotify - \ No newline at end of file + diff --git a/docker/ports.conf b/docker/ports.conf new file mode 100644 index 0000000..8202f9b --- /dev/null +++ b/docker/ports.conf @@ -0,0 +1,12 @@ +# If you just change the port or add more ports here, you will likely also +# have to change the VirtualHost statement in +# /etc/apache2/sites-enabled/000-default.conf + +Listen 8080 + + + Listen 8081 + + + +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet diff --git a/docker/startup.sh b/docker/startup.sh index b7251ef..f48be25 100644 --- a/docker/startup.sh +++ b/docker/startup.sh @@ -1,4 +1,4 @@ #!/bin/bash -cd /home/root/analytics-for-spotify/ +cd /home/www/analytics-for-spotify/ python3 setup.py -exec "$@" \ No newline at end of file +exec "$@" diff --git a/dockerfile b/dockerfile index 40b3f56..a911b41 100644 --- a/dockerfile +++ b/dockerfile @@ -2,17 +2,27 @@ FROM python:3.9-slim RUN apt-get update RUN apt-get install -y apache2 libapache2-mod-wsgi-py3 libmariadb-dev gcc -COPY . /home/root/analytics-for-spotify/ +COPY . /home/www/analytics-for-spotify/ COPY docker/000-default.conf /etc/apache2/sites-available/000-default.conf -RUN pip3 install -r /home/root/analytics-for-spotify/requirements.txt -ADD ./webFrontend/node_modules.tar.xz /home/root/analytics-for-spotify/webFrontend/ -RUN chown -R www-data:www-data /home/root/analytics-for-spotify/ -RUN chmod +x /home/root/analytics-for-spotify/docker/startup.sh +RUN pip3 install -r /home/www/analytics-for-spotify/requirements.txt +ADD ./webFrontend/node_modules.tar.xz /home/www/analytics-for-spotify/webFrontend/ +RUN chown -R 10033:10033 /home/www/analytics-for-spotify/ +RUN chmod +x /home/www/analytics-for-spotify/docker/startup.sh RUN echo "ServerName 127.0.0.1" >> /etc/apache2/apache2.conf RUN echo 'LogFormat "%{%a %b %d %H:%M:%S %Y}t %H %m %U" customLog' >> /etc/apache2/apache2.conf RUN echo 'ErrorLogFormat "%t %M"' >> /etc/apache2/apache2.conf -ENTRYPOINT ["/home/root/analytics-for-spotify/docker/startup.sh"] +COPY docker/ports.conf /etc/apache2/ports.conf +RUN sed -i "s,www-data,www,g" /etc/apache2/envvars + + +RUN useradd -u 10033 www +RUN chown -R 10033:10033 /var/log/apache2/ +RUN chown -R 10033:10033 /var/run/apache2/ +RUN chown -R 10033:10033 /proc/self/fd/1 +USER 10033 + +ENTRYPOINT ["/home/www/analytics-for-spotify/docker/startup.sh"] CMD ["/usr/sbin/apachectl", "-D", "FOREGROUND"] -EXPOSE 80 +EXPOSE 8080 diff --git a/kubernetes/configmap.yaml b/kubernetes/configmap.yaml index eb8aa5c..2924f6f 100644 --- a/kubernetes/configmap.yaml +++ b/kubernetes/configmap.yaml @@ -8,6 +8,6 @@ data: CLIENT_SECRET: "" HOST: DATABASE: - USER: - PASSWORD: + DB_USER: + DB_PASSWORD: REDIRECT_URL: "" diff --git a/kubernetes/deployment.yaml b/kubernetes/deployment.yaml index ccfecf9..081e99e 100644 --- a/kubernetes/deployment.yaml +++ b/kubernetes/deployment.yaml @@ -20,8 +20,15 @@ spec: annotations: enable.version-checker.io/analytics-for-spotify: "false" spec: + securityContext: + runAsUser: 10033 + runAsGroup: 30000 + fsGroup: 20000 containers: - - env: + - securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + env: - name: CLIENT_ID valueFrom: configMapKeyRef: @@ -44,26 +51,26 @@ spec: configMapKeyRef: key: HOST name: env - - name: PASSWORD + - name: DB_PASSWORD valueFrom: configMapKeyRef: - key: PASSWORD + key: DB_PASSWORD name: env - name: REDIRECT_URL valueFrom: configMapKeyRef: key: REDIRECT_URL name: env - - name: USER + - name: DB_USER valueFrom: configMapKeyRef: - key: USER + key: DB_USER name: env image: : imagePullPolicy: IfNotPresent name: analytics-for-spotify ports: - - containerPort: 80 + - containerPort: 8080 resources: limits: cpu: 350m diff --git a/kubernetes/service.yaml b/kubernetes/service.yaml index 7656b2e..54fb2e9 100644 --- a/kubernetes/service.yaml +++ b/kubernetes/service.yaml @@ -7,7 +7,7 @@ spec: ports: - protocol: TCP name: web - port: 80 - targetPort: 80 + port: 8080 + targetPort: 8080 selector: app: analytics-for-spotify diff --git a/kubernetes/traefik.yaml b/kubernetes/traefik.yaml index 384c19b..f285947 100644 --- a/kubernetes/traefik.yaml +++ b/kubernetes/traefik.yaml @@ -11,4 +11,4 @@ spec: kind: Rule services: - name: analytics-for-spotify - port: 80 + port: 8080 diff --git a/setup.py b/setup.py index f39c4f6..fc7a825 100644 --- a/setup.py +++ b/setup.py @@ -20,8 +20,8 @@ def dockerSetup(): IP = os.environ.get('HOST') DB = os.environ.get('DATABASE') - USER = os.environ.get('USER') - PASS = os.environ.get('PASSWORD') + USER = os.environ.get('DB_USER') + PASS = os.environ.get('DB_PASSWORD') db = myCNF(IP, DB, USER, PASS) setup(db, API) @@ -38,8 +38,7 @@ def myCNF(IP, DB, USER, PASS): "password = "+PASS, "default-character-set = utf8", ] - with open("AnalyticsforSpotify/my.cnf", 'w+') as f: - f.writelines('\n'.join(MYSQL)) + db = MySQLdb.connect( host=IP, user=USER, From d95c96a83bf83598223a85e340d578e5d850550e Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 2 Nov 2021 22:25:06 -0400 Subject: [PATCH 04/16] Kubernetes Read Only Root File System --- dockerfile | 30 ++++++++++++++++++++++-------- kubernetes/deployment.yaml | 1 + 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/dockerfile b/dockerfile index a911b41..a8d6322 100644 --- a/dockerfile +++ b/dockerfile @@ -1,28 +1,42 @@ # WIP, TESTING ONLY, DO NOT DEPLOY + +# Base OS & Dependencies FROM python:3.9-slim RUN apt-get update RUN apt-get install -y apache2 libapache2-mod-wsgi-py3 libmariadb-dev gcc + +# Custom Apache Logs +RUN echo 'LogFormat "%{%a %b %d %H:%M:%S %Y}t %H %m %U" customLog' >> /etc/apache2/apache2.conf +RUN echo 'ErrorLogFormat "%t %M"' >> /etc/apache2/apache2.conf + +# Supress Apache Server Name Error +RUN echo "ServerName 127.0.0.1" >> /etc/apache2/apache2.conf + +# Required Read Only Root File System +RUN sed -i "s,ErrorLog \${APACHE_LOG_DIR}/error.log,ErrorLog /proc/self/fd/1,g" /etc/apache2/apache2.conf +RUN sed -i "s,CustomLog \${APACHE_LOG_DIR}/other_vhosts_access.log,CustomLog /proc/self/fd/1,g" /etc/apache2/conf-available/other-vhosts-access-log.conf +RUN sed -i "s,/var/run/apache2\$SUFFIX/,/dev/shm/,g" /etc/apache2/envvars +RUN sed -i "s,/var/run/apache2\$SUFFIX,/dev/shm/apache2\$SUFFIX,g" /etc/apache2/envvars +RUN sed -i "s,#WSGISocketPrefix /var/run/apache2/,WSGISocketPrefix /dev/shm/apache2/,g" /etc/apache2/mods-available/wsgi.conf + +# Setup Analytics For Spotify COPY . /home/www/analytics-for-spotify/ COPY docker/000-default.conf /etc/apache2/sites-available/000-default.conf RUN pip3 install -r /home/www/analytics-for-spotify/requirements.txt ADD ./webFrontend/node_modules.tar.xz /home/www/analytics-for-spotify/webFrontend/ -RUN chown -R 10033:10033 /home/www/analytics-for-spotify/ RUN chmod +x /home/www/analytics-for-spotify/docker/startup.sh -RUN echo "ServerName 127.0.0.1" >> /etc/apache2/apache2.conf -RUN echo 'LogFormat "%{%a %b %d %H:%M:%S %Y}t %H %m %U" customLog' >> /etc/apache2/apache2.conf -RUN echo 'ErrorLogFormat "%t %M"' >> /etc/apache2/apache2.conf +EXPOSE 8080 +# Non Root User Settings COPY docker/ports.conf /etc/apache2/ports.conf RUN sed -i "s,www-data,www,g" /etc/apache2/envvars - - RUN useradd -u 10033 www +RUN chown -R 10033:10033 /home/www/analytics-for-spotify/ RUN chown -R 10033:10033 /var/log/apache2/ RUN chown -R 10033:10033 /var/run/apache2/ RUN chown -R 10033:10033 /proc/self/fd/1 USER 10033 +# Entry Point ENTRYPOINT ["/home/www/analytics-for-spotify/docker/startup.sh"] CMD ["/usr/sbin/apachectl", "-D", "FOREGROUND"] - -EXPOSE 8080 diff --git a/kubernetes/deployment.yaml b/kubernetes/deployment.yaml index 081e99e..788ca3d 100644 --- a/kubernetes/deployment.yaml +++ b/kubernetes/deployment.yaml @@ -27,6 +27,7 @@ spec: containers: - securityContext: allowPrivilegeEscalation: false + readOnlyRootFilesystem: true runAsNonRoot: true env: - name: CLIENT_ID From 3858dfdd33b158911a571ce1eb6b383f466e2d0a Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 2 Nov 2021 23:27:41 -0400 Subject: [PATCH 05/16] livenessProbe & Resource Tweak & Port Config Tweak --- docker/ports.conf | 12 ------------ dockerfile | 4 +++- kubernetes/deployment.yaml | 34 ++++++++++++++++++++-------------- 3 files changed, 23 insertions(+), 27 deletions(-) delete mode 100644 docker/ports.conf diff --git a/docker/ports.conf b/docker/ports.conf deleted file mode 100644 index 8202f9b..0000000 --- a/docker/ports.conf +++ /dev/null @@ -1,12 +0,0 @@ -# If you just change the port or add more ports here, you will likely also -# have to change the VirtualHost statement in -# /etc/apache2/sites-enabled/000-default.conf - -Listen 8080 - - - Listen 8081 - - - -# vim: syntax=apache ts=4 sw=4 sts=4 sr noet diff --git a/dockerfile b/dockerfile index a8d6322..d3ffeb4 100644 --- a/dockerfile +++ b/dockerfile @@ -27,8 +27,10 @@ ADD ./webFrontend/node_modules.tar.xz /home/www/analytics-for-spotify/webFronten RUN chmod +x /home/www/analytics-for-spotify/docker/startup.sh EXPOSE 8080 +# Ports +RUN sed -i "s,80,8080,g" /etc/apache2/ports.conf + # Non Root User Settings -COPY docker/ports.conf /etc/apache2/ports.conf RUN sed -i "s,www-data,www,g" /etc/apache2/envvars RUN useradd -u 10033 www RUN chown -R 10033:10033 /home/www/analytics-for-spotify/ diff --git a/kubernetes/deployment.yaml b/kubernetes/deployment.yaml index 788ca3d..db6c4ae 100644 --- a/kubernetes/deployment.yaml +++ b/kubernetes/deployment.yaml @@ -20,6 +20,8 @@ spec: annotations: enable.version-checker.io/analytics-for-spotify: "false" spec: + hostname: analytics-for-spotify + restartPolicy: Always securityContext: runAsUser: 10033 runAsGroup: 30000 @@ -29,6 +31,24 @@ spec: allowPrivilegeEscalation: false readOnlyRootFilesystem: true runAsNonRoot: true + livenessProbe: + httpGet: + path: /analytics/authenticated/ + port: 8080 + initialDelaySeconds: 15 + periodSeconds: 60 + image: : + imagePullPolicy: IfNotPresent + name: analytics-for-spotify + ports: + - containerPort: 8080 + resources: + limits: + cpu: 200m + memory: 250M + requests: + cpu: 100m + memory: 200M env: - name: CLIENT_ID valueFrom: @@ -67,17 +87,3 @@ spec: configMapKeyRef: key: DB_USER name: env - image: : - imagePullPolicy: IfNotPresent - name: analytics-for-spotify - ports: - - containerPort: 8080 - resources: - limits: - cpu: 350m - memory: 250M - requests: - cpu: 150m - memory: 175M - hostname: analytics-for-spotify - restartPolicy: Always From 8806cf75b91b014cd962d949e41251b457994f84 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 3 Nov 2021 07:00:49 -0400 Subject: [PATCH 06/16] Kubernetes: configmap to secret --- .gitlab-ci.yml | 24 ++++++++++++++-------- kubernetes/deployment.yaml | 14 ++++++------- kubernetes/{configmap.yaml => secret.yaml} | 2 +- 3 files changed, 24 insertions(+), 16 deletions(-) rename kubernetes/{configmap.yaml => secret.yaml} (94%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 60ed5a5..1002e7b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -105,14 +105,22 @@ DeployKubernetes: - sed -i "s//${CI_COMMIT_SHA}/g" kubernetes/deployment.yaml - kubectl apply -f kubernetes/deployment.yaml - - sed -i "s,,${CLIENT_ID},g" kubernetes/configmap.yaml - - sed -i "s//${CLIENT_SECRET}/g" kubernetes/configmap.yaml - - sed -i "s,,${REDIRECT_URL},g" kubernetes/configmap.yaml - - sed -i "s,,${HOST},g" kubernetes/configmap.yaml - - sed -i "s,,${DATABASE},g" kubernetes/configmap.yaml - - sed -i "s,,${DB_USER},g" kubernetes/configmap.yaml - - sed -i "s,,${DB_PASSWORD},g" kubernetes/configmap.yaml - - kubectl apply -f kubernetes/configmap.yaml + - CLIENT_ID=$(echo -n ${CLIENT_ID} | base64 -w 0 | tr -d \\n) + - CLIENT_SECRET=$(echo -n ${CLIENT_SECRET} | base64 -w 0 | tr -d \\n) + - REDIRECT_URL=$(echo -n ${REDIRECT_URL} | base64 -w 0 | tr -d \\n) + - HOST=$(echo -n ${HOST} | base64 -w 0 | tr -d \\n) + - DATABASE=$(echo -n ${DATABASE} | base64 -w 0 | tr -d \\n) + - DB_USER=$(echo -n ${DB_USER} | base64 -w 0 | tr -d \\n) + - DB_PASSWORD=$(echo -n ${DB_PASSWORD} | base64 -w 0 | tr -d \\n) + + - sed -i "s,,${CLIENT_ID},g" kubernetes/secret.yaml + - sed -i "s//${CLIENT_SECRET}/g" kubernetes/secret.yaml + - sed -i "s,,${REDIRECT_URL},g" kubernetes/secret.yaml + - sed -i "s,,${HOST},g" kubernetes/secret.yaml + - sed -i "s,,${DATABASE},g" kubernetes/secret.yaml + - sed -i "s,,${DB_USER},g" kubernetes/secret.yaml + - sed -i "s,,${DB_PASSWORD},g" kubernetes/secret.yaml + - kubectl apply -f kubernetes/secret.yaml - kubectl apply -f kubernetes/service.yaml diff --git a/kubernetes/deployment.yaml b/kubernetes/deployment.yaml index db6c4ae..b12c7c5 100644 --- a/kubernetes/deployment.yaml +++ b/kubernetes/deployment.yaml @@ -52,38 +52,38 @@ spec: env: - name: CLIENT_ID valueFrom: - configMapKeyRef: + secretKeyRef: key: CLIENT_ID name: env - name: CLIENT_SECRET valueFrom: - configMapKeyRef: + secretKeyRef: key: CLIENT_SECRET name: env - name: DATABASE valueFrom: - configMapKeyRef: + secretKeyRef: key: DATABASE name: env - name: DOCKER value: "YES" - name: HOST valueFrom: - configMapKeyRef: + secretKeyRef: key: HOST name: env - name: DB_PASSWORD valueFrom: - configMapKeyRef: + secretKeyRef: key: DB_PASSWORD name: env - name: REDIRECT_URL valueFrom: - configMapKeyRef: + secretKeyRef: key: REDIRECT_URL name: env - name: DB_USER valueFrom: - configMapKeyRef: + secretKeyRef: key: DB_USER name: env diff --git a/kubernetes/configmap.yaml b/kubernetes/secret.yaml similarity index 94% rename from kubernetes/configmap.yaml rename to kubernetes/secret.yaml index 2924f6f..0abc752 100644 --- a/kubernetes/configmap.yaml +++ b/kubernetes/secret.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -kind: ConfigMap +kind: Secret metadata: name: env namespace: analytics-for-spotify From f5d890b825bfd9c50930de843ecf1d24d9f71947 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 3 Nov 2021 16:54:16 -0400 Subject: [PATCH 07/16] Pipeline Tweak --- .gitlab-ci.yml | 41 ----------------------------------------- 1 file changed, 41 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1002e7b..49bdd9b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,6 @@ variables: stages: - test - build - - push - deploy PythonTests: @@ -50,46 +49,6 @@ DockerBuild: - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_BRANCH || true - docker build --network host --cache-from $CI_REGISTRY_IMAGE:$CI_COMMIT_BRANCH -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA . - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - -PushProduction: - stage: push - image: docker:latest - variables: - GIT_STRATEGY: none - DOCKER_HOST: tcp://docker:2375 - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - services: - - name: docker:dind - alias: docker - command: ["--tls=false"] - only: - - production - before_script: - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - script: - - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:$CI_COMMIT_BRANCH - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_BRANCH - -PushDevelop: - stage: push - image: docker:latest - variables: - GIT_STRATEGY: none - DOCKER_HOST: tcp://docker:2375 - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - services: - - name: docker:dind - alias: docker - command: ["--tls=false"] - only: - - develop - before_script: - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - script: - - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:$CI_COMMIT_BRANCH - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_BRANCH From 763ece1b7e383ee76ccdd076368ea91059f6a180 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 3 Nov 2021 18:41:10 -0400 Subject: [PATCH 08/16] URL Redirect Tweaks --- AnalyticsforSpotify/urls.py | 3 +-- webBackend/urls.py | 1 + webFrontend/chart.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/AnalyticsforSpotify/urls.py b/AnalyticsforSpotify/urls.py index fbc1453..c9faa9d 100644 --- a/AnalyticsforSpotify/urls.py +++ b/AnalyticsforSpotify/urls.py @@ -18,8 +18,7 @@ from django.conf.urls.static import static urlpatterns = [ - path('analytics/', include('webBackend.urls')), path('', include('webBackend.urls')), path('spotify/', include('webBackend.urls')), - + path('analytics/', include('webBackend.urls')), ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) diff --git a/webBackend/urls.py b/webBackend/urls.py index cd80484..90de941 100644 --- a/webBackend/urls.py +++ b/webBackend/urls.py @@ -5,6 +5,7 @@ urlpatterns = [ path('', views.redirect), path('spotify/', views.redirect), + path('analytics/', views.redirect), path('listeningHistory/', views.listeningHistory, name='listeningHistory'), path('songs/', views.songs, name='songs'), path('playlistSongs/', views.playlistSongs, name='playlistSongs'), diff --git a/webFrontend/chart.js b/webFrontend/chart.js index bb4109e..5229f43 100644 --- a/webFrontend/chart.js +++ b/webFrontend/chart.js @@ -7,7 +7,7 @@ function deleteCookies() { document.cookie = `${allCookies[i]}=;expires=${new Date(0).toUTCString()}`; } $.post('/analytics/logout/') - window.location.href = '/spotify'; + window.location.href = '/spotify/index.html'; } function start() { $.post('/analytics/start/'); window.location.href = '/spotify/analytics.html'; }; @@ -33,7 +33,7 @@ window.onload = function () { method: 'GET', success(data) { if (data === 'False') { - window.location.href = '/spotify'; + window.location.href = '/spotify/index.html'; } }, From c05f5a9d8f8b527b324c40ae100caa9e5a9ed975 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 3 Nov 2021 19:16:20 -0400 Subject: [PATCH 09/16] DEBUG = False --- AnalyticsforSpotify/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AnalyticsforSpotify/settings.py b/AnalyticsforSpotify/settings.py index 837266f..c0d2cc0 100644 --- a/AnalyticsforSpotify/settings.py +++ b/AnalyticsforSpotify/settings.py @@ -24,7 +24,7 @@ # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = False ALLOWED_HOSTS = ['*'] From d9159d7dc64197a0181e580dadaa12e13263dfc9 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 3 Nov 2021 19:16:59 -0400 Subject: [PATCH 10/16] HOSTS -> DB_HOSTS --- .github/workflows/tests.yml | 2 +- .gitlab-ci.yml | 6 +++--- AnalyticsforSpotify/settings.py | 2 +- kubernetes/deployment.yaml | 4 ++-- kubernetes/secret.yaml | 2 +- setup.py | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 82c76ff..c446466 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -42,7 +42,7 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with Django run: | - export HOST=localhost + export DB_HOST=localhost export DATABASE=spotifyTest export DB_USER=spotifyTest export DB_PASSWORD=spotifyTest diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 49bdd9b..ce3f88e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,7 +21,7 @@ PythonTests: MARIADB_ROOT_PASSWORD: root MARIADB_DATABASE: spotify before_script: - - export HOST=mariadb + - export DB_HOST=mariadb - export DATABASE=spotify - export DB_USER=root - export DB_PASSWORD=root @@ -67,7 +67,7 @@ DeployKubernetes: - CLIENT_ID=$(echo -n ${CLIENT_ID} | base64 -w 0 | tr -d \\n) - CLIENT_SECRET=$(echo -n ${CLIENT_SECRET} | base64 -w 0 | tr -d \\n) - REDIRECT_URL=$(echo -n ${REDIRECT_URL} | base64 -w 0 | tr -d \\n) - - HOST=$(echo -n ${HOST} | base64 -w 0 | tr -d \\n) + - DB_HOST=$(echo -n ${DB_HOST} | base64 -w 0 | tr -d \\n) - DATABASE=$(echo -n ${DATABASE} | base64 -w 0 | tr -d \\n) - DB_USER=$(echo -n ${DB_USER} | base64 -w 0 | tr -d \\n) - DB_PASSWORD=$(echo -n ${DB_PASSWORD} | base64 -w 0 | tr -d \\n) @@ -75,7 +75,7 @@ DeployKubernetes: - sed -i "s,,${CLIENT_ID},g" kubernetes/secret.yaml - sed -i "s//${CLIENT_SECRET}/g" kubernetes/secret.yaml - sed -i "s,,${REDIRECT_URL},g" kubernetes/secret.yaml - - sed -i "s,,${HOST},g" kubernetes/secret.yaml + - sed -i "s,,${DB_HOST},g" kubernetes/secret.yaml - sed -i "s,,${DATABASE},g" kubernetes/secret.yaml - sed -i "s,,${DB_USER},g" kubernetes/secret.yaml - sed -i "s,,${DB_PASSWORD},g" kubernetes/secret.yaml diff --git a/AnalyticsforSpotify/settings.py b/AnalyticsforSpotify/settings.py index c0d2cc0..891c20d 100644 --- a/AnalyticsforSpotify/settings.py +++ b/AnalyticsforSpotify/settings.py @@ -89,7 +89,7 @@ 'NAME': os.environ.get('DATABASE'), 'USER': os.environ.get('DB_USER'), 'PASSWORD': os.environ.get('DB_PASSWORD'), - 'HOST': os.environ.get('HOST'), + 'HOST': os.environ.get('DB_HOST'), } } WSGI_APPLICATION = 'AnalyticsforSpotify.wsgi.application' diff --git a/kubernetes/deployment.yaml b/kubernetes/deployment.yaml index b12c7c5..c2a58ec 100644 --- a/kubernetes/deployment.yaml +++ b/kubernetes/deployment.yaml @@ -67,10 +67,10 @@ spec: name: env - name: DOCKER value: "YES" - - name: HOST + - name: DB_HOST valueFrom: secretKeyRef: - key: HOST + key: DB_HOST name: env - name: DB_PASSWORD valueFrom: diff --git a/kubernetes/secret.yaml b/kubernetes/secret.yaml index 0abc752..b5bfa1e 100644 --- a/kubernetes/secret.yaml +++ b/kubernetes/secret.yaml @@ -6,7 +6,7 @@ metadata: data: CLIENT_ID: "" CLIENT_SECRET: "" - HOST: + DB_HOST: DATABASE: DB_USER: DB_PASSWORD: diff --git a/setup.py b/setup.py index fc7a825..d89a169 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def dockerSetup(): R_URL = os.environ.get('REDIRECT_URL') API = spotifyAPI(CLIENT, SECRET, R_URL) - IP = os.environ.get('HOST') + IP = os.environ.get('DB_HOST') DB = os.environ.get('DATABASE') USER = os.environ.get('DB_USER') PASS = os.environ.get('DB_PASSWORD') From 381503f299171e5ebeeb8b8e88090b5cfcfb3af8 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 3 Nov 2021 19:20:51 -0400 Subject: [PATCH 11/16] Docker Compose Template Update --- docker/docker-compose.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 8c45cd0..2d1fb78 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -8,8 +8,8 @@ services: analytics-for-spotify: container_name: analytics-for-spotify hostname: analytics-for-spotify - ports: - - "8000:80" + ports: + - "8080:8080" env_file: .env image: ${REPOSITORY} environment: @@ -17,8 +17,7 @@ services: - CLIENT_ID=${CLIENT_ID} - CLIENT_SECRET=${CLIENT_SECRET} - REDIRECT_URL=${REDIRECT_URL} - - HOST=${HOST} + - DB_HOST=${DB_HOST} - DATABASE=${DATABASE} - - USER=${USERNAME} - - PASSWORD=${PASSWORD} - \ No newline at end of file + - DB_USER=${DB_USER} + - DB_PASSWORD=${DB_PASSWORD} From a1f9edf52e6c9369276770b9f3d9496b0b9b3b33 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 3 Nov 2021 22:21:45 -0400 Subject: [PATCH 12/16] django.contrib.auth Removal --- AnalyticsforSpotify/settings.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/AnalyticsforSpotify/settings.py b/AnalyticsforSpotify/settings.py index 891c20d..ae42354 100644 --- a/AnalyticsforSpotify/settings.py +++ b/AnalyticsforSpotify/settings.py @@ -32,7 +32,6 @@ # Application definition INSTALLED_APPS = [ - 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', @@ -58,7 +57,6 @@ 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', # Local Only Acesss 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] @@ -76,7 +74,6 @@ 'context_processors': [ 'django.template.context_processors.debug', 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', 'django.contrib.messages.context_processors.messages', ], }, @@ -97,20 +94,6 @@ # Password validation # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators -AUTH_PASSWORD_VALIDATORS = [ - { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', - }, -] LOGGING = { 'version': 1, From ffad730a7906b5cd0e4d3c8c3dbbfcfccd0e6e32 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 3 Nov 2021 23:03:20 -0400 Subject: [PATCH 13/16] django.contrib.messages Removal --- AnalyticsforSpotify/settings.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/AnalyticsforSpotify/settings.py b/AnalyticsforSpotify/settings.py index ae42354..20896a5 100644 --- a/AnalyticsforSpotify/settings.py +++ b/AnalyticsforSpotify/settings.py @@ -32,9 +32,7 @@ # Application definition INSTALLED_APPS = [ - 'django.contrib.contenttypes', 'django.contrib.sessions', - 'django.contrib.messages', 'webBackend.apps.WebBackendConfig', 'django_nose', ] From 9fdc4235eed4ea08728f1a503c980eece2ff1a9f Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 3 Nov 2021 23:13:35 -0400 Subject: [PATCH 14/16] Settings Cleanup --- AnalyticsforSpotify/settings.py | 42 --------------------------------- 1 file changed, 42 deletions(-) diff --git a/AnalyticsforSpotify/settings.py b/AnalyticsforSpotify/settings.py index 20896a5..9fa8299 100644 --- a/AnalyticsforSpotify/settings.py +++ b/AnalyticsforSpotify/settings.py @@ -1,36 +1,21 @@ """ Django settings for AnalyticsforSpotify project. - Generated by 'django-admin startproject' using Django 3.0.5. - -For more information on this file, see -https://docs.djangoproject.com/en/3.0/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/3.0/ref/settings/ """ - - import os # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ - # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = 'LocalAcessOnly' - # SECURITY WARNING: don't run with debug turned on in production! DEBUG = False ALLOWED_HOSTS = ['*'] - # Application definition - INSTALLED_APPS = [ 'django.contrib.sessions', 'webBackend.apps.WebBackendConfig', @@ -60,24 +45,8 @@ ] CORS_ORIGIN_ALLOW_ALL = True - ROOT_URLCONF = 'AnalyticsforSpotify.urls' -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.messages.context_processors.messages', - ], - }, - }, -] - DATABASES = { 'default': { 'ENGINE': 'django.db.backends.mysql', @@ -89,10 +58,6 @@ } WSGI_APPLICATION = 'AnalyticsforSpotify.wsgi.application' -# Password validation -# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators - - LOGGING = { 'version': 1, 'disable_existing_loggers': False, @@ -113,17 +78,10 @@ }, } -# Internationalization -# https://docs.djangoproject.com/en/3.0/topics/i18n/ - LANGUAGE_CODE = 'en-us' - TIME_ZONE = 'America/Detroit' - USE_I18N = True - USE_L10N = True - USE_TZ = True SESSION_COOKIE_HTTPONLY = True From 1791bfff30e374bb760692bde7d3e4a8b8bfc45c Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 3 Nov 2021 23:14:37 -0400 Subject: [PATCH 15/16] CSRF: Cors Origin --- AnalyticsforSpotify/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AnalyticsforSpotify/settings.py b/AnalyticsforSpotify/settings.py index 9fa8299..7a5aa13 100644 --- a/AnalyticsforSpotify/settings.py +++ b/AnalyticsforSpotify/settings.py @@ -39,11 +39,11 @@ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', - # Local Only Acesss 'django.middleware.csrf.CsrfViewMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', # Comment for Testing 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] -CORS_ORIGIN_ALLOW_ALL = True +CORS_ORIGIN_ALLOW_ALL = False # True For Testing ROOT_URLCONF = 'AnalyticsforSpotify.urls' From bf8240a5df70618344658cd4bcbf2ea48dd9abce Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 3 Nov 2021 23:21:26 -0400 Subject: [PATCH 16/16] docs cleanup --- AnalyticsforSpotify/urls.py | 16 +--------------- AnalyticsforSpotify/wsgi.py | 13 ++----------- webBackend/models.py | 7 ------- 3 files changed, 3 insertions(+), 33 deletions(-) diff --git a/AnalyticsforSpotify/urls.py b/AnalyticsforSpotify/urls.py index c9faa9d..5353d6c 100644 --- a/AnalyticsforSpotify/urls.py +++ b/AnalyticsforSpotify/urls.py @@ -1,18 +1,4 @@ -"""AnalyticsforSpotify URL Configuration - -The `urlpatterns` list routes URLs to views. For more information please see: - https://docs.djangoproject.com/en/3.0/topics/http/urls/ -Examples: -Function views - 1. Add an import: from my_app import views - 2. Add a URL to urlpatterns: path('', views.home, name='home') -Class-based views - 1. Add an import: from other_app.views import Home - 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') -Including another URLconf - 1. Import the include() function: from django.urls import include, path - 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) -""" +"""AnalyticsforSpotify URL Configuration""" from django.urls import include, path from django.conf import settings from django.conf.urls.static import static diff --git a/AnalyticsforSpotify/wsgi.py b/AnalyticsforSpotify/wsgi.py index f520650..b85d4d5 100644 --- a/AnalyticsforSpotify/wsgi.py +++ b/AnalyticsforSpotify/wsgi.py @@ -1,15 +1,6 @@ -""" -WSGI config for AnalyticsforSpotify project. - -It exposes the WSGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/ -""" - -import os - +"""WSGI config for AnalyticsforSpotify project.""" from django.core.wsgi import get_wsgi_application +import os os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'AnalyticsforSpotify.settings') application = get_wsgi_application() diff --git a/webBackend/models.py b/webBackend/models.py index 8f3b017..72059a0 100644 --- a/webBackend/models.py +++ b/webBackend/models.py @@ -1,10 +1,3 @@ -# This is an auto-generated Django model module. -# You'll have to do the following manually to clean this up: -# * Rearrange models' order -# * Make sure each model has one field with primary_key=True -# * Make sure each ForeignKey and OneToOneField has `on_delete` set to the desired behavior -# * Remove `managed = False` lines if you wish to allow Django to create, modify, and delete the table -# Feel free to rename the models, but don't rename db_table values or field names. from django.db import models