Merge pull request #226 from serguun42/bring-docker

Use Docker for CI/CD
serguun42 · Nov 15, 2023 · 9acf860 · 9acf860
2 parents 191d039 + 6e2daba
commit 9acf860
Show file tree

Hide file tree

Showing 51 changed files with 22,650 additions and 2,760 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -8,134 +8,72 @@ on:
   workflow_dispatch:
 
 jobs:
-
-  copy:
-    name: Delete old and copy new
+  configure:
+    name: Configure
     runs-on: self-hosted
 
     steps:
       - uses: actions/checkout@v3
-      - name: Copy fresh project
-        run: rm -rf ~/mss && cd .. && cp -r mss ~/mss
-      - name: Copy production configs
-        run: |
-          cp ~/MSS_CONFIGS/notifier-and-logger.config.json ~/mss/notifier/;
-          cp ~/MSS_CONFIGS/ci.config.json ~/mss/cicd/;
-          cp ~/MSS_CONFIGS/scrapper.config.json ~/mss/scrapper/;
-          cp ~/MSS_CONFIGS/backend.config.json ~/mss/backend/;
-          cp ~/MSS_CONFIGS/telegram-bot.config.json ~/mss/telegram/
-          [[ -f ~/MSS_CONFIGS/tag-manager.html ]] &&
-          cp ~/MSS_CONFIGS/tag-manager.html ~/mss/frontend/src/config/tag-manager.html ||
-          echo "" > ~/mss/frontend/src/config/tag-manager.html;
-          [[ -f ~/MSS_CONFIGS/apps-links.json ]] &&
-          cp ~/MSS_CONFIGS/apps-links.json ~/mss/frontend/src/config/apps-links.json ||
-          echo "[]" > ~/mss/frontend/src/config/apps-links.json;
-          [[ -d ~/MSS_CONFIGS/platforms/ ]] &&
-          cp -r ~/MSS_CONFIGS/platforms ~/mss/frontend/public/img/
-
-  notifier:
-    name: Deploy «notifier»
-    runs-on: self-hosted
-    needs: copy
-
-    steps:
-      - name: npm install
-        run: cd ~/mss/notifier/ && npm i --progress=false --omit=dev
-      - name: Delete old PM2 instance
+      - name: Copy configs
+        env:
+          CONFIGS_LOCATION_BASE: ${{ secrets.CONFIGS_LOCATION_BASE }}
         run: |
-          pm2 delete notifier-and-logger 2> /dev/null &&
-          echo "Process <notifier-and-logger> exited successfully" ||
-          echo "Process <notifier-and-logger> wasn't running"
-      - name: Start new PM2 instance
-        run: cd ~/mss/notifier/ && npm run production
-      - name: Delete previous PM2 logs
-        run: pm2 flush mss-notifier
-
-  cicd:
-    name: Deploy «ci/cd»
-    runs-on: self-hosted
-    needs: notifier
-
-    steps:
-      - name: npm install
-        run: cd ~/mss/cicd/ && npm i --progress=false --omit=dev
-
-  scrapper:
-    name: Deploy «scrapper»
+          cp $CONFIGS_LOCATION_BASE/notifier-and-logger.config.json ./notifier/;
+          cp $CONFIGS_LOCATION_BASE/scrapper.config.json ./scrapper/;
+          cp $CONFIGS_LOCATION_BASE/backend.config.json ./backend/;
+          cp $CONFIGS_LOCATION_BASE/telegram-bot.config.json ./telegram/
+          [[ -f $CONFIGS_LOCATION_BASE/tag-manager.html ]] &&
+          cp $CONFIGS_LOCATION_BASE/tag-manager.html ./frontend/src/config/tag-manager.html ||
+          echo "" > ./frontend/src/config/tag-manager.html;
+          [[ -f $CONFIGS_LOCATION_BASE/apps-links.json ]] &&
+          cp $CONFIGS_LOCATION_BASE/apps-links.json ./frontend/src/config/apps-links.json ||
+          echo "[]" > ./frontend/src/config/apps-links.json;
+          [[ -d $CONFIGS_LOCATION_BASE/platforms/ ]] &&
+          cp -r $CONFIGS_LOCATION_BASE/platforms ./frontend/public/img/
+
+  build:
+    name: Docker Build
     runs-on: self-hosted
-    needs: notifier
+    needs: configure
 
     steps:
-      - name: npm install
-        run: cd ~/mss/scrapper/ && npm i --progress=false --omit=dev
+      - name: Build Docker images
+        env:
+          CONFIGS_LOCATION_BASE: ${{ secrets.CONFIGS_LOCATION_BASE }}
+        run: docker compose --env-file $CONFIGS_LOCATION_BASE/docker.env build --no-cache
 
-  backend:
-    name: Deploy «backend»
+  deploy:
+    name: Docker Deploy
     runs-on: self-hosted
-    needs: notifier
+    needs: build
 
     steps:
-      - name: npm install
-        run: cd ~/mss/backend/ && npm i --progress=false --omit=dev
-      - name: Delete old PM2 instance
-        run: |
-          pm2 delete backend-server 2> /dev/null &&
-          echo "Process <backend-server> exited successfully" ||
-          echo "Process <backend-server> wasn't running"
-      - name: Start new PM2 instance
-        run: cd ~/mss/backend/ && npm run production
-      - name: Delete previous PM2 logs
-        run: pm2 flush mss-backend-server
+      - name: Deploy Docker images
+        env:
+          CONFIGS_LOCATION_BASE: ${{ secrets.CONFIGS_LOCATION_BASE }}
+        run: docker compose --env-file $CONFIGS_LOCATION_BASE/docker.env up -d --force-recreate --no-deps
 
-  telegram:
-    name: Deploy «telegram»
+  clean:
+    name: Clean cache and dangling images
     runs-on: self-hosted
-    needs: notifier
+    needs: deploy
 
     steps:
-      - name: npm install
-        run: cd ~/mss/telegram/ && npm i --progress=false --omit=dev
-      - name: Delete old PM2 instance
+      - name: Clean dangling images
         run: |
-          pm2 delete telegram-bot 2> /dev/null &&
-          echo "Process <telegram-bot> exited successfully" ||
-          echo "Process <telegram-bot> wasn't running"
-      - name: Start new PM2 instance
-        run: cd ~/mss/telegram/ && npm run production
-      - name: Delete previous PM2 logs
-        run: pm2 flush mss-telegram-bot
-
-  frontend:
-    name: Deploy «frontend»
-    runs-on: self-hosted
-    needs:
-      - cicd
-      - scrapper
-      - backend
-      - telegram
-
-    steps:
-      - name: npm install
-        run: cd ~/mss/frontend/ && npm i --progress=false --omit=dev
-      - name: npm run build
-        run: cd ~/mss/frontend/ && npm run build
-      - name: Generate Redoc bundle file
-        run: cd ~/mss/frontend/ && npm exec -- redoc-cli bundle public/docs/api/latest.yml -o dist/docs/api/redoc.html
-      - name: Clear node_modules
-        run: cd ~/mss/frontend/ && rm -rf node_modules
-      - name: Copy to NGINX
-        run: cp -r ~/mss/frontend/dist/* /usr/share/nginx/mss/
+          DANGLING_IMAGES=$(docker images -qf "dangling=true");
+          [[ ! -z "$DANGLING_IMAGES" ]] && docker rmi $DANGLING_IMAGES || echo "No dangling images"
 
-  final:
+  notify:
     name: Send final logs
     runs-on: self-hosted
-    needs: frontend
+    needs: deploy
 
     steps:
-      - name: Show PM2 processes list
-        run: pm2 list
-      - name: Send via notifier
+      - name: Send logs via notifier
+        env:
+          CONFIGS_LOCATION_BASE: ${{ secrets.CONFIGS_LOCATION_BASE }}
         run: |
-          [ -f ~/mss/cicd/ci-notify.js ] &&
-          node ~/mss/cicd/ci-notify.js "Done merge into master, initiator – $GITHUB_ACTOR" ||
+          source $CONFIGS_LOCATION_BASE/docker.env;
+          curl http://127.0.0.1:$NOTIFIER_PUBLISH_PORT -d "{\"error\":false,\"args\":[\"Done merge into master, initiator – $GITHUB_ACTOR\"],\"tag\":\"ci\"}" ||
           echo "Done merge into master, initiator – $GITHUB_ACTOR"
diff --git a/.github/workflows/notify-on-merge.yml b/.github/workflows/notify-on-merge.yml
@@ -3,9 +3,9 @@ name: Notify on merge
 on:
   pull_request:
     branches:
-      - '*'
-      - '*/*'
-      - '**'
+      - "*"
+      - "*/*"
+      - "**"
 
   workflow_dispatch:
 
@@ -14,7 +14,9 @@ jobs:
     runs-on: self-hosted
     steps:
       - name: Send notification
+        env:
+          CONFIGS_LOCATION_BASE: ${{ secrets.CONFIGS_LOCATION_BASE }}
         run: |
-          [ -f ~/mss/cicd/ci-notify.js ] &&
-          node ~/mss/cicd/ci-notify.js "User $GITHUB_ACTOR made a pull request" "From branch: $GITHUB_HEAD_REF" "To branch: $GITHUB_BASE_REF" ||
-          echo "User $GITHUB_ACTOR made a pull request" "From branch: $GITHUB_HEAD_REF" "To branch: $GITHUB_BASE_REF"
+          source $CONFIGS_LOCATION_BASE/docker.env;
+          curl http://127.0.0.1:$NOTIFIER_PUBLISH_PORT -d "{\"error\":false,\"args\":[\"User $GITHUB_ACTOR made a pull request\",\"From branch: $GITHUB_HEAD_REF\",\"To branch: $GITHUB_BASE_REF\"],\"tag\":\"ci\"}" ||
+          echo "Error while sending notification"
diff --git a/.github/workflows/notify-on-push.yml b/.github/workflows/notify-on-push.yml
@@ -3,10 +3,10 @@ name: Notify on push
 on:
   push:
     branches:
-      - '*'
-      - '*/*'
-      - '**'
-      - '!master'
+      - "*"
+      - "*/*"
+      - "**"
+      - "!master"
 
   workflow_dispatch:
 
@@ -15,7 +15,9 @@ jobs:
     runs-on: self-hosted
     steps:
       - name: Send notification
+        env:
+          CONFIGS_LOCATION_BASE: ${{ secrets.CONFIGS_LOCATION_BASE }}
         run: |
-          [ -f ~/mss/cicd/ci-notify.js ] &&
-          node ~/mss/cicd/ci-notify.js "User $GITHUB_ACTOR pushed to branch ${GITHUB_REF#refs/heads/}" ||
-          echo "User $GITHUB_ACTOR pushed to branch ${GITHUB_REF#refs/heads/}"
+          source $CONFIGS_LOCATION_BASE/docker.env;
+          curl http://127.0.0.1:$NOTIFIER_PUBLISH_PORT -d "{\"error\":false,\"args\":[\"User $GITHUB_ACTOR pushed to branch ${GITHUB_REF#refs/heads/}\"],\"tag\":\"ci\"}" ||
+          echo "Error while sending notification"
diff --git a/README.md b/README.md
@@ -5,12 +5,13 @@
 ## Complete system for MIREA schedule
 
 This repo represents all the code, structure and pipelines for MSS project, including
-* [Frontend](./frontend) for [mirea.xyz](https://mirea.xyz) site
-* [Backend](./backend) with API for that
-* [Telegram bot](./telegram) available by [@mirea_table_bot](https://t.me/mirea_table_bot)
-* [Scrapper](./scrapper) – script for parsing tables for each study group
-* [Notifier](./notifier) – collecting and sending logs and errors
-* [Android app](./app) – outdated, [see this page](https://mirea.xyz/apps)
+
+- [Frontend](./frontend) for [mirea.xyz](https://mirea.xyz) site
+- [Backend](./backend) with API for that
+- [Telegram bot](./telegram) available by [@mirea_table_bot](https://t.me/mirea_table_bot)
+- [Scrapper](./scrapper) – script for parsing tables for each study group
+- [Notifier](./notifier) – collecting and sending logs and errors
+- [Android app](./app) – outdated, [see this page](https://mirea.xyz/apps)
 
 ## Project's API
 
@@ -20,17 +21,25 @@ MSS has public API with common methods and actions. You can view and test it wit
 
 Each folder contains its own README with. Here's the list of them with responsible dev and main info for each:
 
-| Folder						| Dev											| What is used and how it works
-| ----------------------------- | :-------------------------------------------: | -----------------------------
-| [Backend](./backend)			| [@serguun42](https://github.com/serguun42)	| Handles back for front, back for API, back for user accounts. All the stuff.
-| [Frontend](./frontend)		| [@serguun42](https://github.com/serguun42)	| Front done with Vue.js. View and save groups' schedule.
-| [Telegram bot](./telegram)	| [@serguun42](https://github.com/serguun42)	| Sends schedule on demand, stores users, does mailing on morning, evening and late evening. Notifies via [notifier](./notifier), uses local Telegram API server (if specified), uses local MongoDB [mirea-table-bot](https://github.com/serguun42/mirea-table-bot) was the base for it.
-| [Scrapper](./scrapper)		| [@serguun42](https://github.com/serguun42)	| Parses schedule page, gets links to `.xlsx`-files, parses them then, builds table models for each and every possible study group, updates DB schedule for each group of those ones.
-| [Notifier](./notifier)		| [@serguun42](https://github.com/serguun42)	| Runs local HTTP server, notifies into *System Telegram*, logs into *stdout*, *stderr*. Use tags (inner and passed), determines which output(s) will be used to log/notify.
-| [CI/CD](./cicd)				| [@serguun42](https://github.com/serguun42)	| Single JS-script used only for notifying on CI/CD events. Also there are 2 Github Action workflow/pipeline scripts:<br><ul><li>[build.yml](.github/workflows/build.yml) contains main workflow for deploying production.</li><li>[notify.yml](.github/workflows/notify.yml) has only one job – to notify on event `push` into every branch except `master`.</li></ul>
-| [Android app](./app)			| [@rodyapal](https://github.com/rodyapal)		| Outdated, [see this page](https://mirea.xyz/apps).
+| Folder                     |                    Dev                     | What is used and how it works                                                                                                                                                                                                                                                          |
+| -------------------------- | :----------------------------------------: | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [Backend](./backend)       | [@serguun42](https://github.com/serguun42) | Handles back for front, back for API, back for user accounts. All the stuff.                                                                                                                                                                                                           |
+| [Frontend](./frontend)     | [@serguun42](https://github.com/serguun42) | Front done with Vue.js. View and save groups' schedule.                                                                                                                                                                                                                                |
+| [Telegram bot](./telegram) | [@serguun42](https://github.com/serguun42) | Sends schedule on demand, stores users, does mailing on morning, evening and late evening. Notifies via [notifier](./notifier), uses local Telegram API server (if specified), uses local MongoDB [mirea-table-bot](https://github.com/serguun42/mirea-table-bot) was the base for it. |
+| [Scrapper](./scrapper)     | [@serguun42](https://github.com/serguun42) | Parses schedule page, gets links to `.xlsx`-files, parses them then, builds table models for each and every possible study group, updates DB schedule for each group of those ones.                                                                                                    |
+| [Notifier](./notifier)     | [@serguun42](https://github.com/serguun42) | Runs local HTTP server, notifies into _System Telegram_, logs into _stdout_, _stderr_. Use tags (inner and passed), determines which output(s) will be used to log/notify.                                                                                                             |
+| [Android app](./app)       |  [@rodyapal](https://github.com/rodyapal)  | Outdated, [see this page](https://mirea.xyz/apps).                                                                                                                                                                                                                                     |
+
+## CI/CD
+
+Build & deploy are now being done with Docker. See workflow files and [docker-compose.yml](./docker-compose.yml). In total there are 3 Github Action workflow scripts:
+
+- [build.yml](.github/workflows/build.yml) contains main workflow for deploying production.
+- [notify-on-merge.yml](.github/workflows/notify-on-merge.yml) serves to notify on event `pull_request` into every branch.
+- [notify-on-push.yml](.github/workflows/notify-on-push.yml) serves to notify on event `push` into every branch except `master`.
 
 ---
 
 ### Этот проект, всё его содержимое и разработчики не связаны с администрацией РТУ МИРЭА.
+
 ### [License](./LICENSE)
diff --git a/backend/.gitignore b/backend/.gitignore
@@ -1,6 +1,5 @@
 .vscode
 /node_modules/
-package-lock.json
 
 /data/
 /out/

diff --git a/backend/Dockerfile b/backend/Dockerfile
@@ -0,0 +1,9 @@
+FROM node:lts-alpine
+
+WORKDIR /usr/src/app
+COPY . /usr/src/app
+
+ENV NODE_ENV=production
+RUN npm ci --omit=optional
+
+CMD [ "npm", "run", "production" ]