From e2650c7a4bf822b9f2f85763940c9dae5b49164f Mon Sep 17 00:00:00 2001 From: ztefanie Date: Thu, 12 Oct 2023 10:51:42 +0200 Subject: [PATCH] 1154: Add circleci config check and create a more saving commit workflow --- .circleci/config.yml | 1144 +++++++++-------- .circleci/src/@common.yml | 11 + .../src/commands/check-circleci-config.yml | 28 + .circleci/src/jobs/check.yml | 10 + .circleci/src/workflows/backend.yml | 1 + .circleci/src/workflows/commit.yml | 11 + .circleci/src/workflows/frontend.yml | 1 + scripts/circleci-update-config.sh | 1 + 8 files changed, 660 insertions(+), 547 deletions(-) create mode 100644 .circleci/src/commands/check-circleci-config.yml create mode 100644 .circleci/src/jobs/check.yml create mode 100644 .circleci/src/workflows/commit.yml create mode 100755 scripts/circleci-update-config.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index cc818be17..c02563250 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,554 +1,604 @@ -version: 2.1 +### AUTO GENERATED. DO NOT MODIFY. ### +# This file should be auto generated by the files in the src folder. +# You can update it by running `yarn run circleci:update-config`. +commands: + check-circleci-config: + description: This command builds the circle config from the files in src and validates that it is up-to-date and valid. + steps: + - run: + command: curl -fLSs https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/master/install.sh | sudo bash + name: Install CircleCI CLI + - run: + command: ./scripts/circleci-update-config.sh + name: Build circle config + - run: + command: | + FILES_MODIFIED="" + setcommit () { + FILES_MODIFIED=$(git status -s | grep -i -E '.*circleci/config.yml') + } + setcommit || true + if [ -z "$FILES_MODIFIED" ] + then + echo "The CircleCI config is up to date." + exit 0; + else + echo "The CircleCI config is not up to date. You can update it by running `yarn run circleci:update-config`." + exit 1; + fi + name: CircleCI config up to date + - run: + command: circleci config validate + name: Validate circle config + install-app-toolbelt: + steps: + - setup-npm-global + - run: + command: npm install --unsafe-perm -g https://github.com/digitalfabrik/app-toolbelt/archive/refs/heads/main.tar.gz + name: Install app-toolbelt + install-dart-linux: + steps: + - run: + command: |- + curl -o dart.deb https://storage.googleapis.com/dart-archive/channels/stable/release/3.0.5/linux_packages/dart_3.0.5-1_amd64.deb + sudo dpkg -i dart.deb + name: Install Dart + install-dart-mac: + steps: + - run: + command: |- + brew tap dart-lang/dart + brew install dart + name: Install Dart + install-fvm: + steps: + - run: + command: | + dart pub global activate fvm + echo 'export PATH=$HOME/.pub-cache/bin:$PATH' >> $BASH_ENV + name: Install FVM + - restore_cache: + keys: + - fvm-0-{{ checksum ".fvm/fvm_config.json" }}-{{ arch }} + - run: + command: fvm install + name: Install Flutter + - save_cache: + key: fvm-0-{{ checksum ".fvm/fvm_config.json" }}-{{ arch }} + paths: + - .fvm + - ~/fvm/ + - run: + command: fvm flutter --version + name: Show Flutter version + - run: + command: fvm flutter config --no-analytics + name: Configure Flutter + install-protobuf-linux: + steps: + - run: + command: | + curl -LO "https://github.com/protocolbuffers/protobuf/releases/download/v21.11/protoc-21.11-linux-x86_64.zip" + unzip protoc*.zip -d $HOME/.local + echo 'export PATH=$HOME/.local/bin:$PATH' >> $BASH_ENV + name: Install protobuf from GitHub + - run: + command: dart pub global activate protoc_plugin + name: Install Flutter plugin + install-protobuf-mac: + steps: + - run: + command: | + curl -LO "https://github.com/protocolbuffers/protobuf/releases/download/v21.11/protoc-21.11-osx-x86_64.zip" + unzip protoc*.zip -d $HOME/.local + echo 'export PATH=$HOME/.local/bin:$PATH' >> $BASH_ENV + name: Install protobuf + - run: + command: dart pub global activate protoc_plugin + name: Install Flutter plugin + setup-npm-global: + steps: + - run: + command: |- + mkdir -p ~/.npm-global + npm config set prefix '~/.npm-global' + echo 'export PATH=~/.npm-global/bin:"$PATH"' >> "$BASH_ENV" + name: Setup npm +jobs: + administration-build: + docker: + - image: cimg/node:19.1.0 + environment: null + steps: + - checkout: + path: ~/project + - restore_cache: + keys: + - v1-node-modules-{{ checksum "package.json" }}-{{checksum "package-lock.json" }} + - install-dart-linux + - install-protobuf-linux + - run: + command: '[ ! -d node_modules ] && npm ci --loglevel warn --yes || echo package.json and package-lock.json unchanged. Using cache.' + name: Install node dependencies + - run: | + npm run generate-graphql + npm run generate-protobuf + - run: + command: npm run lint + name: Lint + - run: + command: npm run test -- --maxWorkers=2 + name: Test + - run: + command: npm run build + name: Build + - save_cache: + key: v1-node-modules-{{ checksum "package.json" }}-{{checksum "package-lock.json" }} + paths: + - node_modules + - store_artifacts: + path: build + - persist_to_workspace: + paths: + - administration/build + root: ~/project + working_directory: ~/project/administration + administration-pack: + docker: + - image: debian:11 + steps: + - checkout: + path: ~/project + - attach_workspace: + at: /tmp/workspace + - run: ~/project/scripts/pack_deb.sh -v "0.$CIRCLE_BUILD_NUM" -f /tmp/workspace/administration/build -d "Administration backend for the Ehrenamtskarte app" -n "eak-administration" + - run: | + mkdir -p /tmp/artifacts/debs + mv *.deb /tmp/artifacts/debs + - store_artifacts: + path: /tmp/artifacts + - persist_to_workspace: + paths: + - debs/*.deb + root: /tmp/artifacts + working_directory: ~/project/administration + backend-build: + docker: + - image: cimg/openjdk:17.0.6-node + environment: + _JAVA_OPTIONS: -Xmx3g + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=2 + steps: + - checkout: + path: ~/project + - run: git submodule sync + - run: git submodule update --init + - restore_cache: + key: v2-gradle-wrapper-{{ checksum "gradle/wrapper/gradle-wrapper.properties" }} + - restore_cache: + key: v2-gradle-cache-{{ checksum "build.gradle.kts" }} + - run: + command: | + ./gradlew run --args="graphql-export ../specs/backend-api.graphql" + git diff --exit-code + name: Check that GraphQL scheme is stable + - run: + command: | + ./gradlew test + name: Test + - run: ./gradlew build + - save_cache: + key: v2-gradle-wrapper-{{ checksum "gradle/wrapper/gradle-wrapper.properties" }} + paths: + - .gradle/wrapper + - save_cache: + key: v2-gradle-cache-{{ checksum "build.gradle.kts" }} + paths: + - .gradle/caches + - store_artifacts: + path: build/libs + - persist_to_workspace: + paths: + - backend/build/distributions/* + root: ~/project + working_directory: ~/project/backend + backend-health-check: + docker: + - image: cimg/base:2023.03 + - environment: + - POSTGRES_DB=ehrenamtskarte + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + image: postgis/postgis:13-3.0-alpine + steps: + - checkout: + path: ~/project + - attach_workspace: + at: /tmp/workspace + - run: + command: | + sudo apt update + sudo apt install curl -y + name: Install Curl + - run: + command: sudo apt install ca-certificates-java + name: Install ca-certificates-java + - run: + command: sudo apt install openjdk-17-jre-headless -y + name: Install openjdk-17-jre-headless + - run: + command: sudo dpkg -i /tmp/workspace/debs/eak-backend*.deb + name: Install backend + - run: + command: /opt/ehrenamtskarte/backend/bin/backend migrate + name: Run migrate + - run: + background: true + command: /opt/ehrenamtskarte/backend/bin/backend execute + name: Start backend + - run: + command: curl --retry 120 --retry-delay 1 --retry-all-errors http://0.0.0.0:8000/health + name: Check health + backend-pack: + docker: + - image: debian:11 + steps: + - checkout: + path: ~/project + - attach_workspace: + at: /tmp/workspace + - run: ~/project/scripts/pack_deb.sh -v "0.$CIRCLE_BUILD_NUM" -t /tmp/workspace/backend/build/distributions/*.tar -s ~/project/scripts/eak-backend.service -d "Backend server for the Ehrenamtskarte app" -n "eak-backend" -c "openjdk-17-jre-headless" + - run: | + mkdir -p /tmp/artifacts/debs + mv *.deb /tmp/artifacts/debs + - store_artifacts: + path: /tmp/artifacts + - persist_to_workspace: + paths: + - debs/*.deb + root: /tmp/artifacts + working_directory: ~/project/backend + check: + docker: + - image: cimg/node:19.1.0-browsers + environment: + TOTAL_CPUS: 1 + TZ: Europe/Berlin + resource_class: small + shell: /bin/bash -eo pipefail + steps: + - checkout + - check-circleci-config + check-frontend: + docker: + - image: cimg/node:19.1.0-browsers + resource_class: small + steps: + - checkout: + path: ~/project + - browser-tools/install-chrome + - install-dart-linux + - install-fvm + - install-app-toolbelt + - install-protobuf-linux + - run: + command: fvm flutter pub get --enforce-lockfile + name: Install Flutter Packages + - run: + command: fvm dart format -l 120 -o none --set-exit-if-changed . + name: Check Formatting + - run: + command: | + # Statically use "bayern" build config for analyzing here + fvm flutter pub run build_runner build --define "df_build_config=name=bayern" + name: Build Runner + - run: + command: | + fvm flutter analyze --fatal-infos --fatal-warnings + fvm flutter analyze pubs/df_build_config --fatal-infos --fatal-warnings + fvm flutter analyze pubs/df_protobuf --fatal-infos --fatal-warnings + name: Check Analyzer and Linting + - run: + command: |- + fvm flutter test + fvm flutter test pubs/df_build_config + fvm flutter test pubs/df_protobuf + name: Tests + working_directory: ~/project/frontend + deploy: + docker: + - image: cimg/base:2022.09 + parameters: + server: + enum: + - entitlementcard.tuerantuer.org + - entitlementcard-test.tuerantuer.org + type: enum + ssh-host-fingerprint: + enum: + - '|1|dkYQrdGB1QML0o+POL3QzAkBbek=|b4Tm0Ymh82UKyZPJfVKy4t+MFV8= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIONuARu18Fktz+j4QosI+nRqgMnFOMgE7OZLuTOwgZ0k' + - '|1|iikuvSrIo3wkj+EqUgLRMsAq6yk=|r9bSjkawWFa94b45qE/se5Oio5k= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIED3MwobbCs+ENMLXdyqlJb3bJ26TuIYt977TA3NrN66' + type: enum + steps: + - attach_workspace: + at: /tmp/workspace + - add_ssh_keys: + fingerprints: + - a1:3f:a7:c3:ff:12:40:1d:85:de:a7:ab:12:3f:cc:05 + - run: + command: |- + echo "<< parameters.ssh-host-fingerprint >>" >> known_hosts + echo "Uploading: " /tmp/workspace/debs/*.deb + sftp -b - -o UserKnownHostsFile=known_hosts ci@<< parameters.server >>:/local-apt-repository/ \<<< "put -r /tmp/workspace/debs/*.deb" + name: SFTP upload + frontend-android-build: + docker: + - image: cimg/android:2022.09.2-node + environment: + GRADLE_OPTS: -Dorg.gradle.jvmargs="-Xmx2048m" -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=2 + parameters: + buildConfig: + description: Name of the build config to use + type: string + flutterFlavor: + description: Flutter Flavor to use + type: string + resource_class: large + steps: + - checkout: + path: ~/project + - install-dart-linux + - install-fvm + - install-app-toolbelt + - install-protobuf-linux + - run: + command: | + fvm flutter pub get --enforce-lockfile + fvm flutter precache --android + name: Install Flutter Packages + - run: + command: | + fvm flutter pub run build_runner build --define "df_build_config=name=<< parameters.buildConfig >>" + name: Build Runner + - run: + command: | + fvm flutter build apk --dart-define=environment=production --flavor << parameters.flutterFlavor >> --release -t lib/main.dart + name: Build + - store_artifacts: + path: build/app/outputs/flutter-apk/ + working_directory: ~/project/frontend + frontend-ios-build: + macos: + xcode: 14.0.0 + parameters: + buildConfig: + description: Name of the build config to use + type: string + flutterFlavor: + description: Flutter Flavor to use + type: string + steps: + - checkout: + path: ~/project + - install-dart-mac + - install-fvm + - install-app-toolbelt + - install-protobuf-mac + - run: + command: | + fvm flutter pub get --enforce-lockfile + fvm flutter precache --ios + name: Install Flutter Packages + - run: + command: | + cd ios + pod update # This command is somehow needed: https://github.com/m0nac0/flutter-maplibre-gl/pull/9 + pod install + name: Update Pods + - run: + command: | + fvm flutter pub run build_runner build --define "df_build_config=name=<< parameters.buildConfig >>" + name: Build Runner + - run: + command: | + app-toolbelt v0 build-config write-xcconfig "<< parameters.buildConfig >>" ios --directory ios/ + fvm flutter build ios --dart-define=environment=production --flavor << parameters.flutterFlavor >> --no-codesign --release -t lib/main.dart + name: Build + working_directory: ~/project/frontend + martin-build: + docker: + - image: rust:bullseye + steps: + - run: + command: | + mkdir -p ~/.ssh + touch ~/.ssh/known_hosts + echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" >> ~/.ssh/known_hosts + git clone git@github.com:urbica/martin.git ~/martin + git config advice.detachedHead false + git checkout ed14582a8f5c3e11bfb165f3b012edccd929b479 + name: Checkout + - restore_cache: + keys: + - v1-cargo-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }} + - run: + command: | + apt update + apt install -y openssl libssl-dev + name: Install OpenSSL + - run: + command: | + cargo build --release --target x86_64-unknown-linux-gnu + name: Build + - save_cache: + key: v1-cargo-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }} + paths: + - ~/.cargo + - target + - run: + command: | + mkdir -p /tmp/artifacts/martin + cp target/x86_64-unknown-linux-gnu/release/martin /tmp/artifacts/martin + name: Prepare storing + - store_artifacts: + path: /tmp/artifacts + - persist_to_workspace: + paths: + - martin/martin + root: /tmp/artifacts + working_directory: ~/martin + martin-pack: + docker: + - image: debian:11 + steps: + - checkout: + path: ~/project + - attach_workspace: + at: /tmp/workspace + - run: + command: | + cp ~/project/docker/martin-config.yaml config.yaml + cp /tmp/workspace/martin/martin . + ~/project/scripts/pack_deb.sh -v "0.$CIRCLE_BUILD_NUM" -d "Martin tile server for the Ehrenamtskarte app" -n "eak-martin" -s ~/project/scripts/eak-martin.service -C "/opt/ehrenamtskarte/martin/config.yaml" -M . + name: Build .deb + - run: + command: | + mkdir -p /tmp/artifacts/debs + mv *.deb /tmp/artifacts/debs + name: Move .deb to artifacts folder + - store_artifacts: + path: /tmp/artifacts + - persist_to_workspace: + paths: + - debs/*.deb + root: /tmp/artifacts + working_directory: ~/project/map-tiles/martin + meta-pack: + docker: + - image: debian:11 + steps: + - checkout: + path: ~/project + - run: ~/project/scripts/pack_deb.sh -v "0.$CIRCLE_BUILD_NUM" -d "Meta package for the Ehrenamtskarte app" -n "eak" -c "eak-backend, eak-administration, eak-martin" + - run: | + mkdir -p /tmp/artifacts/debs + cp *.deb /tmp/artifacts/debs + - store_artifacts: + path: /tmp/artifacts + - persist_to_workspace: + paths: + - debs/*.deb + root: /tmp/artifacts + working_directory: ~/project/administration orbs: - gradle: circleci/gradle@2.2.0 - browser-tools: circleci/browser-tools@1.4.1 + browser-tools: circleci/browser-tools@1.4.1 + gradle: circleci/gradle@2.2.0 +parameters: + run_backend: + default: false + type: boolean + run_commit: + default: false + type: boolean + run_frontend: + default: false + type: boolean +version: 2.1 workflows: - frontend: - jobs: - - check-frontend - - frontend-android-build: - name: frontend-android-build-bayern-floss - buildConfig: "bayern-floss" - flutterFlavor: "BayernFloss" - requires: - - check-frontend - - frontend-android-build: - name: frontend-android-build-bayern - buildConfig: "bayern" - flutterFlavor: "Bayern" - requires: - - check-frontend - - frontend-ios-build: - name: frontend-ios-build-bayern - buildConfig: "bayern" - flutterFlavor: "Bayern" - requires: - - check-frontend - - frontend-android-build: - name: frontend-android-build-nuernberg - buildConfig: "nuernberg" - flutterFlavor: "Nuernberg" - requires: - - check-frontend - - frontend-ios-build: - name: frontend-ios-build-nuernberg - buildConfig: "nuernberg" - flutterFlavor: "Nuernberg" - requires: - - check-frontend - backend: - jobs: - - backend-build - - administration-build - - martin-build - - backend-pack: - requires: + backend: + jobs: - backend-build - - backend-health-check: - requires: - - backend-pack - - administration-pack: - requires: - administration-build - - martin-pack: - requires: - martin-build - - meta-pack - - deploy: - name: deploy-staging - server: entitlementcard-test.tuerantuer.org - ssh-host-fingerprint: '|1|dkYQrdGB1QML0o+POL3QzAkBbek=|b4Tm0Ymh82UKyZPJfVKy4t+MFV8= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIONuARu18Fktz+j4QosI+nRqgMnFOMgE7OZLuTOwgZ0k' - context: - - entitlementcard-apt - requires: - - backend-pack - - backend-health-check - - administration-pack - - martin-pack + - backend-pack: + requires: + - backend-build + - backend-health-check: + requires: + - backend-pack + - administration-pack: + requires: + - administration-build + - martin-pack: + requires: + - martin-build - meta-pack - filters: - branches: - only: - - /release.*/ - - deploy: - name: deploy-production - server: entitlementcard.tuerantuer.org - ssh-host-fingerprint: '|1|iikuvSrIo3wkj+EqUgLRMsAq6yk=|r9bSjkawWFa94b45qE/se5Oio5k= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIED3MwobbCs+ENMLXdyqlJb3bJ26TuIYt977TA3NrN66' - context: - - entitlementcard-apt - requires: - - backend-pack - - administration-pack - - martin-pack - - meta-pack - filters: - branches: - only: - - /release.*/ - -commands: - setup-npm-global: - steps: - - run: - name: Setup npm - command: | - mkdir -p ~/.npm-global - npm config set prefix '~/.npm-global' - echo 'export PATH=~/.npm-global/bin:"$PATH"' >> "$BASH_ENV" - install-app-toolbelt: - steps: - - setup-npm-global - - run: - name: Install app-toolbelt - command: npm install --unsafe-perm -g https://github.com/digitalfabrik/app-toolbelt/archive/refs/heads/main.tar.gz - install-dart-linux: - steps: - - run: - name: Install Dart - command: | - curl -o dart.deb https://storage.googleapis.com/dart-archive/channels/stable/release/3.0.5/linux_packages/dart_3.0.5-1_amd64.deb - sudo dpkg -i dart.deb - install-dart-mac: - steps: - - run: - name: Install Dart - command: | - brew tap dart-lang/dart - brew install dart - install-protobuf-linux: - steps: - - run: - name: Install protobuf from GitHub - command: | - curl -LO "https://github.com/protocolbuffers/protobuf/releases/download/v21.11/protoc-21.11-linux-x86_64.zip" - unzip protoc*.zip -d $HOME/.local - echo 'export PATH=$HOME/.local/bin:$PATH' >> $BASH_ENV - - run: - name: Install Flutter plugin - # Do not use the dart from fvm here for installing protoc_plugin. - # Else we might get a "Can't load Kernel binary: Invalid kernel binary format version" because the dart - # versions do not match. - command: dart pub global activate protoc_plugin - install-protobuf-mac: - steps: - - run: - name: Install protobuf - command: | - curl -LO "https://github.com/protocolbuffers/protobuf/releases/download/v21.11/protoc-21.11-osx-x86_64.zip" - unzip protoc*.zip -d $HOME/.local - echo 'export PATH=$HOME/.local/bin:$PATH' >> $BASH_ENV - - run: - name: Install Flutter plugin - # Use dart directly here. See comment in install-protobuf-linux - command: dart pub global activate protoc_plugin - install-fvm: - steps: - - run: - name: Install FVM - command: | - dart pub global activate fvm - echo 'export PATH=$HOME/.pub-cache/bin:$PATH' >> $BASH_ENV - - restore_cache: - keys: - - fvm-0-{{ checksum ".fvm/fvm_config.json" }}-{{ arch }} - - run: - name: Install Flutter - command: fvm install - - save_cache: - key: fvm-0-{{ checksum ".fvm/fvm_config.json" }}-{{ arch }} - paths: - - .fvm - - ~/fvm/ - - run: - name: Show Flutter version - command: fvm flutter --version - - run: - name: Configure Flutter - command: fvm flutter config --no-analytics - -jobs: - check-frontend: - docker: - - image: cimg/node:19.1.0-browsers - resource_class: small - working_directory: ~/project/frontend - steps: - - checkout: - path: ~/project - - browser-tools/install-chrome - - install-dart-linux - - install-fvm - - install-app-toolbelt - - install-protobuf-linux - - run: - name: Install Flutter Packages - command: fvm flutter pub get --enforce-lockfile - - run: - name: Check Formatting - command: fvm dart format -l 120 -o none --set-exit-if-changed . - - run: - name: Build Runner - command: | - # Statically use "bayern" build config for analyzing here - fvm flutter pub run build_runner build --define "df_build_config=name=bayern" - - run: - name: Check Analyzer and Linting - command: | - fvm flutter analyze --fatal-infos --fatal-warnings - fvm flutter analyze pubs/df_build_config --fatal-infos --fatal-warnings - fvm flutter analyze pubs/df_protobuf --fatal-infos --fatal-warnings - - run: - name: Tests - command: | - fvm flutter test - fvm flutter test pubs/df_build_config - fvm flutter test pubs/df_protobuf - - frontend-android-build: - docker: - - image: cimg/android:2022.09.2-node - parameters: - buildConfig: - description: "Name of the build config to use" - type: string - flutterFlavor: - description: "Flutter Flavor to use" - type: string - environment: - GRADLE_OPTS: '-Dorg.gradle.jvmargs="-Xmx2048m" -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=2' - resource_class: large - working_directory: ~/project/frontend - steps: - - checkout: - path: ~/project - - install-dart-linux - - install-fvm - - install-app-toolbelt - - install-protobuf-linux - - run: - name: Install Flutter Packages - command: | - fvm flutter pub get --enforce-lockfile - fvm flutter precache --android - - run: - name: Build Runner - command: | - fvm flutter pub run build_runner build --define "df_build_config=name=<< parameters.buildConfig >>" - - run: - name: Build - command: | - fvm flutter build apk --dart-define=environment=production --flavor << parameters.flutterFlavor >> --release -t lib/main.dart - - store_artifacts: - path: build/app/outputs/flutter-apk/ - - frontend-ios-build: - macos: - xcode: 14.0.0 - parameters: - buildConfig: - description: "Name of the build config to use" - type: string - flutterFlavor: - description: "Flutter Flavor to use" - type: string - working_directory: ~/project/frontend - steps: - - checkout: - path: ~/project - - install-dart-mac - - install-fvm - - install-app-toolbelt - - install-protobuf-mac - - run: - name: Install Flutter Packages - command: | - fvm flutter pub get --enforce-lockfile - fvm flutter precache --ios - - run: - name: Update Pods - command: | - cd ios - pod update # This command is somehow needed: https://github.com/m0nac0/flutter-maplibre-gl/pull/9 - pod install - - run: - name: Build Runner - command: | - fvm flutter pub run build_runner build --define "df_build_config=name=<< parameters.buildConfig >>" - - run: - name: Build - command: | - app-toolbelt v0 build-config write-xcconfig "<< parameters.buildConfig >>" ios --directory ios/ - fvm flutter build ios --dart-define=environment=production --flavor << parameters.flutterFlavor >> --no-codesign --release -t lib/main.dart - - backend-build: - environment: - _JAVA_OPTIONS: "-Xmx3g" - GRADLE_OPTS: "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=2" - docker: - - image: cimg/openjdk:17.0.6-node - working_directory: ~/project/backend - steps: - - checkout: - path: ~/project - - run: git submodule sync - - run: git submodule update --init - - restore_cache: - key: v2-gradle-wrapper-{{ checksum "gradle/wrapper/gradle-wrapper.properties" }} - - restore_cache: - key: v2-gradle-cache-{{ checksum "build.gradle.kts" }} - - run: - name: Check that GraphQL scheme is stable - command: | - ./gradlew run --args="graphql-export ../specs/backend-api.graphql" - git diff --exit-code - - run: - name: Test - command: | - ./gradlew test - - run: ./gradlew build - - save_cache: - paths: - - .gradle/wrapper - key: v2-gradle-wrapper-{{ checksum "gradle/wrapper/gradle-wrapper.properties" }} - - save_cache: - paths: - - .gradle/caches - key: v2-gradle-cache-{{ checksum "build.gradle.kts" }} - - store_artifacts: - path: build/libs - - persist_to_workspace: - root: ~/project - paths: - - backend/build/distributions/* - - administration-build: - docker: - - image: cimg/node:19.1.0 - environment: - working_directory: ~/project/administration - steps: - - checkout: - path: ~/project - - restore_cache: - keys: - - v1-node-modules-{{ checksum "package.json" }}-{{checksum "package-lock.json" }} - - install-dart-linux - - install-protobuf-linux - - run: - name: Install node dependencies - command: "[ ! -d node_modules ] && npm ci --loglevel warn --yes || echo package.json and package-lock.json unchanged. Using cache." - - run: | - npm run generate-graphql - npm run generate-protobuf - - run: - name: Lint - command: npm run lint - - run: - name: Test - command: npm run test -- --maxWorkers=2 - - run: - name: Build - command: npm run build - - save_cache: - paths: - - node_modules - key: v1-node-modules-{{ checksum "package.json" }}-{{checksum "package-lock.json" }} - - store_artifacts: - path: build - - persist_to_workspace: - root: ~/project - paths: - - administration/build - - martin-build: - docker: - - image: rust:bullseye - working_directory: ~/martin - steps: - - run: - name: "Checkout" - command: | - mkdir -p ~/.ssh - touch ~/.ssh/known_hosts - echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=" >> ~/.ssh/known_hosts - git clone git@github.com:urbica/martin.git ~/martin - git config advice.detachedHead false - git checkout ed14582a8f5c3e11bfb165f3b012edccd929b479 - - restore_cache: - keys: - - v1-cargo-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }} - - run: - name: "Install OpenSSL" - command: | - apt update - apt install -y openssl libssl-dev - - run: - name: "Build" - command: | - cargo build --release --target x86_64-unknown-linux-gnu - - save_cache: - paths: - - "~/.cargo" - - "target" - key: v1-cargo-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }} - - run: - name: "Prepare storing" - command: | - mkdir -p /tmp/artifacts/martin - cp target/x86_64-unknown-linux-gnu/release/martin /tmp/artifacts/martin - - store_artifacts: - path: /tmp/artifacts - - persist_to_workspace: - root: /tmp/artifacts - paths: - - martin/martin - - backend-pack: - docker: - - image: debian:11 # We deploy on debian -> pack on debian - working_directory: ~/project/backend - steps: - - checkout: - path: ~/project - - attach_workspace: - at: /tmp/workspace - - run: ~/project/scripts/pack_deb.sh -v "0.$CIRCLE_BUILD_NUM" -t /tmp/workspace/backend/build/distributions/*.tar -s ~/project/scripts/eak-backend.service -d "Backend server for the Ehrenamtskarte app" -n "eak-backend" -c "openjdk-17-jre-headless" - - run: | - mkdir -p /tmp/artifacts/debs - mv *.deb /tmp/artifacts/debs - - store_artifacts: - path: /tmp/artifacts - - persist_to_workspace: - root: /tmp/artifacts - paths: - - debs/*.deb - - backend-health-check: - docker: - - image: cimg/base:2023.03 - - image: postgis/postgis:13-3.0-alpine - environment: - - POSTGRES_DB=ehrenamtskarte - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=postgres - steps: - - checkout: - path: ~/project - - attach_workspace: - at: /tmp/workspace - - run: - name: Install Curl - command: | - sudo apt update - sudo apt install curl -y - - run: - name: Install ca-certificates-java # workaround for current issue: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1030129 - command: sudo apt install ca-certificates-java - - run: - name: Install openjdk-17-jre-headless - command: sudo apt install openjdk-17-jre-headless -y - - run: - name: Install backend - command: sudo dpkg -i /tmp/workspace/debs/eak-backend*.deb - - run: - name: Run migrate - command: /opt/ehrenamtskarte/backend/bin/backend migrate - - run: - name: Start backend - command: /opt/ehrenamtskarte/backend/bin/backend execute - background: true - - run: - name: Check health - command: curl --retry 120 --retry-delay 1 --retry-all-errors http://0.0.0.0:8000/health - - administration-pack: - docker: - - image: debian:11 # We deploy on debian -> pack on debian - working_directory: ~/project/administration - steps: - - checkout: - path: ~/project - - attach_workspace: - at: /tmp/workspace - - run: ~/project/scripts/pack_deb.sh -v "0.$CIRCLE_BUILD_NUM" -f /tmp/workspace/administration/build -d "Administration backend for the Ehrenamtskarte app" -n "eak-administration" - - run: | - mkdir -p /tmp/artifacts/debs - mv *.deb /tmp/artifacts/debs - - store_artifacts: - path: /tmp/artifacts - - persist_to_workspace: - root: /tmp/artifacts - paths: - - debs/*.deb - - martin-pack: - docker: - - image: debian:11 # We deploy on debian -> pack on debian - working_directory: ~/project/map-tiles/martin - steps: - - checkout: - path: ~/project - - attach_workspace: - at: /tmp/workspace - - run: - name: "Build .deb" - command: | - cp ~/project/docker/martin-config.yaml config.yaml - cp /tmp/workspace/martin/martin . - ~/project/scripts/pack_deb.sh -v "0.$CIRCLE_BUILD_NUM" -d "Martin tile server for the Ehrenamtskarte app" -n "eak-martin" -s ~/project/scripts/eak-martin.service -C "/opt/ehrenamtskarte/martin/config.yaml" -M . - - run: - name: "Move .deb to artifacts folder" - command: | - mkdir -p /tmp/artifacts/debs - mv *.deb /tmp/artifacts/debs - - store_artifacts: - path: /tmp/artifacts - - persist_to_workspace: - root: /tmp/artifacts - paths: - - debs/*.deb - - meta-pack: - docker: - - image: debian:11 # We deploy on debian -> pack on debian - working_directory: ~/project/administration - steps: - - checkout: - path: ~/project - - run: ~/project/scripts/pack_deb.sh -v "0.$CIRCLE_BUILD_NUM" -d "Meta package for the Ehrenamtskarte app" -n "eak" -c "eak-backend, eak-administration, eak-martin" - - run: | - mkdir -p /tmp/artifacts/debs - cp *.deb /tmp/artifacts/debs - - store_artifacts: - path: /tmp/artifacts - - persist_to_workspace: - root: /tmp/artifacts - paths: - - debs/*.deb + - deploy: + context: + - entitlementcard-apt + filters: + branches: + only: + - /release.*/ + name: deploy-staging + requires: + - backend-pack + - backend-health-check + - administration-pack + - martin-pack + - meta-pack + server: entitlementcard-test.tuerantuer.org + ssh-host-fingerprint: '|1|dkYQrdGB1QML0o+POL3QzAkBbek=|b4Tm0Ymh82UKyZPJfVKy4t+MFV8= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIONuARu18Fktz+j4QosI+nRqgMnFOMgE7OZLuTOwgZ0k' + - deploy: + context: + - entitlementcard-apt + filters: + branches: + only: + - /release.*/ + name: deploy-production + requires: + - backend-pack + - administration-pack + - martin-pack + - meta-pack + server: entitlementcard.tuerantuer.org + ssh-host-fingerprint: '|1|iikuvSrIo3wkj+EqUgLRMsAq6yk=|r9bSjkawWFa94b45qE/se5Oio5k= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIED3MwobbCs+ENMLXdyqlJb3bJ26TuIYt977TA3NrN66' + when: << pipeline.parameters.run_backend >> + commit: + jobs: + - check + - backend-build + - backend-pack: + requires: + - backend-build + - backend-health-check: + requires: + - backend-pack + - check-frontend + when: << pipeline.parameters.run_commit >> + frontend: + jobs: + - check-frontend + - frontend-android-build: + buildConfig: bayern-floss + flutterFlavor: BayernFloss + name: frontend-android-build-bayern-floss + requires: + - check-frontend + - frontend-android-build: + buildConfig: bayern + flutterFlavor: Bayern + name: frontend-android-build-bayern + requires: + - check-frontend + - frontend-ios-build: + buildConfig: bayern + flutterFlavor: Bayern + name: frontend-ios-build-bayern + requires: + - check-frontend + - frontend-android-build: + buildConfig: nuernberg + flutterFlavor: Nuernberg + name: frontend-android-build-nuernberg + requires: + - check-frontend + - frontend-ios-build: + buildConfig: nuernberg + flutterFlavor: Nuernberg + name: frontend-ios-build-nuernberg + requires: + - check-frontend + when: << pipeline.parameters.run_frontend >> - deploy: - docker: - - image: cimg/base:2022.09 - parameters: - server: - enum: - - entitlementcard.tuerantuer.org - - entitlementcard-test.tuerantuer.org - type: enum - ssh-host-fingerprint: - enum: - - '|1|dkYQrdGB1QML0o+POL3QzAkBbek=|b4Tm0Ymh82UKyZPJfVKy4t+MFV8= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIONuARu18Fktz+j4QosI+nRqgMnFOMgE7OZLuTOwgZ0k' - - '|1|iikuvSrIo3wkj+EqUgLRMsAq6yk=|r9bSjkawWFa94b45qE/se5Oio5k= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIED3MwobbCs+ENMLXdyqlJb3bJ26TuIYt977TA3NrN66' - type: enum - steps: - - attach_workspace: - at: /tmp/workspace - - add_ssh_keys: - fingerprints: - - "a1:3f:a7:c3:ff:12:40:1d:85:de:a7:ab:12:3f:cc:05" - - run: - name: "SFTP upload" - command: | - echo "<< parameters.ssh-host-fingerprint >>" >> known_hosts - echo "Uploading: " /tmp/workspace/debs/*.deb - sftp -b - -o UserKnownHostsFile=known_hosts ci@<< parameters.server >>:/local-apt-repository/ \<<< "put -r /tmp/workspace/debs/*.deb" diff --git a/.circleci/src/@common.yml b/.circleci/src/@common.yml index b383a641d..f2c3d77c8 100644 --- a/.circleci/src/@common.yml +++ b/.circleci/src/@common.yml @@ -3,3 +3,14 @@ version: 2.1 orbs: gradle: circleci/gradle@2.2.0 browser-tools: circleci/browser-tools@1.4.1 + +parameters: + run_commit: + default: false + type: boolean + run_backend: + default: false + type: boolean + run_frontend: + default: false + type: boolean \ No newline at end of file diff --git a/.circleci/src/commands/check-circleci-config.yml b/.circleci/src/commands/check-circleci-config.yml new file mode 100644 index 000000000..37ec95c5c --- /dev/null +++ b/.circleci/src/commands/check-circleci-config.yml @@ -0,0 +1,28 @@ +description: This command builds the circle config from the files in src and validates that it is up-to-date and valid. +steps: + - run: + name: Install CircleCI CLI + command: curl -fLSs https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/master/install.sh | sudo bash + - run: + name: Build circle config + command: ./scripts/circleci-update-config.sh + - run: # Taken from https://github.com/roopakv/orbs/blob/master/src/commands/fail_if_dirty.yml + name: CircleCI config up to date + # language=bash + command: | + FILES_MODIFIED="" + setcommit () { + FILES_MODIFIED=$(git status -s | grep -i -E '.*circleci/config.yml') + } + setcommit || true + if [ -z "$FILES_MODIFIED" ] + then + echo "The CircleCI config is up to date." + exit 0; + else + echo "The CircleCI config is not up to date. You can update it by running `yarn run circleci:update-config`." + exit 1; + fi + - run: + name: Validate circle config + command: circleci config validate \ No newline at end of file diff --git a/.circleci/src/jobs/check.yml b/.circleci/src/jobs/check.yml new file mode 100644 index 000000000..9f9c3b897 --- /dev/null +++ b/.circleci/src/jobs/check.yml @@ -0,0 +1,10 @@ +docker: + - image: cimg/node:19.1.0-browsers +resource_class: small +environment: + TOTAL_CPUS: 1 # For resource_class small + TZ: 'Europe/Berlin' # Should be removed in IGAPP-39 +shell: /bin/bash -eo pipefail +steps: + - checkout + - check-circleci-config \ No newline at end of file diff --git a/.circleci/src/workflows/backend.yml b/.circleci/src/workflows/backend.yml index 4ee264ac1..9a64b558e 100644 --- a/.circleci/src/workflows/backend.yml +++ b/.circleci/src/workflows/backend.yml @@ -1,3 +1,4 @@ +when: << pipeline.parameters.run_backend >> jobs: - backend-build - administration-build diff --git a/.circleci/src/workflows/commit.yml b/.circleci/src/workflows/commit.yml new file mode 100644 index 000000000..56435b7b4 --- /dev/null +++ b/.circleci/src/workflows/commit.yml @@ -0,0 +1,11 @@ +when: << pipeline.parameters.run_commit >> +jobs: + - check + - backend-build + - backend-pack: + requires: + - backend-build + - backend-health-check: + requires: + - backend-pack + - check-frontend \ No newline at end of file diff --git a/.circleci/src/workflows/frontend.yml b/.circleci/src/workflows/frontend.yml index ca42fbd2a..129746af3 100644 --- a/.circleci/src/workflows/frontend.yml +++ b/.circleci/src/workflows/frontend.yml @@ -1,3 +1,4 @@ +when: << pipeline.parameters.run_frontend >> jobs: - check-frontend - frontend-android-build: diff --git a/scripts/circleci-update-config.sh b/scripts/circleci-update-config.sh new file mode 100755 index 000000000..9c1fe439d --- /dev/null +++ b/scripts/circleci-update-config.sh @@ -0,0 +1 @@ +{ cat .circleci/autogenerated_header.yml; circleci config pack .circleci/src; } > .circleci/config.yml && circleci config validate \ No newline at end of file