From ff86d7bbf58d29194d7fa36d63d369053c32a02a Mon Sep 17 00:00:00 2001 From: Ian Streeter Date: Wed, 15 Dec 2021 17:09:34 +0000 Subject: [PATCH] Migrate from travis to github actions (close #47) --- .github/workflows/deploy.yml | 56 ++++++++++++++++++++++++++++++++++ .github/workflows/lacework.yml | 37 ++++++++++++++++++++++ .github/workflows/snyk.yml | 25 +++++++++++++++ .github/workflows/test.yml | 16 ++++++++++ .travis.yml | 36 ---------------------- .travis/deploy_docker.sh | 20 ------------ .travis/deploy_template.sh | 24 --------------- .travis/release.yml | 32 ------------------- README.md | 48 +++-------------------------- build.sbt | 3 +- 10 files changed, 140 insertions(+), 157 deletions(-) create mode 100644 .github/workflows/deploy.yml create mode 100644 .github/workflows/lacework.yml create mode 100644 .github/workflows/snyk.yml create mode 100644 .github/workflows/test.yml delete mode 100644 .travis.yml delete mode 100755 .travis/deploy_docker.sh delete mode 100755 .travis/deploy_template.sh delete mode 100644 .travis/release.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..cf2c505 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,56 @@ +name: build + +on: + push: + tags: + - '*' + +jobs: + deploy_docker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: coursier/cache-action@v3 + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 8 + + - name: Get current version + id: ver + run: echo "::set-output name=tag::${GITHUB_REF#refs/tags/}" + + - name: Stage the Docker build + run: sbt docker:stage + + - name: Docker metadata + id: meta + uses: docker/metadata-action@v3 + with: + images: snowplow/snowplow-google-cloud-storage-loader + tags: | + type=raw,value=latest,enable=${{ !contains(steps.ver.outputs.tag, 'rc') }} + type=raw,value=${{ steps.ver.outputs.tag }} + flavor: | + latest=false + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Push image + uses: docker/build-push-action@v2 + with: + context: target/docker/stage + file: target/docker/stage/Dockerfile + platforms: linux/amd64,linux/arm64/v8 + tags: ${{ steps.meta.outputs.tags }} + push: true diff --git a/.github/workflows/lacework.yml b/.github/workflows/lacework.yml new file mode 100644 index 0000000..6cd98c5 --- /dev/null +++ b/.github/workflows/lacework.yml @@ -0,0 +1,37 @@ +name: lacework + +on: + push: + tags: + - '*' + +jobs: + scan: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: coursier/cache-action@v3 + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 8 + - name: Get current version + id: ver + run: echo "::set-output name=tag::${GITHUB_REF#refs/tags/}" + + - name: Install lacework scanner + run: | + sudo apt-get update + sudo apt-get -y install curl + curl -L https://github.com/lacework/lacework-vulnerability-scanner/releases/latest/download/lw-scanner-linux-amd64 -o lw-scanner + chmod +x lw-scanner + + - name: Build docker images + run: sbt docker:publishLocal + + - name: Scan image + env: + LW_ACCESS_TOKEN: ${{ secrets.LW_ACCESS_TOKEN }} + LW_ACCOUNT_NAME: ${{ secrets.LW_ACCOUNT_NAME }} + LW_SCANNER_SAVE_RESULTS: ${{ !contains(steps.version.outputs.tag, 'rc') }} + run: ./lw-scanner image evaluate snowplow/snowplow-google-cloud-storage-loader ${{ steps.ver.outputs.tag }} --build-id ${{ github.run_id }} --no-pull diff --git a/.github/workflows/snyk.yml b/.github/workflows/snyk.yml new file mode 100644 index 0000000..596c376 --- /dev/null +++ b/.github/workflows/snyk.yml @@ -0,0 +1,25 @@ +name: Snyk + +on: + push: + branches: [ master ] + +jobs: + security: + runs-on: ubuntu-latest + + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Install Snyk + run: sudo npm install -g snyk + + - name: Add correct sbt-dependency-graph version + run: mkdir -p $HOME/.sbt/1.0/plugins && echo 'addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.10.0-RC1")' >> $HOME/.sbt/1.0/plugins/plugins.sbt + + - name: Run Snyk to check for vulnerabilities + run: snyk monitor --project-name=snowplow-google-cloud-storage-loader diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..6253074 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,16 @@ +name: test + +on: push + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: coursier/cache-action@v3 + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 8 + - name: Run tests + run: sbt test diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 8054e69..0000000 --- a/.travis.yml +++ /dev/null @@ -1,36 +0,0 @@ -dist: trusty -language: scala -services: - - docker -scala: - - 2.11.12 -jdk: -- oraclejdk8 -before_install: - - openssl aes-256-cbc -K $encrypted_a1a0961ea86f_key -iv $encrypted_a1a0961ea86f_iv -in service-account.json.enc -out ${HOME}/service-account.json -d -script: - - sbt test -before_deploy: - - pip install --user release-manager==0.3.0 -deploy: - - provider: script - skip_cleanup: true - script: "./.travis/deploy_template.sh $TRAVIS_TAG" - on: - tags: true - - provider: script - skip_cleanup: true - script: release-manager --config ./.travis/release.yml --check-version --make-version --make-artifact --upload-artifact - on: - tags: true - - provider: script - skip_cleanup: true - script: "./.travis/deploy_docker.sh $TRAVIS_TAG" - on: - tags: true -env: - global: - - secure: ncodfcWlGupVKH/taFMMmypRhJ2x8VRs2ulo9pW/u7UXSf1lOHDFj/GGAprlBRKq0t1ks1fQGr5MMjlBOTbkCHIUP6o5CzpGGcyClnUBHYtWJreCRiJIJmmHZEw3pzSmtTol2wU4xH7c3R0qqGiVb0OZEz4DZonlcRnkDSrP3BgmO14hBR/kcH9S/xJzc9lwoqOzL6nXjAmPyK/Swvxv9NxigMiqKvwNV5FLfQIWIXuwpcDJFabL9zRzCnppsZn7j/UtVGrowziGwyPTFBzDi6flQOhZg8/tDpKomzbemgvHFm6H8baD1tgtKAOdvjCRxnTExcBdqteAa7ocQg6YaePb0D410yPTaaS5lzH5caT1hBszPdce6ksEpE69jupiXnO0PYo3KcPb4uPGwQfE+qf6hIFC3MNcSe0bY9kc0Xo3cAgjIMzrpfzm41fkY/dIV2yPBZf1m3cSnrQBKpHI+Jbl/mnXX4tvmvwxKJYZ7Crd4cYhihWz1zxmhmJP//kD0iHi6rfLvqZOx0CJSG1aFVpnFKIPQxF4hlDEhm0DwUG4SYPBgJHjI1ZltYIvudHdq1n65UbFtw/mkWhEzWsE2zEFtxkMT3hVXRjUKkkbRacAGKJs0ktu8jUFEtkaNG9RXX0BUlSfccHRBADn9uqz5wXovV8d26nRF06AUl0QGhg= - - secure: ntwyTd5eTHrdO44pXQjXKJ930qOwpuM1DdYeazpVfjwTTjr0ZMhjCTeyFUN2kVQUi6Tv6tt0nrrMabM4311zLAm3JZ/OrELfPq90hgAYc8wv4z2c+Yo+0YX3IIXvAAtNVUU2+s3WGGCa8/vZmtzLyCXU7X8bO3th36GPae5Bv2jpBe0JLAquyY8j1jT7jX6xOe24I6udyAWzRK9aW27HGc7LgPu8Zp98sgqRYpsZT/SQBCEYYnADj8jldFEdSyeERHR9XmVUZOJzmpFdUs/sYOhpyNl/LhFlthm7w/q2qWWs3qEVSRLU2pQErGSpGqFvoFZ/ojH7+Uhvdwih8OBbrKnDYxU9HRMBCY+aV5RVW3PY4xuG2P3hqKzRI+4YjVQZjfaV4fKu0WgP2Bj3Nw4NMGE6Lx+bQ8iov/kUGWPrbnm/s7tOm0BPfJgwuA4pZ9ngJHNOGfaiFhy+VM8GbciA/V9Yt5C6YxN2Ffpgmrs00p/i7SKEsBFK82MkwTubV7Nv62WrIF/zlbFO0Lj+Bc12uFcROvTFyH4pSJQYc+vZJKkPOblKMh6fpxRyNHtHZCrqULZV4PU4E/fcNBfpItFeS0PGFlzmvimxInexPuvQ9bbEbj09NPxkoicxsWItp31DYaC4/jhzhT37LHutgEPdO+/yZcSwsA16I/WI0PyHqcg= - - secure: RgOTfPNgF1fPf1LweKCyMW5uxFa09NrR4zCHfPz4cE77YFZoDe/qnKfvKrmnNH6EaXHH6Yqobe0feOwALCXsRmOiHSEwbsbajzUJXnTmtzsMmvrXujgkOt1WnvIViinVeS8aSiIs3WwT2j5NBFSsjWheo9CY3nmnjyS5Ef3r7+YefF5ty60ez8bt5pdJrt39l+nZAkvCEyBP7rSCRNiM5S3Ktm+drJo5AfUF4j3WXaaGzYhjbLhvS9rSyxUX2R1rH/2D4VddvlC/UTOsW8ERkTIwtizgOI9sBkZD02w7Dp7C3wvHnvpZq4JJ1fq7iMLZh7O1LSvrwnJHwOHZO3Qd9IiEzHIlRiRBtvofFdn1xUyrbPFV+ZlLbaiXIxZUmKhdZcr/lA2h5/2pnzFdefT+3Sk7qjcZbZH4j8ICrjr8N20+5W8U18fD8rcx6SekdEdJH6wAK4Yy2JpeLN5FxEao94NMFu7kcux/fSUktMRRzvZo275AeLNF8vfe1xsmLxAnoFlhcAKtzabjsiCn+//CS4tWATpZJrLI4hUJO+JfhHtKBtoevVOyyMM39zfwG7KT3sfzuIC/E7uDLaBfNN6GEpQbYLkiy+W7XMLaKT1UdxR8zrstOW8r4hNZEG07lHa/brpC3182axIlIXT4vZ8/cg7SgLrSEy7aOclvD2zuSvg= - - secure: anz+CgPzs791QYY4z7Bs1/7+Dx95DdEjg53NTV0Ltnw4kJhzqyjQrBOUOjeOiQ0PBrGokYdmlmN/H7Y5N/0Jms/EUltCgk/jMVKqsg/3yaYfQXYLmglRXdZy5TbDJBWox5wmwfUcFFUEZJ+ZT5asDIFNQtp3MB7TbkhQ4LgM1MFzkxTavfjENIltltaL5HGmJCQT8aOV+dcC3YS3BEddsk8Rj00vnlEL/MaUVEhGktQXJMPYBLFajgFdrOLyLEdiE/bMfShW0YBqXnAXX2+QNF0AdNi90RXk2ejnfgXRwcehCUKYr9h3oUCbFYc8qd9ePSMYM1vOOK8EqAtzgkaVbvvWYL3hU/I3PEdRhfnJo/hqRIRo7XPJn4tjI5p0ml/hb1ScHADdc/meLFA7ZMIhc+ClZjyy+QZyYDNaxmRSHp9jH+F6DoEeu0JT0l+aUSN2ajp2HCCYJIU6RRXx5bvh4F23rjUPosV+oPLf9J0VrpRo18QvHNGOi/djGSzecMoUr1eCdPb6jCbWr643366cFIiOxCOsDRpLquRi6v76o5fUaf+5qMSvt1IlpNuSVkC24qrzRW1xKfh+5u+06cxR5FjgN165AbDOLBqDvanpw1umeeaUrrCpjh37pENJAm5RPzNT6RiM4m7UJlvjMRxbuW3+CDarJdLY2WkUOlYe1pQ= diff --git a/.travis/deploy_docker.sh b/.travis/deploy_docker.sh deleted file mode 100755 index d41168b..0000000 --- a/.travis/deploy_docker.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -tag=$1 - -file="${HOME}/.dockercfg" -docker_repo="snowplow-docker-registry.bintray.io" -curl -X GET \ - -u${BINTRAY_SNOWPLOW_DOCKER_USER}:${BINTRAY_SNOWPLOW_DOCKER_API_KEY} \ - https://${docker_repo}/v2/auth > $file - -cd ${TRAVIS_BUILD_DIR} - -project_version=$(sbt -no-colors version | perl -ne 'print "$1\n" if /info.*(\d+\.\d+\.\d+[^\r\n]*)/' | tail -n 1 | tr -d '\n') -if [[ "${tag}" = "${project_version}" ]]; then - sbt docker:publishLocal - docker push "${docker_repo}/snowplow/snowplow-google-cloud-storage-loader:${tag}" -else - echo "Tag version '${tag}' doesn't match version in scala project ('${project_version}'). aborting!" - exit 1 -fi diff --git a/.travis/deploy_template.sh b/.travis/deploy_template.sh deleted file mode 100755 index a6f50e6..0000000 --- a/.travis/deploy_template.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -tag=$1 - -export GOOGLE_APPLICATION_CREDENTIALS="${HOME}/service-account.json" - -cd ${TRAVIS_BUILD_DIR} - -project_version=$(sbt -no-colors version | perl -ne 'print "$1\n" if /info.*(\d+\.\d+\.\d+[^\r\n]*)/' | tail -n 1 | tr -d '\n') -if [[ "${tag}" = *"${project_version}" ]]; then - sbt "runMain com.snowplowanalytics.storage.googlecloudstorage.loader.CloudStorageLoader --project=snowplow-assets \ - --templateLocation=gs://sp-hosted-assets/4-storage/snowplow-google-cloud-storage-loader/${tag}/SnowplowGoogleCloudStorageLoaderTemplate-${tag} \ - --stagingLocation=gs://sp-hosted-assets/4-storage/snowplow-google-cloud-storage-loader/${tag}/staging \ - --runner=DataflowRunner \ - --tempLocation=gs://sp-hosted-assets/tmp \ - --autoscalingAlgorithm=THROUGHPUT_BASED \ - --numWorkers=1 \ - --numShards=1 \ - --diskSizeGb=30 \ - --workerMachineType=n1-standard-1" -else - echo "Tag version '${tag}' doesn't match version in scala project ('${project_version}'). aborting!" - exit 1 -fi diff --git a/.travis/release.yml b/.travis/release.yml deleted file mode 100644 index 4d37cc7..0000000 --- a/.travis/release.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -# --- Variables --- # - -local : - root_dir : <%= ENV['TRAVIS_BUILD_DIR'] %> - -# --- Release Manager Config --- # - -# Required: deployment targets -targets : - - type : "bintray" - user : <%= ENV['BINTRAY_SNOWPLOW_GENERIC_USER'] %> - password : <%= ENV['BINTRAY_SNOWPLOW_GENERIC_API_KEY'] %> - -# Required: packages to be deployed -packages : - - repo : "snowplow-generic" - name : "snowplow-google-cloud-storage-loader" - user_org : "snowplow" - publish : true - override : false - continue_on_conflict : false - version : <%= FUNC['sbt_version(.)'] %> - build_version : <%= ENV['TRAVIS_TAG'] %> - build_commands : - - sbt universal:packageBin - artifacts : - - type : "asis" - prefix : "snowplow_google_cloud_storage_loader_" - suffix : ".zip" - binary_paths : - - "target/universal/snowplow-google-cloud-storage-loader-{{ packages.0.build_version }}.zip" diff --git a/README.md b/README.md index 195cc43..3dd84a8 100644 --- a/README.md +++ b/README.md @@ -30,46 +30,9 @@ sbt docker:publishLocal ## Running -### Through the zip archive - -You can find the archive hosted on [our Bintray][bintray]. - -Once unzipped the artifact can be run as follows: - -```bash -./bin/snowplow-google-cloud-storage-loader \ - --runner=DataFlowRunner \ - --project=[PROJECT] \ - --streaming=true \ - --zone=europe-west2-a \ - --inputSubscription=projects/[PROJECT]/subscriptions/[SUBSCRIPTION] \ - --outputDirectory=gs://[BUCKET] \ - --outputFilenamePrefix=output \ # optional - --shardTemplate=-W-P-SSSSS-of-NNNNN \ # optional - --outputFilenameSuffix=.txt \ # optional - --windowDuration=5 \ # optional, in minutes - --compression=none \ # optional, gzip, bz2 or none - --numShards=1 \ # optional - --dateFormat=YYYY/MM/dd/HH/ \ # optional - --labels={\"label\": \"value\"} \ #OPTIONAL - --partitionedOuptutDirectory=gs://[BUCKET]/[SUBDIR] # optional -``` - -To display the help message: - -```bash -./bin/snowplow-google-cloud-storage-loader --help -``` - -To display documentation about Cloud Storage Loader-specific options: - -```bash -./bin/snowplow-google-cloud-storage-loader --help=com.snowplowanalytics.storage.googlecloudstorage.loader.Options -``` - ### Through a docker container -You can also find the image on [our Bintray][bintray-docker]. +You can also find the image on [Docker hub][docker-hub]. A container can be run as follows: @@ -77,7 +40,7 @@ A container can be run as follows: docker run \ -v $PWD/config:/snowplow/config \ -e GOOGLE_APPLICATION_CREDENTIALS=/snowplow/config/credentials.json \ # if running outside GCP - snowplow-docker-registry.bintray.io/snowplow/snowplow-google-cloud-storage-loader:0.3.0 \ + snowplow/snowplow-google-cloud-storage-loader:0.3.0 \ --runner=DataFlowRunner \ --jobName=[JOB-NAME] \ --project=[PROJECT] \ @@ -99,14 +62,14 @@ docker run \ To display the help message: ```bash -docker run snowplow-docker-registry.bintray.io/snowplow/snowplow-google-cloud-storage-loader:0.3.0 \ +docker run snowplow/snowplow-google-cloud-storage-loader:0.3.0 \ --help ``` To display documentation about Cloud Storage Loader-specific options: ```bash -docker run snowplow-docker-registry.bintray.io/snowplow/snowplow-google-cloud-storage-loader:0.3.0 \ +docker run snowplow/snowplow-google-cloud-storage-loader:0.3.0 \ --help=com.snowplowanalytics.storage.googlecloudstorage.loader.Options ``` @@ -157,8 +120,7 @@ limitations under the License. [dataflow]: https://cloud.google.com/dataflow/ [self-describing-json]: https://snowplowanalytics.com/blog/2014/05/15/introducing-self-describing-jsons/ -[bintray]: https://bintray.com/snowplow/snowplow-generic/snowplow-google-cloud-storage-loader -[bintray-docker]: https://bintray.com/snowplow/registry/snowplow%3Asnowplow-google-cloud-storage-loader +[docker-hub]: https://hub.docker.com/r/snowplow/snowplow-google-cloud-storage-loader [license]: http://www.apache.org/licenses/LICENSE-2.0 diff --git a/build.sbt b/build.sbt index 6942a89..534707a 100644 --- a/build.sbt +++ b/build.sbt @@ -42,9 +42,8 @@ lazy val macroSettings = Seq( ) import com.typesafe.sbt.packager.docker._ -dockerRepository := Some("snowplow-docker-registry.bintray.io") dockerUsername := Some("snowplow") -dockerBaseImage := "snowplow-docker-registry.bintray.io/snowplow/k8s-dataflow:0.1.0" +dockerBaseImage := "snowplow/k8s-dataflow:0.1.0" Docker / maintainer := "Snowplow Analytics Ltd. " Docker / daemonUser := "snowplow" dockerCommands := dockerCommands.value.map{