diff --git a/.github/workflows/code_standards.yaml b/.github/workflows/code_standards.yaml deleted file mode 100644 index 92ff8effee..0000000000 --- a/.github/workflows/code_standards.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: code standards - -on: [push, pull_request] - -jobs: - lint-java: - container: gcr.io/kf-feast/feast-ci:latest - runs-on: [ubuntu-latest] - steps: - - uses: actions/checkout@v2 - - name: lint java - run: make lint-java - - lint-python: - container: gcr.io/kf-feast/feast-ci:latest - runs-on: [ubuntu-latest] - steps: - - uses: actions/checkout@v2 - - name: install dependencies - run: make install-python-ci-dependencies - - name: compile protos - run: make compile-protos-python - - name: lint python - run: make lint-python - - lint-go: - container: gcr.io/kf-feast/feast-ci:latest - runs-on: [ubuntu-latest] - steps: - - uses: actions/checkout@v2 - - name: install dependencies - run: make install-go-ci-dependencies - - name: lint go - run: make lint-go - - lint-versions: - container: gcr.io/kf-feast/feast-ci:latest - runs-on: [ubuntu-latest] - steps: - - uses: actions/checkout@v2 - - name: install dependencies - run: make lint-versions \ No newline at end of file diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml new file mode 100644 index 0000000000..901f9f5e6f --- /dev/null +++ b/.github/workflows/complete.yml @@ -0,0 +1,117 @@ +name: complete workflow + +on: [push, pull_request] + +jobs: + build-push-docker-images: + runs-on: [self-hosted, builder] + strategy: + matrix: + component: [core, serving] + steps: + - uses: actions/checkout@v2 + - name: build image + run: make build-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA} + - name: push image + run: make push-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA} + + lint-java: + container: gcr.io/kf-feast/feast-ci:latest + runs-on: [ubuntu-latest] + steps: + - uses: actions/checkout@v2 + - name: lint java + run: make lint-java + + lint-python: + container: gcr.io/kf-feast/feast-ci:latest + runs-on: [ubuntu-latest] + steps: + - uses: actions/checkout@v2 + - name: install dependencies + run: make install-python-ci-dependencies + - name: compile protos + run: make compile-protos-python + - name: lint python + run: make lint-python + + lint-go: + container: gcr.io/kf-feast/feast-ci:latest + runs-on: [ubuntu-latest] + steps: + - uses: actions/checkout@v2 + - name: install dependencies + run: make install-go-ci-dependencies + - name: lint go + run: make lint-go + + lint-versions: + container: gcr.io/kf-feast/feast-ci:latest + runs-on: [ubuntu-latest] + steps: + - uses: actions/checkout@v2 + - name: install dependencies + run: make lint-versions + + unit-test-java: + runs-on: ubuntu-latest + container: gcr.io/kf-feast/feast-ci:latest + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v1 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: test java + run: make test-java-with-coverage + - uses: actions/upload-artifact@v2 + with: + name: java-coverage-report + path: ${{ github.workspace }}/docs/coverage/java/target/site/jacoco-aggregate/ + + unit-test-python: + runs-on: ubuntu-latest + container: gcr.io/kf-feast/feast-ci:latest + steps: + - uses: actions/checkout@v2 + - name: install python + run: make install-python + - name: test python + run: make test-python + + unit-test-go: + runs-on: ubuntu-latest + container: gcr.io/kf-feast/feast-ci:latest + steps: + - uses: actions/checkout@v2 + - name: install dependencies + run: make compile-protos-go + - name: test go + run: make test-go + + integration-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: '11' + java-package: jdk + architecture: x64 + - name: Run integration tests + run: make test-java-integration + + load-test: + runs-on: [self-hosted, load] + needs: build-push-docker-images + steps: + - uses: actions/checkout@v2 + - name: Run load test + run: make test-load GIT_SHA=${GITHUB_SHA} + - uses: actions/upload-artifact@v2 + with: + name: load-test-results + path: load-test-output/ \ No newline at end of file diff --git a/.github/workflows/docker_compose_tests.yml b/.github/workflows/docker_compose_tests.yml deleted file mode 100644 index 2984ebc37b..0000000000 --- a/.github/workflows/docker_compose_tests.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: docker compose tests - -on: - push: - branches: - - master - -jobs: - basic-redis-e2e-tests-docker-compose: - runs-on: ubuntu-latest - name: basic redis e2e tests on docker compose - steps: - - uses: actions/checkout@v2 - - name: test docker compose - run: ./infra/scripts/test-docker-compose.sh diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml deleted file mode 100644 index 699d5f63d5..0000000000 --- a/.github/workflows/integration_tests.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: integration tests - -on: [push, pull_request] - -jobs: - maven-integration-test: - runs-on: ubuntu-latest - name: integration-test-java - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: '11' - java-package: jdk - architecture: x64 - - name: Run integration tests - run: make test-java-integration \ No newline at end of file diff --git a/.github/workflows/load_test.yml b/.github/workflows/load_test.yml deleted file mode 100644 index 5a95c5d1d5..0000000000 --- a/.github/workflows/load_test.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: load test - -on: [push, pull_request] - -jobs: - load-test: - runs-on: [self-hosted, load] - name: load-test - steps: - - uses: actions/checkout@v2 - - name: Run load test - run: make test-load - - uses: actions/upload-artifact@v2 - with: - name: load-test-results - path: load-test-output/ \ No newline at end of file diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml new file mode 100644 index 0000000000..b2620c4e8d --- /dev/null +++ b/.github/workflows/master_only.yml @@ -0,0 +1,56 @@ +name: master only + +on: + push: + branches: master + tags: + - 'v*.*.*' + +jobs: + build-docker-images: + runs-on: [self-hosted, builder] + strategy: + matrix: + component: [core, serving, jupyter, ci] + steps: + - uses: actions/checkout@v2 + - name: build image + run: make build-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA} + - name: push image + run: make push-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA} + - name: push feast dev + run: | + if [ ${GITHUB_REF#refs/*/} == "master" ]; then + docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:dev + docker push gcr.io/kf-feast/feast-${{ matrix.component }}:dev + fi + - name: get version + id: get_version + run: echo ::set-output name=VERSION::${${GITHUB_REF/refs\/tags\//}:1} + - name: push versioned release + run: | + + # Build and push semver tagged commits + rx='^([0-9]+\.){0,2}(\*|[0-9]+)$' + if [[ ${{ steps.get_version.outputs.VERSION }} =~ $rx ]]; then + + docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:${{ steps.get_version.outputs.VERSION }} + docker push gcr.io/kf-feast/feast-${{ matrix.component }}:${{ steps.get_version.outputs.VERSION }} + + # Also update "latest" image if tagged commit is pushed to stable branch + HIGHEST_SEMVER_TAG=$(git tag -l --sort -version:refname | head -n 1) + echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG" + + if [ ${{ steps.get_version.outputs.VERSION }} == "${HIGHEST_SEMVER_TAG:1}" ] + then + docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:${{ steps.get_version.outputs.VERSION }} + docker push gcr.io/kf-feast/feast-${{ matrix.component }}:${{ steps.get_version.outputs.VERSION }} + fi + fi + + tests-docker-compose: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: test docker compose + run: ./infra/scripts/test-docker-compose.sh diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml deleted file mode 100644 index f71788a90b..0000000000 --- a/.github/workflows/unit_tests.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: unit tests - -on: [push, pull_request] - -jobs: - unit-test-java: - runs-on: ubuntu-latest - container: gcr.io/kf-feast/feast-ci:latest - name: unit test java - steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v1 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - name: test java - run: make test-java-with-coverage - - uses: actions/upload-artifact@v2 - with: - name: java-coverage-report - path: ${{ github.workspace }}/docs/coverage/java/target/site/jacoco-aggregate/ - - unit-test-python: - runs-on: ubuntu-latest - container: gcr.io/kf-feast/feast-ci:latest - name: unit test python - steps: - - uses: actions/checkout@v2 - - name: install python - run: make install-python - - name: test python - run: make test-python - - unit-test-go: - runs-on: ubuntu-latest - container: gcr.io/kf-feast/feast-ci:latest - name: unit test go - steps: - - uses: actions/checkout@v2 - - name: install dependencies - run: make compile-protos-go - - name: test go - run: make test-go diff --git a/.prow/config.yaml b/.prow/config.yaml index 17d3a8af56..8d4e407323 100644 --- a/.prow/config.yaml +++ b/.prow/config.yaml @@ -154,7 +154,7 @@ presubmits: memory: "6144Mi" skip_branches: - ^v0\.(3|4)-branch$ - + - name: test-end-to-end-auth decorate: true always_run: true @@ -300,55 +300,6 @@ presubmits: skip_branches: - ^v0\.(3|4)-branch$ - - name: publish-docker-images - decorate: true - always_run: true - spec: - containers: - - image: google/cloud-sdk:302.0.0 - command: - - bash - - -c - - | - infra/scripts/download-maven-cache.sh \ - --archive-uri gs://feast-templocation-kf-feast/.m2.2020-07-13.tar \ - --output-dir $PWD/ - - infra/scripts/publish-docker-image.sh \ - --repository gcr.io/kf-feast/feast-core \ - --tag ${PULL_PULL_SHA} \ - --file infra/docker/core/Dockerfile \ - --google-service-account-file /etc/gcloud/service-account.json - - infra/scripts/publish-docker-image.sh \ - --repository gcr.io/kf-feast/feast-serving \ - --tag ${PULL_PULL_SHA} \ - --file infra/docker/serving/Dockerfile \ - --google-service-account-file /etc/gcloud/service-account.json - - infra/scripts/publish-docker-image.sh \ - --repository gcr.io/kf-feast/feast-jupyter \ - --tag ${PULL_PULL_SHA} \ - --file infra/docker/jupyter/Dockerfile \ - --google-service-account-file /etc/gcloud/service-account.json - - volumeMounts: - - name: docker-socket - mountPath: /var/run/docker.sock - - name: service-account - mountPath: /etc/gcloud/service-account.json - subPath: service-account.json - readOnly: true - securityContext: - privileged: true - volumes: - - name: docker-socket - hostPath: - path: /var/run/docker.sock - - name: service-account - secret: - secretName: feast-service-account - postsubmits: feast-dev/feast: - name: publish-python-sdk @@ -436,97 +387,6 @@ postsubmits: # Filter on tags with semantic versioning, prefixed with "v". v0.3 and v0.4 only. - ^v0\.(3|4)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$ - - name: publish-docker-images - decorate: true - spec: - containers: - - image: google/cloud-sdk:302.0.0 - command: - - bash - - -c - - | - # Load cache to speed up builds - infra/scripts/download-maven-cache.sh \ - --archive-uri gs://feast-templocation-kf-feast/.m2.2020-07-13.tar \ - --output-dir $PWD/ - - # Build images based on SHAs - infra/scripts/publish-docker-image.sh \ - --repository gcr.io/kf-feast/feast-core \ - --tag ${PULL_BASE_SHA} \ - --file infra/docker/core/Dockerfile \ - --google-service-account-file /etc/gcloud/service-account.json - - infra/scripts/publish-docker-image.sh \ - --repository gcr.io/kf-feast/feast-serving \ - --tag ${PULL_BASE_SHA} \ - --file infra/docker/serving/Dockerfile \ - --google-service-account-file /etc/gcloud/service-account.json - - infra/scripts/publish-docker-image.sh \ - --repository gcr.io/kf-feast/feast-jupyter \ - --tag ${PULL_BASE_SHA} \ - --file infra/docker/jupyter/Dockerfile \ - --google-service-account-file /etc/gcloud/service-account.json - - # Create a "dev" image for the master branch - if [ $PULL_BASE_REF == "master" ]; then - docker tag gcr.io/kf-feast/feast-core:${PULL_BASE_SHA} gcr.io/kf-feast/feast-core:dev - docker push gcr.io/kf-feast/feast-core:dev - - docker tag gcr.io/kf-feast/feast-serving:${PULL_BASE_SHA} gcr.io/kf-feast/feast-serving:dev - docker push gcr.io/kf-feast/feast-serving:dev - fi - - # Build and push semver tagged commits - rx='^([0-9]+\.){0,2}(\*|[0-9]+)$' - if [[ ${PULL_BASE_REF:1} =~ $rx ]]; then - - docker tag gcr.io/kf-feast/feast-core:${PULL_BASE_SHA} gcr.io/kf-feast/feast-core:${PULL_BASE_REF:1} - docker push gcr.io/kf-feast/feast-core:${PULL_BASE_REF:1} - - docker tag gcr.io/kf-feast/feast-serving:${PULL_BASE_SHA} gcr.io/kf-feast/feast-serving:${PULL_BASE_REF:1} - docker push gcr.io/kf-feast/feast-serving:${PULL_BASE_REF:1} - - docker tag gcr.io/kf-feast/feast-jupyter:${PULL_BASE_SHA} gcr.io/kf-feast/feast-jupyter:${PULL_BASE_REF:1} - docker push gcr.io/kf-feast/feast-jupyter:${PULL_BASE_REF:1} - - # Also update "latest" image if tagged commit is pushed to stable branch - HIGHEST_SEMVER_TAG=$(git tag -l --sort -version:refname | head -n 1) - echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG" - - if [ "${PULL_BASE_REF:1}" == "${HIGHEST_SEMVER_TAG:1}" ] - then - docker tag gcr.io/kf-feast/feast-core:${PULL_BASE_SHA} gcr.io/kf-feast/feast-core:latest - docker push gcr.io/kf-feast/feast-core:latest - - docker tag gcr.io/kf-feast/feast-serving:${PULL_BASE_SHA} gcr.io/kf-feast/feast-serving:latest - docker push gcr.io/kf-feast/feast-serving:latest - - docker tag gcr.io/kf-feast/feast-jupyter:${PULL_BASE_SHA} gcr.io/kf-feast/feast-jupyter:latest - docker push gcr.io/kf-feast/feast-jupyter:latest - fi - fi - volumeMounts: - - name: docker-socket - mountPath: /var/run/docker.sock - - name: service-account - mountPath: /etc/gcloud/service-account.json - subPath: service-account.json - readOnly: true - securityContext: - privileged: true - volumes: - - name: docker-socket - hostPath: - path: /var/run/docker.sock - - name: service-account - secret: - secretName: feast-service-account - branches: - - ^master$ - - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$ - - name: publish-helm-chart decorate: true spec: diff --git a/Makefile b/Makefile index 4466b62f9c..d28a14659a 100644 --- a/Makefile +++ b/Makefile @@ -129,6 +129,9 @@ push-serving-docker: push-ci-docker: docker push $(REGISTRY)/feast-ci:latest +push-jupyter-docker: + docker push $(REGISTRY)/feast-jupyter:$(VERSION) + build-core-docker: docker build -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile . @@ -138,6 +141,9 @@ build-serving-docker: build-ci-docker: docker build -t $(REGISTRY)/feast-ci:latest -f infra/docker/ci/Dockerfile . +build-jupyter-docker: + docker build -t $(REGISTRY)/feast-jupyter:$(VERSION) -f infra/docker/jupyter/Dockerfile . + # Documentation install-dependencies-proto-docs: @@ -184,4 +190,4 @@ lint-versions: # Performance test-load: - ./infra/scripts/test-load.sh \ No newline at end of file + ./infra/scripts/test-load.sh $(GIT_SHA) \ No newline at end of file diff --git a/infra/scripts/test-load.sh b/infra/scripts/test-load.sh index c017d0fe41..243da5e11e 100755 --- a/infra/scripts/test-load.sh +++ b/infra/scripts/test-load.sh @@ -24,10 +24,14 @@ clean_up() { exit $ARG } -CURRENT_SHA=$(git rev-parse HEAD) - -if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then - export CURRENT_SHA=$(cat "$GITHUB_EVENT_PATH" | jq -r .pull_request.head.sha) +if [ -z "$1" ] ; then + echo "No SHA provided as argument, using local HEAD"; + CURRENT_SHA=$(git rev-parse HEAD); + export CURRENT_SHA +else + echo "Using ${1} as SHA to test"; + CURRENT_SHA=${1} + export CURRENT_SHA fi export PROJECT_ROOT_DIR=$(git rev-parse --show-toplevel)