Merge pull request #2 from fluent/master

sync master

.dockerignore

@@ -1,3 +1,3 @@
 .git
 build/**
-Dockerfile
+dockerfiles/**

.github/workflows/1.7.yaml

@@ -0,0 +1,167 @@
+on:
+  push:
+    branches:
+      - 1.7
+
+name: Build and publish for branch 1.7
+jobs:
+  docker_build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/checkout@v2
+        with:
+          repository: calyptia/fluent-bit-ci
+          path: ci
+
+      - name: Setup environment
+        run: |
+          sudo apt-get --yes update
+          sudo apt-get install --yes docker.io containerd runc
+          sudo systemctl unmask docker && sudo systemctl start docker
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build a docker image for master branch
+        run: |
+          DOCKER_BUILDKIT=1 docker build --no-cache -f ./dockerfiles/Dockerfile.${{ env.arch }}-1_7 -t ${{ env.dockerhub_organization }}/fluent-bit:${{ env.arch }}-1_7 .
+        env:
+          arch: x86_64
+          dockerhub_organization: fluentbitdev
+
+      - name: Push image to Docker Hub
+        run: |
+          DOCKER_BUILDKIT=1 docker push ${{ env.dockerhub_organization }}/fluent-bit:${{ env.arch }}-1_7
+        env:
+          arch: x86_64
+          dockerhub_organization: fluentbitdev
+  run-integration-tests:
+    name: run integration tests on GCP - k8s ${{ matrix.k8s-release }} for 1.7 branch
+    needs: docker_build
+    strategy:
+      max-parallel: 3
+      fail-fast: false
+      matrix:
+        k8s-release: [ '1.19' ] #, '1.20' ] #, 1.19/stable, 1.18/stable ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          repository: calyptia/fluent-bit-ci
+          path: ci
+
+      - uses: frabert/replace-string-action@master
+        id: formatted_release
+        with:
+          pattern: '(.*)\.(.*)$'
+          string: ${{ matrix.k8s-release }}
+          replace-with: '$1-$2'
+          flags: 'g'
+
+      - name: Replace the k8s release
+        run: |
+          sed -i -e "s/\$K8S_RELEASE/${{ env.k8s_release }}/g" default.auto.tfvars
+          sed -i -e "s/\$K8S_FORMATTED/${{ env.k8s_release_formatted }}/g" default.auto.tfvars
+          sed -i -e "s/\$K8S_FORMATTED/${{ env.k8s_release_formatted }}/g" config.tf
+        working-directory: ci/terraform/gcp/
+        env:
+          k8s_release: ${{ matrix.k8s-release }}
+          k8s_release_formatted: ${{ steps.formatted_release.outputs.replaced }}
+
+      - uses: hashicorp/setup-terraform@v1
+        with:
+          cli_config_credentials_hostname: 'app.terraform.io'
+          cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }}
+
+      - name: Replace the GCP service account key
+        run: |
+          cat <<EOT >> default.auto.tfvars
+          gcp-sa-key    =  <<-EOF
+          ${{ secrets.GCP_SA_KEY }}
+          EOF
+          EOT
+        working-directory: ci/terraform/gcp/
+
+      - name: Terraform fmt
+        id: fmt
+        run: terraform fmt -check
+        continue-on-error: true
+        working-directory: ci/terraform/gcp/
+
+      - name: Terraform Init
+        id: init
+        run: terraform init
+        working-directory: ci/terraform/gcp/
+
+      - name: Terraform Validate
+        id: validate
+        run: terraform validate -no-color
+        working-directory: ci/terraform/gcp/
+
+      - name: Terraform Apply
+        id: apply
+        run: |
+          terraform apply -input=false -auto-approve
+        working-directory: ci/terraform/gcp/
+
+      - name: Get the k8s cluster name from terraform output
+        id: get-k8s-cluster-name
+        run: terraform output -no-color -raw k8s-cluster-name
+        working-directory: ci/terraform/gcp/
+
+      - name: Get the k8s cluster location from terraform output
+        id: get-k8s-cluster-location
+        run: terraform output -no-color -raw k8s-cluster-location
+        working-directory: ci/terraform/gcp/
+
+      - name: Get the k8s project id from terraform output
+        id: get-gcp-project-id
+        run: terraform output -no-color -raw gcp-project-id
+        working-directory: ci/terraform/gcp/
+
+      - name: Get the bigquery dataset id from terraform output
+        id: get-gcp-bigquery-dataset-id
+        run: terraform output -no-color -raw gcp-bigquery-dataset-id
+        working-directory: ci/terraform/gcp/
+
+      - name: Get the bigquery table id from terraform output
+        id: get-gcp-bigquery-table-id
+        run: terraform output -no-color -raw gcp-bigquery-table-id
+        working-directory: ci/terraform/gcp/
+
+      - uses: google-github-actions/setup-gcloud@master
+        with:
+          service_account_key: ${{ secrets.GCP_SA_KEY }}
+          export_default_credentials: true
+
+      - uses: google-github-actions/get-gke-credentials@main
+        with:
+          cluster_name: ${{ steps.get-k8s-cluster-name.outputs.stdout }}
+          location: ${{ steps.get-k8s-cluster-location.outputs.stdout }}
+          credentials: ${{ secrets.GCP_SA_KEY }}
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.16' # The Go version to download (if necessary) and use.
+
+      - uses: azure/setup-helm@v1
+        id: install
+
+      - run: go mod download
+        working-directory: ci/integration/
+
+      - run: make integration
+        env:
+          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+          GCP_BQ_DATASET_ID: ${{ steps.get-gcp-bigquery-dataset-id.outputs.stdout }}
+          GCP_BQ_TABLE_ID: ${{ steps.get-gcp-bigquery-table-id.outputs.stdout }}
+          GCP_PROJECT_ID: fluent-bit-ci
+          IMAGE_REPOSITORY: fluentbitdev/fluent-bit
+          IMAGE_TAG: x86_64-1_7
+          GRAFANA_USERNAME: ${{ secrets.GRAFANA_USERNAME }}
+          GRAFANA_PASSWORD: ${{ secrets.GRAFANA_PASSWORD }}
+        working-directory: ci/

.github/workflows/README.workflows.md

@@ -0,0 +1,25 @@
+### Available workflows
+
+
+| Workflow file                                         | Description               | Run event                                         |
+| :---------------------------------------------------- | ------------------------  | ------------------------------------------------- |
+| [build-release](./build-release.yaml)            | Builds the distro packages and docker images from a tagged release| on new release/tag|
+| [publish-release](./publish-release.yaml)        | Publishes the docker images/manifest on hub.docker.io/fluent/ and the distro packages | on new release/tag on build-release completes|
+| [pr-closed-docker](./pr-closed-docker.yaml)      | Removes docker images for PR on hub.docker.io/fluentbitdev/| on pr closed|
+| [pr-stale](./pr-stale.yaml)                      | Closes stale PR(s) with no activity in 30 days | scheduled daily 01:30 AM UTC|
+| [integration-build-master](./integration-build-master.yaml)     | Builds a docker image to be used in integration testing (master branch) | on new commit/push on master|
+| [integration-build-pr](./integration-build-pr.yaml)     | Builds a docker image to be used in integration testing (pr branch) | on new commit/push on PR(s) |
+| [integration-run-master](./integration-run-master.yaml)     | Runs the integration testing suite on master | on new image built|
+| [integration-run-pr](./integration-run-pr.yaml)     | Runs the integration testing suite on a PR branch | pr opened / label created 'ok-to-test' |
+| [unit-tests](./unit-tests.yaml)     | Runs the unit tests suite on master push or new PR | PR opened, merge in master branch |
+
+### Available labels
+
+| Label name | Description | 
+| :----------|-------------|
+| docs-required| default tag used to request documentation, has to be removed before merge |
+| ok-to-test | run all integration tests |
+| ok-to-merge | run mergebot and merge (rebase) current PR |
+| ci/integration-docker-ok | integration test is able to build docker image |
+| ci/integration-gcp-ok | integration test is able to run on GCP |
+| long-term | long running pull request, don't close |

.github/workflows/benchmark-run-master.yaml

@@ -0,0 +1,118 @@
+name: Run benchmark tests for master
+on:
+  workflow_run:
+    workflows: [ 'Run integration tests for master' ]
+    types:
+      - completed
+jobs:
+  run-benchmark-gcp:
+    name: run-benchmark on GCP - k8s ${{ matrix.k8s-release }}
+    if: github.event.workflow_run.conclusion == 'success'
+    strategy:
+      max-parallel: 3
+      fail-fast: false
+      matrix:
+        k8s-release: [ '1.19' ] #, '1.20' ] #, 1.19/stable, 1.18/stable ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          repository: calyptia/fluent-bit-ci
+          path: ci
+
+      - uses: frabert/replace-string-action@master
+        id: formatted_release
+        with:
+          pattern: '(.*)\.(.*)$'
+          string: ${{ matrix.k8s-release }}
+          replace-with: '$1-$2'
+          flags: 'g'
+
+      - name: Replace the k8s release
+        run: |
+          sed -i -e "s/\$K8S_RELEASE/${{ env.k8s_release }}/g" default.auto.tfvars
+          sed -i -e "s/\$K8S_FORMATTED/${{ env.k8s_release_formatted }}/g" default.auto.tfvars
+          sed -i -e "s/\$K8S_FORMATTED/${{ env.k8s_release_formatted }}/g" config.tf
+        working-directory: ci/terraform/gcp/
+        env:
+          k8s_release: ${{ matrix.k8s-release }}
+          k8s_release_formatted: ${{ steps.formatted_release.outputs.replaced }}
+
+      - uses: hashicorp/setup-terraform@v1
+        with:
+          cli_config_credentials_hostname: 'app.terraform.io'
+          cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }}
+
+      - name: Replace the GCP service account key
+        run: |
+          cat <<EOT >> default.auto.tfvars
+          gcp-sa-key    =  <<-EOF
+          ${{ secrets.GCP_SA_KEY }}
+          EOF
+          EOT
+        working-directory: ci/terraform/gcp/
+
+      - name: Terraform fmt
+        id: fmt
+        run: terraform fmt -check
+        continue-on-error: true
+        working-directory: ci/terraform/gcp/
+
+      - name: Terraform Init
+        id: init
+        run: terraform init
+        working-directory: ci/terraform/gcp/
+
+      - name: Terraform Validate
+        id: validate
+        run: terraform validate -no-color
+        working-directory: ci/terraform/gcp/
+
+      - name: Terraform Apply
+        id: apply
+        run: |
+          terraform apply -input=false -auto-approve
+        working-directory: ci/terraform/gcp/
+
+      - name: Get the k8s cluster name from terraform output
+        id: get-k8s-cluster-name
+        run: terraform output -no-color -raw k8s-cluster-name
+        working-directory: ci/terraform/gcp/
+
+      - name: Get the k8s cluster location from terraform output
+        id: get-k8s-cluster-location
+        run: terraform output -no-color -raw k8s-cluster-location
+        working-directory: ci/terraform/gcp/
+
+      - name: Get the k8s project id from terraform output
+        id: get-gcp-project-id
+        run: terraform output -no-color -raw gcp-project-id
+        working-directory: ci/terraform/gcp/
+
+      - uses: google-github-actions/setup-gcloud@master
+        with:
+          service_account_key: ${{ secrets.GCP_SA_KEY }}
+
+      - uses: google-github-actions/get-gke-credentials@main
+        with:
+          cluster_name: ${{ steps.get-k8s-cluster-name.outputs.stdout }}
+          location: ${{ steps.get-k8s-cluster-location.outputs.stdout }}
+          credentials: ${{ secrets.GCP_SA_KEY }}
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.15' # The Go version to download (if necessary) and use.
+
+      - uses: azure/setup-helm@v1
+        id: install
+
+      - run: go mod download
+        working-directory: ci/integration/
+
+      - run: make benchmark
+        env:
+          IMAGE_REPOSITORY: fluentbitdev/fluent-bit
+          IMAGE_TAG: x86_64-master
+          GRAFANA_USERNAME: ${{ secrets.GRAFANA_USERNAME }}
+          GRAFANA_PASSWORD: ${{ secrets.GRAFANA_PASSWORD }}
+        working-directory: ci/

.github/workflows/build-master-packages.yaml

@@ -0,0 +1,54 @@
+on:
+  push:
+    branches:
+      - master
+
+name: Build packages for master
+jobs:
+  build-distro-packages:
+    name: build packages
+    strategy:
+      max-parallel: 48
+      fail-fast: true
+      matrix:
+        distro: [ ubuntu/16.04, ubuntu/18.04, ubuntu/20.04, debian/buster ]
+
+    runs-on: [ ubuntu-latest ] #self-hosted, Linux, X64, packet-builder]
+    steps:
+      - name: Setup environment
+        run: |
+          sudo apt-get update
+          sudo apt-get install --yes qemu binfmt-support qemu-user-static qemu-utils qemu-efi-aarch64 qemu-system-arm docker.io containerd runc
+          sudo systemctl unmask docker && sudo systemctl start docker
+          docker run --rm --privileged --name qemu multiarch/qemu-user-static:register --reset
+
+      - uses: frabert/replace-string-action@master
+        id: formatted_distro
+        with:
+          pattern: '(.*)\/(.*)$'
+          string: "${{ matrix.distro }}"
+          replace-with: '$1-$2'
+          flags: 'g'
+
+      - uses: actions/checkout@v2
+        with:
+          repository: fluent/fluent-bit-packaging
+          fetch-depth: 1
+          path: packaging
+
+      - name: Build the distro artifacts
+        run: ./build.sh -v master -d ${{ env.distro }} -b master
+        env:
+          distro: ${{ matrix.distro }}
+        working-directory: packaging
+
+      - name: Store the master package artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: packages-${{env.release}}-${{env.bucket-name}}
+          path: |
+            packaging/packages/${{env.distro}}/${{env.release}}/**/*
+        env:
+          bucket-name: ${{ steps.formatted_distro.outputs.replaced }}
+          release: master
+          distro: ${{ matrix.distro }}