Merge branch 'cosmos_binding_track2' of https://github.com/RyanLettie…

…ri/components-contrib into cosmos_binding_track2

.codecov.yaml

@@ -11,5 +11,5 @@ coverage:
       # See https://docs.codecov.io/docs/commit-status#disabling-a-status.
       default: false
 comment:
-  # Delete old comment and post new one for new coverage information.
-  behavior: new
+  # Update old comment with new coverage information if the PR is changed. Avoids triggering multiple emails.
+  behavior: once

.github/workflows/certification.yml

@@ -25,24 +25,11 @@ on:
       - release-*
 
 jobs:
-  pre_job:
-    name: Skip Duplicate Actions
-    runs-on: ubuntu-latest
-    outputs:
-      should_skip: ${{ steps.skip_check.outputs.should_skip }}
-    steps:
-      - id: skip_check
-        uses: fkirc/[email protected]
-        with:
-          cancel_others: 'true'
-          paths_ignore: '["**.md", ".codecov.yaml", ".github/workflows/dapr-automerge.yml"]'
   # Based on whether this is a PR or a scheduled run, we will run a different
   # subset of the certification tests. This allows all the tests not requiring
   # secrets to be executed on pull requests.
   generate-matrix:
     runs-on: ubuntu-latest
-    needs: pre_job
-    if: needs.pre_job.outputs.should_skip != 'true' || github.event_name == 'repository_dispatch'
     steps:
     - name: Install yq
       run: |
@@ -59,6 +46,8 @@ jobs:
         - state.postgresql
         - bindings.alicloud.dubbo
         - bindings.kafka
+        - secretstores.local.env
+        - secretstores.local.file
         EOF
         )
         echo "::set-output name=pr-components::$PR_COMPONENTS"
@@ -107,6 +96,8 @@ jobs:
           required-secrets: AzureBlobStorageAccount, AzureBlobStorageAccessKey
         - component: state.azure.tablestorage
           required-secrets: AzureBlobStorageAccount, AzureBlobStorageAccessKey, AzureCertificationTenantId, AzureCertificationServicePrincipalClientId, AzureCertificationServicePrincipalClientSecret
+        - component: state.azure.blobstorage
+          required-secrets: AzureBlobStorageContainer,AzureBlobStorageAccount, AzureBlobStorageAccessKey, AzureCertificationTenantId, AzureCertificationServicePrincipalClientId, AzureCertificationServicePrincipalClientSecret
         EOF
         )
         echo "::set-output name=cloud-components::$CRON_COMPONENTS"
@@ -156,11 +147,13 @@ jobs:
         export TEST_OUTPUT_FILE_PREFIX=$GITHUB_WORKSPACE/test_report
         echo "TEST_OUTPUT_FILE_PREFIX=$TEST_OUTPUT_FILE_PREFIX" >> $GITHUB_ENV
 
-    - name: Configure certification test path
+    - name: Configure certification test and source path
       run: |
         TEST_COMPONENT=$(echo ${{ matrix.component }} | sed -E 's/\./\//g')
         export TEST_PATH="${PROJECT_PATH}/tests/certification/${TEST_COMPONENT}"
         echo "TEST_PATH=$TEST_PATH" >> $GITHUB_ENV
+        export SOURCE_PATH="github.com/dapr/components-contrib/${TEST_COMPONENT}"
+        echo "SOURCE_PATH=$SOURCE_PATH" >> $GITHUB_ENV
 
     - uses: Azure/login@v1
       with:
@@ -201,6 +194,7 @@ jobs:
       run: |
         go mod download
         go install gotest.tools/gotestsum@latest
+        go install github.com/axw/gocov/[email protected]
 
     - name: Check that go mod tidy is up-to-date
       working-directory: ${{ env.TEST_PATH }}
@@ -218,7 +212,13 @@ jobs:
         set +e
         gotestsum --jsonfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.json \
           --junitfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.xml --format standard-quiet -- \
-          -count=1 -timeout=15m
+          -coverprofile=cover.out -covermode=set -coverpkg=${{ env.SOURCE_PATH }}
+
+        COVERAGE_REPORT=$(gocov convert cover.out | gocov report)
+        COVERAGE_LINE=$(echo $COVERAGE_REPORT  | grep -oP '(?<=Total Coverage:).*') # example: "80.00% (40/50)"
+        COVERAGE_PERCENTAGE=$(echo $COVERAGE_LINE | grep -oP '([0-9\.]*)' | head -n 1) # example "80.00"
+        echo "COVERAGE_LINE=$COVERAGE_LINE" >> $GITHUB_ENV
+        echo "COMPONENT_PERCENTAGE=$COVERAGE_PERCENTAGE" >> $GITHUB_ENV
 
         status=$?
         echo "Completed certification tests for ${{ matrix.component }} ... "
@@ -245,10 +245,69 @@ jobs:
           exit 1
         fi
 
+    - name: Prepare Cert Coverage Info
+      run: |
+       mkdir -p tmp/cov_files
+       SOURCE_PATH_LINEAR=$(echo ${{ env.SOURCE_PATH }} |sed 's#/#\.#g') # converts slashes to dots in this string, so that it doesn't consider them sub-folders
+       echo "${{ env.COVERAGE_LINE }}" >> tmp/cov_files/$SOURCE_PATH_LINEAR.txt
+
+    - name: Upload Cert Coverage Artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: certtest_cov
+        path: tmp/cov_files
+        retention-days: 1
+
+    - name: Component Coverage Discord Notification 
+      env:
+        DISCORD_WEBHOOK: ${{ secrets.DISCORD_MONITORING_WEBHOOK_URL }}
+      uses: Ilshidur/action-discord@0c4b27844ba47cb1c7bee539c8eead5284ce9fa9
+      continue-on-error: true
+      with:
+        args: 'Cert Test Coverage for {{ SOURCE_PATH }} is {{ COVERAGE_LINE }}'
+
     # Upload logs for test analytics to consume
     - name: Upload test results
       if: always()
       uses: actions/upload-artifact@master
       with:
         name: ${{ matrix.component }}_certification_test
         path: ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.*
+
+  post_job:
+    name: Notify Total coverage
+    runs-on: ubuntu-latest
+    needs: certification
+    if: always()
+    steps:
+      - name: Download Cert Coverage Artifact
+        uses: actions/download-artifact@v3
+        continue-on-error: true
+        id: download
+        with:
+          name: certtest_cov
+          path: tmp/cov_files
+
+      - name: Calculate total coverage
+        run: |
+          ls "${{steps.download.outputs.download-path}}" | while read f; do
+          while read LINE;
+              do
+              ratio=$(echo $LINE | cut -d "(" -f2 | cut -d ")" -f1)
+              tempNumerator=$(echo $ratio | cut -d'/' -f1)
+              tempDenominator=$(echo $ratio | cut -d'/' -f2)
+              export numerator=$(($numerator+$tempNumerator))
+              export denominator=$(($denominator+$tempDenominator))
+              totalPer=$(awk "BEGIN { print (($numerator / $denominator) * 100) }")
+              echo "totalPer=$totalPer" >> $GITHUB_ENV
+            done < ${{steps.download.outputs.download-path}}/$f
+          done
+        continue-on-error: true
+
+      - name: Final Coverage Discord Notification
+        env:
+          DISCORD_WEBHOOK: ${{ secrets.DISCORD_MONITORING_WEBHOOK_URL }}
+        uses: Ilshidur/action-discord@0c4b27844ba47cb1c7bee539c8eead5284ce9fa9
+        continue-on-error: true
+        with:
+          args: 'Total Coverage for Certification Tests is {{ totalPer }}%'

.github/workflows/components-contrib.yml

@@ -29,11 +29,11 @@ jobs:
     name: Build ${{ matrix.target_os }}_${{ matrix.target_arch }} binaries
     runs-on: ${{ matrix.os }}
     env:
-      GOVER: 1.18
+      GOVER: "1.18"
       GOOS: ${{ matrix.target_os }}
       GOARCH: ${{ matrix.target_arch }}
       GOPROXY: https://proxy.golang.org
-      GOLANGCI_LINT_VER: v1.45.2
+      GOLANGCI_LINT_VER: "v1.45.2"
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macOS-latest]
@@ -51,12 +51,6 @@ jobs:
           - os: macOS-latest
             target_arch: arm
     steps:
-      - name: Check if need skip
-        id: skip_check
-        uses: fkirc/[email protected]
-        with:
-          cancel_others: 'true'
-          paths_ignore: '["**.md", ".codecov.yaml", ".github/workflows/dapr-automerge.yml"]'
       - name: Set up Go ${{ env.GOVER }}
         if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
         uses: actions/setup-go@v2
@@ -65,19 +59,52 @@ jobs:
       - name: Check out code into the Go module directory
         if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
         uses: actions/checkout@v2
+      - name: Cache Go modules (Linux)
+        if: matrix.target_os == 'linux'
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ matrix.target_os }}-${{ matrix.target_arch }}-go-${{ env.GOVER }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ matrix.target_os }}-${{ matrix.target_arch }}-go-${{ env.GOVER }}-
+      - name: Cache Go modules (Windows)
+        if: matrix.target_os == 'windows'
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~\AppData\Local\go-build
+            ~\go\pkg\mod
+          key: ${{ matrix.target_os }}-${{ matrix.target_arch }}-go-${{ env.GOVER }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ matrix.target_os }}-${{ matrix.target_arch }}-go-${{ env.GOVER }}-
+      - name: Cache Go modules (macOS)
+        if: matrix.target_os == 'darwin'
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/Library/Caches/go-build
+            ~/go/pkg/mod
+          key: ${{ matrix.target_os }}-${{ matrix.target_arch }}-go-${{ env.GOVER }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ matrix.target_os }}-${{ matrix.target_arch }}-go-${{ env.GOVER }}-
       - name: Run golangci-lint
         if: matrix.target_arch == 'amd64' && matrix.target_os == 'linux' && steps.skip_check.outputs.should_skip != 'true'
-        uses: golangci/golangci-lint-action@v3.1.0
+        uses: golangci/golangci-lint-action@v3.2.0
         with:
           version: ${{ env.GOLANGCI_LINT_VER }}
+          skip-cache: true
+          args: --timeout 15m
       - name: Run go mod tidy check diff
         if: matrix.target_arch == 'amd64' && matrix.target_os == 'linux' && steps.skip_check.outputs.should_skip != 'true'
         run: make modtidy-all check-diff
       - name: Run make test
         env:
           COVERAGE_OPTS: "-coverprofile=coverage.txt -covermode=atomic"
+          IPFS_TEST: "1"
         if: matrix.target_arch != 'arm' && steps.skip_check.outputs.should_skip != 'true'
         run: make test
       - name: Codecov
         if: matrix.target_arch == 'amd64' && matrix.target_os == 'linux'
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v3

.github/workflows/conformance.yml

@@ -25,24 +25,11 @@ on:
       - release-*
 
 jobs:
-  pre_job:
-    name: Skip Duplicate Actions
-    runs-on: ubuntu-latest
-    outputs:
-      should_skip: ${{ steps.skip_check.outputs.should_skip }}
-    steps:
-      - id: skip_check
-        uses: fkirc/[email protected]
-        with:
-          cancel_others: 'true'
-          paths_ignore: '["**.md", ".codecov.yaml", ".github/workflows/dapr-automerge.yml"]'
   # Based on whether this is a PR or a scheduled run, we will run a different
   # subset of the conformance tests. This allows all the tests not requiring
   # secrets to be executed on pull requests.
   generate-matrix:
     runs-on: ubuntu-latest
-    needs: pre_job
-    if: needs.pre_job.outputs.should_skip != 'true' || github.event_name == 'repository_dispatch'
     steps:
     - name: Install yq
       run: |
@@ -59,6 +46,7 @@ jobs:
         - bindings.mqtt-mosquitto
         - bindings.mqtt-vernemq
         - bindings.redis
+        - bindings.rabbitmq
         - pubsub.aws.snssqs
         - pubsub.hazelcast
         - pubsub.in-memory

.golangci.yml

@@ -4,7 +4,7 @@ run:
   concurrency: 4
 
   # timeout for analysis, e.g. 30s, 5m, default is 1m
-  deadline: 5m
+  deadline: 15m
 
   # exit code when at least one issue was found, default is 1
   issues-exit-code: 1

bindings/alicloud/rocketmq/rocketmq.go

@@ -90,11 +90,12 @@ func (a *AliCloudRocketMQ) Read(ctx context.Context, handler bindings.Handler) e
 		if topicStr == "" {
 			continue
 		}
-		mqType, mqExpression, topic, err := parseTopic(topicStr)
-		if err != nil {
+
+		var mqType, mqExpression, topic string
+		if mqType, mqExpression, topic, err = parseTopic(topicStr); err != nil {
 			return err
 		}
-		if err := consumer.Subscribe(
+		if err = consumer.Subscribe(
 			topic,
 			mqc.MessageSelector{
 				Type:       mqc.ExpressionType(mqType),
@@ -106,7 +107,7 @@ func (a *AliCloudRocketMQ) Read(ctx context.Context, handler bindings.Handler) e
 		}
 	}
 
-	if err := consumer.Start(); err != nil {
+	if err = consumer.Start(); err != nil {
 		return fmt.Errorf("binding-rocketmq: consumer start failed. %w", err)
 	}
 
@@ -121,7 +122,7 @@ func (a *AliCloudRocketMQ) Read(ctx context.Context, handler bindings.Handler) e
 
 		innerErr := consumer.Shutdown()
 		if innerErr != nil && !errors.Is(innerErr, context.Canceled) {
-			a.logger.Warnf("binding-rocketmq: error while shutting down consumer: %v")
+			a.logger.Warnf("binding-rocketmq: error while shutting down consumer: %v", innerErr)
 		}
 	}()