Ensure failed conformance tests are caught in pipeline & fix local ma…

…ke command (#1203)

* Fail conf tests in pipeline & fix local make command

.github/workflows/conformance.yml

@@ -149,7 +149,11 @@ jobs:
         working-directory: ./conformance
 
       - name: Run conformance tests
-        run: make run-conformance-tests TAG=${{ github.sha }} VERSION=${{ github.ref_name }}
+        run: |
+          make run-conformance-tests TAG=${{ github.sha }} VERSION=${{ github.ref_name }}
+          core_result=$(cat conformance-profile.yaml | yq '.profiles[0].core.result')
+          extended_result=$(cat conformance-profile.yaml | yq '.profiles[0].extended.result')
+          if [ "${core_result}" == "failure" ] || [ "${extended_result}" == "failure" ]; then echo "Conformance test failed, see above for details." && exit 2; fi
         working-directory: ./conformance
 
       - name: Upload profile to release

conformance/Makefile

@@ -79,9 +79,8 @@ run-conformance-tests: ## Run conformance tests
 								--report-output=output.txt; cat output.txt" | tee output.txt
 	sed -e '1,/CONFORMANCE PROFILE/d' output.txt > conformance-profile.yaml
 	rm output.txt
-	$(eval result_core=$(shell cat conformance-profile.yaml | yq '.profiles[0].core.result'))
-	$(eval result_extended=$(shell cat conformance-profile.yaml | yq '.profiles[0].extended.result'))
-	[ "$(result_core)" != "failure" ] && [ "$(result_extended)" != "failure" ] || exit 2
+	[ $(shell cat conformance-profile.yaml | yq '.profiles[0].core.result') != "failure" ] \
+		&& [ $(shell cat conformance-profile.yaml | yq '.profiles[0].extended.result') != "failure" ] || exit 2
 
 .PHONY: cleanup-conformance-tests
 cleanup-conformance-tests: ## Clean up conformance tests fixtures