chore: improve SI error on creation (NR-314131) #1239

Workflow file for this run

.github/workflows/validation.yml at e456957

	name: Validate Recipe (Unit Test)
	on: [pull_request, workflow_dispatch]

	jobs:
	log-context:
	runs-on: ubuntu-latest
	steps:
	# Dump all contexts
	- name: Dump GitHub context
	env:
	GITHUB_CONTEXT: ${{ toJson(github) }}
	run: echo "$GITHUB_CONTEXT"
	- name: Dump job context
	env:
	JOB_CONTEXT: ${{ toJson(job) }}
	run: echo "$JOB_CONTEXT"
	- name: Dump steps context
	env:
	STEPS_CONTEXT: ${{ toJson(steps) }}
	run: echo "$STEPS_CONTEXT"
	- name: Dump runner context
	env:
	RUNNER_CONTEXT: ${{ toJson(runner) }}
	run: echo "$RUNNER_CONTEXT"
	- name: Dump strategy context
	env:
	STRATEGY_CONTEXT: ${{ toJson(strategy) }}
	run: echo "$STRATEGY_CONTEXT"
	- name: Dump matrix context
	env:
	MATRIX_CONTEXT: ${{ toJson(matrix) }}
	run: echo "$MATRIX_CONTEXT"

	validate-schema:
	name: Validation
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2

	- name: Setup Node.js
	uses: actions/setup-node@v1
	with:
	node-version: "14"

	- name: Install dependencies
	run: npm --prefix validator install

	- name: Validate
	run: npm --prefix validator run check

	get-test-definition-files:
	name: Get Test Definition Files
	needs: [validate-schema]
	runs-on: ubuntu-latest
	outputs:
	matrix: ${{ steps.get-test-definition-files.outputs.result }}
	steps:
	- name: Checkout Repo
	uses: actions/checkout@v2
	with:
	fetch-depth: 0

	- name: Ensure no fork repo
	if: ${{ github.event_name == 'pull_request' }}
	run: \|
	SOURCE_NAME=$(echo ${{ github.event.pull_request.head.repo.full_name }})
	REMOTE_NAME=$(echo ${{ github.event.pull_request.base.repo.full_name }})
	if [ $SOURCE_NAME != $REMOTE_NAME ]; then
	echo "PR should be created from a branch on the source repo, not from a fork, so the E2E tests can run. If you need access please reach out to the #help-virtuoso channel."
	exit 1
	fi

	# 1) Check for all incoming changes to files under `recipes` directory
	- name: Get Changed Files
	id: getfile
	run: \|
	RECIPES=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} \| grep "recipes" \|\| true)
	if [ -z "$RECIPES" ]; then
	echo "No recipe files detected."
	else
	echo "RECIPES<<EOF" >> $GITHUB_ENV
	echo $RECIPES >> $GITHUB_ENV
	echo "EOF" >> $GITHUB_ENV
	fi

	TEST_DEFINITIONS=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} \| grep "test/definitions" \|\| true)
	if [ -z "$TEST_DEFINITIONS" ]; then
	echo "No test definitions files detected."
	else
	echo "TEST_DEFINITIONS<<EOF" >> $GITHUB_ENV
	echo $TEST_DEFINITIONS >> $GITHUB_ENV
	echo "EOF" >> $GITHUB_ENV
	fi

	# 2) Use javascript action / yq to pull the test definition file from the recipe yaml
	- name: Get Test Definition Files
	id: get-test-definition-files
	uses: actions/github-script@v3
	with:
	script: \|
	const fs = require('fs');
	const fsp = fs.promises;
	const path = require('path');

	// readdir recursive directory search
	const { resolve } = path;
	const { readdir } = fsp;
	async function getFiles(dir) {
	const dirents = await readdir(dir, { withFileTypes: true });
	const files = await Promise.all(dirents.map((dirent) => {
	const res = path.join(dir, dirent.name);
	return dirent.isDirectory() ? getFiles(res) : res;
	}));
	return Array.prototype.concat(...files);
	}

	var outputTestFiles = []

	if (process.env.TEST_DEFINITIONS != '') {
	// Get incoming added/updated test definitions files
	const testDefinitionFiles = process.env.TEST_DEFINITIONS ? process.env.TEST_DEFINITIONS.split(' ') : []
	console.log(`Detected Test Definitions Files: ${JSON.stringify(testDefinitionFiles, null, 2)}`)

	testDefinitionFiles.forEach(testDefinitionFile => {
	if (!outputTestFiles.includes(testDefinitionFile)) {
	outputTestFiles.push(testDefinitionFile)
	}
	})
	}

	if (process.env.RECIPES != '') {
	// Get incoming added/updated recipe files
	const recipeFiles = process.env.RECIPES ? process.env.RECIPES.split(' ') : []
	console.log(`Detected Recipe Files: ${JSON.stringify(recipeFiles, null, 2)}`)

	if (recipeFiles.length) {
	// Get all deploy config files
	const deployConfigsUS = await getFiles('test/definitions');
	const deployConfigsEU = await getFiles('test/definitions-eu');
	const deployConfigs = deployConfigsUS.concat(deployConfigsEU);
	console.log("All deployConfigs:", deployConfigs);

	// Build up list of Deploy Configs to run based on recipes that have changed
	const testDefinitionFilesToRun = deployConfigs.reduce((p, c) => {
	const contents = require(`${process.env.GITHUB_WORKSPACE}/${c}`);

	var recipes = []
	if (contents.instrumentations && contents.instrumentations.resources) {
	contents.instrumentations.resources.forEach(resource => {
	if (resource.params && resource.params.recipe_content_url) {
	recipes = recipes.concat(resource.params.recipe_content_url);
	}
	});
	}

	// returns list of matched recipes from the deploy config file
	const matchedRecipes = recipes.filter(
	(r) => recipeFiles.filter((rf) => r.includes(rf)).length > 0
	);

	// Add the current deploy config file to our output if there were matches
	return matchedRecipes.length > 0 ? [`${c}`, ...p] : p;
	}, []);
	console.log('testDefinitionFilesToRun:', testDefinitionFilesToRun);

	testDefinitionFilesToRun.forEach(testDefinitionFile => {
	if (!outputTestFiles.includes(testDefinitionFile)) {
	outputTestFiles.push(testDefinitionFile)
	}
	})
	}
	}

	const outputTestFilesMap = outputTestFiles.map(testDefinitionFile => {
	return { testDefinitionFile }
	})
	const output = {
	"include": outputTestFilesMap
	}
	console.log("Output: ", output);
	return output;

	test-deploy-recipe:
	name: Test Deploy Recipe
	needs: [get-test-definition-files]
	if: ${{ fromJSON(needs.get-test-definition-files.outputs.matrix).include[0] }} # Avoids empty matrix validation error
	runs-on: ubuntu-latest
	strategy:
	matrix: ${{ fromJSON(needs.get-test-definition-files.outputs.matrix) }}
	fail-fast: false
	max-parallel: 10
	env:
	MATRIX: ${{ toJSON(matrix) }}
	steps:
	- name: Checkout Repo
	uses: actions/checkout@v2
	with:
	fetch-depth: 0

	- name: Update Test Definition Files URLs
	id: get-test-definition-files
	env:
	TEST_DEFINITION_FILE: ${{ matrix.testDefinitionFile }}
	uses: actions/github-script@v3
	with:
	script: \|
	const fs = require('fs');
	const fsp = fs.promises;
	const path = require('path');

	// before returning, we need to edit the deploy config files in-place so they
	// use the right URLs from the branch
	async function getDeployConfigFile(file, outputDir) {
	const data = await fsp.readFile(path.join(outputDir, file));
	return JSON.parse(data);
	}

	// Get testDefinitonFile from MATRIX env var
	const testDefinitionFile = process.env.TEST_DEFINITION_FILE;
	console.log(`Detected Deploy Config: ${JSON.stringify(testDefinitionFile, null, 2)}`)

	// Update URLs to use branch this PR is opened with
	const data = await getDeployConfigFile(testDefinitionFile, process.env.GITHUB_WORKSPACE);

	// Update github source URLs with branch name
	let jsonContent = JSON.stringify(data, null, 2);
	const replacementString = `$1$2-b ${process.env.GITHUB_HEAD_REF} $3$4`;
	const sourceRepositoryRegex = /(.)(\")(https:\/\/github.com\/newrelic\/open-install-library)(.)/gi;
	jsonContent = jsonContent.replace(sourceRepositoryRegex, replacementString);

	// Update raw URLs with branch name
	const replacementString2 = `$1${process.env.GITHUB_HEAD_REF}$3`;
	const sourceRepositoryRegex2 = /(raw.githubusercontent.com\/newrelic\/open-install-library\/)(main)(\/newrelic\/recipes\/)*/gi;
	jsonContent = jsonContent.replace(sourceRepositoryRegex2, replacementString2);

	// Write file back to workspace
	const outputPath = `${process.env.GITHUB_WORKSPACE}/${testDefinitionFile}`;
	console.log("Updated Deploy Config File: ", outputPath);
	console.log("Deploy Config content: ", jsonContent);
	fs.writeFileSync(outputPath, jsonContent);

	return testDefinitionFile;

	- name: Write AWS Certificate to File
	env:
	AWS_PEM: ${{ secrets.GIT_DEPLOYER_CANADA_AWS_PEM }}
	run: \|
	mkdir -p configs
	rm -f configs/gitdeployerCanada.pem
	echo "$AWS_PEM" > configs/gitdeployerCanada.pem
	sudo chmod 400 configs/gitdeployerCanada.pem

	- name: Write Test Definition File JSON to file
	env:
	USER_JSON: ${{ secrets.GIT_DEPLOYER_DOCKER_USER_CONFIG }}
	run: \|
	echo "$USER_JSON" > configs/gitusdkr${{ github.run_id }}.json

	- name: Pull Deployer image
	run: \|
	docker pull newrelic/deployer:latest
	docker images newrelic/deployer:latest

	- name: Run deployer
	id: deployerRun
	continue-on-error: true
	run: \|
	set +e
	testDefinitionFile=$(echo $MATRIX \| jq -c -r '.testDefinitionFile')
	echo $testDefinitionFile
	docker run \
	-v ${{ github.workspace }}/configs/:/mnt/deployer/configs/\
	-v ${{ github.workspace }}/test/:/mnt/deployer/test/\
	--entrypoint ruby newrelic/deployer:latest main.rb -c configs/gitusdkr${{ github.run_id }}.json -d $testDefinitionFile -l debug
	echo ::set-output name=exit_status::$?

	- name: Teardown any previous deployment
	if: always()
	id: cleanupResources
	continue-on-error: true
	run: \|
	testDefinitionFile=$(echo $MATRIX \| jq -c -r '.testDefinitionFile')
	echo $testDefinitionFile
	docker run \
	-v ${{ github.workspace }}/configs/:/mnt/deployer/configs/\
	-v ${{ github.workspace }}/test/:/mnt/deployer/test/\
	--entrypoint ruby newrelic/deployer:latest main.rb -c configs/gitusdkr${{ github.run_id }}.json -d $testDefinitionFile -t

	- name: Report any error
	if: steps.deployerRun.outputs.exit_status != 0
	run: exit 1

	slack-notify:
	runs-on: ubuntu-latest
	needs: [test-deploy-recipe]
	if: always()
	steps:
	- name: Build Result Slack Notification
	uses: 8398a7/action-slack@v3
	with:
	author_name: GitHub Actions
	status: custom
	fields: commit,repo,ref,author,eventName,message,workflow
	custom_payload: \|
	{
	username: "GitHub Actions",
	icon_emoji: ":octocat:",
	attachments: [{
	color: ${{
	needs.test-deploy-recipe.result == 'success'
	}} === true ? '#43cc11' : '#e05d44',
	blocks: [
	{
	type: "section",
	text: {
	type: "mrkdwn",
	text: `Build for ${process.env.AS_REPO}`
	}
	},
	{
	type: "section",
	fields: [
	{
	type: "mrkdwn",
	text: `Commit:\n${process.env.AS_COMMIT}`
	},
	{
	type: "mrkdwn",
	text: `Author:\n${process.env.AS_AUTHOR}`
	},
	{
	type: "mrkdwn",
	text: `Branch:\n${process.env.AS_REF}`
	},
	{
	type: "mrkdwn",
	text: `Message:\n${process.env.AS_MESSAGE}`
	},
	{
	type: "mrkdwn",
	text: `Type:\n${process.env.AS_EVENT_NAME}`
	},
	{
	type: "mrkdwn",
	text: "PR:\n${{ github.event.pull_request.html_url }}"
	},
	{
	type: "mrkdwn",
	text: `Workflow:\n${ process.env.AS_WORKFLOW }`
	}
	]
	},
	{
	type: "section",
	text: {
	type: "mrkdwn",
	text: [
	"Result:",
	`• ${ ${{ needs.test-deploy-recipe.result == 'success' }} === true ? '✅' : '❌' } AWS recipe validation test: ${{ needs.test-deploy-recipe.result }}`
	].join('\n')
	}
	},
	{
	type: "context",
	elements: [
	{
	type: "image",
	image_url: "https://avatars2.githubusercontent.com/in/15368",
	alt_text: "Github Actions"
	},
	{
	type: "mrkdwn",
	text: "This message was created automatically by GitHub Actions."
	}
	]
	}
	]
	}]
	}
	env:
	GITHUB_TOKEN: ${{ github.token }}
	SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

chore: improve SI error on creation (NR-314131) #1239

Workflow file

chore: improve SI error on creation (NR-314131) #1239

Jobs

Run details

Workflow file for this run