forked from WASdev/azure.websphere-traditional.image
-
Notifications
You must be signed in to change notification settings - Fork 0
329 lines (317 loc) · 18 KB
/
twas-baseBuild.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
# Copyright (c) IBM Corporation.
# Copyright (c) Microsoft Corporation.
name: twas-base CICD
# Controls when the action will run.
on:
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
inputs:
imageVersionNumber:
description: 'Must provide image version number'
required: true
# Allows you to run this workflow using GitHub APIs
# PERSONAL_ACCESS_TOKEN=<GITHUB_PERSONAL_ACCESS_TOKEN>
# REPO_NAME=WASdev/azure.websphere-traditional.image
# curl --verbose -X POST -u "WASdev:${PERSONAL_ACCESS_TOKEN}" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" https://api.github.com/repos/${REPO_NAME}/actions/workflows/twas-baseBuild.yml/dispatches --data '{"ref": "main", "inputs": {"imageVersionNumber": "9.0.20230428"}}'
repository_dispatch:
types: [integration-test-twasbase, integration-test-all]
# sample request
# PERSONAL_ACCESS_TOKEN=<GITHUB_PERSONAL_ACCESS_TOKEN>
# REPO_NAME=WASdev/azure.websphere-traditional.image
# curl --verbose -X POST https://api.github.com/repos/${REPO_NAME}/dispatches -H "Accept: application/vnd.github.everest-preview+json" -H "Authorization: token ${PERSONAL_ACCESS_TOKEN}" --data '{"event_type": "integration-test-twasbase", "client_payload": {"imageVersionNumber": "9.0.20230428"}}'
# The integration-test-all event is used to run all integration tests (twas-base, twas-nd & ihs) in the repository with the same image version number.
# curl --verbose -X POST https://api.github.com/repos/${REPO_NAME}/dispatches -H "Accept: application/vnd.github.everest-preview+json" -H "Authorization: token ${PERSONAL_ACCESS_TOKEN}" --data '{"event_type": "integration-test-all", "client_payload": {"imageVersionNumber": "9.0.20230428"}}'
env:
repoName: azure.websphere-traditional.image
userName: ${{ secrets.USER_NAME }}
azureCredentials: ${{ secrets.AZURE_CREDENTIALS }}
entitledIbmUserId: ${{ secrets.ENTITLED_IBM_USER_ID }}
entitledIbmPassword: ${{ secrets.ENTITLED_IBM_USER_PWD }}
unEntitledIbmUserId: ${{ secrets.UNENTITLED_IBM_USER_ID }}
unEntitledIbmPassword: ${{ secrets.UNENTITLED_IBM_USER_PWD }}
vmAdminId: ${{ secrets.VM_ADMIN_ID }}
vmAdminPassword: ${{ secrets.VM_ADMIN_PASSWORD }}
testResourceGroup: imageTest${{ github.run_id }}${{ github.run_number }}-twasBase
vmName: vm${{ github.run_id }}${{ github.run_number }}
vhdStorageAccountName: storage${{ github.run_id }}${{ github.run_number }}
location: eastus
scriptLocation: https://raw.githubusercontent.com/${{ secrets.USER_NAME }}/azure.websphere-traditional.image/$GITHUB_REF_NAME/twas-base/test/
utilitiesLocation: https://raw.githubusercontent.com/${{ secrets.USER_NAME }}/azure.websphere-traditional.image/$GITHUB_REF_NAME/utilities/
accessToken: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
twasSingleRepo: ${{ secrets.USER_NAME }}/azure.websphere-traditional.singleserver
offerId: "2023-03-27-twas-single-server-base-image"
planId: "2023-03-27-twas-single-server-base-image"
offerType: "vm_image_offer"
clientId: ${{ secrets.CLIENT_ID }}
secretValue: ${{ secrets.SECRET_VALUE }}
tenantId: ${{ secrets.TENANT_ID }}
imageType: "x64Gen1"
operatingSystemFamily: "linux"
operatingSystemType: "redHat"
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
build:
# The type of runner that the job will run on
runs-on: ubuntu-latest
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
- name: Get versions of external dependencies
run: |
curl -Lo external-deps-versions.properties https://raw.githubusercontent.com/Azure/azure-javaee-iaas/main/external-deps-versions.properties
source external-deps-versions.properties
echo "azCliVersion=${AZ_CLI_VERSION}" >> $GITHUB_ENV
echo "refArmttk=${ARM_TTK_REFERENCE}" >> $GITHUB_ENV
echo "refJavaee=${AZURE_JAVAEE_IAAS_REFERENCE}" >> $GITHUB_ENV
- name: Setup environment variables
id: setup-env-variables-based-on-dispatch-event
run: |
if [ ${{ github.event_name }} == 'workflow_dispatch' ]; then
imageVersionNumber=${{ github.event.inputs.imageVersionNumber }}
else
imageVersionNumber=${{ github.event.client_payload.imageVersionNumber }}
fi
echo "##[set-output name=imageVersionNumber;]${imageVersionNumber}"
- name: Set up JDK 11
uses: actions/setup-java@v2
with:
distribution: 'microsoft'
java-version: '11'
- name: Checkout azure-javaee-iaas
uses: actions/checkout@v3
with:
repository: Azure/azure-javaee-iaas
path: azure-javaee-iaas
ref: ${{ env.refJavaee }}
- name: Checkout arm-ttk
uses: actions/checkout@v3
with:
repository: Azure/arm-ttk
path: arm-ttk
ref: ${{ env.refArmttk }}
- name: Checkout ${{ env.repoName }}
uses: actions/checkout@v3
with:
path: ${{ env.repoName }}
ref: ${{ github.event.inputs.ref }}
- name: Build azure-javaee-iaas
run: mvn -DskipTests clean install --file azure-javaee-iaas/pom.xml
- name: Azure login
uses: azure/login@v1
with:
creds: ${{ env.azureCredentials }}
- name: Build ${{ env.repoName }}
run: |
echo "Branch name is $GITHUB_REF_NAME"
mvn -Dgit.repo=${{ env.userName }} -Dgit.tag=$GITHUB_REF_NAME \
-DibmUserId=${{ env.entitledIbmUserId }} -DibmUserPwd=${{ env.entitledIbmPassword }} \
-DstorageAccount=${{ env.vhdStorageAccountName }} -DvmName=${{ env.vmName }} \
-DvmAdminId=${{ env.vmAdminId }} -DvmAdminPwd=${{ env.vmAdminPassword }} \
-Dtest.args="-Test All" -Ptemplate-validation-tests -Dtemplate.validation.tests.directory=../../arm-ttk/arm-ttk \
clean install --file ${{ env.repoName }}/twas-base/pom.xml
- name: Deploy a RHEL 9.x VM and install twas Base server
run: |
cd ${{ env.repoName }}/twas-base/target/cli
chmod a+x deploy.azcli
./deploy.azcli -n testDeployment -g ${{ env.testResourceGroup }} -l ${{ env.location }}
- name: Query public IP of VM
uses: azure/CLI@v1
with:
azcliversion: ${{ env.azCliVersion }}
inlineScript: |
echo "query public ip"
publicIP=$(az vm show \
--resource-group ${{ env.testResourceGroup }} \
--name ${{ env.vmName }} -d \
--query publicIps -o tsv)
echo "publicIP=${publicIP}" >> $GITHUB_ENV
- name: Output installation log
run: |
sudo apt-get install -y sshpass
timeout 1m sh -c 'until nc -zv $0 $1; do echo "nc rc: $?"; sleep 5; done' ${publicIP} 22
echo "Output stdout:"
result=$(sshpass -p ${{ env.vmAdminPassword }} -v ssh -p 22 -o StrictHostKeyChecking=no -o TCPKeepAlive=yes -o ServerAliveCountMax=20 -o ServerAliveInterval=15 -o ConnectTimeout=100 -v -tt ${{ env.vmAdminId }}@${publicIP} 'echo "${{ env.vmAdminPassword }}" | sudo -S cat /var/lib/waagent/custom-script/download/0/stdout')
echo "$result"
- name: Update the packages
run: |
sudo apt-get install -y sshpass
timeout 1m sh -c 'until nc -zv $0 $1; do echo "nc rc: $?"; sleep 5; done' ${publicIP} 22
sshpass -p ${{ env.vmAdminPassword }} -v ssh -p 22 -o StrictHostKeyChecking=no -o TCPKeepAlive=yes -o ServerAliveCountMax=20 -o ServerAliveInterval=15 -o ConnectTimeout=100 -v -tt ${{ env.vmAdminId }}@${publicIP} 'echo "${{ env.vmAdminPassword }}" | sudo -S yum update -y'
- name: Harden the VM using OpenSCAP tool
run: |
az vm extension set --name CustomScript \
--extension-instance-name apply-oscap-rules \
--resource-group ${{ env.testResourceGroup }} --vm-name ${{ env.vmName }} \
--publisher Microsoft.Azure.Extensions --version 2.0 \
--settings "{\"fileUris\": [\"${{ env.utilitiesLocation }}oscap.sh\"]}" \
--protected-settings "{\"commandToExecute\":\"bash oscap.sh ${{ env.vmAdminId }}\"}"
- name: Copy openscap scanning reports
run: |
sudo apt-get install -y sshpass
timeout 1m sh -c 'until nc -zv $0 $1; do echo "nc rc: $?"; sleep 5; done' ${publicIP} 22
sshpass -p ${vmAdminPassword} scp ${vmAdminId}@${publicIP}:/home/${vmAdminId}/scan_report_* .
- name: Upload openscap scanning report before remidation
uses: actions/upload-artifact@v3
with:
name: scan-report-before
path: scan_report_before.html
- name: Upload openscap scanning report after remidation
uses: actions/upload-artifact@v3
with:
name: scan-report-after
path: scan_report_after.html
- name: Deprovision the VM
run: |
sudo apt-get install -y sshpass
timeout 1m sh -c 'until nc -zv $0 $1; do echo "nc rc: $?"; sleep 5; done' ${publicIP} 22
sshpass -p ${{ env.vmAdminPassword }} -v ssh -p 22 -o StrictHostKeyChecking=no -o TCPKeepAlive=yes -o ServerAliveCountMax=20 -o ServerAliveInterval=15 -o ConnectTimeout=100 -v -tt ${{ env.vmAdminId }}@${publicIP} 'echo "${{ env.vmAdminPassword }}" | sudo -S waagent -deprovision+user -force'
- name: Generate the image
run: |
az vm deallocate --resource-group ${{ env.testResourceGroup }} --name ${{ env.vmName }}
az vm generalize --resource-group ${{ env.testResourceGroup }} --name ${{ env.vmName }}
az image create --resource-group ${{ env.testResourceGroup }} --name ${{ env.vmName }} --source ${{ env.vmName }}
- name: Verify the image
run: |
imageResourceId=$(az image show --name ${{ env.vmName }} --resource-group ${{ env.testResourceGroup }} --query id -o tsv)
echo "imageResourceId=${imageResourceId}" >> $GITHUB_ENV
# Deploy VMs using different IBMids and verify installation
vmGroups=( ${{ env.testResourceGroup }}-entitled ${{ env.testResourceGroup }}-unentitled ${{ env.testResourceGroup }}-evaluation )
deploymentModes=( Entitled Unentitled Evaluation )
ibmUserIds=( ${{ env.entitledIbmUserId }} ${{ env.unEntitledIbmUserId }} "" )
ibmUserPwds=( ${{ env.entitledIbmPassword }} ${{ env.unEntitledIbmPassword }} "" )
for (( i=0; i<${#vmGroups[@]}; i++ )); do
rgName=${vmGroups[$i]}
az group create -n $rgName -l ${{ env.location }}
az deployment group create --resource-group $rgName --name testDeployment \
--template-file ${{ env.repoName }}/utilities/verifyTemplate.json \
--parameters deploymentMode=${deploymentModes[$i]} ibmUserId=${ibmUserIds[$i]} ibmUserPwd=${ibmUserPwds[$i]} \
imageResourceId=$imageResourceId vmAdminId=${{ env.vmAdminId }} vmAdminPwd=${{ env.vmAdminPassword }} \
scriptLocation=${{ env.scriptLocation }}
#az group delete -n $rgName --yes --no-wait
done
- name: Verify the image with twas-single integration-test pipeline
if: false
run: |
# Trigger the workflow run
requestData={\"ref\":\"main\",\"inputs\":{\"databaseType\":\"none\",\"imageResourceId\":\"${imageResourceId}\"}}
curl --verbose -L -X POST \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer ${accessToken}"\
-H "X-GitHub-Api-Version: 2022-11-28" \
https://api.github.com/repos/${{ env.twasSingleRepo }}/actions/workflows/integration-test.yaml/dispatches \
-d $(echo $requestData | jq -c)
# Wait until the workflow run starts
idAndStatus=$(curl -L \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer ${accessToken}"\
-H "X-GitHub-Api-Version: 2022-11-28" \
https://api.github.com/repos/${{ env.twasSingleRepo }}/actions/workflows/integration-test.yaml/runs \
| jq -r '.workflow_runs[0] | {id, status}')
status=$(echo $idAndStatus | jq -r '.status')
while [[ $status != queued ]]
do
echo "Wait until the workflow run starts..."
idAndStatus=$(curl -L \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer ${accessToken}"\
-H "X-GitHub-Api-Version: 2022-11-28" \
https://api.github.com/repos/${{ env.twasSingleRepo }}/actions/workflows/integration-test.yaml/runs \
| jq -r '.workflow_runs[0] | {id, status}')
status=$(echo $idAndStatus | jq -r '.status')
echo "The latest workflow run is ${idAndStatus}"
done
workflowRunId=$(echo $idAndStatus | jq '.id')
# Wait until the workflow run completes
status=$(curl -L \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer ${accessToken}"\
-H "X-GitHub-Api-Version: 2022-11-28" \
https://api.github.com/repos/${{ env.twasSingleRepo }}/actions/runs/$workflowRunId \
| jq -r '.status')
while [[ $status != completed ]]
do
sleep 60
echo "Wait until the workflow run ${workflowRunId} completes..."
status=$(curl -L \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer ${accessToken}"\
-H "X-GitHub-Api-Version: 2022-11-28" \
https://api.github.com/repos/${{ env.twasSingleRepo }}/actions/runs/$workflowRunId \
| jq -r '.status')
echo "Workflow run ${workflowRunId} is ${status}"
done
# Verify the workflow run result
jobs=$(curl -L \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer ${accessToken}"\
-H "X-GitHub-Api-Version: 2022-11-28" \
https://api.github.com/repos/${{ env.twasSingleRepo }}/actions/runs/$workflowRunId/jobs)
successIntegTestJobCnt=$(echo $jobs | jq 'select(.jobs != null) | .jobs | map(select(.name=="integration-test" and .conclusion=="success")) | length')
if [[ $successIntegTestJobCnt != 1 ]]; then
echo "twas-single integration test workflow run ${workflowRunId} failed."
exit 1
else
echo "twas-single integration test workflow run ${workflowRunId} succeeded."
fi
- name: Clean up all resources except for vhd storage account
uses: azure/CLI@v1
with:
azcliversion: ${{ env.azCliVersion }}
inlineScript: |
#az image delete --ids \
# $(az image show -g ${{ env.testResourceGroup }} -n ${{ env.vmName }} --query id -o tsv)
az vm delete --yes --ids \
$(az vm show -g ${{ env.testResourceGroup }} -n ${{ env.vmName }} --query id -o tsv)
az network nic delete --ids \
$(az network nic list -g ${{ env.testResourceGroup }} --query [0].id -o tsv)
az network vnet delete --ids \
$(az network vnet list -g ${{ env.testResourceGroup }} --query [0].id -o tsv)
az network public-ip delete --ids \
$(az network public-ip list -g ${{ env.testResourceGroup }} --query [0].id -o tsv)
az network nsg delete --ids \
$(az network nsg list -g ${{ env.testResourceGroup }} --query [0].id -o tsv)
- name: Generate SAS url
id: sas_url
run: |
# Get a minus-24-hour date and a plus-30-day date for the SAS token
minus24HoursUtc=$(date -u --date "$dte -24 hour" +%Y-%m-%dT%H:%MZ)
plus30DaysUtc=$(date -u --date "$dte 30 day" +%Y-%m-%dT%H:%MZ)
vhdStorageAccountAccessKey=$(az storage account keys list --account-name ${{ env.vhdStorageAccountName }} --query "[?keyName=='key1'].value" -o tsv)
sasToken=$(az storage container generate-sas \
--connection-string "DefaultEndpointsProtocol=https;AccountName=${{ env.vhdStorageAccountName }};AccountKey=${vhdStorageAccountAccessKey};EndpointSuffix=core.windows.net" \
--name vhds --permissions rl --start "${minus24HoursUtc}" --expiry "${plus30DaysUtc}" -o tsv)
blobStorageEndpoint=$( az storage account show -n ${{ env.vhdStorageAccountName }} -g ${{ env.testResourceGroup }} -o json | jq -r '.primaryEndpoints.blob' )
osDiskSasUrl=${blobStorageEndpoint}vhds/${{ env.vmName }}.vhd?$sasToken
dataDiskSasUrl=${blobStorageEndpoint}vhds/${{ env.vmName }}datadisk1.vhd?$sasToken
echo "osDiskSasUrl: ${osDiskSasUrl}, dataDiskSasUrl: ${dataDiskSasUrl}" > sas-url-twasbase.txt
echo "##[set-output name=osDiskSasUrl;]${osDiskSasUrl}"
echo "##[set-output name=dataDiskSasUrl;]${dataDiskSasUrl}"
- name: Upload SAS url
uses: actions/upload-artifact@v3
with:
name: sasurl-twasbase
path: sas-url-twasbase.txt
- name: Update offer sas url and version
if: false
uses: microsoft/[email protected]
with:
offerId: ${{ env.offerId }}
planId: ${{ env.planId }}
clientId: ${{ env.clientId }}
tenantId: ${{ env.tenantId }}
secretValue: ${{ env.secretValue }}
imageType: ${{ env.imageType }}
imageVersionNumber: ${{ steps.setup-env-variables-based-on-dispatch-event.outputs.imageVersionNumber }}
offerType: ${{ env.offerType }}
operatingSystemFamily: ${{ env.operatingSystemFamily }}
operatingSystemType: ${{ env.operatingSystemType }}
osDiskSasUrl: ${{steps.sas_url.outputs.osDiskSasUrl}}
dataDiskSasUrl: ${{steps.sas_url.outputs.dataDiskSasUrl}}
verbose: "false"
summary:
needs: build
runs-on: ubuntu-latest
steps:
- name: Output inputs from workflow_dispatch
run: echo "${{ toJSON(github.event.inputs) }}"
- name: Output client_payload from repository_dispatch
run: echo "${{ toJSON(github.event.client_payload) }}"