-
Notifications
You must be signed in to change notification settings - Fork 3.7k
148 lines (127 loc) · 6.24 KB
/
indexer-processor-testing.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
name: Trigger Processor Tests on JSON Change
on:
workflow_dispatch:
pull_request: # Trigger on PR-level events
branches:
- main
# the required permissions to request the ID token
permissions:
id-token: write # This is required for GCP authentication
contents: read # Ensure the workflow has access to repository contents
jobs:
dispatch_event:
runs-on: runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,run-id=${{ github.run_id }}
steps:
- name: Checkout the repository
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.ref }}
- name: Set up Rust
uses: aptos-labs/aptos-core/.github/actions/rust-setup@main
with:
GIT_CREDENTIALS: ${{ secrets.GIT_CREDENTIALS }}
# Install necessary system dependencies
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install build-essential libssl-dev pkg-config
# Ensure Rust is updated
- name: Update Rust toolchain
run: rustup update
- name: Run CLI to Generate JSON Files
run: |
cd ecosystem/indexer-grpc/indexer-transaction-generator
cargo run -- --testing-folder ./example_tests --output-folder ../indexer-test-transactions/new_json_transactions
- name: Install jq
run: sudo apt-get install jq # Ensure jq is installed for JSON processing
# TODO: improve this step to be easily maintainable and extensible
# Prepare Original and New JSON Files
- name: Prepare and Clean JSON Files
run: |
cd ecosystem/indexer-grpc/indexer-test-transactions
for folder in json_transactions/scripted_transactions new_json_transactions/scripted_transactions; do
for file in $folder/*.json; do
echo "Processing $file..."
base_file=$(basename "$file")
jq 'del(.timestamp,
.version,
.info.hash,
.info.stateChangeHash,
.info.accumulatorRootHash,
.info.changes[].writeResource.stateKeyHash,
.info.changes[].writeResource.type.address,
.info.changes[].writeResource.address,
.info.changes[].writeTableItem.stateKeyHash,
.info.changes[].writeTableItem.data.key,
.info.changes[].writeTableItem.data.value,
.epoch,
.blockHeight,
.sizeInfo,
.user.request.sender,
.user.request.expirationTimestampSecs.seconds,
.user.request.signature.ed25519.publicKey,
.user.request.signature.ed25519.signature)
| (.info.changes[].writeResource.data |=
if type == "string" then
(fromjson
| del(.authentication_key)
| walk(if type == "object" and has("addr") then del(.addr) else . end)
| tostring)
else . end)' "$file" > "$folder/cleaned_$base_file"
done
done
- name: Compare JSON Files Across Multiple Folders
id: diff_check
run: |
. scripts/indexer_test_txns_compare_and_diff.sh
- name: Handle New Files and Differences
run: |
echo "Checking outputs from diff_check step..."
echo "New file found: ${{ steps.diff_check.outputs.new_file_found }}"
echo "Diff found: ${{ steps.diff_check.outputs.diff_found }}"
if [ "${{ steps.diff_check.outputs.new_file_found }}" == "true" ]; then
echo "New JSON files detected:"
echo "${{ steps.diff_check.outputs.new_files }}" # Print all new files with paths
exit 0 # Fail the workflow to enforce manual review
elif [ "${{ steps.diff_check.outputs.diff_found }}" == "true" ]; then
echo "Differences detected. Proceeding with dispatch event."
echo "Modified files:"
echo "${{ steps.diff_check.outputs.modified_files }}" # Print modified files with paths
else
echo "No differences or new files detected."
exit 0 # Proceed successfully only if no new files or differences are found
fi
- id: auth
if: steps.diff_check.outputs.diff_found == 'true' && steps.diff_check.outputs.new_file_found == 'false'
uses: "google-github-actions/auth@v2"
with:
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}
- name: Log active service account email
if: steps.diff_check.outputs.diff_found == 'true' && steps.diff_check.outputs.new_file_found == 'false'
run: |
gcloud auth list --filter=status:ACTIVE --format="value(account)"
- id: 'secrets'
if: steps.diff_check.outputs.diff_found == 'true' && steps.diff_check.outputs.new_file_found == 'false'
uses: 'google-github-actions/get-secretmanager-secrets@v2'
with:
secrets: |-
token:aptos-ci/github-actions-repository-dispatch
# Conditionally Dispatch Event to Processor Repo if Differences Found
- name: Dispatch Event to Processor Repo
if: steps.diff_check.outputs.diff_found == 'true' && steps.diff_check.outputs.new_file_found == 'false'
uses: peter-evans/[email protected]
with:
TOKEN: '${{ steps.secrets.outputs.token }}'
repository: 'aptos-labs/aptos-indexer-processors'
event-type: 'test-txn-json-change-detected'
client-payload: '{"commit_hash": "${{ github.sha }}"}'
# Poll Processor Repo for Workflow Run Status and Memorize Run ID to check the job status
- name: Poll for Workflow Run and Wait for Job Completion
if: steps.diff_check.outputs.diff_found == 'true' && steps.diff_check.outputs.new_file_found == 'false'
id: poll_status
run: |
. scripts/indexer_processor_tests_status_poll.sh
env:
GITHUB_TOKEN: ${{ steps.secrets.outputs.token }} # Pass the correct GitHub token
GITHUB_SHA: ${{ github.sha }}