Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dev kmr #134

Closed
wants to merge 3 commits into from
Closed

Dev kmr #134

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9af1b45
update and fix a bug in the check-controller-build action
k-rister Aug 14, 2024
8ab0845
DEBUG - DO_NOT_MERGE
k-rister Aug 14, 2024
0d9d0d0
fixup
k-rister Aug 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 19 additions & 5 deletions .github/actions/check-controller-build/check-controller-build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,24 +21,31 @@ function error() {
# because the code being examined is not from a submitted PR but
# the committed upstream repository -- meaning no controller build
# required
diff_cmd_validate="git log HEAD^1"
diff_cmd="git diff --name-only HEAD^1 HEAD"
available_history="git log --oneline origin/master...HEAD"
diff_cmd="git diff --name-only origin/master...HEAD"

build_controller="no"

echo "Crucible workshop change analysis:"
if pushd ${crucible_directory}; then
if ${diff_cmd_validate} > /dev/null 2>&1; then
available_commits=$(${available_history} | wc -l)
echo "Available commits: ${available_commits}"

if [ ${available_commits} -gt 1 ]; then
# history available

echo "Available commits:"
${available_history}
echo

echo "Files changed:"
${diff_cmd}
if [ $? != 0 ]; then
error "could not obtain git-diff output"
fi
echo

workshop_files_changed=$(${diff_cmd} | grep "^workshop/" | wc -l)
workshop_files_changed=$(${diff_cmd} | grep "^workshop/" | grep -v ".md$" | wc -l)
echo "workshop_files_changed=${workshop_files_changed}"
echo

Expand All @@ -59,9 +66,16 @@ fi

echo "Workshop change analysis:"
if pushd ${workshop_directory}; then
if ${diff_cmd_validate} > /dev/null 2>&1; then
available_commits=$(${available_history} | wc -l)
echo "Available commits: ${available_commits}"

if [ ${available_commits} -gt 1 ]; then
# history available

echo "Available commits:"
${available_history}
echo

echo "Files changed:"
${diff_cmd}
if [ $? != 0 ]; then
Expand Down
16 changes: 10 additions & 6 deletions .github/actions/check-controller-build/validate-inputs.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,16 @@ else
if ! pushd ${crucible_directory}; then
error "Could not pushd to the crucible directory '${crucible_directory}'"
else
echo "Contents of crucible workshop directory:"
if [ -f crucible-install.sh -a -d workshop -a -f workshop/build-controller.sh -a -f workshop/controller-workshop.json ]; then
ls -l workshop/
echo "Contents of crucible directory:"
ls -la
if [ ! -f crucible-install.sh -o ! -d workshop ]; then
error "Could not find required crucible directory contents"
else
ls -l
error "Could not find the required crucible workshop directory contents"
echo "Contents of crucible workshop directory:"
ls -la workshop/
if [ ! -f workshop/build-controller.sh -o ! -f workshop/controller-workshop.json ]; then
error "Could not find required crucible workshop contents"
fi
fi

popd
Expand All @@ -52,7 +56,7 @@ else
error "Could not pushd to the workshop directory '${workshop_directory}'"
else
echo "Contents of the workshop directory:"
ls -l
ls -la
if [ ! -f workshop.pl -o ! -f schema.json ]; then
error "Could not find required workshop directory contents"
fi
Expand Down
168 changes: 2 additions & 166 deletions .github/workflows/core-crucible-ci.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ jobs:
repository: perftool-incubator/${{ inputs.ci_target }}
ref: ${{ inputs.ci_target_branch }}
path: crucible
fetch-depth: 2
fetch-depth: 0

- name: checkout workshop default
if: ${{ inputs.ci_target != 'workshop' }}
Expand All @@ -87,7 +87,7 @@ jobs:
repository: perftool-incubator/${{ inputs.ci_target }}
ref: ${{ inputs.ci_target_branch }}
path: workshop
fetch-depth: 2
fetch-depth: 0

- name: checkout crucible-ci default
if: ${{ inputs.ci_target != 'crucible-ci' && inputs.crucible_ci_test != 'yes' }}
Expand Down Expand Up @@ -247,167 +247,3 @@ jobs:
- name: no build required
if: ${{ needs.gen-params.outputs.build_controller == 'no' }}
run: echo "no controller build required"

github-runners:
runs-on: ubuntu-latest
timeout-minutes: 90
needs:
- gen-params
- display-params
- build-controller
strategy:
fail-fast: false
matrix:
scenario: ${{ fromJSON(needs.gen-params.outputs.github_hosted_scenarios) }}
userenv: ${{ fromJSON(needs.gen-params.outputs.userenvs) }}
ci_controller: [ "${{ needs.gen-params.outputs.build_controller }}" ]
steps:
- name: Matrix Parameters => (${{ matrix.scenario.enabled }}, ${{ matrix.scenario.endpoint }}, ${{ matrix.scenario.benchmark }}, ${{ matrix.userenv }}, ${{ matrix.ci_controller }})
run: |
echo "enabled=${{ matrix.scenario.enabled }}"
echo "endpoint=${{ matrix.scenario.endpoint }}"
echo "benchmark=${{ matrix.scenario.benchmark }}"
echo "userenv=${{ matrix.userenv }}"
echo "ci_controller=${{ matrix.ci_controller }}"

- name: checkout ${{ needs.gen-params.outputs.ci_target_dir }} ci_target
uses: actions/checkout@v4
with:
repository: perftool-incubator/${{ needs.gen-params.outputs.ci_target_dir }}
ref: ${{ inputs.ci_target_branch }}
path: ${{ needs.gen-params.outputs.ci_target_dir }}

- name: checkout crucible-ci default
if: ${{ inputs.ci_target != 'crucible-ci' && inputs.crucible_ci_test != 'yes' }}
uses: actions/checkout@v4
with:
repository: perftool-incubator/crucible-ci
ref: main
path: crucible-ci
- name: checkout crucible-ci crucible_ci_test
if: ${{ inputs.ci_target != 'crucible-ci' && inputs.crucible_ci_test == 'yes' }}
uses: actions/checkout@v4
with:
repository: perftool-incubator/crucible-ci
ref: ${{ inputs.crucible_ci_test_branch }}
path: crucible-ci

- name: import ci secret
env:
ENGINE_REGISTRY_AUTH_SECRET: ${{ secrets.registry_auth != '' && secrets.registry_auth || secrets.ci_registry_auth }}
if: ${{ env.ENGINE_REGISTRY_AUTH_SECRET != '' }}
run: sudo bash -c "echo \"$ENGINE_REGISTRY_AUTH_SECRET\" > /root/crucible-ci-engines-token.json"
- name: import production secret
env:
ENGINE_REGISTRY_AUTH_SECRET: ${{ secrets.production_registry_auth }}
if: ${{ env.ENGINE_REGISTRY_AUTH_SECRET != '' }}
run: sudo bash -c "echo \"$ENGINE_REGISTRY_AUTH_SECRET\" > /root/crucible-production-engines-token.json"

- name: import quay oath token
env:
QUAY_OAUTH_TOKEN: ${{ secrets.quay_oauth_token }}
if: ${{ env.QUAY_OAUTH_TOKEN != '' }}
run: sudo bash -c "echo \"$QUAY_OAUTH_TOKEN\" > /root/quay-oauth.token"

- name: run crucible-ci->integration-tests
if: ${{ matrix.scenario.enabled }}
uses: ./crucible-ci/.github/actions/integration-tests
with:
artifact_tag: "${{ inputs.ci_target }}__github-runners__${{ matrix.scenario.endpoint }}-${{ matrix.scenario.benchmark }}-${{ matrix.userenv }}"
ci_endpoint: "${{ matrix.scenario.endpoint }}"
scenarios: "${{ matrix.scenario.benchmark }}"
userenvs: "${{ matrix.userenv }}"
ci_target: "${{ inputs.ci_target }}"
ci_target_dir: ${{ github.workspace }}/${{ needs.gen-params.outputs.ci_target_dir }}
ci_controller: ${{ matrix.ci_controller }}
controller_tag: ${{ needs.gen-params.outputs.repo_name }}_${{ inputs.ci_target }}_${{ github.run_number }}
- name: skip crucible-ci->integration-tests
if: ${{ ! matrix.scenario.enabled }}
run: echo "crucible-ci->integration-tests not enabled"

self-hosted-runners:
runs-on: [ self-hosted, cpu-partitioning, remotehost, remotehosts ]
timeout-minutes: 90
needs:
- gen-params
- display-params
- build-controller
strategy:
fail-fast: false
matrix:
scenario: ${{ fromJSON(needs.gen-params.outputs.self_hosted_scenarios) }}
userenv: ${{ fromJSON(needs.gen-params.outputs.userenvs) }}
ci_controller: [ "${{ needs.gen-params.outputs.build_controller }}" ]
steps:
- name: Matrix Parameters => (${{ matrix.scenario.enabled }}, ${{ matrix.scenario.endpoint }}, ${{ matrix.scenario.benchmark }}, ${{ matrix.userenv }}, ${{ matrix.ci_controller }})
run: |
echo "enabled=${{ matrix.scenario.enabled }}"
echo "endpoint=${{ matrix.scenario.endpoint }}"
echo "benchmark=${{ matrix.scenario.benchmark }}"
echo "userenv=${{ matrix.userenv }}"
echo "ci_controller=${{ matrix.ci_controller }}"

- name: checkout ${{ needs.gen-params.outputs.ci_target_dir }} ci_target
uses: actions/checkout@v4
with:
repository: perftool-incubator/${{ needs.gen-params.outputs.ci_target_dir }}
ref: ${{ inputs.ci_target_branch }}
path: ${{ needs.gen-params.outputs.ci_target_dir }}

- name: checkout crucible-ci default
if: ${{ inputs.ci_target != 'crucible-ci' && inputs.crucible_ci_test != 'yes' }}
uses: actions/checkout@v4
with:
repository: perftool-incubator/crucible-ci
ref: main
path: crucible-ci
- name: checkout crucible-ci crucible_ci_test
if: ${{ inputs.ci_target != 'crucible-ci' && inputs.crucible_ci_test == 'yes' }}
uses: actions/checkout@v4
with:
repository: perftool-incubator/crucible-ci
ref: ${{ inputs.crucible_ci_test_branch }}
path: crucible-ci

- name: import ci secret
env:
ENGINE_REGISTRY_AUTH_SECRET: ${{ secrets.registry_auth != '' && secrets.registry_auth || secrets.ci_registry_auth }}
if: ${{ env.ENGINE_REGISTRY_AUTH_SECRET != '' }}
run: sudo bash -c "echo \"$ENGINE_REGISTRY_AUTH_SECRET\" > /root/crucible-ci-engines-token.json"
- name: import production secret
env:
ENGINE_REGISTRY_AUTH_SECRET: ${{ secrets.production_registry_auth }}
if: ${{ env.ENGINE_REGISTRY_AUTH_SECRET != '' }}
run: sudo bash -c "echo \"$ENGINE_REGISTRY_AUTH_SECRET\" > /root/crucible-production-engines-token.json"

- name: import quay oath token
env:
QUAY_OAUTH_TOKEN: ${{ secrets.quay_oauth_token }}
if: ${{ env.QUAY_OAUTH_TOKEN != '' }}
run: sudo bash -c "echo \"$QUAY_OAUTH_TOKEN\" > /root/quay-oauth.token"

- name: run crucible-ci->integration-tests
if: ${{ matrix.scenario.enabled }}
uses: ./crucible-ci/.github/actions/integration-tests
with:
artifact_tag: "${{ inputs.ci_target }}__self-hosted-runners__${{ matrix.scenario.endpoint }}-${{ matrix.scenario.benchmark }}-${{ matrix.userenv }}"
ci_endpoint: "${{ matrix.scenario.endpoint }}"
scenarios: "${{ matrix.scenario.benchmark }}"
userenvs: "${{ matrix.userenv }}"
ci_target: "${{ inputs.ci_target }}"
ci_target_dir: ${{ github.workspace }}/${{ needs.gen-params.outputs.ci_target_dir }}
ci_controller: ${{ matrix.ci_controller }}
controller_tag: ${{ needs.gen-params.outputs.repo_name }}_${{ inputs.ci_target }}_${{ github.run_number }}
- name: skip crucible-ci->integration-tests
if: ${{ ! matrix.scenario.enabled }}
run: echo "crucible-ci->integration-tests not enabled"

core-crucible-ci-complete:
runs-on: [ self-hosted, workflow-overhead ]
timeout-minutes: 10
needs:
- github-runners
- self-hosted-runners
steps:
- name: Confirm Success
run: echo "core-crucible-ci-complete"
44 changes: 0 additions & 44 deletions .github/workflows/test-benchmark-crucible-ci.yaml
View file
Open in desktop

This file was deleted.

42 changes: 0 additions & 42 deletions .github/workflows/test-tool-crucible-ci.yaml
View file
Open in desktop

This file was deleted.