Skip to content

Commit

Permalink
tmep change to workflow
Browse files Browse the repository at this point in the history
  • Loading branch information
@tomkoren21
tomkoren21 committed Aug 9, 2024
1 parent 37d57e9 commit 709e73e
Showing 1 changed file with 153 additions and 151 deletions.
304 changes: 153 additions & 151 deletions .github/workflows/end2end.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,166 +17,168 @@ jobs:
update-leap-model-parser:
runs-on: ubuntu-latest
steps:
- name: Check branch name
id: check_branch
run: |
if [ "${{ github.event.inputs.branch }}" = "master" ]; then
echo "::error::Manual runs not allowed from master branch."
exit 1
fi
- name: Extract branch name
id: extract_branch
run: echo "::set-output name=branch_name::$(echo ${GITHUB_REF#refs/heads/})"
- name: Check out repository
uses: actions/checkout@v2
- name: Get package version from pyproject.toml
id: package_version
run: |
PACKAGE_VERSION=$(awk -F' = ' '$1 == "version" {gsub(/"/, "", $2); print $2}' pyproject.toml | head -n 1)
echo "PACKAGE_VERSION=$PACKAGE_VERSION" >> $GITHUB_ENV
echo "$PACKAGE_VERSION"
- name: Set up python
id: setup-python
uses: actions/setup-python@v2
with:
python-version: 3.8.12
- name: Set Poetry Lock Hash
run: |
echo "POETRY_HASH=$(sha256sum poetry.lock | cut -d ' ' -f 1)" >> $GITHUB_ENV
- name: Install Poetry
uses: snok/install-poetry@v1
with:
version: 1.8.2
virtualenvs-create: true
virtualenvs-in-project: true
installer-parallel: true
- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache/restore@v4
with:
path: .venv
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ env.POETRY_HASH }}
- name: Install dependencies
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
run: poetry install --no-interaction --no-root
- name: Save cache
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
uses: actions/cache/save@v4
with:
path: .venv
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ env.POETRY_HASH }}
- name: Check if version exists on PyPI
id: check_version
run: |
URL="https://pypi.org/pypi/onnx2kerastl/$PACKAGE_VERSION/json"
echo "URL: $URL"
RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" "$URL")
if [ $RESPONSE -eq 200 ]; then
echo "Version $PACKAGE_VERSION already exists on PyPI."
echo "IS_O2K_PUBLISHED=true" >> $GITHUB_ENV
else
echo "Version $PACKAGE_VERSION does not exist on PyPI."
echo "IS_O2K_PUBLISHED=false" >> $GITHUB_ENV
fi
- name: Build and publish O2K package
if: success() # Only run this step if the previous steps were successful
run: |
if [ "$IS_O2K_PUBLISHED" != "true" ]; then
poetry build
poetry publish -u __token__ -p ${{ secrets.PYPI_O2K }}
echo "Published O2K version $PACKAGE_VERSION"
else
echo "Skipping publishing as version already exists on PyPI."
fi
- name: checkout leap model parser
uses: actions/checkout@v4
with:
token: ${{ secrets.TENSORLEAP_OPS_GITHUB_TOKEN }}
repository: tensorleap/leap-model-parser
path: leap-model-parser
- name: Configure AWS Credentials
uses: aws-actions/[email protected]
with:
role-to-assume: arn:aws:iam::898022457080:role/TerraformAdmin
aws-region: us-east-1
- name: Configure GIT credentials
run: |
git config --global user.email [email protected]
git config --global user.name github-actions
- name: leap-model-parser test and push
working-directory: leap-model-parser
id: parser_test
env:
PYTHONPATH: ${{ env.PYTHONPATH }}:${{ github.workspace }}/leap-model-parser
run: | #fetch or create branch, update-o2k-version
echo "Project version is $PACKAGE_VERSION"
BRANCH="${{ steps.extract_branch.outputs.branch_name }}-from-o2k"
echo "BRANCH=$BRANCH" >> $GITHUB_ENV
if git ls-remote --exit-code --heads origin "$BRANCH" >/dev/null 2>&1; then
git fetch origin "$BRANCH"
git checkout "$BRANCH"
else
git checkout -b "$BRANCH"
fi
# add packages with retry
clear_cache() {
echo "Clearing Poetry cache..."
poetry cache clear . --all --no-interaction
}
add_packages() {
poetry add onnx2kerastl==$PACKAGE_VERSION
}
max_retries=3
attempt=0
# Try to add packages
while [ $attempt -lt $max_retries ]; do
echo "Attempt $(($attempt + 1)) to add the package..."
# - name: Check branch name
# id: check_branch
# run: |
# if [ "${{ github.event.inputs.branch }}" = "master" ]; then
# echo "::error::Manual runs not allowed from master branch."
# exit 1
# fi
# - name: Extract branch name
# id: extract_branch
# run: echo "::set-output name=branch_name::$(echo ${GITHUB_REF#refs/heads/})"
# - name: Check out repository
# uses: actions/checkout@v2
# - name: Get package version from pyproject.toml
# id: package_version
# run: |
# PACKAGE_VERSION=$(awk -F' = ' '$1 == "version" {gsub(/"/, "", $2); print $2}' pyproject.toml | head -n 1)
# echo "PACKAGE_VERSION=$PACKAGE_VERSION" >> $GITHUB_ENV
# echo "$PACKAGE_VERSION"
# - name: Set up python
# id: setup-python
# uses: actions/setup-python@v2
# with:
# python-version: 3.8.12
# - name: Set Poetry Lock Hash
# run: |
# echo "POETRY_HASH=$(sha256sum poetry.lock | cut -d ' ' -f 1)" >> $GITHUB_ENV
# - name: Install Poetry
# uses: snok/install-poetry@v1
# with:
# version: 1.8.2
# virtualenvs-create: true
# virtualenvs-in-project: true
# installer-parallel: true
# - name: Load cached venv
# id: cached-poetry-dependencies
# uses: actions/cache/restore@v4
# with:
# path: .venv
# key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ env.POETRY_HASH }}
# - name: Install dependencies
# if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
# run: poetry install --no-interaction --no-root
# - name: Save cache
# if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
# uses: actions/cache/save@v4
# with:
# path: .venv
# key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ env.POETRY_HASH }}
# - name: Check if version exists on PyPI
# id: check_version
# run: |
# URL="https://pypi.org/pypi/onnx2kerastl/$PACKAGE_VERSION/json"
# echo "URL: $URL"
# RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" "$URL")
# if [ $RESPONSE -eq 200 ]; then
# echo "Version $PACKAGE_VERSION already exists on PyPI."
# echo "IS_O2K_PUBLISHED=true" >> $GITHUB_ENV
# else
# echo "Version $PACKAGE_VERSION does not exist on PyPI."
# echo "IS_O2K_PUBLISHED=false" >> $GITHUB_ENV
# fi
# - name: Build and publish O2K package
# if: success() # Only run this step if the previous steps were successful
# run: |
# if [ "$IS_O2K_PUBLISHED" != "true" ]; then
# poetry build
# poetry publish -u __token__ -p ${{ secrets.PYPI_O2K }}
# echo "Published O2K version $PACKAGE_VERSION"
# else
# echo "Skipping publishing as version already exists on PyPI."
# fi
# - name: checkout leap model parser
# uses: actions/checkout@v4
# with:
# token: ${{ secrets.TENSORLEAP_OPS_GITHUB_TOKEN }}
# repository: tensorleap/leap-model-parser
# path: leap-model-parser
# - name: Configure AWS Credentials
# uses: aws-actions/[email protected]
# with:
# role-to-assume: arn:aws:iam::898022457080:role/TerraformAdmin
# aws-region: us-east-1
# - name: Configure GIT credentials
# run: |
# git config --global user.email [email protected]
# git config --global user.name github-actions
# - name: leap-model-parser test and push
# working-directory: leap-model-parser
# id: parser_test
# env:
# PYTHONPATH: ${{ env.PYTHONPATH }}:${{ github.workspace }}/leap-model-parser
# run: | #fetch or create branch, update-o2k-version
# echo "Project version is $PACKAGE_VERSION"
# BRANCH="${{ steps.extract_branch.outputs.branch_name }}-from-o2k"
# echo "BRANCH=$BRANCH" >> $GITHUB_ENV
# if git ls-remote --exit-code --heads origin "$BRANCH" >/dev/null 2>&1; then
# git fetch origin "$BRANCH"
# git checkout "$BRANCH"
# else
# git checkout -b "$BRANCH"
# fi
# # add packages with retry
# clear_cache() {
# echo "Clearing Poetry cache..."
# poetry cache clear . --all --no-interaction
# }
# add_packages() {
# poetry add onnx2kerastl==$PACKAGE_VERSION
# }
# max_retries=3
# attempt=0
# # Try to add packages
# while [ $attempt -lt $max_retries ]; do
# echo "Attempt $(($attempt + 1)) to add the package..."

# Attempt to add the package
if add_packages; then
echo "Package added successfully."
break
else
echo "Package add failed."
# # Attempt to add the package
# if add_packages; then
# echo "Package added successfully."
# break
# else
# echo "Package add failed."

# Clear the cache before retrying
clear_cache
# # Clear the cache before retrying
# clear_cache

# Increment the attempt counter
attempt=$((attempt + 1))
# # Increment the attempt counter
# attempt=$((attempt + 1))

# If this was the last attempt, exit with failure
if [ $attempt -ge $max_retries ]; then
echo "Failed to add the package after $max_retries attempts."
exit 1
fi
fi
done
source .venv/bin/activate
cd tests
if pytest -s test_branch_model.py --cloud_dir ${{ github.event.inputs.cloud_dir }} --model_name ${{ github.event.inputs.model_name }}; then
TEST_CONCLUSION=success
echo "Tests passed, Pushing Branch if needed"
cd ..
git add -u
if git status --porcelain | grep -q '^M'; then
echo "Files are modified - updating branch"
git commit -m "update O2K --model_name ${{ github.event.inputs.model_name }} --cloud_dir ${{ github.event.inputs.cloud_dir }} --sha $GITHUB_SHA"
git push origin "$BRANCH"
else
echo "No changes were made to leap-model-parser - not pushing the branch"
fi
else
TEST_CONCLUSION=failure
echo "Tests Failed, check pytest output"
fi
echo "TEST_CONCLUSION=$TEST_CONCLUSION" >> $GITHUB_ENV
# # If this was the last attempt, exit with failure
# if [ $attempt -ge $max_retries ]; then
# echo "Failed to add the package after $max_retries attempts."
# exit 1
# fi
# fi
# done
# source .venv/bin/activate
# cd tests
# if pytest -s test_branch_model.py --cloud_dir ${{ github.event.inputs.cloud_dir }} --model_name ${{ github.event.inputs.model_name }}; then
# TEST_CONCLUSION=success
# echo "Tests passed, Pushing Branch if needed"
# cd ..
# git add -u
# if git status --porcelain | grep -q '^M'; then
# echo "Files are modified - updating branch"
# git commit -m "update O2K --model_name ${{ github.event.inputs.model_name }} --cloud_dir ${{ github.event.inputs.cloud_dir }} --sha $GITHUB_SHA"
# git push origin "$BRANCH"
# else
# echo "No changes were made to leap-model-parser - not pushing the branch"
# fi
# else
# TEST_CONCLUSION=failure
# echo "Tests Failed, check pytest output"
# fi
# echo "TEST_CONCLUSION=$TEST_CONCLUSION" >> $GITHUB_ENV
- name: add commit status
id: commit_status
run: |
TEST_CONCLUSION=success
echo "TEST_CONCLUSION=$TEST_CONCLUSION" >> $GITHUB_ENV
curl -X POST \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer ${{ secrets.TENSORLEAP_OPS_GITHUB_TOKEN }}" \
-H "X-GitHub-Api-Version: 2022-11-28" \
https://api.github.com/repos/tensorleap/onnx2keras/statuses/$GITHUB_SHA \
-d '{"state":"${TEST_CONCLUSION}","target_url":"https://github.com/${GITHUB_REPOSITORY}/runs/${GITHUB_RUN_ID}/jobs/${GITHUB_JOB}","description":"Leap model parser passed dynamic test","context":"end2end/parser-dynamic-test"}'
-d '{"state":"'${TEST_CONCLUSION}'","target_url":"https://github.com/'${GITHUB_REPOSITORY}'/actions/runs/'${GITHUB_RUN_ID}'","description":"Leap model parser passed dynamic test","context":"end2end/parser-dynamic-test"}'

0 comments on commit 709e73e

Please sign in to comment.