Bump werkzeug from 0.16.1 to 3.0.1 #49
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: benchmarks | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| types: [labeled] | |
| jobs: | |
| deploy-cloud-runner: | |
| if: ${{ (github.event.action == 'labeled' && github.event.label.name == 'benchmark') || github.event.action == 'workflow_dispatch' }} | |
| runs-on: [ubuntu-latest] | |
| container: docker://dvcorg/cml | |
| steps: | |
| - name: deploy | |
| env: | |
| repo_token: ${{ secrets.HAYSTACK_BOT_TOKEN }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CI_ACCESS_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CI_SECRET_ACCESS_KEY }} | |
| VPC: ${{ secrets.AWS_CI_VPC }} | |
| run: | | |
| echo "Deploying..." | |
| RUNNER_LABELS="cml,aws" | |
| RUNNER_REPO="https://github.com/${GITHUB_REPOSITORY}" | |
| MACHINE="cml$(date +%s)" | |
| docker-machine create \ | |
| --driver amazonec2 \ | |
| --amazonec2-instance-type p3.8xlarge \ | |
| --amazonec2-vpc-id $VPC \ | |
| --amazonec2-region us-east-1 \ | |
| --amazonec2-zone c \ | |
| --amazonec2-ssh-user ubuntu \ | |
| --amazonec2-ami ami-06a25ee8966373068 \ | |
| --amazonec2-root-size 150 \ | |
| $MACHINE | |
| eval "$(docker-machine env --shell sh $MACHINE)" | |
| ( | |
| docker-machine ssh $MACHINE "sudo mkdir -p \ | |
| /docker_machine && \ | |
| sudo chmod 777 /docker_machine" && \ | |
| docker-machine scp -r -q ~/.docker/machine/ \ | |
| $MACHINE:/docker_machine && \ | |
| docker run --name runner -d \ | |
| --gpus all \ | |
| -v /docker_machine/machine:/root/.docker/machine \ | |
| --net host \ | |
| --ipc host \ | |
| -e DOCKER_MACHINE=$MACHINE \ | |
| -e repo_token=$repo_token \ | |
| -e RUNNER_LABELS=$RUNNER_LABELS \ | |
| -e RUNNER_REPO=$RUNNER_REPO \ | |
| -e RUNNER_IDLE_TIMEOUT=120 \ | |
| dvcorg/cml-py3:latest && \ | |
| sleep 20 && echo "Deployed $MACHINE" | |
| ) || (echo "Shut down machine" && docker-machine rm -y -f $MACHINE && exit 1) | |
| run-benchmark: | |
| if: ${{ (github.event.action == 'labeled' && github.event.label.name == 'benchmark') || github.event.action == 'workflow_dispatch' }} | |
| needs: deploy-cloud-runner | |
| runs-on: [self-hosted,cml] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: cml_run | |
| env: | |
| repo_token: ${{ secrets.HAYSTACK_BOT_TOKEN }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CI_ACCESS_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CI_SECRET_ACCESS_KEY }} | |
| run: | | |
| apt-get update -y | |
| apt-get install python3-dev -y | |
| pip install -r requirements.txt | |
| pip install . | |
| cd test/benchmarks && python question_answering_accuracy.py | |
| echo -en "## Benchmarks: QA Accuracy\n" >> accuracy_report.md | |
| cat results_accuracy.md >> accuracy_report.md | |
| cml-send-comment accuracy_report.md | |
| python question_answering_components.py | |
| echo -en "## Benchmarks: QA per component\n" >> components_report.md | |
| cat results_per_component.md >> components_report.md | |
| cml-send-comment components_report.md |