From 873a5b5a6706794e18d6a7999a770f4bec89b1be Mon Sep 17 00:00:00 2001 From: Alexander Guschin <1aguschin@gmail.com> Date: Fri, 31 Mar 2023 14:31:56 +0600 Subject: [PATCH 1/4] remove annotate and move artifacts to dvc.yaml --- .../.github/workflows/deploy-model-with-mlem.yml | 6 ++++-- example-gto/generate.sh | 16 +++++++++++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/example-gto/code/mlem/.github/workflows/deploy-model-with-mlem.yml b/example-gto/code/mlem/.github/workflows/deploy-model-with-mlem.yml index 4264be1d..0e3a84c0 100644 --- a/example-gto/code/mlem/.github/workflows/deploy-model-with-mlem.yml +++ b/example-gto/code/mlem/.github/workflows/deploy-model-with-mlem.yml @@ -16,13 +16,15 @@ jobs: - uses: actions/checkout@v3 - name: "GTO: figure out what was registered/promoted and show the Registry state" id: gto - uses: iterative/gto-action@v1 + uses: iterative/gto-action@v2 + with: + download: true # we define the Job outputs here to let the next Job use them outputs: name: ${{ steps.gto.outputs.name }} stage: ${{ steps.gto.outputs.stage }} event: ${{ steps.gto.outputs.event }} - path: ${{ steps.gto.outputs.path }} + path: ${{ steps.gto.outputs.path }} # the path the model was `dvc get` to deploy-model: name: Deploy a MLEM model (act on assigning a new stage) needs: parse-git-tag diff --git a/example-gto/generate.sh b/example-gto/generate.sh index bb2e8e99..da957e09 100755 --- a/example-gto/generate.sh +++ b/example-gto/generate.sh @@ -91,9 +91,19 @@ git add models requirements.txt tick git commit -am "Create models" -gto annotate churn --type model --path models/churn.pkl --must-exist -gto annotate segment --type model --path s3://mycorp/proj-ml/segm-model-2022-04-15.pt -gto annotate cv-class --type model --path s3://mycorp/proj-ml/classif-v2.pt +cat >> dvc.yaml<< EOF +artifacts: + churn: + type: model + path: models/churn.pkl + segment: + type: model + path: s3://mycorp/proj-ml/segm-model-2022-04-15.pt + cv-class: + type: model + path: s3://mycorp/proj-ml/classif-v2.pt +EOF + git add artifacts.yaml tick git commit -m "Annotate models with GTO" From 5addb17f5918858aaca0560af0374aa515a916f1 Mon Sep 17 00:00:00 2001 From: Alexander Guschin <1aguschin@gmail.com> Date: Tue, 23 May 2023 18:46:20 +0300 Subject: [PATCH 2/4] adapting to gto-action@v2 --- example-gto/code/.github/workflows/gto-act-on-tags.yml | 2 +- example-gto/generate.sh | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/example-gto/code/.github/workflows/gto-act-on-tags.yml b/example-gto/code/.github/workflows/gto-act-on-tags.yml index a40392f5..aad969e9 100644 --- a/example-gto/code/.github/workflows/gto-act-on-tags.yml +++ b/example-gto/code/.github/workflows/gto-act-on-tags.yml @@ -12,7 +12,7 @@ jobs: - uses: actions/checkout@v3 - name: "GTO: figure out what was registered/promoted and show the Registry state" id: gto - uses: iterative/gto-action@v1 + uses: iterative/gto-action@v2 - uses: actions/setup-python@v2 - name: Install dependencies run: | diff --git a/example-gto/generate.sh b/example-gto/generate.sh index da957e09..84263458 100755 --- a/example-gto/generate.sh +++ b/example-gto/generate.sh @@ -44,7 +44,7 @@ source $BUILD_PATH/.venv/bin/activate TOTAL_TAGS=15 STEP_TIME=100000 -SLEEP_TIME=30 +SLEEP_TIME=90 BEGIN_TIME=$(($(date +%s) - (${TOTAL_TAGS} * ${STEP_TIME}))) export TAG_TIME=${BEGIN_TIME} export GIT_AUTHOR_DATE="${TAG_TIME} +0000" @@ -84,6 +84,9 @@ if $PUSH; then git ls-remote --tags origin | awk '/^(.*)(\s+)(.*[a-zA-Z0-9])$/ {print ":" $2}' | xargs git push origin fi +echo "Initialize DVC" +dvc init +git commit -m "Initialize DVC" echo "Create new models" mkdir models echo "1st version" > models/churn.pkl @@ -103,8 +106,8 @@ artifacts: type: model path: s3://mycorp/proj-ml/classif-v2.pt EOF +git add dvc.yaml -git add artifacts.yaml tick git commit -m "Annotate models with GTO" if $PUSH; then From 99b6bede26cbafdb56c5ad4c85bb50ea527cd654 Mon Sep 17 00:00:00 2001 From: Alexander Guschin <1aguschin@gmail.com> Date: Tue, 23 May 2023 18:49:05 +0300 Subject: [PATCH 3/4] Update example-gto/code/mlem/.github/workflows/deploy-model-with-mlem.yml --- .../code/mlem/.github/workflows/deploy-model-with-mlem.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example-gto/code/mlem/.github/workflows/deploy-model-with-mlem.yml b/example-gto/code/mlem/.github/workflows/deploy-model-with-mlem.yml index 0e3a84c0..44e8dc63 100644 --- a/example-gto/code/mlem/.github/workflows/deploy-model-with-mlem.yml +++ b/example-gto/code/mlem/.github/workflows/deploy-model-with-mlem.yml @@ -24,7 +24,7 @@ jobs: name: ${{ steps.gto.outputs.name }} stage: ${{ steps.gto.outputs.stage }} event: ${{ steps.gto.outputs.event }} - path: ${{ steps.gto.outputs.path }} # the path the model was `dvc get` to + path: ${{ steps.gto.outputs.path }} deploy-model: name: Deploy a MLEM model (act on assigning a new stage) needs: parse-git-tag From 5e83ee28c8be1dd25d3f27f943521efb450c7595 Mon Sep 17 00:00:00 2001 From: Alexander Guschin <1aguschin@gmail.com> Date: Tue, 23 May 2023 18:49:25 +0300 Subject: [PATCH 4/4] Update example-gto/code/mlem/.github/workflows/deploy-model-with-mlem.yml --- .../code/mlem/.github/workflows/deploy-model-with-mlem.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/example-gto/code/mlem/.github/workflows/deploy-model-with-mlem.yml b/example-gto/code/mlem/.github/workflows/deploy-model-with-mlem.yml index 44e8dc63..eee75741 100644 --- a/example-gto/code/mlem/.github/workflows/deploy-model-with-mlem.yml +++ b/example-gto/code/mlem/.github/workflows/deploy-model-with-mlem.yml @@ -17,8 +17,6 @@ jobs: - name: "GTO: figure out what was registered/promoted and show the Registry state" id: gto uses: iterative/gto-action@v2 - with: - download: true # we define the Job outputs here to let the next Job use them outputs: name: ${{ steps.gto.outputs.name }}