Remove Service Principal identities (#294)

* Use WMI for agro workflow pods Co-authored-by: Tom Augspurger <[email protected]>
microsoft · May 28, 2024 · 71bf445 · 71bf445
1 parent 9c242e6
commit 71bf445
Show file tree

Hide file tree

Showing 91 changed files with 318 additions and 1,661 deletions.
diff --git a/datasets/aster/dataset.yaml b/datasets/aster/dataset.yaml
@@ -9,11 +9,6 @@ code:
   src: ${{ local.path(./aster.py) }}
   requirements: ${{ local.path(./requirements.txt) }}
 
-environment:
-  AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-  AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-  AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
-
 collections:
   - id: aster-l1t
     template: ${{ local.path(./collection) }}

diff --git a/datasets/aster/update-geometries-ingest.yaml b/datasets/aster/update-geometries-ingest.yaml
@@ -17,10 +17,6 @@ jobs:
         options:
           insert_group_size: 5000
           insert_only: false
-      environment:
-        AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-        AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-        AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
       schema_version: 1.0.0
 schema_version: 1.0.0
 id: aster-update-geometries-ingest

diff --git a/datasets/aster/update-geometries.yaml b/datasets/aster/update-geometries.yaml
@@ -15,10 +15,6 @@ jobs:
         args:
           url: https://planetarycomputer.microsoft.com/api/stac/v1/collections/aster-l1t
           asset_key: geoparquet-items
-        environment:
-          AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-          AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-          AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
   create-chunks:
     foreach:
       items: ${{ jobs.get-partitions.tasks.get-partitions.output.assets }}
@@ -34,10 +30,6 @@ jobs:
           asset: ${{ item }}
           dst_uri: blob://astersa/aster-etl-data/chunks/update-geometries-2023-01-23-00
           chunk_size: 5000
-        environment:
-          AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-          AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-          AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
         tags:
           batch_pool_id: high_memory_pool
   update-items:
@@ -58,10 +50,6 @@ jobs:
           chunk_id: ${{ item.id }}
           item_chunkset_uri: blob://astersa/aster-etl-data/items/update-geometries-2023-01-23-00
           simplify_tolerance: 0.001
-        environment:
-          AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-          AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-          AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
       - id: ingest-items
         image_key: ingest
         task: pctasks.ingest_task.task:ingest_task
@@ -73,7 +61,3 @@ jobs:
           options:
             insert_group_size: 5000
             insert_only: false
-        environment:
-          AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-          AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-          AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
diff --git a/datasets/chesapeake_lulc/dataset.yaml b/datasets/chesapeake_lulc/dataset.yaml
@@ -8,11 +8,6 @@ code:
   src: ${{ local.path(./chesapeake_lulc.py) }}
   requirements: ${{ local.path(./requirements.txt) }}
 
-environment:
-  AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-  AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-  AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
-
 collections:
   - id: chesapeake-lc-7
     template: ${{ local.path(./collection/chesapeake-lc-7) }}

diff --git a/datasets/cil-gdpcir/dataset.yaml b/datasets/cil-gdpcir/dataset.yaml
@@ -3,11 +3,6 @@ image: ${{ args.registry }}/pctasks-task-base:latest
 args:
   - registry
 
-environment:
-  AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-  AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-  AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
-
 collections:
   - id: cil-gdpcir-cc-by
     template: ${{ local.path(./collection/cil-gdpcir-cc-by) }}

diff --git a/datasets/deltaresfloods/dataset.yaml b/datasets/deltaresfloods/dataset.yaml
@@ -1,10 +1,6 @@
 id: deltaresfloods
 image: pccomponentstest.azurecr.io/pctasks-basic:2022.6.16.2
 target_environment: staging
-environment:
-  AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-  AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-  AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
 
 collections:
   - id: deltares-floods

diff --git a/datasets/eclipse/dataset.yaml b/datasets/eclipse/dataset.yaml
@@ -4,11 +4,6 @@ image: ${{ args.registry }}/pctasks-task-base:latest
 args:
 - registry
 
-environment:
-  AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-  AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-  AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
-
 collections:
   - id: eclipse
     template: ${{ local.path(./collection) }}

diff --git a/datasets/ecmwf-forecast/dataset.yaml b/datasets/ecmwf-forecast/dataset.yaml
@@ -8,9 +8,6 @@ code:
   src: ${{ local.path(./ecmwf_forecast.py) }}
 
 environment:
-  AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-  AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-  AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
   APPLICATIONINSIGHTS_CONNECTION_STRING: ${{ secrets.task-application-insights-connection-string }}
 
 collections:

diff --git a/datasets/ecmwf-forecast/streaming.yaml b/datasets/ecmwf-forecast/streaming.yaml
@@ -37,9 +37,6 @@ jobs:
           collection_id: "ecmwf-forecast"
 
         environment:
-          AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-          AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-          AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
           APPLICATIONINSIGHTS_CONNECTION_STRING: ${{ secrets.task-application-insights-connection-string }}
           PCTASKS_COSMOSDB__URL: ${{ args.cosmosdb_url }}
         schema_version: 1.0.0

diff --git a/datasets/era5-pds/dataset.yaml b/datasets/era5-pds/dataset.yaml
@@ -4,11 +4,6 @@ image: ${{ args.registry }}/pctasks-task-base:latest
 args:
 - registry
 
-environment:
-  AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-  AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-  AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
-
 collections:
   - id: goes-cmi
     template: ${{ local.path(./collection) }}

diff --git a/datasets/esa-cci-lc/dataset.yaml b/datasets/esa-cci-lc/dataset.yaml
@@ -9,11 +9,6 @@ code:
   # included in the container image
   # requirements: ${{ local.path(./requirements.txt) }}
 
-environment:
-  AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-  AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-  AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
-
 collections:
   - id: esa-cci-lc-netcdf
     template: ${{ local.path(./collection/esa-cci-lc-netcdf) }}

diff --git a/datasets/esa-worldcover/dataset.yaml b/datasets/esa-worldcover/dataset.yaml
@@ -7,11 +7,6 @@ args:
 code:
   src: ${{ local.path(./esa_worldcover.py) }}
 
-environment:
-  AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-  AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-  AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
-
 collections:
   - id: esa-worldcover
     template: ${{ local.path(./collection) }}

diff --git a/datasets/esa-worldcover/workflows/esa-worldcover-process-items-2020.yaml b/datasets/esa-worldcover/workflows/esa-worldcover-process-items-2020.yaml
@@ -24,10 +24,6 @@ jobs:
             chunk_file_name: uris-list
             chunk_extension: .csv
         options: {}
-      environment:
-        AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-        AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-        AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
       schema_version: 1.0.0
   create-chunks:
     id: create-chunks
@@ -41,10 +37,6 @@ jobs:
         src_uri: ${{ item.uri }}
         dst_uri: blob://ai4edataeuwest/esa-worldcover-etl-data/pctasks/2020-all/assets
         options: ${{ item.chunk_options }}
-      environment:
-        AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-        AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-        AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
       schema_version: 1.0.0
     foreach:
       items: ${{ jobs.create-splits.tasks.create-splits.output.splits }}
@@ -66,10 +58,6 @@ jobs:
         collection_id: esa-worldcover
         options:
           skip_validation: false
-      environment:
-        AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-        AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-        AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
       schema_version: 1.0.0
     - id: ingest-items
       image_key: ingest
@@ -82,10 +70,6 @@ jobs:
         options:
           insert_group_size: 5000
           insert_only: false
-      environment:
-        AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-        AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-        AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
       schema_version: 1.0.0
     foreach:
       items: ${{ jobs.create-chunks.tasks.create-chunks.output.chunks }}

diff --git a/datasets/esa-worldcover/workflows/esa-worldcover-process-items-2021.yaml b/datasets/esa-worldcover/workflows/esa-worldcover-process-items-2021.yaml
@@ -24,10 +24,6 @@ jobs:
             chunk_file_name: uris-list
             chunk_extension: .csv
         options: {}
-      environment:
-        AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-        AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-        AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
       schema_version: 1.0.0
   create-chunks:
     id: create-chunks
@@ -41,10 +37,6 @@ jobs:
         src_uri: ${{ item.uri }}
         dst_uri: blob://ai4edataeuwest/esa-worldcover-etl-data/pctasks/2021-all/assets
         options: ${{ item.chunk_options }}
-      environment:
-        AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-        AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-        AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
       schema_version: 1.0.0
     foreach:
       items: ${{ jobs.create-splits.tasks.create-splits.output.splits }}
@@ -66,10 +58,6 @@ jobs:
         collection_id: esa-worldcover
         options:
           skip_validation: false
-      environment:
-        AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-        AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-        AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
       schema_version: 1.0.0
     - id: ingest-items
       image_key: ingest
@@ -82,10 +70,6 @@ jobs:
         options:
           insert_group_size: 5000
           insert_only: false
-      environment:
-        AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-        AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-        AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
       schema_version: 1.0.0
     foreach:
       items: ${{ jobs.create-chunks.tasks.create-chunks.output.chunks }}

diff --git a/datasets/fws-nwi/dataset.yaml b/datasets/fws-nwi/dataset.yaml
@@ -5,10 +5,6 @@ args:
 code:
   src: ${{ local.path(./fws_nwi.py) }}
   # requirements: ${{ local.path(./requirements.txt) }}
-environment:
-  AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-  AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-  AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
 
 task_config:
   fws-nwi:

diff --git a/datasets/gbif/dataset.yaml b/datasets/gbif/dataset.yaml
@@ -9,9 +9,6 @@ code:
   requirements: ${{ local.path(./requirements.txt) }}
 
 environment:
-  AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-  AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-  AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
   APPLICATIONINSIGHTS_CONNECTION_STRING: ${{ secrets.task-application-insights-connection-string }}
 
 collections:

diff --git a/datasets/goes/goes-cmi/dataset.yaml b/datasets/goes/goes-cmi/dataset.yaml
@@ -9,9 +9,6 @@ code:
   src: ${{ local.path(./goes_cmi) }}
 
 environment:
-  AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-  AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-  AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
   APPLICATIONINSIGHTS_CONNECTION_STRING: ${{ secrets.task-application-insights-connection-string }}
 
 collections:

diff --git a/datasets/goes/goes-cmi/streaming.yaml b/datasets/goes/goes-cmi/streaming.yaml
@@ -37,9 +37,6 @@ jobs:
           collection_id: "goes-cmi"
 
         environment:
-          AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-          AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-          AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
           APPLICATIONINSIGHTS_CONNECTION_STRING: ${{ secrets.task-application-insights-connection-string }}
           PCTASKS_COSMOSDB__URL: ${{ args.cosmosdb_url }}
         schema_version: 1.0.0

diff --git a/datasets/goes/goes-glm/dataset.yaml b/datasets/goes/goes-glm/dataset.yaml
@@ -11,9 +11,6 @@ code:
   # requirements: ${{ local.path(./requirements.txt) }}
 
 environment:
-  AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-  AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-  AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
   APPLICATIONINSIGHTS_CONNECTION_STRING: ${{ secrets.task-application-insights-connection-string }}
 
 collections:

diff --git a/datasets/goes/goes-glm/streaming.yaml b/datasets/goes/goes-glm/streaming.yaml
@@ -37,9 +37,6 @@ jobs:
           collection_id: "goes-glm"
 
         environment:
-          AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-          AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-          AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
           APPLICATIONINSIGHTS_CONNECTION_STRING: ${{ secrets.task-application-insights-connection-string }}
           PCTASKS_COSMOSDB__URL: ${{ args.cosmosdb_url }}
         schema_version: 1.0.0

diff --git a/datasets/goes/goes-glm/tests/dataset-dev-cluster.yaml b/datasets/goes/goes-glm/tests/dataset-dev-cluster.yaml
@@ -8,11 +8,6 @@ code:
   src: ${{ local.path(../goes_glm.py) }}
   requirements: ${{ local.path(../requirements.txt) }}
 
-environment:
-  AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-  AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-  AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
-
 collections:
   - id: goes-glm
     template: ${{ local.path(../collection) }}

diff --git a/datasets/goes/goes-glm/workflows/goes-glm-update.yaml b/datasets/goes/goes-glm/workflows/goes-glm-update.yaml
@@ -46,10 +46,6 @@ jobs:
             chunk_extension: .csv
             since: ${{ args.since }}
         options: {}
-      environment:
-        AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-        AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-        AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
       schema_version: 1.0.0
   create-chunks:
     id: create-chunks
@@ -63,10 +59,6 @@ jobs:
         src_uri: ${{ item.uri }}
         dst_uri: blob://goeseuwest/noaa-goes-etl-data/pctasks/glm/${{ args.since }}/assets
         options: ${{ item.chunk_options }}
-      environment:
-        AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-        AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-        AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
       schema_version: 1.0.0
     foreach:
       items: ${{ jobs.create-splits.tasks.create-splits.output.splits }}
@@ -89,10 +81,6 @@ jobs:
         collection_id: goes-glm
         options:
           skip_validation: false
-      environment:
-        AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-        AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-        AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
       schema_version: 1.0.0
     - id: ingest-items
       image_key: ingest
@@ -105,10 +93,6 @@ jobs:
         options:
           insert_group_size: 5000
           insert_only: false
-      environment:
-        AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-        AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-        AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
       schema_version: 1.0.0
     foreach:
       items: ${{ jobs.create-chunks.tasks.create-chunks.output.chunks }}

diff --git a/datasets/io-biodiversity/dataset.yaml b/datasets/io-biodiversity/dataset.yaml
@@ -7,11 +7,6 @@ args:
 code:
   src: ${{ local.path(./io_biodiversity.py) }}
 
-environment:
-  AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-  AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-  AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
-
 collections:
   - id: io-biodiversity
     template: ${{ local.path(./collection) }}

diff --git a/datasets/io-land-cover/dataset.yaml b/datasets/io-land-cover/dataset.yaml
@@ -7,11 +7,6 @@ args:
 code:
   src: ${{ local.path(./io_lulc.py) }}
 
-environment:
-  AZURE_TENANT_ID: ${{ secrets.task-tenant-id }}
-  AZURE_CLIENT_ID: ${{ secrets.task-client-id }}
-  AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }}
-
 collections:
   - id: io-lulc
     template: ${{ local.path(./collection/io-lulc) }}