revert to previous yamls

tests/ci/azure_pipeline_test/dsvm_nightly_linux_cpu.yml

@@ -1,15 +1,5 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
-#
-# A Github Service Connection must also be created with the name "AI-GitHub"
-# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/demands?view=azure-devops&tabs=yaml
-
-resources:
-  repositories:
-    - repository: aitemplates
-      type: github
-      name: microsoft/AI
-      endpoint: AI-GitHub
 
 
 schedules:
@@ -41,9 +31,16 @@ jobs:
     name: $(Agent_Pool)
 
   steps:
-  - template: .ci/steps/reco_config_conda_linux.yml@aitemplates
-    parameters:
-      conda_env: nightly_reco_base
+  - bash: |
+      echo "##vso[task.prependpath]/data/anaconda/bin"
+      conda env list
+    displayName: 'Add Conda to PATH'
+
+  - script: |
+      conda env remove -n nightly_reco_base && \
+      python ./scripts/generate_conda_file.py --name nightly_reco_base && \
+      conda env create --quiet -f nightly_reco_base.yaml 2> log
+    displayName: 'Setup Conda Env'
 
   - script: |
       . /anaconda/etc/profile.d/conda.sh && \
@@ -62,6 +59,9 @@ jobs:
       failTaskOnFailedTests: true
     condition: succeededOrFailed()
 
-  - template: .ci/steps/reco_conda_clean_linux.yml@aitemplates
-    parameters:
-      conda_env: nightly_reco_base
+  - script: |
+      conda env remove -n nightly_reco_base -y   
+    workingDirectory: tests
+    displayName: 'Conda remove'
+    continueOnError: true
+    condition: always() # this step will always run, even if the pipeline is canceled

tests/ci/azure_pipeline_test/dsvm_nightly_linux_gpu.yml

@@ -1,29 +1,17 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
-#
-# A Github Service Connection must also be created with the name "AI-GitHub"
-# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/demands?view=azure-devops&tabs=yaml
-
-resources:
-  repositories:
-    - repository: aitemplates
-      type: github
-      name: microsoft/AI
-      endpoint: AI-GitHub
 
 schedules:
 - cron: "7 4 * * *"
   displayName: Nightly build master
   branches:
     include:
     - master
-  always: true
 - cron: "7 16 * * *"
   displayName: Nightly build staging
   branches:
     include:
     - staging
-  always: true
 
 trigger: none
 
@@ -40,9 +28,16 @@ jobs:
     name: $(Agent_Pool)
 
   steps:
-  - template: .ci/steps/reco_config_conda_linux.yml@aitemplates
-    parameters:
-      conda_env: nightly_reco_gpu
+  - bash: |
+      echo "##vso[task.prependpath]/data/anaconda/bin"
+      conda env list
+    displayName: 'Add Conda to PATH'
+
+  - script: |
+      conda env remove -n nightly_reco_gpu && \
+      python ./scripts/generate_conda_file.py --gpu --name nightly_reco_gpu && \
+      conda env create --quiet -f nightly_reco_gpu.yaml 2> log
+    displayName: 'Setup Conda Env'
 
   - script: |
       . /anaconda/etc/profile.d/conda.sh && \
@@ -61,6 +56,9 @@ jobs:
       failTaskOnFailedTests: true
     condition: succeededOrFailed()
 
-  - template: .ci/steps/reco_conda_clean_linux.yml@aitemplates
-    parameters:
-      conda_env: nightly_reco_gpu
+  - script: |
+      conda env remove -n nightly_reco_gpu -y   
+    workingDirectory: tests
+    displayName: 'Conda remove'
+    continueOnError: true
+    condition: always() # this step will always run, even if the pipeline is canceled

tests/ci/azure_pipeline_test/dsvm_nightly_linux_pyspark.yml

@@ -1,29 +1,17 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
-#
-# A Github Service Connection must also be created with the name "AI-GitHub"
-# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/demands?view=azure-devops&tabs=yaml
-
-resources:
-  repositories:
-    - repository: aitemplates
-      type: github
-      name: microsoft/AI
-      endpoint: AI-GitHub
 
 schedules:
 - cron: "7 8 * * *"
   displayName: Nightly build master
   branches:
     include:
     - master
-  always: true
 - cron: "7 20 * * *"
   displayName: Nightly build staging
   branches:
     include:
     - staging
-  always: true
 
 trigger: none
 
@@ -40,9 +28,16 @@ jobs:
     name: $(Agent_Pool)
 
   steps:
-  - template: .ci/steps/reco_config_conda_linux.yml@aitemplates
-    parameters:
-      conda_env: nightly_reco_pyspark
+  - bash: |
+      echo "##vso[task.prependpath]/data/anaconda/bin"
+      conda env list
+    displayName: 'Add Conda to PATH'
+
+  - script: |
+      conda env remove -n nightly_reco_pyspark && \
+      python ./scripts/generate_conda_file.py --pyspark --name nightly_reco_pyspark && \
+      conda env create --quiet -f nightly_reco_pyspark.yaml 2> log
+    displayName: 'Setup Conda Env'
 
   - script: |
       . /anaconda/etc/profile.d/conda.sh && \
@@ -61,6 +56,9 @@ jobs:
       failTaskOnFailedTests: true
     condition: succeededOrFailed()
 
-  - template: .ci/steps/reco_conda_clean_linux.yml@aitemplates
-    parameters:
-      conda_env: nightly_reco_pyspark
+  - script: |
+      conda env remove -n nightly_reco_pyspark -y   
+    workingDirectory: tests
+    displayName: 'Conda remove'
+    continueOnError: true
+    condition: always() # this step will always run, even if the pipeline is canceled

tests/ci/azure_pipeline_test/dsvm_nightly_win_cpu.yml

@@ -1,29 +1,17 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
-#
-# A Github Service Connection must also be created with the name "AI-GitHub"
-# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/demands?view=azure-devops&tabs=yaml
-
-resources:
-  repositories:
-    - repository: aitemplates
-      type: github
-      name: microsoft/AI
-      endpoint: AI-GitHub
 
 schedules:
 - cron: "7 0 * * *"
   displayName: Nightly build master
   branches:
     include:
     - master
-  always: true
 - cron: "7 12 * * *"
   displayName: Nightly build staging
   branches:
     include:
     - staging
-  always: true
 
 trigger: none
 
@@ -40,9 +28,15 @@ jobs:
     name: $(Agent_Pool)
 
   steps:
-  - template: .ci/steps/reco_config_conda_win.yml@aitemplates
-    parameters:
-      conda_env: nightly_reco_base
+  - script: |
+      call conda env remove -n nightly_reco_base -y
+      if exist C:\Anaconda\envs\nightly_reco_base rmdir /s /q C:\Anaconda\envs\nightly_reco_base
+    displayName: 'Remove Conda Env if it exists'
+
+  - script: |
+      python ./scripts/generate_conda_file.py --name nightly_reco_base
+      call conda env create -f nightly_reco_base.yaml
+    displayName: 'Setup Conda Env'
 
   - script: |
       call conda activate nightly_reco_base
@@ -59,7 +53,16 @@ jobs:
       failTaskOnFailedTests: true
     condition: succeededOrFailed()
 
-  - template: .ci/steps/reco_conda_clean_win.yml@aitemplates
-    parameters:
-      conda_env: nightly_reco_base
+  - script: |
+      call conda env remove -n nightly_reco_base -y
+      if exist C:\Anaconda\envs\nightly_reco_base rmdir /s /q C:\Anaconda\envs\nightly_reco_base
+    workingDirectory: tests
+    displayName: 'Conda remove'
+    continueOnError: true
+    condition: always() # this step will always run, even if the pipeline is canceled
 
+  - script: |
+      del /q /S %LOCALAPPDATA%\Temp\*
+      for /d %%i in (%LOCALAPPDATA%\Temp\*) do @rmdir /s /q "%%i"
+    displayName: 'Remove Temp Files'
+    condition: succeededOrFailed()

tests/ci/azure_pipeline_test/dsvm_nightly_win_gpu.yml

@@ -1,29 +1,17 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
-#
-# A Github Service Connection must also be created with the name "AI-GitHub"
-# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/demands?view=azure-devops&tabs=yaml
-
-resources:
-  repositories:
-    - repository: aitemplates
-      type: github
-      name: microsoft/AI
-      endpoint: AI-GitHub
 
 schedules:
 - cron: "7 4 * * *"
   displayName: Nightly build master
   branches:
     include:
     - master
-  always: true
 - cron: "7 16 * * *"
   displayName: Nightly build staging
   branches:
     include:
     - staging
-  always: true
 
 trigger: none
 
@@ -40,9 +28,15 @@ jobs:
     name: $(Agent_Pool)
 
   steps:
-  - template: .ci/steps/reco_config_conda_win.yml@aitemplates
-    parameters:
-      conda_env: nightly_reco_base
+  - script: |
+      call conda env remove -n nightly_reco_gpu -y
+      if exist C:\Anaconda\envs\nightly_reco_gpu rmdir /s /q C:\Anaconda\envs\nightly_reco_gpu
+    displayName: 'Remove Conda Env if it exists'
+
+  - script: |
+      python ./scripts/generate_conda_file.py --gpu --name nightly_reco_gpu
+      conda env create --quiet -f nightly_reco_gpu.yaml --verbose
+    displayName: 'Setup Conda Env'
 
   - script: |
       call conda activate nightly_reco_gpu
@@ -59,6 +53,10 @@ jobs:
       failTaskOnFailedTests: true
     condition: succeededOrFailed()
 
-  - template: .ci/steps/reco_conda_clean_win.yml@aitemplates
-    parameters:
-      conda_env: nightly_reco_gpu
+  - script: |
+      call conda env remove -n nightly_reco_gpu -y
+      if exist C:\Anaconda\envs\nightly_reco_gpu rmdir /s /q C:\Anaconda\envs\nightly_reco_gpu
+    workingDirectory: tests
+    displayName: 'Conda remove'
+    continueOnError: true
+    condition: always() # this step will always run, even if the pipeline is canceled

tests/ci/azure_pipeline_test/dsvm_nightly_win_pyspark.yml

@@ -1,29 +1,17 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
-#
-# A Github Service Connection must also be created with the name "AI-GitHub"
-# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/demands?view=azure-devops&tabs=yaml
-
-resources:
-  repositories:
-    - repository: aitemplates
-      type: github
-      name: microsoft/AI
-      endpoint: AI-GitHub
 
 schedules:
 - cron: "7 8 * * *"
   displayName: Nightly build master
   branches:
     include:
     - master
-  always: true
 - cron: "7 20 * * *"
   displayName: Nightly build staging
   branches:
     include:
     - staging
-  always: true
 
 trigger: none
 
@@ -40,9 +28,15 @@ jobs:
     name: $(Agent_Pool)
 
   steps:
-  - template: .ci/steps/reco_config_conda_win.yml@aitemplates
-    parameters:
-      conda_env: nightly_reco_base
+  - script: |
+      call conda env remove -n nightly_reco_pyspark -y
+      if exist C:\Anaconda\envs\nightly_reco_pyspark rmdir /s /q C:\Anaconda\envs\nightly_reco_pyspark
+    displayName: 'Remove Conda Env if it exists'
+
+  - script: |
+      python ./scripts/generate_conda_file.py --pyspark --name nightly_reco_pyspark
+      conda env create --quiet -f nightly_reco_pyspark.yaml --verbose
+    displayName: 'Setup Conda Env'
 
   - script: |
       call conda activate nightly_reco_pyspark
@@ -63,6 +57,10 @@ jobs:
       failTaskOnFailedTests: true
     condition: succeededOrFailed()
 
-  - template: .ci/steps/reco_conda_clean_win.yml@aitemplates
-    parameters:
-      conda_env: nightly_reco_pyspark
+  - script: |
+      call conda env remove -n nightly_reco_pyspark -y
+      if exist C:\Anaconda\envs\nightly_reco_pyspark rmdir /s /q C:\Anaconda\envs\nightly_reco_pyspark
+    workingDirectory: tests
+    displayName: 'Conda remove'
+    continueOnError: true
+    condition: always() # this step will always run, even if the pipeline is canceled