Merge pull request #78 from anandhu-eng/fixGithubActionError

Code change for rclone and gdown
mlcommons · Jun 26, 2024 · 098785e · 098785e
2 parents 0c25b88 + e5702dc
commit 098785e
Show file tree

Hide file tree

Showing 4 changed files with 42 additions and 3 deletions.
diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml
@@ -37,7 +37,7 @@ jobs:
         cm pull repo mlcommons@cm4abtf --branch=poc
     - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on docker
       run: |
-        cm run script --tags=run-abtf,inference,_poc-demo --adr.compiler.tags=gcc --quiet --docker --docker_it=no  -v
+        cm run script --tags=run-abtf,inference,_poc-demo --adr.compiler.tags=gcc --quiet  -v
 
   build2:
     runs-on: ${{ matrix.os }}
@@ -67,3 +67,29 @@ jobs:
       run: |
         cm run script --tags=run-abtf,inference,_poc-demo --adr.compiler.tags=gcc --quiet  -v
     
+  build3:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [windows-latest]
+        python-version: [ "3.8", "3.12" ]
+        backend: [ "pytorch" ]
+        implementation: [ "python" ]
+        exclude:
+        - python-version: "3.8"
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python3 -m pip install cmind
+        cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
+        cm pull repo mlcommons@cm4abtf --branch=poc
+    - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on ${{ matrix.os }}
+      run: |
+        cm run script --tags=run-abtf,inference,_poc-demo --quiet --env.CM_MLPERF_LOADGEN_BUILD_FROM_SRC=off  -v
diff --git a/script/download-file/customize.py b/script/download-file/customize.py
@@ -48,7 +48,7 @@ def preprocess(i):
         verify_ssl = env.get('CM_VERIFY_SSL', "True")
         if str(verify_ssl).lower() in [ "no", "false" ]:
             verify_ssl = False
-            if tool == 'wget':
+            if tool == 'wget' or tool == "gdown":
                 extra_download_options += " --no-check-certificate"
         else:
             verify_ssl = True
@@ -113,7 +113,13 @@ def preprocess(i):
             if env.get('CM_RCLONE_CONFIG_CMD', '') != '':
                 env['CM_DOWNLOAD_CONFIG_CMD'] = env['CM_RCLONE_CONFIG_CMD']
             rclone_copy_using = env.get('CM_RCLONE_COPY_USING', 'sync')
-            env['CM_DOWNLOAD_CMD'] = f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), env['CM_DOWNLOAD_FILENAME'])}{q} -P"
+            if env["CM_HOST_OS_TYPE"] == "windows":
+                # have to modify the variable from url to temp_url if it is going to be used anywhere after this point
+                url = url.replace("%", "%%")
+                temp_download_file = env['CM_DOWNLOAD_FILENAME'].replace("%", "%%")
+                env['CM_DOWNLOAD_CMD'] = f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), temp_download_file)}{q} -P"
+            else:
+                env['CM_DOWNLOAD_CMD'] = f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), env['CM_DOWNLOAD_FILENAME'])}{q} -P"
 
         filename = env['CM_DOWNLOAD_FILENAME']
         env['CM_DOWNLOAD_DOWNLOADED_FILENAME'] = filename

diff --git a/script/extract-file/customize.py b/script/extract-file/customize.py
@@ -25,6 +25,9 @@ def preprocess(i):
     filename = env.get('CM_EXTRACT_FILEPATH','')
     if filename == '':
         return {'return': 1, 'error': 'Extract with no download requested and CM_EXTRACT_FILEPATH is not set'}
+
+    if windows:
+        filename = filename.replace("%", "%%")
 
     env['CM_EXTRACT_FILENAME'] = filename
 

diff --git a/script/get-rclone/customize.py b/script/get-rclone/customize.py
@@ -53,6 +53,10 @@ def preprocess(i):
                 cur_dir = os.getcwd()
                 path_bin = os.path.join(cur_dir, file_name)
                 env['CM_RCLONE_BIN_WITH_PATH'] = path_bin
+
+                if not env.get('+PATH', []):
+                    env['+PATH'] = []
+                env['+PATH'].append(cur_dir)
 
                 if not env.get('+PATH', []):
                     env['+PATH'] = []