[gitlab] Improve go modules cache accuracy and speed (DataDog#29487)

Co-authored-by: Nicolas Schweitzer <[email protected]>
kkpan11 · Sep 23, 2024 · dddc0c7 · dddc0c7
1 parent e0b516c
commit dddc0c7
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 12 deletions.
diff --git a/.gitlab/deps_fetch/deps_fetch.yml b/.gitlab/deps_fetch/deps_fetch.yml
@@ -4,15 +4,15 @@
 # to reuse them in further jobs that need them.
 
 .retrieve_linux_go_deps:
-  - mkdir -p $GOPATH/pkg/mod && tar xJf modcache.tar.xz -C $GOPATH/pkg/mod
+  - mkdir -p $GOPATH/pkg/mod/cache && tar xJf modcache.tar.xz -C $GOPATH/pkg/mod/cache
   - rm -f modcache.tar.xz
 
 .retrieve_linux_go_tools_deps:
-  - mkdir -p $GOPATH/pkg/mod && tar xJf modcache_tools.tar.xz -C $GOPATH/pkg/mod
+  - mkdir -p $GOPATH/pkg/mod/cache && tar xJf modcache_tools.tar.xz -C $GOPATH/pkg/mod/cache
   - rm -f modcache_tools.tar.xz
 
 .retrieve_linux_go_e2e_deps:
-  - mkdir -p $GOPATH/pkg/mod && tar xJf modcache_e2e.tar.xz -C $GOPATH/pkg/mod
+  - mkdir -p $GOPATH/pkg/mod/cache && tar xJf modcache_e2e.tar.xz -C $GOPATH/pkg/mod/cache
   - rm -f modcache_e2e.tar.xz
 
 .cache:
@@ -42,7 +42,7 @@ go_deps:
     - if [ -f modcache.tar.xz  ]; then exit 0; fi
     - inv -e deps --verbose
     - inv -e install-tools
-    - cd $GOPATH/pkg/mod/ && tar c -I "pxz -T${KUBERNETES_CPU_REQUEST}" -f $CI_PROJECT_DIR/modcache.tar.xz .
+    - cd $GOPATH/pkg/mod/cache/ && tar c -I "pxz -T${KUBERNETES_CPU_REQUEST}" -f $CI_PROJECT_DIR/modcache.tar.xz .
   artifacts:
     expire_in: 1 day
     paths:
@@ -57,7 +57,7 @@ go_deps:
         files:
           - \**/go.mod
           - .gitlab/deps_fetch/deps_fetch.yml
-        prefix: "go_deps"
+        prefix: "go_deps_modcache"
       paths:
         - modcache.tar.xz
 
@@ -66,7 +66,7 @@ go_tools_deps:
   script:
     - if [ -f modcache_tools.tar.xz  ]; then exit 0; fi
     - inv -e install-tools
-    - cd $GOPATH/pkg/mod/ && tar c -I "pxz -T${KUBERNETES_CPU_REQUEST}" -f $CI_PROJECT_DIR/modcache_tools.tar.xz .
+    - cd $GOPATH/pkg/mod/cache/ && tar c -I "pxz -T${KUBERNETES_CPU_REQUEST}" -f $CI_PROJECT_DIR/modcache_tools.tar.xz .
   artifacts:
     expire_in: 1 day
     paths:
@@ -76,7 +76,7 @@ go_tools_deps:
         files:
           - ./**/go.mod
           - .gitlab/deps_fetch/deps_fetch.yml
-        prefix: "go_tools_deps"
+        prefix: "go_tools_deps_modcache"
       paths:
         - modcache_tools.tar.xz
 
@@ -85,7 +85,7 @@ go_e2e_deps:
   script:
     - if [ -f modcache_e2e.tar.xz  ]; then exit 0; fi
     - inv -e new-e2e-tests.deps
-    - cd $GOPATH/pkg/mod/ && tar c -I "pxz -T${KUBERNETES_CPU_REQUEST}" -f $CI_PROJECT_DIR/modcache_e2e.tar.xz .
+    - cd $GOPATH/pkg/mod/cache/ && tar c -I "pxz -T${KUBERNETES_CPU_REQUEST}" -f $CI_PROJECT_DIR/modcache_e2e.tar.xz .
   artifacts:
     expire_in: 1 day
     paths:
@@ -95,6 +95,6 @@ go_e2e_deps:
         files:
           - ./test/new-e2e/go.mod
           - .gitlab/deps_fetch/deps_fetch.yml
-        prefix: "go_e2e_deps"
+        prefix: "go_e2e_deps_modcache"
       paths:
         - modcache_e2e.tar.xz
diff --git a/tasks/libs/common/go.py b/tasks/libs/common/go.py
@@ -11,8 +11,10 @@
 
 def download_go_dependencies(ctx: Context, paths: list[str], verbose: bool = False, max_retry: int = 3):
     print("downloading dependencies")
-    with timed("go mod download"):
+    with timed("go mod download && go mod tidy"):
         verbosity = ' -x' if verbose else ''
         for path in paths:
             with ctx.cd(path):
-                run_command_with_retry(ctx, f"go mod download{verbosity}", max_retry=max_retry)
+                run_command_with_retry(
+                    ctx, f"go mod download{verbosity} && go mod tidy{verbosity}", max_retry=max_retry
+                )
diff --git a/tasks/winbuildscripts/extract-modcache.bat b/tasks/winbuildscripts/extract-modcache.bat
@@ -33,7 +33,7 @@ if exist %MODCACHE_XZ_FILE% (
     REM This shouldn't have any negative impact: since modules are
     REM stored per version and hash, files that get replaced will
     REM get replaced by the same files
-    Powershell -C "7z x %MODCACHE_TAR_FILE% -o%GOMODCACHE% -aoa -bt"
+    Powershell -C "7z x %MODCACHE_TAR_FILE% -o%GOMODCACHE%\cache -aoa -bt"
     @echo Modcache extracted
 ) else (
     @echo %MODCACHE_XZ_FILE% not found, dependencies will be downloaded