Actually use dependency caching in CI (#310)

This commit enables dependency caching in CI. It uses root go.sum file to detect changes in dependencies but this is a good enough first step. We can concat go.sum files from all modules and hash that to detect changes in future if needed.
open-telemetry · Jun 15, 2020 · a304ef1 · a304ef1
1 parent 4e63242
commit a304ef1
Show file tree

Hide file tree

Showing 5 changed files with 72 additions and 23 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -16,6 +16,11 @@ executors:
       - image: cimg/go:1.14
 
 commands:
+  restore_workspace:
+    steps:
+      - attach_to_workspace
+      - restore_module_cache
+
   attach_to_workspace:
     steps:
       - attach_workspace:
@@ -24,7 +29,7 @@ commands:
   save_module_cache:
     steps:
       - save_cache:
-          key: cimg-go-pkg-mod-{{ checksum "go.sum" }}-v2
+          key: cimg-go-pkg-mod-{{ checksum "go.sum" }}-v4
           paths:
             - "/home/circleci/go/pkg/mod"
 
@@ -35,10 +40,7 @@ commands:
           command: mkdir -p ~/go/pkg/mod
       - restore_cache: # restores saved cache if no changes are detected since last run
           keys:
-            - cimg-go-pkg-mod-{{ checksum "go.sum" }}-v2
-      - persist_to_workspace:
-          root: ~/
-          paths: go/pkg/mod
+            - cimg-go-pkg-mod-{{ checksum "go.sum" }}-v4
 
   publish_docker_images:
     parameters:
@@ -90,7 +92,7 @@ workflows:
               only: /.*/
       - lint:
           requires:
-            - setup 
+            - setup
           filters:
             tags:
               only: /.*/
@@ -108,7 +110,7 @@ workflows:
               only: /.*/
       - cross-compile:
           requires:
-            - build
+            - setup
           filters:
             tags:
               only: /.*/
@@ -149,33 +151,36 @@ jobs:
     steps:
       - checkout
       - restore_module_cache
-      - persist_to_workspace:
-          root: ~/
-          paths: project
+      - run:
+          name: Install deps
+          command: make -j8 for-all-target TARGET='dep'
       - run:
           name: Install tools
           command: make install-tools
       - run:
           name: Install testbed tools
           command: make -C testbed install-tools
+      - save_module_cache
       - persist_to_workspace:
           root: ~/
-          paths: go/bin
+          paths:
+            - project
+            - go/bin
   lint:
     executor: golang
     steps:
-      - attach_to_workspace
+      - restore_workspace
       - run:
           name: Lint
-          command: CMD="make lint" make -j4 for-all
+          command: make -j8 for-all-target TARGET="lint"
       - run:
           name: Checks
           command: make -j4 checklicense impi misspell
 
   build:
     executor: golang
     steps:
-      - attach_to_workspace
+      - restore_workspace
       - run:
           name: Build collector for linux_amd64
           command: make otelcontribcol
@@ -187,7 +192,7 @@ jobs:
     docker:
       - image: cimg/go:1.14
     steps:
-      - attach_to_workspace
+      - restore_workspace
       - setup_remote_docker
       - run:
           name: Build trace example
@@ -197,7 +202,7 @@ jobs:
     executor: golang
     parallelism: 4
     steps:
-      - attach_to_workspace
+      - restore_workspace
       - run:
           name: Build collector for all archs
           command: grep ^otelcontribcol-all-sys Makefile|fmt -w 1|tail -n +2|circleci tests split|xargs make
@@ -208,7 +213,7 @@ jobs:
   test:
     executor: golang
     steps:
-      - attach_to_workspace
+      - restore_workspace
       - run:
           name: Coverage tests
           command: make test-with-cover
@@ -220,7 +225,7 @@ jobs:
     executor: golang
     resource_class: medium+
     steps:
-      - attach_to_workspace
+      - restore_workspace
       - run:
           name: Loadtest
           command: make -C testbed runtests
@@ -233,7 +238,7 @@ jobs:
     docker:
       - image: cimg/go:1.14
     steps:
-      - attach_to_workspace
+      - restore_workspace
       - setup_remote_docker
       - publish_docker_images:
           repo: opentelemetry-collector-contrib
@@ -248,7 +253,7 @@ jobs:
   publish-dev:
     executor: golang
     steps:
-      - attach_to_workspace
+      - restore_workspace 
       - setup_remote_docker
       - publish_docker_images:
           repo: opentelemetry-collector-contrib-dev
@@ -278,7 +283,7 @@ jobs:
   run-stability-tests:
     executor: golang
     steps:
-      - attach_to_workspace
+      - restore_workspace 
       - run:
           name: Run stability tests
           command: make stability-tests

diff --git a/Makefile b/Makefile
@@ -60,6 +60,14 @@ for-all:
 	 	$${CMD} ); \
 	done
 
+GOMODULES = $(ALL_MODULES) $(PWD)
+.PHONY: $(GOMODULES)
+MODULEDIRS = $(GOMODULES:%=for-all-target-%)
+for-all-target: $(MODULEDIRS)
+$(MODULEDIRS): 
+	$(MAKE) -C $(@:for-all-target-%=%) $(TARGET) 
+.PHONY: for-all-target
+
 .PHONY: install-tools
 install-tools:
 	go install github.com/client9/misspell/cmd/misspell

diff --git a/Makefile.Common b/Makefile.Common
@@ -92,7 +92,7 @@ fmt:
 
 .PHONY: lint
 lint: lint-static-check
-	$(LINT) run
+	$(LINT) run --allow-parallel-runners
 
 .PHONY: misspell
 misspell:
@@ -105,3 +105,7 @@ misspell-correction:
 .PHONY: impi
 impi:
 	@$(IMPI) --local github.com/open-telemetry/opentelemetry-collector-contrib --scheme stdThirdPartyLocal ./...
+
+.PHONY: dep 
+dep:
+	go mod download
diff --git a/go.mod b/go.mod
@@ -4,7 +4,7 @@ go 1.14
 
 require (
 	github.com/client9/misspell v0.3.4
-	github.com/golangci/golangci-lint v1.24.0
+	github.com/golangci/golangci-lint v1.27.0
 	github.com/google/addlicense v0.0.0-20200301095109-7c013a14f2e2
 	github.com/jstemmer/go-junit-report v0.9.1
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/alibabacloudlogserviceexporter v0.0.0