add sops.make for mozillas sops encryption. cleaned up comments (#24)

* added a new sops.make for mozillas sops.

* added sops decrypt to tests

* test for sops

* get test in for vault.decrypt

* change builder base

* fix gpg import and base docker

* minor readme tweak.

.circleci/config.yml

@@ -2,13 +2,14 @@ version: 2.1
 jobs:
   build:
     docker:
-      - image: yakworks/builder:bash-make
+      - image: yakworks/builder:k8s
     steps:
       - checkout
       - run: make install-shellcheck-alpine
       - run: make install-file-alpine
       - run: make check
       # test some other stuff
+      - run: make vault.decrypt-files
       - run: make git.config-bot-user
       # if it meets the criteria to be IS_RELEASABLE, this will bump version and tag release on github
       - run: make ship.version

.env.example

@@ -1,22 +1,16 @@
 # NEVER CHECK IN
-# dotenv is for user environment for testing and development
-# below are some options for override
-# for a list of what your make build is currently using run `make log-vars`
+# Copy and rename this to a .env to override variables for testing and development
+# Copy to a secrets.env if data is sensitive
+# `make log-vars` shows what is defined in .env and will not show whats in .secrets.env
 
-# if not storing GITHUB_TOKEN in bash ENV then set it here
+# if not storing GITHUB_TOKEN in bash ENV then set it in .secrects.env
 # GITHUB_TOKEN=gp123l4j1234lkj
-
-# Some releasing targets depend on ACTIVE_BRANCH being set, which is normally only set when its a CI
-# these can help with simulation or manual deploys
-# Set this to a "releasable" branch and some targets such as publish-release or kube.deploy will be active
-# ACTIVE_BRANCH=master
+# MAVEN_REPO_KEY = xxx
+# MAVEN_REPO_USER = yyy
 
 # to test some of the CIRCLE targets
 # CIRCLECI=true
 # CIRCLE_BRANCH=master
+# this is the org
 # CIRCLE_PROJECT_USERNAME=yakworks
 # CIRCLE_PROJECT_REPONAME=gorm-tools
-
-# `make log-vars` should show this
-# dummy, should show up in log-vars
-FOO=bar

.sops.yaml

@@ -0,0 +1,7 @@
+# josh 54B2, joanna FC1B, 9cibot C575, Ken EFC9
+creation_rules:
+    - pgp: >-
+        54B28047704C36FDCB30E63B472AC57FC3EB83C8,
+        C5758A15DF91198CF5347A90F8E8B460302AAEFA,
+        FC1B46ABD5AE6A99E31165E42DCB557950992300,
+        EFC9CC2C54F7387675F435F4D51CED1BA3F5E95E

Makefile

@@ -1,27 +1,28 @@
-# tell shipkit where the main build.sh is, not used in test
-# build.sh := ./build.sh
+
 # If setting any vars needed for the $(shell init_vars..) in Shipkit.make then track the MAKE_VARS so those dont get added
-# if not git is installed then need to add the PROJECT_FULLNAME, much easier to add a build.sh and do it there
+# if not git is installed then need to add the PROJECT_FULLNAME, much easier to add a build.yml and do it there
 # export PROJECT_FULLNAME = yakworks/shipkit
 # BUILD_VARS = PROJECT_FULLNAME # need this in order for it to build what vars get passed the $(shell)
 # core include, creates the makefile.env for the BUILD_VARS that evrything else depends on
 include Shipkit.make
 include $(SHIPKIT_MAKEFILES)/base-build.make
 include $(SHIPKIT_MAKEFILES)/docker.make
-include $(SHIPKIT_MAKEFILES)/secrets.make
+# include $(SHIPKIT_MAKEFILES)/secrets.make
+include $(SHIPKIT_MAKEFILES)/vault.make
 include $(SHIPKIT_MAKEFILES)/git-tools.make
 include $(SHIPKIT_MAKEFILES)/ship-version.make
 include $(SHIPKIT_MAKEFILES)/circle.make
 include $(SHIPKIT_MAKEFILES)/bats-testing.make
 
 # -- Variables ---
 export BOT_EMAIL ?= [email protected]
+export VAULT_URL = https://github.com/9ci/vault.git
 # can be set here but best do it on command line with make
 # export LOGIT_DEBUG_ENABLED := true
 
 # --- Dockers ---
 docker_tools := $(SHIPKIT_BIN)/docker_tools
-DOCK_SHELL_URL = yakworks/builder:bash-make
+DOCK_SHELL_URL = yakworks/builder:base
 
 ## docker shell for testing
 docker.shell:

README.md

@@ -26,7 +26,7 @@ We have normalized on a set of `Bash` scripts and `Make` targets for all of our
 
 ## Why Bash?
 
-Yes we need it. Its pretty much every where. And when using a small alpine docker image, adding bash only adds about 2.5 mb
+Yes we need it. Its pretty much every where. And when using a small alpine docker image, adding bash only adds about 800k compressed, and 2.2 mb uncompressed.
 
 | Image                                                        | size    | uncompressed |
 |--------------------------------------------------------------|--------:|-------------:|

bin/circle

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 # ---
 # working with CI circle and publishing,
-# should be imported into the main build.sh
 # ---
 set -euo pipefail # strict mode https://bit.ly/36MvF0T
 source "$(dirname "${BASH_SOURCE[0]}")"/core/main

bin/dotenv

@@ -2,7 +2,7 @@
 # shellcheck disable=SC2034 # appears unused, many are used in templates
 # ---
 # default functions to setup BUILD_VARS
-# BUILD_VARS are used in the build.sh and a makefile.env is created
+# BUILD_VARS are used to create a makefile.env
 # that is imcluded at the start of the Makefile to share vars
 # ---
 set -eo pipefail # strict mode https://bit.ly/36MvF0T

bin/init_env

@@ -17,7 +17,7 @@ core.import "setVar"
 # version.properties should be in root dir
 if [ -e "version.properties" ]; then source version.properties; fi
 
-# main make env, this is called early shipkit or can be overriden in build.sh.
+# main make env, this is called early.
 # $1 - BUILD_ENV (test, dev, seed)
 function make_env {
   make_env_init "${1:-}"
@@ -53,7 +53,7 @@ function init_env {
   load_env ".env"
 
   # if build/vault/bot.env has been cloned this will import the vars
-  parse_bot_env_file
+  # parse_bot_env_file
 
   # check for a build.env
   load_env "build.env"

bin/setVar

@@ -3,6 +3,7 @@
 # The original purpose here was to be able to set a variable that will be tracked and can then be
 # eported back out to a file can be created and included into Make
 set -eo pipefail # strict mode https://bit.ly/36MvF0T
+core.import "logit"
 
 # sets the variable value if its not already set and adds it for tracking in BUILD_VARS
 # args $1 - the variable name
@@ -14,7 +15,7 @@ function setVar {
   # curVal="${!varName}"
   # [[ ! ${!varName+x} ]]
   if [[ -z ${!varName} ]]; then
-    # echo "curVal is empty, adding"
+    # logit.info "$varName=\"\$2\""
     eval "$varName=\"\$2\""
   fi
   add_build_vars "$varName"

docs/Makefiles.md

@@ -56,16 +56,4 @@ A target with a variable right after it like `foo : BAR = true`
 - passing through `$` variable references will require a double `$$` to escape it, without it make will think
   its and internal reference
 
-- each line in a target runs in a new shell. To run them together its recomended to create a
-  function in the build.sh and call that as its can get a little ugly in `make`. 
-  if you need to do it in make then each exec should end 
-  with `;` and a `\` if its multiline. for example (note if you copy this example make sure to convert indent spaces to tabs):
-  ```
-  foo: ## shows foos
-      @FOOS="buzz bar bazz"; \
-      for f in $$FOOS; do \
-        echo "$$f"; \
-      done;
-  ```
-
 - `make` can call it self using the `$(MAKE)` built in. so to call a build target `$(MAKE) build` can be used

makefiles/Shipkit-main.make

@@ -13,12 +13,12 @@ export MAKE_ENV_FILE ?= $(BUILD_DIR)/make/makefile$(MAKELEVEL).env
 SHELL_VARS += VERBOSE_LOG BUILD_DIR MAKE_ENV_FILE DBMS env dry_run
 #shell doesn't get the exported vars so we need to spin the ones we want, which should be in BUILD_VARS
 SHELL_EXPORTS := $(foreach v,$(SHELL_VARS), $(v)='$($(v))')
-# if no build.sh var is not set then use the the init_env script directly
-# if its set then call build.sh and assume it sourced in /init_env and will
+# if no init_env.sh var is not set then use the the init_env script directly
+# if its set then call init_env.sh and assume it sourced in /init_env and will
 # be setting up variables and/or potentially overriding make_env
-build.sh ?= $(SHIPKIT_BIN)/init_env
+init_env.sh ?= $(SHIPKIT_BIN)/init_env
 # we do this in subshell so it forces the file to be generated before the sindlues happens
-SH_RESULTS := $(shell $(SHELL_EXPORTS) $(build.sh) make_env $(BUILD_ENV))
+SH_RESULTS := $(shell $(SHELL_EXPORTS) $(init_env.sh) make_env $(BUILD_ENV))
 ifneq ($(.SHELLSTATUS),0)
   $(error init_env error -> $(SH_RESULTS))
 endif
@@ -70,8 +70,13 @@ FORCE:
 log-vars: FORCE
 	printf "$(ccyan)dry_run$(creset) = $(dry_run)\n"
 	printf "$(culine)Variable:\n\n$(creset)"
-	for v in $(sort $(BUILD_VARS)); do
-		printf "$(ccyan)$$v $(creset)=$(cbold) $${!v:-} $(creset)\n"
+	printf "$(ccyan)VAULT_ENV_VARS $(creset)= $(cbold) $(VAULT_ENV_VARS) $(creset)\n"
+	for varName in $(sort $(BUILD_VARS)); do
+		varVal=$${!varName:-}
+		if [[ $${varName^^} =~ TOKEN|PASSWORD|GPG.*KEY|REPO.*KEY ]]; then
+			varVal="*********"
+		fi
+		printf "$(ccyan)$$varName $(creset)=$(cbold) "$$varVal"$(creset)\n"
 	done;
 
 log-make-vars: FORCE
@@ -215,7 +220,7 @@ endef
 #
 OS ?=
 ifeq (Windows_NT,$(OS))
-OS_NAME := Windows
+OS_NAME := windows
 OS_CPU  := $(call _lower,$(PROCESSOR_ARCHITECTURE))
 OS_ARCH := $(if $(findstring amd64,$(OS_CPU)),x86_64,i686)
 else

makefiles/vault.make

@@ -0,0 +1,81 @@
+# Opinionated process to clone and decrypt a github project use as a "vault
+# installs sops if not present on OS, will not be for build dockers, for local dev should already be there
+# clones the github repo in VAULT_DIR, we call it the vault
+
+SOP_VERSION := 3.7.1
+SOP_URL     := https://github.com/mozilla/sops/releases/download/v$(SOP_VERSION)/sops-v$(SOP_VERSION).linux
+VAULT_DIR   ?= $(BUILD_DIR)/vault
+VAULT_FILES ?= bot.enc.env
+VAULT_BOT_ENV_FILE := $(VAULT_DIR)/bot.env
+
+# --- look for build/vault/bot.env , run  sops.decrypt-vault-files --
+# we import it straight into make since these are secrets, dont want them in BUILD_VARS where they can get logged
+
+ifneq ($(wildcard $(VAULT_BOT_ENV_FILE)),)
+  include $(VAULT_BOT_ENV_FILE)
+  VAULT_ENV_VARS := $(shell sed '/^#.*/d; /^$$/d; s/=.*//g;' $(VAULT_BOT_ENV_FILE))
+  # $(info including $(VAULT_ENV_VARS) from $(VAULT_ENV_FILE))
+  export $(VAULT_ENV_VARS)
+endif
+# ---
+
+SOP_SH := $(shell which sops 2> /dev/null)
+
+# if doesn't already exists then above will be empty
+ifeq ($(SOP_SH),)
+
+ SOP_SH := $(SHIPKIT_INSTALLS)/sops
+ # $(info sops is NOT installed)
+
+endif
+
+# on demand clone and install of git-secret for build dockers
+# as a make reminder, if the file ref SOP_SH doesn't exist then this runs, if its there already then this does nothing
+$(SOP_SH):
+	$(logr) "intalling sops $(SOP_URL)"
+	# make sure installs is created
+	mkdir -p $(SHIPKIT_INSTALLS)
+	curl -qsL $(SOP_URL) -o $(SOP_SH)
+	chmod +x $(SOP_SH)
+	$(logr.done)
+
+# easier for testing
+sops.install: $(SOP_SH)
+
+vault.clone: | _verify_VAULT_URL
+	[ ! -e $(VAULT_DIR) ] && git clone $(VAULT_URL) $(VAULT_DIR) || :;
+
+vault.decrypt-files: $(SOP_SH) gpg.import-private-key vault.clone
+	cd $(VAULT_DIR)
+	for vfile in $(VAULT_FILES); do
+		outFile="$${vfile/.enc./.}" # remove .enc.
+		outFile="$${outFile/.encrypted./.}" # remove .encrypted.
+		$(logr) "$$vfile > $$outFile"
+		$(SOP_SH) -d $$vfile > $$outFile
+	done
+	$(logr.done)
+
+# to test the gpg stuff set this to the base64 encoded key, DO NOT CHECK IN
+# GPG_KEY=ZZZ
+# set this to the gpg passphrase if needed, not base64
+# GPG_PASS := xxx
+
+# imports private key from GPG_PRIVATE_KEY var
+gpg.import-private-key:
+	if [ "$(GPG_KEY)"  ]; then
+		echo "$(GPG_KEY)" | base64 --decode | gpg -v --batch --import --quiet --no-verbose
+		$(logr) "GPG_KEY imported"
+	else
+		$(logr) "GPG_KEY not set, no key to import"
+	fi
+
+# encrypts a dummy file so that it doesnt ask again for passphrase when sops is run
+# this is only needed if using a private key that has passphrase
+gpg.passphrase:
+	if [ "$(GPG_PASS)" ]; then
+		touch build/dummy.txt
+		# a bit remarkable that this is what it takes but it is.
+		echo $(GPG_PASS) | gpg -q --sign --batch --pinentry-mode loopback --passphrase-fd 0 --output /dev/null --yes build/dummy.txt
+	fi
+
+# gpg above --batch doesn't ask for prompt and -v is verbose