From e0f92412b84d112178353569da98aaf82915f714 Mon Sep 17 00:00:00 2001 From: Joshua B Date: Sun, 26 Sep 2021 09:55:49 -0600 Subject: [PATCH] add sops.make for mozillas sops encryption. cleaned up comments (#24) * added a new sops.make for mozillas sops. * added sops decrypt to tests * test for sops * get test in for vault.decrypt * change builder base * fix gpg import and base docker * minor readme tweak. --- .circleci/config.yml | 3 +- .env.example | 20 ++++----- .sops.yaml | 7 ++++ Makefile | 11 ++--- README.md | 2 +- bin/circle | 1 - bin/dotenv | 2 +- bin/init_env | 4 +- bin/setVar | 3 +- docs/Makefiles.md | 12 ------ makefiles/Shipkit-main.make | 19 +++++---- makefiles/vault.make | 81 +++++++++++++++++++++++++++++++++++++ 12 files changed, 121 insertions(+), 44 deletions(-) create mode 100644 .sops.yaml create mode 100644 makefiles/vault.make diff --git a/.circleci/config.yml b/.circleci/config.yml index 2b4507d..74fb6b8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,13 +2,14 @@ version: 2.1 jobs: build: docker: - - image: yakworks/builder:bash-make + - image: yakworks/builder:k8s steps: - checkout - run: make install-shellcheck-alpine - run: make install-file-alpine - run: make check # test some other stuff + - run: make vault.decrypt-files - run: make git.config-bot-user # if it meets the criteria to be IS_RELEASABLE, this will bump version and tag release on github - run: make ship.version diff --git a/.env.example b/.env.example index 9991273..7448194 100644 --- a/.env.example +++ b/.env.example @@ -1,22 +1,16 @@ # NEVER CHECK IN -# dotenv is for user environment for testing and development -# below are some options for override -# for a list of what your make build is currently using run `make log-vars` +# Copy and rename this to a .env to override variables for testing and development +# Copy to a secrets.env if data is sensitive +# `make log-vars` shows what is defined in .env and will not show whats in .secrets.env -# if not storing GITHUB_TOKEN in bash ENV then set it here +# if not storing GITHUB_TOKEN in bash ENV then set it in .secrects.env # GITHUB_TOKEN=gp123l4j1234lkj - -# Some releasing targets depend on ACTIVE_BRANCH being set, which is normally only set when its a CI -# these can help with simulation or manual deploys -# Set this to a "releasable" branch and some targets such as publish-release or kube.deploy will be active -# ACTIVE_BRANCH=master +# MAVEN_REPO_KEY = xxx +# MAVEN_REPO_USER = yyy # to test some of the CIRCLE targets # CIRCLECI=true # CIRCLE_BRANCH=master +# this is the org # CIRCLE_PROJECT_USERNAME=yakworks # CIRCLE_PROJECT_REPONAME=gorm-tools - -# `make log-vars` should show this -# dummy, should show up in log-vars -FOO=bar diff --git a/.sops.yaml b/.sops.yaml new file mode 100644 index 0000000..e11990e --- /dev/null +++ b/.sops.yaml @@ -0,0 +1,7 @@ +# josh 54B2, joanna FC1B, 9cibot C575, Ken EFC9 +creation_rules: + - pgp: >- + 54B28047704C36FDCB30E63B472AC57FC3EB83C8, + C5758A15DF91198CF5347A90F8E8B460302AAEFA, + FC1B46ABD5AE6A99E31165E42DCB557950992300, + EFC9CC2C54F7387675F435F4D51CED1BA3F5E95E diff --git a/Makefile b/Makefile index 46b4704..ba288ce 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,14 @@ -# tell shipkit where the main build.sh is, not used in test -# build.sh := ./build.sh + # If setting any vars needed for the $(shell init_vars..) in Shipkit.make then track the MAKE_VARS so those dont get added -# if not git is installed then need to add the PROJECT_FULLNAME, much easier to add a build.sh and do it there +# if not git is installed then need to add the PROJECT_FULLNAME, much easier to add a build.yml and do it there # export PROJECT_FULLNAME = yakworks/shipkit # BUILD_VARS = PROJECT_FULLNAME # need this in order for it to build what vars get passed the $(shell) # core include, creates the makefile.env for the BUILD_VARS that evrything else depends on include Shipkit.make include $(SHIPKIT_MAKEFILES)/base-build.make include $(SHIPKIT_MAKEFILES)/docker.make -include $(SHIPKIT_MAKEFILES)/secrets.make +# include $(SHIPKIT_MAKEFILES)/secrets.make +include $(SHIPKIT_MAKEFILES)/vault.make include $(SHIPKIT_MAKEFILES)/git-tools.make include $(SHIPKIT_MAKEFILES)/ship-version.make include $(SHIPKIT_MAKEFILES)/circle.make @@ -16,12 +16,13 @@ include $(SHIPKIT_MAKEFILES)/bats-testing.make # -- Variables --- export BOT_EMAIL ?= 9cibot@9ci.com +export VAULT_URL = https://github.com/9ci/vault.git # can be set here but best do it on command line with make # export LOGIT_DEBUG_ENABLED := true # --- Dockers --- docker_tools := $(SHIPKIT_BIN)/docker_tools -DOCK_SHELL_URL = yakworks/builder:bash-make +DOCK_SHELL_URL = yakworks/builder:base ## docker shell for testing docker.shell: diff --git a/README.md b/README.md index 50c83ca..aa0c631 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ We have normalized on a set of `Bash` scripts and `Make` targets for all of our ## Why Bash? -Yes we need it. Its pretty much every where. And when using a small alpine docker image, adding bash only adds about 2.5 mb +Yes we need it. Its pretty much every where. And when using a small alpine docker image, adding bash only adds about 800k compressed, and 2.2 mb uncompressed. | Image | size | uncompressed | |--------------------------------------------------------------|--------:|-------------:| diff --git a/bin/circle b/bin/circle index e2b8e96..1e45c22 100755 --- a/bin/circle +++ b/bin/circle @@ -1,7 +1,6 @@ #!/usr/bin/env bash # --- # working with CI circle and publishing, -# should be imported into the main build.sh # --- set -euo pipefail # strict mode https://bit.ly/36MvF0T source "$(dirname "${BASH_SOURCE[0]}")"/core/main diff --git a/bin/dotenv b/bin/dotenv index 0b403e6..88eac4f 100755 --- a/bin/dotenv +++ b/bin/dotenv @@ -2,7 +2,7 @@ # shellcheck disable=SC2034 # appears unused, many are used in templates # --- # default functions to setup BUILD_VARS -# BUILD_VARS are used in the build.sh and a makefile.env is created +# BUILD_VARS are used to create a makefile.env # that is imcluded at the start of the Makefile to share vars # --- set -eo pipefail # strict mode https://bit.ly/36MvF0T diff --git a/bin/init_env b/bin/init_env index ab4d5d0..846247d 100755 --- a/bin/init_env +++ b/bin/init_env @@ -17,7 +17,7 @@ core.import "setVar" # version.properties should be in root dir if [ -e "version.properties" ]; then source version.properties; fi -# main make env, this is called early shipkit or can be overriden in build.sh. +# main make env, this is called early. # $1 - BUILD_ENV (test, dev, seed) function make_env { make_env_init "${1:-}" @@ -53,7 +53,7 @@ function init_env { load_env ".env" # if build/vault/bot.env has been cloned this will import the vars - parse_bot_env_file + # parse_bot_env_file # check for a build.env load_env "build.env" diff --git a/bin/setVar b/bin/setVar index 3f7d3f7..8bbbffb 100755 --- a/bin/setVar +++ b/bin/setVar @@ -3,6 +3,7 @@ # The original purpose here was to be able to set a variable that will be tracked and can then be # eported back out to a file can be created and included into Make set -eo pipefail # strict mode https://bit.ly/36MvF0T +core.import "logit" # sets the variable value if its not already set and adds it for tracking in BUILD_VARS # args $1 - the variable name @@ -14,7 +15,7 @@ function setVar { # curVal="${!varName}" # [[ ! ${!varName+x} ]] if [[ -z ${!varName} ]]; then - # echo "curVal is empty, adding" + # logit.info "$varName=\"\$2\"" eval "$varName=\"\$2\"" fi add_build_vars "$varName" diff --git a/docs/Makefiles.md b/docs/Makefiles.md index 54b78fb..beefecb 100644 --- a/docs/Makefiles.md +++ b/docs/Makefiles.md @@ -56,16 +56,4 @@ A target with a variable right after it like `foo : BAR = true` - passing through `$` variable references will require a double `$$` to escape it, without it make will think its and internal reference -- each line in a target runs in a new shell. To run them together its recomended to create a - function in the build.sh and call that as its can get a little ugly in `make`. - if you need to do it in make then each exec should end - with `;` and a `\` if its multiline. for example (note if you copy this example make sure to convert indent spaces to tabs): - ``` - foo: ## shows foos - @FOOS="buzz bar bazz"; \ - for f in $$FOOS; do \ - echo "$$f"; \ - done; - ``` - - `make` can call it self using the `$(MAKE)` built in. so to call a build target `$(MAKE) build` can be used diff --git a/makefiles/Shipkit-main.make b/makefiles/Shipkit-main.make index f1e7091..60f55ea 100644 --- a/makefiles/Shipkit-main.make +++ b/makefiles/Shipkit-main.make @@ -13,12 +13,12 @@ export MAKE_ENV_FILE ?= $(BUILD_DIR)/make/makefile$(MAKELEVEL).env SHELL_VARS += VERBOSE_LOG BUILD_DIR MAKE_ENV_FILE DBMS env dry_run #shell doesn't get the exported vars so we need to spin the ones we want, which should be in BUILD_VARS SHELL_EXPORTS := $(foreach v,$(SHELL_VARS), $(v)='$($(v))') -# if no build.sh var is not set then use the the init_env script directly -# if its set then call build.sh and assume it sourced in /init_env and will +# if no init_env.sh var is not set then use the the init_env script directly +# if its set then call init_env.sh and assume it sourced in /init_env and will # be setting up variables and/or potentially overriding make_env -build.sh ?= $(SHIPKIT_BIN)/init_env +init_env.sh ?= $(SHIPKIT_BIN)/init_env # we do this in subshell so it forces the file to be generated before the sindlues happens -SH_RESULTS := $(shell $(SHELL_EXPORTS) $(build.sh) make_env $(BUILD_ENV)) +SH_RESULTS := $(shell $(SHELL_EXPORTS) $(init_env.sh) make_env $(BUILD_ENV)) ifneq ($(.SHELLSTATUS),0) $(error init_env error -> $(SH_RESULTS)) endif @@ -70,8 +70,13 @@ FORCE: log-vars: FORCE printf "$(ccyan)dry_run$(creset) = $(dry_run)\n" printf "$(culine)Variable:\n\n$(creset)" - for v in $(sort $(BUILD_VARS)); do - printf "$(ccyan)$$v $(creset)=$(cbold) $${!v:-} $(creset)\n" + printf "$(ccyan)VAULT_ENV_VARS $(creset)= $(cbold) $(VAULT_ENV_VARS) $(creset)\n" + for varName in $(sort $(BUILD_VARS)); do + varVal=$${!varName:-} + if [[ $${varName^^} =~ TOKEN|PASSWORD|GPG.*KEY|REPO.*KEY ]]; then + varVal="*********" + fi + printf "$(ccyan)$$varName $(creset)=$(cbold) "$$varVal"$(creset)\n" done; log-make-vars: FORCE @@ -215,7 +220,7 @@ endef # OS ?= ifeq (Windows_NT,$(OS)) -OS_NAME := Windows +OS_NAME := windows OS_CPU := $(call _lower,$(PROCESSOR_ARCHITECTURE)) OS_ARCH := $(if $(findstring amd64,$(OS_CPU)),x86_64,i686) else diff --git a/makefiles/vault.make b/makefiles/vault.make new file mode 100644 index 0000000..2fe4615 --- /dev/null +++ b/makefiles/vault.make @@ -0,0 +1,81 @@ +# Opinionated process to clone and decrypt a github project use as a "vault +# installs sops if not present on OS, will not be for build dockers, for local dev should already be there +# clones the github repo in VAULT_DIR, we call it the vault + +SOP_VERSION := 3.7.1 +SOP_URL := https://github.com/mozilla/sops/releases/download/v$(SOP_VERSION)/sops-v$(SOP_VERSION).linux +VAULT_DIR ?= $(BUILD_DIR)/vault +VAULT_FILES ?= bot.enc.env +VAULT_BOT_ENV_FILE := $(VAULT_DIR)/bot.env + +# --- look for build/vault/bot.env , run sops.decrypt-vault-files -- +# we import it straight into make since these are secrets, dont want them in BUILD_VARS where they can get logged + +ifneq ($(wildcard $(VAULT_BOT_ENV_FILE)),) + include $(VAULT_BOT_ENV_FILE) + VAULT_ENV_VARS := $(shell sed '/^#.*/d; /^$$/d; s/=.*//g;' $(VAULT_BOT_ENV_FILE)) + # $(info including $(VAULT_ENV_VARS) from $(VAULT_ENV_FILE)) + export $(VAULT_ENV_VARS) +endif +# --- + +SOP_SH := $(shell which sops 2> /dev/null) + +# if doesn't already exists then above will be empty +ifeq ($(SOP_SH),) + + SOP_SH := $(SHIPKIT_INSTALLS)/sops + # $(info sops is NOT installed) + +endif + +# on demand clone and install of git-secret for build dockers +# as a make reminder, if the file ref SOP_SH doesn't exist then this runs, if its there already then this does nothing +$(SOP_SH): + $(logr) "intalling sops $(SOP_URL)" + # make sure installs is created + mkdir -p $(SHIPKIT_INSTALLS) + curl -qsL $(SOP_URL) -o $(SOP_SH) + chmod +x $(SOP_SH) + $(logr.done) + +# easier for testing +sops.install: $(SOP_SH) + +vault.clone: | _verify_VAULT_URL + [ ! -e $(VAULT_DIR) ] && git clone $(VAULT_URL) $(VAULT_DIR) || :; + +vault.decrypt-files: $(SOP_SH) gpg.import-private-key vault.clone + cd $(VAULT_DIR) + for vfile in $(VAULT_FILES); do + outFile="$${vfile/.enc./.}" # remove .enc. + outFile="$${outFile/.encrypted./.}" # remove .encrypted. + $(logr) "$$vfile > $$outFile" + $(SOP_SH) -d $$vfile > $$outFile + done + $(logr.done) + +# to test the gpg stuff set this to the base64 encoded key, DO NOT CHECK IN +# GPG_KEY=ZZZ +# set this to the gpg passphrase if needed, not base64 +# GPG_PASS := xxx + +# imports private key from GPG_PRIVATE_KEY var +gpg.import-private-key: + if [ "$(GPG_KEY)" ]; then + echo "$(GPG_KEY)" | base64 --decode | gpg -v --batch --import --quiet --no-verbose + $(logr) "GPG_KEY imported" + else + $(logr) "GPG_KEY not set, no key to import" + fi + +# encrypts a dummy file so that it doesnt ask again for passphrase when sops is run +# this is only needed if using a private key that has passphrase +gpg.passphrase: + if [ "$(GPG_PASS)" ]; then + touch build/dummy.txt + # a bit remarkable that this is what it takes but it is. + echo $(GPG_PASS) | gpg -q --sign --batch --pinentry-mode loopback --passphrase-fd 0 --output /dev/null --yes build/dummy.txt + fi + +# gpg above --batch doesn't ask for prompt and -v is verbose