add hashi vault integration tests (#452)

* add hashi vault integration tests

* adding hashi vault tests

* forgot to add hashi cert destination dir

---------

Co-authored-by: stephengaudet <[email protected]>

.github/workflows/build.yaml

@@ -170,6 +170,7 @@ jobs:
          . ./.env.${{ matrix.testenvs }};
          envsubst < gcp-token-template.json > gcp-token.json;
          echo $VAULT_AZ_SP_KEY |base64 -d >service-principal.key;
+         chmod 777 ./hashicerts;
          docker compose up -d --wait --pull always;
          docker exec octez sudo chown -R tezos /home/tezos/.tezos-client;
          go test ./...;

integration_test/config.go

@@ -50,8 +50,8 @@ type TezosPolicy struct {
 }
 
 type VaultConfig struct {
-	Driver string             `yaml:"driver"`
-	Conf   map[string]*string `yaml:"config"`
+	Driver string                 `yaml:"driver"`
+	Conf   map[string]interface{} `yaml:"config"`
 }
 
 type FileVault struct {

integration_test/docker-compose.yml

@@ -1,6 +1,7 @@
 version: "3.9"
 networks:
   ecadnet: {}
+
 services:
 
   flextesa:
@@ -75,6 +76,7 @@ services:
     volumes:
       - ./.watermarks:/var/lib/signatory
       - ./coverage:/opt/coverage
+      - ./hashicerts:/opt/hashicerts
     configs:
       - source: sigy-config
         target: /etc/signatory.yaml
@@ -115,6 +117,24 @@ services:
       retries: 10
       start_period: 10s
 
+  hashi:
+    container_name: hashi
+    image: hashicorp/vault:1.14
+    ports:
+      - "8200:8200"
+    networks:
+      - ecadnet
+    environment:
+      - VAULT_DEV_ROOT_TOKEN_ID=root
+      - VAULT_TOKEN=root
+      - VAULT_ADDR=https://127.0.0.1:8200
+      - VAULT_CACERT=/opt/signatory/certs/vault-ca.pem
+    command: server -dev-tls -dev-tls-cert-dir=/opt/signatory/certs
+    volumes:
+      - ./hashicerts:/opt/signatory/certs
+    cap_add:
+      - IPC_LOCK      
+
 configs:
   sigy-config:
     file: ./signatory.yaml

integration_test/hashicerts/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

integration_test/vault_aws_test.go

@@ -26,7 +26,7 @@ func TestAWSVault(t *testing.T) {
 	c.Read()
 	var v VaultConfig
 	v.Driver = "awskms"
-	v.Conf = map[string]*string{"user_name": &user, "access_key_id": &key, "secret_access_key": &secret, "region": &region}
+	v.Conf = map[string]interface{}{"user_name": &user, "access_key_id": &key, "secret_access_key": &secret, "region": &region}
 	c.Vaults["aws"] = &v
 	var p TezosPolicy
 	p.LogPayloads = true

integration_test/vault_az_test.go

@@ -31,7 +31,7 @@ func TestAZVault(t *testing.T) {
 	c.Read()
 	var v VaultConfig
 	v.Driver = "azure"
-	v.Conf = map[string]*string{"vault": &vault, "tenant_id": &tenantid, "client_id": &clientid, "client_private_key": &spkey, "client_certificate_thumbprint": &thumb, "subscription_id": &subid, "resource_group": &resgroup}
+	v.Conf = map[string]interface{}{"vault": &vault, "tenant_id": &tenantid, "client_id": &clientid, "client_private_key": &spkey, "client_certificate_thumbprint": &thumb, "subscription_id": &subid, "resource_group": &resgroup}
 	c.Vaults["azure"] = &v
 	var p TezosPolicy
 	p.LogPayloads = true

integration_test/vault_gcp_test.go

@@ -22,7 +22,7 @@ func TestGCPVault(t *testing.T) {
 	c.Read()
 	var v VaultConfig
 	v.Driver = "cloudkms"
-	v.Conf = map[string]*string{"project": &project, "location": &location, "key_ring": &keyring}
+	v.Conf = map[string]interface{}{"project": &project, "location": &location, "key_ring": &keyring}
 	c.Vaults["gcp"] = &v
 	var p TezosPolicy
 	p.LogPayloads = true

integration_test/vault_hashi_test.go

@@ -0,0 +1,176 @@
+package integrationtest
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os/exec"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHashiVault(t *testing.T) {
+
+	roleid, secretid := hashiBootstrap()
+	cn := hashiGetValidCN()
+	address := "https://" + cn + ":8200"
+	mountPoint := "transit/"
+
+	var c Config
+	c.Read()
+	var v VaultConfig
+	v.Driver = "hashicorpvault"
+	v.Conf = map[string]interface{}{"address": &address, "roleID": &roleid, "secretID": &secretid, "transitConfig": map[string]string{"mountPoint": mountPoint}, "tlsCaCert": "/opt/hashicerts/vault-ca.pem"}
+	c.Vaults["hashicorp"] = &v
+	backup_then_update_config(c)
+	defer restore_config()
+
+	pkh := hashiGetTz1()
+	var p TezosPolicy
+	p.LogPayloads = true
+	p.Allow = map[string][]string{"generic": {"reveal", "transaction"}}
+	c.Tezos[pkh] = &p
+	c.Write()
+	restart_signatory()
+
+	out, err := OctezClient("import", "secret", "key", "hashitz1", "http://signatory:6732/"+pkh)
+	assert.NoError(t, err)
+	assert.Contains(t, string(out), "Tezos address added: "+pkh)
+	defer OctezClient("forget", "address", "hashitz1", "--force")
+
+	out, err = OctezClient("transfer", "100", "from", "alice", "to", "hashitz1", "--burn-cap", "0.06425")
+	assert.NoError(t, err)
+	require.Contains(t, string(out), "Operation successfully injected in the node")
+
+	out, err = OctezClient("transfer", "1", "from", "hashitz1", "to", "alice", "--burn-cap", "0.06425")
+	assert.NoError(t, err)
+	require.Contains(t, string(out), "Operation successfully injected in the node")
+
+	require.Contains(t, GetPublicKey(pkh), "edpk")
+}
+
+func hashiBootstrap() (roleId string, secretId string) {
+	body, _ := json.Marshal(map[string]string{"policy": "path \"transit/*\" { capabilities = [\"list\", \"read\", \"create\", \"update\"]}"})
+	code, res := hashiAPI(http.MethodPost, "sys/policy/transit-policy", body)
+	hashiCheckErr(code, "create transit-policy")
+
+	body, _ = json.Marshal(map[string]string{"type": "transit"})
+	code, res = hashiAPI(http.MethodPost, "sys/mounts/transit", body)
+	hashiCheckErr(code, "enable transit secrets engine")
+
+	body, _ = json.Marshal(map[string]string{"type": "ed25519"})
+	code, res = hashiAPI(http.MethodPost, "transit/keys/tz1key", body)
+	hashiCheckErr(code, "generate a key")
+
+	body, _ = json.Marshal(map[string]string{"type": "approle"})
+	code, res = hashiAPI(http.MethodPost, "sys/auth/approle", body)
+	hashiCheckErr(code, "enable auth method approle")
+
+	body, _ = json.Marshal(map[string]string{"secret_id_ttl": "0m", "token_ttl": "10m", "token_max_ttl": "20m", "token_policies": "transit-policy"})
+	code, res = hashiAPI(http.MethodPost, "auth/approle/role/my-approle", body)
+	hashiCheckErr(code, "create an approle")
+
+	code, res = hashiAPI(http.MethodGet, "auth/approle/role/my-approle/role-id", nil)
+	hashiCheckErr(code, "get the role id")
+	var rid HashiResponse
+	dec := json.NewDecoder(bytes.NewReader(res))
+	dec.Decode(&rid)
+	roleid := rid.Data["role_id"]
+
+	code, res = hashiAPI(http.MethodPost, "auth/approle/role/my-approle/secret-id", nil)
+	hashiCheckErr(code, "generate new secret id")
+	var sid HashiResponse
+	dec = json.NewDecoder(bytes.NewReader(res))
+	dec.Decode(&sid)
+	secretid := sid.Data["secret_id"]
+
+	return roleid, secretid
+}
+
+func hashiAPI(method string, path string, body []byte) (int, []byte) {
+	url := "https://127.0.0.1:8200/v1/" + path
+	reqbody := bytes.NewReader(body)
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	client := &http.Client{Transport: tr}
+	req, err := http.NewRequest(method, url, reqbody)
+	if err != nil {
+		panic(err)
+	}
+	req.Header.Add("X-Vault-Token", "root")
+	resp, err := client.Do(req)
+	if err != nil {
+		panic(err)
+	}
+	defer resp.Body.Close()
+	bytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		panic(err)
+	}
+	return resp.StatusCode, bytes
+}
+
+type HashiResponse struct {
+	Data map[string]string `json:"data"`
+}
+
+func hashiCheckErr(code int, message string) {
+	if !(code >= 200 && code < 300) {
+		panic("hashi config error: " + message + " : error code " + fmt.Sprint(code))
+	}
+}
+
+func hashiGetTz1() string {
+	out, err := SignatoryCli("list")
+	if err != nil {
+		panic("hashiGetTz1: signatory-cli returned an error: " + string(out))
+	}
+	var tz1 string
+	lines := strings.Split(string(out), "\n")
+	for _, line := range lines {
+		if strings.Contains(line, "tz1") {
+			fields := strings.Fields(line)
+			for _, field := range fields {
+				if strings.Contains(field, "tz1") {
+					tz1 = field
+				}
+			}
+		}
+		if strings.Contains(line, "HASHICORP_VAULT") {
+			return tz1
+		}
+	}
+	return "KEY_NOT_FOUND"
+}
+
+// the vault hostname in Signatory config file needs to match a CN in the SSL Cert
+func hashiGetValidCN() string {
+	//the cert autogenerated by hashi vault -dev-tls mode just so happens to include the container hash as a valid CN
+	var cmd = "docker"
+	var args = []string{"ps"}
+
+	out, err := exec.Command(cmd, args...).CombinedOutput()
+	if err != nil {
+		panic("hashiGetValidCN error: " + string(out))
+	}
+	var cn string
+	lines := strings.Split(string(out), "\n")
+	for _, line := range lines {
+		if strings.Contains(line, "hashi") {
+			fields := strings.Fields(line)
+			cn = fields[0]
+			break
+		}
+	}
+	if len(cn) < 12 {
+		panic("hashiGetValidCN: did not discover a valid CN: " + cn)
+	}
+	return cn
+}