merged devons work

Dockerfile

@@ -24,7 +24,7 @@
 
 ### build-base stage ###
 # Build base just has the packages installed we need.
-FROM artifactory.algol60.net/docker.io/library/golang:1.16-alpine AS build-base
+FROM golang:1.16-alpine AS build-base
 
 RUN set -ex \
     && apk -U upgrade \
@@ -39,7 +39,6 @@ RUN go env -w GO111MODULE=auto
 # Copy all the necessary files to the image.
 COPY cmd $GOPATH/src/github.com/Cray-HPE/hms-bss/cmd
 COPY pkg $GOPATH/src/github.com/Cray-HPE/hms-bss/pkg
-COPY vendor $GOPATH/src/github.com/Cray-HPE/hms-bss/vendor
 COPY .version $GOPATH/src/github.com/Cray-HPE/hms-bss/.version
 
 ### Build Stage ###
@@ -48,7 +47,7 @@ FROM base AS builder
 RUN set -ex && go build -v -i -o /usr/local/bin/boot-script-service github.com/Cray-HPE/hms-bss/cmd/boot-script-service
 
 ### Final Stage ###
-FROM artifactory.algol60.net/docker.io/alpine:3.15
+FROM alpine:3.15
 LABEL maintainer="Hewlett Packard Enterprise"
 EXPOSE 27778
 STOPSIGNAL SIGTERM
@@ -94,7 +93,7 @@ COPY --from=builder /usr/local/bin/boot-script-service /usr/local/bin/.
 
 COPY .version /
 
-# noboby 65534:65534
+# nobody 65534:65534
 USER 65534:65534
 
 # Set up the command to start the service, the run the init script.

test/ct/Dockerfile → Dockerfile.bss-postgres-test.Dockerfile

@@ -1,6 +1,6 @@
 # MIT License
 #
-# (C) Copyright [2020-2022] Hewlett Packard Enterprise Development LP
+# (C) Copyright [2021] Hewlett Packard Enterprise Development LP
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -20,20 +20,16 @@
 # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 # OTHER DEALINGS IN THE SOFTWARE.
 
-FROM artifactory.algol60.net/csm-docker/stable/hms-test:5.0.0
+# Dockerfile for running postgres integration tests.
 
-COPY smoke/ /src/app
-COPY api/ /src/app/api
-COPY tavern_global_config_ct_test.yaml /src/app/tavern_global_config_ct_test.yaml
-COPY bss_test_utils.py /src/libs/bss_test_utils.py
+### build-base stage ###
+# Build base just has the packages installed we need.
+FROM ghcr.io/orange-opensource/hurl:latest
 
-ENV PATH="/src/libs:${PATH}"
-ENV PATH="/src/app:${PATH}"
-ENV PYTHONPATH=$PYTHONPATH:/src/libs 
+# Copy all the necessary files to the image.
+COPY test/ct/postgres /postgres
 
-USER root
-RUN chown -R 65534:65534 /src
-USER 65534:65534
+WORKDIR /postgres
 
-# this is inherited from the hms-test container
-ENTRYPOINT [ "entrypoint.sh" ]
+# Run unit tests.
+ENTRYPOINT ["/postgres/run.sh"]

Dockerfile.bss-postgres.Dockerfile

@@ -0,0 +1,91 @@
+# MIT License
+#
+# (C) Copyright [2018-2021] Hewlett Packard Enterprise Development LP
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+
+# Dockerfile for building HMS bss.
+
+### build-base stage ###
+# Build base just has the packages installed we need.
+FROM golang:1.16-alpine AS build-base
+
+RUN set -ex \
+    && apk -U upgrade \
+    && apk add build-base
+
+### base stage ###
+# Base copies in the files we need to test/build.
+FROM build-base AS base
+
+# Copy all the necessary files to the image.
+COPY go.mod /hms-bss/go.mod
+COPY go.sum /hms-bss/go.sum
+COPY cmd /hms-bss/cmd
+COPY pkg /hms-bss/pkg
+COPY internal /hms-bss/internal
+COPY .version /hms-bss/.version
+
+### Build Stage ###
+FROM base AS builder
+
+RUN set -ex \
+    && cd /hms-bss/cmd/boot-script-service \
+    && go build -v -o /usr/local/bin/boot-script-service
+
+### Final Stage ###
+FROM alpine:3.15
+LABEL maintainer="Hewlett Packard Enterprise"
+EXPOSE 27778
+STOPSIGNAL SIGTERM
+
+# Setup environment variables.
+ENV HSM_URL=http://smd:27779
+ENV NFD_URL=http://cray-hmnfd
+
+# WARNING: Our containers currently do not have certificates set up correctly
+#          to allow for https connections to other containers.  Therefore, we
+#          will use an insecure connection.  This needs to be corrected before
+#          release.  Once the certificates are properly set up, the --insecure
+#          option needs to be removed.
+ENV BSS_OPTS="--insecure --postgres-insecure"
+
+ENV BSS_RETRY_DELAY=30
+ENV BSS_HSM_RETRIEVAL_DELAY=10
+
+# Other potentially useful env variables:
+# BSS_IPXE_SERVER defaults to "api-gw-service-nmn.local"
+# BSS_CHAIN_PROTO defaults to "https"
+# BSS_GW_URI defaults to "/apis/bss"
+
+# Include curl in the final image.
+RUN set -ex \
+    && apk -U upgrade \
+    && apk add --no-cache curl
+
+# Get the boot-script-service from the builder stage.
+COPY --from=builder /usr/local/bin/boot-script-service /usr/local/bin/.
+
+COPY .version /
+
+# nobody 65534:65534
+USER 65534:65534
+
+# Set up the command to start the service, the run the init script.
+CMD boot-script-service $BSS_OPTS --cloud-init-address localhost --hsm=$HSM_URL --postgres --postgres-host $POSTGRES_HOST --postgres-port $POSTGRES_PORT --retry-delay=$BSS_RETRY_DELAY --hsm $HSM_URL --hsm-retrieval-delay=$BSS_HSM_RETRIEVAL_DELAY

cmd/boot-script-service/boot_data.go

@@ -238,6 +238,19 @@ func nidName(nid int) string {
 func Remove(bp bssTypes.BootParams) error {
 	debugf("Remove(): Ready to remove %v\n", bp)
 	var err error
+	if useSQL {
+		var (
+			nodesDeleted []string
+			bcsDeleted   []string
+		)
+		nodesDeleted, bcsDeleted, err = bssdb.Delete(bp)
+		if err != nil {
+			return err
+		}
+		debugf("Node IDs deleted: %v", nodesDeleted)
+		debugf("Boot Config IDs deleted: %v", bcsDeleted)
+		return err
+	}
 	for _, h := range bp.Hosts {
 		e := removeHost(h)
 		if err == nil {
@@ -340,6 +353,12 @@ func extractParamName(x hmetcd.Kvi_KV) (ret string) {
 }
 
 func StoreNew(bp bssTypes.BootParams) (error, string) {
+	// postgres.Add will handle duplicates, and it is called in New().
+	// Therefore, if Postgres is enabled, simply call Store().
+	if useSQL {
+		return Store(bp)
+	}
+
 	item := ""
 	// Go through the entire struct.  We must be storing to new hosts or this
 	// request must fail.
@@ -393,6 +412,16 @@ func StoreNew(bp bssTypes.BootParams) (error, string) {
 func Store(bp bssTypes.BootParams) (error, string) {
 	debugf("Store(%v)\n", bp)
 
+	if useSQL {
+		debugf("postgres.Add(%v)\n", bp)
+		result, err := bssdb.Add(bp)
+		if err != nil {
+			return err, ""
+		}
+		debugf("postgres.Add result: %v\n", result)
+		return err, uuid.New().String()
+	}
+
 	var kernel_id, initrd_id string
 	if bp.Kernel != "" {
 		kernel_id = imageStore(bp.Kernel, kernelImageType)
@@ -925,6 +954,27 @@ func LookupByName(name string) (BootData, SMComponent) {
 		comp_name = comp.ID
 		role = comp.Role
 	}
+	if useSQL {
+		var result BootData
+		bps, err := bssdb.GetBootParamsByName([]string{name})
+		if err != nil {
+			err = fmt.Errorf("Could not retrieve boot parameters with name %q: %v", name, err)
+			log.Printf("ERROR: %v", err)
+			return result, comp
+		}
+		if len(bps) == 0 {
+			// Not found.
+			log.Printf("WARNING: Name %q did not return any results.", name)
+			return result, comp
+		} else if len(bps) > 1 {
+			debugf("BootParams returned: %v", bps)
+			log.Printf("WARNING: More than 1 node found for name %q, taking first one: %v", name, bps[0])
+		}
+		result.Kernel = ImageData{bps[0].Kernel, ""}
+		result.Initrd = ImageData{bps[0].Initrd, ""}
+		result.Params = bps[0].Params
+		return result, comp
+	}
 	return lookup(comp_name, name, role, DefaultTag), comp
 }
 
@@ -936,6 +986,27 @@ func LookupByMAC(mac string) (BootData, SMComponent) {
 		comp_name = comp.ID
 		role = comp.Role
 	}
+	if useSQL {
+		var result BootData
+		bps, err := bssdb.GetBootParamsByMac([]string{mac})
+		if err != nil {
+			err = fmt.Errorf("Could not retrieve boot parameters with mac %q: %v", mac, err)
+			log.Printf("ERROR: %v", err)
+			return result, comp
+		}
+		if len(bps) == 0 {
+			// Not found.
+			log.Printf("WARNING: MAC %q did not return any results.", mac)
+			return result, comp
+		} else if len(bps) > 1 {
+			debugf("BootParams returned: %v", bps)
+			log.Printf("WARNING: More than 1 node found for MAC %q, taking first one: %v", mac, bps[0])
+		}
+		result.Kernel = ImageData{bps[0].Kernel, ""}
+		result.Initrd = ImageData{bps[0].Initrd, ""}
+		result.Params = bps[0].Params
+		return result, comp
+	}
 	return lookup(comp_name, mac, role, DefaultTag), comp
 }
 
@@ -948,6 +1019,27 @@ func LookupByNid(nid int) (BootData, SMComponent) {
 		comp_name = comp.ID
 		role = comp.Role
 	}
+	if useSQL {
+		var result BootData
+		bps, err := bssdb.GetBootParamsByNid([]int32{int32(nid)})
+		if err != nil {
+			err = fmt.Errorf("Could not retrieve boot parameters with NID %d: %v", nid, err)
+			log.Printf("ERROR: %v", err)
+			return result, comp
+		}
+		if len(bps) == 0 {
+			// Not found.
+			log.Printf("WARNING: NID %d did not return any results.", nid)
+			return result, comp
+		} else if len(bps) > 1 {
+			debugf("BootParams returned: %v", bps)
+			log.Printf("WARNING: More than 1 node found for NID %d, taking first one: %v", nid, bps[0])
+		}
+		result.Kernel = ImageData{bps[0].Kernel, ""}
+		result.Initrd = ImageData{bps[0].Initrd, ""}
+		result.Params = bps[0].Params
+		return result, comp
+	}
 	return lookup(comp_name, nid_str, role, DefaultTag), comp
 }
 

cmd/boot-script-service/default_api.go

@@ -175,38 +175,55 @@ func checkURL(u string) (string, error) {
 
 func BootparametersGetAll(w http.ResponseWriter, r *http.Request) {
 	var results []bssTypes.BootParams
-	for _, image := range GetKernelInfo() {
-		var bp bssTypes.BootParams
-		bp.Params = image.Params
-		bp.Kernel = image.Path
-		results = append(results, bp)
-	}
-	for _, image := range GetInitrdInfo() {
-		var bp bssTypes.BootParams
-		bp.Params = image.Params
-		bp.Initrd = image.Path
-		results = append(results, bp)
-	}
-	var names []string
-	if kvl, e := getTags(); e == nil {
-		for _, x := range kvl {
-			name := extractParamName(x)
-			names = append(names, name)
-			var bds BootDataStore
-			e = json.Unmarshal([]byte(x.Value), &bds)
-			if e == nil {
-				bd := bdConvert(bds)
-				var bp bssTypes.BootParams
-				bp.Hosts = append(bp.Hosts, name)
-				bp.Params = bd.Params
-				bp.Kernel = bd.Kernel.Path
-				bp.Initrd = bd.Initrd.Path
-				bp.CloudInit = bd.CloudInit
-				results = append(results, bp)
+	if useSQL {
+		var (
+			err error
+			nodes []string
+		)
+		results, err = bssdb.GetBootParamsAll()
+		if err != nil {
+			log.Printf("Yikes, I couldn't retrieve boot parameters from Postgres: %v\n", err)
+		}
+		for _, bp := range results {
+			for _, node := range bp.Hosts {
+				nodes = append(nodes, node)
 			}
 		}
+		debugf("Retrieved boot configs for nodes: %v", nodes)
+	} else {
+		for _, image := range GetKernelInfo() {
+			var bp bssTypes.BootParams
+			bp.Params = image.Params
+			bp.Kernel = image.Path
+			results = append(results, bp)
+		}
+		for _, image := range GetInitrdInfo() {
+			var bp bssTypes.BootParams
+			bp.Params = image.Params
+			bp.Initrd = image.Path
+			results = append(results, bp)
+		}
+		var names []string
+		if kvl, e := getTags(); e == nil {
+			for _, x := range kvl {
+				name := extractParamName(x)
+				names = append(names, name)
+				var bds BootDataStore
+				e = json.Unmarshal([]byte(x.Value), &bds)
+				if e == nil {
+					bd := bdConvert(bds)
+					var bp bssTypes.BootParams
+					bp.Hosts = append(bp.Hosts, name)
+					bp.Params = bd.Params
+					bp.Kernel = bd.Kernel.Path
+					bp.Initrd = bd.Initrd.Path
+					bp.CloudInit = bd.CloudInit
+					results = append(results, bp)
+				}
+			}
+		}
+		debugf("Retrieved names: %v", names)
 	}
-	debugf("Retreived names: %v", names)
 	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
 	w.WriteHeader(http.StatusOK)
 	err := json.NewEncoder(w).Encode(results)