From ea5ae9ce0393b576eca05bc7b2533937afcf22a7 Mon Sep 17 00:00:00 2001 From: Laurent Demailly Date: Sat, 24 Feb 2024 21:54:49 -0800 Subject: [PATCH] adding probes... except they don't work as promised --- .gitignore | 1 + Makefile | 19 ++++++- chart/Chart.yaml | 2 +- chart/templates/02_config.yaml | 2 +- chart/templates/03_memstore.yaml | 19 +++++++ chart/templates/04_svc.yaml | 17 ++++++- go.mod | 4 ++ go.sum | 8 +++ mstore/mstore.go | 2 +- probes/probes.go | 85 ++++++++++++++++++++++++++++++++ proto.go | 16 ++++++ 11 files changed, 168 insertions(+), 7 deletions(-) create mode 100644 probes/probes.go diff --git a/.gitignore b/.gitignore index 32f1387..371f03c 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ # vendor/ .vscode memstore +memstore-*.tgz diff --git a/Makefile b/Makefile index ea7e7ce..3eef482 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ test: # In CI it takes quite a while for the go process to run # prime it by compiling/running: go run . help - ( sleep 10; pkill -int memstore) & # make sure this happens even if the curl below fails + ( sleep 10; pkill -term memstore) & # make sure this happens even if the curl below fails ( sleep 4; curl -G http://localhost:7999/set \ --data-urlencode name=peers \ --data-urlencode value=c,b,e,f ; echo) & @@ -12,11 +12,26 @@ test: go run . -peers a,b,c -config-port 7999 # Works with docker-desktop for instance: + +LOCAL_HELM_OVERRIDES:=--set image.pullPolicy=Never --set debug=true +HELM:=helm +CHART_NAME:=memstore +CHART_DIR:=chart/ +HELM_INSTALL_ARGS:=upgrade --install $(CHART_NAME) $(CHART_DIR) $(LOCAL_HELM_OVERRIDES) + local-k8s: CGO_ENABLED=0 GOOS=linux go build -ldflags "-s -w" . -kubectl delete statefulset -n memstore memstore # so it'll reload the image docker buildx build --load --tag fortio/memstore:latest . - helm upgrade --install memstore chart/ --set image.pullPolicy=Never --set debug=true + $(HELM) $(HELM_INSTALL_ARGS) + +# Needs helm plugin install https://github.com/databus23/helm-diff +local-diff: + $(HELM) diff $(HELM_INSTALL_ARGS) + +# Logs of first pod, colorized with logc (go install fortio.org/logc@latest) +tail-log: + kubectl logs -f -n memstore memstore-0 | logc debug-pod: kubectl run debug --image=ubuntu --restart=Never -- /bin/sleep infinity diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 195b9e0..6d2f49b 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: memstore version: 0.1.0 -description: A Helm chart for deploying the memstore application +description: A Helm chart for deploying the memstore service keywords: - memstore - helm diff --git a/chart/templates/02_config.yaml b/chart/templates/02_config.yaml index ba19782..1c11cad 100644 --- a/chart/templates/02_config.yaml +++ b/chart/templates/02_config.yaml @@ -6,6 +6,6 @@ metadata: data: loglevel: debug # peers: memstore-0,memstore-1,memstore-2 - dns: memstore.memstore.svc.cluster.local + dns: memstore-internal.memstore.svc.cluster.local dns-interval: 5s statefulset: "true" diff --git a/chart/templates/03_memstore.yaml b/chart/templates/03_memstore.yaml index 0879ae5..5a1936b 100644 --- a/chart/templates/03_memstore.yaml +++ b/chart/templates/03_memstore.yaml @@ -27,6 +27,25 @@ spec: volumeMounts: - name: config-volume mountPath: /etc/memstore + startupProbe: + httpGet: + path: /startup + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 10 + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 15 + periodSeconds: 20 volumes: - name: config-volume configMap: diff --git a/chart/templates/04_svc.yaml b/chart/templates/04_svc.yaml index 3c11566..a26670a 100644 --- a/chart/templates/04_svc.yaml +++ b/chart/templates/04_svc.yaml @@ -3,12 +3,25 @@ apiVersion: v1 kind: Service metadata: - name: memstore + name: memstore-internal namespace: memstore spec: ports: - port: 8080 name: http2 - clusterIP: None # Headless + clusterIP: None # Headless - internal use to get all the pod IPs individually irrespective of readiness + selector: + app: memstore +--- +apiVersion: v1 +kind: Service +metadata: + name: memstore + namespace: memstore +spec: selector: app: memstore + ports: + - name: http2 + port: 8080 + type: ClusterIP diff --git a/go.mod b/go.mod index f3638c0..88e908d 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.18 require ( fortio.org/dflag v1.7.0 + fortio.org/fortio v1.63.3 fortio.org/log v1.12.0 fortio.org/scli v1.14.1 fortio.org/sets v1.0.3 @@ -16,6 +17,9 @@ require ( fortio.org/struct2env v0.4.0 // indirect fortio.org/version v1.0.3 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 // indirect + golang.org/x/net v0.21.0 // indirect golang.org/x/sys v0.17.0 // indirect + golang.org/x/text v0.14.0 // indirect ) diff --git a/go.sum b/go.sum index e0ec60e..5763a76 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,8 @@ fortio.org/cli v1.5.1 h1:lqPvkxRVSajsVwLfblaN62BPAICPp05Oab+yjRvI3DU= fortio.org/cli v1.5.1/go.mod h1:Tp7AypudP1mJomTUN/J/vlOTlZDWTMsok09MMyA99ow= fortio.org/dflag v1.7.0 h1:4Vpo5hMly0rx9VMuyBaDGFK1Mx2S3qjxx1iAIA3KBgU= fortio.org/dflag v1.7.0/go.mod h1:FUxv/s3DXhCpy7GsuZa4FJWLR92gsYvG3ylkia8MbBM= +fortio.org/fortio v1.63.3 h1:t4FoQ70znmYEEST3eyMLDqGaoNJPUDZNvoPRRH0WsaQ= +fortio.org/fortio v1.63.3/go.mod h1:HFYGCHKrxS+Yuuw/7gcO7hhsEvrKt6t7sh1Xkn/kggw= fortio.org/log v1.12.0 h1:5Yg4pL9Pp0jcWeJYixm2xikMCldVaSDMgDFDmQJZfho= fortio.org/log v1.12.0/go.mod h1:1tMBG/Elr6YqjmJCWiejJp2FPvXg7/9UAN0Rfpkyt1o= fortio.org/scli v1.14.1 h1:MLxVRfuCXU8AIhHbWRj3JGE0uzTSQ/cWmcKdpuF55qk= @@ -15,7 +17,13 @@ fortio.org/version v1.0.3 h1:5gJ3plj6isAOMq52cI5ifo4cC+QHmJF76Wevc5Cp4x0= fortio.org/version v1.0.3/go.mod h1:2JQp9Ax+tm6QKiGuzR5nJY63kFeANcgrZ0osoQFDVm0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 h1:/RIbNt/Zr7rVhIkQhooTxCxFcdWLGIKnZA4IXNFSrvo= golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= diff --git a/mstore/mstore.go b/mstore/mstore.go index a8e228b..d1934e4 100644 --- a/mstore/mstore.go +++ b/mstore/mstore.go @@ -47,7 +47,7 @@ func dnsChange(oldValue, newValue string) { } func Start(name string) { - log.Infof("memstore Start() name is %q", name) + log.Infof("memstore starting with name %q", name) myName = name // peerChange does get call for even initial flag value } diff --git a/probes/probes.go b/probes/probes.go new file mode 100644 index 0000000..ff38a2e --- /dev/null +++ b/probes/probes.go @@ -0,0 +1,85 @@ +// Implement kubernetes probes. +package probes + +import ( + "net/http" + "sync" + + "fortio.org/log" +) + +type state struct { + started bool + live bool + ready bool + // mutex + mutex sync.Mutex +} + +var State = state{} + +func (s *state) SetLive(live bool) { + s.mutex.Lock() + s.live = live + s.mutex.Unlock() +} + +func (s *state) SetReady(ready bool) { + s.mutex.Lock() + s.ready = ready + s.mutex.Unlock() +} + +func (s *state) SetStarted(started bool) { + s.mutex.Lock() + s.started = started + s.mutex.Unlock() +} + +func (s *state) IsLive() bool { + s.mutex.Lock() + defer s.mutex.Unlock() + return s.live +} + +func (s *state) IsReady() bool { + s.mutex.Lock() + defer s.mutex.Unlock() + return s.ready +} + +func (s *state) IsStarted() bool { + s.mutex.Lock() + defer s.mutex.Unlock() + return s.started +} + +func StartupProbe(w http.ResponseWriter, _ *http.Request) { + if State.IsStarted() { + w.WriteHeader(http.StatusOK) + } else { + w.WriteHeader(http.StatusServiceUnavailable) + } +} + +func LivenessProbe(w http.ResponseWriter, _ *http.Request) { + if State.IsLive() { + w.WriteHeader(http.StatusOK) + } else { + w.WriteHeader(http.StatusServiceUnavailable) + } +} + +func ReadinessProbe(w http.ResponseWriter, _ *http.Request) { + if State.IsReady() { + w.WriteHeader(http.StatusOK) + } else { + w.WriteHeader(http.StatusServiceUnavailable) + } +} + +func Setup(mux *http.ServeMux) { + mux.HandleFunc("/startup", log.LogAndCall("startup", StartupProbe)) + mux.HandleFunc("/ready", log.LogAndCall("readiness", ReadinessProbe)) + mux.HandleFunc("/health", log.LogAndCall("liveness", LivenessProbe)) +} diff --git a/proto.go b/proto.go index dfad89b..6fa2db4 100644 --- a/proto.go +++ b/proto.go @@ -5,15 +5,20 @@ package main import ( + "flag" "os" + "time" "fortio.org/dflag" + "fortio.org/fortio/fhttp" "fortio.org/log" "fortio.org/memstore/mstore" + "fortio.org/memstore/probes" "fortio.org/scli" ) func main() { + port := flag.String("port", "8080", "Port to listen on") dflag.Flag("peers", mstore.Peers) dflag.Flag("dns", mstore.DNSWatch) dflag.Flag("dns-interval", mstore.DNSWatchSleepTime) @@ -30,6 +35,17 @@ func main() { log.Fatalf("No NAME env var found for statefulset mode (to this pod's name)") } mstore.Start(myName) + mux, addr := fhttp.HTTPServer("memstore", *port) + if addr == nil { + log.Fatalf("Failed to start http server") + } + probes.Setup(mux) + probes.State.SetLive(true) + probes.State.SetStarted(true) + probes.State.SetReady(true) + time.Sleep(50 * time.Second) // give time for the probes to be ready + log.Warnf("Switching back to not ready") + probes.State.SetReady(false) scli.UntilInterrupted() mstore.Stop() }