Skip to content

Commit

Permalink
adding probes... except they don't work as promised
Browse files Browse the repository at this point in the history
  • Loading branch information
ldemailly committed Feb 25, 2024
1 parent 91cee27 commit ea5ae9c
Show file tree
Hide file tree
Showing 11 changed files with 168 additions and 7 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,4 @@
# vendor/
.vscode
memstore
memstore-*.tgz
19 changes: 17 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,34 @@ test:
# In CI it takes quite a while for the go process to run
# prime it by compiling/running:
go run . help
( sleep 10; pkill -int memstore) & # make sure this happens even if the curl below fails
( sleep 10; pkill -term memstore) & # make sure this happens even if the curl below fails
( sleep 4; curl -G http://localhost:7999/set \
--data-urlencode name=peers \
--data-urlencode value=c,b,e,f ; echo) &
@echo 'Expect to see: Success "peers" -> "c,b,e,f"'
go run . -peers a,b,c -config-port 7999

# Works with docker-desktop for instance:

LOCAL_HELM_OVERRIDES:=--set image.pullPolicy=Never --set debug=true
HELM:=helm
CHART_NAME:=memstore
CHART_DIR:=chart/
HELM_INSTALL_ARGS:=upgrade --install $(CHART_NAME) $(CHART_DIR) $(LOCAL_HELM_OVERRIDES)

local-k8s:
CGO_ENABLED=0 GOOS=linux go build -ldflags "-s -w" .
-kubectl delete statefulset -n memstore memstore # so it'll reload the image
docker buildx build --load --tag fortio/memstore:latest .
helm upgrade --install memstore chart/ --set image.pullPolicy=Never --set debug=true
$(HELM) $(HELM_INSTALL_ARGS)

# Needs helm plugin install https://github.com/databus23/helm-diff
local-diff:
$(HELM) diff $(HELM_INSTALL_ARGS)

# Logs of first pod, colorized with logc (go install fortio.org/logc@latest)
tail-log:
kubectl logs -f -n memstore memstore-0 | logc

debug-pod:
kubectl run debug --image=ubuntu --restart=Never -- /bin/sleep infinity
2 changes: 1 addition & 1 deletion chart/Chart.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
apiVersion: v2
name: memstore
version: 0.1.0
description: A Helm chart for deploying the memstore application
description: A Helm chart for deploying the memstore service
keywords:
- memstore
- helm
Expand Down
2 changes: 1 addition & 1 deletion chart/templates/02_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,6 @@ metadata:
data:
loglevel: debug
# peers: memstore-0,memstore-1,memstore-2
dns: memstore.memstore.svc.cluster.local
dns: memstore-internal.memstore.svc.cluster.local
dns-interval: 5s
statefulset: "true"
19 changes: 19 additions & 0 deletions chart/templates/03_memstore.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,25 @@ spec:
volumeMounts:
- name: config-volume
mountPath: /etc/memstore
startupProbe:
httpGet:
path: /startup
port: 8080
initialDelaySeconds: 5
periodSeconds: 10
failureThreshold: 10
readinessProbe:
httpGet:
path: /ready
port: 8080
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
httpGet:
path: /health
port: 8080
initialDelaySeconds: 15
periodSeconds: 20
volumes:
- name: config-volume
configMap:
Expand Down
17 changes: 15 additions & 2 deletions chart/templates/04_svc.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,25 @@
apiVersion: v1
kind: Service
metadata:
name: memstore
name: memstore-internal
namespace: memstore
spec:
ports:
- port: 8080
name: http2
clusterIP: None # Headless
clusterIP: None # Headless - internal use to get all the pod IPs individually irrespective of readiness
selector:
app: memstore
---
apiVersion: v1
kind: Service
metadata:
name: memstore
namespace: memstore
spec:
selector:
app: memstore
ports:
- name: http2
port: 8080
type: ClusterIP
4 changes: 4 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.18

require (
fortio.org/dflag v1.7.0
fortio.org/fortio v1.63.3
fortio.org/log v1.12.0
fortio.org/scli v1.14.1
fortio.org/sets v1.0.3
Expand All @@ -16,6 +17,9 @@ require (
fortio.org/struct2env v0.4.0 // indirect
fortio.org/version v1.0.3 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/google/uuid v1.6.0 // indirect
golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 // indirect
golang.org/x/net v0.21.0 // indirect
golang.org/x/sys v0.17.0 // indirect
golang.org/x/text v0.14.0 // indirect
)
8 changes: 8 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ fortio.org/cli v1.5.1 h1:lqPvkxRVSajsVwLfblaN62BPAICPp05Oab+yjRvI3DU=
fortio.org/cli v1.5.1/go.mod h1:Tp7AypudP1mJomTUN/J/vlOTlZDWTMsok09MMyA99ow=
fortio.org/dflag v1.7.0 h1:4Vpo5hMly0rx9VMuyBaDGFK1Mx2S3qjxx1iAIA3KBgU=
fortio.org/dflag v1.7.0/go.mod h1:FUxv/s3DXhCpy7GsuZa4FJWLR92gsYvG3ylkia8MbBM=
fortio.org/fortio v1.63.3 h1:t4FoQ70znmYEEST3eyMLDqGaoNJPUDZNvoPRRH0WsaQ=
fortio.org/fortio v1.63.3/go.mod h1:HFYGCHKrxS+Yuuw/7gcO7hhsEvrKt6t7sh1Xkn/kggw=
fortio.org/log v1.12.0 h1:5Yg4pL9Pp0jcWeJYixm2xikMCldVaSDMgDFDmQJZfho=
fortio.org/log v1.12.0/go.mod h1:1tMBG/Elr6YqjmJCWiejJp2FPvXg7/9UAN0Rfpkyt1o=
fortio.org/scli v1.14.1 h1:MLxVRfuCXU8AIhHbWRj3JGE0uzTSQ/cWmcKdpuF55qk=
Expand All @@ -15,7 +17,13 @@ fortio.org/version v1.0.3 h1:5gJ3plj6isAOMq52cI5ifo4cC+QHmJF76Wevc5Cp4x0=
fortio.org/version v1.0.3/go.mod h1:2JQp9Ax+tm6QKiGuzR5nJY63kFeANcgrZ0osoQFDVm0=
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 h1:/RIbNt/Zr7rVhIkQhooTxCxFcdWLGIKnZA4IXNFSrvo=
golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
2 changes: 1 addition & 1 deletion mstore/mstore.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ func dnsChange(oldValue, newValue string) {
}

func Start(name string) {
log.Infof("memstore Start() name is %q", name)
log.Infof("memstore starting with name %q", name)
myName = name
// peerChange does get call for even initial flag value
}
Expand Down
85 changes: 85 additions & 0 deletions probes/probes.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
// Implement kubernetes probes.
package probes

import (
"net/http"
"sync"

"fortio.org/log"
)

type state struct {
started bool
live bool
ready bool
// mutex
mutex sync.Mutex
}

var State = state{}

func (s *state) SetLive(live bool) {
s.mutex.Lock()
s.live = live
s.mutex.Unlock()
}

func (s *state) SetReady(ready bool) {
s.mutex.Lock()
s.ready = ready
s.mutex.Unlock()
}

func (s *state) SetStarted(started bool) {
s.mutex.Lock()
s.started = started
s.mutex.Unlock()
}

func (s *state) IsLive() bool {
s.mutex.Lock()
defer s.mutex.Unlock()
return s.live
}

func (s *state) IsReady() bool {
s.mutex.Lock()
defer s.mutex.Unlock()
return s.ready
}

func (s *state) IsStarted() bool {
s.mutex.Lock()
defer s.mutex.Unlock()
return s.started
}

func StartupProbe(w http.ResponseWriter, _ *http.Request) {
if State.IsStarted() {
w.WriteHeader(http.StatusOK)
} else {
w.WriteHeader(http.StatusServiceUnavailable)
}
}

func LivenessProbe(w http.ResponseWriter, _ *http.Request) {
if State.IsLive() {
w.WriteHeader(http.StatusOK)
} else {
w.WriteHeader(http.StatusServiceUnavailable)
}
}

func ReadinessProbe(w http.ResponseWriter, _ *http.Request) {
if State.IsReady() {
w.WriteHeader(http.StatusOK)
} else {
w.WriteHeader(http.StatusServiceUnavailable)
}
}

func Setup(mux *http.ServeMux) {
mux.HandleFunc("/startup", log.LogAndCall("startup", StartupProbe))
mux.HandleFunc("/ready", log.LogAndCall("readiness", ReadinessProbe))
mux.HandleFunc("/health", log.LogAndCall("liveness", LivenessProbe))
}
16 changes: 16 additions & 0 deletions proto.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,20 @@
package main

import (
"flag"
"os"
"time"

"fortio.org/dflag"
"fortio.org/fortio/fhttp"
"fortio.org/log"
"fortio.org/memstore/mstore"
"fortio.org/memstore/probes"
"fortio.org/scli"
)

func main() {
port := flag.String("port", "8080", "Port to listen on")
dflag.Flag("peers", mstore.Peers)
dflag.Flag("dns", mstore.DNSWatch)
dflag.Flag("dns-interval", mstore.DNSWatchSleepTime)
Expand All @@ -30,6 +35,17 @@ func main() {
log.Fatalf("No NAME env var found for statefulset mode (to this pod's name)")
}
mstore.Start(myName)
mux, addr := fhttp.HTTPServer("memstore", *port)
if addr == nil {
log.Fatalf("Failed to start http server")
}
probes.Setup(mux)
probes.State.SetLive(true)
probes.State.SetStarted(true)
probes.State.SetReady(true)
time.Sleep(50 * time.Second) // give time for the probes to be ready
log.Warnf("Switching back to not ready")
probes.State.SetReady(false)
scli.UntilInterrupted()
mstore.Stop()
}

0 comments on commit ea5ae9c

Please sign in to comment.