From 6ba7d769718e046f6fa7b867ecf3a4b0a337c661 Mon Sep 17 00:00:00 2001 From: Nithish Date: Wed, 31 Jul 2019 10:12:01 -0500 Subject: [PATCH] Adding healthz endpoint to IPamD --- Makefile | 1 + client/health-check/grpc_health_probe.go | 138 +++++++++++ config/v1.5/aws-k8s-cni.yaml | 8 + ipamd/rpc_handler.go | 8 + scripts/dockerfiles/Dockerfile.release | 1 + .../grpc/health/grpc_health_v1/health.pb.go | 227 ++++++++++++++++++ vendor/modules.txt | 7 +- 7 files changed, 387 insertions(+), 3 deletions(-) create mode 100644 client/health-check/grpc_health_probe.go create mode 100644 vendor/google.golang.org/grpc/health/grpc_health_v1/health.pb.go diff --git a/Makefile b/Makefile index 1c0cb07724..bb96eedb4d 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,7 @@ endif build-linux: GOOS=linux GOARCH=$(ARCH) CGO_ENABLED=0 go build -o aws-k8s-agent -ldflags "$(LDFLAGS)" GOOS=linux GOARCH=$(ARCH) CGO_ENABLED=0 go build -o aws-cni -ldflags "$(LDFLAGS)" ./plugins/routed-eni/ + GOOS=linux GOARCH=$(ARCH) CGO_ENABLED=0 go build -o grpc_health_probe -ldflags "$(LDFLAGS)" ./client/health-check/ # Download portmap plugin download-portmap: diff --git a/client/health-check/grpc_health_probe.go b/client/health-check/grpc_health_probe.go new file mode 100644 index 0000000000..ec3c4e1720 --- /dev/null +++ b/client/health-check/grpc_health_probe.go @@ -0,0 +1,138 @@ +package main + +import ( + "context" + "flag" + "log" + "os" + "os/signal" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" +) + +var ( + userAgent string + remoteURL string + serviceName string + connTimeoutDur time.Duration = time.Second + rpcTimeoutDur time.Duration = time.Second + verbose bool +) + +const ( + // StatusInvalidArguments indicates specified invalid arguments. + StatusInvalidArguments = 1 + // StatusConnectionFailure indicates connection failed. + StatusConnectionFailure = 2 + // StatusRPCFailure indicates rpc failed. + StatusRPCFailure = 3 + // StatusUnhealthy indicates rpc succeeded but indicates unhealthy service. + StatusUnhealthy = 4 +) + +func init() { + log.SetFlags(0) + flag.StringVar(&remoteURL, "addr", "", "(required) tcp host:port to connect") + flag.StringVar(&serviceName, "service", "", "service name to check (default: \"\")") + flag.StringVar(&userAgent, "user-agent", "grpc_health_probe", "user-agent header value of health check requests") + // timeouts + flag.DurationVar(&connTimeoutDur, "connect-timeout", connTimeoutDur, "timeout for establishing connection") + flag.DurationVar(&rpcTimeoutDur, "rpc-timeout", rpcTimeoutDur, "timeout for health check rpc") + // verbose + flag.BoolVar(&verbose, "v", false, "verbose logs") + + flag.Parse() + + argError := func(s string, v ...interface{}) { + log.Printf("error: "+s, v...) + os.Exit(StatusInvalidArguments) + } + + if remoteURL == "" { + argError("--addr not specified") + } + + if connTimeoutDur <= 0 { + argError("--connect-timeout must be greater than zero (specified: %v)", connTimeoutDur) + } + if rpcTimeoutDur <= 0 { + argError("--rpc-timeout must be greater than zero (specified: %v)", rpcTimeoutDur) + } + if verbose { + log.Printf("parsed options:") + log.Printf("> remoteUrl=%s conn-timeout=%v rpc-timeout=%v", remoteURL, connTimeoutDur, rpcTimeoutDur) + } +} + +func main() { + ctx, cancel := context.WithCancel(context.Background()) + + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + go func() { + sig := <-c + if sig == os.Interrupt { + log.Printf("cancellation received") + cancel() + return + } + }() + + opts := []grpc.DialOption{ + grpc.WithUserAgent(userAgent), + grpc.WithBlock()} + + opts = append(opts, grpc.WithInsecure()) + + if verbose { + log.Print("establishing connection") + } + connStart := time.Now() + dialCtx, cancel2 := context.WithTimeout(ctx, connTimeoutDur) + defer cancel2() + conn, err := grpc.DialContext(dialCtx, remoteURL, opts...) + if err != nil { + if err == context.DeadlineExceeded { + log.Printf("timeout: failed to connect service %q within %v", remoteURL, connTimeoutDur) + } else { + log.Printf("error: failed to connect service at %q: %+v", remoteURL, err) + } + os.Exit(StatusConnectionFailure) + } + connDuration := time.Since(connStart) + defer conn.Close() + if verbose { + log.Printf("connection establisted (took %v)", connDuration) + } + + rpcStart := time.Now() + rpcCtx, rpcCancel := context.WithTimeout(ctx, rpcTimeoutDur) + defer rpcCancel() + resp, err := healthpb.NewHealthClient(conn).Check(rpcCtx, &healthpb.HealthCheckRequest{Service: serviceName}) + log.Print(resp) + if err != nil { + if stat, ok := status.FromError(err); ok && stat.Code() == codes.Unimplemented { + log.Printf("error: this server does not implement the grpc health protocol (grpc.health.v1.Health)") + } else if stat, ok := status.FromError(err); ok && stat.Code() == codes.DeadlineExceeded { + log.Printf("timeout: health rpc did not complete within %v", rpcTimeoutDur) + } else { + log.Printf("error: health rpc failed: %+v", err) + } + os.Exit(StatusRPCFailure) + } + rpcDuration := time.Since(rpcStart) + + if resp.GetStatus() != healthpb.HealthCheckResponse_SERVING { + log.Printf("service unhealthy (responded with %q)", resp.GetStatus().String()) + os.Exit(StatusUnhealthy) + } + if verbose { + log.Printf("time elapsed: connect=%v rpc=%v", connDuration, rpcDuration) + } + log.Printf("status: %v", resp.GetStatus().String()) +} + diff --git a/config/v1.5/aws-k8s-cni.yaml b/config/v1.5/aws-k8s-cni.yaml index d6c2caa7c2..958b0ff977 100644 --- a/config/v1.5/aws-k8s-cni.yaml +++ b/config/v1.5/aws-k8s-cni.yaml @@ -87,6 +87,14 @@ spec: - containerPort: 61678 name: metrics name: aws-node + #readinessProbe: + # exec: + # command: ["/app/grpc_health_probe", "-addr=:50051"] + # initialDelaySeconds: 5 + #livenessProbe: + # exec: + # command: ["/app/grpc_health_probe", "-addr=:50051"] + # initialDelaySeconds: 5 env: - name: AWS_VPC_K8S_CNI_LOGLEVEL value: DEBUG diff --git a/ipamd/rpc_handler.go b/ipamd/rpc_handler.go index 4f4ede580b..1afa7c2f13 100644 --- a/ipamd/rpc_handler.go +++ b/ipamd/rpc_handler.go @@ -22,6 +22,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "golang.org/x/net/context" "google.golang.org/grpc" + healthpb "google.golang.org/grpc/health/grpc_health_v1" "google.golang.org/grpc/reflection" log "github.com/cihub/seelog" @@ -34,10 +35,16 @@ const ( ipamdgRPCaddress = "127.0.0.1:50051" ) +// server controls RPC service responses. type server struct { ipamContext *IPAMContext } +// Check is for health checking. +func (s *server) Check(ctx context.Context, req *healthpb.HealthCheckRequest) (*healthpb.HealthCheckResponse, error) { + return &healthpb.HealthCheckResponse{Status: healthpb.HealthCheckResponse_SERVING}, nil +} + // AddNetwork processes CNI add network request and return an IP address for container func (s *server) AddNetwork(ctx context.Context, in *pb.AddNetworkRequest) (*pb.AddNetworkReply, error) { log.Infof("Received AddNetwork for NS %s, Pod %s, NameSpace %s, Container %s, ifname %s", @@ -98,6 +105,7 @@ func (c *IPAMContext) RunRPCHandler() error { } s := grpc.NewServer() pb.RegisterCNIBackendServer(s, &server{ipamContext: c}) + healthpb.RegisterHealthServer(s, &server{}) // Register reflection service on gRPC server. reflection.Register(s) if err := s.Serve(lis); err != nil { diff --git a/scripts/dockerfiles/Dockerfile.release b/scripts/dockerfiles/Dockerfile.release index 7ec835cfc7..11bcf42942 100644 --- a/scripts/dockerfiles/Dockerfile.release +++ b/scripts/dockerfiles/Dockerfile.release @@ -31,6 +31,7 @@ COPY --from=builder /go/src/github.com/aws/amazon-vpc-cni-k8s/misc/10-aws.confli COPY --from=builder /go/src/github.com/aws/amazon-vpc-cni-k8s/portmap /app COPY --from=builder /go/src/github.com/aws/amazon-vpc-cni-k8s/aws-k8s-agent /app +COPY --from=builder /go/src/github.com/aws/amazon-vpc-cni-k8s/grpc_health_probe /app COPY --from=builder /go/src/github.com/aws/amazon-vpc-cni-k8s/scripts/aws-cni-support.sh /app COPY --from=builder /go/src/github.com/aws/amazon-vpc-cni-k8s/scripts/install-aws.sh /app ENTRYPOINT /app/install-aws.sh diff --git a/vendor/google.golang.org/grpc/health/grpc_health_v1/health.pb.go b/vendor/google.golang.org/grpc/health/grpc_health_v1/health.pb.go new file mode 100644 index 0000000000..a1fda2801b --- /dev/null +++ b/vendor/google.golang.org/grpc/health/grpc_health_v1/health.pb.go @@ -0,0 +1,227 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: grpc/health/v1/health.proto + +package grpc_health_v1 // import "google.golang.org/grpc/health/grpc_health_v1" + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type HealthCheckResponse_ServingStatus int32 + +const ( + HealthCheckResponse_UNKNOWN HealthCheckResponse_ServingStatus = 0 + HealthCheckResponse_SERVING HealthCheckResponse_ServingStatus = 1 + HealthCheckResponse_NOT_SERVING HealthCheckResponse_ServingStatus = 2 +) + +var HealthCheckResponse_ServingStatus_name = map[int32]string{ + 0: "UNKNOWN", + 1: "SERVING", + 2: "NOT_SERVING", +} +var HealthCheckResponse_ServingStatus_value = map[string]int32{ + "UNKNOWN": 0, + "SERVING": 1, + "NOT_SERVING": 2, +} + +func (x HealthCheckResponse_ServingStatus) String() string { + return proto.EnumName(HealthCheckResponse_ServingStatus_name, int32(x)) +} +func (HealthCheckResponse_ServingStatus) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_health_85731b6c49265086, []int{1, 0} +} + +type HealthCheckRequest struct { + Service string `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *HealthCheckRequest) Reset() { *m = HealthCheckRequest{} } +func (m *HealthCheckRequest) String() string { return proto.CompactTextString(m) } +func (*HealthCheckRequest) ProtoMessage() {} +func (*HealthCheckRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_health_85731b6c49265086, []int{0} +} +func (m *HealthCheckRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_HealthCheckRequest.Unmarshal(m, b) +} +func (m *HealthCheckRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_HealthCheckRequest.Marshal(b, m, deterministic) +} +func (dst *HealthCheckRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_HealthCheckRequest.Merge(dst, src) +} +func (m *HealthCheckRequest) XXX_Size() int { + return xxx_messageInfo_HealthCheckRequest.Size(m) +} +func (m *HealthCheckRequest) XXX_DiscardUnknown() { + xxx_messageInfo_HealthCheckRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_HealthCheckRequest proto.InternalMessageInfo + +func (m *HealthCheckRequest) GetService() string { + if m != nil { + return m.Service + } + return "" +} + +type HealthCheckResponse struct { + Status HealthCheckResponse_ServingStatus `protobuf:"varint,1,opt,name=status,proto3,enum=grpc.health.v1.HealthCheckResponse_ServingStatus" json:"status,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *HealthCheckResponse) Reset() { *m = HealthCheckResponse{} } +func (m *HealthCheckResponse) String() string { return proto.CompactTextString(m) } +func (*HealthCheckResponse) ProtoMessage() {} +func (*HealthCheckResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_health_85731b6c49265086, []int{1} +} +func (m *HealthCheckResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_HealthCheckResponse.Unmarshal(m, b) +} +func (m *HealthCheckResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_HealthCheckResponse.Marshal(b, m, deterministic) +} +func (dst *HealthCheckResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_HealthCheckResponse.Merge(dst, src) +} +func (m *HealthCheckResponse) XXX_Size() int { + return xxx_messageInfo_HealthCheckResponse.Size(m) +} +func (m *HealthCheckResponse) XXX_DiscardUnknown() { + xxx_messageInfo_HealthCheckResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_HealthCheckResponse proto.InternalMessageInfo + +func (m *HealthCheckResponse) GetStatus() HealthCheckResponse_ServingStatus { + if m != nil { + return m.Status + } + return HealthCheckResponse_UNKNOWN +} + +func init() { + proto.RegisterType((*HealthCheckRequest)(nil), "grpc.health.v1.HealthCheckRequest") + proto.RegisterType((*HealthCheckResponse)(nil), "grpc.health.v1.HealthCheckResponse") + proto.RegisterEnum("grpc.health.v1.HealthCheckResponse_ServingStatus", HealthCheckResponse_ServingStatus_name, HealthCheckResponse_ServingStatus_value) +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// HealthClient is the client API for Health service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type HealthClient interface { + Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) +} + +type healthClient struct { + cc *grpc.ClientConn +} + +func NewHealthClient(cc *grpc.ClientConn) HealthClient { + return &healthClient{cc} +} + +func (c *healthClient) Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) { + out := new(HealthCheckResponse) + err := c.cc.Invoke(ctx, "/grpc.health.v1.Health/Check", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// HealthServer is the server API for Health service. +type HealthServer interface { + Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) +} + +func RegisterHealthServer(s *grpc.Server, srv HealthServer) { + s.RegisterService(&_Health_serviceDesc, srv) +} + +func _Health_Check_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(HealthCheckRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(HealthServer).Check(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.health.v1.Health/Check", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(HealthServer).Check(ctx, req.(*HealthCheckRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _Health_serviceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.health.v1.Health", + HandlerType: (*HealthServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Check", + Handler: _Health_Check_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "grpc/health/v1/health.proto", +} + +func init() { proto.RegisterFile("grpc/health/v1/health.proto", fileDescriptor_health_85731b6c49265086) } + +var fileDescriptor_health_85731b6c49265086 = []byte{ + // 271 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4e, 0x2f, 0x2a, 0x48, + 0xd6, 0xcf, 0x48, 0x4d, 0xcc, 0x29, 0xc9, 0xd0, 0x2f, 0x33, 0x84, 0xb2, 0xf4, 0x0a, 0x8a, 0xf2, + 0x4b, 0xf2, 0x85, 0xf8, 0x40, 0x92, 0x7a, 0x50, 0xa1, 0x32, 0x43, 0x25, 0x3d, 0x2e, 0x21, 0x0f, + 0x30, 0xc7, 0x39, 0x23, 0x35, 0x39, 0x3b, 0x28, 0xb5, 0xb0, 0x34, 0xb5, 0xb8, 0x44, 0x48, 0x82, + 0x8b, 0xbd, 0x38, 0xb5, 0xa8, 0x2c, 0x33, 0x39, 0x55, 0x82, 0x51, 0x81, 0x51, 0x83, 0x33, 0x08, + 0xc6, 0x55, 0x9a, 0xc3, 0xc8, 0x25, 0x8c, 0xa2, 0xa1, 0xb8, 0x20, 0x3f, 0xaf, 0x38, 0x55, 0xc8, + 0x93, 0x8b, 0xad, 0xb8, 0x24, 0xb1, 0xa4, 0xb4, 0x18, 0xac, 0x81, 0xcf, 0xc8, 0x50, 0x0f, 0xd5, + 0x22, 0x3d, 0x2c, 0x9a, 0xf4, 0x82, 0x41, 0x86, 0xe6, 0xa5, 0x07, 0x83, 0x35, 0x06, 0x41, 0x0d, + 0x50, 0xb2, 0xe2, 0xe2, 0x45, 0x91, 0x10, 0xe2, 0xe6, 0x62, 0x0f, 0xf5, 0xf3, 0xf6, 0xf3, 0x0f, + 0xf7, 0x13, 0x60, 0x00, 0x71, 0x82, 0x5d, 0x83, 0xc2, 0x3c, 0xfd, 0xdc, 0x05, 0x18, 0x85, 0xf8, + 0xb9, 0xb8, 0xfd, 0xfc, 0x43, 0xe2, 0x61, 0x02, 0x4c, 0x46, 0x51, 0x5c, 0x6c, 0x10, 0x8b, 0x84, + 0x02, 0xb8, 0x58, 0xc1, 0x96, 0x09, 0x29, 0xe1, 0x75, 0x09, 0xd8, 0xbf, 0x52, 0xca, 0x44, 0xb8, + 0xd6, 0x29, 0x91, 0x4b, 0x30, 0x33, 0x1f, 0x4d, 0xa1, 0x13, 0x37, 0x44, 0x65, 0x00, 0x28, 0x70, + 0x03, 0x18, 0xa3, 0x74, 0xd2, 0xf3, 0xf3, 0xd3, 0x73, 0x52, 0xf5, 0xd2, 0xf3, 0x73, 0x12, 0xf3, + 0xd2, 0xf5, 0xf2, 0x8b, 0xd2, 0xf5, 0x91, 0x63, 0x03, 0xc4, 0x8e, 0x87, 0xb0, 0xe3, 0xcb, 0x0c, + 0x57, 0x31, 0xf1, 0xb9, 0x83, 0x4c, 0x83, 0x18, 0xa1, 0x17, 0x66, 0x98, 0xc4, 0x06, 0x8e, 0x24, + 0x63, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0xec, 0x66, 0x81, 0xcb, 0xc3, 0x01, 0x00, 0x00, +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 779b92268c..b4bad8a8ad 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -91,8 +91,8 @@ github.com/golang/glog github.com/golang/mock/gomock # github.com/golang/protobuf v1.2.0 github.com/golang/protobuf/proto -github.com/golang/protobuf/protoc-gen-go/descriptor github.com/golang/protobuf/ptypes +github.com/golang/protobuf/protoc-gen-go/descriptor github.com/golang/protobuf/ptypes/any github.com/golang/protobuf/ptypes/duration github.com/golang/protobuf/ptypes/timestamp @@ -186,10 +186,12 @@ golang.org/x/time/rate google.golang.org/genproto/googleapis/rpc/status # google.golang.org/grpc v1.14.0 google.golang.org/grpc +google.golang.org/grpc/codes +google.golang.org/grpc/health/grpc_health_v1 +google.golang.org/grpc/status google.golang.org/grpc/reflection google.golang.org/grpc/balancer google.golang.org/grpc/balancer/roundrobin -google.golang.org/grpc/codes google.golang.org/grpc/connectivity google.golang.org/grpc/credentials google.golang.org/grpc/encoding @@ -209,7 +211,6 @@ google.golang.org/grpc/resolver google.golang.org/grpc/resolver/dns google.golang.org/grpc/resolver/passthrough google.golang.org/grpc/stats -google.golang.org/grpc/status google.golang.org/grpc/tap google.golang.org/grpc/reflection/grpc_reflection_v1alpha google.golang.org/grpc/balancer/base