Remove default role and use retry with backoff to deal with race cond…

…ition on pod creation

README.md

@@ -110,7 +110,7 @@ the traffic to `169.254.169.254` must be proxied for docker containers.
 
 ### kubernetes annotation
 
-Add an `iam/role` annotation to your containers with the role that you want to assume for this container.
+Add an `iam/role` annotation to your pods with the role that you want to assume for this pod.
 
 ```
 ---
@@ -121,7 +121,7 @@ metadata:
   labels:
 	name: aws-cli
   annotations:
-	iam/role: role-name
+	iam.amazonaws.com/role: role-name
 spec:
   containers:
   - image: fstab/aws-cli
@@ -135,9 +135,9 @@ spec:
 
 ### Options
 
-By default, `kube2iam` will use the in-cluster method to connect to the kubernetes master, and use the `iam/role`
+By default, `kube2iam` will use the in-cluster method to connect to the kubernetes master, and use the `iam.amazonaws.com/role`
 annotation to retrieve the role for the container. Either set the `base-role-arn` option to apply to all roles
-and only pass the role name in the `iam/role` annotation, otherwise pass the full role ARN in the annotation.
+and only pass the role name in the `iam.amazonaws.com/role` annotation, otherwise pass the full role ARN in the annotation.
 
 ```
 $ kube2iam --help
@@ -146,8 +146,7 @@ Usage of kube2iam:
       --api-token string               Token to authenticate with the api server
       --app-port string                Http port (default "8181")
       --base-role-arn string           Base role ARN
-      --default-role string            Default IAM role (default "default")
-      --iam-role-key string            Pod annotation key used to retrieve the IAM role (default "iam/role")
+      --iam-role-key string            Pod annotation key used to retrieve the IAM role (default "iam.amazonaws.com/role")
       --insecure                       Kubernetes server should be accessed without verifying the TLS. Testing only
       --log-flush-frequency duration   Maximum number of seconds between log flushes (default 5s)
       --metadata-addr string           Address for the ec2 metadata (default "169.254.169.254")

cmd/server.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	log "github.com/Sirupsen/logrus"
+	"github.com/cenk/backoff"
 	"github.com/gorilla/mux"
 )
 
@@ -18,7 +19,6 @@ type Server struct {
 	APIToken        string
 	AppPort         string
 	BaseRoleARN     string
-	DefaultRole     string
 	IAMRoleKey      string
 	MetadataAddress string
 	Insecure        bool
@@ -50,26 +50,53 @@ func parseRemoteAddr(addr string) string {
 	return hostname
 }
 
+func (s *Server)  getRole(IP string) (string, error) {
+	var role string
+	var err error
+	operation := func() error {
+		role, err = s.store.Get(IP)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+
+	err = backoff.Retry(operation, backoff.NewExponentialBackOff())
+	if err != nil {
+		return "", err
+	}
+
+	return role, nil
+}
+
 func (s *Server) securityCredentialsHandler(w http.ResponseWriter, r *http.Request) {
 	remoteIP := parseRemoteAddr(r.RemoteAddr)
-	roleARN := s.iam.roleARN(s.store.Get(remoteIP))
+	role, err := s.getRole(remoteIP)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusNotFound)
+	}
+	roleARN := s.iam.roleARN(role)
 	idx := strings.LastIndex(roleARN, "/")
 	write(w, roleARN[idx+1:])
 }
 
 func (s *Server) roleHandler(w http.ResponseWriter, r *http.Request) {
 	remoteIP := parseRemoteAddr(r.RemoteAddr)
-	roleARN := s.iam.roleARN(s.store.Get(remoteIP))
+	role, err := s.store.Get(remoteIP)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusNotFound)
+	}
+	roleARN := s.iam.roleARN(role)
 	credentials, err := s.iam.assumeRole(roleARN, remoteIP)
 	if err != nil {
 		log.Errorf("Error assuming role %+v", err)
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
 	if err := json.NewEncoder(w).Encode(credentials); err != nil {
 		log.Errorf("Error sending json %+v", err)
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
 	}
 }
 
@@ -97,7 +124,7 @@ func (s *Server) Run(host, token string, insecure bool) error {
 		return err
 	}
 	s.k8s = k8s
-	s.store = newStore(s.IAMRoleKey, s.DefaultRole)
+	s.store = newStore(s.IAMRoleKey)
 	s.k8s.watchForPods(s.store)
 	s.iam = newIAM(s.BaseRoleARN)
 	r := mux.NewRouter()
@@ -116,8 +143,7 @@ func (s *Server) Run(host, token string, insecure bool) error {
 func NewServer() *Server {
 	return &Server{
 		AppPort:         "8181",
-		DefaultRole:     "default",
-		IAMRoleKey:      "iam/role",
+		IAMRoleKey:      "iam.amazonaws.com/role",
 		MetadataAddress: "169.254.169.254",
 	}
 }

cmd/store.go

@@ -1,34 +1,34 @@
 package cmd
 
 import (
+	"fmt"
 	"sync"
 
 	"k8s.io/kubernetes/pkg/api"
 )
 
 // store implements the k8s framework ResourceEventHandler interface.
 type store struct {
-	defaultRole string
 	iamRoleKey  string
 	mutex       sync.RWMutex
 	rolesByIP   map[string]string
 }
 
 // Get returns the iam role based on IP address.
-func (s *store) Get(IP string) string {
+func (s *store) Get(IP string) (string, error) {
 	s.mutex.RLock()
 	defer s.mutex.RUnlock()
 	if role, ok := s.rolesByIP[IP]; ok {
-		return role
+		return role, nil
 	}
-	return s.defaultRole
+	return "", fmt.Errorf("Unable to find role for IP %s", IP)
 }
 
 // OnAdd is called when a pod is added.
 func (s *store) OnAdd(obj interface{}) {
 	if pod, ok := obj.(*api.Pod); ok {
-		if role, ok := pod.Annotations[s.iamRoleKey]; ok {
-			if pod.Status.PodIP != "" {
+		if pod.Status.PodIP != "" {
+			if role, ok := pod.Annotations[s.iamRoleKey]; ok {
 				s.mutex.Lock()
 				s.rolesByIP[pod.Status.PodIP] = role
 				s.mutex.Unlock()
@@ -66,9 +66,8 @@ func (s *store) OnDelete(obj interface{}) {
 	}
 }
 
-func newStore(key, defaultRole string) *store {
+func newStore(key string) *store {
 	return &store{
-		defaultRole: defaultRole,
 		iamRoleKey:  key,
 		rolesByIP:   make(map[string]string),
 	}

glide.yaml

@@ -19,3 +19,4 @@ import:
   - pkg/controller/framework
   - pkg/fields
   - pkg/util/wait
+- package: github.com/cenk/backoff

main.go

@@ -35,7 +35,6 @@ func addFlags(s *cmd.Server, fs *pflag.FlagSet) {
 	fs.StringVar(&s.APIToken, "api-token", s.APIToken, "Token to authenticate with the api server")
 	fs.StringVar(&s.AppPort, "app-port", s.AppPort, "Http port")
 	fs.StringVar(&s.BaseRoleARN, "base-role-arn", s.BaseRoleARN, "Base role ARN")
-	fs.StringVar(&s.DefaultRole, "default-role", s.DefaultRole, "Default IAM role")
 	fs.StringVar(&s.IAMRoleKey, "iam-role-key", s.IAMRoleKey, "Pod annotation key used to retrieve the IAM role")
 	fs.BoolVar(&s.Insecure, "insecure", false, "Kubernetes server should be accessed without verifying the TLS. Testing only")
 	fs.StringVar(&s.MetadataAddress, "metadata-addr", s.MetadataAddress, "Address for the ec2 metadata")