Moved IP Reconciler code into IP Control Loop

Signed-off-by: nicklesimba <[email protected]>

.github/CODEOWNERS

@@ -1,7 +1,6 @@
 # https://help.github.com/en/articles/about-code-owners
 * @dougbtv
 cmd/controlloop @maiqueb
-cmd/reconciler @maiqueb
 e2e/ @maiqueb
 pkg/reconciler/ @maiqueb
-
+pkg/controlloop @maiqueb

Dockerfile

@@ -9,6 +9,5 @@ FROM alpine:latest
 LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/whereabouts
 COPY --from=0 /go/src/github.com/k8snetworkplumbingwg/whereabouts/bin/whereabouts .
 COPY --from=0 /go/src/github.com/k8snetworkplumbingwg/whereabouts/bin/ip-control-loop .
-COPY --from=0 /go/src/github.com/k8snetworkplumbingwg/whereabouts/bin/ip-reconciler .
 COPY script/install-cni.sh .
 CMD ["/install-cni.sh"]

Dockerfile.arm64

@@ -13,6 +13,5 @@ FROM arm64v8/alpine:latest
 LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/whereabouts
 COPY --from=0 /go/src/github.com/k8snetworkplumbingwg/whereabouts/bin/whereabouts .
 COPY --from=0 /go/src/github.com/k8snetworkplumbingwg/whereabouts/bin/ip-control-loop .
-COPY --from=0 /go/src/github.com/k8snetworkplumbingwg/whereabouts/bin/ip-reconciler .
 COPY script/install-cni.sh .
 CMD ["/install-cni.sh"]

Dockerfile.openshift

@@ -5,15 +5,13 @@ WORKDIR /go/src/github.com/k8snetworkplumbingwg/whereabouts
 ENV CGO_ENABLED=1
 ENV GO111MODULE=on
 RUN go build -mod vendor -o bin/whereabouts     cmd/whereabouts.go
-RUN go build -mod vendor -o bin/ip-reconciler   cmd/reconciler/ip.go cmd/reconciler/errors.go
-RUN go build -mod vendor -o bin/ip-control-loop cmd/controlloop/main.go
+RUN go build -mod vendor -o bin/ip-control-loop cmd/controlloop/controlloop.go
 WORKDIR /
 
 FROM openshift/origin-base
 RUN mkdir -p /usr/src/whereabouts/images && \
        mkdir -p /usr/src/whereabouts/bin
 COPY --from=builder /go/src/github.com/k8snetworkplumbingwg/whereabouts/bin/whereabouts     /usr/src/whereabouts/bin
-COPY --from=builder /go/src/github.com/k8snetworkplumbingwg/whereabouts/bin/ip-reconciler   /usr/src/whereabouts/bin
 COPY --from=builder /go/src/github.com/k8snetworkplumbingwg/whereabouts/bin/ip-control-loop /usr/src/whereabouts/bin
 
 LABEL org.opencontainers.image.source https://github.com/k8snetworkplumbingwg/whereabouts

README.md

@@ -45,8 +45,7 @@ git clone https://github.com/k8snetworkplumbingwg/whereabouts && cd whereabouts
 kubectl apply \
     -f doc/crds/daemonset-install.yaml \
     -f doc/crds/whereabouts.cni.cncf.io_ippools.yaml \
-    -f doc/crds/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml \
-    -f doc/crds/ip-reconciler-job.yaml
+    -f doc/crds/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml
 ```
 
 The daemonset installation requires Kubernetes Version 1.16 or later.

cmd/controlloop/main.go → cmd/controlloop/controlloop.go

@@ -5,7 +5,9 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
+	"time"
 
+	gocron "github.com/go-co-op/gocron"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
@@ -21,17 +23,23 @@ import (
 
 	wbclient "github.com/k8snetworkplumbingwg/whereabouts/pkg/client/clientset/versioned"
 	wbinformers "github.com/k8snetworkplumbingwg/whereabouts/pkg/client/informers/externalversions"
+	"github.com/k8snetworkplumbingwg/whereabouts/pkg/config"
+	"github.com/k8snetworkplumbingwg/whereabouts/pkg/controlloop"
 	"github.com/k8snetworkplumbingwg/whereabouts/pkg/logging"
-	"github.com/k8snetworkplumbingwg/whereabouts/pkg/reconciler/controlloop"
+	"github.com/k8snetworkplumbingwg/whereabouts/pkg/reconciler"
+	"github.com/k8snetworkplumbingwg/whereabouts/pkg/types"
 )
 
 const (
 	allNamespaces  = ""
-	controllerName = "pod-ip-reconciler"
+	controllerName = "pod-ip-controlloop"
 )
 
 const (
 	couldNotCreateController = 1
+	couldNotReadFlatfile     = 1
+	couldNotGetFlatIPAM      = 1
+	cronExpressionError      = 1
 )
 
 const (
@@ -46,7 +54,9 @@ func main() {
 	logging.SetLogStderr(true)
 
 	stopChan := make(chan struct{})
+	errorChan := make(chan error)
 	defer close(stopChan)
+	defer close(errorChan)
 	handleSignals(stopChan, os.Interrupt)
 
 	networkController, err := newPodController(stopChan)
@@ -57,8 +67,34 @@ func main() {
 
 	networkController.Start(stopChan)
 	defer networkController.Shutdown()
-	<-stopChan
-	logging.Verbosef("shutting down network controller")
+
+	s := gocron.NewScheduler(time.UTC)
+	schedule := cronExpressionFromFlatFile()
+
+	_, err = s.Cron(schedule).Do(func() { // user configurable cron expression in install-cni.sh
+		reconciler.ReconcileIPs(errorChan)
+	})
+	if err != nil {
+		_ = logging.Errorf("error with cron expression schedule: %v", err)
+		os.Exit(cronExpressionError)
+	}
+
+	s.StartAsync()
+
+	for {
+		select {
+		case <-stopChan:
+			logging.Verbosef("shutting down network controller")
+			s.Stop()
+			return
+		case err := <-errorChan:
+			if err == nil {
+				logging.Verbosef("reconciler success")
+			} else {
+				logging.Verbosef("reconciler failure: %s", err)
+			}
+		}
+	}
 }
 
 func handleSignals(stopChannel chan struct{}, signals ...os.Signal) {
@@ -133,3 +169,12 @@ func newEventBroadcaster(k8sClientset kubernetes.Interface) record.EventBroadcas
 func newEventRecorder(broadcaster record.EventBroadcaster) record.EventRecorder {
 	return broadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerName})
 }
+
+func cronExpressionFromFlatFile() string {
+	flatipam, _, err := config.GetFlatIPAM(true, &types.IPAMConfig{}, "")
+	if err != nil {
+		_ = logging.Errorf("could not get flatipam: %v", err)
+		os.Exit(couldNotGetFlatIPAM)
+	}
+	return flatipam.IPAM.ReconcilerCronExpression
+}

doc/extended-configuration.md

@@ -134,6 +134,12 @@ spec:
 
 You'll note that in the `ipam` section there's a lot less parameters than are used in the previous examples.
 
+### Reconciler Cron Expression Configuration (optional)
+
+You may want to provide a cron expression to configure how frequently the ip-reconciler runs. This is done via the flatfile.
+
+Look for the following parameter `"reconciler_cron_expression"` located in `script/install-cni.sh` and change to your desired schedule.
+
 ## Installing etcd. (optional)
 
 etcd installation is optional. By default, we recommend the custom resource backend (given in the first example configuration).

hack/build-go.sh

@@ -45,5 +45,4 @@ VERSION_LDFLAGS="-X github.com/k8snetworkplumbingwg/whereabouts/pkg/version.Vers
 GLDFLAGS="${GLDFLAGS} ${VERSION_LDFLAGS}"
 
 CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} ${GO} build ${GOFLAGS} -ldflags "${GLDFLAGS}" -o bin/${cmd} cmd/${cmd}.go
-CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} ${GO} build ${GOFLAGS} -ldflags "${GLDFLAGS}" -o bin/ip-reconciler cmd/reconciler/*.go
 CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} ${GO} build ${GOFLAGS} -ldflags "${GLDFLAGS}" -o bin/ip-control-loop cmd/controlloop/*.go

hack/e2e-setup-kind-cluster.sh

@@ -98,7 +98,7 @@ trap "rm /tmp/whereabouts-img.tar || true" EXIT
 kind load image-archive --name "$KIND_CLUSTER_NAME" /tmp/whereabouts-img.tar
 
 echo "## install whereabouts"
-for file in "daemonset-install.yaml" "ip-reconciler-job.yaml" "whereabouts.cni.cncf.io_ippools.yaml" "whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml"; do
+for file in "daemonset-install.yaml" "whereabouts.cni.cncf.io_ippools.yaml" "whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml"; do
   retry kubectl apply -f "$ROOT/doc/crds/$file"
 done
 retry kubectl wait -n kube-system --for=condition=ready -l app=whereabouts pod --timeout=$TIMEOUT_K8

pkg/config/config.go

@@ -53,41 +53,9 @@ func LoadIPAMConfig(bytes []byte, envArgs string, extraConfigPaths ...string) (*
 	n.IPAM.PodName = string(args.K8S_POD_NAME)
 	n.IPAM.PodNamespace = string(args.K8S_POD_NAMESPACE)
 
-	// Once we have our basics, let's look for our (optional) configuration file
-	confdirs := []string{"/etc/kubernetes/cni/net.d/whereabouts.d/whereabouts.conf", "/etc/cni/net.d/whereabouts.d/whereabouts.conf"}
-	confdirs = append(confdirs, extraConfigPaths...)
-	// We prefix the optional configuration path (so we look there first)
-	if n.IPAM.ConfigurationPath != "" {
-		confdirs = append([]string{n.IPAM.ConfigurationPath}, confdirs...)
-	}
-
-	// Cycle through the path and parse the JSON config
-	flatipam := types.Net{}
-	foundflatfile := ""
-	for _, confpath := range confdirs {
-		if pathExists(confpath) {
-
-			jsonFile, err := os.Open(confpath)
-
-			if err != nil {
-				return nil, "", fmt.Errorf("error opening flat configuration file @ %s with: %s", confpath, err)
-			}
-
-			defer jsonFile.Close()
-
-			jsonBytes, err := ioutil.ReadAll(jsonFile)
-			if err != nil {
-				return nil, "", fmt.Errorf("LoadIPAMConfig Flatfile (%s) - ioutil.ReadAll error: %s", confpath, err)
-			}
-
-			if err := json.Unmarshal(jsonBytes, &flatipam.IPAM); err != nil {
-				return nil, "", fmt.Errorf("LoadIPAMConfig Flatfile (%s) - JSON Parsing Error: %s / bytes: %s", confpath, err, jsonBytes)
-			}
-
-			foundflatfile = confpath
-
-			break
-		}
+	flatipam, foundflatfile, err := GetFlatIPAM(false, n.IPAM, extraConfigPaths...)
+	if err != nil {
+		return nil, "", err
 	}
 
 	// Now let's try to merge the configurations...
@@ -139,7 +107,6 @@ func LoadIPAMConfig(bytes []byte, envArgs string, extraConfigPaths ...string) (*
 		n.IPAM.Datastore = types.DatastoreETCD
 	}
 
-	var err error
 	storageError := "You have not configured the storage engine (looks like you're using an invalid `%s` parameter in your config)"
 	switch n.IPAM.Datastore {
 	case types.DatastoreKubernetes:
@@ -164,7 +131,6 @@ func LoadIPAMConfig(bytes []byte, envArgs string, extraConfigPaths ...string) (*
 		}
 		n.IPAM.Gateway = gwip
 	}
-
 	for i := range n.IPAM.OmitRanges {
 		_, _, err := netutils.ParseCIDRSloppy(n.IPAM.OmitRanges[i])
 		if err != nil {
@@ -252,6 +218,47 @@ func configureStatic(n *types.Net, args types.IPAMEnvArgs) error {
 
 }
 
+func GetFlatIPAM(isControlLoop bool, IPAM *types.IPAMConfig, extraConfigPaths ...string) (types.Net, string, error) {
+	// Once we have our basics, let's look for our (optional) configuration file
+	confdirs := []string{"/etc/kubernetes/cni/net.d/whereabouts.d/whereabouts.conf", "/etc/cni/net.d/whereabouts.d/whereabouts.conf", "/host/etc/cni/net.d/whereabouts.d/whereabouts.conf"}
+	confdirs = append(confdirs, extraConfigPaths...)
+	// We prefix the optional configuration path (so we look there first)
+
+	if !isControlLoop && IPAM != nil {
+		if IPAM.ConfigurationPath != "" {
+			confdirs = append([]string{IPAM.ConfigurationPath}, confdirs...)
+		}
+	}
+
+	// Cycle through the path and parse the JSON config
+	flatipam := types.Net{}
+	foundflatfile := ""
+	for _, confpath := range confdirs {
+		if pathExists(confpath) {
+			jsonFile, err := os.Open(confpath)
+			if err != nil {
+				return flatipam, foundflatfile, fmt.Errorf("error opening flat configuration file @ %s with: %s", confpath, err)
+			}
+
+			defer jsonFile.Close()
+
+			jsonBytes, err := ioutil.ReadAll(jsonFile)
+			if err != nil {
+				return flatipam, foundflatfile, fmt.Errorf("LoadIPAMConfig Flatfile (%s) - ioutil.ReadAll error: %s", confpath, err)
+			}
+
+			if err := json.Unmarshal(jsonBytes, &flatipam.IPAM); err != nil {
+				return flatipam, foundflatfile, fmt.Errorf("LoadIPAMConfig Flatfile (%s) - JSON Parsing Error: %s / bytes: %s", confpath, err, jsonBytes)
+			}
+
+			foundflatfile = confpath
+			return flatipam, foundflatfile, err
+		}
+	}
+	var err error
+	return flatipam, foundflatfile, err
+}
+
 func handleEnvArgs(n *types.Net, numV6 int, numV4 int, args types.IPAMEnvArgs) (int, int, error) {
 
 	if args.IP != "" {