FLUO: Add reboot test

This commit adds a reboot test for FLUO, the test verifies if the FLUO
can reboot a node running stateless applications.

Signed-off-by: Suraj Deshmukh <[email protected]>

ci/packet_fluo/packet_fluo-cluster.lokocfg.envsubst

@@ -35,6 +35,10 @@ EOF
   worker_pool "general" {
     count     = 1
     node_type = "c2.medium.x86"
+
+    labels = {
+      "fluo-test-pool" = "true"
+    }
   }
 
   worker_pool "storage" {

test/components/flatcar-linux-update-operator/fluo_reboot_test.go

@@ -0,0 +1,98 @@
+// Copyright 2020 The Lokomotive Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build packet_fluo
+// +build disruptivee2e
+
+package fluo_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	testutil "github.com/kinvolk/lokomotive/test/components/util"
+)
+
+//nolint:funlen
+func TestReboot(t *testing.T) {
+	t.Parallel()
+
+	client := testutil.CreateKubeClient(t)
+
+	// Select a node from the general worker pool.
+	nodesList, err := client.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{
+		LabelSelector: "fluo-test-pool=true",
+	})
+	if err != nil {
+		t.Fatalf("Listing nodes with label 'fluo-test-pool=true': %v", err)
+	}
+
+	nodes := nodesList.Items
+	if len(nodes) < 1 {
+		t.Fatalf("Wanted one or more nodes with label 'fluo-test-pool=true', found none.")
+	}
+
+	// Select a node to remove annotation
+	chosenNode := nodes[0]
+	t.Logf("Chosen node to reboot: %s", chosenNode.Name)
+
+	// Remove annotation that disables node reboot by FLUO. This test assumes that the node has
+	// annotation set.
+	annotation := "flatcar-linux-update.v1.flatcar-linux.net/reboot-paused"
+
+	// This annotation should be set on the node during the cluster creation, if it is not set then
+	// something changed in the cluster setup process in the CI.
+	if _, ok := chosenNode.Annotations[annotation]; !ok {
+		t.Fatalf("Annotation %q not found on the node.", annotation)
+	}
+
+	// Remove the annotation that disables node reboot.
+	delete(chosenNode.Annotations, annotation)
+
+	if _, err := client.CoreV1().Nodes().Update(context.Background(), &chosenNode, metav1.UpdateOptions{}); err != nil {
+		t.Fatalf("Removing node annotation %q: %v", annotation, err)
+	}
+
+	// Wait for the FLUO to add following annotation key value pair to the chosen node. This may
+	// involve node reboot if the OS is outdated.
+	// "flatcar-linux-update.v1.flatcar-linux.net/status": "UPDATE_STATUS_IDLE"
+	statusAnnotationKey := "flatcar-linux-update.v1.flatcar-linux.net/status"
+	statusAnnotationVal := "UPDATE_STATUS_IDLE"
+
+	if err := wait.PollImmediate(testutil.RetryInterval, testutil.Timeout, func() (done bool, err error) {
+		node, err := client.CoreV1().Nodes().Get(context.Background(), chosenNode.Name, metav1.GetOptions{})
+		if err != nil {
+			return false, fmt.Errorf("Getting node %q: %v", chosenNode.Name, err)
+		}
+
+		val, ok := node.Annotations[statusAnnotationKey]
+		if !ok {
+			return false, nil
+		}
+
+		// Test passed since the annotation is added by the FLUO to the nodes.
+		if val == statusAnnotationVal {
+			return true, nil
+		}
+
+		return false, nil
+	}); err != nil {
+		t.Fatalf("Waiting for the node to add annotation '%s:%s' %v",
+			statusAnnotationKey, statusAnnotationVal, err)
+	}
+}