Add collecting of cluster wide resources (#93)

Signed-off-by: David Kornel <[email protected]>

test-frame-common/src/main/java/io/skodjob/testframe/clients/cmdClient/BaseCmdKubeClient.java

@@ -482,6 +482,20 @@ public List<String> list(String resourceType) {
                 .filter(s -> !s.trim().isEmpty()).collect(Collectors.toList());
     }
 
+    /**
+     * Lists cluster wide resources of a certain type.
+     *
+     * @param resourceType The type of the resource.
+     * @return A list of resource names.
+     */
+    @Override
+    public List<String> listClusterWide(String resourceType) {
+        return Arrays.stream(Exec.exec(command(List.of(GET, resourceType,
+                                "-o", "jsonpath={range .items[*]}{.metadata.name} ")))
+                        .out().trim().split(" +"))
+                .filter(s -> !s.trim().isEmpty()).collect(Collectors.toList());
+    }
+
     /**
      * Retrieves a resource as JSON.
      *
@@ -517,6 +531,29 @@ public String getResourcesAsYaml(String resourceType) {
         return Exec.exec(namespacedCommand(GET, resourceType, "-o", "yaml")).out();
     }
 
+    /**
+     * Retrieves a  cluster wide resource as YAML.
+     *
+     * @param resourceType The type of the resource.
+     * @param resourceName The name of the resource.
+     * @return The resource as YAML.
+     */
+    @Override
+    public String getClusterWideResourceAsYaml(String resourceType, String resourceName) {
+        return Exec.exec(command(List.of(GET, resourceType, resourceName, "-o", "yaml"))).out();
+    }
+
+    /**
+     * Retrieves resources as YAML.
+     *
+     * @param resourceType The type of the resource.
+     * @return The resources as YAML.
+     */
+    @Override
+    public String getClusterWideResourcesAsYaml(String resourceType) {
+        return Exec.exec(command(List.of(GET, resourceType, "-o", "yaml"))).out();
+    }
+
     /**
      * Creates a resource from a template and applies it.
      *

test-frame-common/src/main/java/io/skodjob/testframe/clients/cmdClient/KubeCmdClient.java

@@ -286,6 +286,14 @@ default K delete(String... files) {
      */
     List<String> list(String resourceType);
 
+    /**
+     * Retrieves a list of cluster wide resources by type.
+     *
+     * @param resourceType The type of the resources.
+     * @return The list of resources.
+     */
+    List<String> listClusterWide(String resourceType);
+
     /**
      * Retrieves the YAML content of a resource by type and name.
      *
@@ -303,6 +311,23 @@ default K delete(String... files) {
      */
     String getResourcesAsYaml(String resourceType);
 
+    /**
+     * Retrieves the YAML content of a cluster wide resource by type and name.
+     *
+     * @param resourceType The type of the resource.
+     * @param resourceName The name of the resource.
+     * @return The YAML content of the resource.
+     */
+    String getClusterWideResourceAsYaml(String resourceType, String resourceName);
+
+    /**
+     * Retrieves the YAML content of cluster wide resources by type.
+     *
+     * @param resourceType The type of the resources.
+     * @return The YAML content of the resources.
+     */
+    String getClusterWideResourcesAsYaml(String resourceType);
+
     /**
      * Creates a resource from a template and applies it.
      *

test-frame-log-collector/README.md

@@ -40,7 +40,7 @@ import io.skodjob.testframe.LogCollector;
 import io.skodjob.testframe.LogCollectorBuilder;
 
 LogCollector logCollector = new LogCollectorBuilder()
-    .withResources("secret", "deployment", "my-custom-resource")
+    .withNamespacedResources("secret", "deployment", "my-custom-resource")
     .withRootFolderPath("/path/to/logs/folder")
     .build();
 ```
@@ -66,7 +66,7 @@ import io.skodjob.testframe.LogCollector;
 import io.skodjob.testframe.LogCollectorBuilder;
 
 LogCollector logCollector = new LogCollectorBuilder()
-    .withResources("secret", "deployment", "my-custom-resource")
+    .withNamespacedResources("secret", "deployment", "my-custom-resource")
     .withRootFolderPath("/path/to/logs/folder")
     .build();
 
@@ -92,7 +92,7 @@ import io.skodjob.testframe.LogCollector;
 import io.skodjob.testframe.LogCollectorBuilder;
 
 LogCollector logCollector = new LogCollectorBuilder()
-    .withResources("secret", "deployment", "my-custom-resource")
+    .withNamespacedResources("secret", "deployment", "my-custom-resource")
     .withRootFolderPath("/path/to/logs/folder")
     .build();
 
@@ -127,17 +127,46 @@ import io.skodjob.testframe.LogCollector;
 import io.skodjob.testframe.LogCollectorBuilder;
 
 LogCollector logCollector = new LogCollectorBuilder()
-    .withResources("secret", "deployment", "my-custom-resource")
+    .withNamespacedResources("secret", "deployment", "my-custom-resource")
     .withRootFolderPath("/path/to/logs/folder")
     .build();
 
 public static void collectFromNamespaces() {
     logCollector.collectFromNamespacesWithLabels(new LabelSelectorBuilder()
-        .withMatchLabels(Map.of("my-label", "my-value"))
+            .withMatchLabels(Map.of("my-label", "my-value"))
     );
 }
 ```
 
+#### 4. Collect cluster wide resource lists
+
+Collect cluster wide resource lists yaml like `nodes`, `pvs`:
+
+```java
+import io.fabric8.kubernetes.api.model.LabelSelectorBuilder;
+import io.skodjob.testframe.LogCollector;
+import io.skodjob.testframe.LogCollectorBuilder;
+
+LogCollector logCollector = new LogCollectorBuilder()
+    .withClusterWideResources("nodes", "pvs")
+    .withRootFolderPath("/path/to/logs/folder")
+    .build();
+
+public static void collectClusterWideRes() {
+    logCollector.collectClusterWideResources();
+}
+```
+the logs path will then look like this:
+```bash
+/path/to/logs/folder
+└── cluster-wide-resources
+    ├── nodes
+    │   ├── node1.yaml
+    │   └── node2.yaml
+    └── pvs
+        ├── pv1.yaml
+        └── pv2.yaml
+```
 The tree path will look similarly to above examples, there will be folders for Namespaces matching the specified labels.
 
 ### Specifying additional folder path

test-frame-log-collector/src/main/java/io/skodjob/testframe/CollectorConstants.java

@@ -32,4 +32,9 @@ public interface CollectorConstants {
      * Prefix for description files
      */
     String DESCRIBE = "describe";
+
+    /**
+     * Cluster wide resources folder name
+     */
+    String CLUSTER_WIDE_FOLDER = "cluster-wide-resources";
 }

test-frame-log-collector/src/main/java/io/skodjob/testframe/LogCollector.java

@@ -10,7 +10,6 @@
 import io.skodjob.testframe.clients.KubeClient;
 import io.skodjob.testframe.clients.cmdClient.KubeCmdClient;
 import io.skodjob.testframe.clients.cmdClient.Kubectl;
-import io.skodjob.testframe.resources.KubeResourceManager;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -27,10 +26,11 @@
  * LogCollector class containing all methods used for logs and YAML collection.
  */
 public class LogCollector {
-    private static final Logger LOGGER = LogManager.getLogger(KubeResourceManager.class);
-    protected final List<String> resources;
+    private static final Logger LOGGER = LogManager.getLogger(LogCollector.class);
+    protected final List<String> namespacedResources;
+    protected final List<String> clusterWideResources;
     protected String rootFolderPath;
-    private KubeCmdClient<Kubectl> kubeCmdClient = new Kubectl();
+    private KubeCmdClient<?> kubeCmdClient = new Kubectl();
     private KubeClient kubeClient = new KubeClient();
 
     /**
@@ -39,33 +39,24 @@ public class LogCollector {
      * @param builder   {@link LogCollectorBuilder} with configuration for {@link LogCollector}
      */
     public LogCollector(LogCollectorBuilder builder) {
-        this.resources = builder.getResources() == null ? Collections.emptyList() : builder.getResources();
+        this.namespacedResources = builder.getNamespacedResources() == null ?
+                Collections.emptyList() : builder.getNamespacedResources();
+        this.clusterWideResources = builder.getClusterWideResources() == null ?
+                Collections.emptyList() : builder.getClusterWideResources();
 
         if (builder.getRootFolderPath() == null) {
             throw new RuntimeException("rootFolderPath should be filled, but it's empty");
         }
 
-        this.rootFolderPath = builder.getRootFolderPath();
-    }
+        if (builder.getKubeClient() != null) {
+            this.kubeClient = builder.getKubeClient();
+        }
 
-    /**
-     * Sets {@link KubeClient} for the LogCollector.
-     * Mainly for testing purposes (to mock the client).
-     *
-     * @param kubeClient     {@link KubeClient} - usually mocked.
-     */
-    /* test */ public void setKubeClient(KubeClient kubeClient) {
-        this.kubeClient = kubeClient;
-    }
+        if (builder.getKubeCmdClient() != null) {
+            this.kubeCmdClient = builder.getKubeCmdClient();
+        }
 
-    /**
-     * Sets {@link KubeCmdClient} for the LogCollector.
-     * Mainly for testing purposes (to mock the client).
-     *
-     * @param kubeCmdClient     {@link KubeCmdClient} - usually mocked.
-     */
-    /* test */ public void setKubeCmdClient(KubeCmdClient<Kubectl> kubeCmdClient) {
-        this.kubeCmdClient = kubeCmdClient;
+        this.rootFolderPath = builder.getRootFolderPath();
     }
 
     /**
@@ -129,6 +120,24 @@ public void collectFromNamespace(String namespaceName) {
         collectFromNamespaceToFolder(namespaceName, null);
     }
 
+    /**
+     * Method that collects YAML of cluster wide resources
+     * {@link #rootFolderPath}.
+     */
+    public void collectClusterWideResources() {
+        collectClusterWideResources(true);
+    }
+
+    /**
+     * Method that collects YAML of cluster wide resources
+     * {@link #rootFolderPath}.
+     *
+     * @param logPerResource    flag enables cluster wide resource per file
+     */
+    public void collectClusterWideResources(boolean logPerResource) {
+        collectClusterWideResourcesToFolder(logPerResource, null);
+    }
+
     /**
      * Method that collects all logs and YAML files from specified Namespace, collected into
      * {@link #rootFolderPath} with {@param folderPath}.
@@ -151,6 +160,43 @@ public void collectFromNamespaceToFolder(String namespaceName, String folderPath
         }
     }
 
+    /**
+     * Method that collects YAML of cluster wide resources
+     * {@link #rootFolderPath} with {@param folderPath}.
+     *
+     * @param folderPath        folder path for the log collection
+     */
+    public void collectClusterWideResourcesToFolder(String folderPath) {
+        collectClusterWideResourcesToFolder(true, folderPath);
+    }
+
+    /**
+     * Method that collects YAML of cluster wide resources
+     * {@link #rootFolderPath} with {@param folderPath}.
+     *
+     * @param logPerFile        flag enables cluster wide resource per file
+     * @param folderPath        folder path for the log collection
+     */
+    public void collectClusterWideResourcesToFolder(boolean logPerFile, String folderPath) {
+        clusterWideResources.forEach(resourceType -> {
+            LOGGER.info("Collecting YAMLs of {}", resourceType);
+
+            String clusterWideFolderPath = createNamespaceDirectory(CollectorConstants.CLUSTER_WIDE_FOLDER,
+                LogCollectorUtils.getFolderPath(rootFolderPath, folderPath));
+            createLogDirOnPath(clusterWideFolderPath);
+
+            if (logPerFile) {
+                collectClusterWideResourcesPerFile(clusterWideFolderPath, resourceType);
+            } else {
+                String yaml = kubeCmdClient.getClusterWideResourcesAsYaml(resourceType);
+                String resFileName = LogCollectorUtils.getYamlFileNameForResource(resourceType);
+                String filePath = LogCollectorUtils
+                    .getFullPathForFolderPathAndFileName(clusterWideFolderPath, resFileName);
+                writeDataToFile(filePath, yaml);
+            }
+        });
+    }
+
     /**
      * Method for collecting logs from all Pods (and their containers) in specified Namespace.
      * At the start, it creates a folder in the Namespace dir for the `pod` resource.
@@ -238,6 +284,27 @@ private void collectPodDescription(String namespaceName, String podsFolderPath,
         writeDataToFile(filePath, podDesc);
     }
 
+    /**
+     * Collect cluster wide resource per file
+     *
+     * @param clusterWideFolderPath root path of cluster wide resource
+     * @param resourceType resource kind for collect
+     */
+    private void collectClusterWideResourcesPerFile(String clusterWideFolderPath,String resourceType) {
+        List<String> resources = kubeCmdClient.listClusterWide(resourceType);
+        if (resources != null && !resources.isEmpty()) {
+            String fullFolderPath = createResourceDirectoryInNamespaceDir(clusterWideFolderPath, resourceType);
+
+            resources.forEach(resourceName -> {
+                String yaml = kubeCmdClient.getClusterWideResourceAsYaml(resourceType, resourceName);
+
+                String resFileName = LogCollectorUtils.getYamlFileNameForResource(resourceName);
+                String fileName = LogCollectorUtils.getFullPathForFolderPathAndFileName(fullFolderPath, resFileName);
+                writeDataToFile(fileName, yaml);
+            });
+        }
+    }
+
     /**
      * Method that collects all Events (kubectl get events) from Namespace
      *
@@ -261,7 +328,7 @@ public void collectEventsFromNamespace(String namespaceName, String namespaceFol
      * @param namespaceFolderPath   path to Namespace folder where the resource directories will be created
      */
     private void collectResourcesDescInNamespace(String namespaceName, String namespaceFolderPath) {
-        resources.forEach(resource ->
+        namespacedResources.forEach(resource ->
             collectDescriptionOfResourceInNamespace(namespaceName, namespaceFolderPath, resource));
     }