Merge pull request #31 from TheRealSibasishBehera/main

Elastisearch client in Relay server

deployments/ek-dashboards/es-sts.yaml

@@ -0,0 +1,77 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: es-cluster
+  namespace: kube-system
+spec:
+  serviceName: elasticsearch
+  replicas: 3
+  selector:
+    matchLabels:
+      app: elasticsearch
+  template:
+    metadata:
+      labels:
+        app: elasticsearch
+    spec:
+      containers:
+      - name: elasticsearch
+        image: docker.elastic.co/elasticsearch/elasticsearch:7.2.0
+        resources:
+            limits:
+              cpu: 1000m
+            requests:
+              cpu: 100m
+        ports:
+        - containerPort: 9200
+          name: rest
+          protocol: TCP
+        - containerPort: 9300
+          name: inter-node
+          protocol: TCP
+        volumeMounts:
+        - name: data
+          mountPath: /usr/share/elasticsearch/data
+        env:
+          - name: cluster.name
+            value: k8s-logs
+          - name: node.name
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: discovery.seed_hosts
+            value: "es-cluster-0.elasticsearch,es-cluster-1.elasticsearch,es-cluster-2.elasticsearch"
+          - name: cluster.initial_master_nodes
+            value: "es-cluster-0,es-cluster-1,es-cluster-2"
+          - name: ES_JAVA_OPTS
+            value: "-Xms512m -Xmx512m"
+      initContainers:
+      - name: fix-permissions
+        image: busybox
+        command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"]
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: data
+          mountPath: /usr/share/elasticsearch/data
+      - name: increase-vm-max-map
+        image: busybox
+        command: ["sysctl", "-w", "vm.max_map_count=262144"]
+        securityContext:
+          privileged: true
+      - name: increase-fd-ulimit
+        image: busybox
+        command: ["sh", "-c", "ulimit -n 65536"]
+        securityContext:
+          privileged: true
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+      labels:
+        app: elasticsearch
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      # storageClassName: ""
+      resources:
+        requests:
+          storage: 5Gi

deployments/ek-dashboards/es-svc.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: elasticsearch
+  namespace: kube-system
+  labels:
+    app: elasticsearch
+    namespace: kube-system
+spec:
+  selector:
+    app: elasticsearch
+  clusterIP: None
+  ports:
+    - port: 9200
+      name: rest
+    - port: 9300
+      name: inter-node
+

deployments/ek-dashboards/kibana-deployment.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kibana
+  labels:
+    app: kibana
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kibana
+  template:
+    metadata:
+      labels:
+        app: kibana
+    spec:
+      containers:
+      - name: kibana
+        image: docker.elastic.co/kibana/kibana:7.2.0
+        resources:
+          limits:
+            cpu: 1000m
+          requests:
+            cpu: 100m
+        env:
+          - name: ELASTICSEARCH_URL
+            value: http://elasticsearch:9200
+        ports:
+        - containerPort: 5601

deployments/ek-dashboards/kibana-svc.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kibana-np
+  namespace: kube-system
+spec:
+  selector: 
+    app: kibana
+  type: NodePort  
+  ports:
+    - port: 8080
+      targetPort: 5601 
+      nodePort: 30000

deployments/relay-deployment.yaml

@@ -33,6 +33,13 @@ spec:
       containers:
       - name: kubearmor-relay-server
         image: kubearmor/kubearmor-relay-server:latest
+        env:
+          - name: ENABLE_DASHBOARDS
+            value: "true"
+          - name: ES_URL
+            value: "http://elasticsearch:9200"
+          - name: KUBEARMOR_SERVICE
+            value: ""
         imagePullPolicy: Always
         ports:
         - containerPort: 32767

docs/ek-dashboards/ek-dashboards.md

@@ -0,0 +1,65 @@
+# kubearmor-elasticstack-logging
+
+
+## Elastic Stack Visualisation
+
+
+There would be 2 additional components along with the Relay server:
+
+1. **Elasticsearch** is a real-time, distributed, and scalable search engine which allows for full-text and structured search, as well as analytics. Relay server logs can be indexed and searched through which would be produced in large volumes of log data.
+
+
+2. **Kibana** is a data visualization frontend and dashboard for Elasticsearch. Kibana allows user to explore the log data in a visual manner that is stored in the Elasticsearch instance with the help of a web interface. Users would also be allowed to build dashboards or view existing ones which would help to answer and quickly gain insight about the pods managed by KubeArmor:
+
+- Alert Metric
+- Alert from Different Pods
+- Alert from Different Namespace 
+- Alert based on Operations
+- Policy and Action Summary 
+- NameSpace Matched Policy Count
+- Namespace Severity Summary
+- Alert Based on Tags
+
+Kibana will be a part of deployment , while  ElasticSearch will be a part of StatefulSet that can run in any node
+
+```
+kubectl apply -f deployments/ek-dashboards
+```
+
+For the log data to be sent to elasticsearch, change the values of ```ENABLE_DASHBOARDS``` in ```deployments/relay-deployment.yaml>spec>template>spec>container>env``` to ```true``` , it should look like 
+
+```
+    .......
+
+      containers:
+      - name: kubearmor-relay-server
+        image: kubearmor/kubearmor-relay-server:latest
+        env:
+          - name: ENABLE_DASHBOARDS
+            value: "true"
+    .......
+
+```
+
+To View the DashBoards
+
+* Portforward the Kibana service
+```
+kubectl port-forward deployment/kibana -n kube-system 5601:5601
+```
+* Open up a browser and go to [localhost:5601](localhost:5601)
+* Go to sidebar and open ``Mangement`` -> ``Saved Objects`` -> ``Import``
+
+Drag and drop the file from ```docs/ek-dashboards/export.ndjson```
+
+* Go to ``Dashboard`` section , selct ``KA``
+
+* The visalisations should be ready !!
+
+Here are some example visulisation with [multiubuntu](https://github.com/kubearmor/KubeArmor/blob/main/examples/multiubuntu.md) and [wordpress-mysql](https://github.com/kubearmor/KubeArmor/blob/main/examples/wordpress-mysql.md) example
+
+![Dash Board 2](./dash-2.png)
+![Dash Board 1](./dash-1.png)
+
+
+