diff --git a/addon-logs/logs-kube-kafka.yml b/addon-logs/logs-kube-kafka.yml deleted file mode 100644 index 7b1f49ee..00000000 --- a/addon-logs/logs-kube-kafka.yml +++ /dev/null @@ -1,46 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: DaemonSet -metadata: - name: logs-kafka - namespace: kube-system -spec: - template: - metadata: - labels: - k8s-app: logs-kafka - version: v1 - kubernetes.io/cluster-service: "true" - spec: - containers: - - name: kafkacat - image: solsson/kafkacat@sha256:ebebf47061300b14a4b4c2e1e4303ab29f65e4b95d34af1b14bb8f7ec6da7cef - command: - - sh - - -ec - - > - cd /var/log/containers/; - tail - -n 0 - --follow=name - *.log - | - kafkacat - -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 - -t ops-kube-logs-raw-001 - -P - #-z snappy - resources: - limits: - memory: 100Mi - requests: - cpu: 100m - memory: 100Mi - volumeMounts: - - name: varlog - mountPath: /var/log - readOnly: true - terminationGracePeriodSeconds: 10 - volumes: - - name: varlog - hostPath: - path: /var/log diff --git a/logs-streaming/logs-kube-kafka-raw.yml b/logs-streaming/logs-kube-kafka-raw.yml new file mode 100644 index 00000000..4470ed56 --- /dev/null +++ b/logs-streaming/logs-kube-kafka-raw.yml @@ -0,0 +1,82 @@ +apiVersion: apps/v1beta2 +kind: DaemonSet +metadata: + name: logs-kafka-raw + namespace: logs-kafka +spec: + selector: + matchLabels: + k8s-app: logs-kafka + log-wrapping: none + container-delimiters: tail + version: v1 + kubernetes.io/cluster-service: "true" + template: + metadata: + labels: + k8s-app: logs-kafka + log-wrapping: none + container-delimiters: tail + version: v1 + kubernetes.io/cluster-service: "true" + spec: + # maybe the /bin/sh does not forward signals? + terminationGracePeriodSeconds: 5 + containers: + - name: kafkacat + image: solsson/kafkacat@sha256:b32eedf936f3cde44cd164ddc77dfcf7565a8af4e357ff6de1abe4389ca530c9 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - sh + - -ec + - > + find /var/log/containers/ -maxdepth 1 -name *.log | grep -v "/$HOSTNAME" | tee /tmp/start-list; + cd /var/log/containers/; + tail + -n 0 + --follow=name + *.log + | + kafkacat + -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 + -t ops-kube-logs-raw-001 + -P + -z snappy + -v + -d broker,topic + -k $NODE_NAME + livenessProbe: + exec: + command: + - /bin/sh + - -c + - find /var/log/containers/ -maxdepth 1 -name *.log | grep -v "/$HOSTNAME" | diff - /tmp/start-list + failureThreshold: 1 + initialDelaySeconds: 5 + periodSeconds: 5 + resources: + requests: + cpu: 2m + memory: 4Mi + limits: + cpu: 10m + memory: 4Mi + volumeMounts: + - name: varlog + mountPath: /var/log + readOnly: true + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + terminationGracePeriodSeconds: 10 + volumes: + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers diff --git a/logs-streaming/test/logs-raw.yml b/logs-streaming/test/logs-raw.yml new file mode 100644 index 00000000..dbe9dc1d --- /dev/null +++ b/logs-streaming/test/logs-raw.yml @@ -0,0 +1,75 @@ +--- +kind: ConfigMap +metadata: + name: logs-raw + namespace: test-kafka +apiVersion: v1 +data: + + setup.sh: |- + touch /tmp/testlog + + tail -f /tmp/testlog + + test.sh: |- + exec >> /tmp/testlog + exec 2>&1 + + echo "" + echo "{\"#---\":\"$(date -u --iso-8601='ns')\"}" + kafkacat -b $BOOTSTRAP -C -t $TOPIC -f '\n%t@%p [%o] %k: %s\n' -o -10 -e + + exit 0 + + quit-on-nonzero-exit.sh: |- + exit 0 + +--- +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: logs-raw + namespace: test-kafka +spec: + replicas: 1 + selector: + matchLabels: + test-target: logs-streaming-raw + test-type: readiness + template: + metadata: + labels: + test-target: logs-streaming-raw + test-type: readiness + spec: + containers: + - name: testcase + image: solsson/kafkacat@sha256:ebebf47061300b14a4b4c2e1e4303ab29f65e4b95d34af1b14bb8f7ec6da7cef + env: + - name: BOOTSTRAP + value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 + - name: TOPIC + value: ops-kube-logs-raw-001 + command: + - /bin/bash + - -e + - /test/setup.sh + readinessProbe: + exec: + command: + - /bin/bash + - -e + - /test/test.sh + livenessProbe: + exec: + command: + - /bin/bash + - -e + - /test/quit-on-nonzero-exit.sh + volumeMounts: + - name: config + mountPath: /test + volumes: + - name: config + configMap: + name: logs-raw diff --git a/addon-logs/topic-ops-kube-logs-raw.yml b/logs-streaming/topic-ops-kube-logs-raw.yml similarity index 83% rename from addon-logs/topic-ops-kube-logs-raw.yml rename to logs-streaming/topic-ops-kube-logs-raw.yml index 9fd9418b..a271cbac 100644 --- a/addon-logs/topic-ops-kube-logs-raw.yml +++ b/logs-streaming/topic-ops-kube-logs-raw.yml @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: Job metadata: name: topic-ops-kube-logs-raw - namespace: kafka + namespace: logs-kafka spec: template: metadata: @@ -17,7 +17,7 @@ spec: command: - ./bin/kafka-topics.sh - --zookeeper - - zookeeper:2181 + - zookeeper.kafka:2181 - --create - --if-not-exists - --topic @@ -25,8 +25,7 @@ spec: - --partitions - "1" - --replication-factor - - "1" + - "2" - --config - # this might be eight days - - retention.ms=69125000 + - retention.ms=6912500000 restartPolicy: Never diff --git a/test/basic-produce-consume.yml b/test/basic-produce-consume.yml index 128a00a9..66dda328 100644 --- a/test/basic-produce-consume.yml +++ b/test/basic-produce-consume.yml @@ -18,9 +18,6 @@ data: tail -f /tmp/testlog - continue.sh: |- - exit 0 - run.sh: |- exec >> /tmp/testlog exec 2>&1 @@ -37,6 +34,9 @@ data: exit 0 + continue.sh: |- + exit 0 + --- apiVersion: apps/v1beta1 kind: Deployment diff --git a/test/basic-with-kafkacat.yml b/test/basic-with-kafkacat.yml index 405a7980..cb6f6675 100644 --- a/test/basic-with-kafkacat.yml +++ b/test/basic-with-kafkacat.yml @@ -10,9 +10,6 @@ data: touch /tmp/testlog tail -f /tmp/testlog - continue.sh: |- - exit 0 - run.sh: |- exec >> /tmp/testlog exec 2>&1 @@ -24,6 +21,9 @@ data: exit 0 + continue.sh: |- + exit 0 + --- apiVersion: batch/v1 kind: Job