Tomcat and MySQL example (open-telemetry#1214)

examples/tomcat-mysql/.gitignore

@@ -0,0 +1 @@
+sample.war

examples/tomcat-mysql/README.md

@@ -0,0 +1,26 @@
+# Tomcat and MySQL example
+
+This example shows how the OpenTelemetry Collector can collect data from Apache Tomcat and MySQL, and send it to Splunk Enterprise.
+
+## Set up
+
+This example will download the sample.war file from the [Apache Tomcat website](https://tomcat.apache.org/tomcat-7.0-doc/appdev/sample/).
+
+
+To deploy the example:
+1. Check out the [Splunk OpenTelemetry Collector repository](https://github.com/signalfx/splunk-otel-collector).
+2. Open a terminal.
+3. Type the following commands:
+```bash
+$> cd examples/tomcat-mysql
+$> curl https://tomcat.apache.org/tomcat-7.0-doc/appdev/sample/sample.war
+$> docker-compose up
+```
+You can stop the example by pressing Ctrl + C.
+
+Splunk Enterprise becomes available on port 18000. Log in to [http://localhost:18000](http://localhost:18000) with the user name `admin` and password `changeme`.
+
+From there, you can see logs flowing in by searching for `index="logs"`.
+
+You can visit `http://localhost:8080/sample` to visit the sample application. This will generate Apache Tomcat access logs.
+

examples/tomcat-mysql/checkpoint/.gitignore

@@ -0,0 +1 @@
+*

examples/tomcat-mysql/docker-compose.yml

@@ -0,0 +1,47 @@
+services:
+  mysql:
+    image: "mysql:8.0.28"
+    container_name: mysql
+    environment:
+      - "MYSQL_ROOT_PASSWORD=root"
+    volumes:
+      - ./logs/:/var/log/mysql/
+      - ./mysql_logging.cnf:/etc/mysql/conf.d/mysql_logging.cnf
+  tomcat:
+    image: "tomcat:8-jdk8-corretto"
+    container_name: tomcat
+    volumes:
+      - ./logs/:/usr/local/tomcat/logs
+      - ./sample.war:/usr/local/tomcat/webapps/sample.war
+    ports:
+      - "8080:8080"
+  # Splunk Enterprise server:
+  splunk:
+    image: splunk/splunk:latest
+    container_name: splunk
+    environment:
+      - SPLUNK_START_ARGS=--accept-license
+      - SPLUNK_HEC_TOKEN=00000000-0000-0000-0000-0000000000000
+      - SPLUNK_PASSWORD=changeme
+    ports:
+      - 18000:8000
+    healthcheck:
+      test: [ 'CMD', 'curl', '-f', 'http://localhost:8000' ]
+      interval: 5s
+      timeout: 5s
+      retries: 20
+    volumes:
+      - ./splunk.yml:/tmp/defaults/default.yml
+      - /opt/splunk/var
+      - /opt/splunk/etc
+  # OpenTelemetry Collector
+  otelcollector:
+    image: quay.io/signalfx/splunk-otel-collector:0.43.0
+    container_name: otelcollector
+    command: [ "--config=/etc/otel-collector-config.yml" ]
+    volumes:
+      - ./otel-collector-config.yml:/etc/otel-collector-config.yml
+      - ./logs:/logs
+      - ./checkpoint:/checkpoint
+    depends_on:
+      - splunk

examples/tomcat-mysql/logs/.gitignore

@@ -0,0 +1,2 @@
+*.log
+*.txt

examples/tomcat-mysql/mysql_logging.cnf

@@ -0,0 +1,3 @@
+[mysqld]
+log-error   = "/var/log/mysql/mysql_error.log"
+general_log  = "on"

examples/tomcat-mysql/otel-collector-config.yml

@@ -0,0 +1,100 @@
+receivers:
+  filelog/tomcat:
+    include: [ /logs/catalina.*, /logs/localhost*.log, /logs/manager*, /logs/host-manager* ]
+    start_at: beginning
+    operators:
+      # 08-Feb-2022 00:29:42.924 INFO
+      - type: regex_parser
+        regex: '^(?P<timestamp_field>\d{2}-\w{3}-\d{4} \d{2}:\d{2}:\d{2}\.\d{3})'
+        preserve_to: $$body
+        timestamp:
+          parse_from: timestamp_field
+          layout_type: strptime
+          layout: '%d-%b-%Y %H:%M:%S.%f'
+  filelog/accesslogs:
+    include: [ /logs/*access_log*.txt ]
+    start_at: beginning
+    operators:
+      # 172.18.0.1 - - [08/Feb/2022:01:27:46 +0000] "GET /sample/ HTTP/1.1" 304 -
+      - type: regex_parser
+        regex: '^.*\[(?P<timestamp_field>.*)\].*'
+        preserve_to: $$body
+        timestamp:
+          parse_from: timestamp_field
+          layout_type: strptime
+          layout: '%d/%b/%Y:%H:%M:%S %z'
+  filelog/mysql:
+    include: [ /logs/mysql_error.log ]
+    start_at: beginning
+    operators:
+      # 2022-02-08T01:08:24.758298Z 0 [Warning] [MY-010068] [Server] CA certificate ca.pem is self signed.
+      - type: regex_parser
+        regex: '^(?P<timestamp_field>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}).*$'
+        preserve_to: $$body
+        timestamp:
+          parse_from: timestamp_field
+          layout_type: strptime
+          layout: '%Y-%m-%dT%H:%M:%S.%f'
+
+exporters:
+  splunk_hec/logs:
+    # Splunk HTTP Event Collector token.
+    token: "00000000-0000-0000-0000-0000000000000"
+    # URL to a Splunk instance to send data to.
+    endpoint: "https://splunk:8088/services/collector"
+    # Splunk index, optional name of the Splunk index targeted.
+    index: "logs"
+    # Maximum HTTP connections to use simultaneously when sending data. Defaults to 100.
+    max_connections: 20
+    # Whether to disable gzip compression over HTTP. Defaults to false.
+    disable_compression: false
+    # HTTP timeout when sending data. Defaults to 10s.
+    timeout: 10s
+    # Whether to skip checking the certificate of the HEC endpoint when sending data over HTTPS. Defaults to false.
+    # For this demo, we use a self-signed certificate on the Splunk docker instance, so this flag is set to true.
+    tls:
+      insecure_skip_verify: true
+
+processors:
+  batch:
+  attributes/mysql:
+    actions:
+      - action: insert
+        key: com.splunk.source
+        value: mysql
+  attributes/tomcat:
+    actions:
+      - action: insert
+        key: com.splunk.source
+        value: tomcat
+  attributes/accesslogs:
+    actions:
+      - action: insert
+        key: com.splunk.source
+        value: accesslogs
+
+extensions:
+  health_check:
+    endpoint: 0.0.0.0:13133
+  pprof:
+    endpoint: :1888
+  zpages:
+    endpoint: :55679
+  file_storage:
+    directory: /checkpoint/
+
+service:
+  extensions: [ pprof, zpages, health_check, file_storage ]
+  pipelines:
+    logs/tomcat:
+      receivers: [ filelog/tomcat ]
+      processors: [ batch, attributes/tomcat ]
+      exporters: [ splunk_hec/logs ]
+    logs/accesslogs:
+      receivers: [ filelog/accesslogs ]
+      processors: [ batch, attributes/accesslogs ]
+      exporters: [ splunk_hec/logs ]
+    logs/mysql:
+      receivers: [ filelog/mysql ]
+      processors: [ batch,attributes/mysql ]
+      exporters: [ splunk_hec/logs ]

examples/tomcat-mysql/splunk.yml

@@ -0,0 +1,11 @@
+splunk:
+    conf:
+      indexes:
+        directory: /opt/splunk/etc/apps/search/local
+        content:
+          logs:
+            coldPath: $SPLUNK_DB/logs/colddb
+            datatype: event
+            homePath: $SPLUNK_DB/logs/db
+            maxTotalDataSizeMB: 512000
+            thawedPath: $SPLUNK_DB/logs/thaweddb