diff --git a/examples/tomcat-mysql/.gitignore b/examples/tomcat-mysql/.gitignore new file mode 100644 index 00000000000..f0ed23f0859 --- /dev/null +++ b/examples/tomcat-mysql/.gitignore @@ -0,0 +1 @@ +sample.war \ No newline at end of file diff --git a/examples/tomcat-mysql/README.md b/examples/tomcat-mysql/README.md new file mode 100644 index 00000000000..ff61cb524b3 --- /dev/null +++ b/examples/tomcat-mysql/README.md @@ -0,0 +1,26 @@ +# Tomcat and MySQL example + +This example shows how the OpenTelemetry Collector can collect data from Apache Tomcat and MySQL, and send it to Splunk Enterprise. + +## Set up + +This example will download the sample.war file from the [Apache Tomcat website](https://tomcat.apache.org/tomcat-7.0-doc/appdev/sample/). + + +To deploy the example: +1. Check out the [Splunk OpenTelemetry Collector repository](https://github.com/signalfx/splunk-otel-collector). +2. Open a terminal. +3. Type the following commands: +```bash +$> cd examples/tomcat-mysql +$> curl https://tomcat.apache.org/tomcat-7.0-doc/appdev/sample/sample.war +$> docker-compose up +``` +You can stop the example by pressing Ctrl + C. + +Splunk Enterprise becomes available on port 18000. Log in to [http://localhost:18000](http://localhost:18000) with the user name `admin` and password `changeme`. + +From there, you can see logs flowing in by searching for `index="logs"`. + +You can visit `http://localhost:8080/sample` to visit the sample application. This will generate Apache Tomcat access logs. + diff --git a/examples/tomcat-mysql/checkpoint/.gitignore b/examples/tomcat-mysql/checkpoint/.gitignore new file mode 100644 index 00000000000..f59ec20aabf --- /dev/null +++ b/examples/tomcat-mysql/checkpoint/.gitignore @@ -0,0 +1 @@ +* \ No newline at end of file diff --git a/examples/tomcat-mysql/docker-compose.yml b/examples/tomcat-mysql/docker-compose.yml new file mode 100644 index 00000000000..62eec480ff3 --- /dev/null +++ b/examples/tomcat-mysql/docker-compose.yml @@ -0,0 +1,47 @@ +services: + mysql: + image: "mysql:8.0.28" + container_name: mysql + environment: + - "MYSQL_ROOT_PASSWORD=root" + volumes: + - ./logs/:/var/log/mysql/ + - ./mysql_logging.cnf:/etc/mysql/conf.d/mysql_logging.cnf + tomcat: + image: "tomcat:8-jdk8-corretto" + container_name: tomcat + volumes: + - ./logs/:/usr/local/tomcat/logs + - ./sample.war:/usr/local/tomcat/webapps/sample.war + ports: + - "8080:8080" + # Splunk Enterprise server: + splunk: + image: splunk/splunk:latest + container_name: splunk + environment: + - SPLUNK_START_ARGS=--accept-license + - SPLUNK_HEC_TOKEN=00000000-0000-0000-0000-0000000000000 + - SPLUNK_PASSWORD=changeme + ports: + - 18000:8000 + healthcheck: + test: [ 'CMD', 'curl', '-f', 'http://localhost:8000' ] + interval: 5s + timeout: 5s + retries: 20 + volumes: + - ./splunk.yml:/tmp/defaults/default.yml + - /opt/splunk/var + - /opt/splunk/etc + # OpenTelemetry Collector + otelcollector: + image: quay.io/signalfx/splunk-otel-collector:0.43.0 + container_name: otelcollector + command: [ "--config=/etc/otel-collector-config.yml" ] + volumes: + - ./otel-collector-config.yml:/etc/otel-collector-config.yml + - ./logs:/logs + - ./checkpoint:/checkpoint + depends_on: + - splunk diff --git a/examples/tomcat-mysql/logs/.gitignore b/examples/tomcat-mysql/logs/.gitignore new file mode 100644 index 00000000000..7bd20a6a77a --- /dev/null +++ b/examples/tomcat-mysql/logs/.gitignore @@ -0,0 +1,2 @@ +*.log +*.txt \ No newline at end of file diff --git a/examples/tomcat-mysql/mysql_logging.cnf b/examples/tomcat-mysql/mysql_logging.cnf new file mode 100644 index 00000000000..8b618057cf1 --- /dev/null +++ b/examples/tomcat-mysql/mysql_logging.cnf @@ -0,0 +1,3 @@ +[mysqld] +log-error = "/var/log/mysql/mysql_error.log" +general_log = "on" \ No newline at end of file diff --git a/examples/tomcat-mysql/otel-collector-config.yml b/examples/tomcat-mysql/otel-collector-config.yml new file mode 100644 index 00000000000..1e892eed038 --- /dev/null +++ b/examples/tomcat-mysql/otel-collector-config.yml @@ -0,0 +1,100 @@ +receivers: + filelog/tomcat: + include: [ /logs/catalina.*, /logs/localhost*.log, /logs/manager*, /logs/host-manager* ] + start_at: beginning + operators: + # 08-Feb-2022 00:29:42.924 INFO + - type: regex_parser + regex: '^(?P\d{2}-\w{3}-\d{4} \d{2}:\d{2}:\d{2}\.\d{3})' + preserve_to: $$body + timestamp: + parse_from: timestamp_field + layout_type: strptime + layout: '%d-%b-%Y %H:%M:%S.%f' + filelog/accesslogs: + include: [ /logs/*access_log*.txt ] + start_at: beginning + operators: + # 172.18.0.1 - - [08/Feb/2022:01:27:46 +0000] "GET /sample/ HTTP/1.1" 304 - + - type: regex_parser + regex: '^.*\[(?P.*)\].*' + preserve_to: $$body + timestamp: + parse_from: timestamp_field + layout_type: strptime + layout: '%d/%b/%Y:%H:%M:%S %z' + filelog/mysql: + include: [ /logs/mysql_error.log ] + start_at: beginning + operators: + # 2022-02-08T01:08:24.758298Z 0 [Warning] [MY-010068] [Server] CA certificate ca.pem is self signed. + - type: regex_parser + regex: '^(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}).*$' + preserve_to: $$body + timestamp: + parse_from: timestamp_field + layout_type: strptime + layout: '%Y-%m-%dT%H:%M:%S.%f' + +exporters: + splunk_hec/logs: + # Splunk HTTP Event Collector token. + token: "00000000-0000-0000-0000-0000000000000" + # URL to a Splunk instance to send data to. + endpoint: "https://splunk:8088/services/collector" + # Splunk index, optional name of the Splunk index targeted. + index: "logs" + # Maximum HTTP connections to use simultaneously when sending data. Defaults to 100. + max_connections: 20 + # Whether to disable gzip compression over HTTP. Defaults to false. + disable_compression: false + # HTTP timeout when sending data. Defaults to 10s. + timeout: 10s + # Whether to skip checking the certificate of the HEC endpoint when sending data over HTTPS. Defaults to false. + # For this demo, we use a self-signed certificate on the Splunk docker instance, so this flag is set to true. + tls: + insecure_skip_verify: true + +processors: + batch: + attributes/mysql: + actions: + - action: insert + key: com.splunk.source + value: mysql + attributes/tomcat: + actions: + - action: insert + key: com.splunk.source + value: tomcat + attributes/accesslogs: + actions: + - action: insert + key: com.splunk.source + value: accesslogs + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + pprof: + endpoint: :1888 + zpages: + endpoint: :55679 + file_storage: + directory: /checkpoint/ + +service: + extensions: [ pprof, zpages, health_check, file_storage ] + pipelines: + logs/tomcat: + receivers: [ filelog/tomcat ] + processors: [ batch, attributes/tomcat ] + exporters: [ splunk_hec/logs ] + logs/accesslogs: + receivers: [ filelog/accesslogs ] + processors: [ batch, attributes/accesslogs ] + exporters: [ splunk_hec/logs ] + logs/mysql: + receivers: [ filelog/mysql ] + processors: [ batch,attributes/mysql ] + exporters: [ splunk_hec/logs ] diff --git a/examples/tomcat-mysql/splunk.yml b/examples/tomcat-mysql/splunk.yml new file mode 100644 index 00000000000..a7c67c12448 --- /dev/null +++ b/examples/tomcat-mysql/splunk.yml @@ -0,0 +1,11 @@ +splunk: + conf: + indexes: + directory: /opt/splunk/etc/apps/search/local + content: + logs: + coldPath: $SPLUNK_DB/logs/colddb + datatype: event + homePath: $SPLUNK_DB/logs/db + maxTotalDataSizeMB: 512000 + thawedPath: $SPLUNK_DB/logs/thaweddb