From 6ed999b0cbf852a6ef3b6bd4449e190bae059cea Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Thu, 11 Jul 2024 09:39:50 -0700 Subject: [PATCH] [exporter/splunkhec] Increase the performance of JSON marshaling (#34011) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Description:** Switch JSON libs to get more performance out of the JSON marshaler used to marshal events. Before: ``` goarch: arm64 pkg: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/splunkhecexporter Benchmark_pushLogData_10_10_1024-10 11065 106969 ns/op 85002 B/op 1285 allocs/op Benchmark_pushLogData_10_10_8K-10 12757 94831 ns/op 74142 B/op 1120 allocs/op Benchmark_pushLogData_10_10_2M-10 13096 92878 ns/op 73534 B/op 1109 allocs/op Benchmark_pushLogData_10_200_2M-10 650 1797755 ns/op 1460043 B/op 22010 allocs/op Benchmark_pushLogData_100_200_2M-10 63 16883869 ns/op 14664365 B/op 220024 allocs/op Benchmark_pushLogData_100_200_5M-10 61 16599464 ns/op 14769204 B/op 220013 allocs/op Benchmark_pushLogData_compressed_10_10_1024-10 1755 710944 ns/op 3102954 B/op 1193 allocs/op Benchmark_pushLogData_compressed_10_10_8K-10 6984 167371 ns/op 75065 B/op 1109 allocs/op Benchmark_pushLogData_compressed_10_10_2M-10 7285 165637 ns/op 74095 B/op 1109 allocs/op Benchmark_pushLogData_compressed_10_200_2M-10 334 3591145 ns/op 1465106 B/op 22010 allocs/op Benchmark_pushLogData_compressed_100_200_2M-10 31 36341606 ns/op 14599767 B/op 220015 allocs/op Benchmark_pushLogData_compressed_100_200_5M-10 31 36417519 ns/op 14598477 B/op 220014 allocs/op Benchmark_pushMetricData_10_10_1024-10 8422 142076 ns/op 87314 B/op 1810 allocs/op Benchmark_pushMetricData_10_10_8K-10 8280 143053 ns/op 94532 B/op 1810 allocs/op Benchmark_pushMetricData_10_10_2M-10 4083 300601 ns/op 2185220 B/op 1812 allocs/op Benchmark_pushMetricData_10_200_2M-10 397 3053150 ns/op 3818987 B/op 36012 allocs/op Benchmark_pushMetricData_100_200_2M-10 37 33856887 ns/op 21432556 B/op 360034 allocs/op Benchmark_pushMetricData_100_200_5M-10 30 41485322 ns/op 27750779 B/op 360034 allocs/op Benchmark_pushMetricData_compressed_10_10_1024-10 3810 309415 ns/op 89075 B/op 1810 allocs/op Benchmark_pushMetricData_compressed_10_10_8K-10 3828 307569 ns/op 95172 B/op 1810 allocs/op Benchmark_pushMetricData_compressed_10_10_2M-10 2688 453805 ns/op 2184373 B/op 1811 allocs/op Benchmark_pushMetricData_compressed_10_200_2M-10 177 6467414 ns/op 3815937 B/op 36012 allocs/op Benchmark_pushMetricData_compressed_100_200_2M-10 16 66096492 ns/op 19305614 B/op 360017 allocs/op Benchmark_pushMetricData_compressed_100_200_5M-10 16 65041961 ns/op 22450356 B/op 360017 allocs/op Benchmark_pushMetricData_10_10_1024_MultiMetric-10 3320 366943 ns/op 249636 B/op 3934 allocs/op Benchmark_pushMetricData_10_10_8K_MultiMetric-10 3319 361330 ns/op 249566 B/op 3934 allocs/op Benchmark_pushMetricData_10_10_2M_MultiMetric-10 3318 358159 ns/op 249583 B/op 3934 allocs/op Benchmark_pushMetricData_10_200_2M_MultiMetric-10 165 7502432 ns/op 4989958 B/op 78073 allocs/op Benchmark_pushMetricData_100_200_2M_MultiMetric-10 16 69800424 ns/op 49863629 B/op 780369 allocs/op Benchmark_pushMetricData_100_200_5M_MultiMetric-10 15 79718683 ns/op 49880688 B/op 780368 allocs/op Benchmark_pushMetricData_compressed_10_10_1024_MultiMetric-10 2259 539714 ns/op 250326 B/op 3934 allocs/op Benchmark_pushMetricData_compressed_10_10_8K_MultiMetric-10 2360 540889 ns/op 250269 B/op 3934 allocs/op Benchmark_pushMetricData_compressed_10_10_2M_MultiMetric-10 2355 505609 ns/op 249576 B/op 3934 allocs/op Benchmark_pushMetricData_compressed_10_200_2M_MultiMetric-10 100 10467474 ns/op 5000816 B/op 78072 allocs/op Benchmark_pushMetricData_compressed_100_200_2M_MultiMetric-10 10 103846029 ns/op 49736248 B/op 780363 allocs/op Benchmark_pushMetricData_compressed_100_200_5M_MultiMetric-10 10 102504771 ns/op 49734080 B/op 780358 allocs/op BenchmarkConsumeLogsRejected-10 1354 903909 ns/op 731212 B/op 11012 allocs/op ``` After: ``` goarch: arm64 pkg: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/splunkhecexporter Benchmark_pushLogData_10_10_1024-10 14031 84592 ns/op 74842 B/op 821 allocs/op Benchmark_pushLogData_10_10_8K-10 16302 72281 ns/op 65259 B/op 716 allocs/op Benchmark_pushLogData_10_10_2M-10 16766 71721 ns/op 64664 B/op 709 allocs/op Benchmark_pushLogData_10_200_2M-10 824 1448491 ns/op 1314978 B/op 14014 allocs/op Benchmark_pushLogData_100_200_2M-10 73 13899896 ns/op 13208607 B/op 140034 allocs/op Benchmark_pushLogData_100_200_5M-10 78 13690774 ns/op 13282520 B/op 140024 allocs/op Benchmark_pushLogData_compressed_10_10_1024-10 1940 570420 ns/op 2538497 B/op 784 allocs/op Benchmark_pushLogData_compressed_10_10_8K-10 8893 133297 ns/op 66024 B/op 709 allocs/op Benchmark_pushLogData_compressed_10_10_2M-10 8931 134021 ns/op 65465 B/op 709 allocs/op Benchmark_pushLogData_compressed_10_200_2M-10 385 3116794 ns/op 1315532 B/op 14015 allocs/op Benchmark_pushLogData_compressed_100_200_2M-10 37 30985287 ns/op 13149922 B/op 140035 allocs/op Benchmark_pushLogData_compressed_100_200_5M-10 38 31037704 ns/op 13149626 B/op 140033 allocs/op Benchmark_pushMetricData_10_10_1024-10 7464 152713 ns/op 83334 B/op 1410 allocs/op Benchmark_pushMetricData_10_10_8K-10 8011 152011 ns/op 90480 B/op 1410 allocs/op Benchmark_pushMetricData_10_10_2M-10 3855 327686 ns/op 2181771 B/op 1417 allocs/op Benchmark_pushMetricData_10_200_2M-10 367 3238910 ns/op 3735843 B/op 28020 allocs/op Benchmark_pushMetricData_100_200_2M-10 37 30327247 ns/op 20711030 B/op 280055 allocs/op Benchmark_pushMetricData_100_200_5M-10 37 30944880 ns/op 27001581 B/op 280051 allocs/op Benchmark_pushMetricData_compressed_10_10_1024-10 4766 250411 ns/op 84060 B/op 1410 allocs/op Benchmark_pushMetricData_compressed_10_10_8K-10 4627 245920 ns/op 91252 B/op 1410 allocs/op Benchmark_pushMetricData_compressed_10_10_2M-10 2776 411300 ns/op 2185072 B/op 1418 allocs/op Benchmark_pushMetricData_compressed_10_200_2M-10 246 4832722 ns/op 3737099 B/op 28022 allocs/op Benchmark_pushMetricData_compressed_100_200_2M-10 25 46085088 ns/op 18552164 B/op 280037 allocs/op Benchmark_pushMetricData_compressed_100_200_5M-10 25 45760102 ns/op 21754765 B/op 280037 allocs/op Benchmark_pushMetricData_10_10_1024_MultiMetric-10 4110 284592 ns/op 153998 B/op 1835 allocs/op Benchmark_pushMetricData_10_10_8K_MultiMetric-10 4045 284535 ns/op 154088 B/op 1835 allocs/op Benchmark_pushMetricData_10_10_2M_MultiMetric-10 4251 288325 ns/op 154016 B/op 1835 allocs/op Benchmark_pushMetricData_10_200_2M_MultiMetric-10 206 5787823 ns/op 3077042 B/op 36068 allocs/op Benchmark_pushMetricData_100_200_2M_MultiMetric-10 19 57907099 ns/op 30804149 B/op 360348 allocs/op Benchmark_pushMetricData_100_200_5M_MultiMetric-10 19 57568178 ns/op 30805770 B/op 360359 allocs/op Benchmark_pushMetricData_compressed_10_10_1024_MultiMetric-10 3140 370639 ns/op 154425 B/op 1835 allocs/op Benchmark_pushMetricData_compressed_10_10_8K_MultiMetric-10 3278 368228 ns/op 154187 B/op 1835 allocs/op Benchmark_pushMetricData_compressed_10_10_2M_MultiMetric-10 3106 375295 ns/op 154715 B/op 1835 allocs/op Benchmark_pushMetricData_compressed_10_200_2M_MultiMetric-10 168 7128948 ns/op 3077226 B/op 36067 allocs/op Benchmark_pushMetricData_compressed_100_200_2M_MultiMetric-10 15 70611422 ns/op 30676065 B/op 360371 allocs/op Benchmark_pushMetricData_compressed_100_200_5M_MultiMetric-10 15 70289731 ns/op 30673553 B/op 360355 allocs/op BenchmarkConsumeLogsRejected-10 1681 716325 ns/op 655652 B/op 7013 allocs/op PASS ok github.com/open-telemetry/opentelemetry-collector-contrib/exporter/splunkhecexporter 49.913s ``` Benchstat ``` goarch: arm64 pkg: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/splunkhecexporter │ before.txt │ after.txt │ │ sec/op │ sec/op vs base │ _pushLogData_10_10_1024-10 106.97µ ± ∞ ¹ 84.59µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_10_10_8K-10 94.83µ ± ∞ ¹ 72.28µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_10_10_2M-10 92.88µ ± ∞ ¹ 71.72µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_10_200_2M-10 1.798m ± ∞ ¹ 1.448m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_100_200_2M-10 16.88m ± ∞ ¹ 13.90m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_100_200_5M-10 16.60m ± ∞ ¹ 13.69m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_10_10_1024-10 710.9µ ± ∞ ¹ 570.4µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_10_10_8K-10 167.4µ ± ∞ ¹ 133.3µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_10_10_2M-10 165.6µ ± ∞ ¹ 134.0µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_10_200_2M-10 3.591m ± ∞ ¹ 3.117m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_100_200_2M-10 36.34m ± ∞ ¹ 30.99m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_100_200_5M-10 36.42m ± ∞ ¹ 31.04m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_1024-10 142.1µ ± ∞ ¹ 152.7µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_8K-10 143.1µ ± ∞ ¹ 152.0µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_2M-10 300.6µ ± ∞ ¹ 327.7µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_200_2M-10 3.053m ± ∞ ¹ 3.239m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_100_200_2M-10 33.86m ± ∞ ¹ 30.33m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_100_200_5M-10 41.49m ± ∞ ¹ 30.94m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_1024-10 309.4µ ± ∞ ¹ 250.4µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_8K-10 307.6µ ± ∞ ¹ 245.9µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_2M-10 453.8µ ± ∞ ¹ 411.3µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_200_2M-10 6.467m ± ∞ ¹ 4.833m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_100_200_2M-10 66.10m ± ∞ ¹ 46.09m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_100_200_5M-10 65.04m ± ∞ ¹ 45.76m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_1024_MultiMetric-10 366.9µ ± ∞ ¹ 284.6µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_8K_MultiMetric-10 361.3µ ± ∞ ¹ 284.5µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_2M_MultiMetric-10 358.2µ ± ∞ ¹ 288.3µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_200_2M_MultiMetric-10 7.502m ± ∞ ¹ 5.788m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_100_200_2M_MultiMetric-10 69.80m ± ∞ ¹ 57.91m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_100_200_5M_MultiMetric-10 79.72m ± ∞ ¹ 57.57m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_1024_MultiMetric-10 539.7µ ± ∞ ¹ 370.6µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_8K_MultiMetric-10 540.9µ ± ∞ ¹ 368.2µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_2M_MultiMetric-10 505.6µ ± ∞ ¹ 375.3µ ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_200_2M_MultiMetric-10 10.467m ± ∞ ¹ 7.129m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_100_200_2M_MultiMetric-10 103.85m ± ∞ ¹ 70.61m ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_100_200_5M_MultiMetric-10 102.50m ± ∞ ¹ 70.29m ± ∞ ¹ ~ (p=1.000 n=1) ² ConsumeLogsRejected-10 903.9µ ± ∞ ¹ 716.3µ ± ∞ ¹ ~ (p=1.000 n=1) ² geomean 2.333m 1.876m -19.60% ¹ need >= 6 samples for confidence interval at level 0.95 ² need >= 4 samples to detect a difference at alpha level 0.05 │ before.txt │ after.txt │ │ B/op │ B/op vs base │ _pushLogData_10_10_1024-10 83.01Ki ± ∞ ¹ 73.09Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_10_10_8K-10 72.40Ki ± ∞ ¹ 63.73Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_10_10_2M-10 71.81Ki ± ∞ ¹ 63.15Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_10_200_2M-10 1.392Mi ± ∞ ¹ 1.254Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_100_200_2M-10 13.99Mi ± ∞ ¹ 12.60Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_100_200_5M-10 14.09Mi ± ∞ ¹ 12.67Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_10_10_1024-10 2.959Mi ± ∞ ¹ 2.421Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_10_10_8K-10 73.31Ki ± ∞ ¹ 64.48Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_10_10_2M-10 72.36Ki ± ∞ ¹ 63.93Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_10_200_2M-10 1.397Mi ± ∞ ¹ 1.255Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_100_200_2M-10 13.92Mi ± ∞ ¹ 12.54Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_100_200_5M-10 13.92Mi ± ∞ ¹ 12.54Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_1024-10 85.27Ki ± ∞ ¹ 81.38Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_8K-10 92.32Ki ± ∞ ¹ 88.36Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_2M-10 2.084Mi ± ∞ ¹ 2.081Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_200_2M-10 3.642Mi ± ∞ ¹ 3.563Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_100_200_2M-10 20.44Mi ± ∞ ¹ 19.75Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_100_200_5M-10 26.47Mi ± ∞ ¹ 25.75Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_1024-10 86.99Ki ± ∞ ¹ 82.09Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_8K-10 92.94Ki ± ∞ ¹ 89.11Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_2M-10 2.083Mi ± ∞ ¹ 2.084Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_200_2M-10 3.639Mi ± ∞ ¹ 3.564Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_100_200_2M-10 18.41Mi ± ∞ ¹ 17.69Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_100_200_5M-10 21.41Mi ± ∞ ¹ 20.75Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_1024_MultiMetric-10 243.8Ki ± ∞ ¹ 150.4Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_8K_MultiMetric-10 243.7Ki ± ∞ ¹ 150.5Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_2M_MultiMetric-10 243.7Ki ± ∞ ¹ 150.4Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_200_2M_MultiMetric-10 4.759Mi ± ∞ ¹ 2.934Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_100_200_2M_MultiMetric-10 47.55Mi ± ∞ ¹ 29.38Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_100_200_5M_MultiMetric-10 47.57Mi ± ∞ ¹ 29.38Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_1024_MultiMetric-10 244.5Ki ± ∞ ¹ 150.8Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_8K_MultiMetric-10 244.4Ki ± ∞ ¹ 150.6Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_2M_MultiMetric-10 243.7Ki ± ∞ ¹ 151.1Ki ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_200_2M_MultiMetric-10 4.769Mi ± ∞ ¹ 2.935Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_100_200_2M_MultiMetric-10 47.43Mi ± ∞ ¹ 29.25Mi ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_100_200_5M_MultiMetric-10 47.43Mi ± ∞ ¹ 29.25Mi ± ∞ ¹ ~ (p=1.000 n=1) ² ConsumeLogsRejected-10 714.1Ki ± ∞ ¹ 640.3Ki ± ∞ ¹ ~ (p=1.000 n=1) ² geomean 1.514Mi 1.229Mi -18.87% ¹ need >= 6 samples for confidence interval at level 0.95 ² need >= 4 samples to detect a difference at alpha level 0.05 │ before.txt │ after.txt │ │ allocs/op │ allocs/op vs base │ _pushLogData_10_10_1024-10 1285.0 ± ∞ ¹ 821.0 ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_10_10_8K-10 1120.0 ± ∞ ¹ 716.0 ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_10_10_2M-10 1109.0 ± ∞ ¹ 709.0 ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_10_200_2M-10 22.01k ± ∞ ¹ 14.01k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_100_200_2M-10 220.0k ± ∞ ¹ 140.0k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_100_200_5M-10 220.0k ± ∞ ¹ 140.0k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_10_10_1024-10 1193.0 ± ∞ ¹ 784.0 ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_10_10_8K-10 1109.0 ± ∞ ¹ 709.0 ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_10_10_2M-10 1109.0 ± ∞ ¹ 709.0 ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_10_200_2M-10 22.01k ± ∞ ¹ 14.02k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_100_200_2M-10 220.0k ± ∞ ¹ 140.0k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushLogData_compressed_100_200_5M-10 220.0k ± ∞ ¹ 140.0k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_1024-10 1.810k ± ∞ ¹ 1.410k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_8K-10 1.810k ± ∞ ¹ 1.410k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_2M-10 1.812k ± ∞ ¹ 1.417k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_200_2M-10 36.01k ± ∞ ¹ 28.02k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_100_200_2M-10 360.0k ± ∞ ¹ 280.1k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_100_200_5M-10 360.0k ± ∞ ¹ 280.1k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_1024-10 1.810k ± ∞ ¹ 1.410k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_8K-10 1.810k ± ∞ ¹ 1.410k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_2M-10 1.811k ± ∞ ¹ 1.418k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_200_2M-10 36.01k ± ∞ ¹ 28.02k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_100_200_2M-10 360.0k ± ∞ ¹ 280.0k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_100_200_5M-10 360.0k ± ∞ ¹ 280.0k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_1024_MultiMetric-10 3.934k ± ∞ ¹ 1.835k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_8K_MultiMetric-10 3.934k ± ∞ ¹ 1.835k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_10_2M_MultiMetric-10 3.934k ± ∞ ¹ 1.835k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_10_200_2M_MultiMetric-10 78.07k ± ∞ ¹ 36.07k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_100_200_2M_MultiMetric-10 780.4k ± ∞ ¹ 360.3k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_100_200_5M_MultiMetric-10 780.4k ± ∞ ¹ 360.4k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_1024_MultiMetric-10 3.934k ± ∞ ¹ 1.835k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_8K_MultiMetric-10 3.934k ± ∞ ¹ 1.835k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_10_2M_MultiMetric-10 3.934k ± ∞ ¹ 1.835k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_10_200_2M_MultiMetric-10 78.07k ± ∞ ¹ 36.07k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_100_200_2M_MultiMetric-10 780.4k ± ∞ ¹ 360.4k ± ∞ ¹ ~ (p=1.000 n=1) ² _pushMetricData_compressed_100_200_5M_MultiMetric-10 780.4k ± ∞ ¹ 360.4k ± ∞ ¹ ~ (p=1.000 n=1) ² ConsumeLogsRejected-10 11.012k ± ∞ ¹ 7.013k ± ∞ ¹ ~ (p=1.000 n=1) ² geomean 18.95k 11.64k -38.56% ¹ need >= 6 samples for confidence interval at level 0.95 ² need >= 4 samples to detect a difference at alpha level 0.05 ``` --- .chloggen/hec_json.yaml | 27 ++++++++++++ exporter/splunkhecexporter/client.go | 42 ++++++------------- exporter/splunkhecexporter/client_test.go | 9 ++-- exporter/splunkhecexporter/go.mod | 3 +- exporter/splunkhecexporter/go.sum | 2 + .../splunkhecexporter/logdata_to_splunk.go | 10 ++--- .../splunkhecexporter/metricdata_to_splunk.go | 5 +-- .../metricdata_to_splunk_test.go | 9 ++-- receiver/splunkhecreceiver/go.mod | 1 + receiver/splunkhecreceiver/go.sum | 2 + 10 files changed, 62 insertions(+), 48 deletions(-) create mode 100644 .chloggen/hec_json.yaml diff --git a/.chloggen/hec_json.yaml b/.chloggen/hec_json.yaml new file mode 100644 index 000000000000..1fc82a492132 --- /dev/null +++ b/.chloggen/hec_json.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: splunkhecexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Increase the performance of JSON marshaling + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [34011] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/exporter/splunkhecexporter/client.go b/exporter/splunkhecexporter/client.go index 35f3ab872453..00f080f29b5f 100644 --- a/exporter/splunkhecexporter/client.go +++ b/exporter/splunkhecexporter/client.go @@ -12,7 +12,7 @@ import ( "net/url" "sync" - jsoniter "github.com/json-iterator/go" + "github.com/goccy/go-json" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/consumer/consumererror" "go.opentelemetry.io/collector/exporter" @@ -58,12 +58,6 @@ type client struct { meter metric.Meter } -var jsonStreamPool = sync.Pool{ - New: func() any { - return jsoniter.NewStream(jsoniter.ConfigDefault, nil, 512) - }, -} - func newClient(set exporter.Settings, cfg *Config, maxContentLength uint) *client { return &client{ config: cfg, @@ -196,8 +190,6 @@ func (c *client) pushLogDataInBatches(ctx context.Context, ld plog.Logs, headers func (c *client) fillLogsBuffer(logs plog.Logs, buf buffer, is iterState) (iterState, []error) { var b []byte var permanentErrors []error - jsonStream := jsonStreamPool.Get().(*jsoniter.Stream) - defer jsonStreamPool.Put(jsonStream) for i := is.resource; i < logs.ResourceLogs().Len(); i++ { rl := logs.ResourceLogs().At(i) @@ -216,7 +208,7 @@ func (c *client) fillLogsBuffer(logs plog.Logs, buf buffer, is iterState) (iterS // JSON encoding event and writing to buffer. var err error - b, err = marshalEvent(event, c.config.MaxEventSize, jsonStream) + b, err = marshalEvent(event, c.config.MaxEventSize) if err != nil { permanentErrors = append(permanentErrors, consumererror.NewPermanent(fmt.Errorf( "dropped log event: %v, error: %w", event, err))) @@ -249,8 +241,6 @@ func (c *client) fillLogsBuffer(logs plog.Logs, buf buffer, is iterState) (iterS func (c *client) fillMetricsBuffer(metrics pmetric.Metrics, buf buffer, is iterState) (iterState, []error) { var permanentErrors []error - jsonStream := jsonStreamPool.Get().(*jsoniter.Stream) - defer jsonStreamPool.Put(jsonStream) tempBuf := bytes.NewBuffer(make([]byte, 0, c.config.MaxContentLengthMetrics)) for i := is.resource; i < metrics.ResourceMetrics().Len(); i++ { @@ -267,7 +257,7 @@ func (c *client) fillMetricsBuffer(metrics pmetric.Metrics, buf buffer, is iterS tempBuf.Reset() for _, event := range events { // JSON encoding event and writing to buffer. - b, err := marshalEvent(event, c.config.MaxEventSize, jsonStream) + b, err := marshalEvent(event, c.config.MaxEventSize) if err != nil { permanentErrors = append(permanentErrors, consumererror.NewPermanent(fmt.Errorf("dropped metric event: %v, error: %w", event, err))) continue @@ -301,13 +291,11 @@ func (c *client) fillMetricsBuffer(metrics pmetric.Metrics, buf buffer, is iterS func (c *client) fillMetricsBufferMultiMetrics(events []*splunk.Event, buf buffer, is iterState) (iterState, []error) { var permanentErrors []error - jsonStream := jsonStreamPool.Get().(*jsoniter.Stream) - defer jsonStreamPool.Put(jsonStream) for i := is.record; i < len(events); i++ { event := events[i] // JSON encoding event and writing to buffer. - b, jsonErr := marshalEvent(event, c.config.MaxEventSize, jsonStream) + b, jsonErr := marshalEvent(event, c.config.MaxEventSize) if jsonErr != nil { permanentErrors = append(permanentErrors, consumererror.NewPermanent(fmt.Errorf("dropped metric event: %v, error: %w", event, jsonErr))) continue @@ -338,8 +326,6 @@ func (c *client) fillMetricsBufferMultiMetrics(events []*splunk.Event, buf buffe func (c *client) fillTracesBuffer(traces ptrace.Traces, buf buffer, is iterState) (iterState, []error) { var permanentErrors []error - jsonStream := jsonStreamPool.Get().(*jsoniter.Stream) - defer jsonStreamPool.Put(jsonStream) for i := is.resource; i < traces.ResourceSpans().Len(); i++ { rs := traces.ResourceSpans().At(i) @@ -354,7 +340,7 @@ func (c *client) fillTracesBuffer(traces ptrace.Traces, buf buffer, is iterState event := mapSpanToSplunkEvent(rs.Resource(), span, c.config) // JSON encoding event and writing to buffer. - b, err := marshalEvent(event, c.config.MaxEventSize, jsonStream) + b, err := marshalEvent(event, c.config.MaxEventSize) if err != nil { permanentErrors = append(permanentErrors, consumererror.NewPermanent(fmt.Errorf("dropped span events: %v, error: %w", event, err))) continue @@ -687,16 +673,14 @@ func buildHTTPHeaders(config *Config, buildInfo component.BuildInfo) map[string] } } -// marshalEvent marshals an event to JSON using a reusable jsoniter stream. -func marshalEvent(event *splunk.Event, sizeLimit uint, stream *jsoniter.Stream) ([]byte, error) { - stream.Reset(nil) - stream.Error = nil - stream.WriteVal(event) - if stream.Error != nil { - return nil, stream.Error +// marshalEvent marshals an event to JSON +func marshalEvent(event *splunk.Event, sizeLimit uint) ([]byte, error) { + b, err := json.Marshal(event) + if err != nil { + return nil, err } - if uint(stream.Buffered()) > sizeLimit { - return nil, fmt.Errorf("event size %d exceeds limit %d", stream.Buffered(), sizeLimit) + if uint(len(b)) > sizeLimit { + return nil, fmt.Errorf("event size %d exceeds limit %d", len(b), sizeLimit) } - return stream.Buffer(), nil + return b, nil } diff --git a/exporter/splunkhecexporter/client_test.go b/exporter/splunkhecexporter/client_test.go index 847cc55b8a1c..dacbfb1beca7 100644 --- a/exporter/splunkhecexporter/client_test.go +++ b/exporter/splunkhecexporter/client_test.go @@ -21,7 +21,7 @@ import ( "testing" "time" - jsoniter "github.com/json-iterator/go" + "github.com/goccy/go-json" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component" @@ -978,16 +978,15 @@ func TestReceiveSpanEvent(t *testing.T) { } // compareWithTestData compares hec output with a json file using maps instead of strings to avoid key ordering -// issues (jsoniter doesn't sort the keys). func compareWithTestData(t *testing.T, actual []byte, file string) { wantStr, err := os.ReadFile(file) require.NoError(t, err) wantMap := map[string]any{} - err = jsoniter.Unmarshal(wantStr, &wantMap) + err = json.Unmarshal(wantStr, &wantMap) require.NoError(t, err) gotMap := map[string]any{} - err = jsoniter.Unmarshal(actual, &gotMap) + err = json.Unmarshal(actual, &gotMap) require.NoError(t, err) assert.Equal(t, wantMap, gotMap) } @@ -1481,7 +1480,7 @@ func TestInvalidJson(t *testing.T) { badEvent := badJSON{ Foo: math.Inf(1), } - _, err := jsoniter.Marshal(badEvent) + _, err := json.Marshal(badEvent) assert.Error(t, err) } diff --git a/exporter/splunkhecexporter/go.mod b/exporter/splunkhecexporter/go.mod index 0012c1bfed33..35c77ea534e6 100644 --- a/exporter/splunkhecexporter/go.mod +++ b/exporter/splunkhecexporter/go.mod @@ -4,7 +4,7 @@ go 1.21.0 require ( github.com/cenkalti/backoff/v4 v4.3.0 - github.com/json-iterator/go v1.1.12 + github.com/goccy/go-json v0.9.7 github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.104.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk v0.104.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/batchperresourceattr v0.104.0 @@ -57,6 +57,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/go-version v1.7.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect github.com/knadh/koanf/providers/confmap v0.1.0 // indirect diff --git a/exporter/splunkhecexporter/go.sum b/exporter/splunkhecexporter/go.sum index 54572b28cb22..98bd5ef47495 100644 --- a/exporter/splunkhecexporter/go.sum +++ b/exporter/splunkhecexporter/go.sum @@ -46,6 +46,8 @@ github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 h1:TQcrn6Wq+sKGkpyPvppOz99zsMBaUOKXq6HSv655U1c= github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM= +github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= diff --git a/exporter/splunkhecexporter/logdata_to_splunk.go b/exporter/splunkhecexporter/logdata_to_splunk.go index b28c4b783a1b..88c55635df30 100644 --- a/exporter/splunkhecexporter/logdata_to_splunk.go +++ b/exporter/splunkhecexporter/logdata_to_splunk.go @@ -8,7 +8,7 @@ import ( "fmt" "time" - jsoniter "github.com/json-iterator/go" + "github.com/goccy/go-json" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/plog" @@ -110,8 +110,8 @@ func mergeValue(dst map[string]any, k string, v any) { if isArrayFlat(element) { dst[k] = v } else { - jsonStr, _ := jsoniter.MarshalToString(element) - dst[k] = jsonStr + b, _ := json.Marshal(element) + dst[k] = string(b) } case map[string]any: flattenAndMergeMap(element, dst, k) @@ -141,8 +141,8 @@ func flattenAndMergeMap(src, dst map[string]any, key string) { if isArrayFlat(element) { dst[current] = element } else { - jsonStr, _ := jsoniter.MarshalToString(element) - dst[current] = jsonStr + b, _ := json.Marshal(element) + dst[current] = string(b) } default: diff --git a/exporter/splunkhecexporter/metricdata_to_splunk.go b/exporter/splunkhecexporter/metricdata_to_splunk.go index d982c958896f..ec95f98aa4d5 100644 --- a/exporter/splunkhecexporter/metricdata_to_splunk.go +++ b/exporter/splunkhecexporter/metricdata_to_splunk.go @@ -9,7 +9,7 @@ import ( "strconv" "strings" - jsoniter "github.com/json-iterator/go" + "github.com/goccy/go-json" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" "go.uber.org/zap" @@ -284,12 +284,11 @@ func mergeEventsToMultiMetricFormat(events []*splunk.Event) ([]*splunk.Event, er hashes := map[uint32]*splunk.Event{} hasher := fnv.New32a() var merged []*splunk.Event - marshaler := jsoniter.ConfigCompatibleWithStandardLibrary for _, e := range events { cloned := copyEventWithoutValues(e) - data, err := marshaler.Marshal(cloned) + data, err := json.Marshal(cloned) if err != nil { return nil, err } diff --git a/exporter/splunkhecexporter/metricdata_to_splunk_test.go b/exporter/splunkhecexporter/metricdata_to_splunk_test.go index 4146814ec6cc..a2ea1bb718d3 100644 --- a/exporter/splunkhecexporter/metricdata_to_splunk_test.go +++ b/exporter/splunkhecexporter/metricdata_to_splunk_test.go @@ -4,14 +4,13 @@ package splunkhecexporter import ( - "encoding/json" "fmt" "io" "math" "testing" "time" - jsoniter "github.com/json-iterator/go" + "github.com/goccy/go-json" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/pdata/pcommon" @@ -706,16 +705,16 @@ func TestMergeEvents(t *testing.T) { json1 := `{"event":"metric","fields":{"IF-Azure":"azure-env","k8s.cluster.name":"devops-uat","k8s.namespace.name":"splunk-collector-tests","k8s.node.name":"myk8snodename","k8s.pod.name":"my-otel-collector-pod","metric_type":"Gauge","metricsIndex":"test_metrics","metricsPlatform":"unset","resourceAttrs":"NO","testNumber":"number42","testRun":"42","metric_name:otel.collector.test":3411}}` json2 := `{"event":"metric","fields":{"IF-Azure":"azure-env","k8s.cluster.name":"devops-uat","k8s.namespace.name":"splunk-collector-tests","k8s.node.name":"myk8snodename","k8s.pod.name":"my-otel-collector-pod","metric_type":"Gauge","metricsIndex":"test_metrics","metricsPlatform":"unset","resourceAttrs":"NO","testNumber":"number42","testRun":"42","metric_name:otel.collector.test2":26059}}` ev1 := &splunk.Event{} - err := jsoniter.Unmarshal([]byte(json1), ev1) + err := json.Unmarshal([]byte(json1), ev1) require.NoError(t, err) ev2 := &splunk.Event{} - err = jsoniter.Unmarshal([]byte(json2), ev2) + err = json.Unmarshal([]byte(json2), ev2) require.NoError(t, err) events := []*splunk.Event{ev1, ev2} merged, err := mergeEventsToMultiMetricFormat(events) require.NoError(t, err) require.Len(t, merged, 1) - b, err := jsoniter.ConfigCompatibleWithStandardLibrary.Marshal(merged[0]) + b, err := json.Marshal(merged[0]) require.NoError(t, err) require.Equal(t, `{"host":"","event":"metric","fields":{"IF-Azure":"azure-env","k8s.cluster.name":"devops-uat","k8s.namespace.name":"splunk-collector-tests","k8s.node.name":"myk8snodename","k8s.pod.name":"my-otel-collector-pod","metric_name:otel.collector.test":3411,"metric_name:otel.collector.test2":26059,"metric_type":"Gauge","metricsIndex":"test_metrics","metricsPlatform":"unset","resourceAttrs":"NO","testNumber":"number42","testRun":"42"}}`, string(b)) } diff --git a/receiver/splunkhecreceiver/go.mod b/receiver/splunkhecreceiver/go.mod index 28f3c7bfcbb1..4bf4e4381def 100644 --- a/receiver/splunkhecreceiver/go.mod +++ b/receiver/splunkhecreceiver/go.mod @@ -38,6 +38,7 @@ require ( github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect + github.com/goccy/go-json v0.9.7 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/hashicorp/go-version v1.7.0 // indirect diff --git a/receiver/splunkhecreceiver/go.sum b/receiver/splunkhecreceiver/go.sum index c1957f2f5a75..85b6cfb9b7a0 100644 --- a/receiver/splunkhecreceiver/go.sum +++ b/receiver/splunkhecreceiver/go.sum @@ -42,6 +42,8 @@ github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 h1:TQcrn6Wq+sKGkpyPvppOz99zsMBaUOKXq6HSv655U1c= github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM= +github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=