Skip to content

Commit

Permalink
Additional regex pattern for Datadog API/APP tokens.
Browse files Browse the repository at this point in the history 
Authored-by: Fredrik Lysén <[email protected]>
  • Loading branch information
@flysen
flysen authored Jan 5, 2024
1 parent 5840c9b commit 881e1b7
Show file tree
Hide file tree
Showing 6 changed files with 217 additions and 1 deletion.
11 changes: 11 additions & 0 deletions src/main/java/com/tsystems/sbs/DefaultRegexpPairs.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,22 @@ public final class DefaultRegexpPairs {
new RegexpPair("(aws_[a-zA-Z_]+=)(\\S+)", "$1********")
));

private final static List<RegexpPair> DEFAULT_REGEXES_DD
= Collections.<RegexpPair>unmodifiableList(
Arrays.<RegexpPair>asList(
new RegexpPair("((?i)(\\bdatadog|dd|dogapi\\b).*)(\\b([a-zA-Z-0-9]{32})\\b)", "$1********"), // Datadog RegExp MASKING API KEY
new RegexpPair("((?i)(\\bdatadog|dd|dogapi\\b).*)(\\b([a-zA-Z-0-9]{40})\\b)", "$1********") // Datadog RegExp MASKING APP KEY
));

public static List<RegexpPair> getDefaultRegexes() {
return DEFAULT_REGEXES;
}

public static List<RegexpPair> getDefaultRegexesAWS() {
return DEFAULT_REGEXES_AWS;
}

public static List<RegexpPair> getDefaultRegexesDD() {
return DEFAULT_REGEXES_DD;
}
}
11 changes: 11 additions & 0 deletions src/main/java/com/tsystems/sbs/LogFileFilterConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ public static LogFileFilterConfig get() {
*/
private boolean enabledDefaultRegexp;
private boolean enabledDefaultRegexpAWS;
private boolean enabledDefaultRegexpDD;

/**
* Represents the custom regexp pairs specified by the user in the global settings.
Expand Down Expand Up @@ -75,6 +76,10 @@ public boolean isEnabledDefaultRegexpAWS() {
return enabledDefaultRegexpAWS;
}

public boolean isEnabledDefaultRegexpDD() {
return enabledDefaultRegexpDD;
}

@DataBoundSetter
public void setEnabledDefaultRegexp(boolean enabledDefaultRegexp) {
this.enabledDefaultRegexp = enabledDefaultRegexp;
Expand All @@ -87,6 +92,12 @@ public void setEnabledDefaultRegexpAWS(boolean enabledDefaultRegexpAWS) {
save();
}

@DataBoundSetter
public void setEnabledDefaultRegexpDD(boolean enabledDefaultRegexpDD) {
this.enabledDefaultRegexpDD = enabledDefaultRegexpDD;
save();
}

Check warning on line 99 in src/main/java/com/tsystems/sbs/LogFileFilterConfig.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered lines 

Lines 80-99 are not covered by tests

public List<RegexpPair> getRegexpPairs() {
return regexpPairs;
}
Expand Down
6 changes: 6 additions & 0 deletions src/main/java/com/tsystems/sbs/LogFileFilterOutputStream.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ public class LogFileFilterOutputStream extends LineTransformationOutputStream {
private final boolean isEnabledGlobally;
private final boolean isEnabledDefaultRegexp;
private final boolean isEnabledDefaultRegexpAWS;
private final boolean isEnabledDefaultRegexpDD;
private final List<RegexpPair> defaultRegexpPairs;
private final List<RegexpPair> customRegexpPairs;
private final String jobName;
Expand All @@ -51,6 +52,7 @@ public LogFileFilterOutputStream(OutputStream out, Charset charset, String jobNa
isEnabledGlobally = config.isEnabledGlobally();
isEnabledDefaultRegexp = config.isEnabledDefaultRegexp();
isEnabledDefaultRegexpAWS = config.isEnabledDefaultRegexpAWS();
isEnabledDefaultRegexpDD = config.isEnabledDefaultRegexpDD();

if (isEnabledGlobally) {
// Load regexes
Expand All @@ -62,6 +64,10 @@ public LogFileFilterOutputStream(OutputStream out, Charset charset, String jobNa
if (isEnabledDefaultRegexpAWS) {
defaultRegexpPairs.addAll(DefaultRegexpPairs.getDefaultRegexesAWS());
}
if (isEnabledDefaultRegexpDD) {
defaultRegexpPairs.addAll(DefaultRegexpPairs.getDefaultRegexesDD());

Check warning on line 68 in src/main/java/com/tsystems/sbs/LogFileFilterOutputStream.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered lines 

Lines 55-68 are not covered by tests
}

// Log defaultRegexpPairs
for (RegexpPair pair : defaultRegexpPairs) {
LOGGER.log(Level.INFO, pair.toString());
Expand Down
6 changes: 5 additions & 1 deletion src/main/resources/com/tsystems/sbs/LogFileFilterConfig/config.jelly
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,11 @@
<f:entry title="Enable Amazon AWS regexp" field="enabledDefaultRegexpAWS">
<f:checkbox checked="${it.enabledDefaultRegexpAWS}" />
</f:entry>


<f:entry title="Enable Datadog API/APP regexp" field="enabledDefaultRegexpDD">
<f:checkbox checked="${it.enabledDefaultRegexpDD}" />
</f:entry>

<!-- Defines the regexp patterns to filter the console logs -->
<f:entry title="Custom regexp pairs" field="regexpPairs">
<f:repeatable name="Regexp Pairs" field="regexpPairs">
Expand Down
40 changes: 40 additions & 0 deletions src/main/resources/com/tsystems/sbs/LogFileFilterConfig/help-enabledDefaultRegexpDD.html
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
<!--THIS FILE IS AUTOGENERATED FROM THE global.properties FILE-->
<div>

<style>
.log-file-filter-plugin th, .log-file-filter-plugin td{
border: solid 1px black;
border-collapse: collapse;
padding: 10px
}
</style>

<p>These are the default regular expressions and their respective replacements. These expressions are fixed and come with the plugin.</p>
<table class="log-file-filter-plugin">
<!-- Table header -->
<tr><th>Description</th><th>Regexp</th><th>Replacement</th><th>Sample</th></tr>

<!-- Table rows (regexes) -->
<tr>
<td>Masks Datadog APP secrets</td>
<td>((?i)(\bdatadog|dd|dogapi\b).*)(\b([a-zA-Z-0-9]{40})\b)</td>
<td>$1********</td>
<td>
<ul>
<li>"datadog key = 3c0c3965368a6b10f7640dbda46abfd2 secret= 3c0c3965368a6b10f7640dbda46abfdca981c2d3" -> <b>datadog key = ******** secret= ********</b></li>
</ul>
</td>
</tr>
<tr>
<td>Masks Datadog API secrets</td>
<td>((?i)(\bdatadog|dd|dogapi\b).*)(\b([a-zA-Z-0-9]{32})\b)</td>
<td>$1********</td>
<td>
<ul>
<li>dAtAdOg token = "3c0c3965368a6b10f7640dbda46abfdc"; -> <b>dAtAdOg token = "********";</b></li>
<li>curl -X GET "https://api.datadoghq.eu/api/v1/validate" -H "Accept: application/json" -H "DD-API-KEY: characteristicallycharacteristic" -> <b>curl -X GET "https://api.datadoghq.eu/api/v1/validate" -H "Accept: application/json" -H "DD-API-KEY: ********"</b></li>
</ul>
</td>
</tr>
</table>
</div>
144 changes: 144 additions & 0 deletions src/test/java/com/tsystems/sbs/DefaultRegexpPairsDDTest.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
package com.tsystems.sbs;

import org.junit.Test;

import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.greaterThan;
import static org.junit.Assert.assertEquals;


public class DefaultRegexpPairsDDTest {
private List<RegexpPair> getDefaultRegexpPairs() {
return DefaultRegexpPairs.getDefaultRegexesDD();
}
@Test
public void testDefaultPairsList() {
List<RegexpPair> defaultRegexpPairs = getDefaultRegexpPairs();
assertThat(defaultRegexpPairs.size(), greaterThan(0));

}

@Test
public void testDefaultPairsApi() {
List<RegexpPair> defaultRegexpPairs = getDefaultRegexpPairs();

// Define the input string 32 characters
String input = "curl -X GET \"https://api.datadoghq.eu/api/v1/validate\" -H \"Accept: application/json\" -H \"DD-API-KEY: characteristicallycharacteristic\"";
String expected = "curl -X GET \"https://api.datadoghq.eu/api/v1/validate\" -H \"Accept: application/json\" -H \"DD-API-KEY: ********\"";


StringBuilder replacedInput = new StringBuilder(input);

for (RegexpPair pair : defaultRegexpPairs) {
String pattern = pair.getRegexp();
String replacement = pair.getReplacement();

Pattern regexPattern = Pattern.compile(pattern);
Matcher matcher = regexPattern.matcher(replacedInput);

while (matcher.find()) {
String matchedPattern = matcher.group();
String replacedString = replacement;

// Replace all occurrences of $n with the matched groups
for (int i = 1; i <= matcher.groupCount(); i++) {
String group = matcher.group(i);
replacedString = replacedString.replace("$" + i, group);
}

replacedInput.replace(matcher.start(), matcher.end(), replacedString);
matcher.region(matcher.start() + replacedString.length(), replacedInput.length());
}
}

String replacedInputString = replacedInput.toString();
System.out.println("Replaced input result: " + replacedInputString);

// Test the behavior
assertEquals(expected, replacedInputString);
}

@Test
public void testDefaultPairsKey() {
List<RegexpPair> defaultRegexpPairs = getDefaultRegexpPairs();

// Define the input string 32 characters
String input = "datadog key = 3c0c3965368a6b10f7640dbda46abfd2 secret= 3c0c3965368a6b10f7640dbda46abfdca981c2d3";
String expected = "datadog key = ******** secret= ********";


StringBuilder replacedInput = new StringBuilder(input);

for (RegexpPair pair : defaultRegexpPairs) {
String pattern = pair.getRegexp();
String replacement = pair.getReplacement();

Pattern regexPattern = Pattern.compile(pattern);
Matcher matcher = regexPattern.matcher(replacedInput);

while (matcher.find()) {
String matchedPattern = matcher.group();
String replacedString = replacement;

// Replace all occurrences of $n with the matched groups
for (int i = 1; i <= matcher.groupCount(); i++) {
String group = matcher.group(i);
replacedString = replacedString.replace("$" + i, group);
}

replacedInput.replace(matcher.start(), matcher.end(), replacedString);
matcher.region(matcher.start() + replacedString.length(), replacedInput.length());
}
}

String replacedInputString = replacedInput.toString();
System.out.println("Replaced input result: " + replacedInputString);

// Test the behavior
assertEquals(expected, replacedInputString);
}
@Test
public void testDefaultPairsToken() {
List<RegexpPair> defaultRegexpPairs = getDefaultRegexpPairs();

// Define the input string 32 characters
String input = "dAtAdOg token = \"3c0c3965368a6b10f7640dbda46abfdc\";";
String expected = "dAtAdOg token = \"********\";";


StringBuilder replacedInput = new StringBuilder(input);

for (RegexpPair pair : defaultRegexpPairs) {
String pattern = pair.getRegexp();
String replacement = pair.getReplacement();

Pattern regexPattern = Pattern.compile(pattern);
Matcher matcher = regexPattern.matcher(replacedInput);

while (matcher.find()) {
String matchedPattern = matcher.group();
String replacedString = replacement;

// Replace all occurrences of $n with the matched groups
for (int i = 1; i <= matcher.groupCount(); i++) {
String group = matcher.group(i);
replacedString = replacedString.replace("$" + i, group);
}

replacedInput.replace(matcher.start(), matcher.end(), replacedString);
matcher.region(matcher.start() + replacedString.length(), replacedInput.length());
}
}

String replacedInputString = replacedInput.toString();
System.out.println("Replaced input result: " + replacedInputString);

// Test the behavior
assertEquals(expected, replacedInputString);
}
}

0 comments on commit 881e1b7

Please sign in to comment.