Skip to content

Commit

Permalink
More efficient filtering of input streams
Browse files Browse the repository at this point in the history
  • Loading branch information
basil committed Oct 18, 2024
1 parent a85622b commit 77b4ff0
Show file tree
Hide file tree
Showing 3 changed files with 120 additions and 12 deletions.
Original file line number Diff line number Diff line change
@@ -1,16 +1,14 @@
package com.cloudbees.jenkins.support.api;

import com.cloudbees.jenkins.support.filter.FilteredInputStream;
import com.cloudbees.jenkins.support.filter.PasswordRedactor;
import com.cloudbees.jenkins.support.impl.SlaveLaunchLogs;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils;
import java.util.function.Function;

/**
* @see SlaveLaunchLogs
Expand All @@ -23,14 +21,8 @@ public LaunchLogsFileContent(String name, String[] filterableParameters, File fi

@Override
protected InputStream getInputStream() throws IOException {
try (FileInputStream inputStream = new FileInputStream(file)) {
List<String> strings = IOUtils.readLines(inputStream, Charset.defaultCharset());
byte[] bytes = strings.stream()
.map(line -> PasswordRedactor.get().redact(line))
.collect(Collectors.joining("\n", "", "\n"))
.getBytes(Charset.defaultCharset());
return new ByteArrayInputStream(bytes);
}
Function<String, String> filter = line -> PasswordRedactor.get().redact(line);
return new FilteredInputStream(new FileInputStream(file), Charset.defaultCharset(), filter);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
package com.cloudbees.jenkins.support.filter;

import edu.umd.cs.findbugs.annotations.NonNull;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.function.Function;

/**
* A custom {@link InputStream} that filters lines of text read from an underlying {@link InputStream} using a provided
* mapping function. Each line read from the input stream is processed by the function before being returned, allowing
* for modifications such as redaction or transformation of the line content.
*
* @author Basil Crow
*/
public class FilteredInputStream extends InputStream {

private final Function<String, String> filter;
private final Charset encoding;
private final BufferedReader reader;
private ByteArrayInputStream buffer;

/**
* Constructs a filtered stream using the provided filter and encoding.
*
* @param is Input stream to read line content from
* @param encoding Character set to use for decoding and encoding bytes read from this stream
* @param filter Filter to apply to lines read from this stream
*/
public FilteredInputStream(
@NonNull InputStream is, @NonNull Charset encoding, @NonNull Function<String, String> filter) {
this.encoding = encoding;
this.reader = new BufferedReader(new InputStreamReader(is, encoding));
this.filter = filter;
}

@Override
public int read() throws IOException {
if (buffer == null) {
String line = reader.readLine();
if (line != null) {
line = filter.apply(line);
line += System.lineSeparator();
buffer = new ByteArrayInputStream(line.getBytes(encoding));
} else {
return -1;
}
}
int ch = buffer.read();
if (ch != -1) {
return ch;
} else {
buffer = null;
return read();
}
}

@Override
public void close() throws IOException {
reader.close();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
package com.cloudbees.jenkins.support.filter;

import static org.junit.Assert.assertEquals;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import org.junit.Test;

public class FilteredInputStreamTest {

@Test
public void testReadSingleLine() throws IOException {
String input = "foo bar";
String output;
try (FilteredInputStream fis = new FilteredInputStream(
new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8)),
StandardCharsets.UTF_8,
String::toUpperCase)) {
output = new String(fis.readAllBytes(), StandardCharsets.UTF_8);
}
assertEquals("FOO BAR" + System.lineSeparator(), output);
}

@Test
public void testReadMultipleLines() throws IOException {
String input = "Line 1\nLine 2\nLine 3";
String output;
try (FilteredInputStream fis = new FilteredInputStream(
new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8)),
StandardCharsets.UTF_8,
String::toUpperCase)) {
output = new String(fis.readAllBytes(), StandardCharsets.UTF_8);
}
assertEquals(
"LINE 1" + System.lineSeparator() + "LINE 2" + System.lineSeparator() + "LINE 3"
+ System.lineSeparator(),
output);
}

@Test
public void testReadEmptyStream() throws IOException {
byte[] input = new byte[0];
String output;
try (FilteredInputStream fis =
new FilteredInputStream(new ByteArrayInputStream(input), StandardCharsets.UTF_8, String::toUpperCase)) {
output = new String(fis.readAllBytes(), StandardCharsets.UTF_8);
}
assertEquals("", output);
}
}

0 comments on commit 77b4ff0

Please sign in to comment.