Skip to content

Commit

Permalink
for filewriter add buffer
Browse files Browse the repository at this point in the history
  • Loading branch information
han committed Oct 17, 2024
1 parent a89fda1 commit a0c58c0
Show file tree
Hide file tree
Showing 2 changed files with 56 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.apache.gravitino.audit;

import com.google.common.annotations.VisibleForTesting;
import java.io.BufferedWriter;
import java.io.FileOutputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
Expand All @@ -39,9 +40,16 @@ public class FileAuditWriter implements AuditLogWriter {

public static final String LOG_FILE_NAME_CONFIG = "fileName";

public static final String APPEND = "append";

public static final String FILE_IMMEDIATE_FLUSH_CONFIG = "immediateFlush";

public static final String BUFFERED = "buffered";

public static final String BUFFER_SIZE = "bufferSize";

public static final String LINE_SEPARATOR = "lineSeparator";

private Formatter formatter;
private Writer outWriter;
@VisibleForTesting String fileName;
Expand All @@ -50,6 +58,12 @@ public class FileAuditWriter implements AuditLogWriter {

String lineSeparator;

boolean append;

boolean buffered;

int bufferSize;

@Override
public Formatter getFormatter() {
return formatter;
Expand All @@ -62,13 +76,20 @@ public void init(Formatter formatter, Map<String, String> properties) {
immediateFlush =
Boolean.parseBoolean(properties.getOrDefault(FILE_IMMEDIATE_FLUSH_CONFIG, "false"));
lineSeparator = properties.getOrDefault(LINE_SEPARATOR, "\n");
append = Boolean.parseBoolean(properties.getOrDefault(APPEND, "false"));
buffered = Boolean.parseBoolean(properties.getOrDefault(BUFFERED, "true"));
bufferSize = Integer.parseInt(properties.getOrDefault(BUFFER_SIZE, "8192"));
try {
OutputStream outputStream = new FileOutputStream(fileName, true);
OutputStream outputStream = new FileOutputStream(fileName, append);
outWriter = new OutputStreamWriter(outputStream, StandardCharsets.UTF_8);
} catch (Exception e) {
throw new GravitinoRuntimeException(
e, "Init audit log writer fail, filename is %s", fileName);
}

if (buffered) {
outWriter = new BufferedWriter(outWriter, bufferSize);
}
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.stream.Stream;
import org.apache.gravitino.Config;
import org.apache.gravitino.Configs;
import org.apache.gravitino.NameIdentifier;
Expand All @@ -51,6 +52,8 @@ public class TestAuditManager {

private static final String DEFAULT_FILE_NAME = "gravitino_audit.log";

private static int EVENT_NUM = 2000;

@BeforeAll
public void setup() {
Path path = Paths.get(DEFAULT_FILE_NAME);
Expand Down Expand Up @@ -139,6 +142,29 @@ public void testFileAuditLog() {
Assertions.assertEquals(formattedAuditLog.toString(), auditLog);
}

@Test
public void testBathEvents() {
Config config = new Config(false) {};
config.set(Configs.AUDIT_LOG_ENABLED_CONF, true);
// set immediate flush to true for testing, so that the audit log will be read immediately
config.set(
new ConfigBuilder("gravitino.audit.writer.file.immediateFlush").stringConf(), "true");

EventListenerManager eventListenerManager = mockEventListenerManager();
AuditLogManager auditLogManager = mockAuditLogManager(config, eventListenerManager);
EventBus eventBus = eventListenerManager.createEventBus();

for (int i = 0; i < EVENT_NUM; i++) {
DummyEvent dummyEvent = mockDummyEvent();
eventBus.dispatchEvent(dummyEvent);
}

FileAuditWriter fileAuditWriter = (FileAuditWriter) auditLogManager.getAuditLogWriter();
String fileName = fileAuditWriter.fileName;
long auditSize = getAuditSize(fileName);
Assertions.assertEquals(EVENT_NUM, auditSize);
}

@AfterEach
public void cleanup() {
try {
Expand All @@ -161,6 +187,14 @@ private String readAuditLog(String fileName) {
}
}

private long getAuditSize(String fileName) {
try (Stream<String> lines = Files.lines(Paths.get(fileName))) {
return lines.count();
} catch (IOException e) {
throw new RuntimeException(e);
}
}

private AuditLogManager mockAuditLogManager(
Config config, EventListenerManager eventListenerManager) {
AuditLogManager auditLogManager = new AuditLogManager();
Expand Down

0 comments on commit a0c58c0

Please sign in to comment.