Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Java tests refactoring #82

Open
wants to merge 8 commits into
base: refactor-pipes-tests
Choose a base branch
from
Open

Java tests refactoring #82

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
292cc3e
[pipes] refactor Pipes tests interface
danielgafni Dec 25, 2024
fbba3f7
fix --test-name
danielgafni Dec 25, 2024
68d19d5
Big tests refactoring
GingerYouth Dec 25, 2024
385428e
[pipes] refactor Pipes tests interface
danielgafni Dec 25, 2024
8c08276
Merge branch 'refactor-pipes-tests' of github.com:dagster-io/communit…
GingerYouth Dec 28, 2024
565244e
Fixed bad merge
GingerYouth Dec 28, 2024
74562ed
Suite.py
GingerYouth Dec 28, 2024
8485cdf
Tests refactoring
GingerYouth Dec 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions ...ries/pipes/implementations/java/dagster-pipes-java/src/main/java/pipes/utils/Example.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,13 @@
import pipes.DagsterPipesException;
import pipes.PipesContext;
import pipes.PipesSession;
import pipes.data.PipesMetadata;
import pipes.loaders.PipesContextLoader;
import pipes.loaders.PipesDefaultContextLoader;
import pipes.loaders.PipesEnvVarParamsLoader;
import pipes.loaders.PipesParamsLoader;
import pipes.writers.PipesDefaultMessageWriter;
import pipes.writers.PipesMessageWriter;
import pipes.writers.PipesMessageWriterChannel;
import types.Type;

import java.util.HashMap;
import java.util.Map;
Expand Down
105 changes: 25 additions & 80 deletions libraries/pipes/implementations/java/dagster-pipes-java/src/test/java/pipes/MainTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,25 +29,6 @@ public class MainTest implements Runnable {
private final ObjectMapper objectMapper = new ObjectMapper();
private Map<String, Object> cachedJson = new ConcurrentHashMap<>();

@CommandLine.Option(
names = {"--context"},
description = "Provide DAGSTER_PIPES_CONTEXT value for testing"
)
private String context;

@CommandLine.Option(
names = {"--messages"},
description = "Provide DAGSTER_PIPES_MESSAGES value for testing"
)
private String messages;

@CommandLine.Option(
names = {"--env"},
description = "Get DAGSTER_PIPES_MESSAGES & DAGSTER_PIPES_CONTEXT values " +
"from environmental variables"
)
private boolean env = false;

@CommandLine.Option(
names = {"--job-name"},
description = "Provide value of 'jobName' for testing"
Expand All @@ -61,13 +42,7 @@ public class MainTest implements Runnable {
private String extras;

@CommandLine.Option(
names = {"--full"},
description = "Flag to test full PipesContext usage"
)
private boolean full = false;

@CommandLine.Option(
names = {"--custom-payload-path"},
names = {"--custom-payload"},
description = "Specify custom payload path"
)
private String customPayloadPath;
Expand All @@ -84,18 +59,6 @@ public class MainTest implements Runnable {
)
private String reportAssetMaterializationJson;

@CommandLine.Option(
names = {"--throw-error"},
description = "Throw exception in PipesSession with specified message"
)
private boolean throwException = false;

@CommandLine.Option(
names = {"--logging"},
description = "Flag to test logging"
)
private boolean logging = false;

@CommandLine.Option(
names = {"--message-writer"},
description = "Specify the type of the message writer: default,s3"
Expand All @@ -108,47 +71,34 @@ public class MainTest implements Runnable {
)
private String contextLoaderType;

@CommandLine.Option(
names = {"--test-name"},
description = "Specify the name of the test"
)
private String testName;

@Override
public void run() {
Map<String, String> input = new HashMap<>();
PipesTests pipesTests = new PipesTests();
try {
if (this.context != null) {
input.put(PipesConstants.CONTEXT_ENV_VAR.name, context);
}
if (this.messages != null) {
input.put(PipesConstants.MESSAGES_ENV_VAR.name, this.messages);
}
pipesTests.setInput(input);

final PipesContextLoader loader;
if (this.contextLoaderType != null && !this.contextLoaderType.isEmpty()) {
switch (this.contextLoaderType) {
case "s3":
S3Client amazonS3Client = S3Client.builder().build();
loader = new PipesS3ContextLoader(amazonS3Client);
break;
case "default":
loader = new PipesDefaultContextLoader();
break;
default:
throw new IllegalArgumentException("Specified unknown context loader type!");
if (this.contextLoaderType.equals("s3")) {
S3Client amazonS3Client = S3Client.builder().build();
loader = new PipesS3ContextLoader(amazonS3Client);
} else {
loader = new PipesDefaultContextLoader();
}
pipesTests.setContextLoader(loader);
}

final PipesMessageWriter<? extends PipesMessageWriterChannel> writer;
if (this.messageWriter != null && !this.messageWriter.isEmpty()) {
switch (this.messageWriter) {
case "s3":
S3Client amazonS3Client = S3Client.builder().build();
writer = new PipesS3MessageWriter(amazonS3Client);
break;
case "default":
writer = new PipesDefaultMessageWriter();
break;
default:
throw new IllegalArgumentException("Specified unknown message writer!");
if (this.messageWriter.equals("s3")) {
S3Client amazonS3Client = S3Client.builder().build();
writer = new PipesS3MessageWriter(amazonS3Client);
} else {
writer = new PipesDefaultMessageWriter();
}
pipesTests.setMessageWriter(writer);
}
Expand All @@ -160,12 +110,12 @@ public void run() {
pipesTests.setPayload(payload);
}

if (this.throwException) {
if (this.testName != null && this.testName.equals("test_error_reporting")) {
pipesTests.testRunPipesSessionWithException();
return;
}

if (this.logging) {
if (this.testName != null && this.testName.equals("test_message_log")) {
pipesTests.testLogging();
return;
}
Expand All @@ -184,31 +134,26 @@ public void run() {
String checkName = loadParamByWrapperKey("checkName", String.class);
boolean passed = loadParamByWrapperKey("passed", Boolean.class);
String assetKey = loadParamByWrapperKey("assetKey", String.class);
PipesAssetCheckSeverity severity = PipesAssetCheckSeverity.valueOf(loadParamByWrapperKey("severity", String.class));
PipesAssetCheckSeverity severity = PipesAssetCheckSeverity.valueOf(
loadParamByWrapperKey("severity", String.class)
);
pipesTests.setCheck(checkName, passed, assetKey, severity);
}

if (this.full) {
pipesTests.fullTest();
return;
} else {
pipesTests.setContextData();
}

if (this.extras != null) {
File jsonFile = new File(this.extras);
ObjectMapper objectMapper = new ObjectMapper();
Map<String, Object> extrasMap = objectMapper.readValue(
jsonFile, new TypeReference<Map<String, Object>>() {}
jsonFile, new TypeReference<Map<String,Object>>() {}
);
pipesTests.setExtras(extrasMap);
pipesTests.testExtras();
}

if (this.jobName != null) {
pipesTests.setJobName(this.jobName);
pipesTests.testJobName();
}

pipesTests.fullTest();
} catch (IOException | DagsterPipesException exception) {
throw new RuntimeException(exception);
}
Expand Down
47 changes: 17 additions & 30 deletions libraries/pipes/implementations/java/dagster-pipes-java/src/test/java/pipes/PipesTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,7 @@
@Disabled
public class PipesTests {

private Map<String, String> input;
private PipesContextLoader contextLoader;
private PipesContextData contextData;
private Map<String, Object> extras;
private String jobName;
private Object payload;
Expand All @@ -41,10 +39,6 @@ public class PipesTests {
//Message writer
private PipesMessageWriter<? extends PipesMessageWriterChannel> pipesMessageWriter;

void setInput(Map<String, String> input) {
this.input = input;
}

void setExtras(Map<String, Object> extras) {
this.extras = extras;
}
Expand All @@ -53,15 +47,10 @@ void setJobName(String jobName) {
this.jobName = jobName;
}

void setContextData() throws DagsterPipesException {
this.contextData = DataLoader.getData(input);
}

void setContextLoader(PipesContextLoader contextLoader) throws DagsterPipesException {
this.contextLoader = contextLoader;
}


void setPayload(Object payload) {
this.payload = payload;
}
Expand All @@ -84,25 +73,6 @@ void setMessageWriter(PipesMessageWriter<? extends PipesMessageWriterChannel> wr
this.pipesMessageWriter = writer;
}

@Test
void testExtras() {
Assertions.assertTrue(
contextData.getExtras().entrySet().containsAll(this.extras.entrySet()),
"Extras does not contain all provided entries."
);
System.out.println("Extras are correct.");
}

@Test
void testJobName() {
Assertions.assertEquals(
this.jobName,
contextData.getJobName(),
"JobName is incorrect."
);
System.out.println("JobName is correct.");
}

@Test
void fullTest() throws DagsterPipesException {
getTestSession().runDagsterPipes(this::fullTest);
Expand All @@ -111,6 +81,23 @@ void fullTest() throws DagsterPipesException {
private void fullTest(PipesContext context) throws DagsterPipesException {
context.reportCustomMessage("Hello from external process!");

if (this.extras != null) {
Assertions.assertTrue(
context.getExtras().entrySet().containsAll(this.extras.entrySet()),
"Extras does not contain all provided entries."
);
System.out.println("Extras are correct.");
}

if (this.jobName != null) {
Assertions.assertEquals(
this.jobName,
context.getJobName(),
"JobName is incorrect."
);
System.out.println("JobName is correct.");
}

if (this.payload != null) {
context.reportCustomMessage(this.payload);
System.out.println("Payload reported with custom message.");
Expand Down
25 changes: 19 additions & 6 deletions libraries/pipes/tests/dagster-pipes-tests/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,17 @@ See the [pipes_config.py](src/dagster_pipes_tests/pipes_config.py) class for mor

In order to run the tests, follow these steps:

1. Install `pytest` and `dagster-pipes-tests`:
1. Install `pytest` and `dagster-pipes-tests`. This can be done with [uv](https://docs.astral.sh/uv/):

```shell
uv pip install pytest
# TODO: publish the package to PyPI
uv pip install <path-to-pipes-tests>
# assuming the command is run in libraries/pipes/implementations/<language>
uv add --group dev pytest --editable ../../tests/dagster-pipes-tests
```

2. Import the test suite in your `pytest` code and configure it with the base arguments (usually containing the testing executable). The executable will be invoked with various arguments, and the test suite will assert certain side effects produced by the executable. Base arguments will be concatenated with additional arguments provided by the test suite.
> [!NOTE]
> To install `dagster-pipes-tests` in a repository other than this one, replace `--editable ../../tests/dagster-pipes-tests` with `git+https://github.com/dagster-io/communioty-integrations.git#subdirectory=libraries/pipes/tests/dagster-pipes-tests`

2. Import the test suite in your `pytest` code (for example, in `tests/test_pipes.py`) and configure it with the base arguments (usually containing the testing executable). The executable will be invoked with various arguments, and the test suite will assert certain side effects produced by the executable. Base arguments will be concatenated with additional arguments provided by the test suite.

For example, for Java:

Expand All @@ -45,8 +47,13 @@ class TestJavaPipes(PipesTestSuite):
]
```

3 [Optional]. When working with compiled languages, it's recommended to setup a `pytest` fixture that compiles the executable before running the tests. This way, the executable is only compiled once, and the tests can be run multiple times without recompiling.
> [!NOTE]
> Each test has it's own `--test-name` argument which can be used to identify the test being run.

> [!WARNING]
> This code must be placed in a file that is discovered by `pytest`, e.g. starts with `test_`.

When working with compiled languages, it's recommended to setup a `pytest` fixture that compiles the executable before running the tests. This way, the executable is only compiled once, and the tests can be run multiple times without recompiling.

For example, for Java, put the following code in `conftest.py`:

Expand All @@ -59,3 +66,9 @@ import subprocess
def built_jar():
subprocess.run(["./gradlew", "build"], check=True)
```

4. Run the tests with `pytest`:

```shell
uv run pytest
```
Loading
Loading