diff --git a/.gitignore b/.gitignore
index 5dbf0258cf41..90ecde5dee45 100644
--- a/.gitignore
+++ b/.gitignore
@@ -283,7 +283,7 @@ instance/
 docs/_build/
 
 # PyBuilder
-target/
+/target/
 
 # Jupyter Notebook
 .ipynb_checkpoints
diff --git a/tests/integration/local/invoke/runtimes/test_with_runtime_zips.py b/tests/integration/local/invoke/runtimes/test_with_runtime_zips.py
index 97c30c5df1c4..6cb8b23938f7 100644
--- a/tests/integration/local/invoke/runtimes/test_with_runtime_zips.py
+++ b/tests/integration/local/invoke/runtimes/test_with_runtime_zips.py
@@ -13,14 +13,16 @@ def setUp(self):
 
         self.template_path = os.path.join(self.test_data_path, "invoke", "runtimes", "template.yaml")
 
-        self.events_file = tempfile.NamedTemporaryFile()
-        self.events_file.write('"yolo"')  # Just empty event
-        self.events_file.flush()
+        # Don't delete on close. Need the file to be present for tests to run.
+        events_file = tempfile.NamedTemporaryFile(delete=False)
+        events_file.write(b'"yolo"')  # Just empty event
+        events_file.flush()
+        events_file.close()
 
-        self.events_file_path = self.events_file.name
+        self.events_file_path = events_file.name
 
     def tearDown(self):
-        self.events_file.close()
+        os.remove(self.events_file_path)
 
     @parameterized.expand([
         param("Go1xFunction"),
diff --git a/tests/integration/testdata/invoke/runtimes/java8/target/HelloWorld-1.0.jar b/tests/integration/testdata/invoke/runtimes/java8/target/HelloWorld-1.0.jar
new file mode 100644
index 000000000000..444b5bde0682
Binary files /dev/null and b/tests/integration/testdata/invoke/runtimes/java8/target/HelloWorld-1.0.jar differ