- Forkwrite ensure dict also for remote.

- Create directories fix for S3 (only create last element, but needs less access writes).
- Test tuning.

drivers/src/main/java/org/gorpipe/oci/driver/OCIObjectStorageSource.java

@@ -15,7 +15,6 @@
 import com.oracle.bmc.objectstorage.responses.HeadObjectResponse;
 import com.oracle.bmc.objectstorage.responses.PutObjectResponse;
 import org.gorpipe.base.streams.LimitedOutputStream;
-import org.gorpipe.exceptions.GorException;
 import org.gorpipe.exceptions.GorResourceException;
 import org.gorpipe.exceptions.GorSystemException;
 import org.gorpipe.gor.binsearch.GorIndexType;
@@ -25,13 +24,14 @@
 import org.gorpipe.gor.driver.providers.stream.RequestRange;
 import org.gorpipe.gor.driver.providers.stream.sources.StreamSource;
 import org.gorpipe.gor.driver.providers.stream.sources.StreamSourceMetadata;
+import org.gorpipe.gor.table.util.PathUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
 
 import java.io.*;
 import java.net.MalformedURLException;
-import java.util.concurrent.ExecutionException;
+import java.nio.file.attribute.FileAttribute;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
@@ -140,11 +140,7 @@ private InputStream openRequest(GetObjectRequest request) {
             Thread.currentThread().interrupt();
             throw new GorSystemException(e);
         } catch (Exception e) {
-            Throwable ex = e;
-            if (e instanceof ExecutionException || e instanceof UncheckedExecutionException) {
-                ex = e.getCause();
-            }
-            throw new GorResourceException("Failed to open S3 object: " + sourceReference.getUrl(), sourceReference.getUrl(), ex).retry();
+            throw new GorResourceException("Failed to open S3 object: " + sourceReference.getUrl(), sourceReference.getUrl(), e).retry();
         }
     }
 
@@ -180,23 +176,15 @@ private StreamSourceMetadata createMetaData(String bucket, String key) {
             Thread.currentThread().interrupt();
             throw new GorSystemException(e);
         } catch (Exception e) {
-            Throwable ex = e;
-            if (e instanceof ExecutionException || e instanceof UncheckedExecutionException) {
-                ex = e.getCause();
-            }
-            throw new GorResourceException("Failed to load metadata for " + bucket + "/" + key, getName(), ex).retry();
+            throw new GorResourceException("Failed to load metadata for " + bucket + "/" + key, getName(), e).retry();
         }
     }
 
     private StreamSourceMetadata loadMetadataFromCache(String bucket, String key) {
         try {
             return metadataCache.get(bucket + key, () -> createMetaData(bucket, key));
-        } catch (ExecutionException | UncheckedExecutionException e) {
-            var cause = e.getCause() != null ? e.getCause() : e;
-            if (cause instanceof GorException) {
-                throw (GorException) cause;
-            }
-            throw new GorResourceException("Failed to load metadata from cache for " + bucket + "/" + key, getName(), cause).retry();
+        } catch (Exception  e) {
+            throw new GorResourceException("Failed to load metadata from cache for " + bucket + "/" + key, getName(), e).retry();
         }
     }
 
@@ -253,10 +241,6 @@ public void delete() {
             Thread.currentThread().interrupt();
             throw new GorSystemException(e);
         } catch (Exception e) {
-            Throwable ex = e;
-            if (e instanceof ExecutionException || e instanceof UncheckedExecutionException) {
-                ex = e.getCause();
-            }
             throw new GorResourceException("Failed to delete " + getName(), getName(), e).retry();
         }
 
@@ -282,4 +266,37 @@ public GorIndexType useIndex() {
     public void close() {
         // No resources to free
     }
+
+    @Override
+    public String createDirectory(FileAttribute<?>... attrs) {
+        try {
+            var folder = PathUtils.markAsFolder(key);
+            var request = PutObjectRequest.builder()
+                    .namespaceName(namespace)
+                    .bucketName(bucket)
+                    .objectName(PathUtils.markAsFolder(key))
+                    .putObjectBody(new InputStream() {
+                        @Override
+                        public int read() {
+                            return -1;
+                        }
+                    })
+                    .contentLength(0L)
+                    .build();
+
+            client.putObject(request, null).get();
+            return folder;
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            throw new GorSystemException(e);
+        } catch (Exception e) {
+            throw new GorResourceException("Failed to create directory " + getName(), getName(), e).retry();
+        }
+    }
+
+    @Override
+    public String createDirectories(FileAttribute<?>... attrs) {
+        // For now just create the last directory in the path.
+        return createDirectory(attrs);
+    }
 }

drivers/src/main/java/org/gorpipe/s3/driver/S3Source.java

@@ -224,7 +224,7 @@ public DataType getDataType() {
     public boolean exists() {
         try {
             // Note: fileExists only handles dirs if the end with /. Therefor we fall back to the much slower Files.exists.
-            //       Do exists with aws filesystem: return Files.exists(Path.of(URI.create(getName())));
+            //       To do a exists with aws filesystem: return Files.exists(Path.of(URI.create(getName())));
             return fileExists() || Files.exists(getPath());
         } catch (Exception e) {
             Credentials cred = S3ClientFileSystemProvider.getCredentials(sourceReference.getSecurityContext(), "s3", bucket);
@@ -256,7 +256,7 @@ public String createDirectory(FileAttribute<?>... attrs) {
             // Files.createDirectory needs elevated access to list all buckets.
             S3Path s3Path = getPath();
             String directoryKey = s3Path.getKey().endsWith("/") ? s3Path.getKey() : s3Path.getKey() + "/";
-            PutObjectRequest request = (PutObjectRequest)PutObjectRequest.builder().bucket(s3Path.getBucketName()).key(directoryKey).cacheControl(s3Path.getFileSystem().getRequestHeaderCacheControlProperty()).contentLength(0L).build();
+            PutObjectRequest request = PutObjectRequest.builder().bucket(s3Path.getBucketName()).key(directoryKey).cacheControl(s3Path.getFileSystem().getRequestHeaderCacheControlProperty()).contentLength(0L).build();
             client.putObject(request, RequestBody.fromBytes(new byte[0]));
             return PathUtils.formatUri(getPath().toUri());
         } catch (Exception e) {
@@ -266,11 +266,14 @@ public String createDirectory(FileAttribute<?>... attrs) {
 
     @Override
     public String createDirectories(FileAttribute<?>... attrs) {
-        try {
-            return PathUtils.formatUri(Files.createDirectories(getPath()).toUri());
-        } catch (IOException e) {
-            throw GorResourceException.fromIOException(e, getPath()).retry();
-        }
+        // Files.createDirectory needs elevated access to list all buckets.
+//        try {
+//            return PathUtils.formatUri(Files.createDirectories(getPath()).toUri());
+//        } catch (IOException e) {
+//            throw GorResourceException.fromIOException(e, getPath()).retry();
+//        }
+        // so for now just create the final directory, not the intermediate ones.
+        return createDirectory(attrs);
     }
 
     @Override

drivers/src/main/java/org/gorpipe/s3/driver/S3SourceProvider.java

@@ -155,7 +155,6 @@ private S3Client createClient(Credentials cred) {
     AwsCredentialsProvider getCredentialsProvider(Credentials cred) {
         if (cred == null || cred.isNull()) {
             log.info("CredentialsProvider: DefaultCredentialsProvider for null creds");
-            //throw new IllegalArgumentException("Credentials are required for S3 access");
             return DefaultCredentialsProvider.create();
         } else {
             var awsKey = cred.getOrDefault(Credentials.Attr.KEY, "");
@@ -180,7 +179,6 @@ AwsCredentialsProvider getCredentialsProvider(Credentials cred) {
                 }
             } else {
                 log.info(String.format("CredentialsProvider: DefaultCredentialsProvider for %s:%s", cred.getService(), cred.getLookupKey()));
-                //throw new IllegalArgumentException(String.format("CredentialsProvider: DefaultCredentialsProvider for %s:%s", cred.getService(), cred.getLookupKey()));
                 return DefaultCredentialsProvider.create();
             }
         }

drivers/src/test/java/org/gorpipe/s3/driver/ITestS3Source.java

@@ -156,7 +156,7 @@ public void testS3WriteMoreThanMaxChunks() {
     public void testS3WritePgorGord() throws IOException {
         String securityContext = DriverUtils.awsSecurityContext(S3_KEY, S3_SECRET);
         String randomId = UUID.randomUUID().toString();
-        String dict = String.format("s3://gdb-unit-test-data/s3write/%s-genes.gord", randomId);
+        String dict = String.format("s3://gdb-unit-test-data/s3write/%s-genes.gord/", randomId);
         TestUtils.runGorPipe("pgor -split 2 ../tests/data/gor/genes.gor | top 2 | write " + dict, false, securityContext);
         String expected = TestUtils.runGorPipe("create x = pgor -split 2 ../tests/data/gor/genes.gor | top 2; gor [x] | select 1-4", false, securityContext);
         String result = TestUtils.runGorPipe("gor " + dict + " | select 1-4", false, securityContext);

drivers/src/test/java/org/gorpipe/s3/table/ITestS3Table.java

@@ -19,6 +19,7 @@
 import org.junit.contrib.java.lang.system.EnvironmentVariables;
 import org.junit.contrib.java.lang.system.ProvideSystemProperty;
 import org.junit.contrib.java.lang.system.RestoreSystemProperties;
+import org.junit.contrib.java.lang.system.SystemErrRule;
 import org.junit.experimental.categories.Category;
 import org.junit.rules.TemporaryFolder;
 
@@ -42,6 +43,8 @@ public class ITestS3Table {
 
     private static String S3_KEY;
     private static String S3_SECRET;
+    private static String S3_REGION = "eu-west-1";
+
     private DriverBackedFileReader fileReader;
 
     private static final String S3_FILE_1 = "s3://gdb-unit-test-data/csa_test_data/data_sets/sim20-micro/source/var/D3_WGC053023D.wgs.genotypes.gor";
@@ -55,6 +58,25 @@ public class ITestS3Table {
     public TemporaryFolder workDir = new TemporaryFolder();
     private Path workDirPath;
 
+    @Rule
+    public final ProvideSystemProperty myPropertyHasMyValue
+            = new ProvideSystemProperty("aws.accessKeyId", S3_KEY);
+
+    @Rule
+    public final ProvideSystemProperty otherPropertyIsMissing
+            = new ProvideSystemProperty("aws.secretKey", S3_SECRET);
+
+    @Rule
+    public final ProvideSystemProperty awsSecretAccessKey
+            = new ProvideSystemProperty("aws.secretAccessKey", S3_SECRET);
+
+    @Rule
+    public final ProvideSystemProperty awsRegion
+            = new ProvideSystemProperty("aws.region", S3_REGION);
+
+    @Rule
+    public final SystemErrRule systemErrRule = new SystemErrRule().enableLog();
+
     @Rule
     public final RestoreSystemProperties restoreSystemProperties = new RestoreSystemProperties();
 

gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala

@@ -107,17 +107,15 @@ case class ForkWrite(forkCol: Int,
 
   def ensureDir(projectContext: ProjectContext, path: String, parent: Boolean = false): Unit = {
     val fileReader = projectContext.getFileReader
-    if (PathUtils.isLocal(path)) {
-      val dir = if (parent) {
-        val parent = Paths.get(path).getParent
-        if (parent != null) parent.toString else null
-      } else {
-        path
-      }
+    val dir = if (parent) {
+      val parent = PathUtils.getParent(path)
+      if (parent != null) parent else null
+    } else {
+      path
+    }
 
-      if (dir != null && !fileReader.exists(dir)) {
-        fileReader.createDirectories(dir)
-      }
+    if (dir != null && !fileReader.exists(dir)) {
+      fileReader.createDirectories(dir)
     }
   }
 

versions.properties

@@ -54,7 +54,9 @@ version.software.amazon.awssdk..s3=2.29.39
 
 #version.software.amazon.nio.s3..aws-java-nio-spi-for-s3=2.0.5
 
-version.org.carlspring.cloud.aws..s3fs-nio=2.0.0
+# Must be careful if we want to update to version 2.0.0.  It looks like some actions do not work as before.
+# For example, testCreateDeleteListEmptyDirectory fails every other run with 2.0.0.
+version.org.carlspring.cloud.aws..s3fs-nio=1.0.6
 ##                             # available=2.0.0
 
 version.com.auth0..java-jwt=4.4.0