Skip to content

Commit

Permalink
Resolve "OCI cloud gor driver"
Browse files Browse the repository at this point in the history
  • Loading branch information
@gmagnu
gmagnu committed Nov 28, 2024
1 parent 4f6282f commit 5247f0e
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 9 deletions.
2 changes: 1 addition & 1 deletion drivers/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ project(':drivers') {
implementation "software.amazon.awssdk:s3:_"
implementation 'org.carlspring.cloud.aws:s3fs-nio:_'
//implementation "software.amazon.nio.s3:aws-java-nio-spi-for-s3:_"
implementation(platform('com.oracle.oci.sdk:oci-java-sdk-bom:_'))
implementation(platform('com.oracle.oci.sdk:oci-java-sdk-bom:_'))
implementation 'com.oracle.oci.sdk:oci-java-sdk-common-httpclient-jersey3'
implementation 'com.oracle.oci.sdk:oci-java-sdk-objectstorage'
implementation 'com.oracle.oci.sdk:oci-java-sdk-identity'
Expand Down
54 changes: 46 additions & 8 deletions drivers/src/main/java/org/gorpipe/oci/driver/OCIObjectStorageSourceProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,17 @@
import com.oracle.bmc.ConfigFileReader;
import com.oracle.bmc.Region;
import com.oracle.bmc.auth.*;
import com.oracle.bmc.http.client.StandardClientProperties;
import com.oracle.bmc.http.client.jersey3.ApacheClientProperties;
import com.oracle.bmc.http.client.jersey3.Jersey3ClientProperties;
import com.oracle.bmc.objectstorage.ObjectStorage;
import com.oracle.bmc.objectstorage.ObjectStorageAsync;
import com.oracle.bmc.objectstorage.ObjectStorageAsyncClient;
import com.oracle.bmc.objectstorage.ObjectStorageClient;
import org.apache.http.HttpResponse;
import org.apache.http.conn.ConnectionKeepAliveStrategy;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.protocol.HttpContext;
import org.gorpipe.base.security.BundledCredentials;
import org.gorpipe.base.security.Credentials;
import org.gorpipe.exceptions.GorResourceException;
Expand All @@ -36,8 +43,14 @@
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;

import static org.gorpipe.oci.driver.OCIUrl.DEFAULT_OCI_ENDPOINT;


@AutoService(SourceProvider.class)
public class OCIObjectStorageSourceProvider extends StreamSourceProvider {
static {
System.setProperty("sun.net.http.allowRestrictedHeaders", "true");
}
private static Logger log = org.slf4j.LoggerFactory.getLogger(OCIObjectStorageSourceProvider.class);

public static final String OCI_AUTH_TYPE_INSTANCE_PRINCIPAL = "INSTANCE_PRINCIPAL";
Expand Down Expand Up @@ -116,11 +129,31 @@ public static ObjectStorageAsync createClientAsync(Credentials cred) {
.maxAsyncThreads(10000)
.build();

return ObjectStorageAsyncClient.builder()
.endpoint(authData.endPoint)
.region(authData.region)
.configuration(clientConfig)
.build(provider);
PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
connectionManager.setMaxTotal(1000); // Default 50
connectionManager.setDefaultMaxPerRoute(connectionManager.getMaxTotal());

ConnectionKeepAliveStrategy keepAliveStrategy = new ConnectionKeepAliveStrategy() {
@Override
public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
return 5000;
}
};

ObjectStorageAsync client = ObjectStorageAsyncClient.builder()
.clientConfigurator(builder -> {
builder.property(Jersey3ClientProperties.USE_APACHE_CONNECTOR, false); // Big performance boost
builder.property(ApacheClientProperties.CONNECTION_MANAGER, connectionManager); // Default PoolingHttpClientConnectionManager
// builder.property(StandardClientProperties.BUFFER_REQUEST, true);
// builder.property(ApacheClientProperties.RETRY_HANDLER, null);
// builder.property(ApacheClientProperties.REUSE_STRATEGY, null);
builder.property(ApacheClientProperties.KEEPALIVE_STRATEGY, keepAliveStrategy);
})
.endpoint(authData.endPoint)
.region(authData.region)
.configuration(clientConfig)
.build(provider);
return client;
}

public static ObjectStorage createClientSync(Credentials cred) {
Expand All @@ -135,6 +168,9 @@ public static ObjectStorage createClientSync(Credentials cred) {
.build();

return ObjectStorageClient.builder()
.clientConfigurator( builder -> {
builder.property(Jersey3ClientProperties.USE_APACHE_CONNECTOR, false);
})
.endpoint(authData.endPoint)
.region(authData.region)
.configuration(clientConfig)
Expand Down Expand Up @@ -222,16 +258,18 @@ private record AuthData(Region region, String userId, String tenantId, String pr
private static final Pattern REGION_PATTERN = Pattern.compile(".*?\\.objectstorage\\.(.*?)\\..*");
public static AuthData from(Credentials creds) {
if (creds != null && !creds.isNull()) {
var endPoint = getOrDefault(creds, Credentials.Attr.API_ENDPOINT, OCIUrl.DEFAULT_OCI_ENDPOINT);
var endPoint = getOrDefault(creds, Credentials.Attr.API_ENDPOINT, DEFAULT_OCI_ENDPOINT);
var region = getRegion(creds, endPoint);
var tenantId = getOrDefault(creds, Credentials.Attr.REALM, OCI_TENANT);
var userId = creds.get(Credentials.Attr.SCOPE);
var userId = creds.getOrDefault(Credentials.Attr.SCOPE, OCI_USER);
var fingerprint = getOrDefault(creds, Credentials.Attr.KEY, OCI_SIMPLE_FINGERPRINT);
var privateKey = getOrDefault(creds, Credentials.Attr.SECRET, OCI_SIMPLE_PRIVATE_KEY).replaceAll("\\\\n", "\n");

return new AuthData(region, userId, tenantId, privateKey, fingerprint, endPoint);
} else {
return new AuthData(Region.US_ASHBURN_1, OCI_USER, OCI_TENANT, OCI_SIMPLE_PRIVATE_KEY, OCI_SIMPLE_FINGERPRINT, OCIUrl.DEFAULT_OCI_ENDPOINT);
return new AuthData(Region.US_ASHBURN_1, OCI_USER, OCI_TENANT,
OCI_SIMPLE_PRIVATE_KEY.replaceAll("\\\\n", "\n"),
OCI_SIMPLE_FINGERPRINT, DEFAULT_OCI_ENDPOINT);
}
}

Expand Down

0 comments on commit 5247f0e

Please sign in to comment.