Add back integration test and test gcs and s3 for real

Signed-off-by: Achal Shah <[email protected]>

java/serving/src/main/java/feast/serving/config/ApplicationProperties.java

@@ -152,6 +152,10 @@ public String getGcpProject() {
     return gcpProject;
   }
 
+  public void setAwsRegion(String awsRegion) {
+    this.awsRegion = awsRegion;
+  }
+
   private String awsRegion;
 
   public String getAwsRegion() {

java/serving/src/main/java/feast/serving/config/RegistryConfig.java

@@ -37,7 +37,7 @@ Storage googleStorage(ApplicationProperties applicationProperties) {
   }
 
   @Provides
-  AmazonS3 awsStorage(ApplicationProperties applicationProperties) {
+  public AmazonS3 awsStorage(ApplicationProperties applicationProperties) {
     return AmazonS3ClientBuilder.standard()
         .withRegion(applicationProperties.getAwsRegion())
         .build();

java/serving/src/main/java/feast/serving/grpc/OnlineServingGrpcServiceV2.java

@@ -21,12 +21,8 @@
 import feast.serving.service.ServingServiceV2;
 import io.grpc.stub.StreamObserver;
 import javax.inject.Inject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 public class OnlineServingGrpcServiceV2 extends ServingServiceGrpc.ServingServiceImplBase {
-  public static final Logger logger = LoggerFactory.getLogger(OnlineServingGrpcServiceV2.class);
-
   private final ServingServiceV2 servingServiceV2;
 
   @Inject
@@ -46,7 +42,6 @@ public void getFeastServingInfo(
   public void getOnlineFeaturesV2(
       ServingAPIProto.GetOnlineFeaturesRequestV2 request,
       StreamObserver<ServingAPIProto.GetOnlineFeaturesResponse> responseObserver) {
-    logger.info("In the getOnlineFeaturesV2");
     responseObserver.onNext(this.servingServiceV2.getOnlineFeatures(request));
     responseObserver.onCompleted();
   }

java/serving/src/test/java/feast/serving/it/ServingBase.java

@@ -21,8 +21,9 @@
 import static org.junit.jupiter.api.Assertions.*;
 
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
 import com.google.inject.*;
+import com.google.inject.Module;
+import com.google.inject.util.Modules;
 import com.google.protobuf.Timestamp;
 import feast.proto.core.FeatureProto;
 import feast.proto.core.FeatureViewProto;
@@ -92,19 +93,10 @@ protected void configure() {
           @Provides
           ApplicationProperties applicationProperties() {
             final ApplicationProperties p = new ApplicationProperties();
-            final ApplicationProperties.FeastProperties feastProperties =
-                new ApplicationProperties.FeastProperties();
-            p.setFeast(feastProperties);
-
-            feastProperties.setRegistry("src/test/resources/docker-compose/feast10/registry.db");
-            feastProperties.setRegistryRefreshInterval(0);
-
-            feastProperties.setActiveStore("online");
+            p.setAwsRegion("us-east-1");
 
-            feastProperties.setStores(
-                ImmutableList.of(
-                    new ApplicationProperties.Store(
-                        "online", "REDIS", ImmutableMap.of("host", "localhost", "port", "6379"))));
+            final ApplicationProperties.FeastProperties feastProperties = createFeastProperties();
+            p.setFeast(feastProperties);
 
             final ApplicationProperties.TracingProperties tracingProperties =
                 new ApplicationProperties.TracingProperties();
@@ -115,10 +107,18 @@ ApplicationProperties applicationProperties() {
           }
         };
 
+    Module overrideConfig = registryConfig();
+    Module registryConfig;
+    if (overrideConfig != null) {
+      registryConfig = Modules.override(new RegistryConfig()).with(registryConfig());
+    } else {
+      registryConfig = new RegistryConfig();
+    }
+
     injector =
         Guice.createInjector(
             new ServingServiceConfigV2(),
-            new RegistryConfig(),
+            registryConfig,
             new InstrumentationConfig(),
             appPropertiesModule);
 
@@ -201,7 +201,7 @@ private static RegistryProto.Registry readLocalRegistry() {
   public void shouldGetOnlineFeatures() {
     ServingAPIProto.GetOnlineFeaturesRequestV2 req = buildOnlineRequest(1005);
     ServingAPIProto.GetOnlineFeaturesResponse featureResponse =
-        servingStub.withDeadlineAfter(100, TimeUnit.MILLISECONDS).getOnlineFeaturesV2(req);
+        servingStub.withDeadlineAfter(1000, TimeUnit.MILLISECONDS).getOnlineFeaturesV2(req);
 
     assertEquals(1, featureResponse.getFieldValuesCount());
 
@@ -297,5 +297,11 @@ public void shouldRefreshRegistryAndServeNewFeatures() throws InterruptedExcepti
         .until(() -> servingStub.getOnlineFeaturesV2(request).getFieldValuesCount(), equalTo(1));
   }
 
+  abstract ApplicationProperties.FeastProperties createFeastProperties();
+
+  AbstractModule registryConfig() {
+    return null;
+  }
+
   abstract void updateRegistryFile(RegistryProto.Registry registry);
 }

java/serving/src/test/java/feast/serving/it/ServingRedisGSRegistryIT.java

@@ -20,7 +20,10 @@
 
 import com.google.cloud.storage.*;
 import com.google.cloud.storage.testing.RemoteStorageHelper;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import feast.proto.core.RegistryProto;
+import feast.serving.config.ApplicationProperties;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import org.junit.jupiter.api.AfterAll;
@@ -37,27 +40,46 @@ public class ServingRedisGSRegistryIT extends ServingBase {
 
   static final String bucket = RemoteStorageHelper.generateBucketName();
 
-  static void putToStorage(RegistryProto.Registry registry) {
-    BlobId blobId = BlobId.of(bucket, "registry.db");
+  static void putToStorage(BlobId blobId, RegistryProto.Registry registry) {
     storage.create(BlobInfo.newBuilder(blobId).build(), registry.toByteArray());
 
     assertArrayEquals(storage.get(blobId).getContent(), registry.toByteArray());
   }
 
+  static BlobId blobId;
+
   @BeforeAll
   static void setUp() {
     storage.create(BucketInfo.of(bucket));
+    blobId = BlobId.of(bucket, "registry.db");
 
-    putToStorage(registryProto);
+    putToStorage(blobId, registryProto);
   }
 
   @AfterAll
   static void tearDown() throws ExecutionException, InterruptedException {
     RemoteStorageHelper.forceDelete(storage, bucket, 5, TimeUnit.SECONDS);
   }
 
+  @Override
+  ApplicationProperties.FeastProperties createFeastProperties() {
+    final ApplicationProperties.FeastProperties feastProperties =
+        new ApplicationProperties.FeastProperties();
+    feastProperties.setRegistry(blobId.toGsUtilUri());
+    feastProperties.setRegistryRefreshInterval(1);
+
+    feastProperties.setActiveStore("online");
+
+    feastProperties.setStores(
+        ImmutableList.of(
+            new ApplicationProperties.Store(
+                "online", "REDIS", ImmutableMap.of("host", "localhost", "port", "6379"))));
+
+    return feastProperties;
+  }
+
   @Override
   void updateRegistryFile(RegistryProto.Registry registry) {
-    putToStorage(registry);
+    putToStorage(blobId, registry);
   }
 }

java/serving/src/test/java/feast/serving/it/ServingRedisLocalRegistryIT.java

@@ -0,0 +1,47 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.serving.it;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import feast.proto.core.RegistryProto;
+import feast.serving.config.ApplicationProperties;
+
+public class ServingRedisLocalRegistryIT extends ServingBase {
+  @Override
+  ApplicationProperties.FeastProperties createFeastProperties() {
+    final ApplicationProperties.FeastProperties feastProperties =
+        new ApplicationProperties.FeastProperties();
+    feastProperties.setRegistry("src/test/resources/docker-compose/feast10/registry.db");
+    feastProperties.setRegistryRefreshInterval(1);
+
+    feastProperties.setActiveStore("online");
+
+    feastProperties.setStores(
+        ImmutableList.of(
+            new ApplicationProperties.Store(
+                "online", "REDIS", ImmutableMap.of("host", "localhost", "port", "6379"))));
+
+    return feastProperties;
+  }
+
+  @Override
+  void updateRegistryFile(RegistryProto.Registry registry) {}
+
+  @Override
+  public void shouldRefreshRegistryAndServeNewFeatures() throws InterruptedException {}
+}

java/serving/src/test/java/feast/serving/it/ServingRedisS3RegistryIT.java

@@ -21,14 +21,18 @@
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.inject.AbstractModule;
+import com.google.inject.Provides;
 import feast.proto.core.RegistryProto;
+import feast.serving.config.ApplicationProperties;
 import java.io.ByteArrayInputStream;
-import java.io.IOException;
 import org.junit.jupiter.api.BeforeAll;
 import org.testcontainers.junit.jupiter.Container;
 
 public class ServingRedisS3RegistryIT extends ServingBase {
-  @Container private static final S3MockContainer s3Mock = new S3MockContainer("2.2.3");
+  @Container static final S3MockContainer s3Mock = new S3MockContainer("2.2.3");
 
   private static AmazonS3 createClient() {
     return AmazonS3ClientBuilder.standard()
@@ -50,15 +54,47 @@ private static void putToStorage(RegistryProto.Registry proto) {
   }
 
   @BeforeAll
-  static void setUp() throws IOException {
+  static void setUp() {
     AmazonS3 s3Client = createClient();
     s3Client.createBucket("test-bucket");
 
     putToStorage(registryProto);
   }
 
+  @Override
+  ApplicationProperties.FeastProperties createFeastProperties() {
+    final ApplicationProperties.FeastProperties feastProperties =
+        new ApplicationProperties.FeastProperties();
+    feastProperties.setRegistry("s3://test-bucket/registry.db");
+    feastProperties.setRegistryRefreshInterval(1);
+
+    feastProperties.setActiveStore("online");
+
+    feastProperties.setStores(
+        ImmutableList.of(
+            new ApplicationProperties.Store(
+                "online", "REDIS", ImmutableMap.of("host", "localhost", "port", "6379"))));
+
+    return feastProperties;
+  }
+
   @Override
   void updateRegistryFile(RegistryProto.Registry registry) {
     putToStorage(registry);
   }
+
+  @Override
+  AbstractModule registryConfig() {
+    return new AbstractModule() {
+      @Provides
+      public AmazonS3 awsStorage() {
+        return AmazonS3ClientBuilder.standard()
+            .withEndpointConfiguration(
+                new AwsClientBuilder.EndpointConfiguration(
+                    String.format("http://localhost:%d", s3Mock.getHttpServerPort()), "us-east-1"))
+            .enablePathStyleAccess()
+            .build();
+      }
+    };
+  }
 }