Remove Job Controller and Beam Ingestion (#1163)

* Remove job controller and ingestion java code Signed-off-by: Willem Pienaar <[email protected]> * Remove feature statistics from Feast Core Signed-off-by: Willem Pienaar <[email protected]> * Remove JC and ingestion dependencies in Serving and Core Signed-off-by: Willem Pienaar <[email protected]> * Remove unused statistics and job controller protobufs and related code Signed-off-by: Willem Pienaar <[email protected]> * Remove Docker Pom addition for serving and core Signed-off-by: Willem Pienaar <[email protected]> * Remove BQ tests and beam dependencies Signed-off-by: Willem Pienaar <[email protected]> * Remove statistics test Signed-off-by: Willem Pienaar <[email protected]> * Add missing test dependencies Signed-off-by: Willem Pienaar <[email protected]> * Add BigQueryDataSource generator back Signed-off-by: Willem Pienaar <[email protected]> * Add okhttp into serving dependencies Signed-off-by: Willem Pienaar <[email protected]> * Add createBigQueryDataSourceSpec Signed-off-by: Willem Pienaar <[email protected]>
feast-dev · Nov 17, 2020 · 977f659 · 977f659
1 parent 5962756
commit 977f659
Show file tree

Hide file tree

Showing 205 changed files with 31 additions and 21,527 deletions.
diff --git a/common-test/src/main/java/feast/common/it/DataGenerator.java b/common-test/src/main/java/feast/common/it/DataGenerator.java
@@ -26,7 +26,6 @@
 import feast.proto.core.DataFormatProto.StreamFormat.AvroFormat;
 import feast.proto.core.DataFormatProto.StreamFormat.ProtoFormat;
 import feast.proto.core.DataSourceProto.DataSource;
-import feast.proto.core.DataSourceProto.DataSource.BigQueryOptions;
 import feast.proto.core.DataSourceProto.DataSource.FileOptions;
 import feast.proto.core.DataSourceProto.DataSource.KafkaOptions;
 import feast.proto.core.DataSourceProto.DataSource.KinesisOptions;
@@ -107,10 +106,6 @@ public static StoreProto.Store createStore(
         StoreProto.Store.RedisConfig redisConfig =
             StoreProto.Store.RedisConfig.newBuilder().build();
         return builder.setRedisConfig(redisConfig).build();
-      case BIGQUERY:
-        StoreProto.Store.BigQueryConfig bqConfig =
-            StoreProto.Store.BigQueryConfig.newBuilder().build();
-        return builder.setBigqueryConfig(bqConfig).build();
       case REDIS_CLUSTER:
         StoreProto.Store.RedisClusterConfig redisClusterConfig =
             StoreProto.Store.RedisClusterConfig.newBuilder().build();
@@ -289,7 +284,8 @@ public static DataSource createBigQueryDataSourceSpec(
       String bigQueryTableRef, String timestampColumn, String datePartitionColumn) {
     return DataSource.newBuilder()
         .setType(DataSource.SourceType.BATCH_BIGQUERY)
-        .setBigqueryOptions(BigQueryOptions.newBuilder().setTableRef(bigQueryTableRef).build())
+        .setBigqueryOptions(
+            DataSource.BigQueryOptions.newBuilder().setTableRef(bigQueryTableRef).build())
         .setEventTimestampColumn(timestampColumn)
         .setDatePartitionColumn(datePartitionColumn)
         .build();

diff --git a/common-test/src/main/java/feast/common/it/SimpleJcClient.java b/common-test/src/main/java/feast/common/it/SimpleJcClient.java
diff --git a/core/pom.xml b/core/pom.xml
@@ -80,12 +80,6 @@
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>dev.feast</groupId>
-            <artifactId>feast-storage-connector-bigquery</artifactId>
-            <version>${project.version}</version>
-            <scope>compile</scope>
-        </dependency>
 
         <!-- Hot reloading for Spring Boot. spring-boot-maven-plugin removes
              this automatically when packaging. -->
@@ -312,5 +306,21 @@
             <version>2.27.0</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.avro</groupId>
+            <artifactId>avro</artifactId>
+            <version>1.8.2</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.squareup.okhttp</groupId>
+            <artifactId>okhttp</artifactId>
+            <version>2.7.4</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-testing</artifactId>
+        </dependency>
     </dependencies>
 </project>
diff --git a/core/src/main/java/feast/core/controller/CoreServiceRestController.java b/core/src/main/java/feast/core/controller/CoreServiceRestController.java
@@ -22,11 +22,7 @@
 import feast.core.model.Project;
 import feast.core.service.ProjectService;
 import feast.core.service.SpecService;
-import feast.core.service.StatsService;
 import feast.proto.core.CoreServiceProto.GetFeastCoreVersionResponse;
-import feast.proto.core.CoreServiceProto.GetFeatureStatisticsRequest;
-import feast.proto.core.CoreServiceProto.GetFeatureStatisticsRequest.Builder;
-import feast.proto.core.CoreServiceProto.GetFeatureStatisticsResponse;
 import feast.proto.core.CoreServiceProto.ListEntitiesRequest;
 import feast.proto.core.CoreServiceProto.ListEntitiesResponse;
 import feast.proto.core.CoreServiceProto.ListFeatureSetsRequest;
@@ -36,7 +32,6 @@
 import feast.proto.core.CoreServiceProto.ListFeaturesRequest;
 import feast.proto.core.CoreServiceProto.ListFeaturesResponse;
 import feast.proto.core.CoreServiceProto.ListProjectsResponse;
-import java.io.IOException;
 import java.time.LocalDate;
 import java.time.LocalTime;
 import java.time.ZoneOffset;
@@ -64,18 +59,13 @@ public class CoreServiceRestController {
 
   private final FeastProperties feastProperties;
   private SpecService specService;
-  private StatsService statsService;
   private ProjectService projectService;
 
   @Autowired
   public CoreServiceRestController(
-      FeastProperties feastProperties,
-      SpecService specService,
-      StatsService statsService,
-      ProjectService projectService) {
+      FeastProperties feastProperties, SpecService specService, ProjectService projectService) {
     this.feastProperties = feastProperties;
     this.specService = specService;
-    this.statsService = statsService;
     this.projectService = projectService;
   }
 
@@ -133,58 +123,6 @@ public ListFeaturesResponse listFeatures(
     return specService.listFeatures(filterBuilder.build());
   }
 
-  /**
-   * GET /feature-statistics : Fetches statistics for a dataset speficied by the parameters. Either
-   * both (start_date, end_date) need to be given or ingestion_ids are required. If both are given,
-   * (start_date, end_date) will be ignored.
-   *
-   * @param ingestionIds Request Parameter: List of ingestion IDs. If missing, both startDate and
-   *     endDate should be provided.
-   * @param startDate Request Parameter: UTC+0 starting date (inclusive) in the ISO format, from
-   *     <code>0001-01-01</code> to <code>9999-12-31</code>. Time given will be ignored. This
-   *     parameter will be ignored if any ingestionIds is provided.
-   * @param endDate Request Parameter: UTC+0 ending date (exclusive) in the ISO format, from <code>
-   *      0001-01-01</code> to <code>9999-12-31</code>. Time given will be ignored. This parameter
-   *     will be ignored if any ingestionIds is provided.
-   * @param store Request Parameter: The name of the historical store used in Feast Serving. Online
-   *     store is not allowed.
-   * @param featureSetId Request Parameter: Feature set ID, which has the form of <code>
-   *                     project/feature_set_name</code>.
-   * @param forceRefresh Request Parameter: whether to override the values in the cache. Accepts
-   *     <code>true</code>, <code>false</code>.
-   * @param features (Optional) Request Parameter: List of features. If none provided, all features
-   *     in the feature set will be used for statistics.
-   * @return (200 OK) Returns {@link GetFeatureStatisticsResponse} in JSON.
-   */
-  @RequestMapping(value = "/v1/feature-statistics", method = RequestMethod.GET)
-  public GetFeatureStatisticsResponse getFeatureStatistics(
-      @RequestParam(name = "feature_set_id") String featureSetId,
-      @RequestParam(required = false) Optional<String[]> features,
-      @RequestParam String store,
-      @RequestParam(name = "start_date", required = false) Optional<String> startDate,
-      @RequestParam(name = "end_date", required = false) Optional<String> endDate,
-      @RequestParam(name = "ingestion_ids", required = false) Optional<String[]> ingestionIds,
-      @RequestParam(name = "force_refresh") boolean forceRefresh)
-      throws IOException {
-
-    Builder requestBuilder =
-        GetFeatureStatisticsRequest.newBuilder()
-            .setForceRefresh(forceRefresh)
-            .setFeatureSetId(featureSetId)
-            .setStore(store);
-
-    // set optional request parameters if they are provided
-    features.ifPresent(theFeatures -> requestBuilder.addAllFeatures(Arrays.asList(theFeatures)));
-    startDate.ifPresent(
-        startDateStr -> requestBuilder.setStartDate(utcTimeStringToTimestamp(startDateStr)));
-    endDate.ifPresent(
-        endDateStr -> requestBuilder.setEndDate(utcTimeStringToTimestamp(endDateStr)));
-    ingestionIds.ifPresent(
-        theIngestionIds -> requestBuilder.addAllIngestionIds(Arrays.asList(theIngestionIds)));
-
-    return statsService.getFeatureStatistics(requestBuilder.build());
-  }
-
   /**
    * GET /projects : Get the list of existing feast projects.
    *

diff --git a/core/src/main/java/feast/core/dao/FeatureStatisticsRepository.java b/core/src/main/java/feast/core/dao/FeatureStatisticsRepository.java
diff --git a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java
@@ -25,7 +25,6 @@
 import feast.core.model.Project;
 import feast.core.service.ProjectService;
 import feast.core.service.SpecService;
-import feast.core.service.StatsService;
 import feast.proto.core.CoreServiceGrpc.CoreServiceImplBase;
 import feast.proto.core.CoreServiceProto.*;
 import feast.proto.core.EntityProto.EntitySpecV2;
@@ -49,21 +48,18 @@ public class CoreServiceImpl extends CoreServiceImplBase {
 
   private final FeastProperties feastProperties;
   private SpecService specService;
-  private StatsService statsService;
   private ProjectService projectService;
   private final AuthorizationService authorizationService;
 
   @Autowired
   public CoreServiceImpl(
       SpecService specService,
       ProjectService projectService,
-      StatsService statsService,
       FeastProperties feastProperties,
       AuthorizationService authorizationService) {
     this.specService = specService;
     this.projectService = projectService;
     this.feastProperties = feastProperties;
-    this.statsService = statsService;
     this.authorizationService = authorizationService;
   }
 
@@ -188,32 +184,6 @@ public void listEntities(
     }
   }
 
-  @Override
-  public void getFeatureStatistics(
-      GetFeatureStatisticsRequest request,
-      StreamObserver<GetFeatureStatisticsResponse> responseObserver) {
-    try {
-      GetFeatureStatisticsResponse response = statsService.getFeatureStatistics(request);
-      responseObserver.onNext(response);
-      responseObserver.onCompleted();
-    } catch (IllegalArgumentException e) {
-      log.error("Illegal arguments provided to GetFeatureStatistics method: ", e);
-      responseObserver.onError(
-          Status.INVALID_ARGUMENT
-              .withDescription(e.getMessage())
-              .withCause(e)
-              .asRuntimeException());
-    } catch (RetrievalException e) {
-      log.error("Unable to fetch feature set requested in GetFeatureStatistics method: ", e);
-      responseObserver.onError(
-          Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asRuntimeException());
-    } catch (Exception e) {
-      log.error("Exception has occurred in GetFeatureStatistics method: ", e);
-      responseObserver.onError(
-          Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException());
-    }
-  }
-
   @Override
   public void updateFeatureSetStatus(
       UpdateFeatureSetStatusRequest request,