datahub-project · david-leifker · Jan 28, 2023 · Dec 29, 2022 · Dec 29, 2022 · Dec 29, 2022
diff --git a/docker/datahub-gms/env/docker-without-neo4j.env b/docker/datahub-gms/env/docker-without-neo4j.env
@@ -10,8 +10,6 @@ ELASTICSEARCH_HOST=elasticsearch
 ELASTICSEARCH_PORT=9200
 ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
 ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
-ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1
-ES_BULK_FLUSH_PERIOD=1
 ES_BULK_REFRESH_POLICY=WAIT_UNTIL
 GRAPH_SERVICE_DIFF_MODE_ENABLED=true
 GRAPH_SERVICE_IMPL=elasticsearch

diff --git a/docker/datahub-gms/env/docker.cassandra.env b/docker/datahub-gms/env/docker.cassandra.env
@@ -6,8 +6,6 @@ ELASTICSEARCH_PORT=9200
 ES_BULK_REFRESH_POLICY=WAIT_UNTIL
 ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
 ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
-ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1
-ES_BULK_FLUSH_PERIOD=1
 NEO4J_HOST=http://neo4j:7474
 NEO4J_URI=bolt://neo4j
 NEO4J_USERNAME=neo4j

diff --git a/docker/datahub-gms/env/docker.env b/docker/datahub-gms/env/docker.env
@@ -11,8 +11,6 @@ ELASTICSEARCH_PORT=9200
 ES_BULK_REFRESH_POLICY=WAIT_UNTIL
 ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
 ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
-ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1
-ES_BULK_FLUSH_PERIOD=1
 NEO4J_HOST=http://neo4j:7474
 NEO4J_URI=bolt://neo4j
 NEO4J_USERNAME=neo4j

diff --git a/docker/datahub-gms/env/docker.mariadb.env b/docker/datahub-gms/env/docker.mariadb.env
@@ -11,8 +11,6 @@ ELASTICSEARCH_PORT=9200
 ES_BULK_REFRESH_POLICY=WAIT_UNTIL
 ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
 ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
-ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1
-ES_BULK_FLUSH_PERIOD=1
 NEO4J_HOST=http://neo4j:7474
 NEO4J_URI=bolt://neo4j
 NEO4J_USERNAME=neo4j

diff --git a/docker/datahub-gms/env/docker.postgres.env b/docker/datahub-gms/env/docker.postgres.env
@@ -14,8 +14,6 @@ ELASTICSEARCH_PORT=9200
 ES_BULK_REFRESH_POLICY=WAIT_UNTIL
 ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
 ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
-ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1
-ES_BULK_FLUSH_PERIOD=1
 NEO4J_HOST=http://neo4j:7474
 NEO4J_URI=bolt://neo4j
 NEO4J_USERNAME=neo4j

diff --git a/docker/datahub-upgrade/env/docker-without-neo4j.env b/docker/datahub-upgrade/env/docker-without-neo4j.env
@@ -11,8 +11,6 @@ ELASTICSEARCH_HOST=elasticsearch
 ELASTICSEARCH_PORT=9200
 ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
 ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
-ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1
-ES_BULK_FLUSH_PERIOD=1
 ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false
 
 GRAPH_SERVICE_IMPL=elasticsearch

diff --git a/docker/datahub-upgrade/env/docker.env b/docker/datahub-upgrade/env/docker.env
@@ -11,8 +11,6 @@ ELASTICSEARCH_HOST=elasticsearch
 ELASTICSEARCH_PORT=9200
 ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
 ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
-ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1
-ES_BULK_FLUSH_PERIOD=1
 ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false
 
 GRAPH_SERVICE_IMPL=neo4j

diff --git a/docker/quickstart/docker-compose-m1.quickstart.yml b/docker/quickstart/docker-compose-m1.quickstart.yml
@@ -82,8 +82,6 @@ services:
     - ES_BULK_REFRESH_POLICY=WAIT_UNTIL
     - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
     - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
-    - ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1
-    - ES_BULK_FLUSH_PERIOD=1
     - NEO4J_HOST=http://neo4j:7474
     - NEO4J_URI=bolt://neo4j
     - NEO4J_USERNAME=neo4j
@@ -122,8 +120,6 @@ services:
     - ELASTICSEARCH_PORT=9200
     - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
     - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
-    - ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1
-    - ES_BULK_FLUSH_PERIOD=1
     - ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false
     - GRAPH_SERVICE_IMPL=elasticsearch
     - DATAHUB_GMS_HOST=datahub-gms

diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml
@@ -64,32 +64,30 @@ services:
     depends_on:
     - mysql
     environment:
-    - UI_INGESTION_ENABLED=true
-    - JAVA_OPTS=-Xms1g -Xmx1g
-    - MCE_CONSUMER_ENABLED=true
+    - EBEAN_DATASOURCE_HOST=mysql:3306
+    - EBEAN_DATASOURCE_PASSWORD=datahub
     - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms
-    - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
     - ES_BULK_REFRESH_POLICY=WAIT_UNTIL
-    - ELASTICSEARCH_HOST=elasticsearch
-    - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
-    - PE_CONSUMER_ENABLED=true
-    - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
-    - MAE_CONSUMER_ENABLED=true
-    - ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1
-    - ENTITY_SERVICE_ENABLE_RETENTION=true
-    - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
     - EBEAN_DATASOURCE_USERNAME=datahub
     - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
-    - ELASTICSEARCH_PORT=9200
+    - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
     - GRAPH_SERVICE_DIFF_MODE_ENABLED=true
-    - EBEAN_DATASOURCE_PASSWORD=datahub
-    - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
-    - ES_BULK_FLUSH_PERIOD=1
+    - ENTITY_SERVICE_ENABLE_RETENTION=true
+    - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
     - GRAPH_SERVICE_IMPL=elasticsearch
-    - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
-    - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
+    - JAVA_OPTS=-Xms1g -Xmx1g
+    - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
     - KAFKA_BOOTSTRAP_SERVER=broker:29092
-    - EBEAN_DATASOURCE_HOST=mysql:3306
+    - MAE_CONSUMER_ENABLED=true
+    - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
+    - UI_INGESTION_ENABLED=true
+    - MCE_CONSUMER_ENABLED=true
+    - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
+    - ELASTICSEARCH_HOST=elasticsearch
+    - ELASTICSEARCH_PORT=9200
+    - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
+    - PE_CONSUMER_ENABLED=true
+    - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
     hostname: datahub-gms
     image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head}
     ports:
@@ -113,8 +111,6 @@ services:
     - ELASTICSEARCH_PORT=9200
     - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
     - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
-    - ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1
-    - ES_BULK_FLUSH_PERIOD=1
     - ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false
     - GRAPH_SERVICE_IMPL=elasticsearch
     - DATAHUB_GMS_HOST=datahub-gms

diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml
@@ -64,32 +64,30 @@ services:
     depends_on:
     - mysql
     environment:
+    - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
     - KAFKA_BOOTSTRAP_SERVER=broker:29092
-    - EBEAN_DATASOURCE_PASSWORD=datahub
+    - ELASTICSEARCH_PORT=9200
+    - GRAPH_SERVICE_DIFF_MODE_ENABLED=true
+    - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
+    - EBEAN_DATASOURCE_USERNAME=datahub
+    - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
+    - EBEAN_DATASOURCE_HOST=mysql:3306
+    - ENTITY_SERVICE_ENABLE_RETENTION=true
     - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
     - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
-    - ENTITY_SERVICE_ENABLE_RETENTION=true
-    - GRAPH_SERVICE_DIFF_MODE_ENABLED=true
+    - MCE_CONSUMER_ENABLED=true
     - JAVA_OPTS=-Xms1g -Xmx1g
-    - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
+    - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
     - MAE_CONSUMER_ENABLED=true
+    - GRAPH_SERVICE_IMPL=elasticsearch
     - ELASTICSEARCH_HOST=elasticsearch
-    - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
+    - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
+    - EBEAN_DATASOURCE_PASSWORD=datahub
+    - ES_BULK_REFRESH_POLICY=WAIT_UNTIL
     - UI_INGESTION_ENABLED=true
-    - GRAPH_SERVICE_IMPL=elasticsearch
-    - ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1
-    - ELASTICSEARCH_PORT=9200
     - PE_CONSUMER_ENABLED=true
-    - EBEAN_DATASOURCE_USERNAME=datahub
     - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms
     - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
-    - ES_BULK_FLUSH_PERIOD=1
-    - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
-    - MCE_CONSUMER_ENABLED=true
-    - EBEAN_DATASOURCE_HOST=mysql:3306
-    - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
-    - ES_BULK_REFRESH_POLICY=WAIT_UNTIL
-    - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
     hostname: datahub-gms
     image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head}
     ports:
@@ -113,8 +111,6 @@ services:
     - ELASTICSEARCH_PORT=9200
     - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
     - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
-    - ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1
-    - ES_BULK_FLUSH_PERIOD=1
     - ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false
     - GRAPH_SERVICE_IMPL=elasticsearch
     - DATAHUB_GMS_HOST=datahub-gms

diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml
@@ -82,8 +82,6 @@ services:
     - ES_BULK_REFRESH_POLICY=WAIT_UNTIL
     - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
     - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
-    - ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1
-    - ES_BULK_FLUSH_PERIOD=1
     - NEO4J_HOST=http://neo4j:7474
     - NEO4J_URI=bolt://neo4j
     - NEO4J_USERNAME=neo4j
@@ -122,8 +120,6 @@ services:
     - ELASTICSEARCH_PORT=9200
     - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
     - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
-    - ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1
-    - ES_BULK_FLUSH_PERIOD=1
     - ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false
     - GRAPH_SERVICE_IMPL=elasticsearch
     - DATAHUB_GMS_HOST=datahub-gms

diff --git a/metadata-service/factories/src/main/resources/application.yml b/metadata-service/factories/src/main/resources/application.yml
@@ -154,7 +154,7 @@ elasticsearch:
   bulkProcessor:
     async: ${ES_BULK_ASYNC:true}
     requestsLimit: ${ES_BULK_REQUESTS_LIMIT:1000}
-    flushPeriod: ${ES_BULK_FLUSH_PERIOD:5}
+    flushPeriod: ${ES_BULK_FLUSH_PERIOD:1}
     numRetries: ${ES_BULK_NUM_RETRIES:3}
     retryInterval: ${ES_BULK_RETRY_INTERVAL:1}
     refreshPolicy: ${ES_BULK_REFRESH_POLICY:NONE}
@@ -163,7 +163,7 @@ elasticsearch:
     numShards: ${ELASTICSEARCH_NUM_SHARDS_PER_INDEX:1}
     numReplicas: ${ELASTICSEARCH_NUM_REPLICAS_PER_INDEX:1}
     numRetries: ${ELASTICSEARCH_INDEX_BUILDER_NUM_RETRIES:3}
-    refreshIntervalSeconds: ${ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS:5} # increase to 30 if expected indexing rates to be greater than 100/s
+    refreshIntervalSeconds: ${ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS:1} # increase to 30 if expected indexing rates to be greater than 100/s
     maxArrayLength: ${SEARCH_DOCUMENT_MAX_ARRAY_LENGTH:1000}
     maxObjectKeys: ${SEARCH_DOCUMENT_MAX_OBJECT_KEYS:1000}
     mainTokenizer: ${ELASTICSEARCH_MAIN_TOKENIZER:#{null}}

diff --git a/smoke-test/requests_wrapper/constants.py b/smoke-test/requests_wrapper/constants.py
@@ -1,2 +1,2 @@
 
-ELASTICSEARCH_REFRESH_INTERVAL_SECONDS = 5
+ELASTICSEARCH_REFRESH_INTERVAL_SECONDS = 3