diff --git a/docker/datahub-gms/env/docker-without-neo4j.env b/docker/datahub-gms/env/docker-without-neo4j.env index c7e261fe0b862..dcc1562ce90c8 100644 --- a/docker/datahub-gms/env/docker-without-neo4j.env +++ b/docker/datahub-gms/env/docker-without-neo4j.env @@ -10,8 +10,6 @@ ELASTICSEARCH_HOST=elasticsearch ELASTICSEARCH_PORT=9200 ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true -ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1 -ES_BULK_FLUSH_PERIOD=1 ES_BULK_REFRESH_POLICY=WAIT_UNTIL GRAPH_SERVICE_DIFF_MODE_ENABLED=true GRAPH_SERVICE_IMPL=elasticsearch diff --git a/docker/datahub-gms/env/docker.cassandra.env b/docker/datahub-gms/env/docker.cassandra.env index 37c80f4956313..5fc981d77b6f3 100644 --- a/docker/datahub-gms/env/docker.cassandra.env +++ b/docker/datahub-gms/env/docker.cassandra.env @@ -6,8 +6,6 @@ ELASTICSEARCH_PORT=9200 ES_BULK_REFRESH_POLICY=WAIT_UNTIL ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true -ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1 -ES_BULK_FLUSH_PERIOD=1 NEO4J_HOST=http://neo4j:7474 NEO4J_URI=bolt://neo4j NEO4J_USERNAME=neo4j diff --git a/docker/datahub-gms/env/docker.env b/docker/datahub-gms/env/docker.env index cccd8d636644e..34b2a82d2a210 100644 --- a/docker/datahub-gms/env/docker.env +++ b/docker/datahub-gms/env/docker.env @@ -11,8 +11,6 @@ ELASTICSEARCH_PORT=9200 ES_BULK_REFRESH_POLICY=WAIT_UNTIL ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true -ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1 -ES_BULK_FLUSH_PERIOD=1 NEO4J_HOST=http://neo4j:7474 NEO4J_URI=bolt://neo4j NEO4J_USERNAME=neo4j diff --git a/docker/datahub-gms/env/docker.mariadb.env b/docker/datahub-gms/env/docker.mariadb.env index 05478d83cf7f4..e7032b069b3fe 100644 --- a/docker/datahub-gms/env/docker.mariadb.env +++ b/docker/datahub-gms/env/docker.mariadb.env @@ -11,8 +11,6 @@ ELASTICSEARCH_PORT=9200 ES_BULK_REFRESH_POLICY=WAIT_UNTIL ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true -ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1 -ES_BULK_FLUSH_PERIOD=1 NEO4J_HOST=http://neo4j:7474 NEO4J_URI=bolt://neo4j NEO4J_USERNAME=neo4j diff --git a/docker/datahub-gms/env/docker.postgres.env b/docker/datahub-gms/env/docker.postgres.env index deabbe30d492b..d907b44b3814a 100644 --- a/docker/datahub-gms/env/docker.postgres.env +++ b/docker/datahub-gms/env/docker.postgres.env @@ -14,8 +14,6 @@ ELASTICSEARCH_PORT=9200 ES_BULK_REFRESH_POLICY=WAIT_UNTIL ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true -ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1 -ES_BULK_FLUSH_PERIOD=1 NEO4J_HOST=http://neo4j:7474 NEO4J_URI=bolt://neo4j NEO4J_USERNAME=neo4j diff --git a/docker/datahub-upgrade/env/docker-without-neo4j.env b/docker/datahub-upgrade/env/docker-without-neo4j.env index 50d248e439212..80dfcaf93544a 100644 --- a/docker/datahub-upgrade/env/docker-without-neo4j.env +++ b/docker/datahub-upgrade/env/docker-without-neo4j.env @@ -11,8 +11,6 @@ ELASTICSEARCH_HOST=elasticsearch ELASTICSEARCH_PORT=9200 ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true -ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1 -ES_BULK_FLUSH_PERIOD=1 ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false GRAPH_SERVICE_IMPL=elasticsearch diff --git a/docker/datahub-upgrade/env/docker.env b/docker/datahub-upgrade/env/docker.env index a92e47f303152..bf5186da83be7 100644 --- a/docker/datahub-upgrade/env/docker.env +++ b/docker/datahub-upgrade/env/docker.env @@ -11,8 +11,6 @@ ELASTICSEARCH_HOST=elasticsearch ELASTICSEARCH_PORT=9200 ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true -ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1 -ES_BULK_FLUSH_PERIOD=1 ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false GRAPH_SERVICE_IMPL=neo4j diff --git a/docker/quickstart/docker-compose-m1.quickstart.yml b/docker/quickstart/docker-compose-m1.quickstart.yml index 0a81c798999f8..5bd1be5a2ea6d 100644 --- a/docker/quickstart/docker-compose-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-m1.quickstart.yml @@ -82,8 +82,6 @@ services: - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1 - - ES_BULK_FLUSH_PERIOD=1 - NEO4J_HOST=http://neo4j:7474 - NEO4J_URI=bolt://neo4j - NEO4J_USERNAME=neo4j @@ -122,8 +120,6 @@ services: - ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1 - - ES_BULK_FLUSH_PERIOD=1 - ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false - GRAPH_SERVICE_IMPL=elasticsearch - DATAHUB_GMS_HOST=datahub-gms diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml index 5bf9f556ce22e..7c2e7aa7c0b0e 100644 --- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml @@ -64,32 +64,30 @@ services: depends_on: - mysql environment: - - UI_INGESTION_ENABLED=true - - JAVA_OPTS=-Xms1g -Xmx1g - - MCE_CONSUMER_ENABLED=true + - EBEAN_DATASOURCE_HOST=mysql:3306 + - EBEAN_DATASOURCE_PASSWORD=datahub - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - PE_CONSUMER_ENABLED=true - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - MAE_CONSUMER_ENABLED=true - - ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1 - - ENTITY_SERVICE_ENABLE_RETENTION=true - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - EBEAN_DATASOURCE_USERNAME=datahub - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - - ELASTICSEARCH_PORT=9200 + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - EBEAN_DATASOURCE_PASSWORD=datahub - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - - ES_BULK_FLUSH_PERIOD=1 + - ENTITY_SERVICE_ENABLE_RETENTION=true + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - GRAPH_SERVICE_IMPL=elasticsearch - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - JAVA_OPTS=-Xms1g -Xmx1g + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - EBEAN_DATASOURCE_HOST=mysql:3306 + - MAE_CONSUMER_ENABLED=true + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - UI_INGESTION_ENABLED=true + - MCE_CONSUMER_ENABLED=true + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - PE_CONSUMER_ENABLED=true + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver hostname: datahub-gms image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: @@ -113,8 +111,6 @@ services: - ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1 - - ES_BULK_FLUSH_PERIOD=1 - ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false - GRAPH_SERVICE_IMPL=elasticsearch - DATAHUB_GMS_HOST=datahub-gms diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index 2fc50dbf2c727..ece7f1e0a8c34 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -64,32 +64,30 @@ services: depends_on: - mysql environment: + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - EBEAN_DATASOURCE_PASSWORD=datahub + - ELASTICSEARCH_PORT=9200 + - GRAPH_SERVICE_DIFF_MODE_ENABLED=true + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - EBEAN_DATASOURCE_USERNAME=datahub + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - EBEAN_DATASOURCE_HOST=mysql:3306 + - ENTITY_SERVICE_ENABLE_RETENTION=true - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - ENTITY_SERVICE_ENABLE_RETENTION=true - - GRAPH_SERVICE_DIFF_MODE_ENABLED=true + - MCE_CONSUMER_ENABLED=true - JAVA_OPTS=-Xms1g -Xmx1g - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - MAE_CONSUMER_ENABLED=true + - GRAPH_SERVICE_IMPL=elasticsearch - ELASTICSEARCH_HOST=elasticsearch - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - EBEAN_DATASOURCE_PASSWORD=datahub + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - UI_INGESTION_ENABLED=true - - GRAPH_SERVICE_IMPL=elasticsearch - - ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1 - - ELASTICSEARCH_PORT=9200 - PE_CONSUMER_ENABLED=true - - EBEAN_DATASOURCE_USERNAME=datahub - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - - ES_BULK_FLUSH_PERIOD=1 - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - - MCE_CONSUMER_ENABLED=true - - EBEAN_DATASOURCE_HOST=mysql:3306 - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true hostname: datahub-gms image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: @@ -113,8 +111,6 @@ services: - ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1 - - ES_BULK_FLUSH_PERIOD=1 - ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false - GRAPH_SERVICE_IMPL=elasticsearch - DATAHUB_GMS_HOST=datahub-gms diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index 78405a2a9bfec..78607447fc100 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -82,8 +82,6 @@ services: - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1 - - ES_BULK_FLUSH_PERIOD=1 - NEO4J_HOST=http://neo4j:7474 - NEO4J_URI=bolt://neo4j - NEO4J_USERNAME=neo4j @@ -122,8 +120,6 @@ services: - ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS=1 - - ES_BULK_FLUSH_PERIOD=1 - ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false - GRAPH_SERVICE_IMPL=elasticsearch - DATAHUB_GMS_HOST=datahub-gms diff --git a/metadata-service/factories/src/main/resources/application.yml b/metadata-service/factories/src/main/resources/application.yml index c7c192fc6b289..304b11c6fb882 100644 --- a/metadata-service/factories/src/main/resources/application.yml +++ b/metadata-service/factories/src/main/resources/application.yml @@ -154,7 +154,7 @@ elasticsearch: bulkProcessor: async: ${ES_BULK_ASYNC:true} requestsLimit: ${ES_BULK_REQUESTS_LIMIT:1000} - flushPeriod: ${ES_BULK_FLUSH_PERIOD:5} + flushPeriod: ${ES_BULK_FLUSH_PERIOD:1} numRetries: ${ES_BULK_NUM_RETRIES:3} retryInterval: ${ES_BULK_RETRY_INTERVAL:1} refreshPolicy: ${ES_BULK_REFRESH_POLICY:NONE} @@ -163,7 +163,7 @@ elasticsearch: numShards: ${ELASTICSEARCH_NUM_SHARDS_PER_INDEX:1} numReplicas: ${ELASTICSEARCH_NUM_REPLICAS_PER_INDEX:1} numRetries: ${ELASTICSEARCH_INDEX_BUILDER_NUM_RETRIES:3} - refreshIntervalSeconds: ${ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS:5} # increase to 30 if expected indexing rates to be greater than 100/s + refreshIntervalSeconds: ${ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS:1} # increase to 30 if expected indexing rates to be greater than 100/s maxArrayLength: ${SEARCH_DOCUMENT_MAX_ARRAY_LENGTH:1000} maxObjectKeys: ${SEARCH_DOCUMENT_MAX_OBJECT_KEYS:1000} mainTokenizer: ${ELASTICSEARCH_MAIN_TOKENIZER:#{null}} diff --git a/smoke-test/requests_wrapper/constants.py b/smoke-test/requests_wrapper/constants.py index c287ec2181406..b13da5786f280 100644 --- a/smoke-test/requests_wrapper/constants.py +++ b/smoke-test/requests_wrapper/constants.py @@ -1,2 +1,2 @@ -ELASTICSEARCH_REFRESH_INTERVAL_SECONDS = 5 +ELASTICSEARCH_REFRESH_INTERVAL_SECONDS = 3