From f29bf5b6e7942cb3d11bf3063d4f9670d1af3bfa Mon Sep 17 00:00:00 2001 From: Andrew Prudhomme Date: Fri, 1 Nov 2024 09:41:29 -0700 Subject: [PATCH 1/2] Clean up references to lucene server --- README.md | 14 ++--- build.gradle | 22 +++++++- docker-compose-config/entrypoint_replica.sh | 6 +-- ...ary.yaml => nrtsearch_primary_config.yaml} | 2 +- ...ica.yaml => nrtsearch_replica_config.yaml} | 0 docker-compose.yaml | 2 +- docs/analysis.rst | 4 +- docs/docker_compose.rst | 38 ++++++------- docs/index_live_settings.rst | 2 +- docs/index_settings.rst | 2 +- docs/introduction.rst | 2 +- docs/server_configuration.rst | 30 +++++------ docs/starter.rst | 12 ++--- .../plugins/example/ExamplePluginTest.java | 2 +- .../server/concurrent/ExecutorFactory.java | 2 +- .../config/ThreadPoolConfiguration.java | 13 +++-- .../grpc/GrpcServerExecutorSupplier.java | 3 +- .../server/grpc/NrtsearchClient.java | 4 +- .../server/grpc/NrtsearchServer.java | 54 +++++++++---------- .../server/modules/NrtsearchModule.java | 11 ++-- .../nrtsearch/server/modules/S3Module.java | 4 +- .../yelp/nrtsearch/server/plugins/Plugin.java | 2 +- .../server/plugins/PluginsService.java | 2 +- .../nrtsearch/server/remote/s3/S3Util.java | 16 +++--- .../server/state/BackendGlobalState.java | 8 +-- .../nrtsearch/server/state/GlobalState.java | 27 +++++----- .../state/backend/RemoteStateBackend.java | 6 +-- .../tools/cli/NrtsearchClientCommand.java | 2 +- ...ion.yaml => nrtsearch_default_config.yaml} | 2 +- ...NodeNameResolverAndLoadBalancingTests.java | 12 ++--- .../yelp/nrtsearch/server/ServerTestCase.java | 18 +++---- .../server/analysis/AnalyzerCreatorTest.java | 2 +- .../SynonymV2GraphFilterFactoryITest.java | 2 +- .../SynonymV2GraphFilterFactoryTest.java | 2 +- .../concurrent/ExecutorFactoryTest.java | 19 ++++--- .../server/config/FileCopyConfigTest.java | 4 +- .../server/config/IndexPreloadConfigTest.java | 2 +- .../server/config/IndexStartConfigTest.java | 2 +- .../server/config/NrtsearchConfigTest.java | 31 +++++------ .../server/config/QueryCacheConfigTest.java | 4 +- .../server/config/ScriptCacheConfigTest.java | 7 ++- .../server/config/StateConfigTest.java | 2 +- .../config/ThreadPoolConfigurationTest.java | 24 ++++----- .../server/field/FieldDefCreatorTest.java | 2 +- .../nrtsearch/server/grpc/CopyFileTest.java | 40 -------------- .../server/grpc/CustomFieldTypeTest.java | 12 ++--- .../nrtsearch/server/grpc/GrpcServer.java | 18 +++---- .../server/grpc/MergeBehaviorTests.java | 12 ++--- .../grpc/NrtsearchServerIdFieldTest.java | 12 ++--- .../server/grpc/NrtsearchServerTest.java | 31 ++++++----- .../yelp/nrtsearch/server/grpc/QueryTest.java | 12 ++--- .../server/grpc/ReplicationServerTest.java | 18 +++---- .../server/index/BackendStateManagerTest.java | 2 +- .../server/index/DirectoryFactoryTest.java | 10 ++-- .../server/index/FieldUpdateUtilsTest.java | 2 +- .../server/index/ImmutableIndexStateTest.java | 6 +-- .../server/logging/HitsLoggerCreatorTest.java | 2 +- .../server/plugins/PluginsServiceTest.java | 2 +- .../server/query/QueryNodeMapperTest.java | 2 +- .../server/rescore/RescorerCreatorTest.java | 2 +- .../server/script/ScoreScriptTest.java | 12 ++--- .../server/script/ScriptServiceTest.java | 2 +- .../server/search/FetchTaskCreatorTest.java | 2 +- .../collectors/CollectorCreatorTest.java | 2 +- .../similarity/SimilarityCreatorTest.java | 2 +- .../server/state/BackendGlobalStateTest.java | 8 +-- ...=> NrtsearchTestConfigurationFactory.java} | 2 +- .../nrtsearch/test_utils/NrtsearchTest.java | 6 +-- ...neServer.java => TestNrtsearchServer.java} | 25 +++++---- .../nrtsearch/tools/cli/ReadyCommandTest.java | 12 ++--- .../yelp_reviews/YelpReviewsTest.java | 42 +++++++-------- .../utils/DocumentGeneratorAndIndexer.java | 8 +-- .../yelp_reviews/utils/IndexerTask.java | 4 +- .../utils/ParallelDocumentIndexer.java | 11 ++-- src/test/resources/config.yaml | 2 +- ...nfiguration.yaml => nrtsearch_config.yaml} | 2 +- ...nfiguration.yaml => nrtsearch_config.yaml} | 2 +- 77 files changed, 355 insertions(+), 399 deletions(-) rename docker-compose-config/{lucene_server_configuration_primary.yaml => nrtsearch_primary_config.yaml} (90%) rename docker-compose-config/{lucene_server_configuration_replica.yaml => nrtsearch_replica_config.yaml} (100%) rename src/main/resources/{lucene_server_default_configuration.yaml => nrtsearch_default_config.yaml} (67%) rename src/test/java/com/yelp/nrtsearch/server/utils/{LuceneServerTestConfigurationFactory.java => NrtsearchTestConfigurationFactory.java} (98%) rename src/test/java/com/yelp/nrtsearch/test_utils/{TestLuceneServer.java => TestNrtsearchServer.java} (70%) rename src/test/resources/yelp_reviews/primary/{lucene_server_configuration.yaml => nrtsearch_config.yaml} (87%) rename src/test/resources/yelp_reviews/replica/{lucene_server_configuration.yaml => nrtsearch_config.yaml} (86%) diff --git a/README.md b/README.md index 23e520f9b..55d3ecec9 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Note: This code has been tested on *Java21* # Run gRPC Server ``` -./build/install/nrtsearch/bin/lucene-server +./build/install/nrtsearch/bin/nrtsearch_server ``` # Build gRPC Gateway @@ -72,7 +72,7 @@ Note: This code has been tested on *Java21* ## Create Index ``` -./build/install/nrtsearch/bin/lucene-client createIndex --indexName testIdx +./build/install/nrtsearch/bin/nrtsearch_client createIndex --indexName testIdx ``` ``` @@ -82,7 +82,7 @@ curl -XPOST localhost:/v1/create_index -d '{"indexName": "testIdx"}' ## Update Settings ``` -./build/install/nrtsearch/bin/lucene-client settings -f settings.json +./build/install/nrtsearch/bin/nrtsearch_client settings -f settings.json cat settings.json { "indexName": "testIdx", "directory": "MMapDirectory", @@ -96,7 +96,7 @@ cat settings.json ## Start Index ``` -./build/install/nrtsearch/bin/lucene-client startIndex -f startIndex.json +./build/install/nrtsearch/bin/nrtsearch_client startIndex -f startIndex.json cat startIndex.json { "indexName" : "testIdx" @@ -106,7 +106,7 @@ cat startIndex.json ## RegisterFields ``` -./build/install/nrtsearch/bin/lucene-client registerFields -f registerFields.json +./build/install/nrtsearch/bin/nrtsearch_client registerFields -f registerFields.json cat registerFields.json { "indexName": "testIdx", "field": @@ -121,7 +121,7 @@ cat registerFields.json ## Add Documents ``` -./build/install/nrtsearch/bin/lucene-client addDocuments -i testIdx -f docs.csv -t csv +./build/install/nrtsearch/bin/nrtsearch_client addDocuments -i testIdx -f docs.csv -t csv cat docs.csv doc_id,vendor_name,license_no 0,first vendor,100;200 @@ -131,7 +131,7 @@ doc_id,vendor_name,license_no ## Search ``` -./build/install/nrtsearch/bin/lucene-client search -f search.json +./build/install/nrtsearch/bin/nrtsearch_client search -f search.json cat search.json { "indexName": "testIdx", diff --git a/build.gradle b/build.gradle index fb8f96925..58eecc754 100644 --- a/build.gradle +++ b/build.gradle @@ -140,6 +140,22 @@ task luceneServerClient(type: CreateStartScripts) { classpath = startScripts.classpath } +task nrtsearchServer(type: CreateStartScripts) { + mainClass = 'com.yelp.nrtsearch.server.grpc.NrtsearchServer' + applicationName = 'nrtsearch_server' + outputDir = new File(project.buildDir, 'tmp-app') + classpath = startScripts.classpath + // Add additional dependencies, e.g. custom loggers + classpath += files('$APP_HOME/additional_libs') +} + +task nrtsearchServerClient(type: CreateStartScripts) { + mainClass = 'com.yelp.nrtsearch.tools.cli.NrtsearchClientCommand' + applicationName = 'nrtsearch_client' + outputDir = new File(project.buildDir, 'tmp-app') + classpath = startScripts.classpath +} + task nrtUtils(type: CreateStartScripts) { mainClass = 'com.yelp.nrtsearch.tools.nrt_utils.NrtUtilsCommand' applicationName = 'nrt_utils' @@ -150,6 +166,8 @@ task nrtUtils(type: CreateStartScripts) { applicationDistribution.into('bin') { from(luceneServer) from(luceneServerClient) + from(nrtsearchServer) + from(nrtsearchServerClient) from(nrtUtils) fileMode = 0755 } @@ -165,7 +183,7 @@ task buildGrpcGateway(dependsOn: installDist, type: Exec) { //e.g. default is to exclude perfTests: ./gradlew test test { finalizedBy 'spotlessJavaCheck' - // Used by LuceneServerConfigurationTest + // Used by NrtsearchConfigTest environment(Map.of('CUSTOM_HOST', 'my_custom_host', 'VAR1', 'v1', 'VAR2', 'v2', 'VAR3', 'v3')) if (project.hasProperty('longRunningTestsOnly')) { include '**/IncrementalDataCleanupCommandTest.class' @@ -233,7 +251,7 @@ publishing { artifact tasks.testsJar pom { name = 'nrtSearch Server' - description = 'GRPC lucene server using near-real-time replication' + description = 'GRPC server using near-real-time replication' url = 'https://github.com/Yelp/nrtsearch' licenses { license { diff --git a/docker-compose-config/entrypoint_replica.sh b/docker-compose-config/entrypoint_replica.sh index e0feb7dc3..ed2555dd9 100755 --- a/docker-compose-config/entrypoint_replica.sh +++ b/docker-compose-config/entrypoint_replica.sh @@ -5,10 +5,10 @@ HOSTNAME=$(hostname -i) echo "hostname: "$HOSTNAME echo "replacing nodeName" -sed -i "s/node-name/$HOSTNAME/g" docker-compose-config/lucene_server_configuration_replica.yaml +sed -i "s/node-name/$HOSTNAME/g" docker-compose-config/nrtsearch_replica_config.yaml echo "replacing nostname" -sed -i "s/host-name-replica/$HOSTNAME/g" docker-compose-config/lucene_server_configuration_replica.yaml +sed -i "s/host-name-replica/$HOSTNAME/g" docker-compose-config/nrtsearch_replica_config.yaml echo "starting service" -/user/app/build/install/nrtsearch/bin/lucene-server /user/app/docker-compose-config/lucene_server_configuration_replica.yaml \ No newline at end of file +/user/app/build/install/nrtsearch/bin/nrtsearch_server /user/app/docker-compose-config/nrtsearch_replica_config.yaml \ No newline at end of file diff --git a/docker-compose-config/lucene_server_configuration_primary.yaml b/docker-compose-config/nrtsearch_primary_config.yaml similarity index 90% rename from docker-compose-config/lucene_server_configuration_primary.yaml rename to docker-compose-config/nrtsearch_primary_config.yaml index d2f121d7e..3316ce743 100644 --- a/docker-compose-config/lucene_server_configuration_primary.yaml +++ b/docker-compose-config/nrtsearch_primary_config.yaml @@ -1,4 +1,4 @@ -nodeName: "lucene_server_primary" +nodeName: "nrtsearch_primary" hostName: "primary-node" port: "8000" replicationPort: "8001" diff --git a/docker-compose-config/lucene_server_configuration_replica.yaml b/docker-compose-config/nrtsearch_replica_config.yaml similarity index 100% rename from docker-compose-config/lucene_server_configuration_replica.yaml rename to docker-compose-config/nrtsearch_replica_config.yaml diff --git a/docker-compose.yaml b/docker-compose.yaml index 5f33a8803..a063cf4c5 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -6,7 +6,7 @@ services: build: ./ container_name: primary-node command: > - bash -c "/user/app/build/install/nrtsearch/bin/lucene-server /user/app/docker-compose-config/lucene_server_configuration_primary.yaml" + bash -c "/user/app/build/install/nrtsearch/bin/nrtsearch_server /user/app/docker-compose-config/nrtsearch_primary_config.yaml" hostname: primary-node replica-node-1: diff --git a/docs/analysis.rst b/docs/analysis.rst index ffe2e44c6..609f8e1c9 100644 --- a/docs/analysis.rst +++ b/docs/analysis.rst @@ -158,7 +158,7 @@ Available character filters: * patternreplace - * mappingV2 - Similar to the ``mapping`` filter, except rules are specified directly in the parameters. See `MappingV2CharFilterFactory `_. + * mappingV2 - Similar to the ``mapping`` filter, except rules are specified directly in the parameters. See `MappingV2CharFilterFactory `_. Available tokenizers: @@ -388,7 +388,7 @@ Available token filters: * synonym - * synonymV2 - Similar to the ``synonymGraph`` filter, except rules are specified directly in the parameters. See `SynonymV2GraphFilterFactory `_. + * synonymV2 - Similar to the ``synonymGraph`` filter, except rules are specified directly in the parameters. See `SynonymV2GraphFilterFactory `_. * synonymGraph diff --git a/docs/docker_compose.rst b/docs/docker_compose.rst index 7fbf95bdd..4259118aa 100644 --- a/docs/docker_compose.rst +++ b/docs/docker_compose.rst @@ -44,7 +44,7 @@ Note: The S3 bucket must exist before hand, and is named in the server configura .. code-block:: - shell% cat docker-compose-config/lucene_server_configuration_primary.yaml # for the replica config as well + shell% cat docker-compose-config/nrtsearch_primary_config.yaml # for the replica config as well ... bucketName: "nrtsearch-bucket" ... @@ -74,10 +74,10 @@ Create the index and settings, register the fields, and start the index: shell% PRIMARY_CONTAINER_ID=$(docker ps | grep nrtsearch_primary-node | awk '{print $1}') shell% docker exec -it $PRIMARY_CONTAINER_ID sh - # ./build/install/nrtsearch/bin/lucene-client -h primary-node -p 8000 createIndex --indexName testIdx - # ./build/install/nrtsearch/bin/lucene-client -h primary-node -p 8000 settings -f docker-compose-config/settings_primary.json - # ./build/install/nrtsearch/bin/lucene-client -h primary-node -p 8000 registerFields -f docker-compose-config/registerFields.json - # ./build/install/nrtsearch/bin/lucene-client -h primary-node -p 8000 startIndex -f docker-compose-config/startIndex_primary.json + # ./build/install/nrtsearch/bin/nrtsearch_client -h primary-node -p 8000 createIndex --indexName testIdx + # ./build/install/nrtsearch/bin/nrtsearch_client -h primary-node -p 8000 settings -f docker-compose-config/settings_primary.json + # ./build/install/nrtsearch/bin/nrtsearch_client -h primary-node -p 8000 registerFields -f docker-compose-config/registerFields.json + # ./build/install/nrtsearch/bin/nrtsearch_client -h primary-node -p 8000 startIndex -f docker-compose-config/startIndex_primary.json 3. Replica: Start Index ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -88,10 +88,10 @@ Next go into any one of the replicas (i.e. nrtsearch_replica-node-1 here), and r shell% REPLICA_1_CONTAINER_ID=$(docker ps | grep nrtsearch_replica-node-1_1 | awk '{print $1}') shell% docker exec -it $REPLICA_1_CONTAINER_ID sh - # ./build/install/nrtsearch/bin/lucene-client -h replica-node-1 -p 8002 createIndex --indexName testIdx - # ./build/install/nrtsearch/bin/lucene-client -h replica-node-1 -p 8002 settings -f docker-compose-config/settings_replica.json - # ./build/install/nrtsearch/bin/lucene-client -h replica-node-1 -p 8002 registerFields -f docker-compose-config/registerFields.json - # ./build/install/nrtsearch/bin/lucene-client -h replica-node-1 -p 8002 startIndex -f docker-compose-config/startIndex_replica.json + # ./build/install/nrtsearch/bin/nrtsearch_client -h replica-node-1 -p 8002 createIndex --indexName testIdx + # ./build/install/nrtsearch/bin/nrtsearch_client -h replica-node-1 -p 8002 settings -f docker-compose-config/settings_replica.json + # ./build/install/nrtsearch/bin/nrtsearch_client -h replica-node-1 -p 8002 registerFields -f docker-compose-config/registerFields.json + # ./build/install/nrtsearch/bin/nrtsearch_client -h replica-node-1 -p 8002 startIndex -f docker-compose-config/startIndex_replica.json 4. Replication ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -101,7 +101,7 @@ Search will work on replicas soon after documents are added on the primary. Add .. code-block:: shell% docker exec -it $PRIMARY_CONTAINER_ID sh - # ./build/install/nrtsearch/bin/lucene-client -h primary-node -p 8000 addDocuments -i testIdx -f docker-compose-config/docs.csv -t csv + # ./build/install/nrtsearch/bin/nrtsearch_client -h primary-node -p 8000 addDocuments -i testIdx -f docker-compose-config/docs.csv -t csv 5. Replica: Search Should Work ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -111,7 +111,7 @@ The search should now work on any of the replicas where the the index was starte .. code-block:: shell% docker exec -it $REPLICA_1_CONTAINER_ID sh - # ./build/install/nrtsearch/bin/lucene-client -h replica-node-1 -p 8002 search -f docker-compose-config/search.json + # ./build/install/nrtsearch/bin/nrtsearch_client -h replica-node-1 -p 8002 search -f docker-compose-config/search.json ... fields { key: "license_no" @@ -134,7 +134,7 @@ In order to backup the index, one can use the 'backupIndex' command with these p .. code-block:: shell% docker exec -it $PRIMARY_CONTAINER_ID sh - # ./build/install/nrtsearch/bin/lucene-client -h primary-node -p 8000 backupIndex --indexName testIdx --serviceName nrtsearch-service-test --resourceName testIdx + # ./build/install/nrtsearch/bin/nrtsearch_client -h primary-node -p 8000 backupIndex --indexName testIdx --serviceName nrtsearch-service-test --resourceName testIdx Now the S3 bucket 'nrtsearch-bucket' should contain the service 'nrtsearch-service-test' data : @@ -152,14 +152,14 @@ To demonstrate how one can start nrtSearch and restore the index data from S3, o .. code-block:: - # update the 2 lucene service configs docker-compose-config/lucene_server_configuration_{primary,replica}.yaml to have this line: - shell% cat docker-compose-config/lucene_server_configuration_primary.yaml + # update the 2 lucene service configs docker-compose-config/nrtsearch_{primary,replica}_config.yaml to have this line: + shell% cat docker-compose-config/nrtsearch_primary_config.yaml ... # previous lines still there, change this line: restoreState: True ... ... - shell% cat docker-compose-config/lucene_server_configuration_replica.yaml + shell% cat docker-compose-config/nrtsearch_replica_config.yaml ... # previous lines still there, change this line: restoreState: True @@ -195,8 +195,8 @@ If one then restarts the containers and restarts the index (do not need to regis shell% docker-compose -f docker-compose.yaml up shell% PRIMARY_CONTAINER_ID=$(docker ps | grep nrtsearch_primary-node | awk '{print $1}') shell% docker exec -it $PRIMARY_CONTAINER_ID sh - # ./build/install/nrtsearch/bin/lucene-client -h primary-node -p 8000 startIndex -f docker-compose-config/startIndex_primary.json - # ./build/install/nrtsearch/bin/lucene-client -h primary-node -p 8000 search -f docker-compose-config/search.json + # ./build/install/nrtsearch/bin/nrtsearch_client -h primary-node -p 8000 startIndex -f docker-compose-config/startIndex_primary.json + # ./build/install/nrtsearch/bin/nrtsearch_client -h primary-node -p 8000 search -f docker-compose-config/search.json ... fields { key: "license_no" @@ -218,9 +218,9 @@ To view the logs in the containers use docker-compose logs: .. code-block:: shell% docker-compose logs - replica-node-1_1 | [INFO ] 2021-12-13 18:58:26.527 [main] LuceneServer - Server started, listening on 8003 for replication messages + replica-node-1_1 | [INFO ] 2021-12-13 18:58:26.527 [main] NrtsearchServer - Server started, listening on 8003 for replication messages replica-node-1_2 | hostname: 172.24.0.2 - primary-node | [INFO ] 2021-12-13 18:58:28.530 [main] LuceneServer - Server started, listening on 8001 for replication messages + primary-node | [INFO ] 2021-12-13 18:58:28.530 [main] NrtsearchServer - Server started, listening on 8001 for replication messages Stop """"""""""""""""""""""""""" diff --git a/docs/index_live_settings.rst b/docs/index_live_settings.rst index d542e40f2..9d8c32fab 100644 --- a/docs/index_live_settings.rst +++ b/docs/index_live_settings.rst @@ -10,7 +10,7 @@ Changes to these settings can be made any time after the index is created. There * Directly use the `liveSetting/liveSettingV2 `_ gRPC server endpoints * Use the '/v1/live_settings' or '/v2/live_settings' endpoints with the gRPC gateway -* Use the lucene-client `liveSettings/liveSettingsV2 `_ commands +* Use the nrtsearch_client `liveSettings/liveSettingsV2 `_ commands Properties ----------------------------- diff --git a/docs/index_settings.rst b/docs/index_settings.rst index 667f89b55..77da0781c 100644 --- a/docs/index_settings.rst +++ b/docs/index_settings.rst @@ -10,7 +10,7 @@ Changes to these settings can be made after the index is created, but before it * Directly use the `setting/settingV2 `_ gRPC server endpoints * Use the '/v1/settings' or '/v2/settings' endpoints with the gRPC gateway -* Use the lucene-client `settings/settingsV2 `_ commands +* Use the nrtsearch_client `settings/settingsV2 `_ commands Properties ----------------------------- diff --git a/docs/introduction.rst b/docs/introduction.rst index 864c41efb..cf9fc007e 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -65,7 +65,7 @@ The server can be run locally like this: .. code-block:: - ./build/install/nrtsearch/bin/lucene-server + ./build/install/nrtsearch/bin/nrtsearch_server Run REST Server --------------------------- diff --git a/docs/server_configuration.rst b/docs/server_configuration.rst index 806be17ec..bef75c0e2 100644 --- a/docs/server_configuration.rst +++ b/docs/server_configuration.rst @@ -6,13 +6,13 @@ Usage .. code-block:: - ./build/install/nrtsearch/bin/lucene-server server_configuration.yaml + ./build/install/nrtsearch/bin/nrtsearch_server server_configuration.yaml Example server configuration .. code-block:: yaml - nodeName: "lucene_server_primary" + nodeName: "nrtsearch_primary" hostName: "primary-node" port: "8000" replicationPort: "8001" @@ -28,7 +28,7 @@ Example server configuration serviceName: "nrtsearch-service-test" -.. list-table:: `LuceneServerConfiguration `_ +.. list-table:: `NrtsearchConfig `_ :widths: 25 10 50 25 :header-rows: 1 @@ -39,17 +39,17 @@ Example server configuration * - nodeName - str - - Name of this NrtSearch instance. Currently used for emitting metrics labels. + - Name of this NrtSearch instance. Currently used for registering replicas with primary. This property supports `environment variable substitution `_. - main * - hostName - str - - Hostname of this NrtSearch instance. Replicas use this property when registering with the primary. This property supports `environment variable substitution `_. + - Hostname of this NrtSearch instance. Replicas use this property when registering with the primary. This property supports `environment variable substitution `_. - localhost * - port - str - - Port for LuceneServer gRPC requests + - Port for NrtsearchServer gRPC requests - 50051 * - replicationPort @@ -60,12 +60,12 @@ Example server configuration * - stateDir - str - Path of global state directory - - ` `_/default_state + - ` `_/default_state * - indexDir - str - Path of directory containing index state and segments - - ` `_/default_index + - ` `_/default_index * - bucketName - str @@ -146,20 +146,20 @@ Example server configuration - Name prefix for threads created by indexing threadpool executor - LuceneIndexingExecutor - * - luceneserver.maxThreads + * - server.maxThreads - int - - Size of LuceneServer threadpool executor + - Size of NrtsearchServer threadpool executor - numCPUs + 1 - * - luceneserver.maxBufferedItems + * - server.maxBufferedItems - int - - Max tasks that can be queued by LuceneServer threadpool executor + - Max tasks that can be queued by NrtsearchServer threadpool executor - max(200, 2 * (numCPUs + 1)) - * - luceneserver.threadNamePrefix + * - server.threadNamePrefix - string - - Name prefix for threads created by LuceneServer threadpool executor - - GrpcLuceneServerExecutor + - Name prefix for threads created by NrtsearchServer threadpool executor + - GrpcServerExecutor * - replicationserver.maxThreads - int diff --git a/docs/starter.rst b/docs/starter.rst index 1b547f6a6..5f27c80b7 100644 --- a/docs/starter.rst +++ b/docs/starter.rst @@ -17,7 +17,7 @@ Create Index .. code-block:: - # ./build/install/nrtsearch/bin/lucene-client createIndex --indexName testIdx + # ./build/install/nrtsearch/bin/nrtsearch_client createIndex --indexName testIdx Configure the index @@ -26,7 +26,7 @@ Configure the index .. code-block:: - # ./build/install/nrtsearch/bin/lucene-client settings -f settings.json + # ./build/install/nrtsearch/bin/nrtsearch_client settings -f settings.json # cat settings.json { "indexName": "testIdx", "directory": "MMapDirectory", @@ -40,7 +40,7 @@ Register Fields ^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: - # ./build/install/nrtsearch/bin/lucene-client registerFields -f registerFields.json + # ./build/install/nrtsearch/bin/nrtsearch_client registerFields -f registerFields.json # cat registerFields.json { "indexName": "testIdx", "field": @@ -56,7 +56,7 @@ Start Index ^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: - # ./build/install/nrtsearch/bin/lucene-client startIndex -f startIndex.json + # ./build/install/nrtsearch/bin/nrtsearch_client startIndex -f startIndex.json # cat startIndex.json { "indexName" : "testIdx" @@ -66,7 +66,7 @@ Add documents to your NRTSearch index --------------------------- .. code-block:: - # ./build/install/nrtsearch/bin/lucene-client addDocuments -i testIdx -f docs.csv -t csv + # ./build/install/nrtsearch/bin/nrtsearch_client addDocuments -i testIdx -f docs.csv -t csv # cat docs.csv doc_id,vendor_name,license_no 0,first vendor,100;200 @@ -76,7 +76,7 @@ Query your NRTSearch server --------------------------- .. code-block:: - # ./build/install/nrtsearch/bin/lucene-client search -f search.json + # ./build/install/nrtsearch/bin/nrtsearch_client search -f search.json # cat search.json { "indexName": "testIdx", diff --git a/example-plugin/src/test/java/com/yelp/nrtsearch/plugins/example/ExamplePluginTest.java b/example-plugin/src/test/java/com/yelp/nrtsearch/plugins/example/ExamplePluginTest.java index 1b9eb747b..991dd85ff 100644 --- a/example-plugin/src/test/java/com/yelp/nrtsearch/plugins/example/ExamplePluginTest.java +++ b/example-plugin/src/test/java/com/yelp/nrtsearch/plugins/example/ExamplePluginTest.java @@ -122,7 +122,7 @@ public void testAnalysisForSearch() { } private static NrtsearchConfig getConfig() { - String config = "nodeName: \"lucene_server_foo\""; + String config = "nodeName: \"server_foo\""; return new NrtsearchConfig(new ByteArrayInputStream(config.getBytes())); } } diff --git a/src/main/java/com/yelp/nrtsearch/server/concurrent/ExecutorFactory.java b/src/main/java/com/yelp/nrtsearch/server/concurrent/ExecutorFactory.java index 5c5707fd4..d8c302636 100644 --- a/src/main/java/com/yelp/nrtsearch/server/concurrent/ExecutorFactory.java +++ b/src/main/java/com/yelp/nrtsearch/server/concurrent/ExecutorFactory.java @@ -36,7 +36,7 @@ public class ExecutorFactory { public enum ExecutorType { SEARCH, INDEX, - LUCENESERVER, + SERVER, REPLICATIONSERVER, FETCH, GRPC, diff --git a/src/main/java/com/yelp/nrtsearch/server/config/ThreadPoolConfiguration.java b/src/main/java/com/yelp/nrtsearch/server/config/ThreadPoolConfiguration.java index 912b90764..8c2531513 100644 --- a/src/main/java/com/yelp/nrtsearch/server/config/ThreadPoolConfiguration.java +++ b/src/main/java/com/yelp/nrtsearch/server/config/ThreadPoolConfiguration.java @@ -34,9 +34,8 @@ public class ThreadPoolConfiguration { public static final int DEFAULT_INDEXING_BUFFERED_ITEMS = Math.max(200, 2 * DEFAULT_INDEXING_THREADS); - public static final int DEFAULT_GRPC_LUCENESERVER_THREADS = DEFAULT_INDEXING_THREADS; - public static final int DEFAULT_GRPC_LUCENESERVER_BUFFERED_ITEMS = - DEFAULT_INDEXING_BUFFERED_ITEMS; + public static final int DEFAULT_GRPC_SERVER_THREADS = DEFAULT_INDEXING_THREADS; + public static final int DEFAULT_GRPC_SERVER_BUFFERED_ITEMS = DEFAULT_INDEXING_BUFFERED_ITEMS; public static final int DEFAULT_GRPC_REPLICATIONSERVER_THREADS = DEFAULT_INDEXING_THREADS; public static final int DEFAULT_GRPC_REPLICATIONSERVER_BUFFERED_ITEMS = @@ -75,11 +74,11 @@ public record ThreadPoolSettings(int maxThreads, int maxBufferedItems, String th DEFAULT_INDEXING_THREADS, DEFAULT_INDEXING_BUFFERED_ITEMS, "LuceneIndexingExecutor"), - ExecutorFactory.ExecutorType.LUCENESERVER, + ExecutorFactory.ExecutorType.SERVER, new ThreadPoolSettings( - DEFAULT_GRPC_LUCENESERVER_THREADS, - DEFAULT_GRPC_LUCENESERVER_BUFFERED_ITEMS, - "GrpcLuceneServerExecutor"), + DEFAULT_GRPC_SERVER_THREADS, + DEFAULT_GRPC_SERVER_BUFFERED_ITEMS, + "GrpcServerExecutor"), ExecutorFactory.ExecutorType.REPLICATIONSERVER, new ThreadPoolSettings( DEFAULT_GRPC_REPLICATIONSERVER_THREADS, diff --git a/src/main/java/com/yelp/nrtsearch/server/grpc/GrpcServerExecutorSupplier.java b/src/main/java/com/yelp/nrtsearch/server/grpc/GrpcServerExecutorSupplier.java index 87fd802a7..2f70c4621 100644 --- a/src/main/java/com/yelp/nrtsearch/server/grpc/GrpcServerExecutorSupplier.java +++ b/src/main/java/com/yelp/nrtsearch/server/grpc/GrpcServerExecutorSupplier.java @@ -43,8 +43,7 @@ public class GrpcServerExecutorSupplier implements ServerCallExecutorSupplier { private final ExecutorService grpcExecutor; public GrpcServerExecutorSupplier() { - serverExecutor = - ExecutorFactory.getInstance().getExecutor(ExecutorFactory.ExecutorType.LUCENESERVER); + serverExecutor = ExecutorFactory.getInstance().getExecutor(ExecutorFactory.ExecutorType.SERVER); metricsExecutor = ExecutorFactory.getInstance().getExecutor(ExecutorFactory.ExecutorType.METRICS); grpcExecutor = ExecutorFactory.getInstance().getExecutor(ExecutorFactory.ExecutorType.GRPC); diff --git a/src/main/java/com/yelp/nrtsearch/server/grpc/NrtsearchClient.java b/src/main/java/com/yelp/nrtsearch/server/grpc/NrtsearchClient.java index c9583d03c..cbba5465a 100644 --- a/src/main/java/com/yelp/nrtsearch/server/grpc/NrtsearchClient.java +++ b/src/main/java/com/yelp/nrtsearch/server/grpc/NrtsearchClient.java @@ -52,7 +52,7 @@ public LuceneServerGrpc.LuceneServerStub getAsyncStub() { private final LuceneServerGrpc.LuceneServerBlockingStub blockingStub; private final LuceneServerGrpc.LuceneServerStub asyncStub; - /** Construct client connecting to LuceneServer server at {@code host:port}. */ + /** Construct client connecting to NrtsearchServer server at {@code host:port}. */ public NrtsearchClient(String host, int port) { this( ManagedChannelBuilder.forAddress(host, port) @@ -63,7 +63,7 @@ public NrtsearchClient(String host, int port) { .build()); } - /** Construct client for accessing LuceneServer server using the existing channel. */ + /** Construct client for accessing NrtsearchServer server using the existing channel. */ NrtsearchClient(ManagedChannel channel) { this.channel = channel; blockingStub = diff --git a/src/main/java/com/yelp/nrtsearch/server/grpc/NrtsearchServer.java b/src/main/java/com/yelp/nrtsearch/server/grpc/NrtsearchServer.java index db361dfa3..fabf6d97b 100644 --- a/src/main/java/com/yelp/nrtsearch/server/grpc/NrtsearchServer.java +++ b/src/main/java/com/yelp/nrtsearch/server/grpc/NrtsearchServer.java @@ -125,7 +125,7 @@ import org.slf4j.LoggerFactory; import picocli.CommandLine; -/** Server that manages startup/shutdown of a {@code LuceneServer} server. */ +/** Server that manages startup/shutdown of a {@code NrtsearchServer} server. */ public class NrtsearchServer { private static final Logger logger = LoggerFactory.getLogger(NrtsearchServer.class.getName()); private final RemoteBackend remoteBackend; @@ -134,18 +134,17 @@ public class NrtsearchServer { private Server server; private Server replicationServer; - private final NrtsearchConfig luceneServerConfiguration; + private final NrtsearchConfig configuration; @Inject public NrtsearchServer( - NrtsearchConfig luceneServerConfiguration, + NrtsearchConfig configuration, RemoteBackend remoteBackend, PrometheusRegistry prometheusRegistry) { - this.luceneServerConfiguration = luceneServerConfiguration; + this.configuration = configuration; this.remoteBackend = remoteBackend; this.prometheusRegistry = prometheusRegistry; - this.pluginsService = - new PluginsService(luceneServerConfiguration, remoteBackend, prometheusRegistry); + this.pluginsService = new PluginsService(configuration, remoteBackend, prometheusRegistry); } @VisibleForTesting @@ -153,36 +152,34 @@ public void start() throws IOException { List plugins = pluginsService.loadPlugins(); LuceneServerImpl serverImpl = - new LuceneServerImpl(luceneServerConfiguration, remoteBackend, prometheusRegistry, plugins); + new LuceneServerImpl(configuration, remoteBackend, prometheusRegistry, plugins); GlobalState globalState = serverImpl.getGlobalState(); registerMetrics(globalState); - if (luceneServerConfiguration.getMaxConcurrentCallsPerConnectionForReplication() != -1) { + if (configuration.getMaxConcurrentCallsPerConnectionForReplication() != -1) { replicationServer = - NettyServerBuilder.forPort(luceneServerConfiguration.getReplicationPort()) + NettyServerBuilder.forPort(configuration.getReplicationPort()) .addService( new ReplicationServerImpl( - globalState, luceneServerConfiguration.getVerifyReplicationIndexId())) + globalState, configuration.getVerifyReplicationIndexId())) .executor( ExecutorFactory.getInstance() .getExecutor(ExecutorFactory.ExecutorType.REPLICATIONSERVER)) .maxInboundMessageSize(MAX_MESSAGE_BYTES_SIZE) .maxConcurrentCallsPerConnection( - luceneServerConfiguration.getMaxConcurrentCallsPerConnectionForReplication()) - .maxConnectionAge( - luceneServerConfiguration.getMaxConnectionAgeForReplication(), TimeUnit.SECONDS) + configuration.getMaxConcurrentCallsPerConnectionForReplication()) + .maxConnectionAge(configuration.getMaxConnectionAgeForReplication(), TimeUnit.SECONDS) .maxConnectionAgeGrace( - luceneServerConfiguration.getMaxConnectionAgeGraceForReplication(), - TimeUnit.SECONDS) + configuration.getMaxConnectionAgeGraceForReplication(), TimeUnit.SECONDS) .build() .start(); } else { replicationServer = - ServerBuilder.forPort(luceneServerConfiguration.getReplicationPort()) + ServerBuilder.forPort(configuration.getReplicationPort()) .addService( new ReplicationServerImpl( - globalState, luceneServerConfiguration.getVerifyReplicationIndexId())) + globalState, configuration.getVerifyReplicationIndexId())) .executor( ExecutorFactory.getInstance() .getExecutor(ExecutorFactory.ExecutorType.REPLICATIONSERVER)) @@ -192,7 +189,7 @@ public void start() throws IOException { } logger.info( "Server started, listening on " - + luceneServerConfiguration.getReplicationPort() + + configuration.getReplicationPort() + " for replication messages"); // Inform global state that the replication server is started, and it is safe to start indices @@ -201,12 +198,12 @@ public void start() throws IOException { NrtsearchMonitoringServerInterceptor monitoringInterceptor = NrtsearchMonitoringServerInterceptor.create( Configuration.allMetrics() - .withLatencyBuckets(luceneServerConfiguration.getMetricsBuckets()) + .withLatencyBuckets(configuration.getMetricsBuckets()) .withPrometheusRegistry(prometheusRegistry)); /* The port on which the server should run */ GrpcServerExecutorSupplier executorSupplier = new GrpcServerExecutorSupplier(); server = - ServerBuilder.forPort(luceneServerConfiguration.getPort()) + ServerBuilder.forPort(configuration.getPort()) .addService(ServerInterceptors.intercept(serverImpl, monitoringInterceptor)) .addService(ProtoReflectionService.newInstance()) // Set executor supplier to use different thread pool for metrics method @@ -219,8 +216,7 @@ public void start() throws IOException { .decompressorRegistry(LuceneServerStubBuilder.DECOMPRESSOR_REGISTRY) .build() .start(); - logger.info( - "Server started, listening on " + luceneServerConfiguration.getPort() + " for messages"); + logger.info("Server started, listening on " + configuration.getPort() + " for messages"); } @VisibleForTesting @@ -272,7 +268,7 @@ public static void main(String[] args) { } @CommandLine.Command( - name = "lucene-server", + name = "nrtsearch_server", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class, description = "Start NRT search server") @@ -281,7 +277,7 @@ public static class NrtsearchServerCommand implements Callable { arity = "0..1", paramLabel = "server_yaml_config_file", description = - "Optional yaml config file. Defaults to /lucene_server_default_configuration.yaml") + "Optional yaml config file. Defaults to /nrtsearch_default_config.yaml") private File optionalConfigFile; public Optional maybeConfigFile() { @@ -290,11 +286,11 @@ public Optional maybeConfigFile() { @Override public Integer call() throws Exception { - NrtsearchServer luceneServer; + NrtsearchServer server; try { Injector injector = Guice.createInjector(new NrtsearchModule(this)); - luceneServer = injector.getInstance(NrtsearchServer.class); - luceneServer.start(); + server = injector.getInstance(NrtsearchServer.class); + server.start(); Runtime.getRuntime() .addShutdownHook( @@ -303,14 +299,14 @@ public Integer call() throws Exception { // Use stderr here since the logger may have been reset by its JVM shutdown // hook. logger.error("*** shutting down gRPC server since JVM is shutting down"); - luceneServer.stop(); + server.stop(); logger.error("*** server shut down"); })); } catch (Throwable t) { logger.error("Uncaught exception", t); throw t; } - luceneServer.blockUntilShutdown(); + server.blockUntilShutdown(); return 0; } } diff --git a/src/main/java/com/yelp/nrtsearch/server/modules/NrtsearchModule.java b/src/main/java/com/yelp/nrtsearch/server/modules/NrtsearchModule.java index 20add1f40..e56785709 100644 --- a/src/main/java/com/yelp/nrtsearch/server/modules/NrtsearchModule.java +++ b/src/main/java/com/yelp/nrtsearch/server/modules/NrtsearchModule.java @@ -28,8 +28,7 @@ import java.util.Optional; public class NrtsearchModule extends AbstractModule { - private static final String DEFAULT_CONFIG_FILE_RESOURCE = - "/lucene_server_default_configuration.yaml"; + private static final String DEFAULT_CONFIG_FILE_RESOURCE = "/nrtsearch_default_config.yaml"; private final NrtsearchServer.NrtsearchServerCommand args; public NrtsearchModule(NrtsearchServer.NrtsearchServerCommand args) { @@ -53,14 +52,14 @@ public PrometheusRegistry providesPrometheusRegistry() { @Singleton @Provides protected NrtsearchConfig providesNrtsearchConfig() throws FileNotFoundException { - NrtsearchConfig luceneServerConfiguration; + NrtsearchConfig configuration; Optional maybeConfigFile = args.maybeConfigFile(); if (maybeConfigFile.isEmpty()) { - luceneServerConfiguration = + configuration = new NrtsearchConfig(getClass().getResourceAsStream(DEFAULT_CONFIG_FILE_RESOURCE)); } else { - luceneServerConfiguration = new NrtsearchConfig(new FileInputStream(maybeConfigFile.get())); + configuration = new NrtsearchConfig(new FileInputStream(maybeConfigFile.get())); } - return luceneServerConfiguration; + return configuration; } } diff --git a/src/main/java/com/yelp/nrtsearch/server/modules/S3Module.java b/src/main/java/com/yelp/nrtsearch/server/modules/S3Module.java index a7bbcb2c5..22c6cb150 100644 --- a/src/main/java/com/yelp/nrtsearch/server/modules/S3Module.java +++ b/src/main/java/com/yelp/nrtsearch/server/modules/S3Module.java @@ -27,7 +27,7 @@ public class S3Module extends AbstractModule { @Inject @Singleton @Provides - protected AmazonS3 providesAmazonS3(NrtsearchConfig luceneServerConfiguration) { - return S3Util.buildS3Client(luceneServerConfiguration); + protected AmazonS3 providesAmazonS3(NrtsearchConfig configuration) { + return S3Util.buildS3Client(configuration); } } diff --git a/src/main/java/com/yelp/nrtsearch/server/plugins/Plugin.java b/src/main/java/com/yelp/nrtsearch/server/plugins/Plugin.java index 12b1cf269..ebf0812fa 100644 --- a/src/main/java/com/yelp/nrtsearch/server/plugins/Plugin.java +++ b/src/main/java/com/yelp/nrtsearch/server/plugins/Plugin.java @@ -21,7 +21,7 @@ /** Common base class for all nrtsearch plugins. */ public abstract class Plugin implements Closeable { /** - * Cleanup function called during lucene server stop. + * Cleanup function called during nrtsearch server stop. * * @throws IOException */ diff --git a/src/main/java/com/yelp/nrtsearch/server/plugins/PluginsService.java b/src/main/java/com/yelp/nrtsearch/server/plugins/PluginsService.java index af4b2d294..196a0499f 100644 --- a/src/main/java/com/yelp/nrtsearch/server/plugins/PluginsService.java +++ b/src/main/java/com/yelp/nrtsearch/server/plugins/PluginsService.java @@ -37,7 +37,7 @@ * the config provided plugin search path for a folder matching the plugin name. A classloader is * created with the jars provided in the plugin directory. The plugin provides a config file * containing the {@link Plugin} classname that should be loaded. Reflection is used to give the - * loaded plugin access to the lucene server config. + * loaded plugin access to the server config. */ public class PluginsService { private static final Logger logger = LoggerFactory.getLogger(PluginsService.class); diff --git a/src/main/java/com/yelp/nrtsearch/server/remote/s3/S3Util.java b/src/main/java/com/yelp/nrtsearch/server/remote/s3/S3Util.java index 3eb502868..d02cdb4bd 100644 --- a/src/main/java/com/yelp/nrtsearch/server/remote/s3/S3Util.java +++ b/src/main/java/com/yelp/nrtsearch/server/remote/s3/S3Util.java @@ -46,19 +46,19 @@ private S3Util() {} /** * Create a new s3 client from the given configuration. * - * @param luceneServerConfiguration configuration + * @param configuration server configuration * @return s3 client */ - public static AmazonS3 buildS3Client(NrtsearchConfig luceneServerConfiguration) { + public static AmazonS3 buildS3Client(NrtsearchConfig configuration) { AWSCredentialsProvider awsCredentialsProvider; - if (luceneServerConfiguration.getBotoCfgPath() == null) { + if (configuration.getBotoCfgPath() == null) { awsCredentialsProvider = new DefaultAWSCredentialsProviderChain(); } else { - Path botoCfgPath = Paths.get(luceneServerConfiguration.getBotoCfgPath()); + Path botoCfgPath = Paths.get(configuration.getBotoCfgPath()); final ProfilesConfigFile profilesConfigFile = new ProfilesConfigFile(botoCfgPath.toFile()); awsCredentialsProvider = new ProfileCredentialsProvider(profilesConfigFile, "default"); } - final boolean globalBucketAccess = luceneServerConfiguration.getEnableGlobalBucketAccess(); + final boolean globalBucketAccess = configuration.getEnableGlobalBucketAccess(); AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard() @@ -70,7 +70,7 @@ public static AmazonS3 buildS3Client(NrtsearchConfig luceneServerConfiguration) .withCredentials(awsCredentialsProvider) .withForceGlobalBucketAccessEnabled(globalBucketAccess) .build(); - String region = s3ClientInterim.getBucketLocation(luceneServerConfiguration.getBucketName()); + String region = s3ClientInterim.getBucketLocation(configuration.getBucketName()); // In useast-1, the region is returned as "US" which is an equivalent to "us-east-1" // https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/Region.html#US_Standard // However, this causes an UnknownHostException so we override it to the full region name @@ -83,7 +83,7 @@ public static AmazonS3 buildS3Client(NrtsearchConfig luceneServerConfiguration) } catch (SdkClientException sdkClientException) { logger.warn( "failed to get the location of S3 bucket: " - + luceneServerConfiguration.getBucketName() + + configuration.getBucketName() + ". This could be caused by missing credentials and/or regions, or wrong bucket name.", sdkClientException); logger.info("return a dummy AmazonS3."); @@ -94,7 +94,7 @@ public static AmazonS3 buildS3Client(NrtsearchConfig luceneServerConfiguration) .build(); } - int maxRetries = luceneServerConfiguration.getMaxS3ClientRetries(); + int maxRetries = configuration.getMaxS3ClientRetries(); if (maxRetries > 0) { RetryPolicy retryPolicy = new RetryPolicy( diff --git a/src/main/java/com/yelp/nrtsearch/server/state/BackendGlobalState.java b/src/main/java/com/yelp/nrtsearch/server/state/BackendGlobalState.java index 75c9875ec..d5a25127c 100644 --- a/src/main/java/com/yelp/nrtsearch/server/state/BackendGlobalState.java +++ b/src/main/java/com/yelp/nrtsearch/server/state/BackendGlobalState.java @@ -99,13 +99,13 @@ public static String getUniqueIndexName(String indexName, String id) { /** * Constructor. * - * @param luceneServerConfiguration server config + * @param configuration server config * @param remoteBackend backend for persistent remote storage * @throws IOException on filesystem error */ - public BackendGlobalState(NrtsearchConfig luceneServerConfiguration, RemoteBackend remoteBackend) + public BackendGlobalState(NrtsearchConfig configuration, RemoteBackend remoteBackend) throws IOException { - super(luceneServerConfiguration, remoteBackend); + super(configuration, remoteBackend); stateBackend = createStateBackend(); GlobalStateInfo globalStateInfo = stateBackend.loadOrCreateGlobalState(); // init index state managers @@ -115,7 +115,7 @@ public BackendGlobalState(NrtsearchConfig luceneServerConfiguration, RemoteBacke createIndexStateManager( entry.getKey(), entry.getValue().getId(), - luceneServerConfiguration.getLiveSettingsOverride(entry.getKey()), + configuration.getLiveSettingsOverride(entry.getKey()), stateBackend); stateManager.load(); managerMap.put(entry.getKey(), stateManager); diff --git a/src/main/java/com/yelp/nrtsearch/server/state/GlobalState.java b/src/main/java/com/yelp/nrtsearch/server/state/GlobalState.java index ccc621bcb..0cc6abf7b 100644 --- a/src/main/java/com/yelp/nrtsearch/server/state/GlobalState.java +++ b/src/main/java/com/yelp/nrtsearch/server/state/GlobalState.java @@ -67,27 +67,26 @@ public abstract class GlobalState implements Closeable { private final ExecutorService fetchExecutor; private final ExecutorService searchExecutor; - public static GlobalState createState( - NrtsearchConfig luceneServerConfiguration, RemoteBackend remoteBackend) throws IOException { - return new BackendGlobalState(luceneServerConfiguration, remoteBackend); + public static GlobalState createState(NrtsearchConfig configuration, RemoteBackend remoteBackend) + throws IOException { + return new BackendGlobalState(configuration, remoteBackend); } public RemoteBackend getRemoteBackend() { return remoteBackend; } - protected GlobalState(NrtsearchConfig luceneServerConfiguration, RemoteBackend remoteBackend) + protected GlobalState(NrtsearchConfig configuration, RemoteBackend remoteBackend) throws IOException { this.remoteBackend = remoteBackend; - this.nodeName = luceneServerConfiguration.getNodeName(); - this.serviceName = luceneServerConfiguration.getServiceName(); - this.stateDir = Paths.get(luceneServerConfiguration.getStateDir()); - this.indexDirBase = Paths.get(luceneServerConfiguration.getIndexDir()); - this.hostName = luceneServerConfiguration.getHostName(); - this.port = luceneServerConfiguration.getPort(); - this.replicaReplicationPortPingInterval = - luceneServerConfiguration.getReplicaReplicationPortPingInterval(); - this.threadPoolConfiguration = luceneServerConfiguration.getThreadPoolConfiguration(); + this.nodeName = configuration.getNodeName(); + this.serviceName = configuration.getServiceName(); + this.stateDir = Paths.get(configuration.getStateDir()); + this.indexDirBase = Paths.get(configuration.getIndexDir()); + this.hostName = configuration.getHostName(); + this.port = configuration.getPort(); + this.replicaReplicationPortPingInterval = configuration.getReplicaReplicationPortPingInterval(); + this.threadPoolConfiguration = configuration.getThreadPoolConfiguration(); if (Files.exists(stateDir) == false) { Files.createDirectories(stateDir); } @@ -97,7 +96,7 @@ protected GlobalState(NrtsearchConfig luceneServerConfiguration, RemoteBackend r ExecutorFactory.getInstance().getExecutor(ExecutorFactory.ExecutorType.SEARCH); this.fetchExecutor = ExecutorFactory.getInstance().getExecutor(ExecutorFactory.ExecutorType.FETCH); - this.configuration = luceneServerConfiguration; + this.configuration = configuration; } public NrtsearchConfig getConfiguration() { diff --git a/src/main/java/com/yelp/nrtsearch/server/state/backend/RemoteStateBackend.java b/src/main/java/com/yelp/nrtsearch/server/state/backend/RemoteStateBackend.java index 1176b3f9a..3f097a0ad 100644 --- a/src/main/java/com/yelp/nrtsearch/server/state/backend/RemoteStateBackend.java +++ b/src/main/java/com/yelp/nrtsearch/server/state/backend/RemoteStateBackend.java @@ -51,12 +51,12 @@ public static class RemoteBackendConfig { /** * Read backend config from server config. * - * @param luceneServerConfiguration server configuration + * @param configuration server configuration * @return config for remote backend */ - public static RemoteBackendConfig fromConfig(NrtsearchConfig luceneServerConfiguration) { + public static RemoteBackendConfig fromConfig(NrtsearchConfig configuration) { boolean readOnly = - luceneServerConfiguration.getConfigReader().getBoolean(CONFIG_PREFIX + "readOnly", true); + configuration.getConfigReader().getBoolean(CONFIG_PREFIX + "readOnly", true); return new RemoteBackendConfig(readOnly); } diff --git a/src/main/java/com/yelp/nrtsearch/tools/cli/NrtsearchClientCommand.java b/src/main/java/com/yelp/nrtsearch/tools/cli/NrtsearchClientCommand.java index 40590aa0c..ae40350fc 100644 --- a/src/main/java/com/yelp/nrtsearch/tools/cli/NrtsearchClientCommand.java +++ b/src/main/java/com/yelp/nrtsearch/tools/cli/NrtsearchClientCommand.java @@ -23,7 +23,7 @@ /*Each new command needs to be added here*/ @CommandLine.Command( - name = "lucene-client", + name = "nrtsearch_client", synopsisSubcommandLabel = "COMMAND", subcommands = { AddDocumentsCommand.class, diff --git a/src/main/resources/lucene_server_default_configuration.yaml b/src/main/resources/nrtsearch_default_config.yaml similarity index 67% rename from src/main/resources/lucene_server_default_configuration.yaml rename to src/main/resources/nrtsearch_default_config.yaml index bc38aadae..a319cead0 100644 --- a/src/main/resources/lucene_server_default_configuration.yaml +++ b/src/main/resources/nrtsearch_default_config.yaml @@ -1,4 +1,4 @@ -nodeName: "lucene_server" +nodeName: "nrtsearch_server" hostName: "localhost" port: "6000" replicationPort: "6001" diff --git a/src/test/java/com/yelp/nrtsearch/clientlib/NodeNameResolverAndLoadBalancingTests.java b/src/test/java/com/yelp/nrtsearch/clientlib/NodeNameResolverAndLoadBalancingTests.java index 9c90b3d51..28ba4b138 100644 --- a/src/test/java/com/yelp/nrtsearch/clientlib/NodeNameResolverAndLoadBalancingTests.java +++ b/src/test/java/com/yelp/nrtsearch/clientlib/NodeNameResolverAndLoadBalancingTests.java @@ -37,7 +37,7 @@ import com.yelp.nrtsearch.server.grpc.SearchRequest; import com.yelp.nrtsearch.server.grpc.SearchResponse; import com.yelp.nrtsearch.server.grpc.StartIndexRequest; -import com.yelp.nrtsearch.server.utils.LuceneServerTestConfigurationFactory; +import com.yelp.nrtsearch.server.utils.NrtsearchTestConfigurationFactory; import io.grpc.Status; import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; @@ -111,16 +111,16 @@ public void setup() throws IOException, InterruptedException { } private GrpcServer createGrpcServer() throws IOException { - NrtsearchConfig luceneServerConfiguration = - LuceneServerTestConfigurationFactory.getConfig(Mode.STANDALONE, folder.getRoot()); + NrtsearchConfig configuration = + NrtsearchTestConfigurationFactory.getConfig(Mode.STANDALONE, folder.getRoot()); return new GrpcServer( grpcCleanup, - luceneServerConfiguration, + configuration, folder, null, - luceneServerConfiguration.getIndexDir(), + configuration.getIndexDir(), TEST_INDEX, - luceneServerConfiguration.getPort()); + configuration.getPort()); } private void startIndexAndAddDocuments(GrpcServer server, int id) diff --git a/src/test/java/com/yelp/nrtsearch/server/ServerTestCase.java b/src/test/java/com/yelp/nrtsearch/server/ServerTestCase.java index 731efe8b3..2dd1c261d 100644 --- a/src/test/java/com/yelp/nrtsearch/server/ServerTestCase.java +++ b/src/test/java/com/yelp/nrtsearch/server/ServerTestCase.java @@ -35,7 +35,7 @@ import com.yelp.nrtsearch.server.grpc.StartIndexRequest; import com.yelp.nrtsearch.server.plugins.Plugin; import com.yelp.nrtsearch.server.state.GlobalState; -import com.yelp.nrtsearch.server.utils.LuceneServerTestConfigurationFactory; +import com.yelp.nrtsearch.server.utils.NrtsearchTestConfigurationFactory; import io.grpc.stub.StreamObserver; import io.grpc.testing.GrpcCleanupRule; import io.prometheus.metrics.model.registry.PrometheusRegistry; @@ -63,8 +63,8 @@ import org.junit.rules.TemporaryFolder; /** - * Base class for tests that want a lucene server that is setup once and used for all class tests. - * Protected methods may be overridden to specify arbitrary indices and add documents. + * Base class for tests that want an nrtsearch server that is setup once and used for all class + * tests. Protected methods may be overridden to specify arbitrary indices and add documents. */ public class ServerTestCase { public static final String DEFAULT_TEST_INDEX = "test_index"; @@ -217,21 +217,21 @@ public void setUpClass() throws Exception { private GrpcServer setUpGrpcServer(PrometheusRegistry prometheusRegistry) throws IOException { String testIndex = "test_index"; - NrtsearchConfig luceneServerConfiguration = - LuceneServerTestConfigurationFactory.getConfig( + NrtsearchConfig configuration = + NrtsearchTestConfigurationFactory.getConfig( Mode.STANDALONE, folder.getRoot(), getExtraConfig()); GrpcServer server = new GrpcServer( prometheusRegistry, grpcCleanup, - luceneServerConfiguration, + configuration, folder, null, - luceneServerConfiguration.getIndexDir(), + configuration.getIndexDir(), testIndex, - luceneServerConfiguration.getPort(), + configuration.getPort(), null, - getPlugins(luceneServerConfiguration)); + getPlugins(configuration)); globalState = server.getGlobalState(); return server; } diff --git a/src/test/java/com/yelp/nrtsearch/server/analysis/AnalyzerCreatorTest.java b/src/test/java/com/yelp/nrtsearch/server/analysis/AnalyzerCreatorTest.java index 48dfdaee0..e8f990c14 100644 --- a/src/test/java/com/yelp/nrtsearch/server/analysis/AnalyzerCreatorTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/analysis/AnalyzerCreatorTest.java @@ -81,7 +81,7 @@ private void init(List plugins) { } private NrtsearchConfig getEmptyConfig() { - String config = "nodeName: \"lucene_server_foo\""; + String config = "nodeName: \"server_foo\""; return new NrtsearchConfig(new ByteArrayInputStream(config.getBytes())); } diff --git a/src/test/java/com/yelp/nrtsearch/server/analysis/SynonymV2GraphFilterFactoryITest.java b/src/test/java/com/yelp/nrtsearch/server/analysis/SynonymV2GraphFilterFactoryITest.java index 70969db30..7aa28fb52 100644 --- a/src/test/java/com/yelp/nrtsearch/server/analysis/SynonymV2GraphFilterFactoryITest.java +++ b/src/test/java/com/yelp/nrtsearch/server/analysis/SynonymV2GraphFilterFactoryITest.java @@ -45,7 +45,7 @@ private void init(List plugins) { } private NrtsearchConfig getEmptyConfig() { - String config = "nodeName: \"lucene_server_foo\""; + String config = "nodeName: \"server_foo\""; return new NrtsearchConfig(new ByteArrayInputStream(config.getBytes())); } diff --git a/src/test/java/com/yelp/nrtsearch/server/analysis/SynonymV2GraphFilterFactoryTest.java b/src/test/java/com/yelp/nrtsearch/server/analysis/SynonymV2GraphFilterFactoryTest.java index 29b9ba28d..4dc459d21 100644 --- a/src/test/java/com/yelp/nrtsearch/server/analysis/SynonymV2GraphFilterFactoryTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/analysis/SynonymV2GraphFilterFactoryTest.java @@ -49,7 +49,7 @@ private void init(List plugins) { } private NrtsearchConfig getEmptyConfig() { - String config = "nodeName: \"lucene_server_foo\""; + String config = "nodeName: \"server_foo\""; return new NrtsearchConfig(new ByteArrayInputStream(config.getBytes())); } diff --git a/src/test/java/com/yelp/nrtsearch/server/concurrent/ExecutorFactoryTest.java b/src/test/java/com/yelp/nrtsearch/server/concurrent/ExecutorFactoryTest.java index 997d44117..a2f64d1af 100644 --- a/src/test/java/com/yelp/nrtsearch/server/concurrent/ExecutorFactoryTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/concurrent/ExecutorFactoryTest.java @@ -32,9 +32,9 @@ private void init() { } private void init(String config) { - NrtsearchConfig luceneServerConfiguration = + NrtsearchConfig serverConfiguration = new NrtsearchConfig(new ByteArrayInputStream(config.getBytes())); - ExecutorFactory.init(luceneServerConfiguration.getThreadPoolConfiguration()); + ExecutorFactory.init(serverConfiguration.getThreadPoolConfiguration()); } @Test @@ -104,30 +104,29 @@ public void testIndexThreadPool_set() { } @Test - public void testLuceneServerThreadPool_default() { + public void testServerThreadPool_default() { init(); ThreadPoolExecutor executor = (ThreadPoolExecutor) - ExecutorFactory.getInstance().getExecutor(ExecutorFactory.ExecutorType.LUCENESERVER); - assertEquals( - executor.getCorePoolSize(), ThreadPoolConfiguration.DEFAULT_GRPC_LUCENESERVER_THREADS); + ExecutorFactory.getInstance().getExecutor(ExecutorFactory.ExecutorType.SERVER); + assertEquals(executor.getCorePoolSize(), ThreadPoolConfiguration.DEFAULT_GRPC_SERVER_THREADS); assertEquals( executor.getQueue().remainingCapacity(), - ThreadPoolConfiguration.DEFAULT_GRPC_LUCENESERVER_BUFFERED_ITEMS); + ThreadPoolConfiguration.DEFAULT_GRPC_SERVER_BUFFERED_ITEMS); } @Test - public void testLuceneServerThreadPool_set() { + public void testServerThreadPool_set() { init( String.join( "\n", "threadPoolConfiguration:", - " luceneserver:", + " server:", " maxThreads: 5", " maxBufferedItems: 10")); ThreadPoolExecutor executor = (ThreadPoolExecutor) - ExecutorFactory.getInstance().getExecutor(ExecutorFactory.ExecutorType.LUCENESERVER); + ExecutorFactory.getInstance().getExecutor(ExecutorFactory.ExecutorType.SERVER); assertEquals(executor.getCorePoolSize(), 5); assertEquals(executor.getQueue().remainingCapacity(), 10); } diff --git a/src/test/java/com/yelp/nrtsearch/server/config/FileCopyConfigTest.java b/src/test/java/com/yelp/nrtsearch/server/config/FileCopyConfigTest.java index 15331f31b..2e9d1ee16 100644 --- a/src/test/java/com/yelp/nrtsearch/server/config/FileCopyConfigTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/config/FileCopyConfigTest.java @@ -30,7 +30,7 @@ private static FileCopyConfig getConfig(String configFile) { @Test public void testDefault() { - String configFile = "nodeName: \"lucene_server_foo\""; + String configFile = "nodeName: \"server_foo\""; FileCopyConfig config = getConfig(configFile); assertFalse(config.getAckedCopy()); assertEquals(FileCopyConfig.DEFAULT_CHUNK_SIZE, config.getChunkSize()); @@ -43,7 +43,7 @@ public void testConfig() { String configFile = String.join( "\n", - "nodeName: \"lucene_server_foo\"", + "nodeName: \"server_foo\"", "FileCopyConfig:", " ackedCopy: true", " chunkSize: 100", diff --git a/src/test/java/com/yelp/nrtsearch/server/config/IndexPreloadConfigTest.java b/src/test/java/com/yelp/nrtsearch/server/config/IndexPreloadConfigTest.java index 47f56289a..c912f9974 100644 --- a/src/test/java/com/yelp/nrtsearch/server/config/IndexPreloadConfigTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/config/IndexPreloadConfigTest.java @@ -34,7 +34,7 @@ private static IndexPreloadConfig getConfig(String configFile) { @Test public void testDefault() { - String configFile = "nodeName: \"lucene_server_foo\""; + String configFile = "nodeName: \"server_foo\""; IndexPreloadConfig config = getConfig(configFile); assertFalse(config.getPreload()); assertEquals(config.getExtensions(), Collections.singleton(IndexPreloadConfig.ALL_EXTENSIONS)); diff --git a/src/test/java/com/yelp/nrtsearch/server/config/IndexStartConfigTest.java b/src/test/java/com/yelp/nrtsearch/server/config/IndexStartConfigTest.java index 37f1320ef..6cfef2af0 100644 --- a/src/test/java/com/yelp/nrtsearch/server/config/IndexStartConfigTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/config/IndexStartConfigTest.java @@ -33,7 +33,7 @@ private static IndexStartConfig getConfig(String configFile) { @Test public void testDefaultConfig() { - String configFile = "nodeName: \"lucene_server_foo\""; + String configFile = "nodeName: \"server_foo\""; IndexStartConfig startConfig = getConfig(configFile); assertTrue(startConfig.getAutoStart()); assertEquals(Mode.STANDALONE, startConfig.getMode()); diff --git a/src/test/java/com/yelp/nrtsearch/server/config/NrtsearchConfigTest.java b/src/test/java/com/yelp/nrtsearch/server/config/NrtsearchConfigTest.java index 5d42e0b44..b14cdbd33 100644 --- a/src/test/java/com/yelp/nrtsearch/server/config/NrtsearchConfigTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/config/NrtsearchConfigTest.java @@ -34,15 +34,14 @@ private NrtsearchConfig getForConfig(String config) { @Test public void testGetsHostName() { - String config = String.join("\n", "nodeName: \"lucene_server_foo\"", "hostName: my_host_name"); + String config = String.join("\n", "nodeName: \"server_foo\"", "hostName: my_host_name"); NrtsearchConfig luceneConfig = getForConfig(config); assertEquals("my_host_name", luceneConfig.getHostName()); } @Test public void testGetEnvHostName() { - String config = - String.join("\n", "nodeName: \"lucene_server_foo\"", "hostName: ${CUSTOM_HOST}"); + String config = String.join("\n", "nodeName: \"server_foo\"", "hostName: ${CUSTOM_HOST}"); NrtsearchConfig luceneConfig = getForConfig(config); assertEquals("my_custom_host", luceneConfig.getHostName()); } @@ -50,8 +49,7 @@ public void testGetEnvHostName() { @Test public void testGetMultiEnvHostName() { String config = - String.join( - "\n", "nodeName: \"lucene_server_foo\"", "hostName: my_${VAR1}_${VAR2}_${VAR1}_host"); + String.join("\n", "nodeName: \"server_foo\"", "hostName: my_${VAR1}_${VAR2}_${VAR1}_host"); NrtsearchConfig luceneConfig = getForConfig(config); assertEquals("my_v1_v2_v1_host", luceneConfig.getHostName()); } @@ -59,15 +57,14 @@ public void testGetMultiEnvHostName() { @Test public void testMissingEnvHostName() { String config = - String.join( - "\n", "nodeName: \"lucene_server_foo\"", "hostName: my_${VAR4}_${VAR3}_${VAR4}_host"); + String.join("\n", "nodeName: \"server_foo\"", "hostName: my_${VAR4}_${VAR3}_${VAR4}_host"); NrtsearchConfig luceneConfig = getForConfig(config); assertEquals("my__v3__host", luceneConfig.getHostName()); } @Test public void testDefaultDiscoveryFileUpdateInterval() { - String config = "nodeName: \"lucene_server_foo\""; + String config = "nodeName: \"server_foo\""; NrtsearchConfig luceneConfig = getForConfig(config); assertEquals( ReplicationServerClient.FILE_UPDATE_INTERVAL_MS, @@ -77,14 +74,14 @@ public void testDefaultDiscoveryFileUpdateInterval() { @Test public void testSetDiscoveryFileUpdateInterval() { String config = - String.join("\n", "nodeName: \"lucene_server_foo\"", "discoveryFileUpdateIntervalMs: 100"); + String.join("\n", "nodeName: \"server_foo\"", "discoveryFileUpdateIntervalMs: 100"); NrtsearchConfig luceneConfig = getForConfig(config); assertEquals(100, luceneConfig.getDiscoveryFileUpdateIntervalMs()); } @Test public void testDefaultCompletionCodecLoadMode() { - String config = "nodeName: \"lucene_server_foo\""; + String config = "nodeName: \"server_foo\""; NrtsearchConfig luceneConfig = getForConfig(config); assertEquals(FSTLoadMode.ON_HEAP, luceneConfig.getCompletionCodecLoadMode()); } @@ -98,7 +95,7 @@ public void testSetCompletionCodecLoadMode() { @Test public void testInitialSyncPrimaryWaitMs_default() { - String config = "nodeName: \"lucene_server_foo\""; + String config = "nodeName: \"server_foo\""; NrtsearchConfig luceneConfig = getForConfig(config); assertEquals( NrtsearchConfig.DEFAULT_INITIAL_SYNC_PRIMARY_WAIT_MS, @@ -114,7 +111,7 @@ public void testInitialSyncPrimaryWaitMs_set() { @Test public void testInitialSyncMaxTimeMs_default() { - String config = "nodeName: \"lucene_server_foo\""; + String config = "nodeName: \"server_foo\""; NrtsearchConfig luceneConfig = getForConfig(config); assertEquals( NrtsearchConfig.DEFAULT_INITIAL_SYNC_MAX_TIME_MS, luceneConfig.getInitialSyncMaxTimeMs()); @@ -129,7 +126,7 @@ public void testInitialSyncMaxTimeMs_set() { @Test public void testMaxS3ClientRetries_default() { - String config = "nodeName: \"lucene_server_foo\""; + String config = "nodeName: \"server_foo\""; NrtsearchConfig luceneConfig = getForConfig(config); assertEquals( NrtsearchConfig.DEFAULT_MAX_S3_CLIENT_RETRIES, luceneConfig.getMaxS3ClientRetries()); @@ -144,7 +141,7 @@ public void testMaxS3ClientRetries_set() { @Test public void testLiveSettingsOverride_default() { - String config = "nodeName: \"lucene_server_foo\""; + String config = "nodeName: \"server_foo\""; NrtsearchConfig luceneConfig = getForConfig(config); assertEquals( IndexLiveSettings.newBuilder().build(), luceneConfig.getLiveSettingsOverride("test_index")); @@ -182,7 +179,7 @@ public void testLiveSettingsOverride_set() { @Test public void testLowPriorityCopyPercentage_default() { - String config = "nodeName: \"lucene_server_foo\""; + String config = "nodeName: \"server_foo\""; NrtsearchConfig luceneConfig = getForConfig(config); assertEquals(0, luceneConfig.getLowPriorityCopyPercentage()); } @@ -196,7 +193,7 @@ public void testLowPriorityCopyPercentage_set() { @Test public void testVerifyReplicationIndexId_default() { - String config = "nodeName: \"lucene_server_foo\""; + String config = "nodeName: \"server_foo\""; NrtsearchConfig luceneConfig = getForConfig(config); assertTrue(luceneConfig.getVerifyReplicationIndexId()); } @@ -210,7 +207,7 @@ public void testVerifyReplicationIndexId_set() { @Test public void testMMapGrouping_default() { - String config = "nodeName: \"lucene_server_foo\""; + String config = "nodeName: \"server_foo\""; NrtsearchConfig luceneConfig = getForConfig(config); assertEquals(DirectoryFactory.MMapGrouping.SEGMENT, luceneConfig.getMMapGrouping()); } diff --git a/src/test/java/com/yelp/nrtsearch/server/config/QueryCacheConfigTest.java b/src/test/java/com/yelp/nrtsearch/server/config/QueryCacheConfigTest.java index cf2fa3916..2bc3cb2a3 100644 --- a/src/test/java/com/yelp/nrtsearch/server/config/QueryCacheConfigTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/config/QueryCacheConfigTest.java @@ -35,7 +35,7 @@ private static QueryCacheConfig getConfig(String configFile) { @Test public void testDefault() { - String configFile = "nodeName: \"lucene_server_foo\""; + String configFile = "nodeName: \"server_foo\""; QueryCacheConfig config = getConfig(configFile); assertTrue(config.getEnabled()); assertEquals(config.getMaxQueries(), QueryCacheConfig.DEFAULT_MAX_QUERIES); @@ -54,7 +54,7 @@ public void testSetConfig() { String configFile = String.join( "\n", - "nodeName: \"lucene_server_foo\"", + "nodeName: \"server_foo\"", "queryCache:", " enabled: false", " maxQueries: 5000", diff --git a/src/test/java/com/yelp/nrtsearch/server/config/ScriptCacheConfigTest.java b/src/test/java/com/yelp/nrtsearch/server/config/ScriptCacheConfigTest.java index cb9a2fa2a..d05dc4496 100644 --- a/src/test/java/com/yelp/nrtsearch/server/config/ScriptCacheConfigTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/config/ScriptCacheConfigTest.java @@ -31,7 +31,7 @@ private static ScriptCacheConfig getConfig(String configFile) { @Test public void testDefault() { - String configFile = "nodeName: \"lucene_server_foo\""; + String configFile = "nodeName: \"server_foo\""; ScriptCacheConfig config = getConfig(configFile); assertEquals(ScriptCacheConfig.DEFAULT_CONCURRENCY_LEVEL, config.getConcurrencyLevel()); assertEquals(ScriptCacheConfig.DEFAULT_SIZE, config.getMaximumSize()); @@ -44,7 +44,7 @@ public void testValidConfig() { String configFile = String.join( "\n", - "nodeName: \"lucene_server_foo\"", + "nodeName: \"server_foo\"", "ScriptCacheConfig:", " concurrencyLevel: 2", " maximumSize: 2000", @@ -60,8 +60,7 @@ public void testValidConfig() { @Test public void testInvalidTimeUnitConfig() { String configFile = - String.join( - "\n", "nodeName: \"lucene_server_foo\"", "ScriptCacheConfig:", " timeUnit: \"ABCD\""); + String.join("\n", "nodeName: \"server_foo\"", "ScriptCacheConfig:", " timeUnit: \"ABCD\""); IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> getConfig(configFile)); assertEquals(exception.getMessage(), "No enum constant java.util.concurrent.TimeUnit.ABCD"); diff --git a/src/test/java/com/yelp/nrtsearch/server/config/StateConfigTest.java b/src/test/java/com/yelp/nrtsearch/server/config/StateConfigTest.java index 680886fe7..9fef5cbeb 100644 --- a/src/test/java/com/yelp/nrtsearch/server/config/StateConfigTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/config/StateConfigTest.java @@ -31,7 +31,7 @@ private static StateConfig getConfig(String configFile) { @Test public void testDefaultConfig() { - String configFile = "nodeName: \"lucene_server_foo\""; + String configFile = "nodeName: \"server_foo\""; StateConfig stateConfig = getConfig(configFile); assertEquals(StateBackendType.LOCAL, stateConfig.getBackendType()); } diff --git a/src/test/java/com/yelp/nrtsearch/server/config/ThreadPoolConfigurationTest.java b/src/test/java/com/yelp/nrtsearch/server/config/ThreadPoolConfigurationTest.java index 2420bb9ae..1ff151768 100644 --- a/src/test/java/com/yelp/nrtsearch/server/config/ThreadPoolConfigurationTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/config/ThreadPoolConfigurationTest.java @@ -38,11 +38,11 @@ private int cpus() { public void testConfiguration() throws FileNotFoundException { String config = Paths.get("src", "test", "resources", "config.yaml").toAbsolutePath().toString(); - NrtsearchConfig luceneServerConfiguration = new NrtsearchConfig(new FileInputStream(config)); - assertEquals("lucene_server_foo", luceneServerConfiguration.getNodeName()); - assertEquals("foohost", luceneServerConfiguration.getHostName()); + NrtsearchConfig configuration = new NrtsearchConfig(new FileInputStream(config)); + assertEquals("server_foo", configuration.getNodeName()); + assertEquals("foohost", configuration.getHostName()); ThreadPoolConfiguration.ThreadPoolSettings threadPoolSettings = - luceneServerConfiguration + configuration .getThreadPoolConfiguration() .getThreadPoolSettings(ExecutorFactory.ExecutorType.SEARCH); assertEquals(threadPoolSettings.maxThreads(), 16); @@ -117,34 +117,34 @@ public void testIndexThreadPool_set() { } @Test - public void testLuceneServerThreadPool_default() { + public void testServerThreadPool_default() { String config = "nodeName: node1"; ThreadPoolConfiguration threadPoolConfiguration = new ThreadPoolConfiguration(getReaderForConfig(config)); ThreadPoolConfiguration.ThreadPoolSettings threadPoolSettings = - threadPoolConfiguration.getThreadPoolSettings(ExecutorFactory.ExecutorType.LUCENESERVER); + threadPoolConfiguration.getThreadPoolSettings(ExecutorFactory.ExecutorType.SERVER); assertEquals( - threadPoolSettings.maxThreads(), ThreadPoolConfiguration.DEFAULT_GRPC_LUCENESERVER_THREADS); + threadPoolSettings.maxThreads(), ThreadPoolConfiguration.DEFAULT_GRPC_SERVER_THREADS); assertEquals( threadPoolSettings.maxBufferedItems(), - ThreadPoolConfiguration.DEFAULT_GRPC_LUCENESERVER_BUFFERED_ITEMS); - assertEquals("GrpcLuceneServerExecutor", threadPoolSettings.threadNamePrefix()); + ThreadPoolConfiguration.DEFAULT_GRPC_SERVER_BUFFERED_ITEMS); + assertEquals("GrpcServerExecutor", threadPoolSettings.threadNamePrefix()); } @Test - public void testLuceneServerThreadPool_set() { + public void testServerThreadPool_set() { String config = String.join( "\n", "threadPoolConfiguration:", - " luceneserver:", + " server:", " maxThreads: 5", " maxBufferedItems: 10", " threadNamePrefix: customName"); ThreadPoolConfiguration threadPoolConfiguration = new ThreadPoolConfiguration(getReaderForConfig(config)); ThreadPoolConfiguration.ThreadPoolSettings threadPoolSettings = - threadPoolConfiguration.getThreadPoolSettings(ExecutorFactory.ExecutorType.LUCENESERVER); + threadPoolConfiguration.getThreadPoolSettings(ExecutorFactory.ExecutorType.SERVER); assertEquals(threadPoolSettings.maxThreads(), 5); assertEquals(threadPoolSettings.maxBufferedItems(), 10); assertEquals("customName", threadPoolSettings.threadNamePrefix()); diff --git a/src/test/java/com/yelp/nrtsearch/server/field/FieldDefCreatorTest.java b/src/test/java/com/yelp/nrtsearch/server/field/FieldDefCreatorTest.java index 5d8c75498..80dda7649 100644 --- a/src/test/java/com/yelp/nrtsearch/server/field/FieldDefCreatorTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/field/FieldDefCreatorTest.java @@ -44,7 +44,7 @@ private void init(List plugins) { } private NrtsearchConfig getEmptyConfig() { - String config = "nodeName: \"lucene_server_foo\""; + String config = "nodeName: \"server_foo\""; return new NrtsearchConfig(new ByteArrayInputStream(config.getBytes())); } diff --git a/src/test/java/com/yelp/nrtsearch/server/grpc/CopyFileTest.java b/src/test/java/com/yelp/nrtsearch/server/grpc/CopyFileTest.java index eeb9966dd..7d6f7663b 100644 --- a/src/test/java/com/yelp/nrtsearch/server/grpc/CopyFileTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/grpc/CopyFileTest.java @@ -110,46 +110,6 @@ public void sendRawFile() throws Exception { ((SendRawFileStreamObserver) responseObserver).getTransferStatus().getMessage()); } - // @Test - // public void recvRawFileOnBlockingClient() throws Exception { - // // Generate a unique in-process server name. - // String serverName = InProcessServerBuilder.generateName(); - // - // // Create a server, add service, start, and register for automatic graceful shutdown. - // grpcCleanup.register(InProcessServerBuilder - // .forName(serverName).directExecutor().addService(new - // LuceneServer.ReplicationServerImpl(null)).build().start()); - // - // ReplicationServerGrpc.ReplicationServerBlockingStub blockingStub = - // ReplicationServerGrpc.newBlockingStub( - // // Create a client channel and register for automatic graceful shutdown. - // - // grpcCleanup.register(InProcessChannelBuilder.forName(serverName).directExecutor().build())); - // ReplicationServerGrpc.ReplicationServerStub stub = ReplicationServerGrpc.newStub( - // - // grpcCleanup.register(InProcessChannelBuilder.forName(serverName).directExecutor().build())); - // - // //create and write to file total_chunk * chunk_size - // File tempFile = folder.newFile("temp"); - // OutputStream outputStream = new FileOutputStream(tempFile); - // Random random = new Random(); - // byte[] chunk = new byte[CHUNK_SIZE]; - // for (int i = 0; i < TOTAL_CHUNKS; i++) { - // random.nextBytes(chunk); - // outputStream.write(chunk); - // } - // outputStream.close(); - // - // Iterator rawFileChunks = - // blockingStub.recvRawFile(FileInfo.newBuilder().setFileName(tempFile.getAbsolutePath()).setFpStart(0).build()); - // long totalBytesReceived = 0; - // while(rawFileChunks.hasNext()){ - // RawFileChunk rawFileChunk = rawFileChunks.next(); - // totalBytesReceived+=rawFileChunk.getContent().size(); - // } - // assertEquals(CHUNK_SIZE * TOTAL_CHUNKS, totalBytesReceived); - // } - static class SendRawFileStreamObserver implements StreamObserver { private final CountDownLatch finishLatch; diff --git a/src/test/java/com/yelp/nrtsearch/server/grpc/CustomFieldTypeTest.java b/src/test/java/com/yelp/nrtsearch/server/grpc/CustomFieldTypeTest.java index b315a13ff..50523947a 100644 --- a/src/test/java/com/yelp/nrtsearch/server/grpc/CustomFieldTypeTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/grpc/CustomFieldTypeTest.java @@ -25,7 +25,7 @@ import com.yelp.nrtsearch.server.field.IndexableFieldDef; import com.yelp.nrtsearch.server.plugins.FieldTypePlugin; import com.yelp.nrtsearch.server.plugins.Plugin; -import com.yelp.nrtsearch.server.utils.LuceneServerTestConfigurationFactory; +import com.yelp.nrtsearch.server.utils.NrtsearchTestConfigurationFactory; import io.grpc.StatusRuntimeException; import io.grpc.testing.GrpcCleanupRule; import io.prometheus.metrics.model.registry.PrometheusRegistry; @@ -81,17 +81,17 @@ public void setUp() throws IOException { private GrpcServer setUpGrpcServer(PrometheusRegistry prometheusRegistry) throws IOException { String testIndex = "test_index"; - NrtsearchConfig luceneServerConfiguration = - LuceneServerTestConfigurationFactory.getConfig(Mode.STANDALONE, folder.getRoot()); + NrtsearchConfig configuration = + NrtsearchTestConfigurationFactory.getConfig(Mode.STANDALONE, folder.getRoot()); return new GrpcServer( prometheusRegistry, grpcCleanup, - luceneServerConfiguration, + configuration, folder, null, - luceneServerConfiguration.getIndexDir(), + configuration.getIndexDir(), testIndex, - luceneServerConfiguration.getPort(), + configuration.getPort(), null, Collections.singletonList(new TestFieldTypePlugin())); } diff --git a/src/test/java/com/yelp/nrtsearch/server/grpc/GrpcServer.java b/src/test/java/com/yelp/nrtsearch/server/grpc/GrpcServer.java index c59d7f296..4db6207d8 100644 --- a/src/test/java/com/yelp/nrtsearch/server/grpc/GrpcServer.java +++ b/src/test/java/com/yelp/nrtsearch/server/grpc/GrpcServer.java @@ -65,7 +65,7 @@ public class GrpcServer { private GlobalState globalState; private NrtsearchConfig configuration; - private Server luceneServer; + private Server server; private ManagedChannel luceneServerManagedChannel; private Server replicationServer; private ManagedChannel replicationServerManagedChannel; @@ -168,10 +168,10 @@ private RemoteBackend getRemoteBackend() { } public void shutdown() { - if (luceneServer != null && luceneServerManagedChannel != null) { - luceneServer.shutdown(); + if (server != null && luceneServerManagedChannel != null) { + server.shutdown(); luceneServerManagedChannel.shutdown(); - luceneServer = null; + server = null; luceneServerManagedChannel = null; } if (replicationServer != null && replicationServerManagedChannel != null) { @@ -183,10 +183,10 @@ public void shutdown() { } public void forceShutdown() { - if (luceneServer != null && luceneServerManagedChannel != null) { - luceneServer.shutdownNow(); + if (server != null && luceneServerManagedChannel != null) { + server.shutdownNow(); luceneServerManagedChannel.shutdownNow(); - luceneServer = null; + server = null; luceneServerManagedChannel = null; } if (replicationServer != null && replicationServerManagedChannel != null) { @@ -247,7 +247,7 @@ private void invoke( // Create a client channel and register for automatic graceful shutdown. LuceneServerStubBuilder stubBuilder = new LuceneServerStubBuilder("localhost", port); grpcCleanup.register(stubBuilder.channel); - luceneServer = server; + this.server = server; luceneServerManagedChannel = stubBuilder.channel; blockingStub = stubBuilder.createBlockingStub(); stub = stubBuilder.createAsyncStub(); @@ -279,7 +279,7 @@ private void invoke( blockingStub = null; stub = null; - luceneServer = null; + this.server = null; luceneServerManagedChannel = null; } } diff --git a/src/test/java/com/yelp/nrtsearch/server/grpc/MergeBehaviorTests.java b/src/test/java/com/yelp/nrtsearch/server/grpc/MergeBehaviorTests.java index fbecdf189..0c2f10f3a 100644 --- a/src/test/java/com/yelp/nrtsearch/server/grpc/MergeBehaviorTests.java +++ b/src/test/java/com/yelp/nrtsearch/server/grpc/MergeBehaviorTests.java @@ -22,7 +22,7 @@ import com.google.common.collect.Sets; import com.yelp.nrtsearch.server.config.NrtsearchConfig; import com.yelp.nrtsearch.server.state.BackendGlobalState; -import com.yelp.nrtsearch.server.utils.LuceneServerTestConfigurationFactory; +import com.yelp.nrtsearch.server.utils.NrtsearchTestConfigurationFactory; import io.grpc.testing.GrpcCleanupRule; import io.prometheus.metrics.model.registry.PrometheusRegistry; import java.io.IOException; @@ -85,17 +85,17 @@ public void setUp() throws IOException { private GrpcServer setUpGrpcServer(PrometheusRegistry prometheusRegistry) throws IOException { String testIndex = "test_index"; - NrtsearchConfig luceneServerConfiguration = - LuceneServerTestConfigurationFactory.getConfig(Mode.STANDALONE, folder.getRoot()); + NrtsearchConfig configuration = + NrtsearchTestConfigurationFactory.getConfig(Mode.STANDALONE, folder.getRoot()); return new GrpcServer( prometheusRegistry, grpcCleanup, - luceneServerConfiguration, + configuration, folder, null, - luceneServerConfiguration.getIndexDir(), + configuration.getIndexDir(), testIndex, - luceneServerConfiguration.getPort(), + configuration.getPort(), null, Collections.emptyList()); } diff --git a/src/test/java/com/yelp/nrtsearch/server/grpc/NrtsearchServerIdFieldTest.java b/src/test/java/com/yelp/nrtsearch/server/grpc/NrtsearchServerIdFieldTest.java index d16df63c9..40315df8b 100644 --- a/src/test/java/com/yelp/nrtsearch/server/grpc/NrtsearchServerIdFieldTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/grpc/NrtsearchServerIdFieldTest.java @@ -21,7 +21,7 @@ import static org.junit.Assert.assertTrue; import com.yelp.nrtsearch.server.config.NrtsearchConfig; -import com.yelp.nrtsearch.server.utils.LuceneServerTestConfigurationFactory; +import com.yelp.nrtsearch.server.utils.NrtsearchTestConfigurationFactory; import io.grpc.StatusRuntimeException; import io.grpc.testing.GrpcCleanupRule; import io.prometheus.metrics.model.registry.PrometheusRegistry; @@ -75,17 +75,17 @@ public void setUp() throws IOException { private GrpcServer setUpGrpcServer(PrometheusRegistry prometheusRegistry) throws IOException { String testIndex = "test_index"; - NrtsearchConfig luceneServerConfiguration = - LuceneServerTestConfigurationFactory.getConfig(Mode.STANDALONE, folder.getRoot()); + NrtsearchConfig configuration = + NrtsearchTestConfigurationFactory.getConfig(Mode.STANDALONE, folder.getRoot()); return new GrpcServer( prometheusRegistry, grpcCleanup, - luceneServerConfiguration, + configuration, folder, null, - luceneServerConfiguration.getIndexDir(), + configuration.getIndexDir(), testIndex, - luceneServerConfiguration.getPort(), + configuration.getPort(), null, Collections.emptyList()); } diff --git a/src/test/java/com/yelp/nrtsearch/server/grpc/NrtsearchServerTest.java b/src/test/java/com/yelp/nrtsearch/server/grpc/NrtsearchServerTest.java index 242145b4d..f6f165e29 100644 --- a/src/test/java/com/yelp/nrtsearch/server/grpc/NrtsearchServerTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/grpc/NrtsearchServerTest.java @@ -34,7 +34,7 @@ import com.yelp.nrtsearch.server.remote.s3.S3Backend; import com.yelp.nrtsearch.server.search.cache.NrtQueryCache; import com.yelp.nrtsearch.server.state.StateUtils; -import com.yelp.nrtsearch.server.utils.LuceneServerTestConfigurationFactory; +import com.yelp.nrtsearch.server.utils.NrtsearchTestConfigurationFactory; import com.yelp.nrtsearch.test_utils.AmazonS3Provider; import io.grpc.StatusRuntimeException; import io.grpc.testing.GrpcCleanupRule; @@ -99,7 +99,7 @@ public class NrtsearchServerTest { Arrays.asList( "doc_id", "vendor_name", "vendor_name_atom", "license_no", "lat_lon", "lat_lon_multi"); - private final String bucketName = "lucene-server-unittest"; + private final String bucketName = "server-unittest"; /** * This rule manages automatic graceful shutdown for the registered servers and channels at the @@ -141,13 +141,13 @@ private void tearDownReplicaGrpcServer() throws IOException { @Before public void setUp() throws IOException { - NrtsearchConfig luceneServerConfiguration = - LuceneServerTestConfigurationFactory.getConfig( + NrtsearchConfig configuration = + NrtsearchTestConfigurationFactory.getConfig( Mode.STANDALONE, folder.getRoot(), "bucketName: " + bucketName); prometheusRegistry = new PrometheusRegistry(); - remoteBackend = setUpRemoteBackend(luceneServerConfiguration); - grpcServer = setUpGrpcServer(luceneServerConfiguration, prometheusRegistry); + remoteBackend = setUpRemoteBackend(configuration); + grpcServer = setUpGrpcServer(configuration, prometheusRegistry); replicaGrpcServer = setUpReplicaGrpcServer(prometheusRegistry); setUpWarmer(); } @@ -178,19 +178,18 @@ private List getTestSearchRequestsAsJsonStrings() { } private GrpcServer setUpGrpcServer( - NrtsearchConfig luceneServerConfiguration, PrometheusRegistry prometheusRegistry) - throws IOException { + NrtsearchConfig configuration, PrometheusRegistry prometheusRegistry) throws IOException { String testIndex = "test_index"; return new GrpcServer( prometheusRegistry, grpcCleanup, - luceneServerConfiguration, + configuration, folder, null, - luceneServerConfiguration.getIndexDir(), + configuration.getIndexDir(), testIndex, - luceneServerConfiguration.getPort(), + configuration.getPort(), remoteBackend, Collections.emptyList()); } @@ -198,18 +197,18 @@ private GrpcServer setUpGrpcServer( private GrpcServer setUpReplicaGrpcServer(PrometheusRegistry prometheusRegistry) throws IOException { String testIndex = "test_index"; - NrtsearchConfig luceneServerReplicaConfiguration = - LuceneServerTestConfigurationFactory.getConfig( + NrtsearchConfig replicaConfiguration = + NrtsearchTestConfigurationFactory.getConfig( Mode.REPLICA, folder.getRoot(), getExtraConfig()); return new GrpcServer( grpcCleanup, - luceneServerReplicaConfiguration, + replicaConfiguration, folder, null, - luceneServerReplicaConfiguration.getIndexDir(), + replicaConfiguration.getIndexDir(), testIndex, - luceneServerReplicaConfiguration.getPort(), + replicaConfiguration.getPort(), remoteBackend); } diff --git a/src/test/java/com/yelp/nrtsearch/server/grpc/QueryTest.java b/src/test/java/com/yelp/nrtsearch/server/grpc/QueryTest.java index c2a63aef6..429c13927 100644 --- a/src/test/java/com/yelp/nrtsearch/server/grpc/QueryTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/grpc/QueryTest.java @@ -21,7 +21,7 @@ import com.google.protobuf.Any; import com.google.protobuf.InvalidProtocolBufferException; import com.yelp.nrtsearch.server.config.NrtsearchConfig; -import com.yelp.nrtsearch.server.utils.LuceneServerTestConfigurationFactory; +import com.yelp.nrtsearch.server.utils.NrtsearchTestConfigurationFactory; import io.grpc.testing.GrpcCleanupRule; import java.io.IOException; import java.nio.file.Paths; @@ -76,16 +76,16 @@ public void setUp() throws Exception { private GrpcServer setUpGrpcServer() throws IOException { String testIndex = "test_index"; - NrtsearchConfig luceneServerConfiguration = - LuceneServerTestConfigurationFactory.getConfig(Mode.STANDALONE, folder.getRoot()); + NrtsearchConfig configuration = + NrtsearchTestConfigurationFactory.getConfig(Mode.STANDALONE, folder.getRoot()); return new GrpcServer( grpcCleanup, - luceneServerConfiguration, + configuration, folder, null, - luceneServerConfiguration.getIndexDir(), + configuration.getIndexDir(), testIndex, - luceneServerConfiguration.getPort()); + configuration.getPort()); } @Test diff --git a/src/test/java/com/yelp/nrtsearch/server/grpc/ReplicationServerTest.java b/src/test/java/com/yelp/nrtsearch/server/grpc/ReplicationServerTest.java index 877cf06b7..b63fc180e 100644 --- a/src/test/java/com/yelp/nrtsearch/server/grpc/ReplicationServerTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/grpc/ReplicationServerTest.java @@ -54,7 +54,7 @@ public void cleanup() { @Test public void recvCopyState() throws IOException, InterruptedException { - initDefaultLuceneServer(); + initDefaultServer(); primaryServer.addSimpleDocs("test_index", 1, 2); primaryServer.refresh("test_index"); @@ -144,7 +144,7 @@ public void copyFiles() throws IOException, InterruptedException { @Test public void basicReplication() throws IOException, InterruptedException { - initDefaultLuceneServer(); + initDefaultServer(); // index 2 documents to primary primaryServer.addSimpleDocs("test_index", 1, 2); @@ -166,7 +166,7 @@ public void basicReplication() throws IOException, InterruptedException { @Test public void getConnectedNodes() throws IOException, InterruptedException { - initDefaultLuceneServer(); + initDefaultServer(); // primary should have registered replica in its connected nodes list GetNodesResponse getNodesResponse = @@ -255,7 +255,7 @@ public void testInitialSyncMaxTime() throws IOException, InterruptedException { @Test public void testInitialSyncTimeout() throws IOException { - initLuceneServers("initialSyncPrimaryWaitMs: 1000"); + initServers("initialSyncPrimaryWaitMs: 1000"); primaryServer.stopIndex("test_index"); @@ -311,7 +311,7 @@ public void testInitialSyncWithCurrentVersion() throws IOException, InterruptedE @Test public void testAddDocumentsOnReplicaFailure() throws IOException, InterruptedException { - initDefaultLuceneServer(); + initDefaultServer(); try { replicaServer.addSimpleDocs("test_index", 1, 2); @@ -323,15 +323,15 @@ public void testAddDocumentsOnReplicaFailure() throws IOException, InterruptedEx } } - private void initDefaultLuceneServer() throws IOException { - initLuceneServers(""); + private void initDefaultServer() throws IOException { + initServers(""); } private void initServerSyncInitialNrtPointFalse() throws IOException { - initLuceneServers("syncInitialNrtPoint: false"); + initServers("syncInitialNrtPoint: false"); } - private void initLuceneServers(String extraConfig) throws IOException { + private void initServers(String extraConfig) throws IOException { primaryServer = TestServer.builder(folder) .withAutoStartConfig( diff --git a/src/test/java/com/yelp/nrtsearch/server/index/BackendStateManagerTest.java b/src/test/java/com/yelp/nrtsearch/server/index/BackendStateManagerTest.java index 9ce75f43b..40b938ce4 100644 --- a/src/test/java/com/yelp/nrtsearch/server/index/BackendStateManagerTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/index/BackendStateManagerTest.java @@ -67,7 +67,7 @@ public class BackendStateManagerTest { @BeforeClass public static void setup() { - String configFile = "nodeName: \"lucene_server_foo\""; + String configFile = "nodeName: \"server_foo\""; NrtsearchConfig dummyConfig = new NrtsearchConfig(new ByteArrayInputStream(configFile.getBytes())); List dummyPlugins = Collections.emptyList(); diff --git a/src/test/java/com/yelp/nrtsearch/server/index/DirectoryFactoryTest.java b/src/test/java/com/yelp/nrtsearch/server/index/DirectoryFactoryTest.java index 8dcbd834e..75f993f01 100644 --- a/src/test/java/com/yelp/nrtsearch/server/index/DirectoryFactoryTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/index/DirectoryFactoryTest.java @@ -42,17 +42,15 @@ public class DirectoryFactoryTest { private static final DirectoryFactory mmFactory = DirectoryFactory.get( "MMapDirectory", - new NrtsearchConfig( - new ByteArrayInputStream("nodeName: \"lucene_server_foo\"".getBytes()))); + new NrtsearchConfig(new ByteArrayInputStream("nodeName: \"server_foo\"".getBytes()))); private static final DirectoryFactory fsFactory = DirectoryFactory.get( "FSDirectory", - new NrtsearchConfig( - new ByteArrayInputStream("nodeName: \"lucene_server_foo\"".getBytes()))); + new NrtsearchConfig(new ByteArrayInputStream("nodeName: \"server_foo\"".getBytes()))); @Test public void testMMapDefault() throws IOException { - String configFile = "nodeName: \"lucene_server_foo\""; + String configFile = "nodeName: \"server_foo\""; IndexPreloadConfig config = IndexPreloadConfig.fromConfig( new YamlConfigReader(new ByteArrayInputStream(configFile.getBytes()))); @@ -93,7 +91,7 @@ public void testMMapPreloadAll() throws IOException { @Test public void testFSDefault() throws IOException { - String configFile = "nodeName: \"lucene_server_foo\""; + String configFile = "nodeName: \"server_foo\""; IndexPreloadConfig config = IndexPreloadConfig.fromConfig( new YamlConfigReader(new ByteArrayInputStream(configFile.getBytes()))); diff --git a/src/test/java/com/yelp/nrtsearch/server/index/FieldUpdateUtilsTest.java b/src/test/java/com/yelp/nrtsearch/server/index/FieldUpdateUtilsTest.java index a31a75424..3ccb3ea3b 100644 --- a/src/test/java/com/yelp/nrtsearch/server/index/FieldUpdateUtilsTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/index/FieldUpdateUtilsTest.java @@ -84,7 +84,7 @@ public class FieldUpdateUtilsTest { @BeforeClass public static void setup() { - String configFile = "nodeName: \"lucene_server_foo\""; + String configFile = "nodeName: \"server_foo\""; NrtsearchConfig dummyConfig = new NrtsearchConfig(new ByteArrayInputStream(configFile.getBytes())); List dummyPlugins = Collections.emptyList(); diff --git a/src/test/java/com/yelp/nrtsearch/server/index/ImmutableIndexStateTest.java b/src/test/java/com/yelp/nrtsearch/server/index/ImmutableIndexStateTest.java index 152f8c96c..61bc48005 100644 --- a/src/test/java/com/yelp/nrtsearch/server/index/ImmutableIndexStateTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/index/ImmutableIndexStateTest.java @@ -87,7 +87,7 @@ public class ImmutableIndexStateTest { @BeforeClass public static void setup() { - String configFile = "nodeName: \"lucene_server_foo\""; + String configFile = "nodeName: \"server_foo\""; NrtsearchConfig dummyConfig = new NrtsearchConfig(new ByteArrayInputStream(configFile.getBytes())); List dummyPlugins = Collections.emptyList(); @@ -129,7 +129,7 @@ private ImmutableIndexState getIndexState( IndexStateManager mockManager = mock(IndexStateManager.class); GlobalState mockGlobalState = mock(GlobalState.class); - String configFile = "nodeName: \"lucene_server_foo\""; + String configFile = "nodeName: \"server_foo\""; NrtsearchConfig dummyConfig = new NrtsearchConfig(new ByteArrayInputStream(configFile.getBytes())); @@ -947,7 +947,7 @@ private ImmutableIndexState getIndexStateForLiveSettingsOverrides( IndexStateManager mockManager = mock(IndexStateManager.class); GlobalState mockGlobalState = mock(GlobalState.class); - String configFile = "nodeName: \"lucene_server_foo\""; + String configFile = "nodeName: \"server_foo\""; NrtsearchConfig dummyConfig = new NrtsearchConfig(new ByteArrayInputStream(configFile.getBytes())); diff --git a/src/test/java/com/yelp/nrtsearch/server/logging/HitsLoggerCreatorTest.java b/src/test/java/com/yelp/nrtsearch/server/logging/HitsLoggerCreatorTest.java index fcf00243b..7c9e7613b 100644 --- a/src/test/java/com/yelp/nrtsearch/server/logging/HitsLoggerCreatorTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/logging/HitsLoggerCreatorTest.java @@ -45,7 +45,7 @@ private void init(List plugins) { } private NrtsearchConfig getEmptyConfig() { - String config = "nodeName: \"lucene_server_foo\""; + String config = "nodeName: \"server_foo\""; return new NrtsearchConfig(new ByteArrayInputStream(config.getBytes())); } diff --git a/src/test/java/com/yelp/nrtsearch/server/plugins/PluginsServiceTest.java b/src/test/java/com/yelp/nrtsearch/server/plugins/PluginsServiceTest.java index 5ba5280dd..f97ff39fa 100644 --- a/src/test/java/com/yelp/nrtsearch/server/plugins/PluginsServiceTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/plugins/PluginsServiceTest.java @@ -37,7 +37,7 @@ public class PluginsServiceTest { @Rule public TemporaryFolder folder = new TemporaryFolder(); private NrtsearchConfig getEmptyConfig() { - String config = "nodeName: \"lucene_server_foo\""; + String config = "nodeName: \"server_foo\""; return new NrtsearchConfig(new ByteArrayInputStream(config.getBytes())); } diff --git a/src/test/java/com/yelp/nrtsearch/server/query/QueryNodeMapperTest.java b/src/test/java/com/yelp/nrtsearch/server/query/QueryNodeMapperTest.java index 612739f6a..2039fa488 100644 --- a/src/test/java/com/yelp/nrtsearch/server/query/QueryNodeMapperTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/query/QueryNodeMapperTest.java @@ -61,7 +61,7 @@ private void init(List plugins) { } private NrtsearchConfig getEmptyConfig() { - String config = "nodeName: \"lucene_server_foo\""; + String config = "nodeName: \"server_foo\""; return new NrtsearchConfig(new ByteArrayInputStream(config.getBytes())); } diff --git a/src/test/java/com/yelp/nrtsearch/server/rescore/RescorerCreatorTest.java b/src/test/java/com/yelp/nrtsearch/server/rescore/RescorerCreatorTest.java index 8881b7ab5..a6aee35ac 100644 --- a/src/test/java/com/yelp/nrtsearch/server/rescore/RescorerCreatorTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/rescore/RescorerCreatorTest.java @@ -46,7 +46,7 @@ private void init(List plugins) { } private NrtsearchConfig getEmptyConfig() { - String config = "nodeName: \"lucene_server_foo\""; + String config = "nodeName: \"server_foo\""; return new NrtsearchConfig(new ByteArrayInputStream(config.getBytes())); } diff --git a/src/test/java/com/yelp/nrtsearch/server/script/ScoreScriptTest.java b/src/test/java/com/yelp/nrtsearch/server/script/ScoreScriptTest.java index f9f8fec12..d1398d9e9 100644 --- a/src/test/java/com/yelp/nrtsearch/server/script/ScoreScriptTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/script/ScoreScriptTest.java @@ -41,7 +41,7 @@ import com.yelp.nrtsearch.server.grpc.VirtualField; import com.yelp.nrtsearch.server.plugins.Plugin; import com.yelp.nrtsearch.server.plugins.ScriptPlugin; -import com.yelp.nrtsearch.server.utils.LuceneServerTestConfigurationFactory; +import com.yelp.nrtsearch.server.utils.NrtsearchTestConfigurationFactory; import io.grpc.testing.GrpcCleanupRule; import io.prometheus.metrics.model.registry.PrometheusRegistry; import java.io.IOException; @@ -98,17 +98,17 @@ public void setUp() throws IOException { private GrpcServer setUpGrpcServer(PrometheusRegistry prometheusRegistry) throws IOException { String testIndex = "test_index"; - NrtsearchConfig luceneServerConfiguration = - LuceneServerTestConfigurationFactory.getConfig(Mode.STANDALONE, folder.getRoot()); + NrtsearchConfig configuration = + NrtsearchTestConfigurationFactory.getConfig(Mode.STANDALONE, folder.getRoot()); return new GrpcServer( prometheusRegistry, grpcCleanup, - luceneServerConfiguration, + configuration, folder, null, - luceneServerConfiguration.getIndexDir(), + configuration.getIndexDir(), testIndex, - luceneServerConfiguration.getPort(), + configuration.getPort(), null, Collections.singletonList(new ScoreScriptTestPlugin())); } diff --git a/src/test/java/com/yelp/nrtsearch/server/script/ScriptServiceTest.java b/src/test/java/com/yelp/nrtsearch/server/script/ScriptServiceTest.java index 97358bfc9..490b2a44b 100644 --- a/src/test/java/com/yelp/nrtsearch/server/script/ScriptServiceTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/script/ScriptServiceTest.java @@ -45,7 +45,7 @@ private void init(List plugins) { } private NrtsearchConfig getEmptyConfig() { - String config = "nodeName: \"lucene_server_foo\""; + String config = "nodeName: \"server_foo\""; return new NrtsearchConfig(new ByteArrayInputStream(config.getBytes())); } diff --git a/src/test/java/com/yelp/nrtsearch/server/search/FetchTaskCreatorTest.java b/src/test/java/com/yelp/nrtsearch/server/search/FetchTaskCreatorTest.java index d4e21a2fe..14d0746ed 100644 --- a/src/test/java/com/yelp/nrtsearch/server/search/FetchTaskCreatorTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/search/FetchTaskCreatorTest.java @@ -43,7 +43,7 @@ private void init(List plugins) { } private NrtsearchConfig getEmptyConfig() { - String config = "nodeName: \"lucene_server_foo\""; + String config = "nodeName: \"server_foo\""; return new NrtsearchConfig(new ByteArrayInputStream(config.getBytes())); } diff --git a/src/test/java/com/yelp/nrtsearch/server/search/collectors/CollectorCreatorTest.java b/src/test/java/com/yelp/nrtsearch/server/search/collectors/CollectorCreatorTest.java index 3ed0c41c9..b7d4bff0a 100644 --- a/src/test/java/com/yelp/nrtsearch/server/search/collectors/CollectorCreatorTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/search/collectors/CollectorCreatorTest.java @@ -58,7 +58,7 @@ private void init(List plugins) { } private NrtsearchConfig getEmptyConfig() { - String config = "nodeName: \"lucene_server_foo\""; + String config = "nodeName: \"server_foo\""; return new NrtsearchConfig(new ByteArrayInputStream(config.getBytes())); } diff --git a/src/test/java/com/yelp/nrtsearch/server/similarity/SimilarityCreatorTest.java b/src/test/java/com/yelp/nrtsearch/server/similarity/SimilarityCreatorTest.java index e5c3a381f..3692b53ca 100644 --- a/src/test/java/com/yelp/nrtsearch/server/similarity/SimilarityCreatorTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/similarity/SimilarityCreatorTest.java @@ -48,7 +48,7 @@ private void init(List plugins) { } private NrtsearchConfig getEmptyConfig() { - String config = "nodeName: \"lucene_server_foo\""; + String config = "nodeName: \"server_foo\""; return new NrtsearchConfig(new ByteArrayInputStream(config.getBytes())); } diff --git a/src/test/java/com/yelp/nrtsearch/server/state/BackendGlobalStateTest.java b/src/test/java/com/yelp/nrtsearch/server/state/BackendGlobalStateTest.java index bc3cc4535..e5eeb019c 100644 --- a/src/test/java/com/yelp/nrtsearch/server/state/BackendGlobalStateTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/state/BackendGlobalStateTest.java @@ -84,7 +84,7 @@ public class BackendGlobalStateTest { @BeforeClass public static void setup() { - String configFile = "nodeName: \"lucene_server_foo\""; + String configFile = "nodeName: \"server_foo\""; NrtsearchConfig dummyConfig = new NrtsearchConfig(new ByteArrayInputStream(configFile.getBytes())); List dummyPlugins = Collections.emptyList(); @@ -109,11 +109,11 @@ static class MockBackendGlobalState extends BackendGlobalState { /** * Constructor. * - * @param luceneServerConfiguration server config + * @param configuration server config * @throws IOException on filesystem error */ - public MockBackendGlobalState(NrtsearchConfig luceneServerConfiguration) throws IOException { - super(luceneServerConfiguration, null); + public MockBackendGlobalState(NrtsearchConfig configuration) throws IOException { + super(configuration, null); } @Override diff --git a/src/test/java/com/yelp/nrtsearch/server/utils/LuceneServerTestConfigurationFactory.java b/src/test/java/com/yelp/nrtsearch/server/utils/NrtsearchTestConfigurationFactory.java similarity index 98% rename from src/test/java/com/yelp/nrtsearch/server/utils/LuceneServerTestConfigurationFactory.java rename to src/test/java/com/yelp/nrtsearch/server/utils/NrtsearchTestConfigurationFactory.java index 7156f845e..138083b4c 100644 --- a/src/test/java/com/yelp/nrtsearch/server/utils/LuceneServerTestConfigurationFactory.java +++ b/src/test/java/com/yelp/nrtsearch/server/utils/NrtsearchTestConfigurationFactory.java @@ -22,7 +22,7 @@ import java.nio.file.Paths; import java.util.concurrent.atomic.AtomicLong; -public class LuceneServerTestConfigurationFactory { +public class NrtsearchTestConfigurationFactory { static AtomicLong atomicLong = new AtomicLong(); public static NrtsearchConfig getConfig(Mode mode, File dataRootDir) { diff --git a/src/test/java/com/yelp/nrtsearch/test_utils/NrtsearchTest.java b/src/test/java/com/yelp/nrtsearch/test_utils/NrtsearchTest.java index c8c3c3bc4..730303b5c 100644 --- a/src/test/java/com/yelp/nrtsearch/test_utils/NrtsearchTest.java +++ b/src/test/java/com/yelp/nrtsearch/test_utils/NrtsearchTest.java @@ -54,8 +54,8 @@ public class NrtsearchTest { private final Path pluginSearchPath = TEMPORARY_FOLDER.newFolder("plugin_search_path").toPath(); @Rule - public final TestLuceneServer testLuceneServer = - new TestLuceneServer(getConfig(), S3_PROVIDER.getAmazonS3()); + public final TestNrtsearchServer testServer = + new TestNrtsearchServer(getConfig(), S3_PROVIDER.getAmazonS3()); public NrtsearchTest() throws IOException {} @@ -121,7 +121,7 @@ private NrtsearchConfig getConfig() { * @return {@link NrtsearchClient} */ protected NrtsearchClient getClient() { - return testLuceneServer.getClient(); + return testServer.getClient(); } /** diff --git a/src/test/java/com/yelp/nrtsearch/test_utils/TestLuceneServer.java b/src/test/java/com/yelp/nrtsearch/test_utils/TestNrtsearchServer.java similarity index 70% rename from src/test/java/com/yelp/nrtsearch/test_utils/TestLuceneServer.java rename to src/test/java/com/yelp/nrtsearch/test_utils/TestNrtsearchServer.java index 42fc1946c..66d23f1bb 100644 --- a/src/test/java/com/yelp/nrtsearch/test_utils/TestLuceneServer.java +++ b/src/test/java/com/yelp/nrtsearch/test_utils/TestNrtsearchServer.java @@ -29,29 +29,29 @@ * A JUnit {@link org.junit.Rule} that creates a {@link NrtsearchServer} instance and starts the * grpc server from it. After the test it will delete the index data and stop the server. */ -public class TestLuceneServer extends ExternalResource { +public class TestNrtsearchServer extends ExternalResource { - private final NrtsearchConfig luceneServerConfiguration; + private final NrtsearchConfig configuration; private final AmazonS3 amazonS3; - private NrtsearchServer luceneServer; + private NrtsearchServer server; private NrtsearchClient client; - public TestLuceneServer(NrtsearchConfig luceneServerConfiguration, AmazonS3 amazonS3) { - this.luceneServerConfiguration = luceneServerConfiguration; + public TestNrtsearchServer(NrtsearchConfig configuration, AmazonS3 amazonS3) { + this.configuration = configuration; this.amazonS3 = amazonS3; } @Override protected void before() throws Throwable { - this.luceneServer = createTestServer(); - this.client = new NrtsearchClient("localhost", luceneServerConfiguration.getPort()); + this.server = createTestServer(); + this.client = new NrtsearchClient("localhost", configuration.getPort()); } @Override protected void after() { deleteIndexData(); client.close(); - luceneServer.stop(); + server.stop(); } protected NrtsearchClient getClient() { @@ -59,11 +59,10 @@ protected NrtsearchClient getClient() { } private NrtsearchServer createTestServer() throws IOException { - Injector injector = - Guice.createInjector(new TestNrtsearchModule(luceneServerConfiguration, amazonS3)); - NrtsearchServer luceneServer = injector.getInstance(NrtsearchServer.class); - luceneServer.start(); - return luceneServer; + Injector injector = Guice.createInjector(new TestNrtsearchModule(configuration, amazonS3)); + NrtsearchServer server = injector.getInstance(NrtsearchServer.class); + server.start(); + return server; } private void deleteIndexData() { diff --git a/src/test/java/com/yelp/nrtsearch/tools/cli/ReadyCommandTest.java b/src/test/java/com/yelp/nrtsearch/tools/cli/ReadyCommandTest.java index d9067fa37..a7259a298 100644 --- a/src/test/java/com/yelp/nrtsearch/tools/cli/ReadyCommandTest.java +++ b/src/test/java/com/yelp/nrtsearch/tools/cli/ReadyCommandTest.java @@ -22,7 +22,7 @@ import com.yelp.nrtsearch.server.grpc.GrpcServer; import com.yelp.nrtsearch.server.grpc.Mode; import com.yelp.nrtsearch.server.grpc.StartIndexRequest; -import com.yelp.nrtsearch.server.utils.LuceneServerTestConfigurationFactory; +import com.yelp.nrtsearch.server.utils.NrtsearchTestConfigurationFactory; import io.grpc.testing.GrpcCleanupRule; import java.io.IOException; import org.junit.After; @@ -48,17 +48,17 @@ public void cleanup() { private GrpcServer server; private void startServer() throws IOException { - NrtsearchConfig luceneServerPrimaryConfiguration = - LuceneServerTestConfigurationFactory.getConfig(Mode.PRIMARY, folder.getRoot()); + NrtsearchConfig configuration = + NrtsearchTestConfigurationFactory.getConfig(Mode.PRIMARY, folder.getRoot()); server = new GrpcServer( grpcCleanup, - luceneServerPrimaryConfiguration, + configuration, folder, null, - luceneServerPrimaryConfiguration.getIndexDir(), + configuration.getIndexDir(), "test_index", - luceneServerPrimaryConfiguration.getPort(), + configuration.getPort(), null); } diff --git a/src/test/java/com/yelp/nrtsearch/yelp_reviews/YelpReviewsTest.java b/src/test/java/com/yelp/nrtsearch/yelp_reviews/YelpReviewsTest.java index e24376444..c4080bb59 100644 --- a/src/test/java/com/yelp/nrtsearch/yelp_reviews/YelpReviewsTest.java +++ b/src/test/java/com/yelp/nrtsearch/yelp_reviews/YelpReviewsTest.java @@ -72,7 +72,7 @@ public class YelpReviewsTest { private static final Logger logger = LoggerFactory.getLogger(YelpReviewsTest.class.getName()); - public static final String LUCENE_SERVER_CONFIGURATION_YAML = "lucene_server_configuration.yaml"; + public static final String NRTSEARCH_CONFIG_YAML = "nrtsearch_config.yaml"; public static final String INDEX_NAME = "yelp_reviews_test_0"; public static final String CLIENT_LOG = "client.log"; public static final String SERVER_LOG = "server.log"; @@ -249,14 +249,12 @@ public static void main(String[] args) throws IOException, InterruptedException logger.info("Temporary directory: {}", yelp_reviews_test_base_path); Process primaryServerProcess = - startServer( - primaryDir.resolve(SERVER_LOG).toString(), getLuceneServerPrimaryConfigurationYaml()); + startServer(primaryDir.resolve(SERVER_LOG).toString(), getServerPrimaryConfigurationYaml()); Process replicaServerProcess = - startServer( - replicaDir.resolve(SERVER_LOG).toString(), getLuceneServerReplicaConfigurationYaml()); + startServer(replicaDir.resolve(SERVER_LOG).toString(), getServerReplicaConfigurationYaml()); - HostPort primaryHostPort = new HostPort(getLuceneServerPrimaryConfigurationYaml()); - HostPort secondaryHostPort = new HostPort(getLuceneServerReplicaConfigurationYaml()); + HostPort primaryHostPort = new HostPort(getServerPrimaryConfigurationYaml()); + HostPort secondaryHostPort = new HostPort(getServerReplicaConfigurationYaml()); NrtsearchClient primaryServerClient = new NrtsearchClient(primaryHostPort.hostName, primaryHostPort.port); NrtsearchClient secondaryServerClient = @@ -417,7 +415,7 @@ private static void createIndex(NrtsearchClient serverClient) { private static Process startServer(String logFilename, String configFileName) throws IOException { String command = String.format( - "%s/build/install/nrtsearch/bin/lucene-server %s", + "%s/build/install/nrtsearch/bin/nrtsearch_server %s", System.getProperty("user.dir"), configFileName); return issueCommand(logFilename, command); } @@ -459,12 +457,12 @@ private static String getPathAsStr(String resourceName, ServerType serverType) { } } - private static String getLuceneServerPrimaryConfigurationYaml() { - return getPathAsStr(LUCENE_SERVER_CONFIGURATION_YAML, ServerType.primary); + private static String getServerPrimaryConfigurationYaml() { + return getPathAsStr(NRTSEARCH_CONFIG_YAML, ServerType.primary); } - private static String getLuceneServerReplicaConfigurationYaml() { - return getPathAsStr(LUCENE_SERVER_CONFIGURATION_YAML, ServerType.replica); + private static String getServerReplicaConfigurationYaml() { + return getPathAsStr(NRTSEARCH_CONFIG_YAML, ServerType.replica); } private static String readResourceAsString(String resourceName, ServerType serverType) @@ -488,12 +486,11 @@ public String toString() { return sb.toString(); } - HostPort(String confiFileName) throws FileNotFoundException { - NrtsearchConfig luceneServerConfiguration = - new NrtsearchConfig(new FileInputStream(confiFileName)); - this.hostName = luceneServerConfiguration.getHostName(); - this.port = luceneServerConfiguration.getPort(); - this.replicationPort = luceneServerConfiguration.getReplicationPort(); + HostPort(String configFileName) throws FileNotFoundException { + NrtsearchConfig configuration = new NrtsearchConfig(new FileInputStream(configFileName)); + this.hostName = configuration.getHostName(); + this.port = configuration.getPort(); + this.replicationPort = configuration.getReplicationPort(); } } @@ -569,11 +566,11 @@ public AddDocumentRequest buildOneDoc(String line, Gson gson) { private static class SearchTask implements Callable { - private final NrtsearchClient luceneServerClient; + private final NrtsearchClient nrtsearchClient; private final AtomicBoolean indexingDone; - SearchTask(NrtsearchClient luceneServerClient, AtomicBoolean indexingDone) { - this.luceneServerClient = luceneServerClient; + SearchTask(NrtsearchClient nrtsearchClient, AtomicBoolean indexingDone) { + this.nrtsearchClient = nrtsearchClient; this.indexingDone = indexingDone; } @@ -621,8 +618,7 @@ public double getSearchTotalHits(long searcherVersion) { } SearchRequest searchRequest = searchRequestBuilder.build(); long t1 = System.nanoTime(); - SearchResponse searchResponse = - this.luceneServerClient.getBlockingStub().search(searchRequest); + SearchResponse searchResponse = this.nrtsearchClient.getBlockingStub().search(searchRequest); long timeMs = (System.nanoTime() - t1) / (1000 * 1000); long totalHits = searchResponse.getTotalHits().getValue(); String threadId = Thread.currentThread().getName() + Thread.currentThread().getId(); diff --git a/src/test/java/com/yelp/nrtsearch/yelp_reviews/utils/DocumentGeneratorAndIndexer.java b/src/test/java/com/yelp/nrtsearch/yelp_reviews/utils/DocumentGeneratorAndIndexer.java index e18a370de..c306c4a8c 100644 --- a/src/test/java/com/yelp/nrtsearch/yelp_reviews/utils/DocumentGeneratorAndIndexer.java +++ b/src/test/java/com/yelp/nrtsearch/yelp_reviews/utils/DocumentGeneratorAndIndexer.java @@ -27,15 +27,15 @@ public class DocumentGeneratorAndIndexer implements Callable { private final Stream lines; private final Gson gson = new Gson(); - final NrtsearchClient luceneServerClient; + final NrtsearchClient nrtsearchClient; private static final Logger logger = LoggerFactory.getLogger(DocumentGeneratorAndIndexer.class.getName()); private final OneDocBuilder oneDocBuilder; public DocumentGeneratorAndIndexer( - OneDocBuilder oneDocBuilder, Stream lines, NrtsearchClient luceneServerClient) { + OneDocBuilder oneDocBuilder, Stream lines, NrtsearchClient nrtsearchClient) { this.lines = lines; - this.luceneServerClient = luceneServerClient; + this.nrtsearchClient = nrtsearchClient; this.oneDocBuilder = oneDocBuilder; } @@ -62,7 +62,7 @@ public Long call() throws Exception { String.format("threadId: %s took %s milliSecs to buildDocs ", threadId, timeMilliSecs)); t1 = System.nanoTime(); - Long genId = new IndexerTask().index(luceneServerClient, addDocumentRequestStream); + Long genId = new IndexerTask().index(nrtsearchClient, addDocumentRequestStream); t2 = System.nanoTime(); timeMilliSecs = (t2 - t1) / (1000 * 100); logger.info( diff --git a/src/test/java/com/yelp/nrtsearch/yelp_reviews/utils/IndexerTask.java b/src/test/java/com/yelp/nrtsearch/yelp_reviews/utils/IndexerTask.java index 109b7cc76..c5ead15fd 100644 --- a/src/test/java/com/yelp/nrtsearch/yelp_reviews/utils/IndexerTask.java +++ b/src/test/java/com/yelp/nrtsearch/yelp_reviews/utils/IndexerTask.java @@ -30,7 +30,7 @@ public class IndexerTask { private String genId; public Long index( - NrtsearchClient luceneServerClient, Stream addDocumentRequestStream) + NrtsearchClient nrtsearchClient, Stream addDocumentRequestStream) throws Exception { String threadId = Thread.currentThread().getName() + Thread.currentThread().getId(); @@ -68,7 +68,7 @@ public void onCompleted() { // The requestObserver handles the sending of stream of client requests to server (i.e. multiple // onNext and 1 completed) StreamObserver requestObserver = - luceneServerClient.getAsyncStub().addDocuments(responseObserver); + nrtsearchClient.getAsyncStub().addDocuments(responseObserver); try { addDocumentRequestStream.forEach( addDocumentRequest -> requestObserver.onNext(addDocumentRequest)); diff --git a/src/test/java/com/yelp/nrtsearch/yelp_reviews/utils/ParallelDocumentIndexer.java b/src/test/java/com/yelp/nrtsearch/yelp_reviews/utils/ParallelDocumentIndexer.java index 1a2480f9d..1725afe56 100644 --- a/src/test/java/com/yelp/nrtsearch/yelp_reviews/utils/ParallelDocumentIndexer.java +++ b/src/test/java/com/yelp/nrtsearch/yelp_reviews/utils/ParallelDocumentIndexer.java @@ -37,7 +37,7 @@ public static List> buildAndIndexDocs( OneDocBuilder oneDocBuilder, Path path, ExecutorService executorService, - NrtsearchClient luceneServerClient) + NrtsearchClient nrtsearchClient) throws IOException, InterruptedException { try (BufferedReader br = new BufferedReader(new FileReader(path.toFile()))) { String line; @@ -52,7 +52,7 @@ public static List> buildAndIndexDocs( "Launching DocumentGeneratorAndIndexer task for {} docs", DOCS_PER_INDEX_REQUEST); List copiedRawLines = new ArrayList<>(rawLines); Future genIdFuture = - submitTask(oneDocBuilder, executorService, luceneServerClient, copiedRawLines); + submitTask(oneDocBuilder, executorService, nrtsearchClient, copiedRawLines); futures.add(genIdFuture); rawLines.clear(); } @@ -61,7 +61,7 @@ public static List> buildAndIndexDocs( // convert left over docs logger.info("Launching DocumentGeneratorAndIndexer task for {} docs", rawLines.size()); Future genIdFuture = - submitTask(oneDocBuilder, executorService, luceneServerClient, rawLines); + submitTask(oneDocBuilder, executorService, nrtsearchClient, rawLines); futures.add(genIdFuture); } return futures; @@ -71,7 +71,7 @@ public static List> buildAndIndexDocs( private static Future submitTask( OneDocBuilder oneDocBuilder, ExecutorService executorService, - NrtsearchClient luceneServerClient, + NrtsearchClient nrtsearchClient, List rawLines) throws InterruptedException { Future genIdFuture; @@ -79,8 +79,7 @@ private static Future submitTask( try { genIdFuture = executorService.submit( - new DocumentGeneratorAndIndexer( - oneDocBuilder, rawLines.stream(), luceneServerClient)); + new DocumentGeneratorAndIndexer(oneDocBuilder, rawLines.stream(), nrtsearchClient)); return genIdFuture; } catch (RejectedExecutionException e) { logger.warn("Waiting for 1s for LinkedBlockingQueue to have more capacity", e); diff --git a/src/test/resources/config.yaml b/src/test/resources/config.yaml index 1ef945258..5a94439e6 100644 --- a/src/test/resources/config.yaml +++ b/src/test/resources/config.yaml @@ -1,4 +1,4 @@ -nodeName: "lucene_server_foo" +nodeName: "server_foo" hostName: "foohost" port: "9998" replicationPort: "9999" diff --git a/src/test/resources/yelp_reviews/primary/lucene_server_configuration.yaml b/src/test/resources/yelp_reviews/primary/nrtsearch_config.yaml similarity index 87% rename from src/test/resources/yelp_reviews/primary/lucene_server_configuration.yaml rename to src/test/resources/yelp_reviews/primary/nrtsearch_config.yaml index 49c5e1fac..eadef349d 100644 --- a/src/test/resources/yelp_reviews/primary/lucene_server_configuration.yaml +++ b/src/test/resources/yelp_reviews/primary/nrtsearch_config.yaml @@ -1,4 +1,4 @@ -nodeName: "lucene_server_primary" +nodeName: "nrtsearch_primary" hostName: "localhost" port: "8000" replicationPort: "8001" diff --git a/src/test/resources/yelp_reviews/replica/lucene_server_configuration.yaml b/src/test/resources/yelp_reviews/replica/nrtsearch_config.yaml similarity index 86% rename from src/test/resources/yelp_reviews/replica/lucene_server_configuration.yaml rename to src/test/resources/yelp_reviews/replica/nrtsearch_config.yaml index 8b3996b7d..d6f2933e2 100644 --- a/src/test/resources/yelp_reviews/replica/lucene_server_configuration.yaml +++ b/src/test/resources/yelp_reviews/replica/nrtsearch_config.yaml @@ -1,4 +1,4 @@ -nodeName: "lucene_server_replica_0" +nodeName: "nrtsearch_replica_0" hostName: "localhost" port: "8002" replicationPort: "8003" From 5f552f86ee0104e06ea7cacec64e028c9d5d254c Mon Sep 17 00:00:00 2001 From: Andrew Prudhomme Date: Mon, 11 Nov 2024 10:00:58 -0800 Subject: [PATCH 2/2] Additional fixes --- docs/server_configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/server_configuration.rst b/docs/server_configuration.rst index bef75c0e2..dd5380f49 100644 --- a/docs/server_configuration.rst +++ b/docs/server_configuration.rst @@ -265,7 +265,7 @@ Example server configuration - Offset in threads formula: (numCPUs * multiplier) + offset - 0 -.. list-table:: `Warmer Configuration `_ (``warmer.*``) +.. list-table:: `Warmer Configuration `_ (``warmer.*``) :widths: 25 10 50 25 :header-rows: 1