-
Notifications
You must be signed in to change notification settings - Fork 30
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Rework schema initialization for better performance (#1078)
Co-authored-by: Bryan Burkholder <[email protected]>
- Loading branch information
Showing
14 changed files
with
371 additions
and
144 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
29 changes: 29 additions & 0 deletions
29
astra/src/main/java/com/slack/astra/logstore/opensearch/AstraIndexAnalyzer.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| package com.slack.astra.logstore.opensearch; | ||
|
|
||
| import static java.util.Collections.emptyMap; | ||
| import static java.util.Collections.singletonMap; | ||
|
|
||
| import org.apache.lucene.analysis.standard.StandardAnalyzer; | ||
| import org.opensearch.index.analysis.AnalyzerScope; | ||
| import org.opensearch.index.analysis.IndexAnalyzers; | ||
| import org.opensearch.index.analysis.NamedAnalyzer; | ||
|
|
||
| public class AstraIndexAnalyzer { | ||
|
|
||
| private static IndexAnalyzers indexAnalyzers; | ||
|
|
||
| private AstraIndexAnalyzer() {} | ||
|
|
||
| public static IndexAnalyzers getInstance() { | ||
| if (indexAnalyzers == null) { | ||
| indexAnalyzers = | ||
| new IndexAnalyzers( | ||
| singletonMap( | ||
| "default", | ||
| new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer())), | ||
| emptyMap(), | ||
| emptyMap()); | ||
| } | ||
| return indexAnalyzers; | ||
| } | ||
| } |
64 changes: 64 additions & 0 deletions
64
astra/src/main/java/com/slack/astra/logstore/opensearch/AstraIndexSettings.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| package com.slack.astra.logstore.opensearch; | ||
|
|
||
| import static org.opensearch.common.settings.IndexScopedSettings.BUILT_IN_INDEX_SETTINGS; | ||
|
|
||
| import com.slack.astra.logstore.LogMessage; | ||
| import java.util.HashSet; | ||
| import org.opensearch.Version; | ||
| import org.opensearch.cluster.metadata.IndexMetadata; | ||
| import org.opensearch.common.settings.IndexScopedSettings; | ||
| import org.opensearch.common.settings.Settings; | ||
| import org.opensearch.index.IndexSettings; | ||
| import org.opensearch.index.mapper.MapperService; | ||
|
|
||
| public class AstraIndexSettings { | ||
| private static IndexSettings indexSettings = null; | ||
|
|
||
| // we can make this configurable when SchemaAwareLogDocumentBuilderImpl enforces a limit | ||
| // set this to a high number for now | ||
| private static final int TOTAL_FIELDS_LIMIT = | ||
| Integer.parseInt(System.getProperty("astra.mapping.totalFieldsLimit", "2500")); | ||
|
|
||
| private AstraIndexSettings() {} | ||
|
|
||
| public static IndexSettings getInstance() { | ||
| if (indexSettings == null) { | ||
| indexSettings = buildIndexSettings(); | ||
| } | ||
| return indexSettings; | ||
| } | ||
|
|
||
| /** Builds the minimal amount of IndexSettings required for using Aggregations */ | ||
| private static IndexSettings buildIndexSettings() { | ||
| Settings settings = | ||
| Settings.builder() | ||
| .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) | ||
| .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) | ||
| .put(IndexMetadata.SETTING_VERSION_CREATED, Version.V_2_11_0) | ||
| .put( | ||
| MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), TOTAL_FIELDS_LIMIT) | ||
|
|
||
| // Astra time sorts the indexes while building it | ||
| // {LuceneIndexStoreImpl#buildIndexWriterConfig} | ||
| // When we were using the lucene query parser the sort info was leveraged by lucene | ||
| // automatically ( as the sort info persists in the segment info ) at query time. | ||
| // However the OpenSearch query parser has a custom implementation which relies on the | ||
| // index sort info to be present as a setting here. | ||
| .put("index.sort.field", LogMessage.SystemField.TIME_SINCE_EPOCH.fieldName) | ||
| .put("index.sort.order", "desc") | ||
| .put("index.query.default_field", LogMessage.SystemField.ALL.fieldName) | ||
| .put("index.query_string.lenient", false) | ||
| .build(); | ||
|
|
||
| Settings nodeSetings = | ||
| Settings.builder().put("indices.query.query_string.analyze_wildcard", true).build(); | ||
|
|
||
| IndexScopedSettings indexScopedSettings = | ||
| new IndexScopedSettings(settings, new HashSet<>(BUILT_IN_INDEX_SETTINGS)); | ||
|
|
||
| return new IndexSettings( | ||
| IndexMetadata.builder("index").settings(settings).build(), | ||
| nodeSetings, | ||
| indexScopedSettings); | ||
| } | ||
| } |
26 changes: 26 additions & 0 deletions
26
astra/src/main/java/com/slack/astra/logstore/opensearch/AstraMapperRegistry.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| package com.slack.astra.logstore.opensearch; | ||
|
|
||
| import static java.util.Collections.emptyList; | ||
|
|
||
| import java.util.Collections; | ||
| import java.util.Map; | ||
| import org.opensearch.index.mapper.Mapper; | ||
| import org.opensearch.index.mapper.MetadataFieldMapper; | ||
| import org.opensearch.indices.IndicesModule; | ||
| import org.opensearch.indices.mapper.MapperRegistry; | ||
| import org.opensearch.plugins.MapperPlugin; | ||
|
|
||
| public class AstraMapperRegistry { | ||
| private static Map<String, Mapper.TypeParser> mappers; | ||
| private static Map<String, MetadataFieldMapper.TypeParser> metadataMappers; | ||
|
|
||
| private AstraMapperRegistry() {} | ||
|
|
||
| public static MapperRegistry buildNewInstance() { | ||
| if (mappers == null || metadataMappers == null) { | ||
| mappers = Collections.unmodifiableMap(IndicesModule.getMappers(emptyList())); | ||
| metadataMappers = Collections.unmodifiableMap(IndicesModule.getMetadataMappers(emptyList())); | ||
| } | ||
| return new MapperRegistry(mappers, metadataMappers, MapperPlugin.NOOP_FIELD_FILTER); | ||
| } | ||
| } |
18 changes: 18 additions & 0 deletions
18
astra/src/main/java/com/slack/astra/logstore/opensearch/AstraSimilarityService.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| package com.slack.astra.logstore.opensearch; | ||
|
|
||
| import static java.util.Collections.emptyMap; | ||
|
|
||
| import org.opensearch.index.similarity.SimilarityService; | ||
|
|
||
| public class AstraSimilarityService { | ||
| private static SimilarityService similarityService = null; | ||
|
|
||
| private AstraSimilarityService() {} | ||
|
|
||
| public static SimilarityService getInstance() { | ||
| if (similarityService == null) { | ||
| similarityService = new SimilarityService(AstraIndexSettings.getInstance(), null, emptyMap()); | ||
| } | ||
| return similarityService; | ||
| } | ||
| } |
Oops, something went wrong.