From 7ca650db33407bc5f95218750de36ac89e91c842 Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Thu, 3 Jan 2019 18:14:54 +0800 Subject: [PATCH] Adds Elasticsearch support for Autocomplete tags (#2333) --- ...pkinCassandraStorageAutoConfiguration.java | 2 +- ...CassandraStorageAutoConfigurationTest.java | 2 +- ...kinCassandra3StorageAutoConfiguration.java | 2 +- ...CassandraStorageAutoConfigurationTest.java | 2 +- ...ElasticsearchStorageAutoConfiguration.java | 7 +- ...ticsearchStorageAutoConfigurationTest.java | 14 ++++ .../v1/CassandraAutocompleteTags.java | 2 +- .../cassandra/v1/CassandraSpanConsumer.java | 2 +- .../cassandra/v1/CassandraStorage.java | 2 +- .../zipkin2/storage/cassandra/v1/Indexer.java | 2 +- .../cassandra/v1/InsertAutocompleteValue.java | 3 +- .../cassandra/v1/InsertServiceName.java | 2 +- .../storage/cassandra/v1/InsertSpanName.java | 2 +- .../zipkin2/storage/cassandra/v1/Schema.java | 2 +- .../v1/SelectAutocompleteValues.java | 2 +- .../zipkin2/storage/cassandra/v1/Tables.java | 2 +- .../cassandra/v1/ITCassandraStorage.java | 2 +- .../storage/cassandra/v1/ITEnsureSchema.java | 2 +- .../storage/cassandra/v1/ITSpanConsumer.java | 2 +- .../cassandra/CassandraAutocompleteTags.java | 2 +- .../cassandra/CassandraSpanConsumer.java | 2 +- .../storage/cassandra/CassandraSpanStore.java | 2 +- .../storage/cassandra/CassandraStorage.java | 2 +- .../storage/cassandra/CassandraUtil.java | 2 +- .../cassandra/InsertAutocompleteValue.java | 3 +- .../storage/cassandra/InsertServiceSpan.java | 2 +- .../zipkin2/storage/cassandra/Schema.java | 2 +- .../cassandra/SelectAutocompleteValues.java | 3 +- .../storage/cassandra/SelectSpanNames.java | 2 +- .../internal/call/DeduplicatingCall.java | 2 +- .../storage/cassandra/ITCassandraStorage.java | 2 +- .../storage/cassandra/ITEnsureSchema.java | 2 +- .../storage/cassandra/ITSpanConsumer.java | 2 +- .../internal/call/DeduplicatingCallTest.java | 2 +- .../ElasticsearchAutocompleteTags.java | 67 +++++++++++++++++++ .../ElasticsearchSpanConsumer.java | 40 ++++++++++- .../elasticsearch/ElasticsearchSpanStore.java | 3 +- .../elasticsearch/ElasticsearchStorage.java | 21 +++++- .../zipkin2/elasticsearch/IndexTemplates.java | 6 +- .../VersionSpecificTemplates.java | 46 ++++++++++++- .../ElasticsearchAutocompleteTagsTest.java | 54 +++++++++++++++ .../ElasticsearchSpanConsumerTest.java | 3 + .../ElasticsearchStorageTest.java | 2 + .../zipkin2/elasticsearch/TestResponses.java | 47 ++++++++++++- .../integration/ITElasticsearchStorageV2.java | 13 ++++ .../integration/ITElasticsearchStorageV5.java | 13 ++++ .../integration/ITElasticsearchStorageV6.java | 13 ++++ 47 files changed, 372 insertions(+), 44 deletions(-) create mode 100644 zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchAutocompleteTags.java create mode 100644 zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchAutocompleteTagsTest.java diff --git a/zipkin-autoconfigure/storage-cassandra/src/main/java/zipkin2/autoconfigure/storage/cassandra/ZipkinCassandraStorageAutoConfiguration.java b/zipkin-autoconfigure/storage-cassandra/src/main/java/zipkin2/autoconfigure/storage/cassandra/ZipkinCassandraStorageAutoConfiguration.java index e629b0b6e01..69acadde060 100644 --- a/zipkin-autoconfigure/storage-cassandra/src/main/java/zipkin2/autoconfigure/storage/cassandra/ZipkinCassandraStorageAutoConfiguration.java +++ b/zipkin-autoconfigure/storage-cassandra/src/main/java/zipkin2/autoconfigure/storage/cassandra/ZipkinCassandraStorageAutoConfiguration.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-autoconfigure/storage-cassandra/src/test/java/zipkin2/storage/cassandra/v1/ZipkinCassandraStorageAutoConfigurationTest.java b/zipkin-autoconfigure/storage-cassandra/src/test/java/zipkin2/storage/cassandra/v1/ZipkinCassandraStorageAutoConfigurationTest.java index e6b694460e8..aa61d7c6c0c 100644 --- a/zipkin-autoconfigure/storage-cassandra/src/test/java/zipkin2/storage/cassandra/v1/ZipkinCassandraStorageAutoConfigurationTest.java +++ b/zipkin-autoconfigure/storage-cassandra/src/test/java/zipkin2/storage/cassandra/v1/ZipkinCassandraStorageAutoConfigurationTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-autoconfigure/storage-cassandra3/src/main/java/zipkin2/autoconfigure/storage/cassandra3/ZipkinCassandra3StorageAutoConfiguration.java b/zipkin-autoconfigure/storage-cassandra3/src/main/java/zipkin2/autoconfigure/storage/cassandra3/ZipkinCassandra3StorageAutoConfiguration.java index fada30f2cc2..f09401bf985 100644 --- a/zipkin-autoconfigure/storage-cassandra3/src/main/java/zipkin2/autoconfigure/storage/cassandra3/ZipkinCassandra3StorageAutoConfiguration.java +++ b/zipkin-autoconfigure/storage-cassandra3/src/main/java/zipkin2/autoconfigure/storage/cassandra3/ZipkinCassandra3StorageAutoConfiguration.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-autoconfigure/storage-cassandra3/src/test/java/zipkin2/storage/cassandra/ZipkinCassandraStorageAutoConfigurationTest.java b/zipkin-autoconfigure/storage-cassandra3/src/test/java/zipkin2/storage/cassandra/ZipkinCassandraStorageAutoConfigurationTest.java index 1a53178fa44..54c4bee3a57 100644 --- a/zipkin-autoconfigure/storage-cassandra3/src/test/java/zipkin2/storage/cassandra/ZipkinCassandraStorageAutoConfigurationTest.java +++ b/zipkin-autoconfigure/storage-cassandra3/src/test/java/zipkin2/storage/cassandra/ZipkinCassandraStorageAutoConfigurationTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-autoconfigure/storage-elasticsearch/src/main/java/zipkin2/autoconfigure/storage/elasticsearch/ZipkinElasticsearchStorageAutoConfiguration.java b/zipkin-autoconfigure/storage-elasticsearch/src/main/java/zipkin2/autoconfigure/storage/elasticsearch/ZipkinElasticsearchStorageAutoConfiguration.java index ec70d58b93f..b91c3a69d02 100644 --- a/zipkin-autoconfigure/storage-elasticsearch/src/main/java/zipkin2/autoconfigure/storage/elasticsearch/ZipkinElasticsearchStorageAutoConfiguration.java +++ b/zipkin-autoconfigure/storage-elasticsearch/src/main/java/zipkin2/autoconfigure/storage/elasticsearch/ZipkinElasticsearchStorageAutoConfiguration.java @@ -13,6 +13,7 @@ */ package zipkin2.autoconfigure.storage.elasticsearch; +import java.util.List; import java.util.logging.Logger; import okhttp3.Interceptor; import okhttp3.OkHttpClient; @@ -65,12 +66,14 @@ ElasticsearchStorage.Builder esHttpBuilder( @Qualifier("zipkinElasticsearchHttp") OkHttpClient client, @Value("${zipkin.query.lookback:86400000}") int namesLookback, @Value("${zipkin.storage.strict-trace-id:true}") boolean strictTraceId, - @Value("${zipkin.storage.search-enabled:true}") boolean searchEnabled) { + @Value("${zipkin.storage.search-enabled:true}") boolean searchEnabled, + @Value("${zipkin.storage.autocomplete-keys:}") List autocompleteKeys) { return elasticsearch .toBuilder(client) .namesLookback(namesLookback) .strictTraceId(strictTraceId) - .searchEnabled(searchEnabled); + .searchEnabled(searchEnabled) + .autocompleteKeys(autocompleteKeys); } static final class HttpLoggingSet implements Condition { diff --git a/zipkin-autoconfigure/storage-elasticsearch/src/test/java/zipkin2/elasticsearch/ZipkinElasticsearchStorageAutoConfigurationTest.java b/zipkin-autoconfigure/storage-elasticsearch/src/test/java/zipkin2/elasticsearch/ZipkinElasticsearchStorageAutoConfigurationTest.java index df97df902d3..6a11a77eb32 100644 --- a/zipkin-autoconfigure/storage-elasticsearch/src/test/java/zipkin2/elasticsearch/ZipkinElasticsearchStorageAutoConfigurationTest.java +++ b/zipkin-autoconfigure/storage-elasticsearch/src/test/java/zipkin2/elasticsearch/ZipkinElasticsearchStorageAutoConfigurationTest.java @@ -376,6 +376,20 @@ public void searchEnabled_false() { assertThat(context.getBean(ElasticsearchStorage.class).searchEnabled()).isFalse(); } + @Test + public void autocompleteKeys_list() { + context = new AnnotationConfigApplicationContext(); + TestPropertyValues.of( + "zipkin.storage.type:elasticsearch", + "zipkin.storage.autocomplete-keys:environment") + .applyTo(context); + Access.registerElasticsearchHttp(context); + context.refresh(); + + assertThat(context.getBean(ElasticsearchStorage.class).autocompleteKeys()) + .containsOnly("environment"); + } + ElasticsearchStorage es() { return context.getBean(ElasticsearchStorage.class); } diff --git a/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/CassandraAutocompleteTags.java b/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/CassandraAutocompleteTags.java index 1db26cadcdd..8436cab8f7f 100644 --- a/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/CassandraAutocompleteTags.java +++ b/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/CassandraAutocompleteTags.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/CassandraSpanConsumer.java b/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/CassandraSpanConsumer.java index 3cc962dc445..3c21656797d 100644 --- a/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/CassandraSpanConsumer.java +++ b/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/CassandraSpanConsumer.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/CassandraStorage.java b/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/CassandraStorage.java index 2cca68faef0..ca04c33af7b 100644 --- a/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/CassandraStorage.java +++ b/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/CassandraStorage.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/Indexer.java b/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/Indexer.java index 3d0680c90f1..721dd607530 100644 --- a/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/Indexer.java +++ b/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/Indexer.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/InsertAutocompleteValue.java b/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/InsertAutocompleteValue.java index 73788f17795..8754bfc1cb2 100644 --- a/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/InsertAutocompleteValue.java +++ b/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/InsertAutocompleteValue.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -11,7 +11,6 @@ * or implied. See the License for the specific language governing permissions and limitations under * the License. */ - package zipkin2.storage.cassandra.v1; import com.datastax.driver.core.PreparedStatement; diff --git a/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/InsertServiceName.java b/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/InsertServiceName.java index e304f97d7fa..00ac4f06c6f 100644 --- a/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/InsertServiceName.java +++ b/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/InsertServiceName.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/InsertSpanName.java b/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/InsertSpanName.java index 54db00aef76..48e8c04c662 100644 --- a/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/InsertSpanName.java +++ b/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/InsertSpanName.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/Schema.java b/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/Schema.java index f43ffbd6870..34cae2481e6 100644 --- a/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/Schema.java +++ b/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/Schema.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/SelectAutocompleteValues.java b/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/SelectAutocompleteValues.java index 5df4e463b0c..890aaf5e7b3 100644 --- a/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/SelectAutocompleteValues.java +++ b/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/SelectAutocompleteValues.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/Tables.java b/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/Tables.java index 2c0af087e5b..aa447278e21 100644 --- a/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/Tables.java +++ b/zipkin-storage/cassandra-v1/src/main/java/zipkin2/storage/cassandra/v1/Tables.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra-v1/src/test/java/zipkin2/storage/cassandra/v1/ITCassandraStorage.java b/zipkin-storage/cassandra-v1/src/test/java/zipkin2/storage/cassandra/v1/ITCassandraStorage.java index 580a813a31d..9b63d460536 100644 --- a/zipkin-storage/cassandra-v1/src/test/java/zipkin2/storage/cassandra/v1/ITCassandraStorage.java +++ b/zipkin-storage/cassandra-v1/src/test/java/zipkin2/storage/cassandra/v1/ITCassandraStorage.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra-v1/src/test/java/zipkin2/storage/cassandra/v1/ITEnsureSchema.java b/zipkin-storage/cassandra-v1/src/test/java/zipkin2/storage/cassandra/v1/ITEnsureSchema.java index c6d157c3e15..34f56653596 100644 --- a/zipkin-storage/cassandra-v1/src/test/java/zipkin2/storage/cassandra/v1/ITEnsureSchema.java +++ b/zipkin-storage/cassandra-v1/src/test/java/zipkin2/storage/cassandra/v1/ITEnsureSchema.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra-v1/src/test/java/zipkin2/storage/cassandra/v1/ITSpanConsumer.java b/zipkin-storage/cassandra-v1/src/test/java/zipkin2/storage/cassandra/v1/ITSpanConsumer.java index de559148049..791be2c3617 100644 --- a/zipkin-storage/cassandra-v1/src/test/java/zipkin2/storage/cassandra/v1/ITSpanConsumer.java +++ b/zipkin-storage/cassandra-v1/src/test/java/zipkin2/storage/cassandra/v1/ITSpanConsumer.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/CassandraAutocompleteTags.java b/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/CassandraAutocompleteTags.java index 0b333a0944f..a79c668803e 100644 --- a/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/CassandraAutocompleteTags.java +++ b/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/CassandraAutocompleteTags.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/CassandraSpanConsumer.java b/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/CassandraSpanConsumer.java index 3e7b0c21a1a..bb3de0b4b8c 100644 --- a/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/CassandraSpanConsumer.java +++ b/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/CassandraSpanConsumer.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/CassandraSpanStore.java b/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/CassandraSpanStore.java index e23dde2c2da..390e0e8bbbd 100644 --- a/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/CassandraSpanStore.java +++ b/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/CassandraSpanStore.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/CassandraStorage.java b/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/CassandraStorage.java index 64ff34ccfc4..9dc970f7639 100644 --- a/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/CassandraStorage.java +++ b/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/CassandraStorage.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/CassandraUtil.java b/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/CassandraUtil.java index 3a3fca2b74b..7efd1c34b1e 100644 --- a/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/CassandraUtil.java +++ b/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/CassandraUtil.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/InsertAutocompleteValue.java b/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/InsertAutocompleteValue.java index 93fbb71f8c6..0ae04c023a0 100644 --- a/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/InsertAutocompleteValue.java +++ b/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/InsertAutocompleteValue.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -11,7 +11,6 @@ * or implied. See the License for the specific language governing permissions and limitations under * the License. */ - package zipkin2.storage.cassandra; import com.datastax.driver.core.PreparedStatement; diff --git a/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/InsertServiceSpan.java b/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/InsertServiceSpan.java index 08fa18d1aaa..0cdcaeeb4b5 100644 --- a/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/InsertServiceSpan.java +++ b/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/InsertServiceSpan.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/Schema.java b/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/Schema.java index 0c0fda73f7f..060959d083a 100644 --- a/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/Schema.java +++ b/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/Schema.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/SelectAutocompleteValues.java b/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/SelectAutocompleteValues.java index d6a5b913d6f..c1e04de601f 100644 --- a/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/SelectAutocompleteValues.java +++ b/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/SelectAutocompleteValues.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -11,7 +11,6 @@ * or implied. See the License for the specific language governing permissions and limitations under * the License. */ - package zipkin2.storage.cassandra; import com.datastax.driver.core.PreparedStatement; diff --git a/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/SelectSpanNames.java b/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/SelectSpanNames.java index 75fc6c05690..adcc2fb5c70 100644 --- a/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/SelectSpanNames.java +++ b/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/SelectSpanNames.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/internal/call/DeduplicatingCall.java b/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/internal/call/DeduplicatingCall.java index 2614154b785..9e5e1ed735e 100644 --- a/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/internal/call/DeduplicatingCall.java +++ b/zipkin-storage/cassandra/src/main/java/zipkin2/storage/cassandra/internal/call/DeduplicatingCall.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra/src/test/java/zipkin2/storage/cassandra/ITCassandraStorage.java b/zipkin-storage/cassandra/src/test/java/zipkin2/storage/cassandra/ITCassandraStorage.java index f9c8f21d099..5c7725d1789 100644 --- a/zipkin-storage/cassandra/src/test/java/zipkin2/storage/cassandra/ITCassandraStorage.java +++ b/zipkin-storage/cassandra/src/test/java/zipkin2/storage/cassandra/ITCassandraStorage.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra/src/test/java/zipkin2/storage/cassandra/ITEnsureSchema.java b/zipkin-storage/cassandra/src/test/java/zipkin2/storage/cassandra/ITEnsureSchema.java index 9f97f99aeb5..de9e6fa82fd 100644 --- a/zipkin-storage/cassandra/src/test/java/zipkin2/storage/cassandra/ITEnsureSchema.java +++ b/zipkin-storage/cassandra/src/test/java/zipkin2/storage/cassandra/ITEnsureSchema.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra/src/test/java/zipkin2/storage/cassandra/ITSpanConsumer.java b/zipkin-storage/cassandra/src/test/java/zipkin2/storage/cassandra/ITSpanConsumer.java index 85829cf19d8..eb814670938 100644 --- a/zipkin-storage/cassandra/src/test/java/zipkin2/storage/cassandra/ITSpanConsumer.java +++ b/zipkin-storage/cassandra/src/test/java/zipkin2/storage/cassandra/ITSpanConsumer.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/cassandra/src/test/java/zipkin2/storage/cassandra/internal/call/DeduplicatingCallTest.java b/zipkin-storage/cassandra/src/test/java/zipkin2/storage/cassandra/internal/call/DeduplicatingCallTest.java index eaa1ad7d31f..65b022f7bcd 100644 --- a/zipkin-storage/cassandra/src/test/java/zipkin2/storage/cassandra/internal/call/DeduplicatingCallTest.java +++ b/zipkin-storage/cassandra/src/test/java/zipkin2/storage/cassandra/internal/call/DeduplicatingCallTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchAutocompleteTags.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchAutocompleteTags.java new file mode 100644 index 00000000000..227e024d425 --- /dev/null +++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchAutocompleteTags.java @@ -0,0 +1,67 @@ +/* + * Copyright 2015-2019 The OpenZipkin Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package zipkin2.elasticsearch; + +import java.util.List; +import java.util.Locale; +import zipkin2.Call; +import zipkin2.elasticsearch.internal.IndexNameFormatter; +import zipkin2.elasticsearch.internal.client.Aggregation; +import zipkin2.elasticsearch.internal.client.SearchCallFactory; +import zipkin2.elasticsearch.internal.client.SearchRequest; +import zipkin2.storage.AutocompleteTags; + +final class ElasticsearchAutocompleteTags implements AutocompleteTags { + static final String AUTOCOMPLETE = "autocomplete"; + + final boolean enabled; + final IndexNameFormatter indexNameFormatter; + final SearchCallFactory search; + final int namesLookback; + final Call> keysCall; + + ElasticsearchAutocompleteTags(ElasticsearchStorage es) { + this.search = new SearchCallFactory(es.http()); + this.indexNameFormatter = es.indexNameFormatter(); + this.enabled = es.searchEnabled() && !es.autocompleteKeys().isEmpty(); + this.namesLookback = es.namesLookback(); + this.keysCall = Call.create(es.autocompleteKeys()); + } + + @Override public Call> getKeys() { + if (!enabled) return Call.emptyList(); + return keysCall.clone(); + } + + @Override public Call> getValues(String key) { + if (key == null) throw new NullPointerException("key == null"); + if (key.isEmpty()) throw new IllegalArgumentException("key was empty"); + if (!enabled) return Call.emptyList(); + + long endMillis = System.currentTimeMillis(); + long beginMillis = endMillis - namesLookback; + List indices = + indexNameFormatter.formatTypeAndRange(AUTOCOMPLETE, beginMillis, endMillis); + + if (indices.isEmpty()) return Call.emptyList(); + + SearchRequest.Filters filters = + new SearchRequest.Filters().addTerm("tagKey", key.toLowerCase(Locale.ROOT)); + + SearchRequest request = SearchRequest.create(indices) + .filters(filters) + .addAggregation(Aggregation.terms("tagValue", Integer.MAX_VALUE)); + return search.newCall(request, BodyConverters.KEYS); + } +} diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchSpanConsumer.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchSpanConsumer.java index a2ce09f3c3d..651afa958a3 100644 --- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchSpanConsumer.java +++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchSpanConsumer.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -15,8 +15,10 @@ import com.squareup.moshi.JsonWriter; import java.io.IOException; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; @@ -31,6 +33,8 @@ import zipkin2.elasticsearch.internal.client.HttpCall; import zipkin2.storage.SpanConsumer; +import static zipkin2.elasticsearch.ElasticsearchAutocompleteTags.AUTOCOMPLETE; + class ElasticsearchSpanConsumer implements SpanConsumer { // not final for testing static final Logger LOG = Logger.getLogger(ElasticsearchSpanConsumer.class.getName()); @@ -66,6 +70,7 @@ void indexSpans(BulkSpanIndexer indexer, List spans) { if (indexTimestamp == 0L) indexTimestamp = System.currentTimeMillis(); } indexer.add(indexTimestamp, span, spanTimestamp); + indexer.addTag(indexTimestamp, span); } } @@ -73,11 +78,13 @@ static final class BulkSpanIndexer { final HttpBulkIndexer indexer; final IndexNameFormatter indexNameFormatter; final boolean searchEnabled; + final Set autocompleteKeys; BulkSpanIndexer(ElasticsearchStorage es) { this.indexer = new HttpBulkIndexer("index-span", es); this.indexNameFormatter = es.indexNameFormatter(); this.searchEnabled = es.searchEnabled(); + this.autocompleteKeys = new LinkedHashSet<>(es.autocompleteKeys()); } void add(long indexTimestamp, Span span, long timestampMillis) { @@ -91,6 +98,37 @@ void add(long indexTimestamp, Span span, long timestampMillis) { index, ElasticsearchSpanStore.SPAN, document, null /* Allow ES to choose an ID */); } + void addTag(long indexTimestamp, Span span) { + if (span.tags().isEmpty()) return; + try { + Buffer query = new Buffer(); + for (Map.Entry tag : span.tags().entrySet()) { + // If the autocomplete whitelist doesn't contain the key, skip storing its value + if (!autocompleteKeys.contains(tag.getKey())) continue; + + JsonWriter writer = JsonWriter.of(query); + writer.beginObject(); + writer.name("tagKey"); + writer.value(tag.getKey()); + writer.name("tagValue"); + writer.value(tag.getValue()); + writer.endObject(); + String index = indexNameFormatter.formatTypeAndTimestamp(AUTOCOMPLETE, indexTimestamp); + byte[] document = query.readByteArray(); + query.clear(); + // Id of the document will be combination of {key,value} so that duplicate autocomplete + // keys can be avoided + indexer.add(index, AUTOCOMPLETE, document, tag.getKey() + "|" + tag.getValue()); + } + } catch (IOException e) { + // very unexpected to have an IOE for an in-memory write + assert false : "Error indexing autocomplete tags for span: " + span; + if (LOG.isLoggable(Level.FINE)) { + LOG.log(Level.FINE, "Error indexing autocomplete tags for span: " + span, e); + } + } + } + HttpCall newCall() { return indexer.newCall(); } diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchSpanStore.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchSpanStore.java index 6780812bf79..3b9b6f98a1a 100644 --- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchSpanStore.java +++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchSpanStore.java @@ -31,10 +31,11 @@ import static java.util.Arrays.asList; -final class ElasticsearchSpanStore implements SpanStore { +final class ElasticsearchSpanStore implements SpanStore{ static final String SPAN = "span"; static final String DEPENDENCY = "dependency"; + /** To not produce unnecessarily long queries, we don't look back further than first ES support */ static final long EARLIEST_MS = 1456790400000L; // March 2016 diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchStorage.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchStorage.java index 223c61c84fd..63d40213a68 100644 --- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchStorage.java +++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchStorage.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -17,6 +17,7 @@ import com.google.auto.value.extension.memoized.Memoized; import com.squareup.moshi.JsonReader; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; @@ -36,9 +37,11 @@ import zipkin2.storage.SpanConsumer; import zipkin2.storage.SpanStore; import zipkin2.storage.StorageComponent; +import zipkin2.storage.AutocompleteTags; import static zipkin2.elasticsearch.ElasticsearchSpanStore.DEPENDENCY; import static zipkin2.elasticsearch.ElasticsearchSpanStore.SPAN; +import static zipkin2.elasticsearch.ElasticsearchAutocompleteTags.AUTOCOMPLETE; import static zipkin2.elasticsearch.internal.JsonReaders.enterPath; @AutoValue @@ -67,7 +70,8 @@ public static Builder newBuilder(OkHttpClient client) { .indexReplicas(1) .namesLookback(86400000) .shutdownClientOnClose(false) - .flushOnWrites(false); + .flushOnWrites(false) + .autocompleteKeys(new ArrayList<>()); } public static Builder newBuilder() { @@ -186,6 +190,9 @@ public final Builder dateSeparator(char dateSeparator) { @Override public abstract Builder searchEnabled(boolean searchEnabled); + @Override + public abstract Builder autocompleteKeys(List keys); + @Override public abstract ElasticsearchStorage build(); @@ -211,6 +218,8 @@ public final Builder dateSeparator(char dateSeparator) { abstract boolean searchEnabled(); + abstract List autocompleteKeys(); + abstract int indexShards(); abstract int indexReplicas(); @@ -225,6 +234,12 @@ public SpanStore spanStore() { return new ElasticsearchSpanStore(this); } + @Override + public AutocompleteTags autocompleteTags() { + ensureIndexTemplates(); + return new ElasticsearchAutocompleteTags(this); + } + @Override public SpanConsumer spanConsumer() { ensureIndexTemplates(); @@ -321,6 +336,8 @@ IndexTemplates ensureIndexTemplates() { EnsureIndexTemplate.apply(http(), index + ":" + SPAN + "_template", templates.span()); EnsureIndexTemplate.apply( http(), index + ":" + DEPENDENCY + "_template", templates.dependency()); + EnsureIndexTemplate.apply( + http(), index + ":" + AUTOCOMPLETE + "_template", templates.autocomplete()); return templates; } catch (IOException e) { throw Platform.get().uncheckedIOException(e); diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/IndexTemplates.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/IndexTemplates.java index 070d7f2bd97..12f2445fb53 100644 --- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/IndexTemplates.java +++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/IndexTemplates.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -27,6 +27,8 @@ static Builder newBuilder() { abstract String dependency(); + abstract String autocomplete(); + @AutoValue.Builder interface Builder { Builder version(float version); @@ -35,6 +37,8 @@ interface Builder { Builder dependency(String dependency); + Builder autocomplete(String autocomplete); + IndexTemplates build(); } } diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/VersionSpecificTemplates.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/VersionSpecificTemplates.java index 674b59f2d0f..5626118e276 100644 --- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/VersionSpecificTemplates.java +++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/VersionSpecificTemplates.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -22,6 +22,7 @@ import static zipkin2.elasticsearch.ElasticsearchSpanStore.DEPENDENCY; import static zipkin2.elasticsearch.ElasticsearchSpanStore.SPAN; +import static zipkin2.elasticsearch.ElasticsearchAutocompleteTags.AUTOCOMPLETE; import static zipkin2.elasticsearch.internal.JsonReaders.enterPath; /** Returns a version-specific span and dependency index template */ @@ -31,6 +32,7 @@ final class VersionSpecificTemplates { final boolean searchEnabled; final String spanIndexTemplate; final String dependencyIndexTemplate; + final String autocompleteIndexTemplate; VersionSpecificTemplates(ElasticsearchStorage es) { this.searchEnabled = es.searchEnabled(); @@ -49,6 +51,10 @@ final class VersionSpecificTemplates { .replace("${__INDEX__}", es.indexNameFormatter().index()) .replace("${__NUMBER_OF_SHARDS__}", String.valueOf(es.indexShards())) .replace("${__NUMBER_OF_REPLICAS__}", String.valueOf(es.indexReplicas())); + this.autocompleteIndexTemplate = AUTOCOMPLETE_INDEX_TEMPLATE + .replace("${__INDEX__}", es.indexNameFormatter().index()) + .replace("${__NUMBER_OF_SHARDS__}", String.valueOf(es.indexShards())) + .replace("${__NUMBER_OF_REPLICAS__}", String.valueOf(es.indexReplicas())); } /** Templatized due to version differences. Only fields used in search are declared */ @@ -161,12 +167,34 @@ String spanIndexTemplate() { + "\": { \"enabled\": false }}\n" + "}"; + // The key filed of a autocompleteKeys is intentionally names as tagKey since it clashes with the + // BodyConverters KEY + static final String AUTOCOMPLETE_INDEX_TEMPLATE = + "{\n" + + " \"TEMPLATE\": \"${__INDEX__}:" + + AUTOCOMPLETE + + "-*\",\n" + + " \"settings\": {\n" + + " \"index.number_of_shards\": ${__NUMBER_OF_SHARDS__},\n" + + " \"index.number_of_replicas\": ${__NUMBER_OF_REPLICAS__},\n" + + " \"index.requests.cache.enable\": true,\n" + + " \"index.mapper.dynamic\": true\n" + + " },\n" + + " \"mappings\": {\"" + + AUTOCOMPLETE + + "\": { \"enabled\": true,\n" + + " \t\"properties\": {\n" + + " \"tagKey\": { KEYWORD },\n" + + " \"tagValue\": { KEYWORD }\n" + + " }}}\n" + + "}"; IndexTemplates get(HttpCall.Factory callFactory) throws IOException { float version = getVersion(callFactory); return IndexTemplates.newBuilder() .version(version) .span(versionSpecificSpanIndexTemplate(version)) .dependency(versionSpecificDependencyLinkIndexTemplate(version)) + .autocomplete(versionSpecificAutocompleteIndexTemplate(version)) .build(); } @@ -224,4 +252,20 @@ private String versionSpecificDependencyLinkIndexTemplate(float version) { return dependencyIndexTemplate.replace( "TEMPLATE", version >= 6 ? "index_patterns" : "template"); } + private String versionSpecificAutocompleteIndexTemplate(float version) { + if (version >= 2 && version < 3) { + return autocompleteIndexTemplate + .replace("TEMPLATE", "template") + .replace("KEYWORD", "\"type\": \"string\", \"norms\": {\"enabled\": false }, \"index\": " + + "\"not_analyzed\""); + } else if (version >= 5) { + return autocompleteIndexTemplate + .replace("TEMPLATE", version >= 6 ? "index_patterns" : "template") + .replace("KEYWORD", "\"type\": \"text\",\"fielddata\": true\n"); + }else { + throw new IllegalStateException( + "Elasticsearch 2.x, 5.x and 6.x are supported, was: " + version); + } + } } + diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchAutocompleteTagsTest.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchAutocompleteTagsTest.java new file mode 100644 index 00000000000..69aba632f92 --- /dev/null +++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchAutocompleteTagsTest.java @@ -0,0 +1,54 @@ +/* + * Copyright 2015-2019 The OpenZipkin Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package zipkin2.elasticsearch; + +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import org.junit.After; +import org.junit.Rule; +import org.junit.Test; + +import static java.util.Arrays.asList; +import static org.assertj.core.api.Assertions.assertThat; + +public class ElasticsearchAutocompleteTagsTest { + @Rule public MockWebServer es = new MockWebServer(); + + ElasticsearchStorage storage = ElasticsearchStorage.newBuilder() + .hosts(asList(es.url("").toString())) + .autocompleteKeys(asList("http#host", "http-url", "http.method")).build(); + ElasticsearchAutocompleteTags tagStore = new ElasticsearchAutocompleteTags(storage); + + @After public void close() { + storage.close(); + } + + @Test public void get_list_of_autocomplete_keys() throws Exception { + // note: we don't enqueue a request! + assertThat(tagStore.getKeys().execute()) + .contains("http#host", "http-url", "http.method"); + } + + @Test public void getValues_requestIncludesKeyName() throws Exception { + es.enqueue(new MockResponse().setBody(TestResponses.AUTOCOMPLETE_VALUES)); + tagStore.getValues("http.method").execute(); + assertThat(es.takeRequest().getBody().readUtf8()).contains("\"tagKey\":\"http.method\""); + } + + @Test public void getValues() throws Exception { + es.enqueue(new MockResponse().setBody(TestResponses.AUTOCOMPLETE_VALUES)); + + assertThat(tagStore.getValues("http.method").execute()).containsOnly("get", "post"); + } +} diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchSpanConsumerTest.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchSpanConsumerTest.java index 83f3ba32499..4ef0e119dec 100644 --- a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchSpanConsumerTest.java +++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchSpanConsumerTest.java @@ -57,10 +57,12 @@ private void ensureIndexTemplates(ElasticsearchStorage storage) throws Interrupt es.enqueue(new MockResponse().setBody("{\"version\":{\"number\":\"6.0.0\"}}")); es.enqueue(new MockResponse()); // get span template es.enqueue(new MockResponse()); // get dependency template + es.enqueue(new MockResponse()); // get tags template storage.ensureIndexTemplates(); es.takeRequest(); // get version es.takeRequest(); // get span template es.takeRequest(); // get dependency template + es.takeRequest(); // get tags template } @After @@ -316,6 +318,7 @@ public void searchDisabled_simplerIndexTemplate() throws Exception { es.enqueue(new MockResponse().setResponseCode(404)); // get span template es.enqueue(new MockResponse()); // put span template es.enqueue(new MockResponse()); // get dependency template + es.enqueue(new MockResponse()); // get tags template storage.ensureIndexTemplates(); es.takeRequest(); // get version es.takeRequest(); // get span template diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchStorageTest.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchStorageTest.java index ec33cfad87f..e483549b9f3 100644 --- a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchStorageTest.java +++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchStorageTest.java @@ -43,6 +43,7 @@ public void memoizesIndexTemplate() throws Exception { es.enqueue(new MockResponse().setBody("{\"version\":{\"number\":\"2.4.0\"}}")); es.enqueue(new MockResponse()); // get span template es.enqueue(new MockResponse()); // get dependency template + es.enqueue(new MockResponse()); // get tags template es.enqueue(new MockResponse()); // dependencies request es.enqueue(new MockResponse()); // dependencies request @@ -53,6 +54,7 @@ public void memoizesIndexTemplate() throws Exception { es.takeRequest(); // get version es.takeRequest(); // get span template es.takeRequest(); // get dependency template + es.takeRequest(); // get tags template assertThat(es.takeRequest().getPath()) .startsWith("/zipkin:dependency-2016-10-01,zipkin:dependency-2016-10-02/_search"); diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/TestResponses.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/TestResponses.java index f690ed1b092..47a875f86ae 100644 --- a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/TestResponses.java +++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/TestResponses.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2018 The OpenZipkin Authors + * Copyright 2015-2019 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -89,4 +89,49 @@ public final class TestResponses { + " }\n" + " }\n" + "}"; + + public static final String AUTOCOMPLETE_VALUES = "{\n" + + " \"took\": 12,\n" + + " \"timed_out\": false,\n" + + " \"_shards\": {\n" + + " \"total\": 5,\n" + + " \"successful\": 5,\n" + + " \"skipped\": 0,\n" + + " \"failed\": 0\n" + + " },\n" + + " \"hits\": {\n" + + " \"total\": 2,\n" + + " \"max_score\": 0,\n" + + " \"hits\": [\n" + + " {\n" + + " \"_index\": \"zipkin:autocomplete-2018-12-08\",\n" + + " \"_type\": \"autocomplete\",\n" + + " \"_id\": \"http.method|POST\",\n" + + " \"_score\": 0\n" + + " },\n" + + " {\n" + + " \"_index\": \"zipkin:autocomplete-2018-12-08\",\n" + + " \"_type\": \"autocomplete\",\n" + + " \"_id\": \"http.method|GET\",\n" + + " \"_score\": 0\n" + + " }\n" + + " ]\n" + + " },\n" + + " \"aggregations\": {\n" + + " \"tagValue\": {\n" + + " \"doc_count_error_upper_bound\": 0,\n" + + " \"sum_other_doc_count\": 0,\n" + + " \"buckets\": [\n" + + " {\n" + + " \"key\": \"get\",\n" + + " \"doc_count\": 1\n" + + " },\n" + + " {\n" + + " \"key\": \"post\",\n" + + " \"doc_count\": 1\n" + + " }\n" + + " ]\n" + + " }\n" + + " }\n" + + "}"; } diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ITElasticsearchStorageV2.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ITElasticsearchStorageV2.java index c65dfcbbea8..d7ceebe6234 100644 --- a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ITElasticsearchStorageV2.java +++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ITElasticsearchStorageV2.java @@ -99,6 +99,19 @@ public static class ITStrictTraceIdFalse extends zipkin2.storage.ITStrictTraceId } } + public static class ITAutocompleteTags extends zipkin2.storage.ITAutocompleteTags { + @ClassRule public static ElasticsearchStorageRule backend = classRule(); + @Rule public TestName testName = new TestName(); + + @Override protected StorageComponent.Builder storageBuilder() { + return backend.computeStorageBuilder().index(index(testName)); + } + + @Before @Override public void clear() throws IOException { + ((ElasticsearchStorage) storage).clear(); + } + } + public static class ITDependencies extends zipkin2.storage.ITDependencies { @ClassRule public static ElasticsearchStorageRule backend = classRule(); @Rule public TestName testName = new TestName(); diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ITElasticsearchStorageV5.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ITElasticsearchStorageV5.java index 1cb04598c3d..9502ccfcb89 100644 --- a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ITElasticsearchStorageV5.java +++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ITElasticsearchStorageV5.java @@ -100,6 +100,19 @@ public static class ITStrictTraceIdFalse extends zipkin2.storage.ITStrictTraceId } } + public static class ITAutocompleteTags extends zipkin2.storage.ITAutocompleteTags { + @ClassRule public static ElasticsearchStorageRule backend = classRule(); + @Rule public TestName testName = new TestName(); + + @Override protected StorageComponent.Builder storageBuilder() { + return backend.computeStorageBuilder().index(index(testName)); + } + + @Before @Override public void clear() throws IOException { + ((ElasticsearchStorage) storage).clear(); + } + } + public static class ITDependencies extends zipkin2.storage.ITDependencies { @ClassRule public static ElasticsearchStorageRule backend = classRule(); @Rule public TestName testName = new TestName(); diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ITElasticsearchStorageV6.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ITElasticsearchStorageV6.java index 9aee6e73ceb..5e0190db487 100644 --- a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ITElasticsearchStorageV6.java +++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ITElasticsearchStorageV6.java @@ -81,6 +81,19 @@ public static class ITSearchEnabledFalse extends zipkin2.storage.ITSearchEnabled } } + public static class ITAutocompleteTags extends zipkin2.storage.ITAutocompleteTags { + @ClassRule public static ElasticsearchStorageRule backend = classRule(); + @Rule public TestName testName = new TestName(); + + @Override protected StorageComponent.Builder storageBuilder() { + return backend.computeStorageBuilder().index(index(testName)); + } + + @Before @Override public void clear() throws IOException { + ((ElasticsearchStorage) storage).clear(); + } + } + public static class ITStrictTraceIdFalse extends zipkin2.storage.ITStrictTraceIdFalse { @ClassRule public static ElasticsearchStorageRule backend = classRule(); @Rule public TestName testName = new TestName();