From b9afa8cb119538c7327697a90b4cb9f3be2d95f9 Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Thu, 19 Aug 2021 08:51:04 +0200 Subject: [PATCH] Use es-index-cleaner golang implementation (#3204) * Use es-index-cleaner golang implementation Signed-off-by: Pavol Loffay * Fix Signed-off-by: Pavol Loffay * Fix Signed-off-by: Pavol Loffay --- .github/workflows/ci-elasticsearch.yml | 2 + Makefile | 13 +- cmd/es-index-cleaner/Dockerfile | 6 + cmd/es-index-cleaner/app/flags.go | 2 +- cmd/es-index-cleaner/app/index_filter.go | 15 +-- cmd/es-index-cleaner/app/index_filter_test.go | 12 +- plugin/storage/es/Dockerfile | 11 -- plugin/storage/es/esCleaner.py | 117 ------------------ .../integration/es_index_cleaner_test.go | 4 +- scripts/build-upload-docker-images.sh | 2 +- 10 files changed, 33 insertions(+), 151 deletions(-) create mode 100644 cmd/es-index-cleaner/Dockerfile delete mode 100644 plugin/storage/es/Dockerfile delete mode 100755 plugin/storage/es/esCleaner.py mode change 100644 => 100755 scripts/build-upload-docker-images.sh diff --git a/.github/workflows/ci-elasticsearch.yml b/.github/workflows/ci-elasticsearch.yml index 65b7f458064..6eec0cd9eb9 100644 --- a/.github/workflows/ci-elasticsearch.yml +++ b/.github/workflows/ci-elasticsearch.yml @@ -41,5 +41,7 @@ jobs: - name: Install tools run: make install-ci + - uses: docker/setup-qemu-action@v1 + - name: Run elasticsearch integration tests run: bash scripts/es-integration-test.sh ${{ matrix.version.distribution }} ${{ matrix.version.image }} diff --git a/Makefile b/Makefile index f2b943806b9..5ee96071b30 100644 --- a/Makefile +++ b/Makefile @@ -115,7 +115,6 @@ grpc-plugin-storage-integration-test: .PHONY: test-compile-es-scripts test-compile-es-scripts: docker run --rm -v ${PWD}:/tmp/jaeger python:3-alpine3.11 /usr/local/bin/python -m py_compile /tmp/jaeger/plugin/storage/es/esRollover.py - docker run --rm -v ${PWD}:/tmp/jaeger python:3-alpine3.11 /usr/local/bin/python -m py_compile /tmp/jaeger/plugin/storage/es/esCleaner.py .PHONY: index-cleaner-integration-test index-cleaner-integration-test: docker-images-elastic @@ -216,6 +215,10 @@ build-esmapping-generator: build-esmapping-generator-linux: GOOS=linux GOARCH=amd64 $(GOBUILD) -o ./plugin/storage/es/esmapping-generator ./cmd/esmapping-generator/main.go +.PHONY: build-es-index-cleaner +build-es-index-cleaner: + $(GOBUILD) -o ./cmd/es-index-cleaner/es-index-cleaner-$(GOOS)-$(GOARCH) ./cmd/es-index-cleaner/main.go + .PHONY: docker-hotrod docker-hotrod: GOOS=linux $(MAKE) build-examples @@ -306,7 +309,8 @@ build-platform-binaries: build-agent \ build-examples \ build-tracegen \ build-anonymizer \ - build-esmapping-generator + build-esmapping-generator \ + build-es-index-cleaner .PHONY: build-all-platforms build-all-platforms: build-binaries-linux build-binaries-windows build-binaries-darwin build-binaries-s390x build-binaries-arm64 build-binaries-ppc64le @@ -317,9 +321,10 @@ docker-images-cassandra: @echo "Finished building jaeger-cassandra-schema ==============" .PHONY: docker-images-elastic -docker-images-elastic: +docker-images-elastic: create-baseimg GOOS=linux GOARCH=$(GOARCH) $(MAKE) build-esmapping-generator - docker build -t $(DOCKER_NAMESPACE)/jaeger-es-index-cleaner:${DOCKER_TAG} plugin/storage/es + GOOS=linux GOARCH=$(GOARCH) $(MAKE) build-es-index-cleaner + docker build -t $(DOCKER_NAMESPACE)/jaeger-es-index-cleaner:${DOCKER_TAG} --build-arg base_image=$(BASE_IMAGE) --build-arg TARGETARCH=$(GOARCH) cmd/es-index-cleaner docker build -t $(DOCKER_NAMESPACE)/jaeger-es-rollover:${DOCKER_TAG} plugin/storage/es -f plugin/storage/es/Dockerfile.rollover --build-arg TARGETARCH=$(GOARCH) @echo "Finished building jaeger-es-indices-clean ==============" diff --git a/cmd/es-index-cleaner/Dockerfile b/cmd/es-index-cleaner/Dockerfile new file mode 100644 index 00000000000..66caa9539fc --- /dev/null +++ b/cmd/es-index-cleaner/Dockerfile @@ -0,0 +1,6 @@ +ARG base_image + +FROM $base_image AS release +ARG TARGETARCH +COPY es-index-cleaner-linux-$TARGETARCH /go/bin/es-index-cleaner-linux +ENTRYPOINT ["/go/bin/es-index-cleaner-linux"] diff --git a/cmd/es-index-cleaner/app/flags.go b/cmd/es-index-cleaner/app/flags.go index 7dfef44c240..53e4d06ea9a 100644 --- a/cmd/es-index-cleaner/app/flags.go +++ b/cmd/es-index-cleaner/app/flags.go @@ -45,7 +45,7 @@ type Config struct { // AddFlags adds flags for TLS to the FlagSet. func (c *Config) AddFlags(flags *flag.FlagSet) { flags.String(indexPrefix, "", "Index prefix") - flags.Bool(archive, false, "Whether to remove archive indices") + flags.Bool(archive, false, "Whether to remove archive indices. It works only for rollover") flags.Bool(rollover, false, "Whether to remove indices created by rollover") flags.Int(timeout, 120, "Number of seconds to wait for master node response") flags.String(indexDateSeparator, "-", "Index date separator") diff --git a/cmd/es-index-cleaner/app/index_filter.go b/cmd/es-index-cleaner/app/index_filter.go index e173ae6dd12..a01582730d6 100644 --- a/cmd/es-index-cleaner/app/index_filter.go +++ b/cmd/es-index-cleaner/app/index_filter.go @@ -42,18 +42,13 @@ func (i *IndexFilter) Filter(indices []Index) []Index { func (i *IndexFilter) filter(indices []Index) []Index { var reg *regexp.Regexp - if !i.Rollover && !i.Archive { - // daily indices - reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-(span|service|dependencies)-\\d{4}%s\\d{2}%s\\d{2}", i.IndexPrefix, i.IndexDateSeparator, i.IndexDateSeparator)) - } else if !i.Rollover && i.Archive { - // daily archive - reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-span-archive", i.IndexPrefix)) - } else if i.Rollover && !i.Archive { - // rollover + if i.Archive { + // archive works only for rollover + reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-span-archive-\\d{6}", i.IndexPrefix)) + } else if i.Rollover { reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-(span|service)-\\d{6}", i.IndexPrefix)) } else { - // rollover archive - reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-span-archive-\\d{6}", i.IndexPrefix)) + reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-(span|service|dependencies)-\\d{4}%s\\d{2}%s\\d{2}", i.IndexPrefix, i.IndexDateSeparator, i.IndexDateSeparator)) } var filtered []Index diff --git a/cmd/es-index-cleaner/app/index_filter_test.go b/cmd/es-index-cleaner/app/index_filter_test.go index da280e6ba68..0f2a58920f5 100644 --- a/cmd/es-index-cleaner/app/index_filter_test.go +++ b/cmd/es-index-cleaner/app/index_filter_test.go @@ -229,19 +229,21 @@ func testIndexFilter(t *testing.T, prefix string) { }, }, { - name: "archive indices, remove older 2 days", + name: "archive indices, remove older 1 days - archive works only for rollover", filter: &IndexFilter{ IndexPrefix: prefix, IndexDateSeparator: "-", Archive: true, Rollover: false, - DeleteBeforeThisDate: time20200807.Add(-time.Hour * 24 * time.Duration(2)), + DeleteBeforeThisDate: time20200807.Add(-time.Hour * 24 * time.Duration(1)), }, expected: []Index{ { - Index: prefix + "jaeger-span-archive", - CreationTime: time.Date(2020, time.August, 0, 15, 0, 0, 0, time.UTC), - Aliases: map[string]bool{}, + Index: prefix + "jaeger-span-archive-000001", + CreationTime: time.Date(2020, time.August, 5, 15, 0, 0, 0, time.UTC), + Aliases: map[string]bool{ + prefix + "jaeger-span-archive-read": true, + }, }, }, }, diff --git a/plugin/storage/es/Dockerfile b/plugin/storage/es/Dockerfile deleted file mode 100644 index 4fab7e98012..00000000000 --- a/plugin/storage/es/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM python:3-alpine3.11 - -# Temporary fix for https://github.com/jaegertracing/jaeger/issues/1494 -RUN pip install urllib3==1.24.3 -# Temporary fix for https://github.com/jaegertracing/jaeger/issues/3179 -RUN pip install elasticsearch==7.13.0 - -RUN pip install elasticsearch elasticsearch-curator -COPY esCleaner.py /es-index-cleaner/ - -ENTRYPOINT ["python3", "/es-index-cleaner/esCleaner.py"] diff --git a/plugin/storage/es/esCleaner.py b/plugin/storage/es/esCleaner.py deleted file mode 100755 index ace05fcd92f..00000000000 --- a/plugin/storage/es/esCleaner.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python3 - -import curator -import elasticsearch -import os -import ssl -import sys - -TIMEOUT=120 - -def main(): - if len(sys.argv) != 3: - print('USAGE: [INDEX_PREFIX=(default "")] [ARCHIVE=(default false)] ... {} NUM_OF_DAYS http://HOSTNAME[:PORT]'.format(sys.argv[0])) - print('NUM_OF_DAYS ... delete indices that are older than the given number of days.') - print('HOSTNAME ... specifies which Elasticsearch hosts URL to search and delete indices from.') - print('TIMEOUT ... number of seconds to wait for master node response (default {}).'.format(TIMEOUT)) - print('INDEX_PREFIX ... specifies index prefix.') - print('INDEX_DATE_SEPARATOR ... specifies index date separator.') - print('ARCHIVE ... specifies whether to remove archive indices (only works for rollover) (default false).') - print('ROLLOVER ... specifies whether to remove indices created by rollover (default false).') - print('ES_USERNAME ... The username required by Elasticsearch.') - print('ES_PASSWORD ... The password required by Elasticsearch.') - print('ES_TLS ... enable TLS (default false).') - print('ES_TLS_CA ... Path to TLS CA file.') - print('ES_TLS_CERT ... Path to TLS certificate file.') - print('ES_TLS_KEY ... Path to TLS key file.') - print('ES_TLS_SKIP_HOST_VERIFY ... (insecure) Skip server\'s certificate chain and host name verification.') - sys.exit(1) - - client = create_client(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"), str2bool(os.getenv("ES_TLS_SKIP_HOST_VERIFY", 'false'))) - ilo = curator.IndexList(client) - empty_list(ilo, 'Elasticsearch has no indices') - - prefix = os.getenv("INDEX_PREFIX", '') - if prefix != '': - prefix += '-' - separator = os.getenv("INDEX_DATE_SEPARATOR", '-') - - if str2bool(os.getenv("ARCHIVE", 'false')): - filter_archive_indices_rollover(ilo, prefix) - else: - if str2bool(os.getenv("ROLLOVER", 'false')): - filter_main_indices_rollover(ilo, prefix) - else: - filter_main_indices(ilo, prefix, separator) - - empty_list(ilo, 'No indices to delete') - - for index in ilo.working_list(): - print("Removing", index) - timeout = int(os.getenv("TIMEOUT", TIMEOUT)) - delete_indices = curator.DeleteIndices(ilo, master_timeout=timeout) - delete_indices.do_action() - - -def filter_main_indices(ilo, prefix, separator): - date_regex = "\d{4}" + separator + "\d{2}" + separator + "\d{2}" - time_string = "%Y" + separator + "%m" + separator + "%d" - - ilo.filter_by_regex(kind='regex', value=prefix + "jaeger-(span|service|dependencies)-" + date_regex) - empty_list(ilo, "No indices to delete") - # This excludes archive index as we use source='name' - # source `creation_date` would include archive index - ilo.filter_by_age(source='name', direction='older', timestring=time_string, unit='days', unit_count=int(sys.argv[1])) - - -def filter_main_indices_rollover(ilo, prefix): - ilo.filter_by_regex(kind='regex', value=prefix + "jaeger-(span|service)-\d{6}") - empty_list(ilo, "No indices to delete") - # do not remove active write indices - ilo.filter_by_alias(aliases=[prefix + 'jaeger-span-write'], exclude=True) - empty_list(ilo, "No indices to delete") - ilo.filter_by_alias(aliases=[prefix + 'jaeger-service-write'], exclude=True) - empty_list(ilo, "No indices to delete") - ilo.filter_by_age(source='creation_date', direction='older', unit='days', unit_count=int(sys.argv[1])) - - -def filter_archive_indices_rollover(ilo, prefix): - # Remove only rollover archive indices - # Do not remove active write archive index - ilo.filter_by_regex(kind='regex', value=prefix + "jaeger-span-archive-\d{6}") - empty_list(ilo, "No indices to delete") - ilo.filter_by_alias(aliases=[prefix + 'jaeger-span-archive-write'], exclude=True) - empty_list(ilo, "No indices to delete") - ilo.filter_by_age(source='creation_date', direction='older', unit='days', unit_count=int(sys.argv[1])) - - -def empty_list(ilo, error_msg): - try: - ilo.empty_list_check() - except curator.NoIndices: - print(error_msg) - sys.exit(0) - - -def str2bool(v): - return v.lower() in ('true', '1') - - -def create_client(username, password, tls, ca, cert, key, skipHostVerify): - context = ssl.create_default_context() - if ca is not None: - context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=ca) - elif skipHostVerify: - context.check_hostname = False - context.verify_mode = ssl.CERT_NONE - if username is not None and password is not None: - return elasticsearch.Elasticsearch(sys.argv[2:], http_auth=(username, password), ssl_context=context) - elif tls: - context.load_cert_chain(certfile=cert, keyfile=key) - return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context) - else: - return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context) - - -if __name__ == "__main__": - main() diff --git a/plugin/storage/integration/es_index_cleaner_test.go b/plugin/storage/integration/es_index_cleaner_test.go index 1eec135f032..b9a19b38dfe 100644 --- a/plugin/storage/integration/es_index_cleaner_test.go +++ b/plugin/storage/integration/es_index_cleaner_test.go @@ -114,10 +114,10 @@ func TestIndexCleaner(t *testing.T) { }, } for _, test := range tests { - t.Run(fmt.Sprintf("%s_no_prefix", test.name), func(t *testing.T) { + t.Run(fmt.Sprintf("%s_no_prefix, %s", test.name, test.envVars), func(t *testing.T) { runIndexCleanerTest(t, client, "", test.expectedIndices, test.envVars) }) - t.Run(fmt.Sprintf("%s_prefix", test.name), func(t *testing.T) { + t.Run(fmt.Sprintf("%s_prefix, %s", test.name, test.envVars), func(t *testing.T) { runIndexCleanerTest(t, client, indexPrefix, test.expectedIndices, append(test.envVars, "INDEX_PREFIX="+indexPrefix)) }) } diff --git a/scripts/build-upload-docker-images.sh b/scripts/build-upload-docker-images.sh old mode 100644 new mode 100755 index fd28807be97..f32793c2eb3 --- a/scripts/build-upload-docker-images.sh +++ b/scripts/build-upload-docker-images.sh @@ -20,7 +20,7 @@ do done # build/upload images for jaeger-es-index-cleaner and jaeger-es-rollover -bash scripts/build-upload-a-docker-image.sh -c jaeger-es-index-cleaner -d plugin/storage/es -p "${platforms}" +bash scripts/build-upload-a-docker-image.sh -b -c jaeger-es-index-cleaner -d cmd/es-index-cleaner -p "${platforms}" -t release bash scripts/build-upload-a-docker-image.sh -c jaeger-es-rollover -d plugin/storage/es -f Dockerfile.rollover -p "${platforms}" # build/upload images for jaeger-tracegen and jaeger-anonymizer