From 42cd9267d5ca0ed2bd644707a61e63c303a08a26 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Thu, 20 Dec 2018 17:03:19 +0100 Subject: [PATCH] Fix ml job because of missing Datafeed in the logs (#9718) The log output from ML changed. Because of this the check on the Beats side has to be adjusted. The old check was kept in place for backward compatiblity. ``` 2018-12-20T14:11:14.356ZERRORinstance/beat.go:906Exiting: 2 errors: Error setting up ML for apache2: 5 errors: [status_exception] A datafeed [datafeed-filebeat-apache2-access-visitor_rate] already exists for job [filebeat-apache2-access-visitor_rate]; [status_exception] A datafeed [datafeed-filebeat-apache2-access-response_code] already exists for job [filebeat-apache2-access-response_code]; [status_exception] A datafeed [datafeed-filebeat-apache2-access-remote_ip_url_count] already exists for job [filebeat-apache2-access-remote_ip_url_count]; [status_exception] A datafeed [datafeed-filebeat-apache2-access-low_request_rate] already exists for job [filebeat-apache2-access-low_request_rate]; [status_exception] A datafeed [datafeed-filebeat-apache2-access-remote_ip_request_rate] already exists for job [filebeat-apache2-access-remote_ip_request_rate]; Error setting up ML for nginx: 5 errors: [status_exception] A datafeed [datafeed-filebeat-nginx-access-visitor_rate] already exists for job [filebeat-nginx-access-visitor_rate]; [status_exception] A datafeed [datafeed-filebeat-nginx-access-remote_ip_url_count] already exists for job [filebeat-nginx-access-remote_ip_url_count]; [status_exception] A datafeed [datafeed-filebeat-nginx-access-response_code] already exists for job [filebeat-nginx-access-response_code]; [status_exception] A datafeed [datafeed-filebeat-nginx-access-remote_ip_request_rate] already exists for job [filebeat-nginx-access-remote_ip_request_rate]; [status_exception] A datafeed [datafeed-filebeat-nginx-access-low_request_rate] already exists for job [filebeat-nginx-access-low_request_rate] ``` --- filebeat/tests/system/test_ml.py | 3 +++ libbeat/ml-importer/importer.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/filebeat/tests/system/test_ml.py b/filebeat/tests/system/test_ml.py index 80786090e7c9..7c798b155d7c 100644 --- a/filebeat/tests/system/test_ml.py +++ b/filebeat/tests/system/test_ml.py @@ -93,6 +93,9 @@ def _run_ml_test(self, setup_flag, modules_flag): "-c", cfgfile ] + # Skipping dashboard loading to speed up tests, unfortunately only works for setup and not --setup + cmd += ["-E", "setup.dashboards.enabled=false"] + if setup_flag: cmd += ["--setup"] else: diff --git a/libbeat/ml-importer/importer.go b/libbeat/ml-importer/importer.go index 892c8dae47ad..fa0c84e0f0d7 100644 --- a/libbeat/ml-importer/importer.go +++ b/libbeat/ml-importer/importer.go @@ -229,7 +229,7 @@ func checkResponse(r []byte) error { for _, feed := range resp.Datafeeds { if !feed.Success { - if strings.HasPrefix(feed.Error.Msg, "[resource_already_exists_exception]") { + if strings.HasPrefix(feed.Error.Msg, "[status_exception] A datafeed") || strings.HasPrefix(feed.Error.Msg, "[resource_already_exists_exception]") { logp.Debug("machine-learning", "Datafeed already exists: %s, error: %s", feed.ID, feed.Error.Msg) continue }