From a1e6c061fb12e2ee574e30e19f8626d54302c3af Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Thu, 20 Dec 2018 17:03:19 +0100 Subject: [PATCH] Fix ml job because of missing Datafeed in the logs (#9718) The log output from ML changed. Because of this the check on the Beats side has to be adjusted. The old check was kept in place for backward compatiblity. ``` 2018-12-20T14:11:14.356ZERRORinstance/beat.go:906Exiting: 2 errors: Error setting up ML for apache2: 5 errors: [status_exception] A datafeed [datafeed-filebeat-apache2-access-visitor_rate] already exists for job [filebeat-apache2-access-visitor_rate]; [status_exception] A datafeed [datafeed-filebeat-apache2-access-response_code] already exists for job [filebeat-apache2-access-response_code]; [status_exception] A datafeed [datafeed-filebeat-apache2-access-remote_ip_url_count] already exists for job [filebeat-apache2-access-remote_ip_url_count]; [status_exception] A datafeed [datafeed-filebeat-apache2-access-low_request_rate] already exists for job [filebeat-apache2-access-low_request_rate]; [status_exception] A datafeed [datafeed-filebeat-apache2-access-remote_ip_request_rate] already exists for job [filebeat-apache2-access-remote_ip_request_rate]; Error setting up ML for nginx: 5 errors: [status_exception] A datafeed [datafeed-filebeat-nginx-access-visitor_rate] already exists for job [filebeat-nginx-access-visitor_rate]; [status_exception] A datafeed [datafeed-filebeat-nginx-access-remote_ip_url_count] already exists for job [filebeat-nginx-access-remote_ip_url_count]; [status_exception] A datafeed [datafeed-filebeat-nginx-access-response_code] already exists for job [filebeat-nginx-access-response_code]; [status_exception] A datafeed [datafeed-filebeat-nginx-access-remote_ip_request_rate] already exists for job [filebeat-nginx-access-remote_ip_request_rate]; [status_exception] A datafeed [datafeed-filebeat-nginx-access-low_request_rate] already exists for job [filebeat-nginx-access-low_request_rate] ``` (cherry picked from commit 837929e647395fa3c0641dd0c4fd87c2a4a3c3aa) --- filebeat/tests/system/test_ml.py | 3 +++ libbeat/ml-importer/importer.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/filebeat/tests/system/test_ml.py b/filebeat/tests/system/test_ml.py index 80786090e7c..7c798b155d7 100644 --- a/filebeat/tests/system/test_ml.py +++ b/filebeat/tests/system/test_ml.py @@ -93,6 +93,9 @@ def _run_ml_test(self, setup_flag, modules_flag): "-c", cfgfile ] + # Skipping dashboard loading to speed up tests, unfortunately only works for setup and not --setup + cmd += ["-E", "setup.dashboards.enabled=false"] + if setup_flag: cmd += ["--setup"] else: diff --git a/libbeat/ml-importer/importer.go b/libbeat/ml-importer/importer.go index 47b8b93b3ae..1c384f7c445 100644 --- a/libbeat/ml-importer/importer.go +++ b/libbeat/ml-importer/importer.go @@ -228,7 +228,7 @@ func checkResponse(r []byte) error { for _, feed := range resp.Datafeeds { if !feed.Success { - if strings.HasPrefix(feed.Error.Msg, "[resource_already_exists_exception]") { + if strings.HasPrefix(feed.Error.Msg, "[status_exception] A datafeed") || strings.HasPrefix(feed.Error.Msg, "[resource_already_exists_exception]") { logp.Debug("machine-learning", "Datafeed already exists: %s, error: %s", feed.ID, feed.Error.Msg) continue }