diff --git a/.readthedocs.yaml b/.readthedocs.yaml index bea8b7fa..7436a551 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -7,9 +7,9 @@ version: 2 # Set the version of Python and other tools you might need build: - os: ubuntu-22.04 + os: ubuntu-24.04 tools: - python: "3.11" + python: "3.12" # Build documentation in the docs/ directory with Sphinx sphinx: diff --git a/CHANGELOG.md b/CHANGELOG.md index 61e85778..96cc09ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - [Docs] Clarify how to reference query_key values in flatline alerts - [#1320](https://github.com/jertel/elastalert2/pull/1320) - @jertel - Fix percentiles aggregation type in Spike Metric Aggregation rules - [#1323](https://github.com/jertel/elastalert2/pull/1323) - @jertel - [Docs] Extend FAQ / troubleshooting section with information on Elasticsearch RBAC - [#1324](https://github.com/jertel/elastalert2/pull/1324) - @chr-b +- Upgrade to Python 3.12 - [#1327](https://github.com/jertel/elastalert2/pull/1327) - @jertel # 2.15.0 diff --git a/Dockerfile b/Dockerfile index a09b6706..9ce0ce3b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim as builder +FROM python:3.12-slim as builder LABEL description="ElastAlert 2 Official Image" LABEL maintainer="Jason Ertel" @@ -10,7 +10,7 @@ RUN mkdir -p /opt/elastalert && \ pip install setuptools wheel && \ python setup.py sdist bdist_wheel -FROM python:3.11-slim +FROM python:3.12-slim ARG GID=1000 ARG UID=1000 diff --git a/docs/source/running_elastalert.rst b/docs/source/running_elastalert.rst index 73b2374a..a64cc103 100644 --- a/docs/source/running_elastalert.rst +++ b/docs/source/running_elastalert.rst @@ -206,11 +206,11 @@ Requirements - Elasticsearch 7.x or 8.x, or OpenSearch 1.x or 2.x - ISO8601 or Unix timestamped data -- Python 3.11. Require OpenSSL 1.1.1 or newer. +- Python 3.12. Require OpenSSL 1.1.1 or newer. - pip -- Packages on Ubuntu 21.x: build-essential python3-pip python3.11 python3.11-dev libffi-dev libssl-dev +- Packages on Ubuntu 21.x: build-essential python3-pip python3.12 python3.12-dev libffi-dev libssl-dev -If you want to install python 3.11 on CentOS, please install python 3.11 from the source code after installing 'Development Tools'. +If you want to install python 3.12 on CentOS, please install python 3.12 from the source code after installing 'Development Tools'. Downloading and Configuring --------------------------- diff --git a/elastalert/alerters/gelf.py b/elastalert/alerters/gelf.py index 1a3f821c..02f8d5bb 100644 --- a/elastalert/alerters/gelf.py +++ b/elastalert/alerters/gelf.py @@ -65,7 +65,8 @@ def sent_tcp(self, gelf_msg): try: if self.ca_cert: - tcp_socket = ssl.wrap_socket(tcp_socket, ca_certs=self.ca_cert) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tcp_socket = ctx.wrap_socket(tcp_socket, ca_certs=self.ca_cert) tcp_socket.sendall(bytes_msg) else: tcp_socket.sendall(bytes_msg) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 96b885b9..d7a9db71 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -18,8 +18,6 @@ from socket import error import statsd - -import dateutil.tz import pytz from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.executors.pool import ThreadPoolExecutor @@ -1130,18 +1128,18 @@ def start(self): name='Internal: Handle Config Change') self.scheduler.start() while self.running: - next_run = datetime.datetime.utcnow() + self.run_every + next_run = datetime.datetime.now(tz=datetime.UTC) + self.run_every # Quit after end_time has been reached if self.args.end: endtime = ts_to_dt(self.args.end) - next_run_dt = next_run.replace(tzinfo=dateutil.tz.tzutc()) + next_run_dt = next_run.replace(tzinfo=timezone.utc) if next_run_dt > endtime: elastalert_logger.info("End time '%s' falls before the next run time '%s', exiting." % (endtime, next_run_dt)) exit(0) - if next_run < datetime.datetime.utcnow(): + if next_run < datetime.datetime.now(tz=datetime.UTC): continue # Show disabled rules @@ -1149,7 +1147,7 @@ def start(self): elastalert_logger.info("Disabled rules are: %s" % (str(self.get_disabled_rules()))) # Wait before querying again - sleep_duration = total_seconds(next_run - datetime.datetime.utcnow()) + sleep_duration = total_seconds(next_run - datetime.datetime.now(tz=datetime.UTC)) self.sleep_for(sleep_duration) def wait_until_responsive(self, timeout, clock=timeit.default_timer): @@ -1209,7 +1207,7 @@ def handle_config_change(self): def handle_rule_execution(self, rule): self.thread_data.alerts_sent = 0 - next_run = datetime.datetime.utcnow() + rule['run_every'] + next_run = datetime.datetime.now(tz=datetime.UTC) + rule['run_every'] # Set endtime based on the rule's delay delay = rule.get('query_delay') if hasattr(self.args, 'end') and self.args.end: @@ -1232,7 +1230,7 @@ def handle_rule_execution(self, rule): # That means that we need to pause execution after this run if endtime_epoch + rule['run_every'].total_seconds() < exec_next - 59: # apscheduler requires pytz tzinfos, so don't use unix_to_dt here! - rule['next_starttime'] = datetime.datetime.utcfromtimestamp(exec_next).replace(tzinfo=pytz.utc) + rule['next_starttime'] = datetime.datetime.fromtimestamp(exec_next, tz=datetime.UTC).replace(tzinfo=pytz.utc) if rule.get('limit_execution_coverage'): rule['next_min_starttime'] = rule['next_starttime'] if not rule['has_run_once']: @@ -1260,7 +1258,7 @@ def handle_rule_execution(self, rule): self.thread_data.alerts_sent = 0 - if next_run < datetime.datetime.utcnow(): + if next_run < datetime.datetime.now(tz=datetime.UTC): # We were processing for longer than our refresh interval # This can happen if --start was specified with a large time period # or if we are running too slow to process events in real time. diff --git a/elastalert/util.py b/elastalert/util.py index 3945dbcd..269c98bc 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -186,7 +186,8 @@ def dt_to_ts_with_format(dt, ts_format): def ts_now(): - return datetime.datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc()) + now = datetime.datetime.now(tz=datetime.UTC) + return now.replace(tzinfo=dateutil.tz.tzutc()) def ts_utc_to_tz(ts, tz_name): @@ -268,8 +269,8 @@ def total_seconds(dt): def dt_to_int(dt): - dt = dt.replace(tzinfo=None) - return int(total_seconds((dt - datetime.datetime.utcfromtimestamp(0))) * 1000) + dt = dt.replace(tzinfo=datetime.UTC) + return int(total_seconds((dt - datetime.datetime.fromtimestamp(0, tz=datetime.UTC))) * 1000) def unixms_to_dt(ts): @@ -277,7 +278,7 @@ def unixms_to_dt(ts): def unix_to_dt(ts): - dt = datetime.datetime.utcfromtimestamp(float(ts)) + dt = datetime.datetime.fromtimestamp(float(ts), tz=datetime.UTC) dt = dt.replace(tzinfo=dateutil.tz.tzutc()) return dt diff --git a/setup.py b/setup.py index 1688125c..825d06cf 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ "Discussion Forum": "https://github.com/jertel/elastalert2/discussions", }, classifiers=[ - 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'License :: OSI Approved :: Apache Software License', 'Operating System :: OS Independent', ], diff --git a/tests/Dockerfile-test b/tests/Dockerfile-test index bf2a1051..e9cb57bd 100644 --- a/tests/Dockerfile-test +++ b/tests/Dockerfile-test @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM python:3.12-slim RUN apt update && apt upgrade -y RUN apt install -y gcc libffi-dev diff --git a/tests/alerters/command_test.py b/tests/alerters/command_test.py index 170cc610..bbbbb4c0 100644 --- a/tests/alerters/command_test.py +++ b/tests/alerters/command_test.py @@ -20,7 +20,7 @@ def test_command_getinfo(): 'nested': {'field': 1}} with mock.patch("elastalert.alerters.command.subprocess.Popen") as mock_popen: alert.alert([match]) - assert mock_popen.called_with(['/bin/test', '--arg', 'foobarbaz'], stdin=subprocess.PIPE, shell=False) + mock_popen.assert_called_with(['/bin/test/', '--arg', 'foobarbaz'], stdin=subprocess.PIPE, shell=False) expected_data = { 'type': 'command', 'command': '/bin/test/ --arg foobarbaz' @@ -39,7 +39,7 @@ def test_command_old_style_string_format1(caplog): alert = CommandAlerter(rule) with mock.patch("elastalert.alerters.command.subprocess.Popen") as mock_popen: alert.alert([match]) - assert mock_popen.called_with('/bin/test --arg foobarbaz', stdin=subprocess.PIPE, shell=False) + mock_popen.assert_called_with(['/bin/test/ --arg foobarbaz'], stdin=subprocess.PIPE, shell=True) assert ('elastalert', logging.WARNING, 'Warning! You could be vulnerable to shell injection!') == caplog.record_tuples[0] assert ('elastalert', logging.INFO, 'Alert sent to Command') == caplog.record_tuples[1] @@ -53,7 +53,7 @@ def test_command_old_style_string_format2(): alert = CommandAlerter(rule) with mock.patch("elastalert.alerters.command.subprocess.Popen") as mock_popen: alert.alert([match]) - assert mock_popen.called_with('/bin/test/foo.sh', stdin=subprocess.PIPE, shell=True) + mock_popen.assert_called_with(['/bin/test/foo.sh'], stdin=subprocess.PIPE, shell=True) def test_command_pipe_match_json(): @@ -67,8 +67,8 @@ def test_command_pipe_match_json(): mock_popen.return_value = mock_subprocess mock_subprocess.communicate.return_value = (None, None) alert.alert([match]) - assert mock_popen.called_with(['/bin/test', '--arg', 'foobarbaz'], stdin=subprocess.PIPE, shell=False) - assert mock_subprocess.communicate.called_with(input=json.dumps(match)) + mock_popen.assert_called_with(['/bin/test/', '--arg', 'foobarbaz'], stdin=subprocess.PIPE, shell=False) + mock_subprocess.communicate.assert_called_with(input=(json.dumps([match]) + '\n').encode()) def test_command_pipe_alert_text(): @@ -83,8 +83,8 @@ def test_command_pipe_alert_text(): mock_popen.return_value = mock_subprocess mock_subprocess.communicate.return_value = (None, None) alert.alert([match]) - assert mock_popen.called_with(['/bin/test', '--arg', 'foobarbaz'], stdin=subprocess.PIPE, shell=False) - assert mock_subprocess.communicate.called_with(input=alert_text.encode()) + mock_popen.assert_called_with(['/bin/test/', '--arg', 'foobarbaz'], stdin=subprocess.PIPE, shell=False) + mock_subprocess.communicate.assert_called_with(input=alert_text.encode()) def test_command_fail_on_non_zero_exit(): @@ -99,7 +99,7 @@ def test_command_fail_on_non_zero_exit(): mock_popen.return_value = mock_subprocess mock_subprocess.wait.return_value = 1 alert.alert([match]) - assert mock_popen.called_with(['/bin/test', '--arg', 'foobarbaz'], stdin=subprocess.PIPE, shell=False) + mock_popen.assert_called_with(['/bin/test/', '--arg', 'foobarbaz'], stdin=subprocess.PIPE, shell=False) assert "Non-zero exit code while running command" in str(exception) diff --git a/tests/alerters/gelf_test.py b/tests/alerters/gelf_test.py index e663264b..ec3b30f0 100644 --- a/tests/alerters/gelf_test.py +++ b/tests/alerters/gelf_test.py @@ -211,7 +211,7 @@ def test_gelf_sent_tcp_with_custom_ca(caplog): expected_data = json.dumps(expected_data).encode('utf-8') + b'\x00' with mock.patch('socket.socket') as mock_socket: - with mock.patch('ssl.wrap_socket') as mock_ssl_wrap_socket: + with mock.patch('ssl.SSLContext.wrap_socket') as mock_ssl_wrap_socket: mock_ssl_wrap_socket.return_value = mock_socket alert.alert([match]) mock_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM) diff --git a/tests/base_test.py b/tests/base_test.py index 0c427f29..5212dd94 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -215,7 +215,7 @@ def test_match(ea): with mock.patch('elastalert.elastalert.elasticsearch_client'): ea.run_rule(ea.rules[0], END, START) - ea.rules[0]['alert'][0].alert.called_with({'@timestamp': END_TIMESTAMP}) + ea.rules[0]['alert'][0].alert.assert_called_with([{'@timestamp': END, 'num_hits': 0, 'num_matches': 1}]) assert ea.rules[0]['alert'][0].alert.call_count == 1 @@ -375,7 +375,6 @@ def test_agg_matchtime_timestamp_field(ea): with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es: ea.send_pending_alerts() # Assert that current_es was refreshed from the aggregate rules - assert mock_es.called_with(host='', port='') assert mock_es.call_count == 2 assert_alerts(ea, [hits_timestamps[:2], hits_timestamps[2:]]) @@ -505,7 +504,6 @@ def test_agg_matchtime(ea): with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es: ea.send_pending_alerts() # Assert that current_es was refreshed from the aggregate rules - assert mock_es.called_with(host='', port='') assert mock_es.call_count == 2 assert_alerts(ea, [hits_timestamps[:2], hits_timestamps[2:]]) @@ -607,7 +605,6 @@ def test_agg_with_aggregation_key(ea): mock_es.return_value = ea.thread_data.current_es ea.send_pending_alerts() # Assert that current_es was refreshed from the aggregate rules - assert mock_es.called_with(host='', port='') assert mock_es.call_count == 2 assert_alerts(ea, [[hits_timestamps[0], hits_timestamps[2]], [hits_timestamps[1]]]) @@ -641,7 +638,7 @@ def test_silence(ea): with mock.patch('elastalert.elastalert.ts_now') as mock_ts: with mock.patch('elastalert.elastalert.elasticsearch_client'): # Converted twice to add tzinfo - mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.utcnow() + datetime.timedelta(hours=5))) + mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.now(tz=datetime.UTC) + datetime.timedelta(hours=5))) ea.run_rule(ea.rules[0], END, START) assert ea.rules[0]['alert'][0].alert.call_count == 1 @@ -684,7 +681,7 @@ def test_silence_query_key(ea): with mock.patch('elastalert.elastalert.ts_now') as mock_ts: with mock.patch('elastalert.elastalert.elasticsearch_client'): # Converted twice to add tzinfo - mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.utcnow() + datetime.timedelta(hours=5))) + mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.now(tz=datetime.UTC) + datetime.timedelta(hours=5))) ea.run_rule(ea.rules[0], END, START) assert ea.rules[0]['alert'][0].alert.call_count == 2 @@ -711,7 +708,7 @@ def test_realert(ea): with mock.patch('elastalert.elastalert.ts_now') as mock_ts: with mock.patch('elastalert.elastalert.elasticsearch_client'): # mock_ts is converted twice to add tzinfo - mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.utcnow() + datetime.timedelta(minutes=10))) + mock_ts.return_value = ts_to_dt(dt_to_ts(datetime.datetime.now(tz=datetime.UTC) + datetime.timedelta(minutes=10))) ea.rules[0]['type'].matches = matches ea.run_rule(ea.rules[0], END, START) assert ea.rules[0]['alert'][0].alert.call_count == 2 diff --git a/tests/tox.ini b/tests/tox.ini index 7933b4a2..79e33217 100644 --- a/tests/tox.ini +++ b/tests/tox.ini @@ -1,6 +1,6 @@ [tox] project = elastalert -envlist = py311,docs +envlist = py312,docs setupdir = .. [testenv] diff --git a/tests/util_test.py b/tests/util_test.py index 9ad1ef6d..6256d732 100644 --- a/tests/util_test.py +++ b/tests/util_test.py @@ -65,8 +65,8 @@ def test_parse_deadline(spec, expected_deadline): # Note: Can't mock ``utcnow`` directly because ``datetime`` is a built-in. class MockDatetime(datetime): @staticmethod - def utcnow(): - return dt('2017-07-07T10:00:00.000Z') + def now(tz=None): + return datetime(2017, 7, 7, 10, 0, 0) with mock.patch('datetime.datetime', MockDatetime): assert parse_deadline(spec) == expected_deadline