From 8bcd60681262ed94ec7ea991246422a2b57aa45b Mon Sep 17 00:00:00 2001 From: ankit Date: Mon, 18 Oct 2021 14:44:34 +0530 Subject: [PATCH 1/2] Updated data time check to handle hourly granularity --- api/ops/tasks/detection/core/detectionTypes/lifetime.py | 2 +- .../tasks/detection/core/detectionTypes/percentageChange.py | 2 +- api/ops/tasks/detection/core/detectionTypes/prophet.py | 4 +--- api/ops/tasks/detection/core/detectionTypes/valueThreshold.py | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/api/ops/tasks/detection/core/detectionTypes/lifetime.py b/api/ops/tasks/detection/core/detectionTypes/lifetime.py index bc1b7d0..79b38e1 100644 --- a/api/ops/tasks/detection/core/detectionTypes/lifetime.py +++ b/api/ops/tasks/detection/core/detectionTypes/lifetime.py @@ -36,7 +36,7 @@ def lifetimeDetect(df, granularity): df["ds"] = pd.to_datetime(df["ds"]) df = df.sort_values("ds") df["ds"] = df["ds"].apply(lambda date: date.isoformat()[:19]) - todayISO = today.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None).isoformat()[:19] + todayISO = today.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None).isoformat()[:19] if granularity == "day" else today.replace(minute=0, second=0, microsecond=0, tzinfo=None).isoformat()[:19] df = df[df["ds"] < todayISO] maxVal = df.y.max() minVal = df.y.min() diff --git a/api/ops/tasks/detection/core/detectionTypes/percentageChange.py b/api/ops/tasks/detection/core/detectionTypes/percentageChange.py index 1cc2bb5..8a7b062 100644 --- a/api/ops/tasks/detection/core/detectionTypes/percentageChange.py +++ b/api/ops/tasks/detection/core/detectionTypes/percentageChange.py @@ -31,7 +31,7 @@ def percentChangeDetect(df, granularity, threshold): df["ds"] = pd.to_datetime(df["ds"]) df = df.sort_values("ds") df["ds"] = df["ds"].apply(lambda date: date.isoformat()[:19]) - todayISO = today.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None).isoformat()[:19] + todayISO = today.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None).isoformat()[:19] if granularity == "day" else today.replace(minute=0, second=0, microsecond=0, tzinfo=None).isoformat()[:19] df = df[df["ds"] < todayISO] df["percentageChange"] = 100 * (df.y - df.y.shift(1)) / df.y.shift(1) df["anomaly"] = (abs(df["percentageChange"]) > threshold) * 14 + 1 diff --git a/api/ops/tasks/detection/core/detectionTypes/prophet.py b/api/ops/tasks/detection/core/detectionTypes/prophet.py index 57b01a0..1ac9ab2 100644 --- a/api/ops/tasks/detection/core/detectionTypes/prophet.py +++ b/api/ops/tasks/detection/core/detectionTypes/prophet.py @@ -43,9 +43,7 @@ def prophetDetect(df, granularity, iterations=None): today = dt.datetime.now() df["ds"] = pd.to_datetime(df["ds"]) df["ds"] = df["ds"].apply(lambda date: date.isoformat()[:19]) - todayISO = today.replace( - hour=0, minute=0, second=0, microsecond=0, tzinfo=None - ).isoformat()[:19] + todayISO = today.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None).isoformat()[:19] if granularity == "day" else today.replace(minute=0, second=0, microsecond=0, tzinfo=None).isoformat()[:19] df = df[df["ds"] < todayISO] lastActualRow = df[-1:] lastISO = df.iloc[-1]["ds"] diff --git a/api/ops/tasks/detection/core/detectionTypes/valueThreshold.py b/api/ops/tasks/detection/core/detectionTypes/valueThreshold.py index fed896e..e4bea1a 100644 --- a/api/ops/tasks/detection/core/detectionTypes/valueThreshold.py +++ b/api/ops/tasks/detection/core/detectionTypes/valueThreshold.py @@ -53,7 +53,7 @@ def valueThresholdDetect(df, granularity, operator, value1, value2): df["ds"] = pd.to_datetime(df["ds"]) df = df.sort_values("ds") df["ds"] = df["ds"].apply(lambda date: date.isoformat()[:19]) - todayISO = today.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None).isoformat()[:19] + todayISO = today.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None).isoformat()[:19] if granularity == "day" else today.replace(minute=0, second=0, microsecond=0, tzinfo=None).isoformat()[:19] df = df[df["ds"] < todayISO] df["anomaly"] = eval(operationDict[operator]) anomalyLatest = checkLatestAnomaly(df, operationStrDict[operator]) From 9a9ba976512e3a08934c195b22e199d690f05c29 Mon Sep 17 00:00:00 2001 From: ankit Date: Wed, 27 Oct 2021 09:34:56 +0530 Subject: [PATCH 2/2] Today selection modified --- api/ops/tasks/detection/core/detectionTypes/lifetime.py | 6 ++++-- .../tasks/detection/core/detectionTypes/percentageChange.py | 6 ++++-- api/ops/tasks/detection/core/detectionTypes/prophet.py | 6 ++++-- .../tasks/detection/core/detectionTypes/valueThreshold.py | 6 ++++-- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/api/ops/tasks/detection/core/detectionTypes/lifetime.py b/api/ops/tasks/detection/core/detectionTypes/lifetime.py index 79b38e1..e1b1c66 100644 --- a/api/ops/tasks/detection/core/detectionTypes/lifetime.py +++ b/api/ops/tasks/detection/core/detectionTypes/lifetime.py @@ -32,11 +32,13 @@ def lifetimeDetect(df, granularity): """ Method to perform anomaly detection on given dataframe """ - today = dt.datetime.now() + today = dt.datetime.now().replace(minute=0, second=0, microsecond=0, tzinfo=None) df["ds"] = pd.to_datetime(df["ds"]) df = df.sort_values("ds") df["ds"] = df["ds"].apply(lambda date: date.isoformat()[:19]) - todayISO = today.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None).isoformat()[:19] if granularity == "day" else today.replace(minute=0, second=0, microsecond=0, tzinfo=None).isoformat()[:19] + if granularity == "day": + today = today.replace(hour=0) + todayISO = today.isoformat()[:19] df = df[df["ds"] < todayISO] maxVal = df.y.max() minVal = df.y.min() diff --git a/api/ops/tasks/detection/core/detectionTypes/percentageChange.py b/api/ops/tasks/detection/core/detectionTypes/percentageChange.py index 8a7b062..f1e2492 100644 --- a/api/ops/tasks/detection/core/detectionTypes/percentageChange.py +++ b/api/ops/tasks/detection/core/detectionTypes/percentageChange.py @@ -27,11 +27,13 @@ def percentChangeDetect(df, granularity, threshold): Method to perform anomaly detection on given dataframe using fbProphet """ threshold = float(threshold) - today = dt.datetime.now() + today = dt.datetime.now().replace(minute=0, second=0, microsecond=0, tzinfo=None) df["ds"] = pd.to_datetime(df["ds"]) df = df.sort_values("ds") df["ds"] = df["ds"].apply(lambda date: date.isoformat()[:19]) - todayISO = today.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None).isoformat()[:19] if granularity == "day" else today.replace(minute=0, second=0, microsecond=0, tzinfo=None).isoformat()[:19] + if granularity == "day": + today = today.replace(hour=0) + todayISO = today.isoformat()[:19] df = df[df["ds"] < todayISO] df["percentageChange"] = 100 * (df.y - df.y.shift(1)) / df.y.shift(1) df["anomaly"] = (abs(df["percentageChange"]) > threshold) * 14 + 1 diff --git a/api/ops/tasks/detection/core/detectionTypes/prophet.py b/api/ops/tasks/detection/core/detectionTypes/prophet.py index 1ac9ab2..95ca856 100644 --- a/api/ops/tasks/detection/core/detectionTypes/prophet.py +++ b/api/ops/tasks/detection/core/detectionTypes/prophet.py @@ -40,10 +40,12 @@ def prophetDetect(df, granularity, iterations=None): """ Method to perform anomaly detection on given dataframe using fbProphet """ - today = dt.datetime.now() + today = dt.datetime.now().replace(minute=0, second=0, microsecond=0, tzinfo=None) df["ds"] = pd.to_datetime(df["ds"]) df["ds"] = df["ds"].apply(lambda date: date.isoformat()[:19]) - todayISO = today.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None).isoformat()[:19] if granularity == "day" else today.replace(minute=0, second=0, microsecond=0, tzinfo=None).isoformat()[:19] + if granularity == "day": + today = today.replace(hour=0) + todayISO = today.isoformat()[:19] df = df[df["ds"] < todayISO] lastActualRow = df[-1:] lastISO = df.iloc[-1]["ds"] diff --git a/api/ops/tasks/detection/core/detectionTypes/valueThreshold.py b/api/ops/tasks/detection/core/detectionTypes/valueThreshold.py index e4bea1a..97ef9b5 100644 --- a/api/ops/tasks/detection/core/detectionTypes/valueThreshold.py +++ b/api/ops/tasks/detection/core/detectionTypes/valueThreshold.py @@ -49,11 +49,13 @@ def valueThresholdDetect(df, granularity, operator, value1, value2): "between": '((df["y"] >= lowerVal) & (df["y"] <= upperVal)) * 14 + 1', "!between": '((df["y"] < lowerVal) | (df["y"] > upperVal)) * 14 + 1' } - today = dt.datetime.now() + today = dt.datetime.now().replace(minute=0, second=0, microsecond=0, tzinfo=None) df["ds"] = pd.to_datetime(df["ds"]) df = df.sort_values("ds") df["ds"] = df["ds"].apply(lambda date: date.isoformat()[:19]) - todayISO = today.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None).isoformat()[:19] if granularity == "day" else today.replace(minute=0, second=0, microsecond=0, tzinfo=None).isoformat()[:19] + if granularity == "day": + today = today.replace(hour=0) + todayISO = today.isoformat()[:19] df = df[df["ds"] < todayISO] df["anomaly"] = eval(operationDict[operator]) anomalyLatest = checkLatestAnomaly(df, operationStrDict[operator])