Update pandas requirement from <2.0.0,>=1.4.0 to >=1.4.0,<3.0.0 (#653)

* Update pandas requirement from <2.0.0,>=1.4.0 to >=1.4.0,<3.0.0 Updates the requirements on [pandas](https://github.com/pandas-dev/pandas) to permit the latest version. - [Release notes](https://github.com/pandas-dev/pandas/releases) - [Changelog](https://github.com/pandas-dev/pandas/blob/main/RELEASE.md) - [Commits](pandas-dev/pandas@v1.4.0...v2.0.0) --- updated-dependencies: - dependency-name: pandas dependency-type: direct:production ... Signed-off-by: dependabot[bot] <[email protected]> * Changes to support pandas 2.0.0 Fixed time conversion in entity_graph_tools.py Removed infer_datetime_format from read_csv Fixed unix timestamp conversion in local_os_query_driver.py --------- Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ian Hellen <[email protected]>
microsoft · Aug 1, 2023 · 7ee9ddd · 7ee9ddd
1 parent d854d9e
commit 7ee9ddd
Show file tree

Hide file tree

Showing 11 changed files with 38 additions and 48 deletions.
diff --git a/conda/conda-reqs.txt b/conda/conda-reqs.txt
@@ -31,7 +31,7 @@ msrest>=0.6.0
 msrestazure>=0.6.0
 networkx>=2.2
 numpy>=1.15.4
-pandas>=1.4.0, <2.0.0
+pandas>=1.4.0, <3.0.0
 panel>=0.14.4
 pygments>=2.0.0
 pyjwt>=2.3.0

diff --git a/msticpy/common/timespan.py b/msticpy/common/timespan.py
@@ -176,7 +176,7 @@ def _parse_time(time_val, prop_name):
             return time_val
         with contextlib.suppress(ValueError, ParserError):
             if isinstance(time_val, str):
-                return pd.to_datetime(time_val, infer_datetime_format=True)
+                return pd.to_datetime(time_val)
         raise ValueError(f"'{prop_name}' must be a datetime or a datetime string.")
 
     @staticmethod

diff --git a/msticpy/data/drivers/local_data_driver.py b/msticpy/data/drivers/local_data_driver.py
@@ -132,9 +132,7 @@ def query(
             )
         if file_path.endswith("csv"):
             try:
-                return pd.read_csv(
-                    file_path, infer_datetime_format=True, parse_dates=["TimeGenerated"]
-                )
+                return pd.read_csv(file_path, parse_dates=["TimeGenerated"])
             except ValueError:
                 return pd.read_csv(file_path)
         data_df = pd.read_pickle(file_path)  # nosec

diff --git a/msticpy/data/drivers/local_osquery_driver.py b/msticpy/data/drivers/local_osquery_driver.py
@@ -16,7 +16,7 @@
 from typing import Any, Dict, List, Optional, Union
 
 import pandas as pd
-from pandas import to_datetime
+from pandas import to_datetime, to_numeric
 from tqdm.auto import tqdm
 
 from ..._version import VERSION
@@ -268,9 +268,8 @@ def _extract_event_type(self, df_all_queries: pd.DataFrame, event_name: str):
             axis=1, how="all"
         )
         for date_column in self.OS_QUERY_DATEIME_COLS & set(query_df.columns):
-            query_df[date_column] = pd.to_datetime(
-                query_df[date_column],
-                unit="s",
+            query_df[date_column] = to_datetime(
+                to_numeric(query_df[date_column]),
                 origin="unix",
                 utc=True,
             )

diff --git a/msticpy/vis/entity_graph_tools.py b/msticpy/vis/entity_graph_tools.py
@@ -4,7 +4,7 @@
 # license information.
 # --------------------------------------------------------------------------
 """Creates an entity graph for a Microsoft Sentinel Incident."""
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import List, Optional, Union
 
 import networkx as nx
@@ -14,6 +14,7 @@
 from bokeh.layouts import column
 from bokeh.models import Circle, HoverTool, Label, LayoutDOM  # type: ignore
 from bokeh.plotting import figure, from_networkx
+from dateutil import parser
 
 from .._version import VERSION
 from ..common.exceptions import MsticpyUserError
@@ -135,6 +136,7 @@ def _plot_with_timeline(self, hide: bool = False, **kwargs) -> LayoutDOM:
         timeline = None
         tl_df = self.to_df()
         tl_type = "duration"
+        # pylint: disable=unsubscriptable-object
         if len(tl_df["EndTime"].unique()) == 1 and not tl_df["EndTime"].unique()[0]:
             tl_type = "discreet"
             if (
@@ -143,9 +145,9 @@ def _plot_with_timeline(self, hide: bool = False, **kwargs) -> LayoutDOM:
             ):
                 print("No timestamps available to create timeline")
                 return self._plot_no_timeline(timeline=False, hide=hide, **kwargs)
-        tl_df["TimeGenerated"] = pd.to_datetime(tl_df["TimeGenerated"], utc=True)
-        tl_df["StartTime"] = pd.to_datetime(tl_df["StartTime"], utc=True)
-        tl_df["EndTime"] = pd.to_datetime(tl_df["EndTime"], utc=True)
+        # tl_df["TimeGenerated"] = pd.to_datetime(tl_df["TimeGenerated"], utc=True)
+        # tl_df["StartTime"] = pd.to_datetime(tl_df["StartTime"], utc=True)
+        # tl_df["EndTime"] = pd.to_datetime(tl_df["EndTime"], utc=True)
         graph = self._plot_no_timeline(hide=True, **kwargs)
         if tl_type == "duration":
             timeline = display_timeline_duration(
@@ -320,35 +322,20 @@ def remove_node(self, name: str):
 
     def to_df(self) -> pd.DataFrame:
         """Generate a dataframe of nodes in the graph."""
-        names = [node[1]["Name"] for node in self.alertentity_graph.nodes.items()]
-        descs = [
-            node[1]["Description"] for node in self.alertentity_graph.nodes.items()
-        ]
-        types = [node[1]["Type"] for node in self.alertentity_graph.nodes.items()]
-        times = [
-            node[1]["TimeGenerated"] if "TimeGenerated" in node[1] else None
-            for node in self.alertentity_graph.nodes.items()
-        ]
-        starttimes = [
-            node[1]["StartTime"] if "StartTime" in node[1] else node[1]["TimeGenerated"]
-            for node in self.alertentity_graph.nodes.items()
-        ]
-        endtimes = [
-            node[1]["EndTime"] if "EndTime" in node[1] else None
-            for node in self.alertentity_graph.nodes.items()
-        ]
-        tl_df = pd.DataFrame(
+        node_list = [
             {
-                "Name": names,
-                "Description": descs,
-                "Type": types,
-                "TimeGenerated": times,
-                "EndTime": endtimes,
-                "StartTime": starttimes,
+                "Name": node.get("Name"),
+                "Description": node.get("Description"),
+                "Type": node.get("Type"),
+                "TimeGenerated": _convert_to_tz_aware_ts(node.get("TimeGenerated")),
+                "EndTime": _convert_to_tz_aware_ts(node.get("EndTime")),
+                "StartTime": _convert_to_tz_aware_ts(
+                    node.get("StartTime", node.get("TimeGenerated"))
+                ),
             }
-        )
-        tl_df.replace("None", np.NaN, inplace=True)
-        return tl_df
+            for node in self.alertentity_graph.nodes.values()
+        ]
+        return pd.DataFrame(node_list).replace("None", np.NaN)
 
     def _add_incident_or_alert_node(self, incident: Union[Incident, Alert, None]):
         """Check what type of entity is passed in and creates relevant graph."""
@@ -402,6 +389,16 @@ def graph(self) -> nx.Graph:
         return self.alertentity_graph
 
 
+def _convert_to_tz_aware_ts(date_string: Optional[str]) -> Optional[datetime]:
+    """Convert a date string to a timezone aware datetime object."""
+    if date_string is None:
+        return None
+    date_time = parser.parse(date_string)
+    if date_time.tzinfo is None:
+        return date_time.replace(tzinfo=timezone.utc)
+    return date_time
+
+
 def _dedupe_entities(alerts, ents) -> list:
     """Deduplicate incident and alert entities."""
     alert_entities = []

diff --git a/requirements-all.txt b/requirements-all.txt
@@ -40,7 +40,7 @@ nest_asyncio>=1.4.0
 networkx>=2.2
 numpy>=1.15.4  # pandas
 openpyxl>=3.0
-pandas>=1.4.0, <2.0.0
+pandas>=1.4.0, <3.0.0
 panel>=0.14.4
 passivetotal>=2.5.3
 pygments>=2.0.0

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -13,7 +13,7 @@ mccabe>=0.6.1
 mypy>=0.812
 nbdime>=2.1.0
 nbconvert>=6.1.0
-pandas>=1.4.0, <2.0.0
+pandas>=1.4.0, <3.0.0
 pep8-naming>=0.10.0
 pep8>=1.7.1
 pipreqs>=0.4.9

diff --git a/requirements.txt b/requirements.txt
@@ -24,7 +24,7 @@ msrestazure>=0.6.0
 nest_asyncio>=1.4.0
 networkx>=2.2
 numpy>=1.15.4  # pandas
-pandas>=1.4.0, <2.0.0
+pandas>=1.4.0, <3.0.0
 pygments>=2.0.0
 pyjwt>=2.3.0
 python-dateutil>=2.8.1  # pandas

diff --git a/tests/vis/test_data_viewer.py b/tests/vis/test_data_viewer.py
@@ -32,7 +32,6 @@ def dataframe():
         str(df_path),
         index_col=0,
         parse_dates=["TimeGenerated"],
-        infer_datetime_format=True,
     )
 
 

diff --git a/tests/vis/test_data_viewer_panel.py b/tests/vis/test_data_viewer_panel.py
@@ -33,7 +33,6 @@ def dataframe():
         str(df_path),
         index_col=0,
         parse_dates=["TimeGenerated"],
-        infer_datetime_format=True,
     )
 
 

diff --git a/tools/mp_demo_data.py b/tools/mp_demo_data.py
@@ -205,9 +205,7 @@ def read_pd_df(data_file, query_name):
         )
 
     if data_file.lower().endswith("csv"):
-        return pd.read_csv(
-            data_file, infer_datetime_format=True, parse_dates=["TimeGenerated"]
-        )
+        return pd.read_csv(data_file, parse_dates=["TimeGenerated"])
     return pd.read_pickle(data_file)