display ID registration prep, formatter registration fixes, index updates, dtype handlers, expanded data generators

.github/workflows/unit-tests.yaml

@@ -28,4 +28,4 @@ jobs:
           poetry install
       - name: Pytest - Unit tests
         run: |
-          poetry run pytest dx/tests -x
+          poetry run pytest dx/tests -xv

.gitignore

@@ -4,4 +4,6 @@ __pycache__/
 dist/
 
 .pytest_cache
+.python-versions
+.venv
 .vscode

CHANGELOG.md

@@ -1,6 +1,33 @@
 All notable changes will be documented here.
 
 ---
+## Unreleased
+_2022-08-21_
+
+### Added
+- `pd.Series` as a default renderable type (to go with the existing `pd.DataFrame` and `np.ndarray` types) 
+- Support for the following data types inside `pd.DataFrame` columns:
+    - `type` and `np.dtype` 
+    - `shapely.geometry` objects
+    - `pd.Timedelta` and `datetime.timedelta`
+    - `pd.Period`
+    - `pd.Interval`
+    - `complex` numbers
+    - `ipaddress.IPv4Address` and `.IPv6Address`
+- Extra dataset generation functions for development/testing under `dx.utils.datatypes`
+  - `dx.quick_random_dataframe(n_rows, n_columns)` to get a `pd.DataFrame` of 0.0-1.0 floats (convenience wrapper for `pd.DataFrame(np.random.rand(n_rows, n_columns))`)
+  - `dx.random_dataframe()` with different boolean values to enable based on available datatypes (`dx.DX_DATATYPES`)
+- `settings_context` context manager to allow temporarily changing a setting (or multiple)
+- Logging via `structlog` (default level: `logging.WARNING`)
+
+### Changed
+- Default sampling method changed from `outer` to `random`
+
+### Fixed
+- Displaying a dataframe with an out-of-order index (like with `.sample()`) no longer resets the index before sending data to the frontend.
+- Index/column flattening and string-formatting is behind settings and is handled more explicitly
+- `dx` should no longer interfere with other media type / mime bundles (e.g. matplotlib) formatted by the existing IPython display formatter
+
 ## `1.1.3`
 _2022-08-05_
 ### Added

dx/__init__.py

@@ -1,9 +1,11 @@
-from .config import *
+from .comms import *
 from .dx import *
 from .formatters import *
-from .helpers import *
+from .loggers import *
 from .settings import *
+from .utils import *
 
-__version__ = "1.1.3"
+__version__ = "1.2.0"
 
+configure_logging()
 set_display_mode("simple")

dx/comms.py

@@ -0,0 +1,37 @@
+import structlog
+from IPython import get_ipython
+
+from dx.settings import get_settings
+
+settings = get_settings()
+logger = structlog.get_logger(__name__)
+
+
+# ref: https://jupyter-notebook.readthedocs.io/en/stable/comms.html#opening-a-comm-from-the-frontend
+def target_func(comm, open_msg):
+    @comm.on_msg
+    def _recv(msg):
+        from dx.filtering import update_display_id
+
+        data = msg["content"]["data"]
+        if "display_id" in data:
+            update_display_id(
+                display_id=data["display_id"],
+                pandas_filter=data.get("pandas_filter"),
+                sql_filter=data.get("sql_filter"),
+                filters=data.get("filters"),
+                output_variable_name=data.get("output_variable_name"),
+                limit=data["limit"],
+            )
+
+    comm.send({"connected": True})
+
+
+ipython_shell = get_ipython()
+if (
+    ipython_shell is not None
+    and getattr(ipython_shell, "kernel", None)
+    and settings.ENABLE_DATALINK
+):
+    COMM_MANAGER = ipython_shell.kernel.comm_manager
+    COMM_MANAGER.register_target("datalink", target_func)

dx/dx.py

@@ -5,7 +5,7 @@
 from IPython.core.interactiveshell import InteractiveShell
 from IPython.display import display as ipydisplay
 
-from dx.settings import set_display_mode, settings
+from dx.settings import settings_context
 from dx.types import DXDisplayMode
 
 
@@ -29,11 +29,9 @@ def display(
             raise ValueError(f"Unsupported file type: `{path.suffix}`")
 
     df = pd.DataFrame(data)
+    with settings_context(display_mode=mode, ipython_shell=ipython_shell):
+        ipydisplay(df)
 
-    orig_mode = settings.DISPLAY_MODE.value
-    set_display_mode(mode, ipython_shell=ipython_shell)
-    ipydisplay(df)
-    set_display_mode(orig_mode, ipython_shell=ipython_shell)
     return
 
 

dx/filtering.py

@@ -0,0 +1,88 @@
+from typing import Optional
+
+import pandas as pd
+import structlog
+from IPython.display import update_display
+
+from dx.formatters.callouts import display_callout
+from dx.settings import get_settings, settings_context
+from dx.utils.formatting import expand_sequences
+from dx.utils.tracking import (
+    DATAFRAME_HASH_TO_VAR_NAME,
+    DISPLAY_ID_TO_DATAFRAME_HASH,
+    SUBSET_TO_DATAFRAME_HASH,
+    generate_df_hash,
+)
+
+logger = structlog.get_logger(__name__)
+
+settings = get_settings()
+
+
+SUBSET_FILTERS = {}
+
+
+def update_display_id(
+    display_id: str,
+    sql_filter: str,
+    pandas_filter: Optional[str] = None,
+    filters: Optional[dict] = None,
+    output_variable_name: Optional[str] = None,
+    limit: Optional[int] = None,
+) -> None:
+    """
+    Filters the dataframe in the cell with the given display_id.
+    """
+    from dx.utils.tracking import sql_engine
+
+    global SUBSET_FILTERS
+
+    row_limit = limit or settings.DISPLAY_MAX_ROWS
+    df_hash = DISPLAY_ID_TO_DATAFRAME_HASH[display_id]
+    df_name = DATAFRAME_HASH_TO_VAR_NAME[df_hash]
+    table_name = f"{df_name}__{df_hash}"
+
+    query_string = sql_filter.format(table_name=table_name)
+    logger.debug(f"sql query string: {query_string}")
+    new_df = pd.read_sql(query_string, sql_engine)
+
+    # in the event there were nested values stored,
+    # try to expand them back to their original datatypes
+    for col in new_df.columns:
+        new_df[col] = new_df[col].apply(expand_sequences)
+
+    # this is associating the subset with the original dataframe,
+    # which will be checked when the DisplayFormatter.format() is called
+    # during update_display(), which will prevent re-registering the display ID to the subset
+    new_df_hash = generate_df_hash(new_df)
+
+    # store filters to be passed through metadata to the frontend
+    logger.debug(f"applying {filters=}")
+    filters = filters or []
+    SUBSET_FILTERS[new_df_hash] = filters
+
+    logger.debug(f"assigning subset {new_df_hash} to parent {df_hash=}")
+    SUBSET_TO_DATAFRAME_HASH[new_df_hash] = df_hash
+
+    # allow temporary override of the display limit
+    with settings_context(DISPLAY_MAX_ROWS=row_limit):
+        logger.debug(f"updating {display_id=} with {min(row_limit, len(new_df))}-row resample")
+        update_display(new_df, display_id=display_id)
+
+    # we can't reference a variable type to suggest to users to perform a `df.query()`
+    # type operation since it was never declared in the first place
+    if not df_name.startswith("unk_dataframe_"):
+        # TODO: replace with custom callout media type
+        output_variable_name = output_variable_name or "new_df"
+        filter_code = f"""{output_variable_name} = {df_name}.query("{pandas_filter.format(df_name=df_name)}", engine="python")"""
+        filter_msg = f"""Copy the following snippet into a cell below to save this subset to a new dataframe:
+        <pre style="background-color:white; padding:0.5rem; border-radius:5px;">{filter_code}</pre>
+        """
+        display_callout(
+            filter_msg,
+            header=False,
+            icon="info",
+            level="success",
+            display_id=display_id + "-primary",
+            update=True,
+        )

dx/formatters/callouts.py

@@ -2,7 +2,7 @@
 import uuid
 from typing import Optional
 
-from IPython.display import HTML, display
+from IPython.display import HTML, display, update_display
 from pydantic import BaseModel
 
 
@@ -37,11 +37,18 @@ def html(self):
             callout_classes.append(f"bp3-icon-{self.icon.value}-sign")
         callout_class_str = " ".join(callout_classes)
 
+        msg = self.message
         if self.use_header:
             heading_html = f"<h6 class='bp3-heading'>{self.level.value.title()}</h6>"
-            return f"""<div class="{callout_class_str}" style="margin-bottom: 0.5rem">{heading_html}{self.message}</div>"""
+            msg = f"{heading_html}{self.message}"
 
-        return f"""<div class="{callout_class_str}" style="margin-bottom: 0.5rem">{self.message}</div>"""
+        style = ";".join(
+            [
+                "margin-bottom: 0.5rem",
+                "margin-top: 0.5rem",
+            ]
+        )
+        return f"""<div class="{callout_class_str}" style="{style}">{msg}</div>"""
 
 
 def display_callout(
@@ -50,6 +57,7 @@ def display_callout(
     header: bool = True,
     icon: Optional[CalloutIcon] = None,
     display_id: str = None,
+    update: bool = False,
 ) -> None:
     callout = Callout(
         message=message,
@@ -61,4 +69,10 @@ def display_callout(
 
     # TODO: coordinate with frontend to replace this with a standalone media type
     # instead of rendering HTML with custom classes/styles
-    display(HTML(callout.html), display_id=display_id)
+    if update:
+        update_display(HTML(callout.html), display_id=display_id)
+    else:
+        display(
+            HTML(callout.html),
+            display_id=display_id,
+        )