Merge pull request #60 from ebmdatalab/always-be-backfillin-slack

Always be backfill slack on every run

metrics/slack/api.py

@@ -1,5 +1,3 @@
-from datetime import datetime, time, timedelta
-
 from slack_bolt import App
 
 
@@ -10,11 +8,8 @@ def get_app(signing_secret, token):
     return App(token=token, signing_secret=signing_secret)
 
 
-def iter_messages(app, channel_id, date=None):
+def iter_messages(app, channel_id):
     start = end = 0
-    if date:
-        start = datetime.combine(date, time()).timestamp()
-        end = (datetime.combine(date, time()) + timedelta(days=1)).timestamp()
 
     for page in app.client.conversations_history(
         channel=channel_id,

metrics/slack/cli.py

@@ -2,12 +2,18 @@
 from datetime import datetime
 
 import click
+import structlog
+from sqlalchemy import create_engine
 
-from ..timescaledb import TimescaleDBWriter
+from ..timescaledb import TimescaleDBWriter, drop_tables
 from ..timescaledb.tables import SlackTechSupport
+from ..timescaledb.writer import TIMESCALEDB_URL
 from .api import get_app, iter_messages
 
 
+log = structlog.get_logger()
+
+
 @click.group()
 @click.option("--signing-secret", required=True, envvar="SLACK_SIGNING_SECRET")
 @click.option("--token", required=True, envvar="SLACK_TOKEN")
@@ -20,21 +26,22 @@ def slack(ctx, signing_secret, token):
 
 
 @slack.command()
-@click.argument("date", type=click.DateTime(), required=False)
 @click.option(
     "--tech-support-channel-id", required=True, envvar="SLACK_TECH_SUPPORT_CHANNEL_ID"
 )
-@click.option("--backfill", is_flag=True)
 @click.pass_context
-def tech_support(ctx, date, tech_support_channel_id, backfill):
-    if backfill and date:
-        raise click.BadParameter("--backfill cannot be used with a date")
-
-    day = None if backfill else date.date()
-
+def tech_support(ctx, tech_support_channel_id):
     app = get_app(ctx.obj["SLACK_SIGNING_SECRET"], ctx.obj["SLACK_TOKEN"])
 
-    messages = iter_messages(app, tech_support_channel_id, date=day)
+    messages = iter_messages(app, tech_support_channel_id)
+
+    log.info("Dropping existing slack_* tables")
+    # TODO: we have this in three places now, can we pull into some kind of
+    # service wrapper?
+    engine = create_engine(TIMESCALEDB_URL)
+    with engine.begin() as connection:
+        drop_tables(connection, prefix="slack_")
+    log.info("Dropped existing slack_* tables")
 
     with TimescaleDBWriter(SlackTechSupport) as writer:
         for date, messages in itertools.groupby(

metrics/timescaledb/db.py

@@ -1,23 +1,10 @@
-import itertools
-
 import structlog
 from sqlalchemy import text
 
+from ..tools.iter import batched
 
-log = structlog.get_logger()
-
-
-def batched(iterable, n):
-    """
-    Backport of 3.12's itertools.batched
-
-    https://docs.python.org/3/library/itertools.html#itertools.batched
 
-    batched('ABCDEFG', 3) --> ABC DEF G
-    """
-    it = iter(iterable)
-    while batch := tuple(itertools.islice(it, n)):
-        yield batch
+log = structlog.get_logger()
 
 
 def delete_rows(connection, name, n=10000):