From 093809d7d28992ffa435318813b7884171d2bf5b Mon Sep 17 00:00:00 2001
From: Iain Dillingham <iain@dillingham.me.uk>
Date: Mon, 25 Apr 2022 09:47:08 +0100
Subject: [PATCH] Optionally output deciles tables

By default, we output deciles tables. Studies using deciles-charts
probably won't be affected, as they probably use a glob pattern that
includes ".png".

We haven't written tests for the new functions, because they are thin
wrappers around `ebmdatalab.charts`. However, we should consider writing
an integration test soon.
---
 README.md                  | 20 +++++++++++++-------
 analysis/deciles_charts.py | 30 +++++++++++++++++++++++++++++-
 project.yaml               |  4 +++-
 3 files changed, 45 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index 09f4155..980b85c 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,8 @@
 # deciles-charts
 
-deciles-charts generates a line chart for each [measure table][1] in an input directory.
-The line chart has time on the horizontal axis (`x`) and value on the vertical axis (`y`).
+deciles-charts generates a table and a line chart for each [measure table][1] in an input directory.
+The table has date, percentile, and value columns.
+The line chart has date on the horizontal axis (`x`) and value on the vertical axis (`y`).
 Deciles are plotted as dashed lines;
 the median is plotted as a solid line.
 For example, the following deciles chart was generated from dummy data:
@@ -48,7 +49,7 @@ generate_measures:
       measure: output/measure_*.csv
 ```
 
-Finally, the following deciles-charts reusable action generates a deciles chart for each measure table.
+Finally, the following deciles-charts reusable action generates a deciles table and a deciles chart for each measure table.
 Remember to replace `[version]` with [a deciles-charts version][4]:
 
 ```yaml
@@ -60,12 +61,14 @@ generate_deciles_charts:
   needs: [generate_measures]
   outputs:
     moderately_sensitive:
-      deciles_charts: output/deciles_chart_*.png
+      deciles_charts: output/deciles_*_*.*
 ```
 
-For each measure table, there will now be a corresponding deciles chart.
+For each measure table, there will now be a deciles table and deciles chart.
 For example, given a measure table called `measure_has_sbp_event_by_stp_code.csv`,
-there will now be a corresponding deciles chart called `deciles_chart_has_sbp_event_by_stp_code.png`.
+there will now be
+a deciles table called `deciles_table_has_sbp_event_by_stp_code.csv` and
+a deciles chart called `deciles_chart_has_sbp_event_by_stp_code.png`.
 
 ## Configuration
 
@@ -80,17 +83,20 @@ generate_deciles_charts:
       --output-dir output
   config:
     show_outer_percentiles: false
+    tables:
+      output: true
     charts:
       output: true
   needs: [generate_measures]
   outputs:
     moderately_sensitive:
-      deciles_charts: output/deciles_chart_*.png
+      deciles_charts: output/deciles_*_*.*
 ```
 
 | Configuration            | Description                                                                                         |
 | ------------------------ | --------------------------------------------------------------------------------------------------- |
 | `show_outer_percentiles` | Show the top and bottom percentiles, as well as the deciles                                         |
+| `tables.output`          | Generate a deciles table for each measure table that is matched by the `--input-files` glob pattern |
 | `charts.output`          | Generate a deciles chart for each measure table that is matched by the `--input-files` glob pattern |
 
 ## Notes for developers
diff --git a/analysis/deciles_charts.py b/analysis/deciles_charts.py
index c6aeac3..5064305 100644
--- a/analysis/deciles_charts.py
+++ b/analysis/deciles_charts.py
@@ -26,6 +26,9 @@
 
 DEFAULT_CONFIG = {
     "show_outer_percentiles": False,
+    "tables": {
+        "output": True,
+    },
     "charts": {
         "output": True,
     },
@@ -36,6 +39,13 @@
     "additionalProperties": False,
     "properties": {
         "show_outer_percentiles": {"type": "boolean"},
+        "tables": {
+            "type": "object",
+            "additionalProperties": False,
+            "properties": {
+                "output": {"type": "boolean"},
+            },
+        },
         "charts": {
             "type": "object",
             "additionalProperties": False,
@@ -76,6 +86,19 @@ def drop_zero_denominator_rows(measure_table):
     return measure_table[is_not_inf].reset_index(drop=True)
 
 
+def get_deciles_table(measure_table, config):
+    return charts.add_percentiles(
+        measure_table,
+        period_column="date",
+        column="value",
+        show_outer_percentiles=config["show_outer_percentiles"],
+    )
+
+
+def write_deciles_table(deciles_table, path):
+    deciles_table.to_csv(path, index=False)
+
+
 def get_deciles_chart(measure_table, config):
     return charts.deciles_chart(
         measure_table,
@@ -139,9 +162,14 @@ def main():
 
     for measure_table in get_measure_tables(input_files):
         measure_table = drop_zero_denominator_rows(measure_table)
+        id_ = measure_table.attrs["id"]
+        if config["tables"]["output"]:
+            deciles_table = get_deciles_table(measure_table, config)
+            fname = f"deciles_table_{id_}.csv"
+            write_deciles_table(deciles_table, output_dir / fname)
+
         if config["charts"]["output"]:
             chart = get_deciles_chart(measure_table, config)
-            id_ = measure_table.attrs["id"]
             fname = f"deciles_chart_{id_}.png"
             write_deciles_chart(chart, output_dir / fname)
 
diff --git a/project.yaml b/project.yaml
index f347d14..f123080 100644
--- a/project.yaml
+++ b/project.yaml
@@ -31,9 +31,11 @@ actions:
         --output-dir output
     config:
       show_outer_percentiles: false
+      tables:
+        output: true
       charts:
         output: true
     needs: [generate_measures]
     outputs:
       moderately_sensitive:
-        deciles_charts: output/deciles_chart_*.png
+        deciles_charts: output/deciles_*_*.*