Add GitHub API script, daily table, Combined PR Age chart.

Fixes microsoft#1043. * `gather_stats.js` + This uses the GitHub API to gather PR/issue statistics. It runs on the console (not in the browser), using the latest version of Node.js (currently 14.5.0). I've significantly improved it since my initial version in microsoft#1038 that wasn't checked into the repo: - This loads a personal access token from a `.env` file, avoiding the danger of hardcoding it in the script. - Because it currently takes about 20 seconds to list all PRs/issues through the GitHub API, the script displays a command-line progress bar. - For efficiency, the script transforms PR/issue data as it's received (instead of repeatedly during analysis). - The script now analyzes the data with daily resolution. Accordingly, I've adjusted the beginning date. - The script now writes a file, `daily_table.js`, complete with a "generated file" warning comment. - The script now calculates combined PR age. * `daily_table.js` + The generation process is stable, so regenerating this file will typically add rows to the bottom with no other changes. However, it's possible for previous rows to change (e.g. due to relabeling or reopening). * `weekly_table.js` + I extracted the classic table, still updated manually, for symmetry. This also makes the source and history of `index.html` easier to read. Eventually, I plan to generate the `cxx20` and `lwg` stats, and we'll stop tracking `vso` after porting all old bugs, leaving only `libcxx` that can't be easily generated yet. * `.gitignore` + We need to ignore the `node_modules` directory that `npm` creates. We also need to ignore the `.env` file, but I can't tell you why - it's a secret. * `package.json` + This is generated by `npm init` and is updated when dependencies are installed. I filled out some reasonable information in response to its questions, but I don't intend to publish this in the registry (as it is hardcoded for our repo and not intended for general use, although it could serve as the basis for such work). * `package-lock.json` + This is also a generated file, and I don't really understand its purpose, except that it's supposed to be checked in (something about pinning the exact versions of all transitive dependencies). * `index.html` + Add `hr` styling, to separate the two charts. + Include `daily_table.js` and `weekly_table.js`. + Move the main `<script>` into the `<head>`, making it easier to read the logic versus the content. (This required only one change, moving the "Toggle Timeframe" button's `addEventListener` to `window.onload`, which is invoked after the page has finished loading.) + Generalize `get_values` because there are two tables now. + Rename `chart_data` to `status_data` (and similarly for `chart_options`) because there are two charts now. + Add `age_data`. + Give `timeframe_all` and `timeframe_github` names, making it easier to refer to them later (`timeframes[0]` was unnecessarily mysterious). + Extract `common_options` used by both charts. + Extract a `make_xAxes` function, as both charts use the same x-axis, but with different timeframes. + `...common_options` is "spread syntax". + Add `age_options` for the new chart. Note that attempting to centralize their `title` configuration (where `display` and `fontSize` are the same, but `text` is different) seems like more trouble than it's worth. + Construct the `age_chart`, and move the button logic as previously mentioned. + Add the `<canvas id="ageChart">` followed by an explanation.
StephanTLavavej · Jul 19, 2020 · 06e958d · 06e958d
1 parent a74aad8
commit 06e958d
Show file tree

Hide file tree

Showing 7 changed files with 992 additions and 219 deletions.
diff --git a/.gitignore b/.gitignore
@@ -11,3 +11,9 @@ __pycache__/
 # Ignore submodules in other branches.
 /llvm-project/
 /vcpkg/
+
+# Ignore installed packages.
+/node_modules/
+
+# Ignore secret credentials.
+/.env
diff --git a/daily_table.js b/daily_table.js
diff --git a/gather_stats.js b/gather_stats.js
@@ -0,0 +1,117 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+const fs = require('fs');
+
+const cliProgress = require('cli-progress');
+const dotenv = require('dotenv');
+const { DateTime, Duration } = require('luxon');
+const { Octokit } = require('@octokit/rest');
+
+{
+    const result = dotenv.config();
+
+    if (result.error) {
+        throw result.error;
+    }
+
+    if (process.env.SECRET_GITHUB_PERSONAL_ACCESS_TOKEN === undefined) {
+        throw 'Missing SECRET_GITHUB_PERSONAL_ACCESS_TOKEN key in .env file.'
+    }
+}
+
+const progress_bar = new cliProgress.SingleBar(
+    { format: '{bar} {percentage}% | ETA: {eta}s | {value}/{total} {name}' },
+    cliProgress.Presets.shades_classic);
+let progress_value = 0;
+
+progress_bar.start(1000 /* placeholder total */, 0, { name: 'PRs and issues received' });
+
+const octokit = new Octokit({
+    auth: process.env.SECRET_GITHUB_PERSONAL_ACCESS_TOKEN,
+});
+
+octokit.paginate(
+    octokit.issues.listForRepo,
+    {
+        owner: 'microsoft',
+        repo: 'STL',
+        state: 'all',
+    },
+    response => {
+        progress_value += response.data.length;
+
+        // PRs/issues are received in descending order by default.
+        // Deleted PRs/issues are skipped, so we recalculate the expected total.
+        // If the last PR/issue we received was number N, we can expect to receive N - 1 more: numbers [1, N - 1].
+        const remaining = response.data[response.data.length - 1]['number'] - 1;
+
+        progress_bar.setTotal(progress_value + remaining);
+        progress_bar.update(progress_value);
+
+        return response.data.map(item => ({
+            opened: DateTime.fromISO(item['created_at']),
+            closed: DateTime.fromISO(item['closed_at'] ?? '2100-01-01'),
+            is_pr: item['pull_request'] !== undefined,
+            label_names: item['labels'].map(label => label['name']),
+        }));
+    }
+).then(transformed_output => {
+    progress_bar.setTotal(progress_value); // Just in case PR/issue number 1 was deleted,
+    progress_bar.update(progress_value); // which would prevent the progress bar from reaching 100%.
+    progress_bar.stop();
+
+    const begin = DateTime.fromISO('2019-09-03' + 'T23:00:00-07');
+    const now = DateTime.local();
+
+    let str = '// Copyright (c) Microsoft Corporation.\n';
+    str += '// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n';
+    str += '\n';
+    const generated_file_warning_comment = '// Generated file - DO NOT EDIT manually!\n';
+    str += generated_file_warning_comment;
+    str += 'const daily_table = [\n';
+
+    progress_bar.start(Math.ceil(now.diff(begin, 'days').as('days')), 0, { name: 'days analyzed' });
+
+    for (let when = begin; when < now; when = when.plus({ days: 1 })) {
+        let num_pr = 0;
+        let num_issue = 0;
+        let num_bug = 0;
+        let combined_pr_age = Duration.fromObject({ seconds: 0 });
+
+        for (const elem of transformed_output) {
+            if (when < elem.opened || elem.closed < when) {
+                // This PR/issue wasn't active; do nothing.
+            } else if (elem.is_pr) {
+                ++num_pr;
+                combined_pr_age = combined_pr_age.plus(when.diff(elem.opened, 'seconds'));
+            } else if (elem.label_names.includes('cxx20')) {
+                // Avoid double-counting C++20 Features and GitHub Issues.
+            } else {
+                ++num_issue;
+
+                if (elem.label_names.includes('bug')) {
+                    ++num_bug;
+                }
+            }
+        }
+
+        str += '    { ';
+        str += [
+            `date: '${when.toISODate()}'`,
+            `pr: ${num_pr}`,
+            `issue: ${num_issue}`,
+            `bug: ${num_bug}`,
+            `months: ${Number.parseFloat(combined_pr_age.as('months')).toFixed(2)}`,
+            '},\n'].join(', ');
+
+        progress_bar.increment();
+    }
+
+    str += '];\n';
+    str += generated_file_warning_comment;
+
+    progress_bar.stop();
+
+    fs.writeFileSync('./daily_table.js', str);
+});