Load data in one pass (#5142)

* Fix layouting and load data in one go. Bonus points if you can fix the rendering for this * fix focus target lines and high and low lines * Remove debug logging. Change profile functions to clear the cache on new data, instead of calculating heavy hashes * Remove unused code * Fix tests * Have Travis run tests on Node 10 and 12, not 13
nightscout · Oct 23, 2019 · 243ff2e · 243ff2e
1 parent fab2453
commit 243ff2e
Show file tree

Hide file tree

Showing 8 changed files with 54 additions and 96 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -26,5 +26,5 @@ matrix:
   include:
     - node_js: "10"
       <<: *node_js-steps
-    - node_js: "node" # Latest Node is not supported, and recommend, but we'll test it to know incompatibility issues
+    - node_js: "12" # Latest Node is not supported, and recommend, but we'll test it to know incompatibility issues
       <<: *node_js-steps
diff --git a/lib/client/chart.js b/lib/client/chart.js
@@ -398,39 +398,39 @@ function init (client, d3, $) {
         // add a y-axis line that shows the high bg threshold
         chart.focus.append('line')
           .attr('class', 'high-line')
-          .attr('x1', chart.xScale(dataRange[0]))
+          .attr('x1', chart.xScale.range()[0])
           .attr('y1', chart.yScale(utils.scaleMgdl(client.settings.thresholds.bgHigh)))
-          .attr('x2', chart.xScale(dataRange[1]))
+          .attr('x2', chart.xScale.range()[1])
           .attr('y2', chart.yScale(utils.scaleMgdl(client.settings.thresholds.bgHigh)))
           .style('stroke-dasharray', ('1, 6'))
           .attr('stroke', '#777');
 
         // add a y-axis line that shows the high bg threshold
         chart.focus.append('line')
           .attr('class', 'target-top-line')
-          .attr('x1', chart.xScale(dataRange[0]))
+          .attr('x1', chart.xScale.range()[0])
           .attr('y1', chart.yScale(utils.scaleMgdl(client.settings.thresholds.bgTargetTop)))
-          .attr('x2', chart.xScale(dataRange[1]))
+          .attr('x2', chart.xScale.range()[1])
           .attr('y2', chart.yScale(utils.scaleMgdl(client.settings.thresholds.bgTargetTop)))
           .style('stroke-dasharray', ('3, 3'))
           .attr('stroke', 'grey');
 
         // add a y-axis line that shows the low bg threshold
         chart.focus.append('line')
           .attr('class', 'target-bottom-line')
-          .attr('x1', chart.xScale(dataRange[0]))
+          .attr('x1', chart.xScale.range()[0])
           .attr('y1', chart.yScale(utils.scaleMgdl(client.settings.thresholds.bgTargetBottom)))
-          .attr('x2', chart.xScale(dataRange[1]))
+          .attr('x2', chart.xScale.range()[1])
           .attr('y2', chart.yScale(utils.scaleMgdl(client.settings.thresholds.bgTargetBottom)))
           .style('stroke-dasharray', ('3, 3'))
           .attr('stroke', 'grey');
 
         // add a y-axis line that shows the low bg threshold
         chart.focus.append('line')
           .attr('class', 'low-line')
-          .attr('x1', chart.xScale(dataRange[0]))
+          .attr('x1', chart.xScale.range()[0])
           .attr('y1', chart.yScale(utils.scaleMgdl(client.settings.thresholds.bgLow)))
-          .attr('x2', chart.xScale(dataRange[1]))
+          .attr('x2', chart.xScale.range()[1])
           .attr('y2', chart.yScale(utils.scaleMgdl(client.settings.thresholds.bgLow)))
           .style('stroke-dasharray', ('1, 6'))
           .attr('stroke', '#777');
@@ -517,27 +517,27 @@ function init (client, d3, $) {
 
         // transition lines to correct location
         chart.focus.select('.high-line')
-          .attr('x1', chart.xScale(currentRange[0]))
+          .attr('x1', chart.xScale.range()[0])
           .attr('y1', chart.yScale(utils.scaleMgdl(client.settings.thresholds.bgHigh)))
-          .attr('x2', chart.xScale(currentRange[1]))
+          .attr('x2', chart.xScale.range()[1])
           .attr('y2', chart.yScale(utils.scaleMgdl(client.settings.thresholds.bgHigh)));
 
         chart.focus.select('.target-top-line')
-          .attr('x1', chart.xScale(currentRange[0]))
+          .attr('x1', chart.xScale.range()[0])
           .attr('y1', chart.yScale(utils.scaleMgdl(client.settings.thresholds.bgTargetTop)))
-          .attr('x2', chart.xScale(currentRange[1]))
+          .attr('x2', chart.xScale.range()[1])
           .attr('y2', chart.yScale(utils.scaleMgdl(client.settings.thresholds.bgTargetTop)));
 
         chart.focus.select('.target-bottom-line')
-          .attr('x1', chart.xScale(currentRange[0]))
+          .attr('x1', chart.xScale.range()[0])
           .attr('y1', chart.yScale(utils.scaleMgdl(client.settings.thresholds.bgTargetBottom)))
-          .attr('x2', chart.xScale(currentRange[1]))
+          .attr('x2', chart.xScale.range()[1])
           .attr('y2', chart.yScale(utils.scaleMgdl(client.settings.thresholds.bgTargetBottom)));
 
         chart.focus.select('.low-line')
-          .attr('x1', chart.xScale(currentRange[0]))
+          .attr('x1', chart.xScale.range()[0])
           .attr('y1', chart.yScale(utils.scaleMgdl(client.settings.thresholds.bgLow)))
-          .attr('x2', chart.xScale(currentRange[1]))
+          .attr('x2', chart.xScale.range()[1])
           .attr('y2', chart.yScale(utils.scaleMgdl(client.settings.thresholds.bgLow)));
 
         // transition open-top line to correct location

diff --git a/lib/client/index.js b/lib/client/index.js
@@ -404,6 +404,7 @@ client.load = function load (serverSettings, callback) {
 
   function brushed () {
     // Brush not initialized
+    console.log("brushed");
     if (!chart.theBrush) {
       return;
     }
@@ -575,7 +576,6 @@ client.load = function load (serverSettings, callback) {
 
     var top = (client.bottomOfPills() + 5);
     $('#chartContainer').css({ top: top + 'px', height: $(window).height() - top - 10 });
-
     container.removeClass('loading');
   }
 
@@ -1204,8 +1204,10 @@ client.load = function load (serverSettings, callback) {
       isInitialData = true;
       if (!headless) {
         chart = client.chart = require('./chart')(client, d3, $);
-        brushed();
         chart.update(true);
+        brushed();
+        chart.update(false);
+//        brushed();
       }
     } else if (!inRetroMode()) {
       if (!headless) chart.update(false);

diff --git a/lib/data/ddata.js b/lib/data/ddata.js
@@ -32,39 +32,11 @@ function init () {
     });
   };
 
-  ddata.splitRecent = function splitRecent (time, cutoff, max, treatmentsToo) {
-    var result = {
-      first: {}
-      , rest: {}
-    };
-
-    function recent (item) {
-      return item.mills >= time - cutoff;
-    }
-
-    function filterMax (item) {
-      return item.mills >= time - max;
-    }
-
-    function partition (field, filter) {
-      var data;
-      if (filter) {
-        data = ddata[field].filter(filterMax);
-      } else {
-        data = ddata[field];
-      }
-
-      var parts = _.partition(data, recent);
-      result.first[field] = parts[0];
-      result.rest[field] = parts[1];
-    }
-
-    partition('treatments', treatmentsToo ? filterMax : false);
-
-    result.first.devicestatus = ddata.recentDeviceStatus(time);
-
-    result.first.sgvs = ddata.sgvs.filter(filterMax);
-    result.first.cals = ddata.cals;
+  ddata.dataWithRecentStatuses = function dataWithRecentStatuses() {
+    var results = {};
+    results.devicestatus = ddata.recentDeviceStatus(Date.now());
+    results.sgvs = ddata.sgvs;
+    results.cals = ddata.cals;
 
     var profiles = _.cloneDeep(ddata.profiles);
     if (profiles && profiles[0]) {
@@ -74,17 +46,14 @@ function init () {
         }
       })
     }
-    result.first.profiles = profiles;
-
-    result.rest.mbgs = ddata.mbgs.filter(filterMax);
-    result.rest.food = ddata.food;
-    result.rest.activity = ddata.activity;
+    results.profiles = profiles;
+    results.mbgs = ddata.mbgs;
+    results.food = ddata.food;
+    results.treatments = ddata.treatments;
 
-    console.log('results.first size', JSON.stringify(result.first).length, 'bytes');
-    console.log('results.rest size', JSON.stringify(result.rest).length, 'bytes');
+    return results;
 
-    return result;
-  };
+  }
 
   ddata.recentDeviceStatus = function recentDeviceStatus (time) {
 

diff --git a/lib/profilefunctions.js b/lib/profilefunctions.js
@@ -4,7 +4,6 @@ var _ = require('lodash');
 var moment = require('moment-timezone');
 var c = require('memory-cache');
 var times = require('./times');
-var crypto = require('crypto');
 
 var cacheTTL = 5000;
 var prevBasalTreatment = null;
@@ -14,6 +13,12 @@ function init (profileData) {
 
   var profile = {};
 
+  profile.clear = function clear() {
+    cache.clear();
+    profile.data = null;
+    prevBasalTreatment = null;
+  }
+
   profile.loadData = function loadData (profileData) {
     if (profileData && profileData.length) {
       profile.data = profile.convertToProfileStore(profileData);
@@ -72,8 +77,7 @@ function init (profileData) {
 
     //round to the minute for better caching
     var minuteTime = Math.round(time / 60000) * 60000;
-
-    var cacheKey = (minuteTime + valueType + spec_profile + profile.profiletreatments_hash);
+    var cacheKey = (minuteTime + valueType + spec_profile);
     var returnValue = cache.get(cacheKey);
 
     if (returnValue) {
@@ -215,9 +219,8 @@ function init (profileData) {
     });
 
     profile.combobolustreatments = combobolustreatments || [];
-    profile.profiletreatments_hash = crypto.createHash('sha1').update(JSON.stringify(profile.profiletreatments)).digest('hex');
-    profile.tempbasaltreatments_hash = crypto.createHash('sha1').update(JSON.stringify(profile.tempbasaltreatments)).digest('hex');
-    profile.combobolustreatments_hash = crypto.createHash('sha1').update(JSON.stringify(profile.combobolustreatments)).digest('hex');
+
+    cache.clear();
   };
 
   profile.activeProfileToTime = function activeProfileToTime (time) {
@@ -236,7 +239,7 @@ function init (profileData) {
   profile.activeProfileTreatmentToTime = function activeProfileTreatmentToTime (time) {
 
     var minuteTime = Math.round(time / 60000) * 60000;
-    var cacheKey = 'profile' + minuteTime + profile.profiletreatments_hash;
+    var cacheKey = 'profileCache' + minuteTime;
     var returnValue = cache.get(cacheKey);
 
     if (returnValue) {
@@ -325,7 +328,7 @@ function init (profileData) {
   profile.getTempBasal = function getTempBasal (time, spec_profile) {
 
     var minuteTime = Math.round(time / 60000) * 60000;
-    var cacheKey = 'basal' + minuteTime + profile.tempbasaltreatments_hash + profile.combobolustreatments_hash + profile.profiletreatments_hash + spec_profile;
+    var cacheKey = 'basalCache' + minuteTime + spec_profile;
     var returnValue = cache.get(cacheKey);
 
     if (returnValue) {

diff --git a/lib/server/websocket.js b/lib/server/websocket.js
@@ -447,20 +447,15 @@ function init (env, ctx, server) {
               filterTreatments = true;
               msecHistory = Math.min(new Date().getTime() - from, msecHistory);
             }
-            // send all data upon new connection
-            if (lastData && lastData.splitRecent) {
-              var split = lastData.splitRecent(Date.now(), times.hours(3).msecs, msecHistory, filterTreatments);
+
+            if (lastData && lastData.dataWithRecentStatuses) {
+              let data = lastData.dataWithRecentStatuses();
+
               if (message.status) {
-                split.first.status = status(split.first.profiles);
+                data.status = status(data.profiles);
               }
-              //send out first chunk
-              socket.emit('dataUpdate', split.first);
-
-              //then send out the rest
-              setTimeout(function sendTheRest() {
-                split.rest.delta = true;
-                socket.emit('dataUpdate', split.rest);
-              }, 500);
+
+              socket.emit('dataUpdate', data);
             }
           }
           console.log(LOG_WS + 'Authetication ID: ', socket.client.id, ' client: ', clientType, ' history: ' + history);

diff --git a/tests/ddata.test.js b/tests/ddata.test.js
@@ -41,19 +41,6 @@ describe('ddata', function ( ) {
     done( );
   });
 
-  it('has #split( )', function (done) {
-    var date = new Date( );
-    var time = date.getTime( );
-    var cutoff = 1000 * 60 * 5;
-    var max = 1000 * 60 * 60 * 24 * 2;
-    var pieces = ctx.ddata.splitRecent(time, cutoff, max);
-    should.exist(pieces);
-    should.exist(pieces.first);
-    should.exist(pieces.rest);
-
-    done( );
-  });
-
   // TODO: ensure partition function gets called via:
   // Properties
   // * ddata.devicestatus

diff --git a/tests/profile.test.js b/tests/profile.test.js
@@ -5,6 +5,10 @@ describe('Profile', function ( ) {
 
   var profile_empty = require('../lib/profilefunctions')();
 
+  beforeEach(function() {
+    profile_empty.clear();
+  });
+
   it('should say it does not have data before it has data', function() {
     var hasData = profile_empty.hasData();
     hasData.should.equal(false);
@@ -30,8 +34,6 @@ describe('Profile', function ( ) {
   };
 
   var profile = require('../lib/profilefunctions')([profileData]);
-//  console.log(profile);
-
   var now = Date.now();
 
   it('should know what the DIA is with old style profiles', function() {