Ui/market watchers [WIP]

config/adapters/mongodb.toml

@@ -0,0 +1,7 @@
+connectionString = "mongodb://mongodb/gekko"
+path = "plugins/mongodb"
+version = 0.1
+
+[[dependencies]]
+module = "mongojs"
+version = "2.4.0"

config/adapters/postgresql.toml

@@ -0,0 +1,7 @@
+connectionString = "postgres://user:pass@localhost:5432"
+path = "plugins/postgresql"
+version = 0.1
+
+[[dependencies]]
+module = "pg"
+version = "6.1.0"

config/adapters/sqlite.toml

@@ -0,0 +1,7 @@
+dataDirectory = "history"
+path = "plugins/sqlite"
+version = 0.1
+
+[[dependencies]]
+module = "sqlite3"
+version = "3.1.4"

config/backtest.toml

@@ -0,0 +1,8 @@
+batchSize = 50
+
+# scan for available dateranges
+daterange = "scan"
+# or specify it like so:
+# [daterange]
+# from = "2015-01-01"
+# to = "2016-01-01"

config/general.toml

@@ -0,0 +1,10 @@
+debug = true
+
+# what database should Gekko use?
+adapter = 'sqlite'
+
+[watch]
+exchange = 'Poloniex'
+currency = 'USDT'
+asset = 'BTC'
+

config/plugins/candleWriter.toml

@@ -1,2 +1 @@
-enabled = true
-sqlite = "sqlite"
+enabled = true

config/plugins/pushbullet.toml

@@ -0,0 +1,16 @@
+enabled = false
+
+# Send 'Gekko starting' message if true
+sendMessageOnStart = true
+
+# disable advice printout if it's soft
+muteSoft = true
+
+# your email, change it unless you are Azor Ahai
+email = "[email protected]"
+
+# your pushbullet API key
+key = "xxx"
+
+# will make Gekko messages start with [GEKKO]
+tag = "[GEKKO]"

config/plugins/trader.toml

@@ -0,0 +1,9 @@
+enabled = false
+
+## NOTE: once you filled in the following
+## never share this file with anyone!
+
+key = ""
+secret = ""
+# this is only required at specific exchanges
+username = ""

config/plugins/tradingAdvisor.toml

@@ -0,0 +1,9 @@
+enabled = true
+
+candleSize = 60
+historySize = 25
+method = "MACD"
+
+[talib]
+enabled = false
+version = "1.0.2"

core/budfox/candleManager.js

@@ -10,6 +10,7 @@ var util = require(__dirname + '/../util');
 var dirs = util.dirs();
 var config = util.getConfig();
 var log = require(dirs.core + 'log');
+var cp = require(dirs.core + 'cp');
 
 var CandleCreator = require(dirs.budfox + 'candleCreator');
 
@@ -20,6 +21,10 @@ var Manager = function() {
 
   this.candleCreator
     .on('candles', this.relayCandles);
+
+  this.messageFirstCandle = _.once(candle => {
+    cp.firstCandle(candle);
+  })
 };
 
 util.makeEventEmitter(Manager);
@@ -29,6 +34,12 @@ Manager.prototype.processTrades = function(tradeBatch) {
 
 Manager.prototype.relayCandles = function(candles) {
   this.emit('candles', candles);
+
+  if(!_.size(candles))
+    return;
+
+  this.messageFirstCandle(_.first(candles));
+  cp.lastCandle(_.last(candles));
 }
 
 module.exports = Manager;

core/budfox/heart.js

@@ -6,7 +6,14 @@ var log = require(util.dirs().core + 'log');
 var _ = require('lodash');
 var moment = require('moment');
 
+if(util.getConfig().watch.exchange === 'okcoin')
+  var TICKRATE = 2;
+else
+  var TICKRATE = 20;
+
 var Heart = function() {
+  this.lastTick = false;
+
   _.bindAll(this);
 }
 
@@ -18,22 +25,22 @@ Heart.prototype.pump = function() {
 }
 
 Heart.prototype.tick = function() {
+  if(this.lastTick) {
+    // make sure the last tick happened not to lang ago
+    // @link https://github.com/askmike/gekko/issues/514
+    if(this.lastTick < moment().unix() - TICKRATE * 3)
+      util.die('Failed to tick in time, see https://github.com/askmike/gekko/issues/514 for details', true);
+  }
+
+  this.lastTick = moment().unix();
   this.emit('tick');
 }
 
-Heart.prototype.determineLiveTickRate = function() {
-  // TODO: fix
-  if(util.getConfig().watch.exchange === 'okcoin')
-    var seconds = 2;
-  else
-    var seconds = 20;
-
-  this.tickRate = +moment.duration(seconds, 's');
-}
-
 Heart.prototype.scheduleTicks = function() {
-  this.determineLiveTickRate();
-  setInterval(this.tick, this.tickRate);
+  setInterval(
+    this.tick,
+    +moment.duration(TICKRATE, 's')
+  );
 
   // start!
   _.defer(this.tick);

core/cp.js

@@ -1,5 +1,4 @@
-// functions that emit information
-// to the parent process.
+// functions that emit data to the parent process.
 //
 // noops if this gekko instance is not a child process!
 
@@ -11,13 +10,40 @@ var moment = require('moment');
 
 var ENV = util.gekkoEnv();
 
+var message = (type, payload) => {
+  payload.type = type;
+  process.send(payload);
+}
+
 var cp = {
-  message: (type, payload) => {
-    payload.type = type;
-    process.send(payload);
-  },
-  update: latest => cp.message('update', { latest }),
-  startAt: startAt => cp.message('startAt', { startAt }),
+  // string like: '2016-12-03T22:23:00.000Z'
+  update: latest => message('update', { latest }),
+  startAt: startAt => message('startAt', { startAt }),
+
+  // object like:
+  //
+  // {
+  //   start: '2016-12-03T22:23:00.000Z',
+  //   open: 765,
+  //   high: 765,
+  //   low: 765,
+  //   close: 765,
+  //   vwp: 765,
+  //   volume: 0,
+  //   trades: 0
+  // }
+  lastCandle: lastCandle => message('lastCandle', { lastCandle }),
+  firstCandle: firstCandle => message('firstCandle', { firstCandle }),
+
+  // object like:
+  //
+  // {
+  //   action: 'sell',
+  //   price: 765,
+  //   date: '2016-12-03T22:23:00.000Z',
+  //   balance: 4242
+  // }
+  trade: trade => message('trade', { trade }),
 }
 
 if(ENV !== 'child-process') {

core/markets/backtest.js

@@ -5,7 +5,7 @@ var dirs = util.dirs();
 var log = require(dirs.core + 'log');
 var moment = require('moment');
 
-var adapter = config.adapters[config.backtest.adapter];
+var adapter = config[config.adapter];
 var Reader = require(dirs.gekko + adapter.path + '/reader');
 var daterange = config.backtest.daterange;
 
@@ -15,6 +15,12 @@ var from = moment.utc(daterange.from);
 if(to <= from)
   util.die('This daterange does not make sense.')
 
+if(!from.isValid())
+  util.die('invalid `from`');
+
+if(!to.isValid())
+  util.die('invalid `to`');
+
 var Market = function() {
 
   _.bindAll(this);

core/markets/importer.js

@@ -6,7 +6,7 @@ var log = require(dirs.core + 'log');
 var moment = require('moment');
 var cp = require(dirs.core + 'cp');
 
-var adapter = config.adapters[config.importer.adapter];
+var adapter = config[config.adapter];
 var daterange = config.importer.daterange;
 
 var from = moment.utc(daterange.from);

core/markets/leech.js

@@ -0,0 +1,103 @@
+// a leech market is "semi-realtime" and pulls out candles of a
+// database (which is expected to be updated reguraly, like with a
+// realtime market running in parralel).
+
+const _ = require('lodash');
+const moment = require('moment');
+
+const util = require('../util');
+const dirs = util.dirs();
+const config = util.getConfig();
+
+const cp = require(dirs.core + 'cp');
+
+const adapter = config[config.adapter];
+const Reader = require(dirs.gekko + adapter.path + '/reader');
+
+const TICKINTERVAL = 20 * 1000; // 20 seconds
+
+const exchanges = require(dirs.gekko + 'exchanges');
+const exchange = _.find(exchanges, function(e) {
+  return e.slug === config.watch.exchange.toLowerCase();
+});
+
+if(!exchange)
+  util.die(`Unsupported exchange: ${config.watch.exchange.toLowerCase()}`)
+
+const exchangeChecker = require(util.dirs().core + 'exchangeChecker');
+
+const error = exchangeChecker.cantMonitor(config.watch);
+if(error)
+  util.die(error, true);
+
+if(config.market.from)
+  var fromTs = moment.utc(config.market.from).unix();
+else
+  var fromTs = moment().startOf('minute').unix();
+
+
+var Market = function() {
+
+  _.bindAll(this);
+
+  Readable.call(this, {objectMode: true});
+
+  this.reader = new Reader();
+  this.latestTs = fromTs;
+
+  setInterval(
+    this.get,
+    TICKINTERVAL
+  );
+}
+
+var Readable = require('stream').Readable;
+Market.prototype = Object.create(Readable.prototype, {
+  constructor: { value: Market }
+});
+
+Market.prototype._read = _.once(function() {
+  this.get();
+});
+
+Market.prototype.get = function() {
+  var future = moment().add(1, 'minute').unix();
+
+  this.reader.get(
+    this.latestTs,
+    future,
+    'full',
+    this.processCandles
+  )
+}
+
+Market.prototype.processCandles = function(err, candles) {
+  var amount = _.size(candles);
+  if(amount === 0) {
+    // no new candles!
+    return;
+  }
+
+
+  // TODO:
+  // verify that the correct amount of candles was passed:
+  //
+  // if `this.latestTs` was at 10:00 and we receive 3 candles with the latest at 11:00
+  // we know we are missing 57 candles...
+
+  _.each(candles, function(c, i) {
+    c.start = moment.unix(c.start).utc();
+    this.push(c);
+  }, this);
+
+  this.sendStartAt(_.first(candles));
+  cp.lastCandle(_.last(candles));
+
+  this.latestTs = _.last(candles).start.unix() + 1;
+}
+
+Market.prototype.sendStartAt = _.once(function(candle) {
+  cp.firstCandle(candle);
+});
+
+module.exports = Market;

core/pipeline.js

@@ -159,8 +159,14 @@ var pipeline = (settings) => {
       prepareMarket
     ],
     function() {
-      // load a market based on the mode
-      var Market = require(dirs.markets + mode);
+      // load a market based on the config (or fallback to mode)
+      let marketType;
+      if(config.market)
+        marketType = config.market.type;
+      else
+        marketType = mode;
+
+      var Market = require(dirs.markets + marketType);
 
       var market = new Market(config);
       var gekko = new GekkoStream(candleConsumers);