Limit how many shards are loaded concurrently

Since loading a shard can allocate a lot of memory, running them all
at once could OOM the process.  This limits the number of shards
loaded to 4.  This will be changed to a config option provided the
approach helps.

tsdb/store.go

@@ -116,6 +116,9 @@ func (s *Store) loadShards() error {
 		s   *Shard
 		err error
 	}
+
+	throttle := newthrottle(4)
+
 	resC := make(chan *res)
 	var n int
 
@@ -140,6 +143,9 @@ func (s *Store) loadShards() error {
 			for _, sh := range shards {
 				n++
 				go func(index *DatabaseIndex, db, rp, sh string) {
+					throttle.take()
+					defer throttle.release()
+
 					start := time.Now()
 					path := filepath.Join(s.path, db, rp, sh)
 					walPath := filepath.Join(s.EngineOptions.Config.WALDir, db, rp, sh)
@@ -862,3 +868,28 @@ func measurementsFromSourcesOrDB(db *DatabaseIndex, sources ...influxql.Source)
 
 	return measurements, nil
 }
+
+// throttle is a simple channel based concurrency limiter.  It uses a fixed
+// size channel to limit callers from proceeding until there is a value avalable
+// in the channel.  If all are in-use, the caller blocks until one is freed.
+type throttle struct {
+	c chan struct{}
+}
+
+func newthrottle(limit int) *throttle {
+	t := &throttle{
+		c: make(chan struct{}, limit),
+	}
+	for i := 0; i < limit; i++ {
+		t.c <- struct{}{}
+	}
+	return t
+}
+
+func (t *throttle) take() {
+	<-t.c
+}
+
+func (t *throttle) release() {
+	t.c <- struct{}{}
+}