From 90e1caedb4b52684da435c30d4b88853eb2271b7 Mon Sep 17 00:00:00 2001
From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com>
Date: Wed, 23 Oct 2019 10:24:19 -0700
Subject: [PATCH 01/22] 1 - check for pkg installed, download and untar ccxt
 source code

---
 scripts/fs_bin_gen.go         | 60 ++++++++++++++++++++++++++++++++---
 support/networking/network.go | 24 ++++++++++++++
 2 files changed, 80 insertions(+), 4 deletions(-)

diff --git a/scripts/fs_bin_gen.go b/scripts/fs_bin_gen.go
index e80cdd0ba..f8e1cc923 100644
--- a/scripts/fs_bin_gen.go
+++ b/scripts/fs_bin_gen.go
@@ -2,15 +2,25 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"log"
 	"net/http"
 	"os/exec"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	"github.com/stellar/kelp/support/networking"
 
 	"github.com/shurcooL/vfsgen"
+	"github.com/stellar/kelp/support/kelpos"
 )
 
+const kelpPrefsDirectory = "build"
 const fsDev_filename = "./scripts/fs_gen/filesystem_vfsdata_dev.go"
 const fs_filename = "./gui/filesystem_vfsdata.go"
+const ccxtDownloadURL = "https://github.com/ccxt-rest/ccxt-rest/archive/v0.0.4.tar.gz"
+const ccxtDownloadFilename = "ccxt-rest_v0.0.4.tar.gz"
+const ccxtUntaredDirName = "ccxt-rest-0.0.4"
 
 func main() {
 	envP := flag.String("env", "dev", "environment to use (dev / release)")
@@ -18,15 +28,18 @@ func main() {
 	env := *envP
 
 	if env == "dev" {
-		generateDev()
+		generateWeb_Dev()
 	} else if env == "release" {
-		generateRelease()
+		generateWeb_Release()
 	} else {
 		panic("unrecognized env flag: " + env)
 	}
+
+	kos := kelpos.GetKelpOS()
+	generateCcxtBinary(kos)
 }
 
-func generateRelease() {
+func generateWeb_Release() {
 	fs := http.Dir("./gui/web/build")
 	e := vfsgen.Generate(fs, vfsgen.Options{
 		Filename:        fs_filename,
@@ -39,10 +52,49 @@ func generateRelease() {
 	}
 }
 
-func generateDev() {
+func generateWeb_Dev() {
 	c := exec.Command("cp", fsDev_filename, fs_filename)
 	e := c.Run()
 	if e != nil {
 		log.Fatalln(e)
 	}
 }
+
+func checkPkgTool(kos *kelpos.KelpOS) {
+	fmt.Printf("checking for presence of `pkg` tool ...\n")
+	_, e := kos.Blocking("pkg", "pkg -v")
+	if e != nil {
+		log.Fatal("ensure that the `pkg` tool is installed correctly. You can get it from here: https://github.com/zeit/pkg")
+	}
+	fmt.Printf("done\n")
+}
+
+func downloadCcxtSource(kos *kelpos.KelpOS) {
+	downloadDir := filepath.Join(kelpPrefsDirectory, "downloads", "ccxt")
+	fmt.Printf("making directory where we can download ccxt file %s ...\n", downloadDir)
+	e := kos.Mkdir(downloadDir)
+	if e != nil {
+		log.Fatal(errors.Wrap(e, "could not make directory for downloadDir "+downloadDir))
+	}
+	fmt.Printf("done\n")
+
+	fmt.Printf("downloading file from URL %s ...\n", ccxtDownloadURL)
+	downloadFilePath := filepath.Join(downloadDir, ccxtDownloadFilename)
+	e = networking.DownloadFile(ccxtDownloadURL, downloadFilePath)
+	if e != nil {
+		log.Fatal(errors.Wrap(e, "could not download ccxt tar.gz file"))
+	}
+	fmt.Printf("done\n")
+
+	fmt.Printf("untaring file %s ...\n", downloadFilePath)
+	_, e = kos.Blocking("tar", fmt.Sprintf("tar xvf %s -C %s", downloadFilePath, downloadDir))
+	if e != nil {
+		log.Fatal(errors.Wrap(e, "could not untar ccxt file"))
+	}
+	fmt.Printf("done\n")
+}
+
+func generateCcxtBinary(kos *kelpos.KelpOS) {
+	checkPkgTool(kos)
+	downloadCcxtSource(kos)
+}
diff --git a/support/networking/network.go b/support/networking/network.go
index 9a613dfa7..27ab0973a 100644
--- a/support/networking/network.go
+++ b/support/networking/network.go
@@ -3,9 +3,11 @@ package networking
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"mime"
 	"net/http"
+	"os"
 	"strings"
 )
 
@@ -78,3 +80,25 @@ func JSONRequest(
 
 	return nil
 }
+
+// DownloadFile downloads a URL to a file on the local disk as it downloads it.
+func DownloadFile(url string, filepath string) error {
+	outfile, e := os.Create(filepath)
+	if e != nil {
+		return fmt.Errorf("could not create file at filepath (%s): %s", filepath, e)
+	}
+	defer outfile.Close()
+
+	resp, e := http.Get(url)
+	if e != nil {
+		return fmt.Errorf("could not get file at URL (%s): %s", url, e)
+	}
+	defer resp.Body.Close()
+
+	// do the download and write to file on disk as we download
+	_, e = io.Copy(outfile, resp.Body)
+	if e != nil {
+		return fmt.Errorf("could not download from URL (%s) to file (%s) in a streaming manner: %s", url, filepath, e)
+	}
+	return nil
+}

From 4eb45359c2ec3e892d0e22363404bcb13cb1e2bf Mon Sep 17 00:00:00 2001
From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com>
Date: Wed, 23 Oct 2019 10:58:11 -0700
Subject: [PATCH 02/22] 2 - split ccxt_bin_gen script from fs_bin_gen script

---
 scripts/build.sh        | 12 ++++++
 scripts/ccxt_bin_gen.go | 91 +++++++++++++++++++++++++++++++++++++++++
 scripts/fs_bin_gen.go   | 52 -----------------------
 3 files changed, 103 insertions(+), 52 deletions(-)
 create mode 100644 scripts/ccxt_bin_gen.go

diff --git a/scripts/build.sh b/scripts/build.sh
index ba4fc502e..b46dcf348 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -148,6 +148,12 @@ then
     OUTFILE=bin/kelp$EXTENSION
     mkdir -p bin
 
+    echo -n "preparing ccxt binary ... "
+    go run ./scripts/ccxt_bin_gen.go -goos $((go env GOOS))
+    check_build_result $?
+    echo "done"
+    echo ""
+
     echo -n "compiling ... "
     go build -ldflags "$LDFLAGS" -o $OUTFILE
     check_build_result $?
@@ -186,6 +192,12 @@ do
         BINARY="$OUTFILE.exe"
     fi
 
+    echo -n "preparing ccxt binary ... "
+    go run ./scripts/ccxt_bin_gen.go -goos $GOOS
+    check_build_result $?
+    echo "done"
+    echo ""
+
     # compile
     env GOOS=$GOOS GOARCH=$GOARCH GOARM=$GOARM go build -ldflags "$LDFLAGS" -o $BINARY
     check_build_result $?
diff --git a/scripts/ccxt_bin_gen.go b/scripts/ccxt_bin_gen.go
new file mode 100644
index 000000000..73f5ab971
--- /dev/null
+++ b/scripts/ccxt_bin_gen.go
@@ -0,0 +1,91 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	"github.com/stellar/kelp/support/kelpos"
+	"github.com/stellar/kelp/support/networking"
+)
+
+const kelpPrefsDirectory = "build"
+const ccxtDownloadURL = "https://github.com/ccxt-rest/ccxt-rest/archive/v0.0.4.tar.gz"
+const ccxtDownloadFilename = "ccxt-rest_v0.0.4.tar.gz"
+const ccxtUntaredDirName = "ccxt-rest-0.0.4"
+const ccxtBinOutputDir = "bin"
+
+func main() {
+	goosP := flag.String("goos", "", "GOOS for which to build")
+	flag.Parse()
+	goos := *goosP
+
+	pkgos = ""
+	if goos == "darwin" {
+		pkgos = "macos"
+	} else if goos == "linux" {
+		pkgos = "linux"
+	} else if goos == "windows" {
+		pkgos = "win"
+	} else {
+		panic("unsupported goos flag: " + goos)
+	}
+	kos := kelpos.GetKelpOS()
+	generateCcxtBinary(kos, pkgos)
+}
+
+func checkPkgTool(kos *kelpos.KelpOS) {
+	fmt.Printf("checking for presence of `pkg` tool ...\n")
+	_, e := kos.Blocking("pkg", "pkg -v")
+	if e != nil {
+		log.Fatal("ensure that the `pkg` tool is installed correctly. You can get it from here: https://github.com/zeit/pkg")
+	}
+	fmt.Printf("done\n")
+}
+
+func downloadCcxtSource(kos *kelpos.KelpOS, downloadDir string) {
+	fmt.Printf("making directory where we can download ccxt file %s ...\n", downloadDir)
+	e := kos.Mkdir(downloadDir)
+	if e != nil {
+		log.Fatal(errors.Wrap(e, "could not make directory for downloadDir "+downloadDir))
+	}
+	fmt.Printf("done\n")
+
+	fmt.Printf("downloading file from URL %s ...\n", ccxtDownloadURL)
+	downloadFilePath := filepath.Join(downloadDir, ccxtDownloadFilename)
+	e = networking.DownloadFile(ccxtDownloadURL, downloadFilePath)
+	if e != nil {
+		log.Fatal(errors.Wrap(e, "could not download ccxt tar.gz file"))
+	}
+	fmt.Printf("done\n")
+
+	fmt.Printf("untaring file %s ...\n", downloadFilePath)
+	_, e = kos.Blocking("tar", fmt.Sprintf("tar xvf %s -C %s", downloadFilePath, downloadDir))
+	if e != nil {
+		log.Fatal(errors.Wrap(e, "could not untar ccxt file"))
+	}
+	fmt.Printf("done\n")
+}
+
+// pkg --targets node8-macos-x64,node8-linux-x64,node8-win-x64 build/downloads/ccxt/ccxt-rest-0.0.4
+func runPkgTool(kos *kelpos.KelpOS, downloadDir string, pkgos string) {
+	sourceDir := filepath.Join(downloadDir, ccxtUntaredDirName)
+	outDir := filepath.Join(downloadDir, ccxtBinOutputDir)
+	target := fmt.Sprintf("node8-%s-x64", pkgos)
+
+	fmt.Printf("running pkg tool on directory %s with output directory as %s on target platform %s ...\n", targetDir, outDir, target)
+	pkgCommand := fmt.Sprintf("pkg --out-path %s --targets %s %s", outDir, target, sourceDir)
+	kos.Blocking("pkg", pkgCommand)
+	fmt.Printf("done\n")
+}
+
+func generateCcxtBinary(kos *kelpos.KelpOS, pkgos string) {
+	checkPkgTool(kos)
+
+	downloadDir := filepath.Join(kelpPrefsDirectory, "downloads", "ccxt")
+	downloadCcxtSource(kos, downloadDir)
+
+	runPkgTool(kos, downloadDir, pkgos)
+}
diff --git a/scripts/fs_bin_gen.go b/scripts/fs_bin_gen.go
index f8e1cc923..f8ac74e68 100644
--- a/scripts/fs_bin_gen.go
+++ b/scripts/fs_bin_gen.go
@@ -2,25 +2,15 @@ package main
 
 import (
 	"flag"
-	"fmt"
 	"log"
 	"net/http"
 	"os/exec"
-	"path/filepath"
-
-	"github.com/pkg/errors"
-	"github.com/stellar/kelp/support/networking"
 
 	"github.com/shurcooL/vfsgen"
-	"github.com/stellar/kelp/support/kelpos"
 )
 
-const kelpPrefsDirectory = "build"
 const fsDev_filename = "./scripts/fs_gen/filesystem_vfsdata_dev.go"
 const fs_filename = "./gui/filesystem_vfsdata.go"
-const ccxtDownloadURL = "https://github.com/ccxt-rest/ccxt-rest/archive/v0.0.4.tar.gz"
-const ccxtDownloadFilename = "ccxt-rest_v0.0.4.tar.gz"
-const ccxtUntaredDirName = "ccxt-rest-0.0.4"
 
 func main() {
 	envP := flag.String("env", "dev", "environment to use (dev / release)")
@@ -34,9 +24,6 @@ func main() {
 	} else {
 		panic("unrecognized env flag: " + env)
 	}
-
-	kos := kelpos.GetKelpOS()
-	generateCcxtBinary(kos)
 }
 
 func generateWeb_Release() {
@@ -59,42 +46,3 @@ func generateWeb_Dev() {
 		log.Fatalln(e)
 	}
 }
-
-func checkPkgTool(kos *kelpos.KelpOS) {
-	fmt.Printf("checking for presence of `pkg` tool ...\n")
-	_, e := kos.Blocking("pkg", "pkg -v")
-	if e != nil {
-		log.Fatal("ensure that the `pkg` tool is installed correctly. You can get it from here: https://github.com/zeit/pkg")
-	}
-	fmt.Printf("done\n")
-}
-
-func downloadCcxtSource(kos *kelpos.KelpOS) {
-	downloadDir := filepath.Join(kelpPrefsDirectory, "downloads", "ccxt")
-	fmt.Printf("making directory where we can download ccxt file %s ...\n", downloadDir)
-	e := kos.Mkdir(downloadDir)
-	if e != nil {
-		log.Fatal(errors.Wrap(e, "could not make directory for downloadDir "+downloadDir))
-	}
-	fmt.Printf("done\n")
-
-	fmt.Printf("downloading file from URL %s ...\n", ccxtDownloadURL)
-	downloadFilePath := filepath.Join(downloadDir, ccxtDownloadFilename)
-	e = networking.DownloadFile(ccxtDownloadURL, downloadFilePath)
-	if e != nil {
-		log.Fatal(errors.Wrap(e, "could not download ccxt tar.gz file"))
-	}
-	fmt.Printf("done\n")
-
-	fmt.Printf("untaring file %s ...\n", downloadFilePath)
-	_, e = kos.Blocking("tar", fmt.Sprintf("tar xvf %s -C %s", downloadFilePath, downloadDir))
-	if e != nil {
-		log.Fatal(errors.Wrap(e, "could not untar ccxt file"))
-	}
-	fmt.Printf("done\n")
-}
-
-func generateCcxtBinary(kos *kelpos.KelpOS) {
-	checkPkgTool(kos)
-	downloadCcxtSource(kos)
-}

From ac3fd3f76eabec1b20f5d052072cc141914ce601 Mon Sep 17 00:00:00 2001
From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com>
Date: Wed, 23 Oct 2019 12:22:40 -0700
Subject: [PATCH 03/22] 3 - move to separate folders in ./scripts

---
 scripts/build.sh                                  | 14 +++++++-------
 scripts/{ => ccxt_bin_gen}/ccxt_bin_gen.go        | 15 ++++++++-------
 .../filesystem_vfsdata_dev.go                     |  0
 scripts/{ => fs_bin_gen}/fs_bin_gen.go            |  2 +-
 4 files changed, 16 insertions(+), 15 deletions(-)
 rename scripts/{ => ccxt_bin_gen}/ccxt_bin_gen.go (90%)
 rename scripts/{fs_gen => fs_bin_gen}/filesystem_vfsdata_dev.go (100%)
 rename scripts/{ => fs_bin_gen}/fs_bin_gen.go (91%)

diff --git a/scripts/build.sh b/scripts/build.sh
index b46dcf348..ba3827466 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -120,7 +120,7 @@ fi
 
 echo ""
 echo "embedding contents of gui/web/build into a .go file (env=$ENV) ..."
-go run ./scripts/fs_bin_gen.go -env $ENV
+go run ./scripts/fs_bin_gen/fs_bin_gen.go -env $ENV
 check_build_result $?
 echo "... finished embedding contents of gui/web/build into a .go file (env=$ENV)"
 echo ""
@@ -148,10 +148,10 @@ then
     OUTFILE=bin/kelp$EXTENSION
     mkdir -p bin
 
-    echo -n "preparing ccxt binary ... "
-    go run ./scripts/ccxt_bin_gen.go -goos $((go env GOOS))
+    echo "preparing ccxt binary ... "
+    go run ./scripts/ccxt_bin_gen/ccxt_bin_gen.go -goos "$(go env GOOS)"
     check_build_result $?
-    echo "done"
+    echo "successful"
     echo ""
 
     echo -n "compiling ... "
@@ -192,10 +192,10 @@ do
         BINARY="$OUTFILE.exe"
     fi
 
-    echo -n "preparing ccxt binary ... "
-    go run ./scripts/ccxt_bin_gen.go -goos $GOOS
+    echo "preparing ccxt binary ... "
+    go run ./scripts/ccxt_bin_gen/ccxt_bin_gen.go -goos $GOOS
     check_build_result $?
-    echo "done"
+    echo "successful"
     echo ""
 
     # compile
diff --git a/scripts/ccxt_bin_gen.go b/scripts/ccxt_bin_gen/ccxt_bin_gen.go
similarity index 90%
rename from scripts/ccxt_bin_gen.go
rename to scripts/ccxt_bin_gen/ccxt_bin_gen.go
index 73f5ab971..d9e17c1e0 100644
--- a/scripts/ccxt_bin_gen.go
+++ b/scripts/ccxt_bin_gen/ccxt_bin_gen.go
@@ -22,7 +22,7 @@ func main() {
 	flag.Parse()
 	goos := *goosP
 
-	pkgos = ""
+	pkgos := ""
 	if goos == "darwin" {
 		pkgos = "macos"
 	} else if goos == "linux" {
@@ -32,6 +32,7 @@ func main() {
 	} else {
 		panic("unsupported goos flag: " + goos)
 	}
+
 	kos := kelpos.GetKelpOS()
 	generateCcxtBinary(kos, pkgos)
 }
@@ -70,12 +71,10 @@ func downloadCcxtSource(kos *kelpos.KelpOS, downloadDir string) {
 }
 
 // pkg --targets node8-macos-x64,node8-linux-x64,node8-win-x64 build/downloads/ccxt/ccxt-rest-0.0.4
-func runPkgTool(kos *kelpos.KelpOS, downloadDir string, pkgos string) {
-	sourceDir := filepath.Join(downloadDir, ccxtUntaredDirName)
-	outDir := filepath.Join(downloadDir, ccxtBinOutputDir)
+func runPkgTool(kos *kelpos.KelpOS, sourceDir string, outDir string, pkgos string) {
 	target := fmt.Sprintf("node8-%s-x64", pkgos)
 
-	fmt.Printf("running pkg tool on directory %s with output directory as %s on target platform %s ...\n", targetDir, outDir, target)
+	fmt.Printf("running pkg tool on source directory %s with output directory as %s on target platform %s ...\n", sourceDir, outDir, target)
 	pkgCommand := fmt.Sprintf("pkg --out-path %s --targets %s %s", outDir, target, sourceDir)
 	kos.Blocking("pkg", pkgCommand)
 	fmt.Printf("done\n")
@@ -85,7 +84,9 @@ func generateCcxtBinary(kos *kelpos.KelpOS, pkgos string) {
 	checkPkgTool(kos)
 
 	downloadDir := filepath.Join(kelpPrefsDirectory, "downloads", "ccxt")
-	downloadCcxtSource(kos, downloadDir)
+	sourceDir := filepath.Join(downloadDir, ccxtUntaredDirName)
+	outDir := filepath.Join(downloadDir, ccxtBinOutputDir)
 
-	runPkgTool(kos, downloadDir, pkgos)
+	downloadCcxtSource(kos, downloadDir)
+	runPkgTool(kos, sourceDir, outDir, pkgos)
 }
diff --git a/scripts/fs_gen/filesystem_vfsdata_dev.go b/scripts/fs_bin_gen/filesystem_vfsdata_dev.go
similarity index 100%
rename from scripts/fs_gen/filesystem_vfsdata_dev.go
rename to scripts/fs_bin_gen/filesystem_vfsdata_dev.go
diff --git a/scripts/fs_bin_gen.go b/scripts/fs_bin_gen/fs_bin_gen.go
similarity index 91%
rename from scripts/fs_bin_gen.go
rename to scripts/fs_bin_gen/fs_bin_gen.go
index f8ac74e68..97b45a67c 100644
--- a/scripts/fs_bin_gen.go
+++ b/scripts/fs_bin_gen/fs_bin_gen.go
@@ -9,7 +9,7 @@ import (
 	"github.com/shurcooL/vfsgen"
 )
 
-const fsDev_filename = "./scripts/fs_gen/filesystem_vfsdata_dev.go"
+const fsDev_filename = "./scripts/fs_bin_gen/filesystem_vfsdata_dev.go"
 const fs_filename = "./gui/filesystem_vfsdata.go"
 
 func main() {

From 7d50ffab99c76c03b6e841e43a459f5b63fc9edb Mon Sep 17 00:00:00 2001
From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com>
Date: Wed, 23 Oct 2019 14:43:52 -0700
Subject: [PATCH 04/22] 4 - run npm install before running pkg tool; copy
 dependency files to output directory

---
 scripts/ccxt_bin_gen/ccxt_bin_gen.go | 36 +++++++++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/scripts/ccxt_bin_gen/ccxt_bin_gen.go b/scripts/ccxt_bin_gen/ccxt_bin_gen.go
index d9e17c1e0..291e9f662 100644
--- a/scripts/ccxt_bin_gen/ccxt_bin_gen.go
+++ b/scripts/ccxt_bin_gen/ccxt_bin_gen.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"log"
 	"path/filepath"
+	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/stellar/kelp/support/kelpos"
@@ -70,13 +71,45 @@ func downloadCcxtSource(kos *kelpos.KelpOS, downloadDir string) {
 	fmt.Printf("done\n")
 }
 
+func npmInstall(kos *kelpos.KelpOS, installDir string) {
+	fmt.Printf("running npm install on directory %s ...\n", installDir)
+	npmCmd := fmt.Sprintf("cd %s && npm install && cd -", installDir)
+	_, e := kos.Blocking("npm", npmCmd)
+	if e != nil {
+		log.Fatal(errors.Wrap(e, "failed to run npm install"))
+	}
+	fmt.Printf("done\n")
+}
+
 // pkg --targets node8-macos-x64,node8-linux-x64,node8-win-x64 build/downloads/ccxt/ccxt-rest-0.0.4
 func runPkgTool(kos *kelpos.KelpOS, sourceDir string, outDir string, pkgos string) {
 	target := fmt.Sprintf("node8-%s-x64", pkgos)
 
 	fmt.Printf("running pkg tool on source directory %s with output directory as %s on target platform %s ...\n", sourceDir, outDir, target)
 	pkgCommand := fmt.Sprintf("pkg --out-path %s --targets %s %s", outDir, target, sourceDir)
-	kos.Blocking("pkg", pkgCommand)
+	outputBytes, e := kos.Blocking("pkg", pkgCommand)
+	if e != nil {
+		log.Fatal(errors.Wrap(e, "failed to run pkg tool"))
+	}
+	fmt.Printf("done\n")
+
+	copyDependencyFiles(kos, outDir, string(outputBytes))
+}
+
+func copyDependencyFiles(kos *kelpos.KelpOS, outDir string, pkgCmdOutput string) {
+	fmt.Printf("copying dependency files to the output directory %s ...\n", outDir)
+	for _, line := range strings.Split(pkgCmdOutput, "\n") {
+		if !strings.Contains(line, "node_modules") {
+			continue
+		}
+		filename := strings.TrimSpace(strings.ReplaceAll(line, "(MISSING)", ""))
+
+		cpCmd := fmt.Sprintf("cp %s %s", filename, outDir)
+		_, e := kos.Blocking("cp", cpCmd)
+		if e != nil {
+			log.Fatal(errors.Wrap(e, "failed to copy dependency file %s"+filename))
+		}
+	}
 	fmt.Printf("done\n")
 }
 
@@ -88,5 +121,6 @@ func generateCcxtBinary(kos *kelpos.KelpOS, pkgos string) {
 	outDir := filepath.Join(downloadDir, ccxtBinOutputDir)
 
 	downloadCcxtSource(kos, downloadDir)
+	npmInstall(kos, sourceDir)
 	runPkgTool(kos, sourceDir, outDir, pkgos)
 }

From 37284ddbfcc454c13fb0377a1f41e5e9eac1de08 Mon Sep 17 00:00:00 2001
From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com>
Date: Wed, 23 Oct 2019 14:45:05 -0700
Subject: [PATCH 05/22] 5 - build script should check build result after
 installing web dependencies and generating static files

---
 scripts/build.sh | 2 ++
 scripts/clean.sh | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/build.sh b/scripts/build.sh
index ba3827466..047bdb740 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -16,6 +16,7 @@ function install_web_dependencies() {
     cd $CURRENT_DIR/gui/web
 
     yarn install
+    check_build_result $?
 
     cd $CURRENT_DIR
     echo "... finished installing web dependencies"
@@ -28,6 +29,7 @@ function generate_static_web_files() {
     cd $CURRENT_DIR/gui/web
 
     yarn build
+    check_build_result $?
 
     cd $CURRENT_DIR
     echo "... finished generating contents of gui/web/build"
diff --git a/scripts/clean.sh b/scripts/clean.sh
index e06760a6c..0e14793a3 100755
--- a/scripts/clean.sh
+++ b/scripts/clean.sh
@@ -18,7 +18,7 @@ fi
 
 echo "removing files ..."
 rm -vrf bin
-rm -vrf build
+delete_large_dir build
 delete_large_dir gui/web/build
 delete_large_dir gui/web/node_modules
 rm -vf gui/filesystem_vfsdata.go

From 63e11db064df7e3a30db05f8d344a871246a09a2 Mon Sep 17 00:00:00 2001
From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com>
Date: Wed, 23 Oct 2019 14:48:52 -0700
Subject: [PATCH 06/22] 6 - wrap underlying error when pkg check fails

---
 scripts/ccxt_bin_gen/ccxt_bin_gen.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/ccxt_bin_gen/ccxt_bin_gen.go b/scripts/ccxt_bin_gen/ccxt_bin_gen.go
index 291e9f662..792ea7044 100644
--- a/scripts/ccxt_bin_gen/ccxt_bin_gen.go
+++ b/scripts/ccxt_bin_gen/ccxt_bin_gen.go
@@ -42,7 +42,7 @@ func checkPkgTool(kos *kelpos.KelpOS) {
 	fmt.Printf("checking for presence of `pkg` tool ...\n")
 	_, e := kos.Blocking("pkg", "pkg -v")
 	if e != nil {
-		log.Fatal("ensure that the `pkg` tool is installed correctly. You can get it from here: https://github.com/zeit/pkg")
+		log.Fatal(errors.Wrap(e, "ensure that the `pkg` tool is installed correctly. You can get it from here: https://github.com/zeit/pkg"))
 	}
 	fmt.Printf("done\n")
 }

From 2c653066c0639c1828b93314ffee85bdb7085282 Mon Sep 17 00:00:00 2001
From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com>
Date: Wed, 23 Oct 2019 16:42:46 -0700
Subject: [PATCH 07/22] 7 - check node version before generating ccxt binary

---
 scripts/ccxt_bin_gen/ccxt_bin_gen.go | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/scripts/ccxt_bin_gen/ccxt_bin_gen.go b/scripts/ccxt_bin_gen/ccxt_bin_gen.go
index 792ea7044..f176ede59 100644
--- a/scripts/ccxt_bin_gen/ccxt_bin_gen.go
+++ b/scripts/ccxt_bin_gen/ccxt_bin_gen.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"log"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -17,6 +18,7 @@ const ccxtDownloadURL = "https://github.com/ccxt-rest/ccxt-rest/archive/v0.0.4.t
 const ccxtDownloadFilename = "ccxt-rest_v0.0.4.tar.gz"
 const ccxtUntaredDirName = "ccxt-rest-0.0.4"
 const ccxtBinOutputDir = "bin"
+const nodeVersionMatchRegex = "v8.[0-9]+.[0-9]+"
 
 func main() {
 	goosP := flag.String("goos", "", "GOOS for which to build")
@@ -38,11 +40,30 @@ func main() {
 	generateCcxtBinary(kos, pkgos)
 }
 
+func checkNodeVersion(kos *kelpos.KelpOS) {
+	fmt.Printf("checking node version ...\n")
+
+	version, e := kos.Blocking("node", "node -v")
+	if e != nil {
+		log.Fatal(errors.Wrap(e, "ensure that the `pkg` tool is installed correctly. You can get it from here https://github.com/zeit/pkg or by running `npm install -g pkg`"))
+	}
+
+	match, e := regexp.Match(nodeVersionMatchRegex, version)
+	if e != nil {
+		log.Fatal(errors.Wrap(e, "could not match regex against node version"))
+	}
+	if !match {
+		log.Fatal("node version will fail to compile a successful binary because of the requirements of ccxt-rest's dependencies, should use v8.x.x instead of " + string(version))
+	}
+
+	fmt.Printf("valid\n")
+}
+
 func checkPkgTool(kos *kelpos.KelpOS) {
 	fmt.Printf("checking for presence of `pkg` tool ...\n")
 	_, e := kos.Blocking("pkg", "pkg -v")
 	if e != nil {
-		log.Fatal(errors.Wrap(e, "ensure that the `pkg` tool is installed correctly. You can get it from here: https://github.com/zeit/pkg"))
+		log.Fatal(errors.Wrap(e, "ensure that the `pkg` tool is installed correctly. You can get it from here https://github.com/zeit/pkg or by running `npm install -g pkg`"))
 	}
 	fmt.Printf("done\n")
 }
@@ -104,16 +125,18 @@ func copyDependencyFiles(kos *kelpos.KelpOS, outDir string, pkgCmdOutput string)
 		}
 		filename := strings.TrimSpace(strings.ReplaceAll(line, "(MISSING)", ""))
 
+		fmt.Printf("copying file %s to the output directory %s ...\n", filename, outDir)
 		cpCmd := fmt.Sprintf("cp %s %s", filename, outDir)
 		_, e := kos.Blocking("cp", cpCmd)
 		if e != nil {
-			log.Fatal(errors.Wrap(e, "failed to copy dependency file %s"+filename))
+			log.Fatal(errors.Wrap(e, "failed to copy dependency file "+filename))
 		}
 	}
 	fmt.Printf("done\n")
 }
 
 func generateCcxtBinary(kos *kelpos.KelpOS, pkgos string) {
+	checkNodeVersion(kos)
 	checkPkgTool(kos)
 
 	downloadDir := filepath.Join(kelpPrefsDirectory, "downloads", "ccxt")

From 0e564abf5340800d0beeec7c75dfbb2e71fdcce8 Mon Sep 17 00:00:00 2001
From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com>
Date: Wed, 23 Oct 2019 16:44:00 -0700
Subject: [PATCH 08/22] 8 - separate -g flag on build script to generate ccxt
 binary

---
 scripts/build.sh | 62 ++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 50 insertions(+), 12 deletions(-)

diff --git a/scripts/build.sh b/scripts/build.sh
index 047bdb740..2ebc8c4a3 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -1,13 +1,13 @@
 #!/bin/bash
 
 function usage() {
-    echo "Usage: $0 [flags]"
+    echo "Usage: $0 [flags] [flag-fields]"
     echo ""
     echo "Flags:"
-    echo "    -d, --deploy        prepare tar archives in build/, only works on a tagged commit in the format v1.0.0 or v1.0.0-rc1"
-    echo "    -t, --test-deploy   test prepare tar archives in build/ for your native platform only"
-    echo "    -h, --help          show this help info"
-    exit 1
+    echo "    -d,   --deploy        prepare tar archives in build/, only works on a tagged commit in the format v1.0.0 or v1.0.0-rc1"
+    echo "    -t,   --test-deploy   test prepare tar archives in build/ for your native platform only"
+    echo "    -g,   --gen-ccxt      generate binary for ccxt-rest executable for to be uploaded to GitHub for use in building kelp binary, takes in arguments (linux, darwin, windows)"
+    echo "    -h,   --help          show this help info"
 }
 
 function install_web_dependencies() {
@@ -46,7 +46,19 @@ function check_build_result() {
     fi
 }
 
-if [[ ($# -gt 1 || $(basename $("pwd")) != "kelp") ]]
+# takes in the GOOS for which to build
+function gen_ccxt_binary() {
+    echo "generating ccxt binary for GOOS=$1 ... "
+    go run ./scripts/ccxt_bin_gen/ccxt_bin_gen.go -goos $1
+    echo "successful"
+}
+
+function fetch_ccxt_binary() {
+    # TODO fetch pre-compiled version from GitHub
+    return
+}
+
+if [[ $(basename $("pwd")) != "kelp" ]]
 then
     echo "need to invoke from the root 'kelp' directory"
     exit 1
@@ -62,10 +74,36 @@ elif [[ ($# -eq 1 && ("$1" == "-t" || "$1" == "--test-deploy")) ]]; then
     IS_TEST_MODE=1
 elif [[ ($# -eq 1 && ("$1" == "-h" || "$1" == "--help")) ]]; then
     usage
-elif [[ $# -eq 1 ]]; then
+    exit 0
+elif [[ (($# -eq 1 || $# -eq 2) && ("$1" == "-g" || "$1" == "--gen-ccxt")) ]]; then
+    if [[ $# -eq 1 ]]; then
+        echo "the $1 flag needs to be followed by the GOOS for which to build the ccxt binary"
+        echo ""
+        usage
+        exit 1
+    fi
+
+    if [[ $# -eq 2 ]]; then
+        if [[ "$2" == "linux" || "$2" == "darwin" || "$2" == "windows" ]]; then
+            gen_ccxt_binary $2
+            echo ""
+            echo "BUILD SUCCESSFUL"
+            exit 0
+        else
+            echo "invalid GOOS type passed in: $2"
+            echo ""
+            usage
+            exit 1
+        fi
+    fi
+
     usage
-else
+    exit 1
+elif [[ $# -eq 0 ]]; then
     ENV=dev
+else
+    usage
+    exit 1
 fi
 
 # version is git tag if it's available, otherwise git hash
@@ -150,8 +188,8 @@ then
     OUTFILE=bin/kelp$EXTENSION
     mkdir -p bin
 
-    echo "preparing ccxt binary ... "
-    go run ./scripts/ccxt_bin_gen/ccxt_bin_gen.go -goos "$(go env GOOS)"
+    echo "fetching ccxt binary ... "
+    fetch_ccxt_binary "$(go env GOOS)"
     check_build_result $?
     echo "successful"
     echo ""
@@ -194,8 +232,8 @@ do
         BINARY="$OUTFILE.exe"
     fi
 
-    echo "preparing ccxt binary ... "
-    go run ./scripts/ccxt_bin_gen/ccxt_bin_gen.go -goos $GOOS
+    echo "fetching ccxt binary ... "
+    fetch_ccxt_binary $GOOS
     check_build_result $?
     echo "successful"
     echo ""

From b000f71cd784de2afd68999761292974d27db208 Mon Sep 17 00:00:00 2001
From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com>
Date: Wed, 23 Oct 2019 16:45:41 -0700
Subject: [PATCH 09/22] 9 - remove /downloads from path of ccxt binary output

---
 scripts/ccxt_bin_gen/ccxt_bin_gen.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/ccxt_bin_gen/ccxt_bin_gen.go b/scripts/ccxt_bin_gen/ccxt_bin_gen.go
index f176ede59..edd3372ba 100644
--- a/scripts/ccxt_bin_gen/ccxt_bin_gen.go
+++ b/scripts/ccxt_bin_gen/ccxt_bin_gen.go
@@ -102,7 +102,7 @@ func npmInstall(kos *kelpos.KelpOS, installDir string) {
 	fmt.Printf("done\n")
 }
 
-// pkg --targets node8-macos-x64,node8-linux-x64,node8-win-x64 build/downloads/ccxt/ccxt-rest-0.0.4
+// pkg --targets node8-macos-x64,node8-linux-x64,node8-win-x64 build/ccxt/ccxt-rest-0.0.4
 func runPkgTool(kos *kelpos.KelpOS, sourceDir string, outDir string, pkgos string) {
 	target := fmt.Sprintf("node8-%s-x64", pkgos)
 
@@ -139,7 +139,7 @@ func generateCcxtBinary(kos *kelpos.KelpOS, pkgos string) {
 	checkNodeVersion(kos)
 	checkPkgTool(kos)
 
-	downloadDir := filepath.Join(kelpPrefsDirectory, "downloads", "ccxt")
+	downloadDir := filepath.Join(kelpPrefsDirectory, "ccxt")
 	sourceDir := filepath.Join(downloadDir, ccxtUntaredDirName)
 	outDir := filepath.Join(downloadDir, ccxtBinOutputDir)
 

From e9c9e6a78632c54299cf9f9d4f9cf29d14ecb235 Mon Sep 17 00:00:00 2001
From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com>
Date: Wed, 23 Oct 2019 17:01:24 -0700
Subject: [PATCH 10/22] 10 - set silent registrations on kelpos and prettify
 ccxt_bin_gen log output

---
 scripts/build.sh                     |  3 ++-
 scripts/ccxt_bin_gen/ccxt_bin_gen.go | 22 +++++++++++++---------
 support/kelpos/kelpos.go             | 14 ++++++++++----
 support/kelpos/process.go            |  8 ++++++--
 4 files changed, 31 insertions(+), 16 deletions(-)

diff --git a/scripts/build.sh b/scripts/build.sh
index 2ebc8c4a3..26d177061 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -48,7 +48,8 @@ function check_build_result() {
 
 # takes in the GOOS for which to build
 function gen_ccxt_binary() {
-    echo "generating ccxt binary for GOOS=$1 ... "
+    echo "generating ccxt binary for GOOS=$1"
+    echo ""
     go run ./scripts/ccxt_bin_gen/ccxt_bin_gen.go -goos $1
     echo "successful"
 }
diff --git a/scripts/ccxt_bin_gen/ccxt_bin_gen.go b/scripts/ccxt_bin_gen/ccxt_bin_gen.go
index edd3372ba..cc6acde27 100644
--- a/scripts/ccxt_bin_gen/ccxt_bin_gen.go
+++ b/scripts/ccxt_bin_gen/ccxt_bin_gen.go
@@ -37,11 +37,12 @@ func main() {
 	}
 
 	kos := kelpos.GetKelpOS()
+	kos.SetSilentRegistrations()
 	generateCcxtBinary(kos, pkgos)
 }
 
 func checkNodeVersion(kos *kelpos.KelpOS) {
-	fmt.Printf("checking node version ...\n")
+	fmt.Printf("checking node version ... ")
 
 	version, e := kos.Blocking("node", "node -v")
 	if e != nil {
@@ -60,7 +61,7 @@ func checkNodeVersion(kos *kelpos.KelpOS) {
 }
 
 func checkPkgTool(kos *kelpos.KelpOS) {
-	fmt.Printf("checking for presence of `pkg` tool ...\n")
+	fmt.Printf("checking for presence of `pkg` tool ... ")
 	_, e := kos.Blocking("pkg", "pkg -v")
 	if e != nil {
 		log.Fatal(errors.Wrap(e, "ensure that the `pkg` tool is installed correctly. You can get it from here https://github.com/zeit/pkg or by running `npm install -g pkg`"))
@@ -69,14 +70,14 @@ func checkPkgTool(kos *kelpos.KelpOS) {
 }
 
 func downloadCcxtSource(kos *kelpos.KelpOS, downloadDir string) {
-	fmt.Printf("making directory where we can download ccxt file %s ...\n", downloadDir)
+	fmt.Printf("making directory where we can download ccxt file %s ... ", downloadDir)
 	e := kos.Mkdir(downloadDir)
 	if e != nil {
 		log.Fatal(errors.Wrap(e, "could not make directory for downloadDir "+downloadDir))
 	}
 	fmt.Printf("done\n")
 
-	fmt.Printf("downloading file from URL %s ...\n", ccxtDownloadURL)
+	fmt.Printf("downloading file from URL %s ... ", ccxtDownloadURL)
 	downloadFilePath := filepath.Join(downloadDir, ccxtDownloadFilename)
 	e = networking.DownloadFile(ccxtDownloadURL, downloadFilePath)
 	if e != nil {
@@ -84,7 +85,7 @@ func downloadCcxtSource(kos *kelpos.KelpOS, downloadDir string) {
 	}
 	fmt.Printf("done\n")
 
-	fmt.Printf("untaring file %s ...\n", downloadFilePath)
+	fmt.Printf("untaring file %s ... ", downloadFilePath)
 	_, e = kos.Blocking("tar", fmt.Sprintf("tar xvf %s -C %s", downloadFilePath, downloadDir))
 	if e != nil {
 		log.Fatal(errors.Wrap(e, "could not untar ccxt file"))
@@ -93,7 +94,7 @@ func downloadCcxtSource(kos *kelpos.KelpOS, downloadDir string) {
 }
 
 func npmInstall(kos *kelpos.KelpOS, installDir string) {
-	fmt.Printf("running npm install on directory %s ...\n", installDir)
+	fmt.Printf("running npm install on directory %s ... ", installDir)
 	npmCmd := fmt.Sprintf("cd %s && npm install && cd -", installDir)
 	_, e := kos.Blocking("npm", npmCmd)
 	if e != nil {
@@ -102,11 +103,11 @@ func npmInstall(kos *kelpos.KelpOS, installDir string) {
 	fmt.Printf("done\n")
 }
 
-// pkg --targets node8-macos-x64,node8-linux-x64,node8-win-x64 build/ccxt/ccxt-rest-0.0.4
+// pkg --targets node8-linux-x64 build/ccxt/ccxt-rest-0.0.4
 func runPkgTool(kos *kelpos.KelpOS, sourceDir string, outDir string, pkgos string) {
 	target := fmt.Sprintf("node8-%s-x64", pkgos)
 
-	fmt.Printf("running pkg tool on source directory %s with output directory as %s on target platform %s ...\n", sourceDir, outDir, target)
+	fmt.Printf("running pkg tool on source directory %s with output directory as %s on target platform %s ... ", sourceDir, outDir, target)
 	pkgCommand := fmt.Sprintf("pkg --out-path %s --targets %s %s", outDir, target, sourceDir)
 	outputBytes, e := kos.Blocking("pkg", pkgCommand)
 	if e != nil {
@@ -118,6 +119,7 @@ func runPkgTool(kos *kelpos.KelpOS, sourceDir string, outDir string, pkgos strin
 }
 
 func copyDependencyFiles(kos *kelpos.KelpOS, outDir string, pkgCmdOutput string) {
+	fmt.Println()
 	fmt.Printf("copying dependency files to the output directory %s ...\n", outDir)
 	for _, line := range strings.Split(pkgCmdOutput, "\n") {
 		if !strings.Contains(line, "node_modules") {
@@ -125,14 +127,16 @@ func copyDependencyFiles(kos *kelpos.KelpOS, outDir string, pkgCmdOutput string)
 		}
 		filename := strings.TrimSpace(strings.ReplaceAll(line, "(MISSING)", ""))
 
-		fmt.Printf("copying file %s to the output directory %s ...\n", filename, outDir)
+		fmt.Printf("    copying file %s to the output directory %s ... ", filename, outDir)
 		cpCmd := fmt.Sprintf("cp %s %s", filename, outDir)
 		_, e := kos.Blocking("cp", cpCmd)
 		if e != nil {
 			log.Fatal(errors.Wrap(e, "failed to copy dependency file "+filename))
 		}
+		fmt.Printf("done\n")
 	}
 	fmt.Printf("done\n")
+	fmt.Println()
 }
 
 func generateCcxtBinary(kos *kelpos.KelpOS, pkgos string) {
diff --git a/support/kelpos/kelpos.go b/support/kelpos/kelpos.go
index 4fd887150..46b3b1d60 100644
--- a/support/kelpos/kelpos.go
+++ b/support/kelpos/kelpos.go
@@ -11,10 +11,16 @@ import (
 
 // KelpOS is a struct that manages all subprocesses started by this Kelp process
 type KelpOS struct {
-	processes   map[string]Process
-	processLock *sync.Mutex
-	bots        map[string]*BotInstance
-	botLock     *sync.Mutex
+	processes           map[string]Process
+	processLock         *sync.Mutex
+	bots                map[string]*BotInstance
+	botLock             *sync.Mutex
+	silentRegistrations bool
+}
+
+// SetSilentRegistrations does not log every time we register and unregister commands
+func (kos *KelpOS) SetSilentRegistrations() {
+	kos.silentRegistrations = true
 }
 
 // Process contains all the pieces that can be used to control a given process
diff --git a/support/kelpos/process.go b/support/kelpos/process.go
index 516860df6..f8a4aa97a 100644
--- a/support/kelpos/process.go
+++ b/support/kelpos/process.go
@@ -130,7 +130,9 @@ func (kos *KelpOS) register(namespace string, p *Process) error {
 	}
 
 	kos.processes[namespace] = *p
-	log.Printf("registered command under namespace '%s' with PID: %d, processes available: %v\n", namespace, p.Cmd.Process.Pid, kos.RegisteredProcesses())
+	if !kos.silentRegistrations {
+		log.Printf("registered command under namespace '%s' with PID: %d, processes available: %v\n", namespace, p.Cmd.Process.Pid, kos.RegisteredProcesses())
+	}
 	return nil
 }
 
@@ -141,7 +143,9 @@ func (kos *KelpOS) Unregister(namespace string) error {
 
 	if p, exists := kos.processes[namespace]; exists {
 		delete(kos.processes, namespace)
-		log.Printf("unregistered command under namespace '%s' with PID: %d, processes available: %v\n", namespace, p.Cmd.Process.Pid, kos.RegisteredProcesses())
+		if !kos.silentRegistrations {
+			log.Printf("unregistered command under namespace '%s' with PID: %d, processes available: %v\n", namespace, p.Cmd.Process.Pid, kos.RegisteredProcesses())
+		}
 		return nil
 	}
 	return fmt.Errorf("process with namespace does not exist: %s", namespace)

From 39c1c12e579cd1b004c7d3adf1f397e68f8815ae Mon Sep 17 00:00:00 2001
From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com>
Date: Wed, 23 Oct 2019 17:23:02 -0700
Subject: [PATCH 11/22] 11 - zip output of ccxt binary

---
 scripts/ccxt_bin_gen/ccxt_bin_gen.go | 37 +++++++++++++++++++++++-----
 1 file changed, 31 insertions(+), 6 deletions(-)

diff --git a/scripts/ccxt_bin_gen/ccxt_bin_gen.go b/scripts/ccxt_bin_gen/ccxt_bin_gen.go
index cc6acde27..24b903b7f 100644
--- a/scripts/ccxt_bin_gen/ccxt_bin_gen.go
+++ b/scripts/ccxt_bin_gen/ccxt_bin_gen.go
@@ -38,7 +38,9 @@ func main() {
 
 	kos := kelpos.GetKelpOS()
 	kos.SetSilentRegistrations()
-	generateCcxtBinary(kos, pkgos)
+
+	zipFilename := fmt.Sprintf("ccxt-rest_node8-%s-%s-x64.zip", goos, pkgos)
+	generateCcxtBinary(kos, pkgos, zipFilename)
 }
 
 func checkNodeVersion(kos *kelpos.KelpOS) {
@@ -139,15 +141,38 @@ func copyDependencyFiles(kos *kelpos.KelpOS, outDir string, pkgCmdOutput string)
 	fmt.Println()
 }
 
-func generateCcxtBinary(kos *kelpos.KelpOS, pkgos string) {
+func mkDir(kos *kelpos.KelpOS, zipDir string) {
+	fmt.Printf("making directory %s ... ", zipDir)
+	e := kos.Mkdir(zipDir)
+	if e != nil {
+		log.Fatal(errors.Wrap(e, "unable to make directory "+zipDir))
+	}
+	fmt.Printf("done\n")
+}
+
+func zipOutput(kos *kelpos.KelpOS, sourceDir string, zipPath string) {
+	fmt.Printf("zipping directory %s as file %s ... ", sourceDir, zipPath)
+	zipCmd := fmt.Sprintf("zip -q %s %s", zipPath, sourceDir)
+	_, e := kos.Blocking("zip", zipCmd)
+	if e != nil {
+		log.Fatal(errors.Wrap(e, "unable to zip folder with ccxt binary and dependencies"))
+	}
+	fmt.Printf("done\n")
+}
+
+func generateCcxtBinary(kos *kelpos.KelpOS, pkgos string, zipFilename string) {
 	checkNodeVersion(kos)
 	checkPkgTool(kos)
 
-	downloadDir := filepath.Join(kelpPrefsDirectory, "ccxt")
-	sourceDir := filepath.Join(downloadDir, ccxtUntaredDirName)
-	outDir := filepath.Join(downloadDir, ccxtBinOutputDir)
+	ccxtDir := filepath.Join(kelpPrefsDirectory, "ccxt")
+	sourceDir := filepath.Join(ccxtDir, ccxtUntaredDirName)
+	outDir := filepath.Join(ccxtDir, ccxtBinOutputDir)
+	zipDir := filepath.Join(ccxtDir, "zipped")
+	zipPath := filepath.Join(zipDir, zipFilename)
 
-	downloadCcxtSource(kos, downloadDir)
+	downloadCcxtSource(kos, ccxtDir)
 	npmInstall(kos, sourceDir)
 	runPkgTool(kos, sourceDir, outDir, pkgos)
+	mkDir(kos, zipDir)
+	zipOutput(kos, outDir, zipPath)
 }

From e614f41500110476b1772ac0031f64f668efbd45 Mon Sep 17 00:00:00 2001
From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com>
Date: Wed, 23 Oct 2019 17:23:26 -0700
Subject: [PATCH 12/22] 12 - check result of running ccxt_bin_gen in main build
 script

---
 scripts/build.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/build.sh b/scripts/build.sh
index 26d177061..369f631f1 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -51,6 +51,7 @@ function gen_ccxt_binary() {
     echo "generating ccxt binary for GOOS=$1"
     echo ""
     go run ./scripts/ccxt_bin_gen/ccxt_bin_gen.go -goos $1
+    check_build_result $?
     echo "successful"
 }
 

From 3365fc95deec2f74ba3aef8b271ce3ceda2dca61 Mon Sep 17 00:00:00 2001
From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com>
Date: Wed, 23 Oct 2019 17:50:00 -0700
Subject: [PATCH 13/22] 13 - keep zipped output file in it's own folder and
 remove inner folder hierarchy

---
 scripts/ccxt_bin_gen/ccxt_bin_gen.go | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/scripts/ccxt_bin_gen/ccxt_bin_gen.go b/scripts/ccxt_bin_gen/ccxt_bin_gen.go
index 24b903b7f..3db6225b6 100644
--- a/scripts/ccxt_bin_gen/ccxt_bin_gen.go
+++ b/scripts/ccxt_bin_gen/ccxt_bin_gen.go
@@ -39,8 +39,8 @@ func main() {
 	kos := kelpos.GetKelpOS()
 	kos.SetSilentRegistrations()
 
-	zipFilename := fmt.Sprintf("ccxt-rest_node8-%s-%s-x64.zip", goos, pkgos)
-	generateCcxtBinary(kos, pkgos, zipFilename)
+	zipFoldername := fmt.Sprintf("ccxt-rest_%s-x64", goos)
+	generateCcxtBinary(kos, pkgos, zipFoldername)
 }
 
 func checkNodeVersion(kos *kelpos.KelpOS) {
@@ -150,9 +150,10 @@ func mkDir(kos *kelpos.KelpOS, zipDir string) {
 	fmt.Printf("done\n")
 }
 
-func zipOutput(kos *kelpos.KelpOS, sourceDir string, zipPath string) {
-	fmt.Printf("zipping directory %s as file %s ... ", sourceDir, zipPath)
-	zipCmd := fmt.Sprintf("zip -q %s %s", zipPath, sourceDir)
+func zipOutput(kos *kelpos.KelpOS, ccxtDir string, sourceDir string, zipFoldername string, zipOutDir string) {
+	zipFilename := zipFoldername + ".zip"
+	fmt.Printf("zipping directory %s as file %s ... ", filepath.Join(ccxtDir, ccxtBinOutputDir), zipFilename)
+	zipCmd := fmt.Sprintf("cd %s && mv %s %s && zip -rq %s %s && cd - && mv %s %s", ccxtDir, ccxtBinOutputDir, zipFoldername, zipFilename, zipFoldername, filepath.Join(ccxtDir, zipFilename), zipOutDir)
 	_, e := kos.Blocking("zip", zipCmd)
 	if e != nil {
 		log.Fatal(errors.Wrap(e, "unable to zip folder with ccxt binary and dependencies"))
@@ -160,19 +161,18 @@ func zipOutput(kos *kelpos.KelpOS, sourceDir string, zipPath string) {
 	fmt.Printf("done\n")
 }
 
-func generateCcxtBinary(kos *kelpos.KelpOS, pkgos string, zipFilename string) {
+func generateCcxtBinary(kos *kelpos.KelpOS, pkgos string, zipFoldername string) {
 	checkNodeVersion(kos)
 	checkPkgTool(kos)
 
 	ccxtDir := filepath.Join(kelpPrefsDirectory, "ccxt")
 	sourceDir := filepath.Join(ccxtDir, ccxtUntaredDirName)
 	outDir := filepath.Join(ccxtDir, ccxtBinOutputDir)
-	zipDir := filepath.Join(ccxtDir, "zipped")
-	zipPath := filepath.Join(zipDir, zipFilename)
+	zipOutDir := filepath.Join(ccxtDir, "zipped")
 
 	downloadCcxtSource(kos, ccxtDir)
 	npmInstall(kos, sourceDir)
 	runPkgTool(kos, sourceDir, outDir, pkgos)
-	mkDir(kos, zipDir)
-	zipOutput(kos, outDir, zipPath)
+	mkDir(kos, zipOutDir)
+	zipOutput(kos, ccxtDir, outDir, zipFoldername, zipOutDir)
 }

From c761f4575dbde5eb828f8388cfbfa09041b20f35 Mon Sep 17 00:00:00 2001
From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com>
Date: Thu, 24 Oct 2019 14:41:48 -0700
Subject: [PATCH 14/22] 14 - clean up directory that was zipped

---
 scripts/ccxt_bin_gen/ccxt_bin_gen.go | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/scripts/ccxt_bin_gen/ccxt_bin_gen.go b/scripts/ccxt_bin_gen/ccxt_bin_gen.go
index 3db6225b6..ae3252145 100644
--- a/scripts/ccxt_bin_gen/ccxt_bin_gen.go
+++ b/scripts/ccxt_bin_gen/ccxt_bin_gen.go
@@ -159,6 +159,15 @@ func zipOutput(kos *kelpos.KelpOS, ccxtDir string, sourceDir string, zipFolderna
 		log.Fatal(errors.Wrap(e, "unable to zip folder with ccxt binary and dependencies"))
 	}
 	fmt.Printf("done\n")
+
+	zipDirPath := filepath.Join(ccxtDir, zipFoldername)
+	fmt.Printf("clean up zipped directory %s ... ", zipDirPath)
+	cleanupCmd := fmt.Sprintf("rm %s/* && rmdir %s", zipDirPath, zipDirPath)
+	_, e = kos.Blocking("zip", cleanupCmd)
+	if e != nil {
+		log.Fatal(errors.Wrap(e, fmt.Sprintf("unable to cleanup zip folder %s with ccxt binary and dependencies", zipDirPath)))
+	}
+	fmt.Printf("done\n")
 }
 
 func generateCcxtBinary(kos *kelpos.KelpOS, pkgos string, zipFoldername string) {

From d0939de89df1ac6df887f85a770c970e01e931f9 Mon Sep 17 00:00:00 2001
From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com>
Date: Fri, 25 Oct 2019 15:20:28 -0700
Subject: [PATCH 15/22] 15 - remove fetch logic of ccxt binary from build
 script

---
 scripts/build.sh | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/scripts/build.sh b/scripts/build.sh
index 369f631f1..3fa0b9824 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -55,11 +55,6 @@ function gen_ccxt_binary() {
     echo "successful"
 }
 
-function fetch_ccxt_binary() {
-    # TODO fetch pre-compiled version from GitHub
-    return
-}
-
 if [[ $(basename $("pwd")) != "kelp" ]]
 then
     echo "need to invoke from the root 'kelp' directory"
@@ -190,12 +185,6 @@ then
     OUTFILE=bin/kelp$EXTENSION
     mkdir -p bin
 
-    echo "fetching ccxt binary ... "
-    fetch_ccxt_binary "$(go env GOOS)"
-    check_build_result $?
-    echo "successful"
-    echo ""
-
     echo -n "compiling ... "
     go build -ldflags "$LDFLAGS" -o $OUTFILE
     check_build_result $?
@@ -234,12 +223,6 @@ do
         BINARY="$OUTFILE.exe"
     fi
 
-    echo "fetching ccxt binary ... "
-    fetch_ccxt_binary $GOOS
-    check_build_result $?
-    echo "successful"
-    echo ""
-
     # compile
     env GOOS=$GOOS GOARCH=$GOARCH GOARM=$GOARM go build -ldflags "$LDFLAGS" -o $BINARY
     check_build_result $?

From f8f1a9bd16b53a1a23fed8e92f389c269b9dc6f5 Mon Sep 17 00:00:00 2001
From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com>
Date: Fri, 25 Oct 2019 15:58:31 -0700
Subject: [PATCH 16/22] update circle config to install yarn

---
 .circleci/config.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index cec473bba..bd3eca2d9 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -6,7 +6,7 @@ commands:
       - restore_cache:
           keys:
             - v2-pkg-cache
-            - v1-node-cache
+            - v2-node-cache
       - run:
           name: Install glide
           command: curl https://glide.sh/get | sh
@@ -36,6 +36,9 @@ commands:
       - run:
           name: Install nvm
           command: curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+      - run:
+          name: Install yarn
+          command: curl -o- -L https://yarnpkg.com/install.sh | bash
       - run:
           name: Use Node 10.5
           command: echo 'export NVM_DIR=$HOME/.nvm' >> $BASH_ENV && echo 'source $NVM_DIR/nvm.sh' >> $BASH_ENV && source $BASH_ENV && nvm install 10.5 && nvm use 10.5
@@ -43,7 +46,7 @@ commands:
           name: Build Kelp
           command: ./scripts/build.sh
       - save_cache:
-          key: v1-node-cache
+          key: v2-node-cache
           paths:
             - "/go/src/github.com/stellar/kelp/gui/web/node_modules"
       - run:

From d6d6201d13c43d0b7d59320a83c5100165b755f8 Mon Sep 17 00:00:00 2001
From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com>
Date: Tue, 29 Oct 2019 18:18:29 -0700
Subject: [PATCH 17/22] 17 - install yarn after usinng correct version of node

---
 .circleci/config.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index bd3eca2d9..ca27ec1d6 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -6,7 +6,7 @@ commands:
       - restore_cache:
           keys:
             - v2-pkg-cache
-            - v2-node-cache
+            - v3-node-cache
       - run:
           name: Install glide
           command: curl https://glide.sh/get | sh
@@ -36,17 +36,17 @@ commands:
       - run:
           name: Install nvm
           command: curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
-      - run:
-          name: Install yarn
-          command: curl -o- -L https://yarnpkg.com/install.sh | bash
       - run:
           name: Use Node 10.5
           command: echo 'export NVM_DIR=$HOME/.nvm' >> $BASH_ENV && echo 'source $NVM_DIR/nvm.sh' >> $BASH_ENV && source $BASH_ENV && nvm install 10.5 && nvm use 10.5
+      - run:
+          name: Install yarn
+          command: curl -o- -L https://yarnpkg.com/install.sh | bash
       - run:
           name: Build Kelp
           command: ./scripts/build.sh
       - save_cache:
-          key: v2-node-cache
+          key: v3-node-cache
           paths:
             - "/go/src/github.com/stellar/kelp/gui/web/node_modules"
       - run:

From b6b6db3f2c3f426784e40be0b13a45482b6110d5 Mon Sep 17 00:00:00 2001
From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com>
Date: Tue, 29 Oct 2019 18:21:37 -0700
Subject: [PATCH 18/22] 18 - command to use yarn in same shell

---
 .circleci/config.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index ca27ec1d6..411dc572f 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -6,7 +6,7 @@ commands:
       - restore_cache:
           keys:
             - v2-pkg-cache
-            - v3-node-cache
+            - v4-node-cache
       - run:
           name: Install glide
           command: curl https://glide.sh/get | sh
@@ -42,11 +42,14 @@ commands:
       - run:
           name: Install yarn
           command: curl -o- -L https://yarnpkg.com/install.sh | bash
+      - run:
+          name: Use yarn
+          command: export PATH="$HOME/.yarn/bin:$HOME/.config/yarn/global/node_modules/.bin:$PATH"
       - run:
           name: Build Kelp
           command: ./scripts/build.sh
       - save_cache:
-          key: v3-node-cache
+          key: v4-node-cache
           paths:
             - "/go/src/github.com/stellar/kelp/gui/web/node_modules"
       - run:

From ccd3e5218abb2020e48291bd0ad7f43ac95d5737 Mon Sep 17 00:00:00 2001
From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com>
Date: Tue, 29 Oct 2019 18:29:23 -0700
Subject: [PATCH 19/22] 19 - update command to use yarn and move setup of nvm
 and yarn to install_deps section

---
 .circleci/config.yml | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 411dc572f..37dcf845b 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -5,8 +5,8 @@ commands:
       - checkout
       - restore_cache:
           keys:
-            - v2-pkg-cache
-            - v4-node-cache
+            - v3-pkg-cache
+            - v5-node-cache
       - run:
           name: Install glide
           command: curl https://glide.sh/get | sh
@@ -19,9 +19,20 @@ commands:
       - run:
           name: Install astilectron-bundler
           command: go install github.com/asticode/go-astilectron-bundler
-
+      - run:
+          name: Install nvm
+          command: curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+      - run:
+          name: Use Node 10.5
+          command: echo 'export NVM_DIR=$HOME/.nvm' >> $BASH_ENV && echo 'source $NVM_DIR/nvm.sh' >> $BASH_ENV && source $BASH_ENV && nvm install 10.5 && nvm use 10.5
+      - run:
+          name: Install yarn
+          command: curl -o- -L https://yarnpkg.com/install.sh | bash
+      - run:
+          name: Use yarn
+          command: echo 'export PATH="$HOME/.yarn/bin:$HOME/.config/yarn/global/node_modules/.bin:$PATH"' >> $BASH_ENV && source $BASH_ENV
       - save_cache:
-          key: v2-pkg-cache
+          key: v3-pkg-cache
           paths:
             - "/go/src/github.com/stellar/kelp/vendor"
 
@@ -33,23 +44,11 @@ commands:
 
   build_kelp:
     steps:
-      - run:
-          name: Install nvm
-          command: curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
-      - run:
-          name: Use Node 10.5
-          command: echo 'export NVM_DIR=$HOME/.nvm' >> $BASH_ENV && echo 'source $NVM_DIR/nvm.sh' >> $BASH_ENV && source $BASH_ENV && nvm install 10.5 && nvm use 10.5
-      - run:
-          name: Install yarn
-          command: curl -o- -L https://yarnpkg.com/install.sh | bash
-      - run:
-          name: Use yarn
-          command: export PATH="$HOME/.yarn/bin:$HOME/.config/yarn/global/node_modules/.bin:$PATH"
       - run:
           name: Build Kelp
           command: ./scripts/build.sh
       - save_cache:
-          key: v4-node-cache
+          key: v5-node-cache
           paths:
             - "/go/src/github.com/stellar/kelp/gui/web/node_modules"
       - run:

From 7b4ba6f6e7e8a314a1e921250df088b60e4565d7 Mon Sep 17 00:00:00 2001
From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com>
Date: Tue, 29 Oct 2019 18:35:38 -0700
Subject: [PATCH 20/22] 20 - use nvm version 10.17

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 37dcf845b..296c31146 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -24,7 +24,7 @@ commands:
           command: curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
       - run:
           name: Use Node 10.5
-          command: echo 'export NVM_DIR=$HOME/.nvm' >> $BASH_ENV && echo 'source $NVM_DIR/nvm.sh' >> $BASH_ENV && source $BASH_ENV && nvm install 10.5 && nvm use 10.5
+          command: echo 'export NVM_DIR=$HOME/.nvm' >> $BASH_ENV && echo 'source $NVM_DIR/nvm.sh' >> $BASH_ENV && source $BASH_ENV && nvm install 10.17 && nvm use 10.17
       - run:
           name: Install yarn
           command: curl -o- -L https://yarnpkg.com/install.sh | bash

From 41775ae2bdaa4f3f4b91ca0043e64a69408cc2e0 Mon Sep 17 00:00:00 2001
From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com>
Date: Tue, 29 Oct 2019 18:41:22 -0700
Subject: [PATCH 21/22] 21 - move filesystem_vfsdata_dev to avoid package
 conflict

---
 scripts/fs_bin_gen/fs_bin_gen.go                       | 2 +-
 scripts/fs_bin_gen/{ => gui}/filesystem_vfsdata_dev.go | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename scripts/fs_bin_gen/{ => gui}/filesystem_vfsdata_dev.go (100%)

diff --git a/scripts/fs_bin_gen/fs_bin_gen.go b/scripts/fs_bin_gen/fs_bin_gen.go
index 97b45a67c..b0e730276 100644
--- a/scripts/fs_bin_gen/fs_bin_gen.go
+++ b/scripts/fs_bin_gen/fs_bin_gen.go
@@ -9,7 +9,7 @@ import (
 	"github.com/shurcooL/vfsgen"
 )
 
-const fsDev_filename = "./scripts/fs_bin_gen/filesystem_vfsdata_dev.go"
+const fsDev_filename = "./scripts/fs_bin_gen/gui/filesystem_vfsdata_dev.go"
 const fs_filename = "./gui/filesystem_vfsdata.go"
 
 func main() {
diff --git a/scripts/fs_bin_gen/filesystem_vfsdata_dev.go b/scripts/fs_bin_gen/gui/filesystem_vfsdata_dev.go
similarity index 100%
rename from scripts/fs_bin_gen/filesystem_vfsdata_dev.go
rename to scripts/fs_bin_gen/gui/filesystem_vfsdata_dev.go

From 8b2a02f0de2e3530f3757f4ca5d4162b121d31fb Mon Sep 17 00:00:00 2001
From: Nikhil Saraf <1028334+nikhilsaraf@users.noreply.github.com>
Date: Wed, 30 Oct 2019 13:34:28 -0700
Subject: [PATCH 22/22] 22 - ReplaceAll is only available from go10.12 onwards

---
 scripts/ccxt_bin_gen/ccxt_bin_gen.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/ccxt_bin_gen/ccxt_bin_gen.go b/scripts/ccxt_bin_gen/ccxt_bin_gen.go
index ae3252145..efb593bf4 100644
--- a/scripts/ccxt_bin_gen/ccxt_bin_gen.go
+++ b/scripts/ccxt_bin_gen/ccxt_bin_gen.go
@@ -127,7 +127,7 @@ func copyDependencyFiles(kos *kelpos.KelpOS, outDir string, pkgCmdOutput string)
 		if !strings.Contains(line, "node_modules") {
 			continue
 		}
-		filename := strings.TrimSpace(strings.ReplaceAll(line, "(MISSING)", ""))
+		filename := strings.TrimSpace(strings.Replace(line, "(MISSING)", "", -1))
 
 		fmt.Printf("    copying file %s to the output directory %s ... ", filename, outDir)
 		cpCmd := fmt.Sprintf("cp %s %s", filename, outDir)