diff --git a/.circleci/config.yml b/.circleci/config.yml index cec473bba..296c31146 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,8 +5,8 @@ commands: - checkout - restore_cache: keys: - - v2-pkg-cache - - v1-node-cache + - v3-pkg-cache + - v5-node-cache - run: name: Install glide command: curl https://glide.sh/get | sh @@ -19,9 +19,20 @@ commands: - run: name: Install astilectron-bundler command: go install github.com/asticode/go-astilectron-bundler - + - run: + name: Install nvm + command: curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash + - run: + name: Use Node 10.5 + command: echo 'export NVM_DIR=$HOME/.nvm' >> $BASH_ENV && echo 'source $NVM_DIR/nvm.sh' >> $BASH_ENV && source $BASH_ENV && nvm install 10.17 && nvm use 10.17 + - run: + name: Install yarn + command: curl -o- -L https://yarnpkg.com/install.sh | bash + - run: + name: Use yarn + command: echo 'export PATH="$HOME/.yarn/bin:$HOME/.config/yarn/global/node_modules/.bin:$PATH"' >> $BASH_ENV && source $BASH_ENV - save_cache: - key: v2-pkg-cache + key: v3-pkg-cache paths: - "/go/src/github.com/stellar/kelp/vendor" @@ -33,17 +44,11 @@ commands: build_kelp: steps: - - run: - name: Install nvm - command: curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash - - run: - name: Use Node 10.5 - command: echo 'export NVM_DIR=$HOME/.nvm' >> $BASH_ENV && echo 'source $NVM_DIR/nvm.sh' >> $BASH_ENV && source $BASH_ENV && nvm install 10.5 && nvm use 10.5 - run: name: Build Kelp command: ./scripts/build.sh - save_cache: - key: v1-node-cache + key: v5-node-cache paths: - "/go/src/github.com/stellar/kelp/gui/web/node_modules" - run: diff --git a/scripts/build.sh b/scripts/build.sh index ba4fc502e..3fa0b9824 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -1,13 +1,13 @@ #!/bin/bash function usage() { - echo "Usage: $0 [flags]" + echo "Usage: $0 [flags] [flag-fields]" echo "" echo "Flags:" - echo " -d, --deploy prepare tar archives in build/, only works on a tagged commit in the format v1.0.0 or v1.0.0-rc1" - echo " -t, --test-deploy test prepare tar archives in build/ for your native platform only" - echo " -h, --help show this help info" - exit 1 + echo " -d, --deploy prepare tar archives in build/, only works on a tagged commit in the format v1.0.0 or v1.0.0-rc1" + echo " -t, --test-deploy test prepare tar archives in build/ for your native platform only" + echo " -g, --gen-ccxt generate binary for ccxt-rest executable for to be uploaded to GitHub for use in building kelp binary, takes in arguments (linux, darwin, windows)" + echo " -h, --help show this help info" } function install_web_dependencies() { @@ -16,6 +16,7 @@ function install_web_dependencies() { cd $CURRENT_DIR/gui/web yarn install + check_build_result $? cd $CURRENT_DIR echo "... finished installing web dependencies" @@ -28,6 +29,7 @@ function generate_static_web_files() { cd $CURRENT_DIR/gui/web yarn build + check_build_result $? cd $CURRENT_DIR echo "... finished generating contents of gui/web/build" @@ -44,7 +46,16 @@ function check_build_result() { fi } -if [[ ($# -gt 1 || $(basename $("pwd")) != "kelp") ]] +# takes in the GOOS for which to build +function gen_ccxt_binary() { + echo "generating ccxt binary for GOOS=$1" + echo "" + go run ./scripts/ccxt_bin_gen/ccxt_bin_gen.go -goos $1 + check_build_result $? + echo "successful" +} + +if [[ $(basename $("pwd")) != "kelp" ]] then echo "need to invoke from the root 'kelp' directory" exit 1 @@ -60,10 +71,36 @@ elif [[ ($# -eq 1 && ("$1" == "-t" || "$1" == "--test-deploy")) ]]; then IS_TEST_MODE=1 elif [[ ($# -eq 1 && ("$1" == "-h" || "$1" == "--help")) ]]; then usage -elif [[ $# -eq 1 ]]; then + exit 0 +elif [[ (($# -eq 1 || $# -eq 2) && ("$1" == "-g" || "$1" == "--gen-ccxt")) ]]; then + if [[ $# -eq 1 ]]; then + echo "the $1 flag needs to be followed by the GOOS for which to build the ccxt binary" + echo "" + usage + exit 1 + fi + + if [[ $# -eq 2 ]]; then + if [[ "$2" == "linux" || "$2" == "darwin" || "$2" == "windows" ]]; then + gen_ccxt_binary $2 + echo "" + echo "BUILD SUCCESSFUL" + exit 0 + else + echo "invalid GOOS type passed in: $2" + echo "" + usage + exit 1 + fi + fi + usage -else + exit 1 +elif [[ $# -eq 0 ]]; then ENV=dev +else + usage + exit 1 fi # version is git tag if it's available, otherwise git hash @@ -120,7 +157,7 @@ fi echo "" echo "embedding contents of gui/web/build into a .go file (env=$ENV) ..." -go run ./scripts/fs_bin_gen.go -env $ENV +go run ./scripts/fs_bin_gen/fs_bin_gen.go -env $ENV check_build_result $? echo "... finished embedding contents of gui/web/build into a .go file (env=$ENV)" echo "" diff --git a/scripts/ccxt_bin_gen/ccxt_bin_gen.go b/scripts/ccxt_bin_gen/ccxt_bin_gen.go new file mode 100644 index 000000000..efb593bf4 --- /dev/null +++ b/scripts/ccxt_bin_gen/ccxt_bin_gen.go @@ -0,0 +1,187 @@ +package main + +import ( + "flag" + "fmt" + "log" + "path/filepath" + "regexp" + "strings" + + "github.com/pkg/errors" + "github.com/stellar/kelp/support/kelpos" + "github.com/stellar/kelp/support/networking" +) + +const kelpPrefsDirectory = "build" +const ccxtDownloadURL = "https://github.com/ccxt-rest/ccxt-rest/archive/v0.0.4.tar.gz" +const ccxtDownloadFilename = "ccxt-rest_v0.0.4.tar.gz" +const ccxtUntaredDirName = "ccxt-rest-0.0.4" +const ccxtBinOutputDir = "bin" +const nodeVersionMatchRegex = "v8.[0-9]+.[0-9]+" + +func main() { + goosP := flag.String("goos", "", "GOOS for which to build") + flag.Parse() + goos := *goosP + + pkgos := "" + if goos == "darwin" { + pkgos = "macos" + } else if goos == "linux" { + pkgos = "linux" + } else if goos == "windows" { + pkgos = "win" + } else { + panic("unsupported goos flag: " + goos) + } + + kos := kelpos.GetKelpOS() + kos.SetSilentRegistrations() + + zipFoldername := fmt.Sprintf("ccxt-rest_%s-x64", goos) + generateCcxtBinary(kos, pkgos, zipFoldername) +} + +func checkNodeVersion(kos *kelpos.KelpOS) { + fmt.Printf("checking node version ... ") + + version, e := kos.Blocking("node", "node -v") + if e != nil { + log.Fatal(errors.Wrap(e, "ensure that the `pkg` tool is installed correctly. You can get it from here https://github.com/zeit/pkg or by running `npm install -g pkg`")) + } + + match, e := regexp.Match(nodeVersionMatchRegex, version) + if e != nil { + log.Fatal(errors.Wrap(e, "could not match regex against node version")) + } + if !match { + log.Fatal("node version will fail to compile a successful binary because of the requirements of ccxt-rest's dependencies, should use v8.x.x instead of " + string(version)) + } + + fmt.Printf("valid\n") +} + +func checkPkgTool(kos *kelpos.KelpOS) { + fmt.Printf("checking for presence of `pkg` tool ... ") + _, e := kos.Blocking("pkg", "pkg -v") + if e != nil { + log.Fatal(errors.Wrap(e, "ensure that the `pkg` tool is installed correctly. You can get it from here https://github.com/zeit/pkg or by running `npm install -g pkg`")) + } + fmt.Printf("done\n") +} + +func downloadCcxtSource(kos *kelpos.KelpOS, downloadDir string) { + fmt.Printf("making directory where we can download ccxt file %s ... ", downloadDir) + e := kos.Mkdir(downloadDir) + if e != nil { + log.Fatal(errors.Wrap(e, "could not make directory for downloadDir "+downloadDir)) + } + fmt.Printf("done\n") + + fmt.Printf("downloading file from URL %s ... ", ccxtDownloadURL) + downloadFilePath := filepath.Join(downloadDir, ccxtDownloadFilename) + e = networking.DownloadFile(ccxtDownloadURL, downloadFilePath) + if e != nil { + log.Fatal(errors.Wrap(e, "could not download ccxt tar.gz file")) + } + fmt.Printf("done\n") + + fmt.Printf("untaring file %s ... ", downloadFilePath) + _, e = kos.Blocking("tar", fmt.Sprintf("tar xvf %s -C %s", downloadFilePath, downloadDir)) + if e != nil { + log.Fatal(errors.Wrap(e, "could not untar ccxt file")) + } + fmt.Printf("done\n") +} + +func npmInstall(kos *kelpos.KelpOS, installDir string) { + fmt.Printf("running npm install on directory %s ... ", installDir) + npmCmd := fmt.Sprintf("cd %s && npm install && cd -", installDir) + _, e := kos.Blocking("npm", npmCmd) + if e != nil { + log.Fatal(errors.Wrap(e, "failed to run npm install")) + } + fmt.Printf("done\n") +} + +// pkg --targets node8-linux-x64 build/ccxt/ccxt-rest-0.0.4 +func runPkgTool(kos *kelpos.KelpOS, sourceDir string, outDir string, pkgos string) { + target := fmt.Sprintf("node8-%s-x64", pkgos) + + fmt.Printf("running pkg tool on source directory %s with output directory as %s on target platform %s ... ", sourceDir, outDir, target) + pkgCommand := fmt.Sprintf("pkg --out-path %s --targets %s %s", outDir, target, sourceDir) + outputBytes, e := kos.Blocking("pkg", pkgCommand) + if e != nil { + log.Fatal(errors.Wrap(e, "failed to run pkg tool")) + } + fmt.Printf("done\n") + + copyDependencyFiles(kos, outDir, string(outputBytes)) +} + +func copyDependencyFiles(kos *kelpos.KelpOS, outDir string, pkgCmdOutput string) { + fmt.Println() + fmt.Printf("copying dependency files to the output directory %s ...\n", outDir) + for _, line := range strings.Split(pkgCmdOutput, "\n") { + if !strings.Contains(line, "node_modules") { + continue + } + filename := strings.TrimSpace(strings.Replace(line, "(MISSING)", "", -1)) + + fmt.Printf(" copying file %s to the output directory %s ... ", filename, outDir) + cpCmd := fmt.Sprintf("cp %s %s", filename, outDir) + _, e := kos.Blocking("cp", cpCmd) + if e != nil { + log.Fatal(errors.Wrap(e, "failed to copy dependency file "+filename)) + } + fmt.Printf("done\n") + } + fmt.Printf("done\n") + fmt.Println() +} + +func mkDir(kos *kelpos.KelpOS, zipDir string) { + fmt.Printf("making directory %s ... ", zipDir) + e := kos.Mkdir(zipDir) + if e != nil { + log.Fatal(errors.Wrap(e, "unable to make directory "+zipDir)) + } + fmt.Printf("done\n") +} + +func zipOutput(kos *kelpos.KelpOS, ccxtDir string, sourceDir string, zipFoldername string, zipOutDir string) { + zipFilename := zipFoldername + ".zip" + fmt.Printf("zipping directory %s as file %s ... ", filepath.Join(ccxtDir, ccxtBinOutputDir), zipFilename) + zipCmd := fmt.Sprintf("cd %s && mv %s %s && zip -rq %s %s && cd - && mv %s %s", ccxtDir, ccxtBinOutputDir, zipFoldername, zipFilename, zipFoldername, filepath.Join(ccxtDir, zipFilename), zipOutDir) + _, e := kos.Blocking("zip", zipCmd) + if e != nil { + log.Fatal(errors.Wrap(e, "unable to zip folder with ccxt binary and dependencies")) + } + fmt.Printf("done\n") + + zipDirPath := filepath.Join(ccxtDir, zipFoldername) + fmt.Printf("clean up zipped directory %s ... ", zipDirPath) + cleanupCmd := fmt.Sprintf("rm %s/* && rmdir %s", zipDirPath, zipDirPath) + _, e = kos.Blocking("zip", cleanupCmd) + if e != nil { + log.Fatal(errors.Wrap(e, fmt.Sprintf("unable to cleanup zip folder %s with ccxt binary and dependencies", zipDirPath))) + } + fmt.Printf("done\n") +} + +func generateCcxtBinary(kos *kelpos.KelpOS, pkgos string, zipFoldername string) { + checkNodeVersion(kos) + checkPkgTool(kos) + + ccxtDir := filepath.Join(kelpPrefsDirectory, "ccxt") + sourceDir := filepath.Join(ccxtDir, ccxtUntaredDirName) + outDir := filepath.Join(ccxtDir, ccxtBinOutputDir) + zipOutDir := filepath.Join(ccxtDir, "zipped") + + downloadCcxtSource(kos, ccxtDir) + npmInstall(kos, sourceDir) + runPkgTool(kos, sourceDir, outDir, pkgos) + mkDir(kos, zipOutDir) + zipOutput(kos, ccxtDir, outDir, zipFoldername, zipOutDir) +} diff --git a/scripts/clean.sh b/scripts/clean.sh index e06760a6c..0e14793a3 100755 --- a/scripts/clean.sh +++ b/scripts/clean.sh @@ -18,7 +18,7 @@ fi echo "removing files ..." rm -vrf bin -rm -vrf build +delete_large_dir build delete_large_dir gui/web/build delete_large_dir gui/web/node_modules rm -vf gui/filesystem_vfsdata.go diff --git a/scripts/fs_bin_gen.go b/scripts/fs_bin_gen/fs_bin_gen.go similarity index 80% rename from scripts/fs_bin_gen.go rename to scripts/fs_bin_gen/fs_bin_gen.go index e80cdd0ba..b0e730276 100644 --- a/scripts/fs_bin_gen.go +++ b/scripts/fs_bin_gen/fs_bin_gen.go @@ -9,7 +9,7 @@ import ( "github.com/shurcooL/vfsgen" ) -const fsDev_filename = "./scripts/fs_gen/filesystem_vfsdata_dev.go" +const fsDev_filename = "./scripts/fs_bin_gen/gui/filesystem_vfsdata_dev.go" const fs_filename = "./gui/filesystem_vfsdata.go" func main() { @@ -18,15 +18,15 @@ func main() { env := *envP if env == "dev" { - generateDev() + generateWeb_Dev() } else if env == "release" { - generateRelease() + generateWeb_Release() } else { panic("unrecognized env flag: " + env) } } -func generateRelease() { +func generateWeb_Release() { fs := http.Dir("./gui/web/build") e := vfsgen.Generate(fs, vfsgen.Options{ Filename: fs_filename, @@ -39,7 +39,7 @@ func generateRelease() { } } -func generateDev() { +func generateWeb_Dev() { c := exec.Command("cp", fsDev_filename, fs_filename) e := c.Run() if e != nil { diff --git a/scripts/fs_gen/filesystem_vfsdata_dev.go b/scripts/fs_bin_gen/gui/filesystem_vfsdata_dev.go similarity index 100% rename from scripts/fs_gen/filesystem_vfsdata_dev.go rename to scripts/fs_bin_gen/gui/filesystem_vfsdata_dev.go diff --git a/support/kelpos/kelpos.go b/support/kelpos/kelpos.go index 4fd887150..46b3b1d60 100644 --- a/support/kelpos/kelpos.go +++ b/support/kelpos/kelpos.go @@ -11,10 +11,16 @@ import ( // KelpOS is a struct that manages all subprocesses started by this Kelp process type KelpOS struct { - processes map[string]Process - processLock *sync.Mutex - bots map[string]*BotInstance - botLock *sync.Mutex + processes map[string]Process + processLock *sync.Mutex + bots map[string]*BotInstance + botLock *sync.Mutex + silentRegistrations bool +} + +// SetSilentRegistrations does not log every time we register and unregister commands +func (kos *KelpOS) SetSilentRegistrations() { + kos.silentRegistrations = true } // Process contains all the pieces that can be used to control a given process diff --git a/support/kelpos/process.go b/support/kelpos/process.go index 516860df6..f8a4aa97a 100644 --- a/support/kelpos/process.go +++ b/support/kelpos/process.go @@ -130,7 +130,9 @@ func (kos *KelpOS) register(namespace string, p *Process) error { } kos.processes[namespace] = *p - log.Printf("registered command under namespace '%s' with PID: %d, processes available: %v\n", namespace, p.Cmd.Process.Pid, kos.RegisteredProcesses()) + if !kos.silentRegistrations { + log.Printf("registered command under namespace '%s' with PID: %d, processes available: %v\n", namespace, p.Cmd.Process.Pid, kos.RegisteredProcesses()) + } return nil } @@ -141,7 +143,9 @@ func (kos *KelpOS) Unregister(namespace string) error { if p, exists := kos.processes[namespace]; exists { delete(kos.processes, namespace) - log.Printf("unregistered command under namespace '%s' with PID: %d, processes available: %v\n", namespace, p.Cmd.Process.Pid, kos.RegisteredProcesses()) + if !kos.silentRegistrations { + log.Printf("unregistered command under namespace '%s' with PID: %d, processes available: %v\n", namespace, p.Cmd.Process.Pid, kos.RegisteredProcesses()) + } return nil } return fmt.Errorf("process with namespace does not exist: %s", namespace) diff --git a/support/networking/network.go b/support/networking/network.go index 9a613dfa7..27ab0973a 100644 --- a/support/networking/network.go +++ b/support/networking/network.go @@ -3,9 +3,11 @@ package networking import ( "encoding/json" "fmt" + "io" "io/ioutil" "mime" "net/http" + "os" "strings" ) @@ -78,3 +80,25 @@ func JSONRequest( return nil } + +// DownloadFile downloads a URL to a file on the local disk as it downloads it. +func DownloadFile(url string, filepath string) error { + outfile, e := os.Create(filepath) + if e != nil { + return fmt.Errorf("could not create file at filepath (%s): %s", filepath, e) + } + defer outfile.Close() + + resp, e := http.Get(url) + if e != nil { + return fmt.Errorf("could not get file at URL (%s): %s", url, e) + } + defer resp.Body.Close() + + // do the download and write to file on disk as we download + _, e = io.Copy(outfile, resp.Body) + if e != nil { + return fmt.Errorf("could not download from URL (%s) to file (%s) in a streaming manner: %s", url, filepath, e) + } + return nil +}