diff --git a/exp/lighthorizon/index/backend/file.go b/exp/lighthorizon/index/backend/file.go index dc7af997f2..2e25e1c1bc 100644 --- a/exp/lighthorizon/index/backend/file.go +++ b/exp/lighthorizon/index/backend/file.go @@ -20,6 +20,10 @@ type FileBackend struct { } func NewFileBackend(dir string, parallel uint32) (*FileBackend, error) { + if parallel <= 0 { + parallel = 1 + } + return &FileBackend{ dir: dir, parallel: parallel, diff --git a/exp/lighthorizon/index/cmd/reduce.sh b/exp/lighthorizon/index/cmd/reduce.sh index 4d681b63f9..e89b91c474 100755 --- a/exp/lighthorizon/index/cmd/reduce.sh +++ b/exp/lighthorizon/index/cmd/reduce.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Combines indices that were built separately in different folders into a single # set of indices. @@ -22,7 +22,7 @@ fi if [[ ! -d "$2" ]]; then echo "Warning: index dest ('$2') does not exist, creating..." - mkdir -p $2 + mkdir -p "$2" fi MAP_JOB_COUNT=$(ls $1 | grep -E 'job_[0-9]+' | wc -l) @@ -50,8 +50,8 @@ do AWS_BATCH_JOB_ARRAY_INDEX=$i MAP_JOB_COUNT=$MAP_JOB_COUNT \ REDUCE_JOB_COUNT=$REDUCE_JOB_COUNT WORKER_COUNT=4 \ INDEX_SOURCE_ROOT=file://$1 INDEX_TARGET=file://$2 \ - ./reduce & - + timeout -k 30s 10s ./reduce & + echo "pid=$!" pids+=($!) done @@ -72,4 +72,4 @@ done rm ./reduce # cleanup echo "All jobs succeeded!" -exit 0 \ No newline at end of file +exit 0 diff --git a/exp/lighthorizon/index/connect.go b/exp/lighthorizon/index/connect.go index f6ac6c64f0..4f8abc368c 100644 --- a/exp/lighthorizon/index/connect.go +++ b/exp/lighthorizon/index/connect.go @@ -4,6 +4,7 @@ import ( "fmt" "net/url" "path/filepath" + "runtime" "github.com/aws/aws-sdk-go/aws" @@ -15,6 +16,10 @@ func Connect(backendUrl string) (Store, error) { } func ConnectWithConfig(config StoreConfig) (Store, error) { + if config.Workers <= 0 { + config.Workers = uint32(runtime.NumCPU()) - 1 + } + parsed, err := url.Parse(config.Url) if err != nil { return nil, err