Merge branch 'core-features' of github.com:bcmi-labs/arduino-cli into…

… lib-features

.gitignore

@@ -0,0 +1,4 @@
+debug
+arduino
+arduino-cli
+.vscode/settings.json

.vscode/launch.json

@@ -49,6 +49,16 @@
             "env": {},
             "args": ["lib", "install", "YoutubeApi"],
             "showLog": true
+        },
+        {
+            "name": "Launch LIB DOWNLOAD",
+            "type": "go",
+            "request": "launch",
+            "mode": "debug",
+            "program": "${workspaceRoot}/main.go",
+            "env": {},
+            "args": ["lib", "download", "Zumo32U4", "YoutubeApi", "YouMadeIt", "XLR8SPI", "XLR8Servo", "XLR8Pong", "XLR8NeoPixel"],
+            "showLog": true
         }
     ]
 }

cmd/arduino_lib.go

@@ -43,6 +43,7 @@ import (
 	"github.com/bcmi-labs/arduino-cli/libraries"
 	"github.com/spf13/cobra"
 	"github.com/zieckey/goini"
+	"gopkg.in/cheggaaa/pb.v1"
 )
 
 const (
@@ -145,46 +146,86 @@ func executeDownloadCommand(cmd *cobra.Command, args []string) error {
 		}
 	}
 
-	status, err := libraries.CreateStatusContextFromIndex(index)
+	status, err := index.CreateStatusContext()
 	if err != nil {
 		status, err = prettyPrints.CorruptedLibIndexFix(index, GlobalFlags.Verbose)
 		if err != nil {
 			return nil
 		}
 	}
 
-	libraryOK := make([]string, 0, len(args))
-	libraryFails := make(map[string]string, len(args))
+	items, failed := extractValidLibraries(args, status)
 
-	for i, libraryName := range args {
-		library := status.Libraries[libraryName]
-		if library != nil {
-			//found
-			_, err = libraries.DownloadAndCache(library, i+1, len(args))
-			if err != nil {
-				libraryFails[libraryName] = err.Error()
-			} else {
-				libraryOK = append(libraryOK, libraryName)
-			}
-		} else {
-			libraryFails[libraryName] = "This library is not in library index"
-		}
+	libraryOK, libraryFails := parallelLibDownloads(items, len(args), true)
+	for _, fail := range failed {
+		libraryFails[fail] = "Not Found"
 	}
 
 	if GlobalFlags.Verbose > 0 {
 		prettyPrints.DownloadLib(libraryOK, libraryFails)
 	} else {
 		for _, library := range libraryOK {
-			fmt.Printf("%-10s -Downloaded\n", library)
+			fmt.Printf("%s\t - Downloaded\n", library)
 		}
 		for library, failure := range libraryFails {
-			fmt.Printf("%-10s -Error : %s\n", library, failure)
+			fmt.Printf("%s\t - Error : %s\n", library, failure)
 		}
 	}
 
 	return nil
 }
 
+func extractValidLibraries(args []string, status *libraries.StatusContext) ([]*libraries.Library, []string) {
+	items := make([]*libraries.Library, 0, len(args))
+	fails := make([]string, 0, len(args))
+
+	for _, libraryName := range args {
+		library := status.Libraries[libraryName]
+		if library != nil {
+			items = append(items, library)
+		} else {
+			fails = append(fails, libraryName)
+		}
+	}
+
+	return items, fails
+}
+
+// parallelLibDownloads executes multiple libraries downloads in parallel
+func parallelLibDownloads(items []*libraries.Library, argC int, forced bool) ([]string, map[string]string) {
+	itemC := len(items)
+	libraryOK := make([]string, 0, itemC)
+	libraryFails := make(map[string]string, argC-itemC)
+
+	tasks := make(map[string]common.TaskWrapper, len(items))
+	progressBars := make([]*pb.ProgressBar, 0, len(items))
+
+	for i, library := range items {
+		if library.Installed == nil || forced {
+			progressBars = append(progressBars, pb.StartNew(library.Latest().Size).SetUnits(pb.U_BYTES).Prefix(fmt.Sprintf("%-20s", library.Name)))
+			tasks[library.Name] = libraries.DownloadAndCache(library, progressBars[i])
+		}
+	}
+
+	if len(tasks) > 0 {
+		pool, _ := pb.StartPool(progressBars...)
+
+		results := common.ExecuteParallelFromMap(tasks, GlobalFlags.Verbose)
+
+		pool.Stop()
+
+		for libraryName, result := range results {
+			if result.Error != nil {
+				libraryFails[libraryName] = result.Error.Error()
+			} else {
+				libraryOK = append(libraryOK, libraryName)
+			}
+		}
+	}
+
+	return libraryOK, libraryFails
+}
+
 func executeInstallCommand(cmd *cobra.Command, args []string) error {
 	if len(args) < 1 {
 		return fmt.Errorf("No library specified for install command")
@@ -199,29 +240,27 @@ func executeInstallCommand(cmd *cobra.Command, args []string) error {
 		}
 	}
 
-	status, err := libraries.CreateStatusContextFromIndex(index)
+	status, err := index.CreateStatusContext()
 	if err != nil {
 		status, err = prettyPrints.CorruptedLibIndexFix(index, GlobalFlags.Verbose)
 		if err != nil {
 			return nil
 		}
 	}
 
-	libraryOK := make([]string, 0, len(args))
-	libraryFails := make(map[string]string, len(args))
+	items, fails := extractValidLibraries(args, status)
 
-	for i, libraryName := range args {
-		library := status.Libraries[libraryName]
-		if library != nil {
-			//found
-			err = libraries.DownloadAndInstall(library, i+1, len(args))
-			if err != nil {
-				libraryFails[libraryName] = err.Error()
-			} else {
-				libraryOK = append(libraryOK, libraryName)
-			}
+	libraryOK, libraryFails := parallelLibDownloads(items, len(args), false)
+	for _, fail := range fails {
+		libraryFails[fail] = "Not Found"
+	}
+
+	for _, library := range items {
+		err = libraries.InstallLib(library, library.Latest().Version)
+		if err != nil {
+			libraryFails[library.Name] = err.Error()
 		} else {
-			libraryFails[libraryName] = "This library is not in library index"
+			libraryOK = append(libraryOK, library.Name)
 		}
 	}
 
@@ -358,7 +397,7 @@ func executeSearch(cmd *cobra.Command, args []string) error {
 		return nil
 	}
 
-	status, err := libraries.CreateStatusContextFromIndex(index)
+	status, err := index.CreateStatusContext()
 	if err != nil {
 		status, err = prettyPrints.CorruptedLibIndexFix(index, GlobalFlags.Verbose)
 		if err != nil {

cmd/logger.go

@@ -0,0 +1,13 @@
+package cmd
+
+import (
+	lcf "github.com/Robpol86/logrus-custom-formatter"
+	"github.com/sirupsen/logrus"
+)
+
+var log *logrus.Entry
+
+func init() {
+	log = logrus.WithFields(logrus.Fields{})
+	logrus.SetFormatter(lcf.NewFormatter("%[message]s", nil))
+}

cmd/pretty_print/pretty_print_lib.go

@@ -128,7 +128,7 @@ func libIndexParse(index *libraries.Index, verbosity int) common.TaskWrapper {
 			"Cannot parse index file",
 		},
 		Task: common.Task(func() common.TaskResult {
-			_, err := libraries.CreateStatusContextFromIndex(index)
+			_, err := index.CreateStatusContext()
 			return common.TaskResult{
 				Result: nil,
 				Error:  err,

common/common.go

@@ -39,7 +39,7 @@ import (
 	"path/filepath"
 	"runtime"
 
-	"github.com/mitchellh/ioprogress"
+	pb "gopkg.in/cheggaaa/pb.v1"
 )
 
 // GetDefaultArduinoFolder returns the default data folder for Arduino platform
@@ -153,7 +153,7 @@ func TruncateDir(dir string) error {
 }
 
 // DownloadPackage downloads a package from arduino repository, applying a label for the progress bar.
-func DownloadPackage(URL string, downloadLabel string, progressFile int, totalFiles int) ([]byte, error) {
+func DownloadPackage(URL string, downloadLabel string, progressBar *pb.ProgressBar) ([]byte, error) {
 	client := http.DefaultClient
 
 	request, err := http.NewRequest("GET", URL, nil)
@@ -164,21 +164,14 @@ func DownloadPackage(URL string, downloadLabel string, progressFile int, totalFi
 	response, err := client.Do(request)
 
 	if err != nil {
-		return nil, fmt.Errorf("Cannot fetch library. Response creation error")
+		return nil, fmt.Errorf("Cannot fetch %s. Response creation error", downloadLabel)
 	} else if response.StatusCode != 200 {
 		response.Body.Close()
-		return nil, fmt.Errorf("Cannot fetch library. Source responded with a status %d code", response.StatusCode)
+		return nil, fmt.Errorf("Cannot fetch %s. Source responded with a status %d code", downloadLabel, response.StatusCode)
 	}
 	defer response.Body.Close()
 
-	// Download completed, now move the archive to temp location and unpack it.
-	rd := &ioprogress.Reader{
-		Reader: response.Body,
-		Size:   response.ContentLength,
-		DrawFunc: ioprogress.DrawTerminalf(os.Stdout, func(progress int64, size int64) string {
-			return fmt.Sprintf("%s ... %s -%.2f %% (%d/%d)", downloadLabel, ioprogress.DrawTextFormatBytes(progress, size), float64(progress)/float64(size)*100, progressFile, totalFiles)
-		}),
-	}
+	rd := progressBar.NewProxyReader(response.Body)
 
 	body, err := ioutil.ReadAll(rd)
 	if err != nil {

common/concurrency.go

@@ -82,24 +82,29 @@ type TaskSequence func() []TaskResult
 func (tw TaskWrapper) Execute(verb int) TaskResult {
 	var maxUsableVerb [3]int
 	var msg string
-	maxUsableVerb[0] = minVerb(verb, tw.BeforeMessage)
-	msg = tw.BeforeMessage[maxUsableVerb[0]]
-	if msg != "" {
-		log.Infof("%s ... ", msg)
+	if tw.BeforeMessage != nil && len(tw.BeforeMessage) > 0 {
+		maxUsableVerb[0] = minVerb(verb, tw.BeforeMessage)
+		msg = tw.BeforeMessage[maxUsableVerb[0]]
+		if msg != "" {
+			log.Infof("%s ... ", msg)
+		}
 	}
 
 	ret := tw.Task()
+	log.Errorf("%v\n", ret)
 
 	if ret.Error != nil {
-		maxUsableVerb[1] = minVerb(verb, tw.ErrorMessage)
-		msg = tw.ErrorMessage[maxUsableVerb[1]]
-		if tw.BeforeMessage[maxUsableVerb[0]] != "" {
-			log.Warn("ERROR\n")
-		}
-		if msg != "" {
-			log.Warnf("%s\n", msg)
+		if tw.ErrorMessage != nil && len(tw.ErrorMessage) > 0 {
+			maxUsableVerb[1] = minVerb(verb, tw.ErrorMessage)
+			msg = tw.ErrorMessage[maxUsableVerb[1]]
+			if tw.BeforeMessage[maxUsableVerb[0]] != "" {
+				log.Warn("ERROR\n")
+			}
+			if msg != "" {
+				log.Warnf("%s\n", msg)
+			}
 		}
-	} else {
+	} else if tw.AfterMessage != nil && len(tw.AfterMessage) > 0 {
 		maxUsableVerb[2] = minVerb(verb, tw.AfterMessage)
 		msg = tw.AfterMessage[maxUsableVerb[2]]
 		if tw.BeforeMessage[maxUsableVerb[0]] != "" {
@@ -131,10 +136,7 @@ func CreateTaskSequence(taskWrappers []TaskWrapper, ignoreOnFailure []bool, verb
 		for i, taskWrapper := range taskWrappers {
 			result := taskWrapper.Execute(verbosity)
 			results = append(results, result)
-			if result.Error != nil {
-				if ignoreOnFailure[i] {
-					break
-				}
+			if result.Error != nil && !ignoreOnFailure[i] {
 				log.Warnf("Warning from task %d: %s", i, result.Error)
 			}
 		}
@@ -159,20 +161,52 @@ func ExecuteSequence(taskWrappers []TaskWrapper, ignoreOnFailure []bool, verbosi
 	return CreateTaskSequence(taskWrappers, ignoreOnFailure, verbosity)()
 }
 
+type resultWithKey struct {
+	Result TaskResult
+	Key    string
+}
+
+// ExecuteParallelFromMap executes a set of taskwrappers in parallel, taking input from a map[string]TaskWrapper.
+func ExecuteParallelFromMap(taskMap map[string]TaskWrapper, verbosity int) map[string]TaskResult {
+	results := make(chan resultWithKey, len(taskMap))
+	wg := sync.WaitGroup{}
+	wg.Add(len(taskMap))
+
+	for key, task := range taskMap {
+		go func(key string, task TaskWrapper, wg *sync.WaitGroup) {
+			results <- resultWithKey{
+				Key: key,
+				Result: func() TaskResult {
+					ret := task.Execute(verbosity)
+					wg.Done()
+					return ret
+				}(),
+			}
+		}(key, task, &wg)
+	}
+	wg.Wait()
+	close(results)
+	mapResult := make(map[string]TaskResult, len(results))
+	for result := range results {
+		//log.Errorf("results : %v %v\n", result.Key, result.Result)
+		mapResult[result.Key] = result.Result
+	}
+	return mapResult
+}
+
 // ExecuteParallel executes a set of TaskWrappers in parallel, handling concurrency for results.
 func ExecuteParallel(taskWrappers []TaskWrapper, verbosity int) []TaskResult {
-	results := make(chan TaskResult)
+	results := make(chan TaskResult, len(taskWrappers))
 	wg := sync.WaitGroup{}
-
 	wg.Add(len(taskWrappers))
-
 	for _, task := range taskWrappers {
-		go func(task TaskWrapper) {
+		go func(task TaskWrapper, wg *sync.WaitGroup) {
 			defer wg.Done()
 			results <- task.Execute(verbosity)
-		}(task)
+		}(task, &wg)
 	}
 	wg.Wait()
+	close(results)
 	array := make([]TaskResult, len(results))
 	for i := range array {
 		array[i] = <-results