Improve version listing

cmd/algolia/main.go

@@ -255,6 +255,10 @@ func indexPackage(p Package, index *algoliasearch.Index) error {
 	github, githuberr := getGitHubMeta(repository)
 	if githuberr != nil {
 		fmt.Printf("%s", githuberr)
+		if !strings.Contains(githuberr.Error(), "404 Not Found") {
+			// 404 Not Found errors are mostly. It's caused by a misconfiguration.
+			return fmt.Errorf("Fatal error `%s`", githuberr)
+		}
 	}
 
 	sri, srierr := getSRI(&p)
@@ -306,12 +310,8 @@ func main() {
 
 		for _, p := range j.Packages {
 			fmt.Printf("%s: ", p.Name)
-			err := indexPackage(p, tmpIndex)
-			if err != nil {
-				fmt.Printf("%s\n", err)
-			} else {
-				fmt.Printf("Ok\n")
-			}
+			util.Check(indexPackage(p, tmpIndex))
+			fmt.Printf("Ok\n")
 		}
 		fmt.Printf("Ok\n")
 

cmd/packages/main.go

@@ -27,7 +27,6 @@ func encodeJson(packages []outputPackage) (string, error) {
 
 	buffer := &bytes.Buffer{}
 	encoder := json.NewEncoder(buffer)
-	encoder.SetIndent("", "  ")
 	encoder.SetEscapeHTML(false)
 	err := encoder.Encode(&out)
 	return buffer.String(), err
@@ -64,7 +63,10 @@ func main() {
 			ctx := util.ContextWithName(f)
 
 			p, err := packages.ReadPackageJSON(ctx, f)
-			util.Check(err)
+			if err != nil {
+				util.Printf(ctx, "error while processing package: %s\n", err)
+				continue
+			}
 
 			if p.Version == "" {
 				util.Printf(ctx, "version is invalid\n")

packages/git.go

@@ -0,0 +1,71 @@
+package packages
+
+import (
+	"context"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/xtuc/cdnjs-go/util"
+)
+
+// We first list all the versions (and top-level package.json)
+// in the package and pass the list to git ls-tree which is
+// going to filter out those not in the tree
+func GitListPackageVersions(ctx context.Context, basePath string) []string {
+	filesOnFs, err := filepath.Glob(path.Join(basePath, "*"))
+	util.Check(err)
+
+	filteredFilesOnFs := make([]string, 0)
+
+	// filter out package.json
+	for _, file := range filesOnFs {
+		if !strings.HasSuffix(file, "package.json") && strings.Trim(file, " ") != "" {
+			filteredFilesOnFs = append(filteredFilesOnFs, file)
+		}
+	}
+
+	args := []string{
+		"ls-tree", "--name-only", "origin/master",
+	}
+	args = append(args, filteredFilesOnFs...)
+
+	out, err := exec.Command("git", args...).Output()
+	util.Check(err)
+
+	outFiles := strings.Split(string(out), "\n")
+
+	filteredOutFiles := make([]string, 0)
+	// remove basePath from the output
+	for _, v := range outFiles {
+		if strings.Trim(v, " ") != "" {
+			filteredOutFiles = append(
+				filteredOutFiles, strings.ReplaceAll(v, basePath+"/", ""))
+		}
+	}
+
+	// Debug mode
+	diff := arrDiff(filteredFilesOnFs, outFiles)
+	if len(diff) > 0 {
+		util.Printf(ctx, "found staged version: %+q\n", diff)
+	}
+	// Debug mode
+
+	return filteredOutFiles
+}
+
+func arrDiff(a, b []string) (diff []string) {
+	m := make(map[string]bool)
+
+	for _, item := range b {
+		m[item] = true
+	}
+
+	for _, item := range a {
+		if _, ok := m[item]; !ok {
+			diff = append(diff, item)
+		}
+	}
+	return
+}

packages/packages.go

@@ -6,17 +6,15 @@ import (
 	"fmt"
 	"io/ioutil"
 	"path"
-	"path/filepath"
 	"sort"
-	"strings"
 
 	"github.com/xtuc/cdnjs-go/util"
 
 	"github.com/pkg/errors"
 )
 
 const (
-	PACKAGES_PATH = "./ajax/libs"
+	PACKAGES_PATH = "ajax/libs"
 )
 
 type Repository struct {
@@ -83,7 +81,7 @@ func ReadPackageJSON(ctx context.Context, file string) (*Package, error) {
 
 	data, err := ioutil.ReadFile(file)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "failed to read %s", file)
 	}
 
 	jsonerr := json.Unmarshal(data, &jsondata)
@@ -233,16 +231,7 @@ func (p *Package) path() string {
 }
 
 func (p *Package) Versions() (versions []string) {
-	files, err := filepath.Glob(path.Join(p.path(), "*"))
-	util.Check(err)
-	// filter out package.json
-	for _, file := range files {
-		if !strings.HasSuffix(file, "package.json") {
-			parts := strings.Split(file, "/")
-			versions = append(versions, parts[3])
-		}
-	}
-	return versions
+	return GitListPackageVersions(p.ctx, p.path())
 }
 
 func (p *Package) files(version string) []string {