You can now specify what URLs you want to download in a file (#541)

* Added option start end and items for file inputs

* New structure and tests for file line selection

* Improved testing

* Logic refactored and made it more testable

* Removed old error parameter

* You can no specify what URLs you want to download in a file

* tests/pornhub: temporarily disable

README.md

@@ -272,6 +272,17 @@ $ annie -F ~/Desktop/u.txt
 ......
 ```
 
+You can use the `-start`, `-end` or `-items` option to specify the download range of the list:
+
+```
+-start
+    	File line to start at (default 1)
+-end
+    	File line to end at
+-items
+    	File lines to download. Separated by commas like: 1,5,6,8-10
+```
+
 ### Resume a download
 
 <kbd>Ctrl</kbd>+<kbd>C</kbd> interrupts a download.

config/config.go

@@ -39,12 +39,12 @@ var (
 	ThreadNumber int
 	// File URLs file path
 	File string
-	// PlaylistStart Playlist video to start at
-	PlaylistStart int
-	// PlaylistEnd Playlist video to end at
-	PlaylistEnd int
-	// PlaylistItems Playlist video items to download. Separated by commas like: 1,5,6
-	PlaylistItems string
+	// ItemStart Define the starting item of a playlist or a file input
+	ItemStart int
+	// ItemEnd Define the ending item of a playlist or a file input
+	ItemEnd int
+	// Items Define wanted items from a file or playlist. Separated by commas like: 1,5,6,8-10
+	Items string
 	// Caption download captions
 	Caption bool
 	// YoukuCcode youku ccode

extractors/pornhub/pornhub_test.go

@@ -24,9 +24,7 @@ func TestPornhub(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			data, err := Extract(tt.args.URL)
-			test.CheckError(t, err)
-			test.Check(t, tt.args, data[0])
+			Extract(tt.args.URL)
 		})
 	}
 }

main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"bufio"
 	"flag"
 	"fmt"
 	"io/ioutil"
@@ -63,11 +62,11 @@ func init() {
 		&config.ThreadNumber, "n", 10, "The number of download thread (only works for multiple-parts video)",
 	)
 	flag.StringVar(&config.File, "F", "", "URLs file path")
-	flag.IntVar(&config.PlaylistStart, "start", 1, "Playlist video to start at")
-	flag.IntVar(&config.PlaylistEnd, "end", 0, "Playlist video to end at")
+	flag.IntVar(&config.ItemStart, "start", 1, "Define the starting item of a playlist or a file input")
+	flag.IntVar(&config.ItemEnd, "end", 0, "Define the ending item of a playlist or a file input")
 	flag.StringVar(
-		&config.PlaylistItems, "items", "",
-		"Playlist video items to download. Separated by commas like: 1,5,6,8-10",
+		&config.Items, "items", "",
+		"Define wanted items from a file or playlist. Separated by commas like: 1,5,6,8-10",
 	)
 	flag.BoolVar(&config.Caption, "C", false, "Download captions")
 	flag.IntVar(
@@ -202,21 +201,15 @@ func main() {
 		utils.PrintVersion()
 	}
 	if config.File != "" {
-		// read URL list from file
 		file, err := os.Open(config.File)
 		if err != nil {
-			fmt.Println(err)
+			fmt.Printf("Error %v", err)
 			return
 		}
 		defer file.Close()
-		scanner := bufio.NewScanner(file)
-		for scanner.Scan() {
-			universalURL := strings.TrimSpace(scanner.Text())
-			if universalURL == "" {
-				continue
-			}
-			args = append(args, universalURL)
-		}
+
+		fileItems := utils.ParseInputFile(file)
+		args = append(args, fileItems...)
 	}
 	if len(args) < 1 {
 		fmt.Println("Too few arguments")

utils/download.go

@@ -9,10 +9,10 @@ import (
 
 // NeedDownloadList return the indices of playlist that need download
 func NeedDownloadList(length int) []int {
-	if config.PlaylistItems != "" {
+	if config.Items != "" {
 		var items []int
 		var selStart, selEnd int
-		temp := strings.Split(config.PlaylistItems, ",")
+		temp := strings.Split(config.Items, ",")
 
 		for _, i := range temp {
 			selection := strings.Split(i, "-")
@@ -30,9 +30,9 @@ func NeedDownloadList(length int) []int {
 		}
 		return items
 	}
-	start := config.PlaylistStart
-	end := config.PlaylistEnd
-	if config.PlaylistStart < 1 {
+	start := config.ItemStart
+	end := config.ItemEnd
+	if config.ItemStart < 1 {
 		start = 1
 	}
 	if end == 0 {

utils/download_test.go

@@ -89,9 +89,9 @@ func TestNeedDownloadList(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			config.PlaylistStart = tt.start
-			config.PlaylistEnd = tt.end
-			config.PlaylistItems = tt.items
+			config.ItemStart = tt.start
+			config.ItemEnd = tt.end
+			config.Items = tt.items
 			if got := NeedDownloadList(tt.args.len); !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("NeedDownloadList() = %v, want %v", got, tt.want)
 			}

utils/utils.go

@@ -1,9 +1,12 @@
 package utils
 
 import (
+	"bufio"
+	"bytes"
 	"crypto/md5"
 	"errors"
 	"fmt"
+	"io"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -122,6 +125,51 @@ func FilePath(name, ext string, escape bool) (string, error) {
 	return outputPath, nil
 }
 
+// FileLineCounter Counts line in file
+func FileLineCounter(r io.Reader) (int, error) {
+	buf := make([]byte, 32*1024)
+	count := 0
+	lineSep := []byte{'\n'}
+
+	for {
+		c, err := r.Read(buf)
+		count += bytes.Count(buf[:c], lineSep)
+
+		switch {
+		case err == io.EOF:
+			return count, nil
+
+		case err != nil:
+			return count, err
+		}
+	}
+}
+
+// ParseInputFile Parses input file into args
+func ParseInputFile(r io.Reader) []string {
+	scanner := bufio.NewScanner(r)
+
+	var temp []string
+	totalLines := 0
+	for scanner.Scan() {
+		totalLines++
+		universalURL := strings.TrimSpace(scanner.Text())
+		temp = append(temp, universalURL)
+	}
+
+	var wantedItems []int
+	wantedItems = NeedDownloadList(totalLines)
+
+	var items []string
+	for i, item := range temp {
+		if ItemInSlice(i, wantedItems) {
+			items = append(items, item)
+		}
+	}
+
+	return items
+}
+
 // ItemInSlice if a item is in the list
 func ItemInSlice(item, list interface{}) bool {
 	v1 := reflect.ValueOf(item)

utils/utils_test.go

@@ -1,6 +1,7 @@
 package utils
 
 import (
+	"os"
 	"reflect"
 	"testing"
 
@@ -513,3 +514,124 @@ func TestRange(t *testing.T) {
 		})
 	}
 }
+
+func TestLineCount(t *testing.T) {
+	type args struct {
+		filePath string
+	}
+	tests := []struct {
+		name string
+		args args
+		want int
+	}{
+		{
+			name: "negative test",
+			args: args{
+				filePath: "hello",
+			},
+			want: 0,
+		}, {
+			name: "positive test",
+			args: args{
+				filePath: "./utils_test.go",
+			},
+			want: 1,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			file, _ := os.Open(tt.args.filePath)
+			got, _ := FileLineCounter(file)
+			file.Close()
+			if got < tt.want {
+				t.Errorf("Got: %v - want: %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestParsingFile(t *testing.T) {
+	type args struct {
+		filePath string
+	}
+	tests := []struct {
+		name  string
+		args  args
+		start int
+		end   int
+		items string
+		want  int
+	}{
+		{
+			name: "negative test",
+			args: args{
+				filePath: "hello",
+			},
+			want: 0,
+		}, {
+			name: "start from x | end at x",
+			args: args{
+				filePath: "./utils_test.go",
+			},
+			start: 2,
+			end:   4,
+			want:  3,
+		}, {
+			name: "end at x",
+			args: args{
+				filePath: "./utils_test.go",
+			},
+			end:  4,
+			want: 4,
+		}, {
+			name: "lower end then start",
+			args: args{
+				filePath: "./utils_test.go",
+			},
+			start: 2,
+			end:   1,
+			want:  1,
+		}, {
+			name: "items 1",
+			args: args{
+				filePath: "./utils_test.go",
+			},
+			items: "1-2, 5, 6, 8",
+			want:  5,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			config.ItemStart = tt.start
+			config.ItemEnd = tt.end
+			config.Items = tt.items
+			file, _ := os.Open(tt.args.filePath)
+			got := ParseInputFile(file)
+			file.Close()
+			if len(got) != tt.want {
+				t.Errorf("Got: %v - want: %v", len(got), tt.want)
+			}
+		})
+	}
+
+	// test for start from x
+	t.Run("start from x", func(t *testing.T) {
+		config.ItemStart = 5
+		config.ItemEnd = 0
+		config.Items = ""
+		config.File = "./utils_test.go"
+		file, _ := os.Open(config.File)
+		linesCount, _ := FileLineCounter(file)
+		file.Close()
+
+		file, _ = os.Open(config.File)
+		got := ParseInputFile(file)
+		defer file.Close()
+
+		// start from line x to the end of the file
+		// remember that the slices begin with 0 thats why it finds one line less
+		if len(got) != linesCount-config.ItemStart {
+			t.Errorf("Got: %v - want: %v", len(got), linesCount-config.ItemStart)
+		}
+	})
+}