From 979081a2ea7bf7f1b92ce65c8f8ee86b3b8ff36c Mon Sep 17 00:00:00 2001
From: Charles <cxjava@users.noreply.github.com>
Date: Thu, 5 Sep 2019 23:21:40 +0800
Subject: [PATCH 1/2] support download video and playlist for tang dou.

---
 README.md                          |  1 +
 extractors/tangdou/tangdou.go      | 98 ++++++++++++++++++++++++++++++
 extractors/tangdou/tangdou_test.go | 73 ++++++++++++++++++++++
 main.go                            |  3 +
 4 files changed, 175 insertions(+)
 create mode 100644 extractors/tangdou/tangdou.go
 create mode 100644 extractors/tangdou/tangdou_test.go

diff --git a/README.md b/README.md
index d48963518..4a9386df3 100644
--- a/README.md
+++ b/README.md
@@ -588,6 +588,7 @@ pixivision | <https://www.pixivision.net> | | ✓ | | |
 YouTube | <https://www.youtube.com> | ✓ | | ✓ | |
 爱奇艺 | <https://www.iqiyi.com> | ✓ | | | |
 芒果TV | <https://www.mgtv.com> | ✓ | | | |
+糖豆广场舞 | <http://www.tangdou.com> | ✓ | | ✓ | |
 Tumblr | <https://www.tumblr.com> | ✓ | ✓ | | |
 Vimeo | <https://vimeo.com> | ✓ | | | |
 Facebook | <https://facebook.com> | ✓ | | | |
diff --git a/extractors/tangdou/tangdou.go b/extractors/tangdou/tangdou.go
new file mode 100644
index 000000000..ab145fe51
--- /dev/null
+++ b/extractors/tangdou/tangdou.go
@@ -0,0 +1,98 @@
+package tangdou
+
+import (
+	"github.com/iawia002/annie/config"
+	"github.com/iawia002/annie/downloader"
+	"github.com/iawia002/annie/request"
+	"github.com/iawia002/annie/utils"
+)
+
+const referer = "http://www.tangdou.com/html/playlist/view/4173"
+
+// Extract is the main function for extracting data
+func Extract(uri string) ([]downloader.Data, error) {
+	var err error
+	if !config.Playlist {
+		return []downloader.Data{tangdouDownload(uri)}, nil
+	}
+	html, err := request.Get(uri, referer, nil)
+	if err != nil {
+		return downloader.EmptyList, err
+	}
+	videoIDs := utils.MatchAll(html, `<a target="tdplayer" href="(.+?)" class="title">`)
+	needDownloadItems := utils.NeedDownloadList(len(videoIDs))
+	extractedData := make([]downloader.Data, len(needDownloadItems))
+	wgp := utils.NewWaitGroupPool(config.ThreadNumber)
+	dataIndex := 0
+	for index, videoID := range videoIDs {
+		if !utils.ItemInSlice(index+1, needDownloadItems) {
+			continue
+		}
+		wgp.Add()
+		go func(index int, videURI string, extractedData []downloader.Data) {
+			defer wgp.Done()
+			extractedData[index] = tangdouDownload(videURI)
+		}(dataIndex, videoID[1], extractedData)
+		dataIndex++
+	}
+	wgp.Wait()
+	return extractedData, nil
+}
+
+// tangdouDownload download function for single url
+func tangdouDownload(uri string) downloader.Data {
+	var err error
+	html, err := request.Get(uri, referer, nil)
+	if err != nil {
+		return downloader.EmptyData(uri, err)
+	}
+
+	title := utils.MatchOneOf(
+		html, `<div class="title">(.+?)</div>`, `<meta name="description" content="(.+?)" />`,
+	)[1]
+
+	var realURL string
+	videoURLs := utils.MatchOneOf(
+		html, `video:'(.+?)'`, `video:"(.+?)"`,
+	)
+	if videoURLs == nil {
+		shareURL := utils.MatchOneOf(
+			html, `<div class="video">\s*<script src="(.+?)">`,
+		)[1]
+		signedVideo, err := request.Get(shareURL, uri, nil)
+		if err != nil {
+			return downloader.EmptyData(uri, err)
+		}
+		realURL = utils.MatchOneOf(
+			signedVideo, `src=\\"(.+?)\\">`,
+		)[1]
+	} else {
+		realURL = videoURLs[1]
+	}
+
+	size, err := request.Size(realURL, uri)
+	if err != nil {
+		return downloader.EmptyData(uri, err)
+	}
+
+	streams := map[string]downloader.Stream{
+		"default": {
+			URLs: []downloader.URL{
+				{
+					URL:  realURL,
+					Size: size,
+					Ext:  "mp4",
+				},
+			},
+			Size: size,
+		},
+	}
+
+	return downloader.Data{
+		Site:    "糖豆广场舞 tangdou.com",
+		Title:   title,
+		Type:    "video",
+		Streams: streams,
+		URL:     uri,
+	}
+}
diff --git a/extractors/tangdou/tangdou_test.go b/extractors/tangdou/tangdou_test.go
new file mode 100644
index 000000000..b5d6b38d9
--- /dev/null
+++ b/extractors/tangdou/tangdou_test.go
@@ -0,0 +1,73 @@
+package tangdou
+
+import (
+	"testing"
+
+	"github.com/iawia002/annie/config"
+	"github.com/iawia002/annie/downloader"
+	"github.com/iawia002/annie/test"
+)
+
+func TestTangDou(t *testing.T) {
+	config.InfoOnly = true
+	config.ThreadNumber = 9
+	tests := []struct {
+		name     string
+		args     test.Args
+		playlist bool
+	}{
+		{
+			name: "contains video URL test directly and can get title from body's div tag",
+			args: test.Args{
+				URL:   "http://www.tangdou.com/v95/dAOQNgMjwT2D5w2.html",
+				Title: "杨丽萍广场舞《好日子天天过》喜庆双扇扇子舞",
+				Size:  87611483,
+			},
+		},
+		{
+			name: "need call share url first and get the signed video URL test and can get title from head's title tag",
+			args: test.Args{
+				URL:   "http://m.tangdou.com/v94/dAOMMYNjwT1T2Q2.html",
+				Title: "吉美广场舞《再唱山歌给党听》民族形体舞 附教学视频在线观看",
+				Size:  50710318,
+			},
+		},
+		{
+			name: "playlist test",
+			args: test.Args{
+				URL:   "http://www.tangdou.com/playlist/view/1882",
+				Title: "青儿广场舞《小朋友们都被接走了》原创32步流行舞",
+				Size:  69448816,
+			},
+			playlist: true,
+		},
+		{
+			name: "playlist test2",
+			args: test.Args{
+				URL:   "http://www.tangdou.com/playlist/view/2816/page/4",
+				Title: "茉莉广场舞 我向草原问个好 原创藏族风民族舞附教学",
+				Size:  66284484,
+			},
+			playlist: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var (
+				data []downloader.Data
+				err  error
+			)
+			if tt.playlist {
+				// playlist mode
+				config.Playlist = true
+				_, err = Extract(tt.args.URL)
+				test.CheckError(t, err)
+			} else {
+				config.Playlist = false
+				data, err = Extract(tt.args.URL)
+				test.CheckError(t, err)
+				test.Check(t, tt.args, data[0])
+			}
+		})
+	}
+}
diff --git a/main.go b/main.go
index 7491eba00..247a1756c 100644
--- a/main.go
+++ b/main.go
@@ -27,6 +27,7 @@ import (
 	"github.com/iawia002/annie/extractors/pixivision"
 	"github.com/iawia002/annie/extractors/pornhub"
 	"github.com/iawia002/annie/extractors/qq"
+	"github.com/iawia002/annie/extractors/tangdou"
 	"github.com/iawia002/annie/extractors/tumblr"
 	"github.com/iawia002/annie/extractors/twitter"
 	"github.com/iawia002/annie/extractors/universal"
@@ -129,6 +130,8 @@ func download(videoURL string) bool {
 		data, err = iqiyi.Extract(videoURL)
 	case "mgtv":
 		data, err = mgtv.Extract(videoURL)
+	case "tangdou":
+		data, err = tangdou.Extract(videoURL)
 	case "tumblr":
 		data, err = tumblr.Extract(videoURL)
 	case "vimeo":

From 0d71648cd10761d5baca65a29d2013d0352945b5 Mon Sep 17 00:00:00 2001
From: Charles <cxjava@users.noreply.github.com>
Date: Sat, 7 Sep 2019 15:00:58 +0800
Subject: [PATCH 2/2] fix bug for share url and add one test for share url

---
 extractors/tangdou/tangdou.go      | 8 ++++----
 extractors/tangdou/tangdou_test.go | 8 ++++++++
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/extractors/tangdou/tangdou.go b/extractors/tangdou/tangdou.go
index ab145fe51..dcdfebcb5 100644
--- a/extractors/tangdou/tangdou.go
+++ b/extractors/tangdou/tangdou.go
@@ -48,23 +48,23 @@ func tangdouDownload(uri string) downloader.Data {
 	}
 
 	title := utils.MatchOneOf(
-		html, `<div class="title">(.+?)</div>`, `<meta name="description" content="(.+?)" />`,
+		html, `<div class="title">(.+?)</div>`, `<meta name="description" content="(.+?)"`, `<title>(.+?)</title>`,
 	)[1]
 
 	var realURL string
 	videoURLs := utils.MatchOneOf(
-		html, `video:'(.+?)'`, `video:"(.+?)"`,
+		html, `video:'(.+?)'`, `video:"(.+?)"`, `<video.*src="(.+?)"`,
 	)
 	if videoURLs == nil {
 		shareURL := utils.MatchOneOf(
-			html, `<div class="video">\s*<script src="(.+?)">`,
+			html, `<div class="video">\s*<script src="(.+?)"`,
 		)[1]
 		signedVideo, err := request.Get(shareURL, uri, nil)
 		if err != nil {
 			return downloader.EmptyData(uri, err)
 		}
 		realURL = utils.MatchOneOf(
-			signedVideo, `src=\\"(.+?)\\">`,
+			signedVideo, `src=\\"(.+?)\\"`,
 		)[1]
 	} else {
 		realURL = videoURLs[1]
diff --git a/extractors/tangdou/tangdou_test.go b/extractors/tangdou/tangdou_test.go
index b5d6b38d9..cd4b44576 100644
--- a/extractors/tangdou/tangdou_test.go
+++ b/extractors/tangdou/tangdou_test.go
@@ -32,6 +32,14 @@ func TestTangDou(t *testing.T) {
 				Size:  50710318,
 			},
 		},
+		{
+			name: "share url",
+			args: test.Args{
+				URL:   "https://share.tangdou.com/play.php?vid=1500667821669",
+				Title: "井岗紫薇广场舞18步双人舞《采槟榔》附分解",
+				Size:  26693149,
+			},
+		},
 		{
 			name: "playlist test",
 			args: test.Args{