implement udn news

extractors/udn/udn.go

@@ -0,0 +1,84 @@
+package udn
+
+import (
+	"errors"
+	"strings"
+
+	"github.com/iawia002/annie/downloader"
+	"github.com/iawia002/annie/request"
+	"github.com/iawia002/annie/utils"
+)
+
+const (
+	startFlag = `',
+            mp4: '//`
+	endFlag = `'
+        },
+        subtitles`
+)
+
+func getCDNUrl(html string) string {
+	if cdnURLs := utils.MatchOneOf(html, startFlag+"(.+?)"+endFlag); len(cdnURLs) > 1 && cdnURLs[1] != "" {
+		return cdnURLs[1]
+	}
+	return ""
+}
+
+func prepareEmbedURL(url string) string {
+	if !strings.Contains(url, "https://video.udn.com/embed/") {
+		newIDs := strings.Split(url, "/")
+		return "https://video.udn.com/embed/news/" + newIDs[len(newIDs)-1]
+	}
+	return url
+}
+
+// Extract is the main function for extracting data
+func Extract(url string) ([]downloader.Data, error) {
+	url = prepareEmbedURL(url)
+	html, err := request.Get(url, url, nil)
+	if err != nil {
+		return downloader.EmptyList, err
+	}
+	var title string
+	desc := utils.MatchOneOf(html, `title: '(.+?)',
+        link:`)
+	if desc != nil {
+		title = desc[1]
+	} else {
+		title = "udn"
+	}
+	cdnURL := getCDNUrl(html)
+	if cdnURL == "" {
+		return downloader.EmptyList, errors.New("empty list")
+	}
+	srcURL, err := request.Get("http://"+cdnURL, url, nil)
+	if err != nil {
+		return downloader.EmptyList, err
+	}
+	size, err := request.Size(srcURL, url)
+	if err != nil {
+		return downloader.EmptyList, err
+	}
+	urlData := downloader.URL{
+		URL:  srcURL,
+		Size: size,
+		Ext:  "mp4",
+	}
+	quality := "normal"
+	streams := map[string]downloader.Stream{
+		quality: {
+			URLs:    []downloader.URL{urlData},
+			Size:    size,
+			Quality: quality,
+		},
+	}
+	return []downloader.Data{
+		{
+			Site:    "udn udn.com",
+			Title:   title,
+			Type:    "video",
+			Streams: streams,
+			URL:     url,
+		},
+	}, nil
+}

extractors/udn/udn_test.go

@@ -0,0 +1,33 @@
+package udn
+
+import (
+	"testing"
+
+	"github.com/iawia002/annie/config"
+	"github.com/iawia002/annie/test"
+)
+
+func TestExtract(t *testing.T) {
+	config.InfoOnly = true
+	config.RetryTimes = 10
+	tests := []struct {
+		name string
+		args test.Args
+	}{
+		{
+			name: "normal test",
+			args: test.Args{
+				URL:   "https://video.udn.com/embed/news/300040",
+				Title: `生物老師男變女 全校挺"做自己"`,
+				Size:  12740874,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			data, err := Extract(tt.args.URL)
+			test.CheckError(t, err)
+			test.Check(t, tt.args, data[0])
+		})
+	}
+}

main.go

@@ -30,6 +30,7 @@ import (
 	"github.com/iawia002/annie/extractors/tangdou"
 	"github.com/iawia002/annie/extractors/tumblr"
 	"github.com/iawia002/annie/extractors/twitter"
+	"github.com/iawia002/annie/extractors/udn"
 	"github.com/iawia002/annie/extractors/universal"
 	"github.com/iawia002/annie/extractors/vimeo"
 	"github.com/iawia002/annie/extractors/weibo"
@@ -161,6 +162,8 @@ func download(videoURL string) bool {
 		data, err = pornhub.Extract(videoURL)
 	case "xvideos":
 		data, err = xvideos.Extract(videoURL)
+	case "udn":
+		data, err = udn.Extract(videoURL)
 	default:
 		data, err = universal.Extract(videoURL)
 	}