From 15b4c8113966efe12ac0bd76cb05484c6b7136ab Mon Sep 17 00:00:00 2001 From: guanw Date: Sun, 2 Jun 2019 13:10:53 -0400 Subject: [PATCH 1/2] pornhub support --- extractors/pornhub/pornhub.go | 73 ++++++++++++++++++++++++++++++ extractors/pornhub/pornhub_test.go | 34 ++++++++++++++ main.go | 6 +++ 3 files changed, 113 insertions(+) create mode 100644 extractors/pornhub/pornhub.go create mode 100644 extractors/pornhub/pornhub_test.go diff --git a/extractors/pornhub/pornhub.go b/extractors/pornhub/pornhub.go new file mode 100644 index 000000000..79a82c3b2 --- /dev/null +++ b/extractors/pornhub/pornhub.go @@ -0,0 +1,73 @@ +package pornhub + +import ( + "encoding/json" + "github.com/iawia002/annie/downloader" + "github.com/iawia002/annie/request" + "github.com/iawia002/annie/utils" +) + +type pornhubData struct { + DefaultQuality bool `json:"defaultQuality"` + Format string `json:"format"` + Quality string `json:"quality"` + VideoUrl string `json:"videoUrl"` +} + +// Extract is the main function for extracting data +func Extract(url string) ([]downloader.Data, error) { + var err error + html, err := request.Get(url, url, nil) + if err != nil { + return downloader.EmptyList, err + } + var title string + desc := utils.MatchOneOf(html, `(.+?)`) + if desc != nil { + title = desc[1] + } else { + title = "pornhub video" + } + + realURLs := utils.MatchOneOf(html, `"mediaDefinitions":\[(.+?)\],"isVertical"`) + + var pornhubs []pornhubData + err = json.Unmarshal([]byte(`[` + realURLs[1] + `]`), &pornhubs) + if err != nil { + return downloader.EmptyList, err + } + + //TODO add support for different quality + var realURL string + for _, downloadlink := range(pornhubs) { + if downloadlink.VideoUrl != "" { + realURL = downloadlink.VideoUrl + break + } + } + + size, err := request.Size(realURL, url) + if err != nil { + return downloader.EmptyList, err + } + urlData := downloader.URL{ + URL: realURL, + Size: size, + Ext: "mp4", + } + streams := map[string]downloader.Stream{ + "default": { + URLs: []downloader.URL{urlData}, + Size: size, + }, + } + return []downloader.Data{ + { + Site: "Pornhub", + Title: title, + Type: "video", + Streams: streams, + URL: url, + }, + }, nil +} \ No newline at end of file diff --git a/extractors/pornhub/pornhub_test.go b/extractors/pornhub/pornhub_test.go new file mode 100644 index 000000000..5b877cf3d --- /dev/null +++ b/extractors/pornhub/pornhub_test.go @@ -0,0 +1,34 @@ +package pornhub + +import ( + "testing" + + "github.com/iawia002/annie/config" + "github.com/iawia002/annie/test" +) + + +func TestYoutube(t *testing.T) { + config.InfoOnly = true + config.RetryTimes = 10 + tests := []struct { + name string + args test.Args + }{ + { + name: "normal test", + args: test.Args{ + URL: "https://www.pornhub.com/view_video.php?viewkey=ph5cb5fc41c6ebd", + Title: "Must Watch MILF Drilled by the Fireplace", + Size: 158868371, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := Extract(tt.args.URL) + test.CheckError(t, err) + test.Check(t, tt.args, data[0]) + }) + } +} \ No newline at end of file diff --git a/main.go b/main.go index 65c16ee9b..56e985c91 100644 --- a/main.go +++ b/main.go @@ -25,6 +25,7 @@ import ( "github.com/iawia002/annie/extractors/miaopai" "github.com/iawia002/annie/extractors/netease" "github.com/iawia002/annie/extractors/pixivision" + "github.com/iawia002/annie/extractors/pornhub" "github.com/iawia002/annie/extractors/qq" "github.com/iawia002/annie/extractors/tumblr" "github.com/iawia002/annie/extractors/twitter" @@ -150,8 +151,13 @@ func download(videoURL string) { data, err = qq.Extract(videoURL) case "yinyuetai": data, err = yinyuetai.Extract(videoURL) +<<<<<<< HEAD case "geekbang": data, err = geekbang.Extract(videoURL) +======= + case "pornhub": + data, err = pornhub.Extract(videoURL) +>>>>>>> pornhub support default: data, err = universal.Extract(videoURL) } From 570d3ce8f195d2f8c55f34c634da575ee44116c2 Mon Sep 17 00:00:00 2001 From: guanw Date: Mon, 3 Jun 2019 07:38:25 -0400 Subject: [PATCH 2/2] address feedback --- extractors/pornhub/pornhub.go | 17 +++++++++-------- main.go | 3 --- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/extractors/pornhub/pornhub.go b/extractors/pornhub/pornhub.go index 79a82c3b2..e7a9eb3a6 100644 --- a/extractors/pornhub/pornhub.go +++ b/extractors/pornhub/pornhub.go @@ -2,16 +2,17 @@ package pornhub import ( "encoding/json" + "github.com/iawia002/annie/downloader" "github.com/iawia002/annie/request" "github.com/iawia002/annie/utils" ) type pornhubData struct { - DefaultQuality bool `json:"defaultQuality"` - Format string `json:"format"` - Quality string `json:"quality"` - VideoUrl string `json:"videoUrl"` + DefaultQuality bool `json:"defaultQuality"` + Format string `json:"format"` + Quality string `json:"quality"` + VideoURL string `json:"videoUrl"` } // Extract is the main function for extracting data @@ -29,10 +30,10 @@ func Extract(url string) ([]downloader.Data, error) { title = "pornhub video" } - realURLs := utils.MatchOneOf(html, `"mediaDefinitions":\[(.+?)\],"isVertical"`) + realURLs := utils.MatchOneOf(html, `"mediaDefinitions":(.+?),"isVertical"`) var pornhubs []pornhubData - err = json.Unmarshal([]byte(`[` + realURLs[1] + `]`), &pornhubs) + err = json.Unmarshal([]byte(realURLs[1]), &pornhubs) if err != nil { return downloader.EmptyList, err } @@ -40,8 +41,8 @@ func Extract(url string) ([]downloader.Data, error) { //TODO add support for different quality var realURL string for _, downloadlink := range(pornhubs) { - if downloadlink.VideoUrl != "" { - realURL = downloadlink.VideoUrl + if downloadlink.VideoURL != "" { + realURL = downloadlink.VideoURL break } } diff --git a/main.go b/main.go index 56e985c91..97984f59f 100644 --- a/main.go +++ b/main.go @@ -151,13 +151,10 @@ func download(videoURL string) { data, err = qq.Extract(videoURL) case "yinyuetai": data, err = yinyuetai.Extract(videoURL) -<<<<<<< HEAD case "geekbang": data, err = geekbang.Extract(videoURL) -======= case "pornhub": data, err = pornhub.Extract(videoURL) ->>>>>>> pornhub support default: data, err = universal.Extract(videoURL) }