Merge pull request #444 from weiyongsheng/master

extractors/geekbang: Add basic support

README.md

@@ -599,6 +599,7 @@ Twitter | <https://twitter.com> | ✓ | | | |
 腾讯视频 | <https://v.qq.com> | ✓ | | | |
 网易云音乐 | <https://music.163.com> | ✓ | | | |
 音悦台 | <https://yinyuetai.com> | ✓ | | | |
+极客时间 | <https://time.geekbang.org> | ✓ | | | |
 
 
 ## Known issues

extractors/geekbang/geekbang.go

@@ -0,0 +1,118 @@
+package geekbang
+
+import (
+	"encoding/json"
+	"errors"
+	"io/ioutil"
+	"strings"
+
+	"github.com/iawia002/annie/downloader"
+	"github.com/iawia002/annie/request"
+	"github.com/iawia002/annie/utils"
+)
+
+type geekData struct {
+	Code  int `json:"code"`
+	Error struct {
+		Code int    `json:"code"`
+		Msg  string `json:"msg"`
+	} `json:"error"`
+	Data struct {
+		Title         string `json:"article_sharetitle"`
+		VideoMediaMap map[string]struct {
+			URL  string `json:"url"`
+			Size int64  `json:"size"`
+		} `json:"video_media_map"`
+	} `json:"data"`
+}
+
+type geekURLInfo struct {
+	URL  string
+	Size int64
+}
+
+func geekM3u8(url string) ([]geekURLInfo, error) {
+	var (
+		data []geekURLInfo
+		temp geekURLInfo
+		size int64
+		err  error
+	)
+	urls, err := utils.M3u8URLs(url)
+	if err != nil {
+		return nil, err
+	}
+	for _, u := range urls {
+		temp = geekURLInfo{
+			URL:  u,
+			Size: size,
+		}
+		data = append(data, temp)
+	}
+	return data, nil
+}
+
+// Extract is the main function for extracting data
+func Extract(url string) ([]downloader.Data, error) {
+	var err error
+	matches := utils.MatchOneOf(url, `https?://time.geekbang.org/course/detail/(\d+)-(\d+)`)
+	if matches == nil {
+		return downloader.EmptyList, errors.New("地址有误")
+	}
+
+	heanders := map[string]string{"Origin": "https://time.geekbang.org", "Content-Type": "application/json", "Referer": url}
+	params := strings.NewReader("{\"id\":" + string(matches[2]+"}"))
+	res, err := request.Request("POST", "https://time.geekbang.org/serv/v1/article", params, heanders)
+	if err != nil {
+		return downloader.EmptyList, err
+	}
+
+	body, err := ioutil.ReadAll(res.Body)
+	defer res.Body.Close()
+	if err != nil {
+		return downloader.EmptyList, err
+	}
+
+	var geekData geekData
+	err = json.Unmarshal(body, &geekData)
+
+	if geekData.Code < 0 {
+		return downloader.EmptyList, errors.New(geekData.Error.Msg)
+	}
+
+	title := geekData.Data.Title
+
+	streams := make(map[string]downloader.Stream, len(geekData.Data.VideoMediaMap))
+
+	for key, media := range geekData.Data.VideoMediaMap {
+		m3u8URLs, err := geekM3u8(media.URL)
+
+		if err != nil {
+			return downloader.EmptyList, err
+		}
+
+		urls := make([]downloader.URL, len(m3u8URLs))
+		for index, u := range m3u8URLs {
+			urls[index] = downloader.URL{
+				URL:  u.URL,
+				Size: u.Size,
+				Ext:  "ts",
+			}
+		}
+
+		streams[key] = downloader.Stream{
+			URLs: urls,
+			Size: media.Size,
+		}
+	}
+
+	return []downloader.Data{
+		{
+			Site:    "极客时间 geekbang.org",
+			Title:   title,
+			Type:    "video",
+			Streams: streams,
+			URL:     url,
+		},
+	}, nil
+}

extractors/geekbang/geekbang_test.go

@@ -0,0 +1,33 @@
+package geekbang
+
+import (
+	"testing"
+
+	"github.com/iawia002/annie/config"
+	"github.com/iawia002/annie/test"
+)
+
+func TestDownload(t *testing.T) {
+	config.InfoOnly = true
+	config.RetryTimes = 100
+	tests := []struct {
+		name string
+		args test.Args
+	}{
+		{
+			name: "normal test",
+			args: test.Args{
+				URL:   "https://time.geekbang.org/course/detail/190-97203",
+				Title: "02 | 内容综述 - 玩转webpack",
+				Size:  38556544,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			data, err := Extract(tt.args.URL)
+			test.CheckError(t, err)
+			test.Check(t, tt.args, data[0])
+		})
+	}
+}

main.go

@@ -18,6 +18,7 @@ import (
 	"github.com/iawia002/annie/extractors/douyin"
 	"github.com/iawia002/annie/extractors/douyu"
 	"github.com/iawia002/annie/extractors/facebook"
+	"github.com/iawia002/annie/extractors/geekbang"
 	"github.com/iawia002/annie/extractors/instagram"
 	"github.com/iawia002/annie/extractors/iqiyi"
 	"github.com/iawia002/annie/extractors/mgtv"
@@ -149,6 +150,8 @@ func download(videoURL string) {
 		data, err = qq.Extract(videoURL)
 	case "yinyuetai":
 		data, err = yinyuetai.Extract(videoURL)
+	case "geekbang":
+		data, err = geekbang.Extract(videoURL)
 	default:
 		data, err = universal.Extract(videoURL)
 	}