Skip to content

Commit

Permalink
feat: add pikpak offline download function (#6648)
Browse files Browse the repository at this point in the history
* add pikpak offline download function

* 完善PikPak离线下载功能

* 删除多余的代码

* add task cache to avoid too many requests about API

* 优化Status函数

* 完善所有功能,目前测试无BUG

* 减少缓存时间,优化添加离线任务的参数
  • Loading branch information
Muione authored Jul 7, 2024
1 parent ca30849 commit 3a3d0ad
Show file tree
Hide file tree
Showing 7 changed files with 339 additions and 3 deletions.
90 changes: 90 additions & 0 deletions drivers/pikpak/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@ package pikpak

import (
"context"
"encoding/json"
"fmt"
"net/http"
"strconv"
"strings"

"github.com/alist-org/alist/v3/drivers/base"
Expand Down Expand Up @@ -207,4 +209,92 @@ func (d *PikPak) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
return err
}

// 离线下载文件
func (d *PikPak) OfflineDownload(ctx context.Context, fileUrl string, parentDir model.Obj, fileName string) (*OfflineTask, error) {
requestBody := base.Json{
"kind": "drive#file",
"name": fileName,
"upload_type": "UPLOAD_TYPE_URL",
"url": base.Json{
"url": fileUrl,
},
"parent_id": parentDir.GetID(),
"folder_type": "",
}

var resp OfflineDownloadResp
_, err := d.request("https://api-drive.mypikpak.com/drive/v1/files", http.MethodPost, func(req *resty.Request) {
req.SetBody(requestBody)
}, &resp)

if err != nil {
return nil, err
}

return &resp.Task, err
}

/*
获取离线下载任务列表
phase 可能的取值:
PHASE_TYPE_RUNNING, PHASE_TYPE_ERROR, PHASE_TYPE_COMPLETE, PHASE_TYPE_PENDING
*/
func (d *PikPak) OfflineList(ctx context.Context, nextPageToken string, phase []string) ([]OfflineTask, error) {
res := make([]OfflineTask, 0)
url := "https://api-drive.mypikpak.com/drive/v1/tasks"

if len(phase) == 0 {
phase = []string{"PHASE_TYPE_RUNNING", "PHASE_TYPE_ERROR", "PHASE_TYPE_COMPLETE", "PHASE_TYPE_PENDING"}
}
params := map[string]string{
"type": "offline",
"thumbnail_size": "SIZE_SMALL",
"limit": "10000",
"page_token": nextPageToken,
"with": "reference_resource",
}

// 处理 phase 参数
if len(phase) > 0 {
filters := base.Json{
"phase": map[string]string{
"in": strings.Join(phase, ","),
},
}
filtersJSON, err := json.Marshal(filters)
if err != nil {
return nil, fmt.Errorf("failed to marshal filters: %w", err)
}
params["filters"] = string(filtersJSON)
}

var resp OfflineListResp
_, err := d.request(url, http.MethodGet, func(req *resty.Request) {
req.SetContext(ctx).
SetQueryParams(params)
}, &resp)

if err != nil {
return nil, fmt.Errorf("failed to get offline list: %w", err)
}
res = append(res, resp.Tasks...)
return res, nil
}

func (d *PikPak) DeleteOfflineTasks(ctx context.Context, taskIDs []string, deleteFiles bool) error {
url := "https://api-drive.mypikpak.com/drive/v1/tasks"
params := map[string]string{
"task_ids": strings.Join(taskIDs, ","),
"delete_files": strconv.FormatBool(deleteFiles),
}
_, err := d.request(url, http.MethodDelete, func(req *resty.Request) {
req.SetContext(ctx).
SetQueryParams(params)
}, nil)
if err != nil {
return fmt.Errorf("failed to delete tasks %v: %w", taskIDs, err)
}
return nil
}

var _ driver.Driver = (*PikPak)(nil)
69 changes: 69 additions & 0 deletions drivers/pikpak/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,3 +99,72 @@ type UploadTaskData struct {

File File `json:"file"`
}

// 添加离线下载响应
type OfflineDownloadResp struct {
File *string `json:"file"`
Task OfflineTask `json:"task"`
UploadType string `json:"upload_type"`
URL struct {
Kind string `json:"kind"`
} `json:"url"`
}

// 离线下载列表
type OfflineListResp struct {
ExpiresIn int64 `json:"expires_in"`
NextPageToken string `json:"next_page_token"`
Tasks []OfflineTask `json:"tasks"`
}

// offlineTask
type OfflineTask struct {
Callback string `json:"callback"`
CreatedTime string `json:"created_time"`
FileID string `json:"file_id"`
FileName string `json:"file_name"`
FileSize string `json:"file_size"`
IconLink string `json:"icon_link"`
ID string `json:"id"`
Kind string `json:"kind"`
Message string `json:"message"`
Name string `json:"name"`
Params Params `json:"params"`
Phase string `json:"phase"` // PHASE_TYPE_RUNNING, PHASE_TYPE_ERROR, PHASE_TYPE_COMPLETE, PHASE_TYPE_PENDING
Progress int64 `json:"progress"`
ReferenceResource ReferenceResource `json:"reference_resource"`
Space string `json:"space"`
StatusSize int64 `json:"status_size"`
Statuses []string `json:"statuses"`
ThirdTaskID string `json:"third_task_id"`
Type string `json:"type"`
UpdatedTime string `json:"updated_time"`
UserID string `json:"user_id"`
}

type Params struct {
Age string `json:"age"`
MIMEType *string `json:"mime_type,omitempty"`
PredictType string `json:"predict_type"`
URL string `json:"url"`
}

type ReferenceResource struct {
Type string `json:"@type"`
Audit interface{} `json:"audit"`
Hash string `json:"hash"`
IconLink string `json:"icon_link"`
ID string `json:"id"`
Kind string `json:"kind"`
Medias []Media `json:"medias"`
MIMEType string `json:"mime_type"`
Name string `json:"name"`
Params map[string]interface{} `json:"params"`
ParentID string `json:"parent_id"`
Phase string `json:"phase"`
Size string `json:"size"`
Space string `json:"space"`
Starred bool `json:"starred"`
Tags []string `json:"tags"`
ThumbnailLink string `json:"thumbnail_link"`
}
1 change: 1 addition & 0 deletions internal/offline_download/all.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,6 @@ package offline_download
import (
_ "github.com/alist-org/alist/v3/internal/offline_download/aria2"
_ "github.com/alist-org/alist/v3/internal/offline_download/http"
_ "github.com/alist-org/alist/v3/internal/offline_download/pikpak"
_ "github.com/alist-org/alist/v3/internal/offline_download/qbit"
)
120 changes: 120 additions & 0 deletions internal/offline_download/pikpak/pikpak.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
package pikpak

import (
"context"
"fmt"

"github.com/alist-org/alist/v3/drivers/pikpak"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/offline_download/tool"
"github.com/alist-org/alist/v3/internal/op"
)

type PikPak struct {
refreshTaskCache bool
}

func (p *PikPak) Name() string {
return "pikpak"
}

func (p *PikPak) Items() []model.SettingItem {
return nil
}

func (p *PikPak) Run(task *tool.DownloadTask) error {
return errs.NotSupport
}

func (p *PikPak) Init() (string, error) {
p.refreshTaskCache = false
return "ok", nil
}

func (p *PikPak) IsReady() bool {
return true
}

func (p *PikPak) AddURL(args *tool.AddUrlArgs) (string, error) {
// 添加新任务刷新缓存
p.refreshTaskCache = true
// args.TempDir 已经被修改为了 DstDirPath
storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir)
if err != nil {
return "", err
}
pikpakDriver, ok := storage.(*pikpak.PikPak)
if !ok {
return "", fmt.Errorf("unsupported storage driver for offline download, only Pikpak is supported")
}

ctx := context.Background()
parentDir, err := op.GetUnwrap(ctx, storage, actualPath)
if err != nil {
return "", err
}

t, err := pikpakDriver.OfflineDownload(ctx, args.Url, parentDir, "")
if err != nil {
return "", fmt.Errorf("failed to add offline download task: %w", err)
}

return t.ID, nil
}

func (p *PikPak) Remove(task *tool.DownloadTask) error {
storage, _, err := op.GetStorageAndActualPath(task.DstDirPath)
if err != nil {
return err
}
pikpakDriver, ok := storage.(*pikpak.PikPak)
if !ok {
return fmt.Errorf("unsupported storage driver for offline download, only Pikpak is supported")
}
ctx := context.Background()
err = pikpakDriver.DeleteOfflineTasks(ctx, []string{task.GID}, false)
if err != nil {
return err
}
return nil
}

func (p *PikPak) Status(task *tool.DownloadTask) (*tool.Status, error) {
storage, _, err := op.GetStorageAndActualPath(task.DstDirPath)
if err != nil {
return nil, err
}
pikpakDriver, ok := storage.(*pikpak.PikPak)
if !ok {
return nil, fmt.Errorf("unsupported storage driver for offline download, only Pikpak is supported")
}
tasks, err := p.GetTasks(pikpakDriver)
if err != nil {
return nil, err
}
s := &tool.Status{
Progress: 0,
NewGID: "",
Completed: false,
Status: "the task has been deleted",
Err: nil,
}
for _, t := range tasks {
if t.ID == task.GID {
s.Progress = float64(t.Progress)
s.Status = t.Message
s.Completed = (t.Phase == "PHASE_TYPE_COMPLETE")
if t.Phase == "PHASE_TYPE_ERROR" {
s.Err = fmt.Errorf(t.Message)
}
return s, nil
}
}
s.Err = fmt.Errorf("the task has been deleted")
return s, nil
}

func init() {
tool.Tools.Add(&PikPak{})
}
43 changes: 43 additions & 0 deletions internal/offline_download/pikpak/util.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package pikpak

import (
"context"
"time"

"github.com/Xhofe/go-cache"
"github.com/alist-org/alist/v3/drivers/pikpak"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/pkg/singleflight"
)

var taskCache = cache.NewMemCache(cache.WithShards[[]pikpak.OfflineTask](16))
var taskG singleflight.Group[[]pikpak.OfflineTask]

func (p *PikPak) GetTasks(pikpakDriver *pikpak.PikPak) ([]pikpak.OfflineTask, error) {
key := op.Key(pikpakDriver, "/drive/v1/task")
if !p.refreshTaskCache {
if tasks, ok := taskCache.Get(key); ok {
return tasks, nil
}
}
p.refreshTaskCache = false
tasks, err, _ := taskG.Do(key, func() ([]pikpak.OfflineTask, error) {
ctx := context.Background()
phase := []string{"PHASE_TYPE_RUNNING", "PHASE_TYPE_ERROR", "PHASE_TYPE_PENDING", "PHASE_TYPE_COMPLETE"}
tasks, err := pikpakDriver.OfflineList(ctx, "", phase)
if err != nil {
return nil, err
}
// 添加缓存 10s
if len(tasks) > 0 {
taskCache.Set(key, tasks, cache.WithEx[[]pikpak.OfflineTask](time.Second*10))
} else {
taskCache.Del(key)
}
return tasks, nil
})
if err != nil {
return nil, err
}
return tasks, nil
}
11 changes: 9 additions & 2 deletions internal/offline_download/tool/add.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,14 @@ package tool

import (
"context"
"path/filepath"

"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/op"
"github.com/google/uuid"
"github.com/pkg/errors"
"github.com/xhofe/tache"
"path/filepath"
)

type DeletePolicy string
Expand Down Expand Up @@ -64,11 +65,17 @@ func AddURL(ctx context.Context, args *AddURLArgs) (tache.TaskWithInfo, error) {

uid := uuid.NewString()
tempDir := filepath.Join(conf.Conf.TempDir, args.Tool, uid)
deletePolicy := args.DeletePolicy
if args.Tool == "pikpak" {
tempDir = args.DstDirPath
// 防止将下载好的文件删除
deletePolicy = DeleteNever
}
t := &DownloadTask{
Url: args.URL,
DstDirPath: args.DstDirPath,
TempDir: tempDir,
DeletePolicy: args.DeletePolicy,
DeletePolicy: deletePolicy,
tool: tool,
}
DownloadTaskManager.Add(t)
Expand Down
Loading

0 comments on commit 3a3d0ad

Please sign in to comment.