From 060f3b9fcf63f4d6954b33248e12435dcef15f41 Mon Sep 17 00:00:00 2001 From: Ludvig Lundgren Date: Wed, 21 Aug 2024 18:32:25 +0200 Subject: [PATCH 1/8] fix(export): add missing tracker to .torrent --- cmd/torrent_export.go | 70 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 3 deletions(-) diff --git a/cmd/torrent_export.go b/cmd/torrent_export.go index 8bdc5f1..82b1e4d 100644 --- a/cmd/torrent_export.go +++ b/cmd/torrent_export.go @@ -11,10 +11,13 @@ import ( "github.com/ludviglundgren/qbittorrent-cli/internal/config" fsutil "github.com/ludviglundgren/qbittorrent-cli/internal/fs" + qbit "github.com/ludviglundgren/qbittorrent-cli/pkg/qbittorrent" + "github.com/anacrolix/torrent/metainfo" "github.com/autobrr/go-qbittorrent" "github.com/pkg/errors" "github.com/spf13/cobra" + "github.com/zeebo/bencode" ) func RunTorrentExport() *cobra.Command { @@ -294,6 +297,9 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To return errors.Wrapf(err, "could not check if dir exists: %s", exportDir) } + // qbittorrent from v4.5.x removes the announce-urls from the .torrent file so we need to add that back + needTrackerFix := false + // check BT_backup dir, pick torrent and fastresume files by id err := filepath.Walk(sourceDir, func(dirPath string, info fs.FileInfo, err error) error { if err != nil { @@ -343,17 +349,75 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To } outFile := filepath.Join(exportDir, fileName) - if err := fsutil.CopyFile(dirPath, outFile); err != nil { - return errors.Wrapf(err, "could not copy file: %s to %s", dirPath, outFile) - } exportCount++ switch ext { case ".torrent": + if needTrackerFix { + + // open file and check if announce is in there. If it's not, open .fastresume and combine before output + torrentFile, err := os.Open(filepath.Join(sourceDir, fileName+".torrent")) + if err != nil { + return err + } + + defer torrentFile.Close() + torrentInfo, err := metainfo.Load(torrentFile) + if err != nil { + return errors.Wrapf(err, "could not open file: %s", outFile) + } + + if torrentInfo.Announce == "" { + needTrackerFix = true + + fastResumeFile, err := os.Open(filepath.Join(sourceDir, fileName+".fastresume")) + if err != nil { + log.Fatalf("could not open fastresume file: %s", err) + } + + // open fastresume and get announce then // open fastresume and get announce then + var fastResume qbit.Fastresume + if err := bencode.NewDecoder(fastResumeFile).Decode(fastResume); err != nil { + return errors.Wrapf(err, "could not open file: %s", fileName+".fastresume") + } + + if len(fastResume.Trackers) == 0 { + return errors.New("no trackers found in fastresume") + } + + torrentInfo.Announce = fastResume.Trackers[0][0] + torrentInfo.AnnounceList = fastResume.Trackers + + if len(torrentInfo.UrlList) == 0 && len(fastResume.UrlList) > 0 { + torrentInfo.UrlList = fastResume.UrlList + } + } + + // write new torrent file to destination path + newTorrentFile, err := os.Create(outFile) + if err != nil { + return errors.Wrapf(err, "could not create file: %s", outFile) + } + defer newTorrentFile.Close() + if err := torrentInfo.Write(newTorrentFile); err != nil { + return errors.Wrapf(err, "could not write new torrent info file %s", outFile) + } + } else { + // only do this if !needTrackerFix + if err := fsutil.CopyFile(dirPath, outFile); err != nil { + return errors.Wrapf(err, "could not copy file: %s to %s", dirPath, outFile) + } + } + exportTorrentCount++ fmt.Printf("(%d/%d) exported: %s\n", exportTorrentCount, len(hashes), fileName) case ".fastresume": + // only do this if !needTrackerFix + if err := fsutil.CopyFile(dirPath, outFile); err != nil { + return errors.Wrapf(err, "could not copy file: %s to %s", dirPath, outFile) + } + exportFastresumeCount++ fmt.Printf("(%d/%d) exported: %s\n", exportFastresumeCount, len(hashes), fileName) } From 3f4c7bc1ff97b39735c0a15c9ea318416f60cc30 Mon Sep 17 00:00:00 2001 From: Ludvig Lundgren Date: Thu, 22 Aug 2024 13:26:17 +0200 Subject: [PATCH 2/8] fix(export): write fastresume from open file --- cmd/torrent_export.go | 138 ++++++++++++++++++++++++++---------------- 1 file changed, 87 insertions(+), 51 deletions(-) diff --git a/cmd/torrent_export.go b/cmd/torrent_export.go index 82b1e4d..5a5a8e8 100644 --- a/cmd/torrent_export.go +++ b/cmd/torrent_export.go @@ -60,8 +60,13 @@ func RunTorrentExport() *cobra.Command { command.MarkFlagRequired("export-dir") command.RunE = func(cmd *cobra.Command, args []string) error { - // get torrents from client by categories - config.InitConfig() + if len(f.includeCategory) > 0 && len(f.excludeCategory) > 0 { + return fmt.Errorf("--include-category and --exclude-category cannot be used together") + } + + if len(f.includeTag) > 0 && len(f.excludeTag) > 0 { + return fmt.Errorf("--include-tag and --exclude-tag cannot be used together") + } if _, err := os.Stat(f.sourceDir); err != nil { if os.IsNotExist(err) { @@ -71,6 +76,9 @@ func RunTorrentExport() *cobra.Command { return err } + // get torrents from client by categories + config.InitConfig() + qbtSettings := qbittorrent.Config{ Host: config.Qbit.Addr, Username: config.Qbit.Login, @@ -183,11 +191,11 @@ func RunTorrentExport() *cobra.Command { } if len(f.hashes) == 0 { - fmt.Printf("Could not find any matching torrents to export from (%s)\n", strings.Join(f.includeCategory, ",")) + log.Printf("Could not find any matching torrents to export from (%s)\n", strings.Join(f.includeCategory, ",")) os.Exit(1) } - fmt.Printf("Found '%d' matching torrents\n", len(f.hashes)) + log.Printf("Found '%d' matching torrents\n", len(f.hashes)) if err := processExport(f.sourceDir, f.exportDir, f.hashes, f.dry, f.verbose); err != nil { return errors.Wrapf(err, "could not process torrents") @@ -217,7 +225,7 @@ func RunTorrentExport() *cobra.Command { fmt.Println("dry-run: successfully wrote manifest to file") } else { if err := exportManifest(f.hashes, f.tags, f.category); err != nil { - fmt.Printf("could not export manifest: %q\n", err) + log.Printf("could not export manifest: %q\n", err) os.Exit(1) } @@ -258,31 +266,28 @@ func exportManifest(hashes map[string]qbittorrent.Torrent, tags map[string]struc }) } - res, err := json.Marshal(data) - if err != nil { - return errors.Wrap(err, "could not marshal manifest to json") - } - currentWorkingDirectory, err := os.Getwd() if err != nil { - return err + return errors.Wrap(err, "could not get current working directory") } - // Create a new file in the current working directory. - fileName := "export-manifest.json" + // Create a new manifestFile in the current working directory. + manifestFileName := "export-manifest.json" - file, err := os.Create(filepath.Join(currentWorkingDirectory, fileName)) + manifestFilePath := filepath.Join(currentWorkingDirectory, manifestFileName) + + manifestFile, err := os.Create(manifestFilePath) if err != nil { - return err + return errors.Wrapf(err, "could not create manifestFile: %s", manifestFilePath) } - defer file.Close() + defer manifestFile.Close() - // Write the string to the file. - _, err = file.WriteString(string(res)) - if err != nil { - return err + if err := json.NewEncoder(manifestFile).Encode(&data); err != nil { + return errors.Wrap(err, "could not encode manifest to json") } + log.Printf("wrote export manifest to %s", manifestFilePath) + return nil } @@ -293,27 +298,30 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To // check if export dir exists, if not then lets create it if err := createDirIfNotExists(exportDir); err != nil { - fmt.Printf("could not check if dir %s exists. err: %q\n", exportDir, err) + log.Printf("could not check if dir %s exists. err: %q\n", exportDir, err) return errors.Wrapf(err, "could not check if dir exists: %s", exportDir) } // qbittorrent from v4.5.x removes the announce-urls from the .torrent file so we need to add that back needTrackerFix := false - // check BT_backup dir, pick torrent and fastresume files by id - err := filepath.Walk(sourceDir, func(dirPath string, info fs.FileInfo, err error) error { + // keep track of processed fastresume files + processedFastResumeHashes := map[string]bool{} + + // check BT_backup dir, pick torrent and fastresume files by hash + err := filepath.WalkDir(sourceDir, func(dirPath string, d fs.DirEntry, err error) error { if err != nil { return err } - if info.IsDir() { + if d.IsDir() { return nil } - fileName := info.Name() + fileName := d.Name() ext := filepath.Ext(fileName) - if !isValidExt(ext) { + if ext != ".torrent" && ext != ".fastresume" { return nil } @@ -327,25 +335,25 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To if dry { if verbose { - fmt.Printf("processing: %s\n", fileName) + log.Printf("processing: %s\n", fileName) } exportCount++ - //fmt.Printf("dry-run: (%d/%d) exported: %s '%s'\n", exportCount, len(hashes), torrentHash, fileName) + //log.Printf("dry-run: (%d/%d) exported: %s '%s'\n", exportCount, len(hashes), torrentHash, fileName) switch ext { case ".torrent": exportTorrentCount++ - fmt.Printf("dry-run: (%d/%d) exported: %s\n", exportTorrentCount, len(hashes), fileName) + log.Printf("dry-run: (%d/%d) exported: %s\n", exportTorrentCount, len(hashes), fileName) case ".fastresume": exportFastresumeCount++ - fmt.Printf("dry-run: (%d/%d) exported: %s\n", exportFastresumeCount, len(hashes), fileName) + log.Printf("dry-run: (%d/%d) exported: %s\n", exportFastresumeCount, len(hashes), fileName) } } else { if verbose { - fmt.Printf("processing: %s\n", fileName) + log.Printf("processing: %s\n", fileName) } outFile := filepath.Join(exportDir, fileName) @@ -354,32 +362,36 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To switch ext { case ".torrent": - if needTrackerFix { + // determine if this should be run on first run and the ones after + if (exportTorrentCount == 0 && !needTrackerFix) || needTrackerFix { // open file and check if announce is in there. If it's not, open .fastresume and combine before output - torrentFile, err := os.Open(filepath.Join(sourceDir, fileName+".torrent")) + //torrentFile, err := os.Open(filepath.Join(sourceDir, fileName+".torrent")) + torrentFile, err := os.Open(dirPath) if err != nil { - return err + return errors.Wrapf(err, "could not open torrent file: %s", dirPath) } - defer torrentFile.Close() + torrentInfo, err := metainfo.Load(torrentFile) if err != nil { - return errors.Wrapf(err, "could not open file: %s", outFile) + return errors.Wrapf(err, "could not open file: %s", dirPath) } if torrentInfo.Announce == "" { needTrackerFix = true - fastResumeFile, err := os.Open(filepath.Join(sourceDir, fileName+".fastresume")) + sourceFastResumeFilePath := filepath.Join(sourceDir, torrentHash+".fastresume") + fastResumeFile, err := os.Open(sourceFastResumeFilePath) if err != nil { - log.Fatalf("could not open fastresume file: %s", err) + return errors.Wrapf(err, "could not open fastresume file: %s", sourceFastResumeFilePath) } + defer fastResumeFile.Close() // open fastresume and get announce then // open fastresume and get announce then var fastResume qbit.Fastresume - if err := bencode.NewDecoder(fastResumeFile).Decode(fastResume); err != nil { - return errors.Wrapf(err, "could not open file: %s", fileName+".fastresume") + if err := bencode.NewDecoder(fastResumeFile).Decode(&fastResume); err != nil { + return errors.Wrapf(err, "could not open file: %s", sourceFastResumeFilePath) } if len(fastResume.Trackers) == 0 { @@ -392,16 +404,35 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To if len(torrentInfo.UrlList) == 0 && len(fastResume.UrlList) > 0 { torrentInfo.UrlList = fastResume.UrlList } + + // copy .fastresume here already since we already have it open + fastresumeFilePath := filepath.Join(exportDir, torrentHash+".fastresume") + newFastResumeFile, err := os.Create(fastresumeFilePath) + if err != nil { + return errors.Wrapf(err, "could not create new fastresume file: %s", fastresumeFilePath) + } + defer newFastResumeFile.Close() + + if err := bencode.NewEncoder(newFastResumeFile).Encode(&fastResume); err != nil { + return errors.Wrapf(err, "could not encode fastresume to file: %s", fastresumeFilePath) + } + + // make sure the fastresume is only written once + processedFastResumeHashes[torrentHash] = true + + exportFastresumeCount++ + log.Printf("[%d/%d] exported: %s\n", exportFastresumeCount, len(hashes), fileName) } // write new torrent file to destination path newTorrentFile, err := os.Create(outFile) if err != nil { - return errors.Wrapf(err, "could not create file: %s", outFile) + return errors.Wrapf(err, "could not create new torrent file: %s", outFile) } defer newTorrentFile.Close() + if err := torrentInfo.Write(newTorrentFile); err != nil { - return errors.Wrapf(err, "could not write new torrent info file %s", outFile) + return errors.Wrapf(err, "could not write torrent info into file %s", outFile) } } else { // only do this if !needTrackerFix @@ -411,18 +442,23 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To } exportTorrentCount++ - fmt.Printf("(%d/%d) exported: %s\n", exportTorrentCount, len(hashes), fileName) + log.Printf("[%d/%d] exported: %s\n", exportTorrentCount, len(hashes), fileName) case ".fastresume": - // only do this if !needTrackerFix - if err := fsutil.CopyFile(dirPath, outFile); err != nil { - return errors.Wrapf(err, "could not copy file: %s to %s", dirPath, outFile) - } + // process if fastresume has not already been copied + _, ok := processedFastResumeHashes[torrentHash] + if !ok { - exportFastresumeCount++ - fmt.Printf("(%d/%d) exported: %s\n", exportFastresumeCount, len(hashes), fileName) + // only do this if !needTrackerFix + if err := fsutil.CopyFile(dirPath, outFile); err != nil { + return errors.Wrapf(err, "could not copy file: %s to %s", dirPath, outFile) + } + + exportFastresumeCount++ + log.Printf("[%d/%d] exported: %s\n", exportFastresumeCount, len(hashes), fileName) + } } - //fmt.Printf("(%d/%d) exported: %s '%s'\n", exportCount, len(hashes), torrentHash, fileName) + //log.Printf("[%d/%d] exported: %s '%s'\n", exportCount, len(hashes), torrentHash, fileName) } return nil @@ -432,7 +468,7 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To return err } - fmt.Printf(` + log.Printf(` found (%d) files in total exported fastresume: %d exported torrent %d From 5abd2a51e28faa039f9ff772354f1459d44c5ca5 Mon Sep 17 00:00:00 2001 From: Ludvig Lundgren Date: Thu, 22 Aug 2024 16:36:02 +0200 Subject: [PATCH 3/8] fix(export): optimizations --- cmd/torrent_export.go | 216 +++++++++++++++++++----------------------- 1 file changed, 100 insertions(+), 116 deletions(-) diff --git a/cmd/torrent_export.go b/cmd/torrent_export.go index 5a5a8e8..694246d 100644 --- a/cmd/torrent_export.go +++ b/cmd/torrent_export.go @@ -92,8 +92,7 @@ func RunTorrentExport() *cobra.Command { ctx := cmd.Context() if err := qb.LoginCtx(ctx); err != nil { - fmt.Fprintf(os.Stderr, "ERROR: connection failed: %v\n", err) - os.Exit(1) + return errors.Wrapf(err, "failed to login") } if len(f.includeCategory) > 0 { @@ -191,11 +190,10 @@ func RunTorrentExport() *cobra.Command { } if len(f.hashes) == 0 { - log.Printf("Could not find any matching torrents to export from (%s)\n", strings.Join(f.includeCategory, ",")) - os.Exit(1) + return errors.Errorf("Could not find any matching torrents to export from (%s)\n", strings.Join(f.includeCategory, ",")) } - log.Printf("Found '%d' matching torrents\n", len(f.hashes)) + log.Printf("Found (%d) matching torrents\n", len(f.hashes)) if err := processExport(f.sourceDir, f.exportDir, f.hashes, f.dry, f.verbose); err != nil { return errors.Wrapf(err, "could not process torrents") @@ -222,18 +220,17 @@ func RunTorrentExport() *cobra.Command { } if f.dry { - fmt.Println("dry-run: successfully wrote manifest to file") + log.Println("dry-run: successfully wrote manifest to file") } else { if err := exportManifest(f.hashes, f.tags, f.category); err != nil { - log.Printf("could not export manifest: %q\n", err) - os.Exit(1) + return errors.Wrapf(err, "could not export manifest") } - fmt.Println("successfully wrote manifest to file") + log.Println("successfully wrote manifest to file") } } - fmt.Println("Successfully exported torrents!") + log.Println("Successfully exported torrents!") return nil } @@ -298,7 +295,6 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To // check if export dir exists, if not then lets create it if err := createDirIfNotExists(exportDir); err != nil { - log.Printf("could not check if dir %s exists. err: %q\n", exportDir, err) return errors.Wrapf(err, "could not check if dir exists: %s", exportDir) } @@ -321,7 +317,7 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To fileName := d.Name() ext := filepath.Ext(fileName) - if ext != ".torrent" && ext != ".fastresume" { + if ext != ".torrent" { return nil } @@ -333,132 +329,124 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To return nil } - if dry { - if verbose { - log.Printf("processing: %s\n", fileName) - } + if verbose { + log.Printf("processing: %s\n", fileName) + } + if dry { exportCount++ - //log.Printf("dry-run: (%d/%d) exported: %s '%s'\n", exportCount, len(hashes), torrentHash, fileName) + exportTorrentCount++ + log.Printf("dry-run: (%d/%d) exported: %s\n", exportTorrentCount, len(hashes), torrentHash+".torrent") - switch ext { - case ".torrent": - exportTorrentCount++ - log.Printf("dry-run: (%d/%d) exported: %s\n", exportTorrentCount, len(hashes), fileName) - case ".fastresume": - exportFastresumeCount++ - log.Printf("dry-run: (%d/%d) exported: %s\n", exportFastresumeCount, len(hashes), fileName) - } + exportFastresumeCount++ + log.Printf("dry-run: (%d/%d) exported: %s\n", exportFastresumeCount, len(hashes), torrentHash+".fastresume") - } else { - if verbose { - log.Printf("processing: %s\n", fileName) - } + return nil + } - outFile := filepath.Join(exportDir, fileName) + outFile := filepath.Join(exportDir, fileName) - exportCount++ + exportCount++ - switch ext { - case ".torrent": - // determine if this should be run on first run and the ones after - if (exportTorrentCount == 0 && !needTrackerFix) || needTrackerFix { + // determine if this should be run on first run and the ones after + if (exportTorrentCount == 0 && !needTrackerFix) || needTrackerFix { - // open file and check if announce is in there. If it's not, open .fastresume and combine before output - //torrentFile, err := os.Open(filepath.Join(sourceDir, fileName+".torrent")) - torrentFile, err := os.Open(dirPath) - if err != nil { - return errors.Wrapf(err, "could not open torrent file: %s", dirPath) - } - defer torrentFile.Close() + // open file and check if announce is in there. If it's not, open .fastresume and combine before output + torrentFile, err := os.Open(dirPath) + if err != nil { + return errors.Wrapf(err, "could not open torrent file: %s", dirPath) + } + defer torrentFile.Close() - torrentInfo, err := metainfo.Load(torrentFile) - if err != nil { - return errors.Wrapf(err, "could not open file: %s", dirPath) - } + torrentInfo, err := metainfo.Load(torrentFile) + if err != nil { + return errors.Wrapf(err, "could not open file: %s", dirPath) + } - if torrentInfo.Announce == "" { - needTrackerFix = true + if torrentInfo.Announce == "" { + needTrackerFix = true - sourceFastResumeFilePath := filepath.Join(sourceDir, torrentHash+".fastresume") - fastResumeFile, err := os.Open(sourceFastResumeFilePath) - if err != nil { - return errors.Wrapf(err, "could not open fastresume file: %s", sourceFastResumeFilePath) - } - defer fastResumeFile.Close() + sourceFastResumeFilePath := filepath.Join(sourceDir, torrentHash+".fastresume") + fastResumeFile, err := os.Open(sourceFastResumeFilePath) + if err != nil { + return errors.Wrapf(err, "could not open fastresume file: %s", sourceFastResumeFilePath) + } + defer fastResumeFile.Close() - // open fastresume and get announce then // open fastresume and get announce then - var fastResume qbit.Fastresume - if err := bencode.NewDecoder(fastResumeFile).Decode(&fastResume); err != nil { - return errors.Wrapf(err, "could not open file: %s", sourceFastResumeFilePath) - } + // open fastresume and get announce then // open fastresume and get announce then + var fastResume qbit.Fastresume + if err := bencode.NewDecoder(fastResumeFile).Decode(&fastResume); err != nil { + return errors.Wrapf(err, "could not open file: %s", sourceFastResumeFilePath) + } - if len(fastResume.Trackers) == 0 { - return errors.New("no trackers found in fastresume") - } + if len(fastResume.Trackers) == 0 { + return errors.New("no trackers found in fastresume") + } - torrentInfo.Announce = fastResume.Trackers[0][0] - torrentInfo.AnnounceList = fastResume.Trackers + torrentInfo.Announce = fastResume.Trackers[0][0] + torrentInfo.AnnounceList = fastResume.Trackers - if len(torrentInfo.UrlList) == 0 && len(fastResume.UrlList) > 0 { - torrentInfo.UrlList = fastResume.UrlList - } + if len(torrentInfo.UrlList) == 0 && len(fastResume.UrlList) > 0 { + torrentInfo.UrlList = fastResume.UrlList + } - // copy .fastresume here already since we already have it open - fastresumeFilePath := filepath.Join(exportDir, torrentHash+".fastresume") - newFastResumeFile, err := os.Create(fastresumeFilePath) - if err != nil { - return errors.Wrapf(err, "could not create new fastresume file: %s", fastresumeFilePath) - } - defer newFastResumeFile.Close() + // copy .fastresume here already since we already have it open + fastresumeFilePath := filepath.Join(exportDir, torrentHash+".fastresume") + newFastResumeFile, err := os.Create(fastresumeFilePath) + if err != nil { + return errors.Wrapf(err, "could not create new fastresume file: %s", fastresumeFilePath) + } + defer newFastResumeFile.Close() - if err := bencode.NewEncoder(newFastResumeFile).Encode(&fastResume); err != nil { - return errors.Wrapf(err, "could not encode fastresume to file: %s", fastresumeFilePath) - } + if err := bencode.NewEncoder(newFastResumeFile).Encode(&fastResume); err != nil { + return errors.Wrapf(err, "could not encode fastresume to file: %s", fastresumeFilePath) + } - // make sure the fastresume is only written once - processedFastResumeHashes[torrentHash] = true + // make sure the fastresume is only written once + processedFastResumeHashes[torrentHash] = true - exportFastresumeCount++ - log.Printf("[%d/%d] exported: %s\n", exportFastresumeCount, len(hashes), fileName) - } + exportFastresumeCount++ + log.Printf("[%d/%d] exported: %s\n", exportFastresumeCount, len(hashes), fileName) + } - // write new torrent file to destination path - newTorrentFile, err := os.Create(outFile) - if err != nil { - return errors.Wrapf(err, "could not create new torrent file: %s", outFile) - } - defer newTorrentFile.Close() + // write new torrent file to destination path + newTorrentFile, err := os.Create(outFile) + if err != nil { + return errors.Wrapf(err, "could not create new torrent file: %s", outFile) + } + defer newTorrentFile.Close() - if err := torrentInfo.Write(newTorrentFile); err != nil { - return errors.Wrapf(err, "could not write torrent info into file %s", outFile) - } - } else { - // only do this if !needTrackerFix - if err := fsutil.CopyFile(dirPath, outFile); err != nil { - return errors.Wrapf(err, "could not copy file: %s to %s", dirPath, outFile) - } - } + if err := torrentInfo.Write(newTorrentFile); err != nil { + return errors.Wrapf(err, "could not write torrent info into file %s", outFile) + } - exportTorrentCount++ - log.Printf("[%d/%d] exported: %s\n", exportTorrentCount, len(hashes), fileName) - case ".fastresume": - // process if fastresume has not already been copied - _, ok := processedFastResumeHashes[torrentHash] - if !ok { + // all good lets return for this file + exportTorrentCount++ + log.Printf("[%d/%d] exported: %s\n", exportTorrentCount, len(hashes), fileName) - // only do this if !needTrackerFix - if err := fsutil.CopyFile(dirPath, outFile); err != nil { - return errors.Wrapf(err, "could not copy file: %s to %s", dirPath, outFile) - } + return nil + } - exportFastresumeCount++ - log.Printf("[%d/%d] exported: %s\n", exportFastresumeCount, len(hashes), fileName) - } + // only do this if !needTrackerFix + if err := fsutil.CopyFile(dirPath, outFile); err != nil { + return errors.Wrapf(err, "could not copy file: %s to %s", dirPath, outFile) + } + + exportTorrentCount++ + log.Printf("[%d/%d] exported: %s\n", exportTorrentCount, len(hashes), fileName) + + // process if fastresume has not already been copied + _, ok = processedFastResumeHashes[torrentHash] + if !ok { + fastResumeFilePath := filepath.Join(exportDir, torrentHash+".fastresume") + + if err := fsutil.CopyFile(dirPath, fastResumeFilePath); err != nil { + return errors.Wrapf(err, "could not copy file: %s to %s", dirPath, fastResumeFilePath) } - //log.Printf("[%d/%d] exported: %s '%s'\n", exportCount, len(hashes), torrentHash, fileName) + exportFastresumeCount++ + log.Printf("[%d/%d] exported: %s\n", exportFastresumeCount, len(hashes), torrentHash+".fastresume") } return nil @@ -468,11 +456,7 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To return err } - log.Printf(` -found (%d) files in total -exported fastresume: %d -exported torrent %d -`, exportCount, exportFastresumeCount, exportTorrentCount) + log.Printf("found (%d) files in total. exported fastresume: %d exported torrent %d", exportCount, exportFastresumeCount, exportTorrentCount) return nil } From 89aafb98dee1caa7c01b3a575f9c0be4597f3daa Mon Sep 17 00:00:00 2001 From: Ludvig Lundgren Date: Thu, 22 Aug 2024 19:12:53 +0200 Subject: [PATCH 4/8] fix(export): further optimizations --- cmd/torrent_export.go | 114 ++++++++++++++---------------------------- 1 file changed, 38 insertions(+), 76 deletions(-) diff --git a/cmd/torrent_export.go b/cmd/torrent_export.go index 694246d..9cf4505 100644 --- a/cmd/torrent_export.go +++ b/cmd/torrent_export.go @@ -95,98 +95,65 @@ func RunTorrentExport() *cobra.Command { return errors.Wrapf(err, "failed to login") } + var torrents []qbittorrent.Torrent + var err error + if len(f.includeCategory) > 0 { for _, category := range f.includeCategory { - torrents, err := qb.GetTorrentsCtx(ctx, qbittorrent.TorrentFilterOptions{Category: category}) + torrentsByCategory, err := qb.GetTorrentsCtx(ctx, qbittorrent.TorrentFilterOptions{Category: category}) if err != nil { return errors.Wrapf(err, "could not get torrents for category: %s", category) } - for _, tor := range torrents { - // only grab completed torrents - //if tor.Progress != 1 { - // continue - //} - - if tor.Tags != "" { - tags := strings.Split(tor.Tags, ", ") - - // check tags and exclude categories - if len(f.includeTag) > 0 && !containsTag(f.includeTag, tags) { - continue - } - - if len(f.excludeTag) > 0 && containsTag(f.excludeTag, tags) { - continue - } - - for _, tag := range tags { - _, ok := f.tags[tag] - if !ok { - f.tags[tag] = struct{}{} - } - } - - } - - if tor.Category != "" { - f.category[tor.Category] = qbittorrent.Category{ - Name: tor.Category, - SavePath: "", - } - } - - // append hash to map of hashes to gather - f.hashes[strings.ToLower(tor.Hash)] = tor - } + torrents = append(torrents, torrentsByCategory...) } } else { - torrents, err := qb.GetTorrentsCtx(ctx, qbittorrent.TorrentFilterOptions{}) + torrents, err = qb.GetTorrentsCtx(ctx, qbittorrent.TorrentFilterOptions{}) if err != nil { return errors.Wrap(err, "could not get torrents") } + } - for _, tor := range torrents { - // only grab completed torrents - //if tor.Progress != 1 { - // continue - //} - - if len(f.excludeCategory) > 0 && containsCategory(f.excludeCategory, tor.Category) { - continue - } + for _, tor := range torrents { + // only grab completed torrents + //if tor.Progress != 1 { + // continue + //} - if tor.Tags != "" { - tags := strings.Split(tor.Tags, ", ") + if len(f.excludeCategory) > 0 && containsCategory(f.excludeCategory, tor.Category) { + continue + } - // check tags and exclude categories - if len(f.includeTag) > 0 && !containsTag(f.includeTag, tags) { - continue - } + if tor.Tags != "" { + tags := strings.Split(tor.Tags, ", ") - if len(f.excludeTag) > 0 && containsTag(f.excludeTag, tags) { - continue - } + // check tags and exclude categories + if len(f.includeTag) > 0 && !containsTag(f.includeTag, tags) { + continue + } - for _, tag := range tags { - _, ok := f.tags[tag] - if !ok { - f.tags[tag] = struct{}{} - } - } + if len(f.excludeTag) > 0 && containsTag(f.excludeTag, tags) { + continue } - if tor.Category != "" { - f.category[tor.Category] = qbittorrent.Category{ - Name: tor.Category, - SavePath: "", + for _, tag := range tags { + _, ok := f.tags[tag] + if !ok { + f.tags[tag] = struct{}{} } } + } - // append hash to map of hashes to gather - f.hashes[strings.ToLower(tor.Hash)] = tor + if tor.Category != "" { + f.category[tor.Category] = qbittorrent.Category{ + Name: tor.Category, + SavePath: "", + } } + + // append hash to map of hashes to gather + f.hashes[strings.ToLower(tor.Hash)] = tor } if len(f.hashes) == 0 { @@ -289,7 +256,6 @@ func exportManifest(hashes map[string]qbittorrent.Torrent, tags map[string]struc } func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.Torrent, dry, verbose bool) error { - exportCount := 0 exportTorrentCount := 0 exportFastresumeCount := 0 @@ -334,8 +300,6 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To } if dry { - exportCount++ - exportTorrentCount++ log.Printf("dry-run: (%d/%d) exported: %s\n", exportTorrentCount, len(hashes), torrentHash+".torrent") @@ -347,8 +311,6 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To outFile := filepath.Join(exportDir, fileName) - exportCount++ - // determine if this should be run on first run and the ones after if (exportTorrentCount == 0 && !needTrackerFix) || needTrackerFix { @@ -407,7 +369,7 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To processedFastResumeHashes[torrentHash] = true exportFastresumeCount++ - log.Printf("[%d/%d] exported: %s\n", exportFastresumeCount, len(hashes), fileName) + log.Printf("[%d/%d] exported: %s\n", exportFastresumeCount, len(hashes), torrentHash+".fastresume") } // write new torrent file to destination path @@ -456,7 +418,7 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To return err } - log.Printf("found (%d) files in total. exported fastresume: %d exported torrent %d", exportCount, exportFastresumeCount, exportTorrentCount) + log.Printf("found (%d) files in total. exported fastresume: %d exported torrent %d", exportFastresumeCount+exportTorrentCount, exportFastresumeCount, exportTorrentCount) return nil } From f1464e726f2abba05221c1823c7f9d0a5a052c60 Mon Sep 17 00:00:00 2001 From: Ludvig Lundgren Date: Fri, 23 Aug 2024 16:55:50 +0200 Subject: [PATCH 5/8] fix(export): pretty print manifest file --- cmd/torrent_export.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cmd/torrent_export.go b/cmd/torrent_export.go index 9cf4505..27f5a17 100644 --- a/cmd/torrent_export.go +++ b/cmd/torrent_export.go @@ -246,11 +246,15 @@ func exportManifest(hashes map[string]qbittorrent.Torrent, tags map[string]struc } defer manifestFile.Close() - if err := json.NewEncoder(manifestFile).Encode(&data); err != nil { + // create new encoder with pretty print + encoder := json.NewEncoder(manifestFile) + encoder.SetIndent("", " ") + + if err := encoder.Encode(&data); err != nil { return errors.Wrap(err, "could not encode manifest to json") } - log.Printf("wrote export manifest to %s", manifestFilePath) + log.Printf("wrote export manifest to %s\n", manifestFilePath) return nil } @@ -418,7 +422,7 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To return err } - log.Printf("found (%d) files in total. exported fastresume: %d exported torrent %d", exportFastresumeCount+exportTorrentCount, exportFastresumeCount, exportTorrentCount) + log.Printf("exported (%d) files in total: fastresume (%d) torrents (%d)\n", exportFastresumeCount+exportTorrentCount, exportFastresumeCount, exportTorrentCount) return nil } From ad7430290aeb38d9f6218d20cd5fcd2eb2e57abe Mon Sep 17 00:00:00 2001 From: Ludvig Lundgren Date: Fri, 23 Aug 2024 17:21:40 +0200 Subject: [PATCH 6/8] fix(export): add timestamp to manifest --- cmd/torrent_export.go | 48 +++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/cmd/torrent_export.go b/cmd/torrent_export.go index 27f5a17..2c4de59 100644 --- a/cmd/torrent_export.go +++ b/cmd/torrent_export.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/ludviglundgren/qbittorrent-cli/internal/config" fsutil "github.com/ludviglundgren/qbittorrent-cli/internal/fs" @@ -76,6 +77,8 @@ func RunTorrentExport() *cobra.Command { return err } + log.Printf("Preparing to export torrents using source-dir: %q export-dir: %q\n", f.sourceDir, f.exportDir) + // get torrents from client by categories config.InitConfig() @@ -157,7 +160,7 @@ func RunTorrentExport() *cobra.Command { } if len(f.hashes) == 0 { - return errors.Errorf("Could not find any matching torrents to export from (%s)\n", strings.Join(f.includeCategory, ",")) + return errors.Errorf("Could not find any matching torrents to export from client") } log.Printf("Found (%d) matching torrents\n", len(f.hashes)) @@ -187,13 +190,11 @@ func RunTorrentExport() *cobra.Command { } if f.dry { - log.Println("dry-run: successfully wrote manifest to file") + log.Println("dry-run: Saved export manifest to file") } else { - if err := exportManifest(f.hashes, f.tags, f.category); err != nil { + if err := exportManifest(f.hashes, f.tags, f.category, f.exportDir); err != nil { return errors.Wrapf(err, "could not export manifest") } - - log.Println("successfully wrote manifest to file") } } @@ -205,7 +206,7 @@ func RunTorrentExport() *cobra.Command { return command } -func exportManifest(hashes map[string]qbittorrent.Torrent, tags map[string]struct{}, categories map[string]qbittorrent.Category) error { +func exportManifest(hashes map[string]qbittorrent.Torrent, tags map[string]struct{}, categories map[string]qbittorrent.Category, exportDir string) error { data := Manifest{ Tags: make([]string, 0), Categories: []qbittorrent.Category{}, @@ -230,15 +231,18 @@ func exportManifest(hashes map[string]qbittorrent.Torrent, tags map[string]struc }) } - currentWorkingDirectory, err := os.Getwd() - if err != nil { - return errors.Wrap(err, "could not get current working directory") - } + //currentWorkingDirectory, err := os.Getwd() + //if err != nil { + // return errors.Wrap(err, "could not get current working directory") + //} + + currentTime := time.Now() + timestamp := currentTime.Format("20060102150405") // Create a new manifestFile in the current working directory. - manifestFileName := "export-manifest.json" + manifestFileName := fmt.Sprintf("export-manifest-%s.json", timestamp) - manifestFilePath := filepath.Join(currentWorkingDirectory, manifestFileName) + manifestFilePath := filepath.Join(exportDir, manifestFileName) manifestFile, err := os.Create(manifestFilePath) if err != nil { @@ -254,7 +258,7 @@ func exportManifest(hashes map[string]qbittorrent.Torrent, tags map[string]struc return errors.Wrap(err, "could not encode manifest to json") } - log.Printf("wrote export manifest to %s\n", manifestFilePath) + log.Printf("Saved export manifest to %s\n", manifestFilePath) return nil } @@ -294,21 +298,21 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To torrentHash := fileNameTrimExt(fileName) // if filename not in hashes return and check next - _, ok := hashes[torrentHash] + torrent, ok := hashes[torrentHash] if !ok { return nil } if verbose { - log.Printf("processing: %s\n", fileName) + log.Printf("Processing: %s\n", fileName) } if dry { exportTorrentCount++ - log.Printf("dry-run: (%d/%d) exported: %s\n", exportTorrentCount, len(hashes), torrentHash+".torrent") + log.Printf("dry-run: [%d/%d] exported: %s %s\n", exportTorrentCount, len(hashes), torrentHash+".torrent", torrent.Name) exportFastresumeCount++ - log.Printf("dry-run: (%d/%d) exported: %s\n", exportFastresumeCount, len(hashes), torrentHash+".fastresume") + log.Printf("dry-run: [%d/%d] exported: %s %s\n", exportFastresumeCount, len(hashes), torrentHash+".fastresume", torrent.Name) return nil } @@ -373,7 +377,7 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To processedFastResumeHashes[torrentHash] = true exportFastresumeCount++ - log.Printf("[%d/%d] exported: %s\n", exportFastresumeCount, len(hashes), torrentHash+".fastresume") + log.Printf("[%d/%d] exported: %s %s\n", exportFastresumeCount, len(hashes), torrentHash+".fastresume", torrent.Name) } // write new torrent file to destination path @@ -389,7 +393,7 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To // all good lets return for this file exportTorrentCount++ - log.Printf("[%d/%d] exported: %s\n", exportTorrentCount, len(hashes), fileName) + log.Printf("[%d/%d] exported: %s %s\n", exportTorrentCount, len(hashes), fileName, torrent.Name) return nil } @@ -400,7 +404,7 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To } exportTorrentCount++ - log.Printf("[%d/%d] exported: %s\n", exportTorrentCount, len(hashes), fileName) + log.Printf("[%d/%d] exported: %s %s\n", exportTorrentCount, len(hashes), fileName, torrent.Name) // process if fastresume has not already been copied _, ok = processedFastResumeHashes[torrentHash] @@ -412,7 +416,7 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To } exportFastresumeCount++ - log.Printf("[%d/%d] exported: %s\n", exportFastresumeCount, len(hashes), torrentHash+".fastresume") + log.Printf("[%d/%d] exported: %s %s\n", exportFastresumeCount, len(hashes), torrentHash+".fastresume", torrent.Name) } return nil @@ -422,7 +426,7 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To return err } - log.Printf("exported (%d) files in total: fastresume (%d) torrents (%d)\n", exportFastresumeCount+exportTorrentCount, exportFastresumeCount, exportTorrentCount) + log.Printf("Exported (%d) files in total: fastresume (%d) torrents (%d)\n", exportFastresumeCount+exportTorrentCount, exportFastresumeCount, exportTorrentCount) return nil } From aa880873538b1771a56f9fa24721a76bf3ffa9c8 Mon Sep 17 00:00:00 2001 From: Ludvig Lundgren Date: Fri, 23 Aug 2024 17:51:36 +0200 Subject: [PATCH 7/8] fix(export): archive exported files --- cmd/torrent_export.go | 42 ++++++++++++++++++++++--- pkg/archive/archive.go | 70 ++++++++++++++++++++++++++++++++++++++++++ pkg/utils/utils.go | 15 +++++++++ 3 files changed, 122 insertions(+), 5 deletions(-) create mode 100644 pkg/archive/archive.go diff --git a/cmd/torrent_export.go b/cmd/torrent_export.go index 2c4de59..34b3b6e 100644 --- a/cmd/torrent_export.go +++ b/cmd/torrent_export.go @@ -3,6 +3,8 @@ package cmd import ( "encoding/json" "fmt" + "github.com/ludviglundgren/qbittorrent-cli/pkg/archive" + "github.com/ludviglundgren/qbittorrent-cli/pkg/utils" "io/fs" "log" "os" @@ -32,6 +34,7 @@ func RunTorrentExport() *cobra.Command { f := export{ dry: false, verbose: false, + archive: false, sourceDir: "", exportDir: "", includeCategory: nil, @@ -46,6 +49,7 @@ func RunTorrentExport() *cobra.Command { command.Flags().BoolVar(&f.dry, "dry-run", false, "dry run") command.Flags().BoolVarP(&f.verbose, "verbose", "v", false, "verbose output") + command.Flags().BoolVarP(&f.archive, "archive", "a", false, "archive export dir to .tar.gz") command.Flags().BoolVar(&skipManifest, "skip-manifest", false, "Do not export all used tags and categories into manifest") command.Flags().StringVar(&f.sourceDir, "source", "", "Dir with torrent and fast-resume files (required)") @@ -60,13 +64,20 @@ func RunTorrentExport() *cobra.Command { command.MarkFlagRequired("source") command.MarkFlagRequired("export-dir") + command.MarkFlagsMutuallyExclusive("include-category", "exclude-category") + command.MarkFlagsMutuallyExclusive("include-tag", "exclude-tag") + command.RunE = func(cmd *cobra.Command, args []string) error { - if len(f.includeCategory) > 0 && len(f.excludeCategory) > 0 { - return fmt.Errorf("--include-category and --exclude-category cannot be used together") + var err error + + f.sourceDir, err = utils.ExpandTilde(f.sourceDir) + if err != nil { + return errors.Wrap(err, "could not read source-dir") } - if len(f.includeTag) > 0 && len(f.excludeTag) > 0 { - return fmt.Errorf("--include-tag and --exclude-tag cannot be used together") + f.exportDir, err = utils.ExpandTilde(f.exportDir) + if err != nil { + return errors.Wrap(err, "could not read export-dir") } if _, err := os.Stat(f.sourceDir); err != nil { @@ -99,7 +110,6 @@ func RunTorrentExport() *cobra.Command { } var torrents []qbittorrent.Torrent - var err error if len(f.includeCategory) > 0 { for _, category := range f.includeCategory { @@ -200,6 +210,27 @@ func RunTorrentExport() *cobra.Command { log.Println("Successfully exported torrents!") + if f.archive { + currentWorkingDirectory, err := os.Getwd() + if err != nil { + return errors.Wrap(err, "could not get current working directory") + } + + archiveFilename := filepath.Join(currentWorkingDirectory, "qbittorrent-export.tar.gz") + + if f.dry { + log.Printf("dry-run: Archive exported dir to: %s\n", archiveFilename) + } else { + log.Printf("Archive exported dir to: %s\n", archiveFilename) + + if err := archive.TarGzDirectory(f.exportDir, archiveFilename); err != nil { + return errors.Wrapf(err, "could not archive exported torrents") + } + } + + log.Println("Successfully archived exported torrents!") + } + return nil } @@ -468,6 +499,7 @@ func createDirIfNotExists(dir string) error { type export struct { dry bool verbose bool + archive bool sourceDir string exportDir string includeCategory []string diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go new file mode 100644 index 0000000..f4357db --- /dev/null +++ b/pkg/archive/archive.go @@ -0,0 +1,70 @@ +package archive + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "os" + "path/filepath" +) + +func TarGzDirectory(source, target string) error { + file, err := os.Create(target) + if err != nil { + return fmt.Errorf("failed to create tar.gz file: %w", err) + } + defer file.Close() + + // Create a new gzip writer + gzipWriter := gzip.NewWriter(file) + defer gzipWriter.Close() + + // Create a new tar writer + tarWriter := tar.NewWriter(gzipWriter) + defer tarWriter.Close() + + err = filepath.Walk(source, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Create a tar header from file info + header, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + + // Preserve the directory structure + header.Name, err = filepath.Rel(filepath.Dir(source), path) + if err != nil { + return err + } + + // Write the header to the tar archive + if err := tarWriter.WriteHeader(header); err != nil { + return err + } + + // If it's a file, write its content to the tar archive + if !info.IsDir() { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + if _, err := io.Copy(tarWriter, file); err != nil { + return err + } + } + + return nil + }) + + if err != nil { + return fmt.Errorf("failed to add files to %s.tar.gz archive: %w", target, err) + } + + return nil +} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index eb14570..4093a15 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -2,6 +2,8 @@ package utils import ( "fmt" + "os/user" + "path/filepath" "regexp" "strings" ) @@ -23,3 +25,16 @@ func ValidateHash(hashes []string) error { return nil } + +// ExpandTilde expands the ~ in the file path to the home directory +func ExpandTilde(path string) (string, error) { + if strings.HasPrefix(path, "~") { + usr, err := user.Current() + if err != nil { + return "", err + } + homeDir := usr.HomeDir + return filepath.Join(homeDir, path[1:]), nil + } + return path, nil +} From 3a21725294116c5fe4f23bdd66cb7cf817a900be Mon Sep 17 00:00:00 2001 From: Ludvig Lundgren Date: Mon, 25 Nov 2024 14:55:37 +0100 Subject: [PATCH 8/8] chore: order imports --- cmd/torrent_export.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/torrent_export.go b/cmd/torrent_export.go index 34b3b6e..6562e5d 100644 --- a/cmd/torrent_export.go +++ b/cmd/torrent_export.go @@ -3,8 +3,6 @@ package cmd import ( "encoding/json" "fmt" - "github.com/ludviglundgren/qbittorrent-cli/pkg/archive" - "github.com/ludviglundgren/qbittorrent-cli/pkg/utils" "io/fs" "log" "os" @@ -14,7 +12,9 @@ import ( "github.com/ludviglundgren/qbittorrent-cli/internal/config" fsutil "github.com/ludviglundgren/qbittorrent-cli/internal/fs" + "github.com/ludviglundgren/qbittorrent-cli/pkg/archive" qbit "github.com/ludviglundgren/qbittorrent-cli/pkg/qbittorrent" + "github.com/ludviglundgren/qbittorrent-cli/pkg/utils" "github.com/anacrolix/torrent/metainfo" "github.com/autobrr/go-qbittorrent"