Add support for download stream media

.github/workflows/testing.yml

@@ -69,6 +69,26 @@ jobs:
         choco install wget
         wget --help
 
+    - name: Install FFmpeg
+      if: matrix.os == 'ubuntu-latest'
+      run: |
+        sudo add-apt-repository universe
+        sudo apt-get -y update
+        sudo apt-get -y install ffmpeg
+        ffmpeg -version
+
+    - name: Install FFmpeg
+      if: matrix.os == 'macos-latest'
+      run: |
+        brew install ffmpeg
+        ffmpeg -version
+
+    - name: Install FFmpeg
+      if: matrix.os == 'windows-latest'
+      run: |
+        choco install ffmpeg
+        ffmpeg -version
+
     - name: Check out code base
       if: github.event_name == 'push'
       uses: actions/checkout@v2

Dockerfile

@@ -71,6 +71,7 @@ ONBUILD RUN set -o pipefail; \
     chromium \
     dbus \
     dumb-init \
+    ffmpeg \
     freetype \
     libstdc++ \
     harfbuzz \

Dockerfile.dev

@@ -63,6 +63,7 @@ ONBUILD RUN set -o pipefail; \
     chromium \
     dbus \
     dumb-init \
+    ffmpeg \
     freetype \
     libstdc++ \
     harfbuzz \

README.md

@@ -25,8 +25,9 @@ Supported Golang version: See [.github/workflows/testing.yml](./.github/workflow
 - Serve as Tor Hidden Service or local web entry
 - Wayback to Internet Archive, archive.today, IPFS and Telegraph easier
 - Interactive with IRC, Martix, Telegram bot, Mastodon and Twitter as daemon service
-- Support publish wayback results to Telegram channel, Mastodon and GitHub Issues
-- Support store archived files to disk
+- Supports publish wayback results to Telegram channel, Mastodon and GitHub Issues
+- Supports store archived files to disk
+- Download stream media (requires [FFmpeg](https://ffmpeg.org/))
 
 ## Installation
 
@@ -207,6 +208,7 @@ You can also specify configuration options either via command flags or via envir
 | -                   | `WAYBACK_POOLING_SIZE`            | `3`                     | Number of worker pool for wayback at once                    |
 | -                   | `WAYBACK_BOLT_PATH`               | `./wayback.db`          | File path of bolt database                                   |
 | -                   | `WAYBACK_STORAGE_DIR`             | -                       | Directory to store binary file, e.g. PDF, html file          |
+| -                   | `WAYBACK_MAX_MEDIA_SIZE`          | `512MB`                 | Max size to limit download stream media                      |
 | `-d`, `--daemon`    | -                                 | -                       | Run as daemon service, e.g. `telegram`, `web`, `mastodon`, `twitter` |
 | `--ia`              | `WAYBACK_ENABLE_IA`               | `true`                  | Wayback webpages to **Internet Archive**                     |
 | `--is`              | `WAYBACK_ENABLE_IS`               | `true`                  | Wayback webpages to **Archive Today**                        |

config/config_test.go

@@ -952,3 +952,43 @@ func TestLogLevel(t *testing.T) {
 		})
 	}
 }
+
+func TestMaxMediaSize(t *testing.T) {
+	t.Parallel()
+
+	var tests = []struct {
+		size     string
+		expected uint64
+	}{
+		{
+			size:     "",
+			expected: 512000000,
+		},
+		{
+			size:     "invalid",
+			expected: 0,
+		},
+		{
+			size:     "10KB",
+			expected: 10000,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.size, func(t *testing.T) {
+			os.Clearenv()
+			os.Setenv("WAYBACK_MAX_MEDIA_SIZE", test.size)
+
+			parser := NewParser()
+			opts, err := parser.ParseEnvironmentVariables()
+			if err != nil {
+				t.Fatalf(`Parsing environment variables failed: %v`, err)
+			}
+
+			got := opts.MaxMediaSize()
+			if got != test.expected {
+				t.Fatalf(`Unexpected set max media size got %d instead of %d`, got, test.expected)
+			}
+		})
+	}
+}

config/options.go

@@ -8,6 +8,7 @@ import (
 	"net/url"
 	"strings"
 
+	"github.com/dustin/go-humanize"
 	"github.com/wabarc/logger"
 )
 
@@ -61,6 +62,7 @@ const (
 	defBoltPathname        = "wayback.db"
 	defPoolingSize         = 3
 	defStorageDir          = ""
+	defMaxMediaSize        = "512MB"
 )
 
 var (
@@ -89,6 +91,7 @@ type Options struct {
 	boltPathname        string
 	poolingSize         int
 	storageDir          string
+	maxMediaSize        string
 }
 
 type ipfs struct {
@@ -158,6 +161,7 @@ func NewOptions() *Options {
 		boltPathname:        defBoltPathname,
 		poolingSize:         defPoolingSize,
 		storageDir:          defStorageDir,
+		maxMediaSize:        defMaxMediaSize,
 		ipfs: &ipfs{
 			host: defIPFSHost,
 			port: defIPFSPort,
@@ -507,3 +511,12 @@ func (o *Options) StorageDir() string {
 func (o *Options) EnabledReduxer() bool {
 	return o.storageDir != ""
 }
+
+// MaxMediaSize returns max size to limit download stream media.
+func (o *Options) MaxMediaSize() uint64 {
+	size, err := humanize.ParseBytes(o.maxMediaSize)
+	if err != nil {
+		return 0
+	}
+	return size
+}

config/parser.go

@@ -165,6 +165,8 @@ func (p *Parser) parseLines(lines []string) (err error) {
 			p.opts.boltPathname = parseString(val, defBoltPathname)
 		case "WAYBACK_STORAGE_DIR":
 			p.opts.storageDir = parseString(val, defStorageDir)
+		case "WAYBACK_MAX_MEDIA_SIZE":
+			p.opts.maxMediaSize = parseString(val, defMaxMediaSize)
 		default:
 			if os.Getenv(key) == "" && val != "" {
 				os.Setenv(key, val)

go.mod

@@ -12,30 +12,37 @@ require (
 	github.com/cretz/bine v0.2.0
 	github.com/dghubble/go-twitter v0.0.0-20201011215211-4b180d0cc78d
 	github.com/dghubble/oauth1 v0.7.0
+	github.com/dustin/go-humanize v1.0.0
+	github.com/fatih/color v1.12.0 // indirect
 	github.com/go-shiori/dom v0.0.0-20210627111528-4e4722cd0d65 // indirect
 	github.com/go-shiori/obelisk v0.0.0-20201115143556-8de0d40b0a9b // indirect
-	github.com/gobwas/ws v1.1.0 // indirect
 	github.com/google/go-github/v37 v37.0.0
 	github.com/gorilla/mux v1.8.0
+	github.com/iawia002/annie v0.0.0-20210714033020-3c20ed1e6f07
 	github.com/klauspost/cpuid/v2 v2.0.8 // indirect
+	github.com/kr/pretty v0.2.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	// github.com/ipsn/go-libtor v1.0.329
 	github.com/libp2p/go-libp2p-core v0.8.5 // indirect
 	github.com/logrusorgru/aurora/v3 v3.0.0
+	github.com/mattn/go-isatty v0.0.13 // indirect
 	github.com/mattn/go-mastodon v0.0.5-0.20210515144304-86627ec7d635
 	github.com/multiformats/go-multiaddr v0.3.3 // indirect
 	github.com/multiformats/go-multihash v0.0.15 // indirect
 	github.com/prometheus/client_golang v1.11.0
 	github.com/prometheus/common v0.29.0
+	github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spf13/cobra v1.2.1
 	github.com/tdewolff/parse/v2 v2.5.19 // indirect
 	github.com/thoj/go-ircevent v0.0.0-20190807115034-8e7ce4b5a1eb
+	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/wabarc/archive.is v1.3.0
 	github.com/wabarc/archive.org v1.2.1-0.20210708220121-cb9b83ff9896
 	github.com/wabarc/helper v0.0.0-20210706220001-6ba9e89c752b
 	github.com/wabarc/logger v0.0.0-20210708144517-a9651f538672
 	github.com/wabarc/playback v0.0.0-20210706162327-6ba67b324cc8
-	github.com/wabarc/screenshot v1.2.1-0.20210708225510-eb68213a95f1
+	github.com/wabarc/screenshot v1.2.1-0.20210713083422-eee9add4752d
 	github.com/wabarc/telegra.ph v0.0.0-20210708231234-c10dbc08962f
 	github.com/wabarc/warcraft v0.1.1-0.20210707001544-e897dbede7c3
 	github.com/wabarc/wbipfs v0.2.0