Add support for Discord

.github/workflows/testing.yml

@@ -35,6 +35,8 @@ jobs:
       RECVER_UID: ${{ secrets.MATRIX_RECVER_UID }}
       RECVER_PWD: ${{ secrets.MATRIX_RECVER_PWD }}
       MATRIX_ROOMID: ${{ secrets.MATRIX_ROOMID }}
+      WAYBACK_DISCORD_BOT_TOKEN: ${{ secrets.WAYBACK_DISCORD_BOT_TOKEN }}
+      WAYBACK_DISCORD_CHANNEL: ${{ secrets.WAYBACK_DISCORD_CHANNEL }}
     steps:
     - name: Set up Go 1.x
       uses: actions/setup-go@v2

README.md

@@ -7,7 +7,7 @@
 [![Releases](https://img.shields.io/github/v/release/wabarc/wayback.svg?include_prereleases&color=blue)](https://github.com/wabarc/wayback/releases)
 
 [![Telegram Bot](https://img.shields.io/badge/Telegram-bot-3dbeff.svg)](https://t.me/wabarc_bot)
-[![Telegram Channel](https://img.shields.io/badge/Telegram-channel-3dbeff.svg)](https://t.me/wabarc)
+[![Discord Bot](https://img.shields.io/badge/Discord-bot-3dbeff.svg)](https://discord.com/api/oauth2/authorize?client_id=863324809206169640&permissions=2147796992&scope=bot%20applications.commands)
 [![Matrix Bot](https://img.shields.io/badge/Matrix-bot-0a976f.svg)](https://matrix.to/#/@wabarc_bot:matrix.org)
 [![Matrix Room](https://img.shields.io/badge/Matrix-room-0a976f.svg)](https://matrix.to/#/#wabarc:matrix.org)
 [![Tor Hidden Service](https://img.shields.io/badge/Tor%20Hidden%20Service-472756.svg)](http://wabarcoww2bxmdbixj7sjwggv3fonh2rpflfiildegcydk5udkdckdyd.onion/)
@@ -24,7 +24,7 @@ Supported Golang version: See [.github/workflows/testing.yml](./.github/workflow
 - Builtin CLI (`wayback`)
 - Serve as Tor Hidden Service or local web entry
 - Wayback to Internet Archive, archive.today, IPFS and Telegraph easier
-- Interactive with IRC, Martix, Telegram bot, Mastodon and Twitter as daemon service
+- Interactive with IRC, Martix, Telegram bot, Discord bot, Mastodon and Twitter as daemon service
 - Supports publish wayback results to Telegram channel, Mastodon and GitHub Issues
 - Supports store archived files to disk
 - Download stream media (requires [FFmpeg](https://ffmpeg.org/))
@@ -209,7 +209,7 @@ You can also specify configuration options either via command flags or via envir
 | -                   | `WAYBACK_BOLT_PATH`               | `./wayback.db`          | File path of bolt database                                   |
 | -                   | `WAYBACK_STORAGE_DIR`             | -                       | Directory to store binary file, e.g. PDF, html file          |
 | -                   | `WAYBACK_MAX_MEDIA_SIZE`          | `512MB`                 | Max size to limit download stream media                      |
-| `-d`, `--daemon`    | -                                 | -                       | Run as daemon service, e.g. `telegram`, `web`, `mastodon`, `twitter` |
+| `-d`, `--daemon`    | -                                 | -                       | Run as daemon service, e.g. `telegram`, `web`, `mastodon`, `twitter`, `discord` |
 | `--ia`              | `WAYBACK_ENABLE_IA`               | `true`                  | Wayback webpages to **Internet Archive**                     |
 | `--is`              | `WAYBACK_ENABLE_IS`               | `true`                  | Wayback webpages to **Archive Today**                        |
 | `--ip`              | `WAYBACK_ENABLE_IP`               | `false`                 | Wayback webpages to **IPFS**                                 |
@@ -222,7 +222,7 @@ You can also specify configuration options either via command flags or via envir
 | -                   | `WAYBACK_GITHUB_REPO`             | -                       | GitHub repository to publish results                         |
 | `-t`, `--token`     | `WAYBACK_TELEGRAM_TOKEN`          | -                       | Telegram Bot API Token                                       |
 | `--chatid`          | `WAYBACK_TELEGRAM_CHANNEL`        | -                       | The Telegram public/private channel id to publish archive result |
-| -                   | `WAYBACK_TELEGRAM_HELPTEXT`       | -                       | The help text for Telegram bot command                       |
+| -                   | `WAYBACK_TELEGRAM_HELPTEXT`       | -                       | The help text for Telegram command                           |
 | -                   | `WAYBACK_MASTODON_SERVER`         | -                       | Domain of Mastodon instance                                  |
 | -                   | `WAYBACK_MASTODON_KEY`            | -                       | The client key of your Mastodon application                  |
 | -                   | `WAYBACK_MASTODON_SECRET`         | -                       | The client secret of your Mastodon application               |
@@ -239,6 +239,9 @@ You can also specify configuration options either via command flags or via envir
 | -                   | `WAYBACK_MATRIX_USERID`           | -                       | Matrix unique user ID, format: `@foo:example.com`            |
 | -                   | `WAYBACK_MATRIX_ROOMID`           | -                       | Matrix internal room ID, format: `!bar:example.com`          |
 | -                   | `WAYBACK_MATRIX_PASSWORD`         | -                       | Matrix password                                              |
+| -                   | `WAYBACK_DISCORD_BOT_TOKEN`       | -                       | Discord bot authorization token                              |
+| -                   | `WAYBACK_DISCORD_CHANNEL`         | -                       | Discord channel ID, [find channel ID](https://support.discord.com/hc/en-us/articles/206346498-Where-can-I-find-my-server-ID-)  |
+| -                   | `WAYBACK_DISCORD_HELPTEXT`        | -                       | The help text for Discord command                            |
 | `--tor`             | `WAYBACK_USE_TOR`                 | `false`                 | Snapshot webpage via Tor anonymity network                   |
 | `--tor-key`         | `WAYBACK_TOR_PRIVKEY`             | -                       | The private key for Tor Hidden Service                       |
 | -                   | `WAYBACK_TOR_LOCAL_PORT`          | `8964`                  | Local port for Tor Hidden Service, also support for a **reverse proxy** |

cmd/wayback/main.go

@@ -68,7 +68,7 @@ func init() {
 	rootCmd.Flags().BoolVarP(&is, "is", "", false, "Wayback webpages to Archive Today.")
 	rootCmd.Flags().BoolVarP(&ip, "ip", "", false, "Wayback webpages to IPFS. (default false)")
 	rootCmd.Flags().BoolVarP(&ph, "ph", "", false, "Wayback webpages to Telegraph. (default false)")
-	rootCmd.Flags().StringSliceVarP(&daemon, "daemon", "d", []string{}, "Run as daemon service, supported services are telegram, web, mastodon, twitter, irc")
+	rootCmd.Flags().StringSliceVarP(&daemon, "daemon", "d", []string{}, "Run as daemon service, supported services are telegram, web, mastodon, twitter, discord, irc")
 	rootCmd.Flags().StringVarP(&host, "ipfs-host", "", "127.0.0.1", "IPFS daemon host, do not require, unless enable ipfs.")
 	rootCmd.Flags().UintVarP(&port, "ipfs-port", "p", 5001, "IPFS daemon port.")
 	rootCmd.Flags().StringVarP(&mode, "ipfs-mode", "m", "pinner", "IPFS mode.")

cmd/wayback/serve.go

@@ -12,6 +12,7 @@ import (
 	"github.com/wabarc/logger"
 	"github.com/wabarc/wayback/config"
 	"github.com/wabarc/wayback/pooling"
+	"github.com/wabarc/wayback/service/discord"
 	"github.com/wabarc/wayback/service/httpd"
 	"github.com/wabarc/wayback/service/mastodon"
 	"github.com/wabarc/wayback/service/matrix"
@@ -63,6 +64,11 @@ func (srv *service) run(ctx context.Context, store *storage.Storage, pool poolin
 			go func(errCh chan error) {
 				errCh <- irc.Serve()
 			}(srv.errCh)
+		case "discord":
+			discord := discord.New(ctx, store, pool)
+			go func(errCh chan error) {
+				errCh <- discord.Serve()
+			}(srv.errCh)
 		case "mastodon", "mstdn":
 			mastodon := mastodon.New(ctx, store, pool)
 			go func(errCh chan error) {

config/config_test.go

@@ -762,6 +762,96 @@ func TestPublishToMatrixRoom(t *testing.T) {
 	}
 }
 
+func TestDiscordBotToken(t *testing.T) {
+	expected := "foo-bar"
+
+	os.Clearenv()
+	os.Setenv("WAYBACK_DISCORD_BOT_TOKEN", expected)
+
+	parser := NewParser()
+	opts, err := parser.ParseEnvironmentVariables()
+	if err != nil {
+		t.Fatalf(`Parsing environment variables failed: %v`, err)
+	}
+
+	got := opts.DiscordBotToken()
+	if got != expected {
+		t.Fatalf(`Unexpected Discord bot token got %v instead of %v`, got, expected)
+	}
+}
+
+func TestDiscordHelptext(t *testing.T) {
+	expected := "some text"
+
+	os.Clearenv()
+	os.Setenv("WAYBACK_DISCORD_HELPTEXT", expected)
+
+	parser := NewParser()
+	opts, err := parser.ParseEnvironmentVariables()
+	if err != nil {
+		t.Fatalf(`Parsing environment variables failed: %v`, err)
+	}
+
+	got := opts.DiscordHelptext()
+	if got != expected {
+		t.Fatalf(`Unexpected Discord help text got %v instead of %v`, got, expected)
+	}
+}
+
+func TestDiscordChannel(t *testing.T) {
+	t.Parallel()
+
+	var tests = []struct {
+		channel, expected string
+	}{
+		{
+			channel:  "",
+			expected: "",
+		},
+		{
+			channel:  "865981235815140000",
+			expected: "865981235815140000",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.channel, func(t *testing.T) {
+			os.Clearenv()
+			os.Setenv("WAYBACK_DISCORD_CHANNEL", test.channel)
+
+			parser := NewParser()
+			opts, err := parser.ParseEnvironmentVariables()
+			if err != nil {
+				t.Fatalf(`Parsing environment variables failed: %v`, err)
+			}
+
+			got := opts.DiscordChannel()
+			if got != test.expected {
+				t.Fatalf(`Unexpected Discord channel got %v instead of %v`, got, test.expected)
+			}
+		})
+	}
+}
+
+func TestPublishToDiscordChannel(t *testing.T) {
+	os.Clearenv()
+	os.Setenv("WAYBACK_DISCORD_BOT_TOKEN", "discord-bot-token")
+	os.Setenv("WAYBACK_DISCORD_CHANNEL", "865981235815140000")
+
+	parser := NewParser()
+	opts, err := parser.ParseEnvironmentVariables()
+	if err != nil {
+		t.Fatalf(`Parsing environment variables failed: %v`, err)
+	}
+
+	expected := true
+	got := opts.PublishToDiscordChannel()
+
+	if got != expected {
+		t.Fatalf(`Unexpected publish to Discord channel got %t instead of %v`, got, expected)
+	}
+}
+
 func TestEnabledChromeRemote(t *testing.T) {
 	addr := "127.0.0.1:1234"
 

config/options.go

@@ -42,6 +42,9 @@ const (
 	defTwitterConsumerSecret = ""
 	defTwitterAccessToken    = ""
 	defTwitterAccessSecret   = ""
+	defDiscordBotToken       = ""
+	defDiscordChannel        = ""
+	defDiscordHelptext       = ""
 
 	defIRCNick     = ""
 	defIRCPassword = ""
@@ -80,6 +83,7 @@ type Options struct {
 	slots    map[string]bool
 	telegram *telegram
 	mastodon *mastodon
+	discord  *discord
 	twitter  *twitter
 	github   *github
 	matrix   *matrix
@@ -113,6 +117,13 @@ type mastodon struct {
 	accessToken  string
 }
 
+type discord struct {
+	appID    string
+	botToken string
+	channel  string
+	helptext string
+}
+
 type twitter struct {
 	consumerKey    string
 	consumerSecret string
@@ -184,6 +195,12 @@ func NewOptions() *Options {
 			clientSecret: defMastodonClientSecret,
 			accessToken:  defMastodonAccessToken,
 		},
+		discord: &discord{
+			appID:    defDiscordBotToken,
+			botToken: defDiscordBotToken,
+			channel:  defDiscordChannel,
+			helptext: defDiscordHelptext,
+		},
 		twitter: &twitter{
 			consumerKey:    defTwitterConsumerKey,
 			consumerSecret: defTwitterConsumerSecret,
@@ -297,12 +314,7 @@ func (o *Options) TelegramChannel() string {
 
 // TelegramHelptext returns the help text for Telegram bot.
 func (o *Options) TelegramHelptext() string {
-	o.telegram.helptext = strings.ReplaceAll(o.telegram.helptext, `\r`, "\n")
-	o.telegram.helptext = strings.ReplaceAll(o.telegram.helptext, `\n`, "\n")
-	o.telegram.helptext = strings.ReplaceAll(o.telegram.helptext, `\r\n`, "\n")
-	o.telegram.helptext = strings.ReplaceAll(o.telegram.helptext, `<br>`, "\n")
-	o.telegram.helptext = strings.ReplaceAll(o.telegram.helptext, `<br/>`, "\n")
-	return o.telegram.helptext
+	return breakLine(o.telegram.helptext)
 }
 
 // PublishToChannel returns whether to publish results to Telegram Channel.
@@ -461,6 +473,32 @@ func (o *Options) PublishToMatrixRoom() bool {
 		o.MatrixPassword() != ""
 }
 
+// DiscordBotToken returns the token of Discord bot
+func (o *Options) DiscordBotToken() string {
+	return o.discord.botToken
+}
+
+// DiscordChannel returns the channel on Discord.
+func (o *Options) DiscordChannel() string {
+	// if strings.HasPrefix(o.discord.channel, "#") {
+	// 	return o.discord.channel
+	// }
+	// if o.discord.channel != "" {
+	// 	return "#" + o.discord.channel
+	// }
+	return o.discord.channel
+}
+
+// DiscordHelptext returns the help text for Discord bot
+func (o *Options) DiscordHelptext() string {
+	return breakLine(o.discord.helptext)
+}
+
+// PublishToDiscordChannel returns whether publish results to Discord channel.
+func (o *Options) PublishToDiscordChannel() bool {
+	return o.DiscordBotToken() != "" && o.DiscordChannel() != ""
+}
+
 // TorPrivKey returns the private key of Tor service.
 func (o *Options) TorPrivKey() string {
 	return o.tor.pvk
@@ -526,6 +564,7 @@ func (o *Options) MaxMediaSize() uint64 {
 func (o *Options) MaxAttachSize(scope string) int64 {
 	scopes := map[string]int64{
 		"telegram": 50000000, // 50MB
+		"discord":  8000000,  // 8MB
 	}
 	return scopes[scope]
 }

config/parser.go

@@ -151,6 +151,12 @@ func (p *Parser) parseLines(lines []string) (err error) {
 			p.opts.matrix.roomID = parseString(val, defMatrixRoomID)
 		case "WAYBACK_MATRIX_PASSWORD":
 			p.opts.matrix.password = parseString(val, defMatrixPassword)
+		case "WAYBACK_DISCORD_BOT_TOKEN":
+			p.opts.discord.botToken = parseString(val, defDiscordBotToken)
+		case "WAYBACK_DISCORD_CHANNEL":
+			p.opts.discord.channel = parseString(val, defDiscordChannel)
+		case "WAYBACK_DISCORD_HELPTEXT":
+			p.opts.discord.helptext = parseString(val, defDiscordHelptext)
 		case "WAYBACK_TOR_PRIVKEY":
 			p.opts.tor.pvk = parseString(val, defTorPrivateKey)
 		case "WAYBACK_TOR_LOCAL_PORT":
@@ -241,3 +247,12 @@ func defaultFilenames() []string {
 		filepath.Join("/etc", name),
 	}
 }
+
+func breakLine(s string) string {
+	s = strings.ReplaceAll(s, `\r`, "\n")
+	s = strings.ReplaceAll(s, `\n`, "\n")
+	s = strings.ReplaceAll(s, `\r\n`, "\n")
+	s = strings.ReplaceAll(s, `<br>`, "\n")
+	s = strings.ReplaceAll(s, `<br/>`, "\n")
+	return s
+}

go.mod

@@ -7,6 +7,7 @@ go 1.16
 require (
 	github.com/PuerkitoBio/goquery v1.7.1 // indirect
 	github.com/btcsuite/btcd v0.22.0-beta // indirect
+	github.com/bwmarrin/discordgo v0.23.3-0.20210627161652-421e14965030
 	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/cretz/bine v0.2.0
 	github.com/davecgh/go-spew v1.1.1
@@ -19,6 +20,7 @@ require (
 	github.com/google/go-github/v37 v37.0.0
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/mux v1.8.0
+	github.com/gorilla/websocket v1.4.2
 	github.com/iawia002/annie v0.0.0-20210714033020-3c20ed1e6f07
 	github.com/klauspost/cpuid/v2 v2.0.8 // indirect
 	github.com/kr/pretty v0.2.1 // indirect

go.sum

@@ -93,6 +93,8 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku
 github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
 github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
+github.com/bwmarrin/discordgo v0.23.3-0.20210627161652-421e14965030 h1:JgAlPJlCqsrUqXcClndnvF6LOjV528hpU/l5YzN++PA=
+github.com/bwmarrin/discordgo v0.23.3-0.20210627161652-421e14965030/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY=
 github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
@@ -663,6 +665,7 @@ golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210415154028-4f45737414dc/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI=

metrics/metrics.go

@@ -19,6 +19,7 @@ const (
 	ServiceIRC      = "irc"
 	ServiceWeb      = "web"
 	ServiceMatrix   = "matrix"
+	ServiceDiscord  = "discord"
 	ServiceMastodon = "mastodon"
 	ServiceTelegram = "telegram"
 	ServiceTwitter  = "twitter"
@@ -28,6 +29,7 @@ const (
 	PublishMstdn   = "mastodon" // Mastodon toot
 	PublishGithub  = "github"   // GitHub issues
 	PublishMatrix  = "room"
+	PublishDiscord = "discord" // Discord channel
 	PublishTwitter = "tweet"
 
 	StatusRequest = "request"