Integration with Ghostarchive (#497)

* Add support Ghostarchive

* Update minimal go version for testing workflow

* Bump minimal go version for Dockerfile

* Update dependencies

* Remove toolchain directive

* Make testing workflow green again

.github/workflows/testing.yml

@@ -46,7 +46,7 @@ jobs:
         # The step `Set up Chrome` do not support arm64 yet, so specified s version to use amd64 arch
         # Refs to https://github.com/actions/runner-images#available-images
         os: [ ubuntu-24.04, macos-13, windows-2022 ]
-        go: [ "1.19", "1.20", "1.21", "1.22" ]
+        go: [ "1.21", "1.22" ]
         include:
           # only update test coverage stats with the most recent go version on linux
           - go: 1.x

Dockerfile

@@ -3,7 +3,7 @@
 # license that can be found in the LICENSE file.
 #
 # syntax=docker/dockerfile:1.2
-ARG GO_VERSION=1.20
+ARG GO_VERSION=1.22
 
 FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS builder
 COPY --from=tonistiigi/xx:golang / /

build/docker/Dockerfile.all

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1.2
-ARG GO_VERSION=1.20
+ARG GO_VERSION=1.22
 
 FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS builder
 COPY --from=tonistiigi/xx:golang / /

build/docker/Dockerfile.dev

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1.2
-ARG GO_VERSION=1.20
+ARG GO_VERSION=1.22
 
 FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS builder
 

build/redhat/Dockerfile

@@ -2,7 +2,7 @@
 # Use of this source code is governed by the GNU GPL v3
 # license that can be found in the LICENSE file.
 #
-FROM golang:1.20-alpine AS builder
+FROM golang:1.22-alpine AS builder
 
 ARG WAYBACK_IPFS_APIKEY
 

cmd/wayback/main.go

@@ -26,6 +26,7 @@ var (
 	is bool
 	ip bool
 	ph bool
+	ga bool
 
 	daemon []string
 
@@ -73,6 +74,7 @@ func init() {
 	rootCmd.Flags().BoolVarP(&is, "is", "", true, "Wayback webpages to Archive Today")
 	rootCmd.Flags().BoolVarP(&ip, "ip", "", true, "Wayback webpages to IPFS")
 	rootCmd.Flags().BoolVarP(&ph, "ph", "", true, "Wayback webpages to Telegraph")
+	rootCmd.Flags().BoolVarP(&ga, "ga", "", true, "Wayback webpages to Ghostarchive")
 	rootCmd.Flags().StringSliceVarP(&daemon, "daemon", "d", []string{}, "Run as daemon service, supported services are telegram, web, mastodon, twitter, discord, slack, irc")
 	rootCmd.Flags().StringVarP(&host, "ipfs-host", "", "127.0.0.1", "IPFS daemon host, do not require, unless enable ipfs")
 	rootCmd.Flags().UintVarP(&port, "ipfs-port", "p", 5001, "IPFS daemon port")
@@ -128,6 +130,9 @@ func setToEnv(cmd *cobra.Command) {
 	if flags.Changed("ph") {
 		os.Setenv("WAYBACK_ENABLE_PH", fmt.Sprint(ph))
 	}
+	if flags.Changed("ga") {
+		os.Setenv("WAYBACK_ENABLE_GA", fmt.Sprint(ga))
+	}
 	if flags.Changed("token") {
 		os.Setenv("WAYBACK_TELEGRAM_TOKEN", token)
 	}
@@ -153,11 +158,12 @@ func setToEnv(cmd *cobra.Command) {
 
 // nolint:gocyclo
 func run(cmd *cobra.Command, args []string) {
-	if !ia && !is && !ip && !ph {
+	if !ia && !is && !ip && !ph && !ga {
 		ia, is, ip = true, true, true
 		os.Setenv("WAYBACK_ENABLE_IA", "true")
 		os.Setenv("WAYBACK_ENABLE_IS", "true")
 		os.Setenv("WAYBACK_ENABLE_IP", "true")
+		os.Setenv("WAYBACK_ENABLE_GA", "true")
 	}
 
 	setToEnv(cmd)

config/config.go

@@ -13,6 +13,7 @@ const (
 	SLOT_IS = "is" // archive.today
 	SLOT_IP = "ip" // IPFS
 	SLOT_PH = "ph" // Telegraph
+	SLOT_GA = "ga" // Ghostarchive
 	SLOT_TT = "tt" // Time Travel
 	SLOT_GC = "gc" // Google Cache
 
@@ -68,6 +69,7 @@ func SlotName(s string) string {
 		SLOT_IS: "archive.today",
 		SLOT_IP: "IPFS",
 		SLOT_PH: "Telegraph",
+		SLOT_GA: "Ghostarchive",
 		SLOT_TT: "Time Travel",
 		SLOT_GC: "Google Cache",
 	}
@@ -86,6 +88,7 @@ func SlotExtra(s string) string {
 		SLOT_IS: "https://archive.today/",
 		SLOT_IP: "https://ipfs.github.io/public-gateway-checker/",
 		SLOT_PH: "https://telegra.ph/",
+		SLOT_GA: "https://ghostarchive.org/",
 		SLOT_TT: "http://timetravel.mementoweb.org/",
 		SLOT_GC: "https://webcache.googleusercontent.com/",
 	}

config/config_test.go

@@ -397,6 +397,7 @@ func TestEnableSlots(t *testing.T) {
 	os.Setenv("WAYBACK_ENABLE_IS", "true")
 	os.Setenv("WAYBACK_ENABLE_IP", "true")
 	os.Setenv("WAYBACK_ENABLE_PH", "true")
+	os.Setenv("WAYBACK_ENABLE_GA", "true")
 
 	parser := NewParser()
 	opts, err := parser.ParseEnvironmentVariables()
@@ -409,10 +410,11 @@ func TestEnableSlots(t *testing.T) {
 		SLOT_IS: true,
 		SLOT_IP: true,
 		SLOT_PH: true,
+		SLOT_GA: true,
 	}
 	got := opts.Slots()
 
-	if got == nil || !got[SLOT_IA] || !got[SLOT_IS] || !got[SLOT_IP] || !got[SLOT_PH] {
+	if got == nil || !got[SLOT_IA] || !got[SLOT_IS] || !got[SLOT_IP] || !got[SLOT_PH] || !got[SLOT_GA] {
 		t.Fatalf(`Unexpected default slots, got %v instead of %v`, got, expected)
 	}
 }
@@ -431,6 +433,7 @@ func TestDefaultSlots(t *testing.T) {
 		SLOT_IS: true,
 		SLOT_IP: true,
 		SLOT_PH: true,
+		SLOT_GA: true,
 	}
 	got := opts.Slots()
 

config/options.go

@@ -34,6 +34,7 @@ const (
 	defEnabledIS = true
 	defEnabledIP = true
 	defEnabledPH = true
+	defEnabledGA = true
 
 	defTelegramToken    = ""
 	defTelegramChannel  = ""
@@ -286,6 +287,7 @@ func NewOptions() *Options {
 			SLOT_IS: defEnabledIS,
 			SLOT_IP: defEnabledIP,
 			SLOT_PH: defEnabledPH,
+			SLOT_GA: defEnabledGA,
 		},
 		telegram: &telegram{
 			token:    defTelegramToken,

docker-bake.hcl

@@ -1,6 +1,6 @@
 // Go version
 variable "GO_VERSION" {
-  default = "1.20"
+  default = "1.22"
 }
 
 variable "WAYBACK_IPFS_APIKEY" {

go.mod

@@ -1,11 +1,11 @@
 module github.com/wabarc/wayback
 
-// +heroku goVersion go1.20
+// +heroku goVersion go1.21
 
-go 1.20
+go 1.21
 
 require (
-	github.com/PuerkitoBio/goquery v1.8.1
+	github.com/PuerkitoBio/goquery v1.9.0
 	github.com/bwmarrin/discordgo v0.23.3-0.20210627161652-421e14965030
 	github.com/cretz/bine v0.2.0
 	github.com/davecgh/go-spew v1.1.1
@@ -28,28 +28,29 @@ require (
 	github.com/nbd-wtf/go-nostr v0.17.1-0.20230426111250-32ca737acf77
 	github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.18.0
-	github.com/prometheus/common v0.46.0
+	github.com/prometheus/client_golang v1.19.1
+	github.com/prometheus/common v0.48.0
 	github.com/rs/xid v1.4.0
 	github.com/slack-go/slack v0.11.2
 	github.com/spf13/cobra v1.6.1
 	github.com/wabarc/archive.is v1.4.0
 	github.com/wabarc/archive.org v1.2.1-0.20210708220121-cb9b83ff9896
+	github.com/wabarc/ghostarchive v0.1.1
 	github.com/wabarc/go-anonfile v0.1.0
 	github.com/wabarc/go-catbox v0.1.0
 	github.com/wabarc/helper v0.0.0-20230418130954-be7440352bcb
 	github.com/wabarc/imgbb v1.0.0
 	github.com/wabarc/ipfs-pinner v1.1.1-0.20230502052510-dc378f9e202b
 	github.com/wabarc/logger v0.0.0-20210730133522-86bd3f31e792
-	github.com/wabarc/playback v0.0.0-20230331122619-84484ab4d599
+	github.com/wabarc/playback v0.0.0-20240229124108-a6ddc935de3c
 	github.com/wabarc/proxier v0.0.0-20230610135141-b55fe1536465
 	github.com/wabarc/rivet v0.1.4-0.20230505152228-2c5c81b4bd10
 	github.com/wabarc/screenshot v1.6.1-0.20240214000834-3820163034f4
 	github.com/wabarc/telegra.ph v0.0.0-20230318134541-a0922e1ace3a
 	github.com/wabarc/warcraft v0.3.1-0.20230308125707-3daa5592ba52
 	go.etcd.io/bbolt v1.3.6
-	golang.org/x/net v0.21.0
-	golang.org/x/sync v0.6.0
+	golang.org/x/net v0.26.0
+	golang.org/x/sync v0.7.0
 	gopkg.in/irc.v4 v4.0.0
 	gopkg.in/telebot.v3 v3.0.0-20220130115853-f0291132d3c3
 	maunium.net/go/mautrix v0.12.0
@@ -62,7 +63,7 @@ require (
 	github.com/MercuryEngineering/CookieMonster v0.0.0-20180304172713-1584578b3403 // indirect
 	github.com/SaveTheRbtz/generic-sync-map-go v0.0.0-20230201052002-6c5833b989be // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
-	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/andybalholm/brotli v1.1.0 // indirect
 	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -77,6 +78,7 @@ require (
 	github.com/chromedp/cdproto v0.0.0-20240202021202-6d0b6a386732 // indirect
 	github.com/chromedp/chromedp v0.9.5 // indirect
 	github.com/chromedp/sysutil v1.0.0 // indirect
+	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
 	github.com/decred/dcrd/crypto/blake256 v1.0.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
@@ -85,15 +87,14 @@ require (
 	github.com/dop251/goja v0.0.0-20221115122301-6c0d9883792e // indirect
 	github.com/fatih/color v1.16.0 // indirect
 	github.com/fortytw2/leaktest v1.3.0 // indirect
-	github.com/gaukas/godicttls v0.0.3 // indirect
 	github.com/go-shiori/dom v0.0.0-20210627111528-4e4722cd0d65 // indirect
 	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
 	github.com/gobwas/httphead v0.1.0 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/gobwas/ws v1.3.2 // indirect
 	github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/google/pprof v0.0.0-20240528025155-186aa0362fba // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/iawia002/lia v0.0.0-20221116085912-1f653221be4b // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
@@ -107,7 +108,7 @@ require (
 	github.com/kallydev/telegraph-go v1.0.1-0.20230318133700-df034d9eed50 // indirect
 	github.com/kennygrant/sanitize v1.2.4 // indirect
 	github.com/kkdai/youtube/v2 v2.7.18 // indirect
-	github.com/klauspost/compress v1.17.6 // indirect
+	github.com/klauspost/compress v1.17.7 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
@@ -132,9 +133,12 @@ require (
 	github.com/multiformats/go-multistream v0.5.0 // indirect
 	github.com/multiformats/go-varint v0.0.7 // indirect
 	github.com/oliamb/cutter v0.2.2 // indirect
+	github.com/onsi/ginkgo/v2 v2.19.0 // indirect
+	github.com/onsi/gomega v1.33.1 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
-	github.com/refraction-networking/utls v1.3.2 // indirect
+	github.com/quic-go/quic-go v0.45.0 // indirect
+	github.com/refraction-networking/utls v1.6.3 // indirect
 	github.com/rivo/uniseg v0.4.3 // indirect
 	github.com/robertkrimen/otto v0.0.0-20211024170158-b87d35c0b86f // indirect
 	github.com/rogpeppe/go-internal v1.10.0 // indirect
@@ -151,14 +155,14 @@ require (
 	github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b // indirect
 	github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
 	github.com/ybbus/httpretry v1.0.2 // indirect
-	golang.org/x/crypto v0.19.0 // indirect
-	golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 // indirect
-	golang.org/x/mod v0.15.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
-	golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
-	golang.org/x/tools v0.17.0 // indirect
-	google.golang.org/protobuf v1.32.0 // indirect
+	golang.org/x/crypto v0.24.0 // indirect
+	golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect
+	golang.org/x/mod v0.18.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	golang.org/x/tools v0.22.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/sourcemap.v1 v1.0.5 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	lukechampine.com/blake3 v1.2.1 // indirect