Propagating error upstream + Random eviction Fix

.github/workflows/build-test.yml

@@ -40,4 +40,8 @@ jobs:
 
       - name: Race Condition Tests
         run: go build -race ./...
-        working-directory: .
+        working-directory: .
+
+      - name: Example Code Tests
+        run: go build .
+        working-directory: examples/

README.md

@@ -787,44 +787,7 @@ On default settings, the daemon listens on the following ports:
 
 ### Use as library
 
-The below example uses interactsh client library to get external interactions for a generated URL by making a http request to the URL.
-
-```go
-package main
-
-import (
-	"fmt"
-	"net/http"
-	"time"
-
-	"github.com/projectdiscovery/interactsh/pkg/client"
-	"github.com/projectdiscovery/interactsh/pkg/server"
-)
-
-func main() {
-	client, err := client.New(client.DefaultOptions)
-	if err != nil {
-		panic(err)
-	}
-	defer client.Close()
-
-	client.StartPolling(time.Duration(1*time.Second), func(interaction *server.Interaction) {
-		fmt.Printf("Got Interaction: %v => %v\n", interaction.Protocol, interaction.FullId)
-	})
-	defer client.StopPolling()
-
-	URL := client.URL()
-
-	resp, err := http.Get("https://" + URL)
-	if err != nil {
-		panic(err)
-	}
-	resp.Body.Close()
-
-	fmt.Printf("Got URL: %v => %v\n", URL, resp)
-	time.Sleep(5 * time.Second)
-}
-```
+The [examples](examples/) uses interactsh client library to get external interactions for a generated URL by making a http request to the URL.
 
 ### Nuclei - OAST
 

cmd/interactsh-client/main.go

@@ -50,6 +50,7 @@ func main() {
 		flagSet.IntVarP(&cliOptions.CorrelationIdLength, "correlation-id-length", "cidl", settings.CorrelationIdLengthDefault, "length of the correlation id preamble"),
 		flagSet.IntVarP(&cliOptions.CorrelationIdNonceLength, "correlation-id-nonce-length", "cidn", settings.CorrelationIdNonceLengthDefault, "length of the correlation id nonce"),
 		flagSet.StringVarP(&cliOptions.SessionFile, "session-file", "sf", "", "store/read from session file"),
+		flagSet.DurationVarP(&cliOptions.KeepAliveInterval, "keep-alive-interval", "kai", time.Minute, "keep alive interval"),
 	)
 
 	flagSet.CreateGroup("filter", "Filter",
@@ -158,7 +159,7 @@ func main() {
 		}
 	}
 
-	_ = client.StartPolling(time.Duration(cliOptions.PollInterval)*time.Second, func(interaction *server.Interaction) {
+	err = client.StartPolling(time.Duration(cliOptions.PollInterval)*time.Second, func(interaction *server.Interaction) {
 		if matcher != nil && !matcher.match(interaction.FullId) {
 			return
 		}
@@ -237,6 +238,9 @@ func main() {
 			}
 		}
 	})
+	if err != nil {
+		gologger.Error().Msgf(err.Error())
+	}
 
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, os.Interrupt)

deploy/deploy.yaml

@@ -74,6 +74,8 @@
       name: "{{ container_name }}"
       image: "{{ container_image }}"
       command: "{{ container_command }}"
+      memory: "4g"
+      memory_swap: "-1"
       network_mode: host
       restart: true
       restart_policy: "unless-stopped"

examples/client.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/projectdiscovery/interactsh/pkg/client"
+	"github.com/projectdiscovery/interactsh/pkg/server"
+)
+
+func main() {
+	client, err := client.New(client.DefaultOptions)
+	if err != nil {
+		panic(err)
+	}
+	defer client.Close()
+
+	client.StartPolling(time.Duration(1*time.Second), func(interaction *server.Interaction) {
+		fmt.Printf("Got Interaction: %v => %v\n", interaction.Protocol, interaction.FullId)
+	})
+	defer client.StopPolling()
+
+	URL := client.URL()
+
+	resp, err := http.Get("https://" + URL)
+	if err != nil {
+		panic(err)
+	}
+	resp.Body.Close()
+
+	fmt.Printf("Got URL: %v => %v\n", URL, resp)
+	time.Sleep(1 * time.Second)
+}

go.mod

@@ -11,7 +11,6 @@ require (
 	github.com/goburrow/cache v0.1.4
 	github.com/google/uuid v1.3.0
 	github.com/json-iterator/go v1.1.12
-	github.com/karlseguin/ccache/v2 v2.0.8
 	github.com/libdns/libdns v0.2.1
 	github.com/mackerelio/go-osstat v0.2.4
 	github.com/miekg/dns v1.1.54

go.sum

@@ -78,10 +78,6 @@ github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/karlseguin/ccache/v2 v2.0.8 h1:lT38cE//uyf6KcFok0rlgXtGFBWxkI6h/qg4tbFyDnA=
-github.com/karlseguin/ccache/v2 v2.0.8/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ=
-github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLMY72r5J4sEfkuE7AFbixEP2qRbEcum/wA=
-github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8=
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid/v2 v2.1.1 h1:t0wUqjowdm8ezddV5k0tLWVklVuvLJpoHeb4WBdydm0=
@@ -198,8 +194,6 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP
 github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ=
 github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
-github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
-github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

pkg/client/client.go

@@ -19,6 +19,7 @@ import (
 	"net/url"
 	"os"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -57,6 +58,7 @@ const (
 
 // Client is a client for communicating with interactsh server instance.
 type Client struct {
+	busy                     sync.RWMutex
 	State                    atomic.Value
 	correlationID            string
 	secretKey                string
@@ -65,6 +67,7 @@ type Client struct {
 	privKey                  *rsa.PrivateKey
 	pubKey                   *rsa.PublicKey
 	quitChan                 chan struct{}
+	quitKeepAliveChan        chan struct{}
 	disableHTTPFallback      bool
 	token                    string
 	correlationIdLength      int
@@ -87,6 +90,8 @@ type Options struct {
 	HTTPClient *retryablehttp.Client
 	// SessionInfo to resume an existing session
 	SessionInfo *options.SessionInfo
+	// keepAliveInterval to renew the session
+	KeepAliveInterval time.Duration
 }
 
 // DefaultOptions is the default options for the interact client
@@ -171,6 +176,38 @@ func New(options *Options) (*Client, error) {
 		}
 	}
 
+	// start a keep alive routine
+	client.quitKeepAliveChan = make(chan struct{})
+	if options.KeepAliveInterval > 0 {
+		ticker := time.NewTicker(options.KeepAliveInterval)
+		go func() {
+			for {
+				// exit if the client is closed
+				if client.State.Load() == Closed {
+					return
+				}
+				select {
+				case <-ticker.C:
+					// todo: internal logic needs a complete redesign
+					pubKeyData, err := encodePublicKey(client.pubKey)
+					if err != nil {
+						return
+					}
+					// attempts to re-register - server will reject is already existing
+					registrationRequest, err := encodeRegistrationRequest(pubKeyData, client.secretKey, client.correlationID)
+					if err != nil {
+						return
+					}
+					// silently fails to re-register if the session is still alive
+					_ = client.performRegistration(client.serverURL.String(), registrationRequest)
+				case <-client.quitKeepAliveChan:
+					ticker.Stop()
+					return
+				}
+			}
+		}()
+	}
+
 	return client, nil
 }
 
@@ -305,7 +342,7 @@ func removeIndex(s []string, index int) []string {
 // InteractionCallback is a callback function for a reported interaction
 type InteractionCallback func(*server.Interaction)
 
-// StartPolling starts polling the server each duration and returns any events
+// StartPolling the server each duration and returns any events
 // that may have been captured by the collaborator server.
 func (c *Client) StartPolling(duration time.Duration, callback InteractionCallback) error {
 	switch c.State.Load() {
@@ -347,6 +384,9 @@ func (c *Client) StartPolling(duration time.Duration, callback InteractionCallba
 
 // getInteractions returns the interactions from the server.
 func (c *Client) getInteractions(callback InteractionCallback) error {
+	c.busy.RLock()
+	defer c.busy.RUnlock()
+
 	builder := &strings.Builder{}
 	builder.WriteString(c.serverURL.String())
 	builder.WriteString("/poll?id=")
@@ -455,8 +495,11 @@ func (c *Client) TryGetAsnInfo(interaction *server.Interaction) error {
 	return nil
 }
 
-// StopPolling stops the polling to the interactsh server.
+// StopPolling the interactsh server.
 func (c *Client) StopPolling() error {
+	c.busy.Lock()
+	defer c.busy.Unlock()
+
 	if c.State.Load() != Polling {
 		return errors.New("client is not polling")
 	}
@@ -470,13 +513,18 @@ func (c *Client) StopPolling() error {
 // Close closes the collaborator client and deregisters from the
 // collaborator server if not explicitly asked by the user.
 func (c *Client) Close() error {
+	c.busy.Lock()
+	defer c.busy.Unlock()
+
 	if c.State.Load() == Polling {
 		return errors.New("client should stop polling before closing")
 	}
 	if c.State.Load() == Closed {
 		return errors.New("client is already closed")
 	}
 
+	close(c.quitKeepAliveChan)
+
 	register := server.DeregisterRequest{
 		CorrelationID: c.correlationID,
 		SecretKey:     c.secretKey,

pkg/options/client_options.go

@@ -1,6 +1,10 @@
 package options
 
-import "github.com/projectdiscovery/goflags"
+import (
+	"time"
+
+	"github.com/projectdiscovery/goflags"
+)
 
 type CLIClientOptions struct {
 	Match                    goflags.StringSlice
@@ -23,4 +27,5 @@ type CLIClientOptions struct {
 	SessionFile              string
 	Asn                      bool
 	DisableUpdateCheck       bool
+	KeepAliveInterval        time.Duration
 }

pkg/storage/storagedb_test.go

@@ -15,7 +15,6 @@ import (
 
 	"github.com/goburrow/cache"
 	"github.com/google/uuid"
-	"github.com/karlseguin/ccache/v2"
 	"github.com/rs/xid"
 	"github.com/stretchr/testify/require"
 )
@@ -118,17 +117,6 @@ func TestStorageAddGetInteractions(t *testing.T) {
 	require.Equal(t, dataOriginal, decoded, "could not get correct decrypted interaction")
 }
 
-func BenchmarkCacheParallel(b *testing.B) {
-	config := ccache.Configure().MaxSize(int64(DefaultOptions.MaxSize)).Buckets(64).GetsPerPromote(10).PromoteBuffer(4096)
-	cache := ccache.New(config)
-
-	b.RunParallel(func(pb *testing.PB) {
-		for pb.Next() {
-			doStuffWithCache(cache)
-		}
-	})
-}
-
 func BenchmarkCacheParallelOther(b *testing.B) {
 	cache := cache.New(cache.WithMaximumSize(DefaultOptions.MaxSize), cache.WithExpireAfterWrite(24*7*time.Hour))
 
@@ -139,13 +127,6 @@ func BenchmarkCacheParallelOther(b *testing.B) {
 	})
 }
 
-func doStuffWithCache(cache *ccache.Cache) {
-	for i := 0; i < 1e2; i++ {
-		cache.Set(strconv.Itoa(i), "test", 1*time.Minute)
-		_ = cache.Get(strconv.Itoa(i))
-	}
-}
-
 func doStuffWithOtherCache(cache cache.Cache) {
 	for i := 0; i < 1e2; i++ {
 		cache.Put(strconv.Itoa(i), "test")