feat(GODT-1917): Expunge benchmark

This benchmark tries to delete random sequence ids or intervals from
separate mailboxes until the desired message count has been deleted.

An alternate mode is also available where all parallel workers delete
from the same inbox rather than from distinct ones.

benchmarks/gluon_bench/benchmarks/expunge.go

@@ -0,0 +1,159 @@
+package benchmarks
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"github.com/bradenaw/juniper/xslices"
+	"github.com/google/uuid"
+	"net"
+
+	"github.com/ProtonMail/gluon/benchmarks/gluon_bench/flags"
+	"github.com/ProtonMail/gluon/benchmarks/gluon_bench/utils"
+	"github.com/emersion/go-imap"
+	"github.com/emersion/go-imap/client"
+)
+
+var (
+	expungeCountFlag    = flag.Uint("expunge-count", 0, "Total number of messages to expunge during expunge benchmarks.")
+	expungeSameMBoxFlag = flag.Bool("expunge-same-mbox", false, "When true run all the expunge test on the same inbox rather than separate ones in parallel.")
+	expungeListFlag     = flag.String("expunge-list", "", "Use a list of predefined sequences to expunge rather than random generated. Only works when -expunge-same-mbox is not set.")
+	expungeAllFlag      = flag.Bool("expunge-all", false, "If set, perform a expunge of the all messages. Only works when -expunge-same-mbox is not set.")
+)
+
+type Expunge struct {
+	seqSets   *ParallelSeqSet
+	mailboxes []string
+}
+
+func NewExpunge() *Expunge {
+	return &Expunge{}
+}
+
+func (*Expunge) Name() string {
+	return "expunge"
+}
+
+func (e *Expunge) Setup(ctx context.Context, addr net.Addr) error {
+	cl, err := utils.NewClient(addr.String())
+	if err != nil {
+		return err
+	}
+
+	defer utils.CloseClient(cl)
+
+	if *expungeSameMBoxFlag {
+		if err := utils.FillBenchmarkSourceMailbox(cl); err != nil {
+			return err
+		}
+
+		status, err := cl.Status(*flags.Mailbox, []imap.StatusItem{imap.StatusMessages})
+		if err != nil {
+			return err
+		}
+
+		messageCount := status.Messages
+
+		if messageCount == 0 {
+			return fmt.Errorf("mailbox '%v' has no messages", *flags.Mailbox)
+		}
+
+		expungeCount := uint32(*expungeCountFlag)
+		if expungeCount == 0 {
+			expungeCount = messageCount / 2
+		}
+
+		e.seqSets = NewParallelSeqSetExpunge(expungeCount,
+			*flags.ParallelClients,
+			*flags.RandomSeqSetIntervals,
+			*flags.UIDMode,
+		)
+
+		for i := uint(0); i < *flags.ParallelClients; i++ {
+			e.mailboxes = append(e.mailboxes, *flags.Mailbox)
+		}
+	} else {
+		e.mailboxes = make([]string, 0, *flags.ParallelClients)
+		for i := uint(0); i < *flags.ParallelClients; i++ {
+			e.mailboxes = append(e.mailboxes, uuid.NewString())
+		}
+
+		for _, v := range e.mailboxes {
+			if err := cl.Create(v); err != nil {
+				return err
+			}
+
+			if err := utils.BuildMailbox(cl, v, int(*flags.FillSourceMailbox)); err != nil {
+				return err
+			}
+		}
+
+		seqSets, err := NewParallelSeqSet(uint32(*flags.FillSourceMailbox),
+			*flags.ParallelClients,
+			*expungeListFlag,
+			*expungeAllFlag,
+			*flags.RandomSeqSetIntervals,
+			true,
+			*flags.UIDMode)
+
+		if err != nil {
+			return err
+		}
+
+		e.seqSets = seqSets
+	}
+
+	return nil
+}
+
+func (e *Expunge) TearDown(ctx context.Context, addr net.Addr) error {
+	cl, err := utils.NewClient(addr.String())
+	if err != nil {
+		return err
+	}
+
+	defer utils.CloseClient(cl)
+
+	if !*expungeSameMBoxFlag {
+		for _, v := range e.mailboxes {
+			if err := cl.Delete(v); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (e *Expunge) Run(ctx context.Context, addr net.Addr) error {
+	mboxInfo := xslices.Map(e.mailboxes, func(m string) utils.MailboxInfo {
+		return utils.MailboxInfo{Name: m, ReadOnly: false}
+	})
+
+	utils.RunParallelClientsWithMailboxes(addr, mboxInfo, func(cl *client.Client, index uint) {
+		var expungeFn func(*client.Client, *imap.SeqSet) error
+		if *flags.UIDMode {
+			expungeFn = func(cl *client.Client, set *imap.SeqSet) error {
+				if err := utils.UIDStore(cl, set, "+FLAGS", true, imap.DeletedFlag); err != nil {
+					return err
+				}
+				return cl.Expunge(nil)
+			}
+		} else {
+			expungeFn = func(cl *client.Client, set *imap.SeqSet) error {
+				if err := utils.Store(cl, set, "+FLAGS", true, imap.DeletedFlag); err != nil {
+					return err
+				}
+				return cl.Expunge(nil)
+			}
+		}
+
+		for _, v := range e.seqSets.Get(index) {
+			if err := expungeFn(cl, v); err != nil {
+				panic(fmt.Sprintf("Seq:%v err:%v", v, err))
+			}
+		}
+	})
+
+	return nil
+}

benchmarks/gluon_bench/benchmarks/move.go

@@ -4,9 +4,10 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"github.com/bradenaw/juniper/xslices"
 	"net"
 
+	"github.com/bradenaw/juniper/xslices"
+
 	"github.com/ProtonMail/gluon/benchmarks/gluon_bench/flags"
 	"github.com/ProtonMail/gluon/benchmarks/gluon_bench/utils"
 	"github.com/emersion/go-imap"

benchmarks/gluon_bench/benchmarks/parallel_seqset.go

@@ -36,6 +36,71 @@ func NewParallelSeqSetFromFile(path string, numWorkers uint) (*ParallelSeqSet, e
 	}, nil
 }
 
+// NewParallelSeqSetExpunge generates sequence ids or intervals that can be used in cases where the data is expunged
+// and/or moved from the original inbox. It also makes sure that concurrent workers can't overlap to avoid operations
+// on messages that no longer exist.
+func NewParallelSeqSetExpunge(count uint32, numWorkers uint, generateIntervals, uid bool) *ParallelSeqSet {
+	lists := make([][]*imap.SeqSet, numWorkers)
+	workerSplit := count / uint32(numWorkers)
+	available := make([]uint32, count)
+
+	for r := uint32(0); r < count; r++ {
+		available[r] = r + 1
+	}
+
+	for i := uint(0); i < numWorkers; i++ {
+		available := available[(uint32(i) * workerSplit):(uint32(i+1) * workerSplit)]
+		list := make([]*imap.SeqSet, 0, workerSplit)
+
+		if generateIntervals {
+			const maxIntervalRange = uint32(40)
+			for len(available) > 0 {
+				intervalRange := rand.Uint32() % maxIntervalRange
+				itemsLeft := uint32(len(available))
+				index := rand.Uint32() % itemsLeft
+
+				if index > intervalRange {
+					index -= intervalRange
+				} else {
+					index = 0
+				}
+
+				if index+intervalRange >= itemsLeft {
+					intervalRange = itemsLeft - index
+				}
+
+				seqSet := &imap.SeqSet{}
+				if uid {
+					seqSet.AddRange(available[index], available[index+intervalRange-1])
+				} else {
+					seqSet.AddRange(index+1, index+intervalRange)
+				}
+
+				list = append(list, seqSet)
+				available = append(available[:index], available[index+intervalRange:]...)
+			}
+		} else {
+			count := uint32(len(available))
+			for r := uint32(0); r < count; r++ {
+				index := rand.Uint32() % (count - r)
+				seqSet := &imap.SeqSet{}
+				if uid {
+					tmp := available[index]
+					available[index] = available[count-r-1]
+					seqSet.AddNum(tmp)
+				} else {
+					seqSet.AddNum(index)
+				}
+				list = append(list, seqSet)
+			}
+		}
+
+		lists[i] = list
+	}
+
+	return &ParallelSeqSet{seqSets: lists}
+}
+
 // NewParallelSeqSetRandom generates count random sequence set for each worker. If generateIntervals is set to true,
 // it will generate intervals rather than a single number. If randomDrain is set to true it will generate unique
 // values that eventually exhaust the problem space.

benchmarks/gluon_bench/main.go

@@ -21,6 +21,7 @@ var benches = []benchmarks.Benchmark{
 	benchmarks.NewCopy(),
 	benchmarks.NewMove(),
 	benchmarks.NewStore(),
+	benchmarks.NewExpunge(),
 }
 
 func main() {

benchmarks/gluon_bench/utils/client.go

@@ -3,14 +3,15 @@ package utils
 import (
 	"bufio"
 	"fmt"
-	"github.com/bradenaw/juniper/xslices"
 	"math/rand"
 	"net"
 	"os"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/bradenaw/juniper/xslices"
+
 	"github.com/ProtonMail/gluon/benchmarks/gluon_bench/flags"
 	"github.com/emersion/go-imap"