Change bp allocation size to match bufsz changes #33

Explicitly free os memory on buffer size changes #33

log buffer removal #33

buffer size logging #33

pool.go

@@ -12,11 +12,12 @@ type qBuf struct {
 }
 
 type bp struct {
-	makes   int
-	get     chan *bytes.Buffer
-	give    chan *bytes.Buffer
-	quit    chan struct{}
-	timeout time.Duration
+	makes    int
+	get      chan *bytes.Buffer
+	give     chan *bytes.Buffer
+	quit     chan struct{}
+	timeout  time.Duration
+	makeSize int64
 }
 
 func makeBuffer(size int64) []byte {
@@ -25,18 +26,20 @@ func makeBuffer(size int64) []byte {
 
 func newBufferPool(bufsz int64) (np *bp) {
 	np = &bp{
-		get:     make(chan *bytes.Buffer),
-		give:    make(chan *bytes.Buffer),
-		quit:    make(chan struct{}),
-		timeout: time.Minute,
+		get:      make(chan *bytes.Buffer),
+		give:     make(chan *bytes.Buffer),
+		quit:     make(chan struct{}),
+		timeout:  time.Minute,
+		makeSize: bufsz,
 	}
 	go func() {
 		q := new(list.List)
 		for {
 			if q.Len() == 0 {
-				size := bufsz + 100*kb // allocate overhead to avoid slice growth
-				q.PushFront(qBuf{when: time.Now(), buffer: bytes.NewBuffer(makeBuffer(int64(size)))})
+				size := np.makeSize + 100*kb // allocate overhead to avoid slice growth
+				q.PushFront(qBuf{when: time.Now(), buffer: bytes.NewBuffer(makeBuffer(size))})
 				np.makes++
+				logger.debugPrintf("buffer %d of %d MB allocated", np.makes, np.makeSize/(1*mb))
 			}
 
 			e := q.Front()
@@ -57,13 +60,14 @@ func newBufferPool(bufsz int64) (np *bp) {
 				for e != nil {
 					n := e.Next()
 					if time.Since(e.Value.(qBuf).when) > np.timeout {
+						logger.debugPrintln("removing buffer from queue")
 						q.Remove(e)
 						e.Value = nil
 					}
 					e = n
 				}
 			case <-np.quit:
-				logger.debugPrintf("%d buffers of %d MB allocated", np.makes, bufsz/(1*mb))
+				logger.debugPrintf("%d buffers of %d MB allocated", np.makes, np.makeSize/(1*mb))
 				return
 			}
 		}

putter.go

@@ -11,6 +11,7 @@ import (
 	"math"
 	"net/http"
 	"net/url"
+	"runtime/debug"
 	"strconv"
 	"strings"
 	"sync"
@@ -146,6 +147,10 @@ func (p *putter) flush() {
 	// while still reaching the 5 Terabyte max object size
 	if p.part%1000 == 0 {
 		p.bufsz = min64(p.bufsz*2, maxPartSize)
+		p.bp.makeSize = p.bufsz
+		logger.debugPrintf("doubling buffer size to %d", p.bufsz)
+		debug.FreeOSMemory()
+		logger.debugPrintln("os memory freed")
 	}
 
 }