forked from earthly/earthly
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Logbus: Don't pre-allocate 80k for each command (earthly#3482)
While looking into BK's memory usage, I noticed some odd consumption (below) when reviewing Earthly's `pprof` info. It turns out that the previously used `circbuf` library was [preallocating](https://github.com/armon/circbuf/blob/master/circbuf.go#L27) ~80K for each command. For large builds like ours, this can represent 100s of MBs (or even GBs). I've added a custom circular buffer implementation that starts with an empty buffer and grows to a maximum size before writing circularly. This is massively reducing memory usage for some of our tests, like `+test-no-qemu`. 
- Loading branch information
1 parent
3b04a98
commit dd6ac1b
Showing
6 changed files
with
133 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| package circbuf | ||
|
|
||
| import "errors" | ||
|
|
||
| // Buffer represents a dynamic circular (ring) buffer. It will append data | ||
| // freely to the underlying buffer until maxSize is hit. Once hit, new writes | ||
| // will be written in a circular fashion. | ||
| type Buffer struct { | ||
| data []byte | ||
| offset int | ||
| maxSize int | ||
| written int | ||
| } | ||
|
|
||
| // NewBuffer creates and returns a Buffer pointer with a max size. | ||
| func NewBuffer(maxSize int) (*Buffer, error) { | ||
| if maxSize < 0 { | ||
| return nil, errors.New("size must be a positive int") | ||
| } | ||
| return &Buffer{ | ||
| maxSize: int(maxSize), | ||
| }, nil | ||
| } | ||
|
|
||
| // Write implements io.Writer. | ||
| func (c *Buffer) Write(buf []byte) (int, error) { | ||
| l := len(c.data) | ||
| n := len(buf) | ||
| c.written += n | ||
|
|
||
| if n > c.maxSize { | ||
| buf = buf[n-c.maxSize:] | ||
| } | ||
|
|
||
| if l < c.maxSize { | ||
| r := c.maxSize - l | ||
| if n > r { | ||
| c.data = append(c.data, buf[:r]...) | ||
| buf = buf[r:] | ||
| c.offset = 0 | ||
| } else { | ||
| c.data = append(c.data, buf...) | ||
| c.offset = n | ||
| return n, nil | ||
| } | ||
| } | ||
|
|
||
| remain := c.maxSize - c.offset | ||
| copy(c.data[c.offset:], buf) | ||
| if len(buf) > remain { | ||
| copy(c.data, buf[remain:]) | ||
| } | ||
|
|
||
| c.offset = (c.offset + len(buf)) % c.maxSize | ||
|
|
||
| return n, nil | ||
| } | ||
|
|
||
| func (c *Buffer) Bytes() []byte { | ||
| if len(c.data) < c.maxSize { | ||
| return c.data | ||
| } | ||
| ret := make([]byte, c.maxSize) | ||
| copy(ret, c.data[c.offset:]) | ||
| copy(ret[c.maxSize-c.offset:], c.data[:c.offset]) | ||
| return ret | ||
| } | ||
|
|
||
| func (c *Buffer) Size() int { | ||
| return len(c.data) | ||
| } | ||
|
|
||
| func (c *Buffer) TotalWritten() int { | ||
| return c.written | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| package circbuf | ||
|
|
||
| import ( | ||
| "io" | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| func TestNewError(t *testing.T) { | ||
| _, err := NewBuffer(-1) | ||
| require.Error(t, err) | ||
| } | ||
|
|
||
| func TestWriteGrow(t *testing.T) { | ||
| b := &Buffer{maxSize: 5} | ||
| n, err := io.WriteString(b, "foo") | ||
| r := require.New(t) | ||
| r.NoError(err) | ||
| r.Equal(3, n) | ||
| r.Len(b.data, 3) | ||
| r.Equal("foo", string(b.Bytes())) | ||
| } | ||
|
|
||
| func TestWriteOverflow(t *testing.T) { | ||
| b := &Buffer{maxSize: 5} | ||
| n, err := io.WriteString(b, "foobarbaz") | ||
|
|
||
| r := require.New(t) | ||
| r.NoError(err) | ||
| r.Equal(9, n) | ||
| r.Equal("arbaz", string(b.data)) | ||
| r.Equal("arbaz", string(b.Bytes())) | ||
| } | ||
|
|
||
| func TestWriteMulti(t *testing.T) { | ||
| b := &Buffer{maxSize: 5} | ||
| r := require.New(t) | ||
|
|
||
| n, err := io.WriteString(b, "mr") | ||
| r.NoError(err) | ||
| r.Equal(2, n) | ||
|
|
||
| n, err = io.WriteString(b, "world") | ||
| r.NoError(err) | ||
| r.Equal(5, n) | ||
|
|
||
| n, err = io.WriteString(b, "wide") | ||
| r.NoError(err) | ||
| r.Equal(4, n) | ||
|
|
||
| r.Equal("edwid", string(b.data)) | ||
| r.Equal(1, b.offset) | ||
|
|
||
| r.Equal("dwide", string(b.Bytes())) | ||
| } |