Skip to content

Commit

Permalink
Unpin extendedQueryBuilder memory immediately after use
Browse files Browse the repository at this point in the history
refs #1110
  • Loading branch information
jackc committed Nov 13, 2021
1 parent 851091f commit a457da8
Show file tree
Hide file tree
Showing 2 changed files with 18 additions and 19 deletions.
6 changes: 6 additions & 0 deletions conn.go
Original file line number Diff line number Diff line change
Expand Up @@ -521,6 +521,7 @@ func (c *Conn) execParams(ctx context.Context, sd *pgconn.StatementDescription,
}

result := c.pgConn.ExecParams(ctx, sd.SQL, c.eqb.paramValues, sd.ParamOIDs, c.eqb.paramFormats, c.eqb.resultFormats).Read()
c.eqb.Reset() // Allow c.eqb internal memory to be GC'ed as soon as possible.
return result.CommandTag, result.Err
}

Expand All @@ -531,6 +532,7 @@ func (c *Conn) execPrepared(ctx context.Context, sd *pgconn.StatementDescription
}

result := c.pgConn.ExecPrepared(ctx, sd.Name, c.eqb.paramValues, c.eqb.paramFormats, c.eqb.resultFormats).Read()
c.eqb.Reset() // Allow c.eqb internal memory to be GC'ed as soon as possible.
return result.CommandTag, result.Err
}

Expand Down Expand Up @@ -678,6 +680,8 @@ optionLoop:
rows.resultReader = c.pgConn.ExecPrepared(ctx, sd.Name, c.eqb.paramValues, c.eqb.paramFormats, resultFormats)
}

c.eqb.Reset() // Allow c.eqb internal memory to be GC'ed as soon as possible.

return rows, rows.err
}

Expand Down Expand Up @@ -825,6 +829,8 @@ func (c *Conn) SendBatch(ctx context.Context, b *Batch) BatchResults {
}
}

c.eqb.Reset() // Allow c.eqb internal memory to be GC'ed as soon as possible.

mrr := c.pgConn.ExecBatch(ctx, batch)

return &batchResults{
Expand Down
31 changes: 12 additions & 19 deletions extended_query_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,6 @@ type extendedQueryBuilder struct {
paramValueBytes []byte
paramFormats []int16
resultFormats []int16

resetCount int
}

func (eqb *extendedQueryBuilder) AppendParam(ci *pgtype.ConnInfo, oid uint32, arg interface{}) error {
Expand All @@ -34,32 +32,27 @@ func (eqb *extendedQueryBuilder) AppendResultFormat(f int16) {
eqb.resultFormats = append(eqb.resultFormats, f)
}

// Reset readies eqb to build another query.
func (eqb *extendedQueryBuilder) Reset() {
eqb.paramValues = eqb.paramValues[0:0]
eqb.paramValueBytes = eqb.paramValueBytes[0:0]
eqb.paramFormats = eqb.paramFormats[0:0]
eqb.resultFormats = eqb.resultFormats[0:0]

eqb.resetCount++

// Every so often shrink our reserved memory if it is abnormally high
if eqb.resetCount%128 == 0 {
if cap(eqb.paramValues) > 64 {
eqb.paramValues = make([][]byte, 0, cap(eqb.paramValues)/2)
}

if cap(eqb.paramValueBytes) > 256 {
eqb.paramValueBytes = make([]byte, 0, cap(eqb.paramValueBytes)/2)
}
if cap(eqb.paramValues) > 64 {
eqb.paramValues = make([][]byte, 0, 64)
}

if cap(eqb.paramFormats) > 64 {
eqb.paramFormats = make([]int16, 0, cap(eqb.paramFormats)/2)
}
if cap(eqb.resultFormats) > 64 {
eqb.resultFormats = make([]int16, 0, cap(eqb.resultFormats)/2)
}
if cap(eqb.paramValueBytes) > 256 {
eqb.paramValueBytes = make([]byte, 0, 256)
}

if cap(eqb.paramFormats) > 64 {
eqb.paramFormats = make([]int16, 0, 64)
}
if cap(eqb.resultFormats) > 64 {
eqb.resultFormats = make([]int16, 0, 64)
}
}

func (eqb *extendedQueryBuilder) encodeExtendedParamValue(ci *pgtype.ConnInfo, oid uint32, formatCode int16, arg interface{}) ([]byte, error) {
Expand Down

6 comments on commit a457da8

@yyijie
Copy link

@yyijie yyijie commented on a457da8 Nov 19, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi,

I see one panic error when importing your packages, could you please help me to check? Thanks.

panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x180 pc=0x120fc22]
goroutine 325 [running]:
sync.(*Mutex).Lock(...)
#11/usr/local/go/src/sync/mutex.go:74
github.com/jackc/pgx.(*Conn).IsAlive(0x0, 0x0)
#11/vendor/github.com/jackc/pgx/conn.go:1679 +0x32

@jackc
Copy link
Owner Author

@jackc jackc commented on a457da8 Nov 19, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like you're using v3. I suggest using v4. It was released over 2 years ago. Also, I have no idea how that problem could relate to this commit.

@yyijie
Copy link

@yyijie yyijie commented on a457da8 Nov 19, 2021 via email

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jackc
Copy link
Owner Author

@jackc jackc commented on a457da8 Nov 19, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, it's available. Versioned package import paths don't exactly correspond to Github URLs. https://github.com/jackc/pgx is the Github page and https://pkg.go.dev/github.com/jackc/pgx/v4 is the documentation.

@yyijie
Copy link

@yyijie yyijie commented on a457da8 Nov 19, 2021 via email

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jackc
Copy link
Owner Author

@jackc jackc commented on a457da8 Nov 20, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.