Skip to content

Commit

Permalink
rowexec: ask for at least 8MiB in the join reader memory limit
Browse files Browse the repository at this point in the history
The join reader doesn't know how to spill to disk, so previously in some
cases (namely, when `distsql_workmem` session variable is low) the
queries would error out. Now this is temporarily fixed by requiring the
memory limit to be at least 8MiB (to accommodate 4MiB scratch input
rows). This shouldn't really matter in the production setting but makes
`tpchvec/disk` roachtest happy.

Release note: None
  • Loading branch information
yuzefovich committed Jul 27, 2021
1 parent d8ba460 commit ce68525
Show file tree
Hide file tree
Showing 3 changed files with 13 additions and 24 deletions.
14 changes: 0 additions & 14 deletions pkg/sql/execinfra/processorsbase.go
Original file line number Diff line number Diff line change
Expand Up @@ -949,20 +949,6 @@ func NewLimitedMonitor(
return limitedMon
}

// NewLimitedMonitorNoDiskSpill is a utility function used by processors to
// create a new limited memory monitor with the given name and start it. The
// returned monitor must be closed. The limit is determined by
// SessionData.WorkMemLimit (stored inside of the flowCtx) but overridden to
// ServerConfig.TestingKnobs.MemoryLimitBytes if that knob is set.
// ServerConfig.TestingKnobs.ForceDiskSpill is ignored by this function.
func NewLimitedMonitorNoDiskSpill(
ctx context.Context, parent *mon.BytesMonitor, flowCtx *FlowCtx, name string,
) *mon.BytesMonitor {
limitedMon := mon.NewMonitorInheritWithLimit(name, GetWorkMemLimitNoDiskSpill(flowCtx), parent)
limitedMon.Start(ctx, parent, mon.BoundAccount{})
return limitedMon
}

// NewLimitedMonitorNoFlowCtx is the same as NewLimitedMonitor and should be
// used when the caller doesn't have an access to *FlowCtx.
func NewLimitedMonitorNoFlowCtx(
Expand Down
8 changes: 0 additions & 8 deletions pkg/sql/execinfra/server_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -268,14 +268,6 @@ func GetWorkMemLimit(flowCtx *FlowCtx) int64 {
if flowCtx.Cfg.TestingKnobs.ForceDiskSpill {
return 1
}
return GetWorkMemLimitNoDiskSpill(flowCtx)
}

// GetWorkMemLimitNoDiskSpill returns the number of bytes determining the amount
// of RAM available to a single processor or operator. This function should be
// used instead of GetWorkMemLimit if the processor cannot spill to disk,
// since ServerConfig.TestingKnobs.ForceDiskSpill is ignored by this function.
func GetWorkMemLimitNoDiskSpill(flowCtx *FlowCtx) int64 {
if flowCtx.Cfg.TestingKnobs.MemoryLimitBytes != 0 {
return flowCtx.Cfg.TestingKnobs.MemoryLimitBytes
}
Expand Down
15 changes: 13 additions & 2 deletions pkg/sql/rowexec/joinreader.go
Original file line number Diff line number Diff line change
Expand Up @@ -373,10 +373,21 @@ func newJoinReader(
}
}

// We will create a memory monitor with at least 8MiB of memory limit since
// the join reader doesn't know how to spill to disk. It is most likely that
// if the target limit is below 8MiB, then we're in a test scenario and we
// don't want to error out.
const minMemoryLimit = 8 << 20
memoryLimit := execinfra.GetWorkMemLimit(flowCtx)
if memoryLimit < minMemoryLimit {
memoryLimit = minMemoryLimit
}

// Initialize memory monitors and bound account for data structures in the joinReader.
jr.MemMonitor = execinfra.NewLimitedMonitorNoDiskSpill(
flowCtx.EvalCtx.Ctx(), flowCtx.EvalCtx.Mon, flowCtx, "joinreader-mem",
jr.MemMonitor = mon.NewMonitorInheritWithLimit(
"joinreader-mem" /* name */, memoryLimit, flowCtx.EvalCtx.Mon,
)
jr.MemMonitor.Start(flowCtx.EvalCtx.Ctx(), flowCtx.EvalCtx.Mon, mon.BoundAccount{})
jr.memAcc = jr.MemMonitor.MakeBoundAccount()

if err := jr.initJoinReaderStrategy(flowCtx, columnTypes, len(columnIDs), rightCols, readerType); err != nil {
Expand Down

0 comments on commit ce68525

Please sign in to comment.