Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

planner: LRUPlanCache memory trace #38069

Merged
merged 43 commits into from
Oct 21, 2022
Merged
Show file tree
Hide file tree
Changes from 41 commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
796bf21
mem
fzzf678 Sep 20, 2022
9c76f9e
mem
fzzf678 Sep 21, 2022
d0c1b3b
mem
fzzf678 Sep 26, 2022
16eb495
Merge remote-tracking branch 'upstream/master' into planCacheValue_me…
fzzf678 Oct 8, 2022
7f8f893
add ut
fzzf678 Oct 8, 2022
19505d9
don't contain self memory
fzzf678 Oct 8, 2022
367124e
test memusage
fzzf678 Oct 10, 2022
bb0bc45
Merge branch 'master' into planCacheValue_memTrace
fzzf678 Oct 10, 2022
07feb79
undo
fzzf678 Oct 11, 2022
d98d3eb
Merge branch 'planCacheValue_memTrace' of https://github.com/fzzf678/…
fzzf678 Oct 11, 2022
23412a0
test
fzzf678 Oct 11, 2022
84cfdda
Update plan_cache_lru.go
fzzf678 Oct 11, 2022
bc4ffd8
Merge branch 'master' into planCacheValue_memTrace
fzzf678 Oct 11, 2022
d2b49d3
test_undo
fzzf678 Oct 11, 2022
177b7e4
Merge branch 'master' into planCacheValue_memTrace
fzzf678 Oct 11, 2022
0adbc7d
Merge branch 'planCacheValue_memTrace' of https://github.com/fzzf678/…
fzzf678 Oct 11, 2022
1296c6c
Update plan_cache_utils.go
fzzf678 Oct 11, 2022
773a9b2
convert fail case
fzzf678 Oct 12, 2022
05475a1
Update util.go
fzzf678 Oct 12, 2022
5e8d919
tracker
fzzf678 Oct 17, 2022
be497fa
converge init
fzzf678 Oct 17, 2022
645af07
Update plan_cache_lru.go
fzzf678 Oct 17, 2022
0bbb28b
Update plan_cache_lru.go
fzzf678 Oct 17, 2022
8ad0b47
Update plan_cache_utils.go
fzzf678 Oct 18, 2022
f3cb4aa
Merge branch 'master' into planCacheValue_memTrace
qw4990 Oct 18, 2022
d16e915
Update plan_cache_lru.go
fzzf678 Oct 18, 2022
ee4f585
Merge branch 'master' into planCacheValue_memTrace
fzzf678 Oct 18, 2022
8855a86
Update plan_cache_lru.go
fzzf678 Oct 18, 2022
2876706
Merge branch 'planCacheValue_memTrace' of https://github.com/fzzf678/…
fzzf678 Oct 18, 2022
1d773ac
fix
fzzf678 Oct 19, 2022
3d44d36
Update plan_cache_lru.go
fzzf678 Oct 19, 2022
f46d272
Update plan_cache_utils.go
fzzf678 Oct 19, 2022
ec528da
Merge branch 'master' into planCacheValue_memTrace
fzzf678 Oct 19, 2022
88f2102
update
fzzf678 Oct 20, 2022
6c391d1
Update plan_cache_lru.go
fzzf678 Oct 20, 2022
c97efcf
Update plan_cache_lru_test.go
fzzf678 Oct 20, 2022
8da7e94
Update plan_cache_utils.go
fzzf678 Oct 21, 2022
7a5fc11
Update plan_cache_utils.go
fzzf678 Oct 21, 2022
2a05173
Update plan_cache_utils.go
fzzf678 Oct 21, 2022
40b3929
fix ut
fzzf678 Oct 21, 2022
05148a6
Update plan_cache_lru_test.go
fzzf678 Oct 21, 2022
0445583
Merge branch 'master' into planCacheValue_memTrace
qw4990 Oct 21, 2022
4b9a204
Merge branch 'master' into planCacheValue_memTrace
ti-chi-bot Oct 21, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions planner/core/plan.go
Original file line number Diff line number Diff line change
Expand Up @@ -584,9 +584,6 @@ func (p *basePhysicalPlan) MemoryUsage() (sum int64) {
sum = p.basePlan.MemoryUsage() + size.SizeOfSlice + int64(cap(p.childrenReqProps))*size.SizeOfPointer +
size.SizeOfSlice + int64(cap(p.children)+1)*size.SizeOfInterface + size.SizeOfFloat64 +
size.SizeOfUint64 + size.SizeOfBool
if p.self != nil {
sum += p.self.MemoryUsage()
}

for _, prop := range p.childrenReqProps {
sum += prop.MemoryUsage()
Expand Down
36 changes: 36 additions & 0 deletions planner/core/plan_cache_lru.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,15 @@ type planCacheEntry struct {
PlanValue kvcache.Value
}

// MemoryUsage return the memory usage of planCacheEntry
func (e *planCacheEntry) MemoryUsage() (sum int64) {
if e == nil {
return
}

return e.PlanKey.(*planCacheKey).MemoryUsage() + e.PlanValue.(*PlanCacheValue).MemoryUsage()
}

// LRUPlanCache is a dedicated least recently used cache, Only used for plan cache.
type LRUPlanCache struct {
capacity uint
Expand All @@ -49,6 +58,9 @@ type LRUPlanCache struct {
// 0 indicates no quota
quota uint64
guard float64

// MemTracker track the memory usage of prepared plan cache
memTracker *memory.Tracker
}

// NewLRUPlanCache creates a PCLRUCache object, whose capacity is "capacity".
Expand All @@ -59,6 +71,7 @@ func NewLRUPlanCache(capacity uint, guard float64, quota uint64,
capacity = 100
logutil.BgLogger().Info("capacity of LRU cache is less than 1, will use default value(100) init cache")
}

return &LRUPlanCache{
capacity: capacity,
size: 0,
Expand All @@ -67,6 +80,7 @@ func NewLRUPlanCache(capacity uint, guard float64, quota uint64,
pickFromBucket: pickFromBucket,
quota: quota,
guard: guard,
memTracker: newTrackerForLRUPC(),
}
}

Expand Down Expand Up @@ -102,6 +116,8 @@ func (l *LRUPlanCache) Put(key kvcache.Key, value kvcache.Value, paramTypes []*t
bucket, bucketExist := l.buckets[hash]
if bucketExist {
if element, exist := l.pickFromBucket(bucket, paramTypes); exist {
l.memTracker.Consume(value.(*PlanCacheValue).MemoryUsage() -
element.Value.(*planCacheEntry).PlanValue.(*PlanCacheValue).MemoryUsage())
element.Value.(*planCacheEntry).PlanValue = value
l.lruList.MoveToFront(element)
return
Expand All @@ -117,6 +133,7 @@ func (l *LRUPlanCache) Put(key kvcache.Key, value kvcache.Value, paramTypes []*t
element := l.lruList.PushFront(newCacheEntry)
l.buckets[hash][element] = struct{}{}
l.size++
l.memTracker.Consume(newCacheEntry.MemoryUsage())
if l.size > l.capacity {
l.removeOldest()
}
Expand All @@ -132,6 +149,7 @@ func (l *LRUPlanCache) Delete(key kvcache.Key) {
bucket, bucketExist := l.buckets[hash]
if bucketExist {
for element := range bucket {
l.memTracker.Consume(-element.Value.(*planCacheEntry).MemoryUsage())
l.lruList.Remove(element)
l.size--
}
Expand All @@ -149,6 +167,7 @@ func (l *LRUPlanCache) DeleteAll() {
l.size--
}
l.buckets = make(map[string]map[*list.Element]struct{}, 1)
l.memTracker = newTrackerForLRUPC()
}

// Size gets the current cache size.
Expand All @@ -174,6 +193,14 @@ func (l *LRUPlanCache) SetCapacity(capacity uint) error {
return nil
}

// MemoryUsage return the memory usage of LRUPlanCache
func (l *LRUPlanCache) MemoryUsage() (sum int64) {
if l == nil {
return
}
return l.memTracker.BytesConsumed()
}

// removeOldest removes the oldest element from the cache.
func (l *LRUPlanCache) removeOldest() {
lru := l.lruList.Back()
Expand All @@ -184,6 +211,7 @@ func (l *LRUPlanCache) removeOldest() {
l.onEvict(lru.Value.(*planCacheEntry).PlanKey, lru.Value.(*planCacheEntry).PlanValue)
}

l.memTracker.Consume(-lru.Value.(*planCacheEntry).MemoryUsage())
l.lruList.Remove(lru)
l.removeFromBucket(lru)
l.size--
Expand Down Expand Up @@ -222,3 +250,11 @@ func PickPlanFromBucket(bucket map[*list.Element]struct{}, paramTypes []*types.F
}
return nil, false
}

// newTrackerForLRUPC return a tracker which consumed emptyLRUPlanCacheSize
// todo: pass label when track general plan cache memory
func newTrackerForLRUPC() *memory.Tracker {
m := memory.NewTracker(memory.LabelForPreparedPlanCache, -1)
//todo: maybe need attach here
return m
}
Loading