Skip to content

Commit

Permalink
Merge pull request #25 from TencentBlueKing/develop
Browse files Browse the repository at this point in the history
v1.8.6
  • Loading branch information
wklken authored Sep 9, 2021
2 parents 72db6f5 + 7d6d779 commit 1f9feeb
Show file tree
Hide file tree
Showing 17 changed files with 92 additions and 40 deletions.
2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.8.5
1.8.6
1 change: 1 addition & 0 deletions build/support-files/sql/0015_iam_20210909-1800_mysql.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ALTER TABLE `bkiam`.`policy` ADD INDEX `idx_update_at` (`updated_at`);
2 changes: 1 addition & 1 deletion pkg/api/policy/handler/util_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,7 @@ func Test_validateSystemMatchClient(t *testing.T) {
return []string{"test"}, nil
}
mockCache := memory.NewCache(
"mockCache", false, retrieveFunc, expiration)
"mockCache", false, retrieveFunc, expiration, nil)
impls.LocalSystemClientsCache = mockCache

type args struct {
Expand Down
58 changes: 45 additions & 13 deletions pkg/cache/impls/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,15 @@ package impls

import (
"errors"
"math/rand"
"time"

gocache "github.com/patrickmn/go-cache"

log "github.com/sirupsen/logrus"

"iam/pkg/cache/cleaner"
"iam/pkg/cache/memory"
"iam/pkg/cache/memory/backend"
"iam/pkg/cache/redis"
)

Expand Down Expand Up @@ -63,6 +64,12 @@ var (
// ErrNotExceptedTypeFromCache ...
var ErrNotExceptedTypeFromCache = errors.New("not expected type from cache")

func newRandomDuration(seconds int) backend.RandomExpirationDurationFunc {
return func() time.Duration {
return time.Duration(rand.Intn(seconds*1000)) * time.Millisecond
}
}

// Cache should only know about get/retrieve data
// ! DO NOT CARE ABOUT WHAT THE DATA WILL BE USED FOR
func InitCaches(disabled bool) {
Expand All @@ -71,62 +78,87 @@ func InitCaches(disabled bool) {
disabled,
retrieveAppCodeAppSecret,
12*time.Hour,
nil,
)

// 影响: engine增量同步

LocalSubjectCache = memory.NewCache(
"local_subject",
disabled,
retrieveSubject,
1*time.Minute,
newRandomDuration(30),
)

LocalSubjectRoleCache = memory.NewCache(
"local_subject_role",
disabled,
retrieveSubjectRole,
1*time.Minute,
)
// 影响: job查询cmdb的资源进行鉴权

LocalRemoteResourceListCache = memory.NewCache(
"local_remote_resource_list",
disabled,
retrieveRemoteResourceList,
30*time.Second,
newRandomDuration(10),
)

// 影响: 每次鉴权

LocalSubjectPKCache = memory.NewCache(
"local_subject_pk",
disabled,
retrieveSubjectPK,
1*time.Minute,
newRandomDuration(30),
)

// 影响: 每次鉴权

LocalSubjectRoleCache = memory.NewCache(
"local_subject_role",
disabled,
retrieveSubjectRole,
1*time.Minute,
newRandomDuration(30),
)

// 影响: 每次鉴权 => system_id比较集中, singleflight可以防止大的并发落db

LocalSystemClientsCache = memory.NewCache(
"local_system_clients",
disabled,
retrieveSystemClients,
1*time.Minute,
newRandomDuration(30),
)

LocalAPIGatewayJWTClientIDCache = memory.NewCache(
"local_apigw_jwt_client_id",
disabled,
retrieveAPIGatewayJWTClientID,
30*time.Second,
)
// 影响: engine接口/policy查询接口

LocalActionCache = memory.NewCache(
"local_action",
disabled,
retrieveAction,
30*time.Minute,
newRandomDuration(30),
)

// 无影响, 重算而已不查db

LocalAPIGatewayJWTClientIDCache = memory.NewCache(
"local_apigw_jwt_client_id",
disabled,
retrieveAPIGatewayJWTClientID,
30*time.Second,
nil,
)

// 无影响, 重算而已不查db

LocalUnmarshaledExpressionCache = memory.NewCache(
"local_unmarshaled_expression",
disabled,
UnmarshalExpression,
30*time.Minute,
nil,
)

// ==========================
Expand Down
4 changes: 2 additions & 2 deletions pkg/cache/impls/local_action_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ func TestGetAction(t *testing.T) {
return svctypes.ThinAction{}, nil
}
mockCache := memory.NewCache(
"mockCache", false, retrieveFunc, expiration)
"mockCache", false, retrieveFunc, expiration, nil)
LocalActionCache = mockCache

_, err := GetAction(1)
Expand All @@ -48,7 +48,7 @@ func TestGetAction(t *testing.T) {
return false, errors.New("error here")
}
mockCache = memory.NewCache(
"mockCache", false, retrieveFunc, expiration)
"mockCache", false, retrieveFunc, expiration, nil)
LocalActionCache = mockCache

_, err = GetAction(1)
Expand Down
2 changes: 1 addition & 1 deletion pkg/cache/impls/local_apigw_jwt_client_id_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ var _ = Describe("LocalApigwJwtClientId", func() {
return true, nil
}
mockCache := memory.NewCache(
"mockCache", false, retrieveFunc, expiration)
"mockCache", false, retrieveFunc, expiration, nil)
LocalAPIGatewayJWTClientIDCache = mockCache
})

Expand Down
4 changes: 2 additions & 2 deletions pkg/cache/impls/local_app_code_secret_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ func TestVerifyAppCodeAppSecret(t *testing.T) {
return true, nil
}
mockCache := memory.NewCache(
"mockCache", false, retrieveFunc, expiration)
"mockCache", false, retrieveFunc, expiration, nil)
LocalAppCodeAppSecretCache = mockCache
assert.True(t, VerifyAppCodeAppSecret("test", "123"))

Expand All @@ -48,7 +48,7 @@ func TestVerifyAppCodeAppSecret(t *testing.T) {
return false, errors.New("error here")
}
mockCache = memory.NewCache(
"mockCache", false, retrieveFunc, expiration)
"mockCache", false, retrieveFunc, expiration, nil)
LocalAppCodeAppSecretCache = mockCache
assert.False(t, VerifyAppCodeAppSecret("test", "123"))
}
4 changes: 2 additions & 2 deletions pkg/cache/impls/local_remote_resource_list_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ func TestListRemoteResources(t *testing.T) {
return []map[string]interface{}{}, nil
}
mockCache := memory.NewCache(
"mockCache", false, retrieveFunc, expiration)
"mockCache", false, retrieveFunc, expiration, nil)
LocalRemoteResourceListCache = mockCache

_, err := ListRemoteResources("test", "app", []string{"1", "2"}, []string{"id", "name"})
Expand All @@ -54,7 +54,7 @@ func TestListRemoteResources(t *testing.T) {
return false, errors.New("error here")
}
mockCache = memory.NewCache(
"mockCache", false, retrieveFunc, expiration)
"mockCache", false, retrieveFunc, expiration, nil)
LocalRemoteResourceListCache = mockCache

_, err = ListRemoteResources("test", "app", []string{"1", "2"}, []string{"id", "name"})
Expand Down
4 changes: 2 additions & 2 deletions pkg/cache/impls/local_subject_role_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ func TestSubjectListSubjectRoleSystemID(t *testing.T) {
return []string{"bk_cmdb", "bk_job"}, nil
}
mockCache := memory.NewCache(
"mockCache", false, retrieveFunc, expiration)
"mockCache", false, retrieveFunc, expiration, nil)
LocalSubjectRoleCache = mockCache

systemIDs, err := ListSubjectRoleSystemID("user", "admin")
Expand All @@ -51,7 +51,7 @@ func TestSubjectListSubjectRoleSystemID(t *testing.T) {
return []string{}, errors.New("error here")
}
mockCache = memory.NewCache(
"mockCache", false, retrieveFunc, expiration)
"mockCache", false, retrieveFunc, expiration, nil)
LocalSubjectRoleCache = mockCache

_, err = ListSubjectRoleSystemID("user", "admin")
Expand Down
4 changes: 2 additions & 2 deletions pkg/cache/impls/local_subject_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ func TestGetSubjectByPK(t *testing.T) {
return svctypes.Subject{}, nil
}
mockCache := memory.NewCache(
"mockCache", false, retrieveFunc, expiration)
"mockCache", false, retrieveFunc, expiration, nil)
LocalSubjectCache = mockCache

_, err := GetSubjectByPK(1)
Expand All @@ -43,7 +43,7 @@ func TestGetSubjectByPK(t *testing.T) {
return false, errors.New("error here")
}
mockCache = memory.NewCache(
"mockCache", false, retrieveFunc, expiration)
"mockCache", false, retrieveFunc, expiration, nil)
LocalSubjectCache = mockCache

_, err = GetSubjectByPK(1)
Expand Down
4 changes: 2 additions & 2 deletions pkg/cache/impls/local_system_clients_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ func TestGetSystemClients(t *testing.T) {
return []string{"a", "b", "c"}, nil
}
mockCache := memory.NewCache(
"mockCache", false, retrieveFunc, expiration)
"mockCache", false, retrieveFunc, expiration, nil)
LocalSystemClientsCache = mockCache

cs, err := GetSystemClients("x")
Expand All @@ -46,7 +46,7 @@ func TestGetSystemClients(t *testing.T) {
return []string{}, errors.New("error here")
}
mockCache = memory.NewCache(
"mockCache", false, retrieveFunc, expiration)
"mockCache", false, retrieveFunc, expiration, nil)
LocalSystemClientsCache = mockCache

_, err = GetSystemClients("x")
Expand Down
22 changes: 17 additions & 5 deletions pkg/cache/memory/backend/memory.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ const (
DefaultCleanupInterval = 5 * time.Minute
)

type RandomExpirationDurationFunc func() time.Duration

// NewTTLCache create cache with expiration and cleanup interval,
// if cleanupInterval is 0, will use DefaultCleanupInterval
func newTTLCache(expiration time.Duration, cleanupInterval time.Duration) *gocache.Cache {
Expand All @@ -36,7 +38,8 @@ type MemoryBackend struct {
name string
cache *gocache.Cache

defaultExpiration time.Duration
defaultExpiration time.Duration
randomDurationFunc RandomExpirationDurationFunc
}

// Set ...
Expand All @@ -45,6 +48,10 @@ func (c *MemoryBackend) Set(key string, value interface{}, duration time.Duratio
duration = c.defaultExpiration
}

if c.randomDurationFunc != nil {
duration += c.randomDurationFunc()
}

c.cache.Set(key, value, duration)
}

Expand All @@ -65,12 +72,17 @@ func (c *MemoryBackend) Delete(key string) error {
}

// NewMemoryBackend ...
func NewMemoryBackend(name string, expiration time.Duration) *MemoryBackend {
func NewMemoryBackend(
name string,
expiration time.Duration,
randomDurationFunc RandomExpirationDurationFunc,
) *MemoryBackend {
cleanupInterval := expiration + (5 * time.Minute)

return &MemoryBackend{
name: name,
cache: newTTLCache(expiration, cleanupInterval),
defaultExpiration: expiration,
name: name,
cache: newTTLCache(expiration, cleanupInterval),
defaultExpiration: expiration,
randomDurationFunc: randomDurationFunc,
}
}
2 changes: 1 addition & 1 deletion pkg/cache/memory/backend/memory_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func TestNewTTLCache(t *testing.T) {
}

func TestMemoryBackend(t *testing.T) {
be := NewMemoryBackend("test", 5*time.Second)
be := NewMemoryBackend("test", 5*time.Second, nil)
assert.NotNil(t, be)

_, found := be.Get("not_exists")
Expand Down
2 changes: 1 addition & 1 deletion pkg/cache/memory/base_cache_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ func retrieveError(k cache.Key) (interface{}, error) {
func TestNewBaseCache(t *testing.T) {
expiration := 5 * time.Minute

be := backend.NewMemoryBackend("test", expiration)
be := backend.NewMemoryBackend("test", expiration, nil)

c := NewBaseCache(false, retrieveTest, be)

Expand Down
11 changes: 7 additions & 4 deletions pkg/cache/memory/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,18 @@ import (
)

// NewCache create a memory cache
func NewCache(name string, disabled bool, retrieveFunc RetrieveFunc,
expiration time.Duration) Cache {
be := backend.NewMemoryBackend(name, expiration)
func NewCache(name string, disabled bool,
retrieveFunc RetrieveFunc,
expiration time.Duration,
randomDurationFunc backend.RandomExpirationDurationFunc,
) Cache {
be := backend.NewMemoryBackend(name, expiration, randomDurationFunc)
return NewBaseCache(disabled, retrieveFunc, be)
}

// NewMockCache create a memory cache for mock
func NewMockCache(retrieveFunc RetrieveFunc) Cache {
be := backend.NewMemoryBackend("mockCache", 5*time.Minute)
be := backend.NewMemoryBackend("mockCache", 5*time.Minute, nil)

return NewBaseCache(false, retrieveFunc, be)
}
2 changes: 1 addition & 1 deletion pkg/cache/memory/cache_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,6 @@ func retrieveOK(k cache.Key) (interface{}, error) {
func TestNewCache(t *testing.T) {
expiration := 5 * time.Minute

c := NewCache("test", false, retrieveOK, expiration)
c := NewCache("test", false, retrieveOK, expiration, nil)
assert.NotNil(t, c)
}
4 changes: 4 additions & 0 deletions release.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# 1.8.6

- add: local cache expire with a random duration

# 1.8.5

- upgrade: update the expression table structure, delete useless columns
Expand Down

0 comments on commit 1f9feeb

Please sign in to comment.