Skip to content

Commit

Permalink
enhance: Enable ReadOnly/ReadWrite/Admin Privilege Group to simplify …
Browse files Browse the repository at this point in the history
…RBAC grant progress (#35472)

issue: #35471

---------

Signed-off-by: Wei Liu <[email protected]>
  • Loading branch information
weiliu1031 authored Aug 16, 2024
1 parent f4a91e1 commit a570567
Show file tree
Hide file tree
Showing 9 changed files with 307 additions and 9 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ require (
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/klauspost/compress v1.17.7
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240815113856-e2789dca8b59
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240815123953-6dab6fcd6454
github.com/minio/minio-go/v7 v7.0.61
github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81
github.com/prometheus/client_golang v1.14.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -598,8 +598,8 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119 h1:9VXijWu
github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240815113856-e2789dca8b59 h1:mKekr0GmCKMpIQh9OJ67TlKVKxDt08600ltARc/JUXY=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240815113856-e2789dca8b59/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240815123953-6dab6fcd6454 h1:JmZCYjMPpiE4ksZw0AUxXWkDY7wwA4fhS+SO1N211Vw=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240815123953-6dab6fcd6454/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A=
github.com/milvus-io/pulsar-client-go v0.6.10/go.mod h1:lQqCkgwDF8YFYjKA+zOheTk1tev2B+bKj5j7+nm8M1w=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
Expand Down
56 changes: 53 additions & 3 deletions internal/proxy/privilege_interceptor.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (

"github.com/casbin/casbin/v2"
"github.com/casbin/casbin/v2/model"
"github.com/samber/lo"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
Expand All @@ -20,6 +21,7 @@ import (
"github.com/milvus-io/milvus/pkg/util"
"github.com/milvus-io/milvus/pkg/util/contextutil"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/paramtable"
)

type PrivilegeFunc func(ctx context.Context, req interface{}) (context.Context, error)
Expand All @@ -39,17 +41,42 @@ p = sub, obj, act
e = some(where (p.eft == allow))
[matchers]
m = r.sub == p.sub && globMatch(r.obj, p.obj) && globMatch(r.act, p.act) || r.sub == "admin" || (r.sub == p.sub && dbMatch(r.obj, p.obj) && p.act == "PrivilegeAll")
m = r.sub == p.sub && globMatch(r.obj, p.obj) && globMatch(r.act, p.act) || r.sub == "admin" || (r.sub == p.sub && dbMatch(r.obj, p.obj) && privilegeGroupContains(r.act, p.act))
`
)

var templateModel = getPolicyModel(ModelStr)

var (
enforcer *casbin.SyncedEnforcer
initOnce sync.Once
enforcer *casbin.SyncedEnforcer
initOnce sync.Once
initPrivilegeGroupsOnce sync.Once
)

var roPrivileges, rwPrivileges, adminPrivileges map[string]struct{}

func initPrivilegeGroups() {
initPrivilegeGroupsOnce.Do(func() {
roGroup := paramtable.Get().CommonCfg.ReadOnlyPrivileges.GetAsStrings()
if len(roGroup) == 0 {
roGroup = util.ReadOnlyPrivilegeGroup
}
roPrivileges = lo.SliceToMap(roGroup, func(item string) (string, struct{}) { return item, struct{}{} })

rwGroup := paramtable.Get().CommonCfg.ReadWritePrivileges.GetAsStrings()
if len(rwGroup) == 0 {
rwGroup = util.ReadWritePrivilegeGroup
}
rwPrivileges = lo.SliceToMap(rwGroup, func(item string) (string, struct{}) { return item, struct{}{} })

adminGroup := paramtable.Get().CommonCfg.AdminPrivileges.GetAsStrings()
if len(adminGroup) == 0 {
adminGroup = util.AdminPrivilegeGroup
}
adminPrivileges = lo.SliceToMap(adminGroup, func(item string) (string, struct{}) { return item, struct{}{} })
})
}

func getEnforcer() *casbin.SyncedEnforcer {
initOnce.Do(func() {
e, err := casbin.NewSyncedEnforcer()
Expand All @@ -60,6 +87,7 @@ func getEnforcer() *casbin.SyncedEnforcer {
adapter := NewMetaCacheCasbinAdapter(func() Cache { return globalMetaCache })
e.InitWithModelAndAdapter(casbinModel, adapter)
e.AddFunction("dbMatch", DBMatchFunc)
e.AddFunction("privilegeGroupContains", PrivilegeGroupContains)
enforcer = e
})
return enforcer
Expand All @@ -75,6 +103,7 @@ func getPolicyModel(modelString string) model.Model {

// UnaryServerInterceptor returns a new unary server interceptors that performs per-request privilege access.
func UnaryServerInterceptor(privilegeFunc PrivilegeFunc) grpc.UnaryServerInterceptor {
initPrivilegeGroups()
return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
newCtx, err := privilegeFunc(ctx, req)
if err != nil {
Expand Down Expand Up @@ -218,3 +247,24 @@ func DBMatchFunc(args ...interface{}) (interface{}, error) {

return db1 == db2, nil
}

func PrivilegeGroupContains(args ...interface{}) (interface{}, error) {
requestPrivilege := args[0].(string)
policyPrivilege := args[1].(string)

switch policyPrivilege {
case commonpb.ObjectPrivilege_PrivilegeAll.String():
return true, nil
case commonpb.ObjectPrivilege_PrivilegeGroupReadOnly.String():
_, ok := roPrivileges[requestPrivilege]
return ok, nil
case commonpb.ObjectPrivilege_PrivilegeGroupReadWrite.String():
_, ok := rwPrivileges[requestPrivilege]
return ok, nil
case commonpb.ObjectPrivilege_PrivilegeGroupAdmin.String():
_, ok := adminPrivileges[requestPrivilege]
return ok, nil
default:
return false, nil
}
}
115 changes: 115 additions & 0 deletions internal/proxy/privilege_interceptor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -231,3 +231,118 @@ func TestResourceGroupPrivilege(t *testing.T) {
assert.NoError(t, err)
})
}

func TestPrivilegeGroup(t *testing.T) {
ctx := context.Background()

t.Run("Read Only", func(t *testing.T) {
paramtable.Get().Save(Params.CommonCfg.AuthorizationEnabled.Key, "true")

var err error
ctx = GetContext(context.Background(), "fooo:123456")
client := &MockRootCoordClientInterface{}
queryCoord := &mocks.MockQueryCoordClient{}
mgr := newShardClientMgr()

client.listPolicy = func(ctx context.Context, in *internalpb.ListPolicyRequest) (*internalpb.ListPolicyResponse, error) {
return &internalpb.ListPolicyResponse{
Status: merr.Success(),
PolicyInfos: []string{
funcutil.PolicyForPrivilege("role1", commonpb.ObjectType_Global.String(), "*", commonpb.ObjectPrivilege_PrivilegeGroupReadOnly.String(), "default"),
},
UserRoles: []string{
funcutil.EncodeUserRoleCache("fooo", "role1"),
},
}, nil
}
InitMetaCache(ctx, client, queryCoord, mgr)
defer CleanPrivilegeCache()

_, err = PrivilegeInterceptor(GetContext(context.Background(), "fooo:123456"), &milvuspb.QueryRequest{})
assert.NoError(t, err)

_, err = PrivilegeInterceptor(GetContext(context.Background(), "fooo:123456"), &milvuspb.SearchRequest{})
assert.NoError(t, err)

_, err = PrivilegeInterceptor(GetContext(context.Background(), "fooo:123456"), &milvuspb.InsertRequest{})
assert.Error(t, err)
})

t.Run("Read Write", func(t *testing.T) {
paramtable.Get().Save(Params.CommonCfg.AuthorizationEnabled.Key, "true")

var err error
ctx = GetContext(context.Background(), "fooo:123456")
client := &MockRootCoordClientInterface{}
queryCoord := &mocks.MockQueryCoordClient{}
mgr := newShardClientMgr()

client.listPolicy = func(ctx context.Context, in *internalpb.ListPolicyRequest) (*internalpb.ListPolicyResponse, error) {
return &internalpb.ListPolicyResponse{
Status: merr.Success(),
PolicyInfos: []string{
funcutil.PolicyForPrivilege("role1", commonpb.ObjectType_Global.String(), "*", commonpb.ObjectPrivilege_PrivilegeGroupReadWrite.String(), "default"),
},
UserRoles: []string{
funcutil.EncodeUserRoleCache("fooo", "role1"),
},
}, nil
}
InitMetaCache(ctx, client, queryCoord, mgr)
defer CleanPrivilegeCache()

_, err = PrivilegeInterceptor(GetContext(context.Background(), "fooo:123456"), &milvuspb.QueryRequest{})
assert.NoError(t, err)

_, err = PrivilegeInterceptor(GetContext(context.Background(), "fooo:123456"), &milvuspb.SearchRequest{})
assert.NoError(t, err)

_, err = PrivilegeInterceptor(GetContext(context.Background(), "fooo:123456"), &milvuspb.InsertRequest{})
assert.NoError(t, err)

_, err = PrivilegeInterceptor(GetContext(context.Background(), "fooo:123456"), &milvuspb.DeleteRequest{})
assert.NoError(t, err)

_, err = PrivilegeInterceptor(GetContext(context.Background(), "fooo:123456"), &milvuspb.CreateResourceGroupRequest{})
assert.Error(t, err)
})

t.Run("Admin", func(t *testing.T) {
paramtable.Get().Save(Params.CommonCfg.AuthorizationEnabled.Key, "true")

var err error
ctx = GetContext(context.Background(), "fooo:123456")
client := &MockRootCoordClientInterface{}
queryCoord := &mocks.MockQueryCoordClient{}
mgr := newShardClientMgr()

client.listPolicy = func(ctx context.Context, in *internalpb.ListPolicyRequest) (*internalpb.ListPolicyResponse, error) {
return &internalpb.ListPolicyResponse{
Status: merr.Success(),
PolicyInfos: []string{
funcutil.PolicyForPrivilege("role1", commonpb.ObjectType_Global.String(), "*", commonpb.ObjectPrivilege_PrivilegeGroupAdmin.String(), "default"),
},
UserRoles: []string{
funcutil.EncodeUserRoleCache("fooo", "role1"),
},
}, nil
}
InitMetaCache(ctx, client, queryCoord, mgr)
defer CleanPrivilegeCache()

_, err = PrivilegeInterceptor(GetContext(context.Background(), "fooo:123456"), &milvuspb.QueryRequest{})
assert.NoError(t, err)

_, err = PrivilegeInterceptor(GetContext(context.Background(), "fooo:123456"), &milvuspb.SearchRequest{})
assert.NoError(t, err)

_, err = PrivilegeInterceptor(GetContext(context.Background(), "fooo:123456"), &milvuspb.InsertRequest{})
assert.NoError(t, err)

_, err = PrivilegeInterceptor(GetContext(context.Background(), "fooo:123456"), &milvuspb.DeleteRequest{})
assert.NoError(t, err)

_, err = PrivilegeInterceptor(GetContext(context.Background(), "fooo:123456"), &milvuspb.CreateResourceGroupRequest{})
assert.NoError(t, err)
})
}
2 changes: 1 addition & 1 deletion pkg/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ require (
github.com/golang/protobuf v1.5.4
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/klauspost/compress v1.17.7
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240815113856-e2789dca8b59
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240815123953-6dab6fcd6454
github.com/nats-io/nats-server/v2 v2.10.12
github.com/nats-io/nats.go v1.34.1
github.com/panjf2000/ants/v2 v2.7.2
Expand Down
4 changes: 2 additions & 2 deletions pkg/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -494,8 +494,8 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119 h1:9VXijWu
github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240815113856-e2789dca8b59 h1:mKekr0GmCKMpIQh9OJ67TlKVKxDt08600ltARc/JUXY=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240815113856-e2789dca8b59/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240815123953-6dab6fcd6454 h1:JmZCYjMPpiE4ksZw0AUxXWkDY7wwA4fhS+SO1N211Vw=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240815123953-6dab6fcd6454/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A=
github.com/milvus-io/pulsar-client-go v0.6.10/go.mod h1:lQqCkgwDF8YFYjKA+zOheTk1tev2B+bKj5j7+nm8M1w=
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
Expand Down
105 changes: 105 additions & 0 deletions pkg/util/constant.go
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,111 @@ var (
commonpb.ObjectPrivilege_PrivilegeGetFlushState.String(),
},
}

ReadOnlyPrivilegeGroup = []string{
commonpb.ObjectPrivilege_PrivilegeQuery.String(),
commonpb.ObjectPrivilege_PrivilegeSearch.String(),
commonpb.ObjectPrivilege_PrivilegeIndexDetail.String(),
commonpb.ObjectPrivilege_PrivilegeGetFlushState.String(),
commonpb.ObjectPrivilege_PrivilegeGetLoadState.String(),
commonpb.ObjectPrivilege_PrivilegeGetLoadingProgress.String(),
commonpb.ObjectPrivilege_PrivilegeHasPartition.String(),
commonpb.ObjectPrivilege_PrivilegeShowPartitions.String(),
commonpb.ObjectPrivilege_PrivilegeShowCollections.String(),
commonpb.ObjectPrivilege_PrivilegeListAliases.String(),
commonpb.ObjectPrivilege_PrivilegeListDatabases.String(),
commonpb.ObjectPrivilege_PrivilegeDescribeCollection.String(),
commonpb.ObjectPrivilege_PrivilegeDescribeDatabase.String(),
commonpb.ObjectPrivilege_PrivilegeDescribeAlias.String(),
commonpb.ObjectPrivilege_PrivilegeGetStatistics.String(),
}
ReadWritePrivilegeGroup = []string{
commonpb.ObjectPrivilege_PrivilegeQuery.String(),
commonpb.ObjectPrivilege_PrivilegeSearch.String(),
commonpb.ObjectPrivilege_PrivilegeIndexDetail.String(),
commonpb.ObjectPrivilege_PrivilegeGetFlushState.String(),
commonpb.ObjectPrivilege_PrivilegeGetLoadState.String(),
commonpb.ObjectPrivilege_PrivilegeGetLoadingProgress.String(),
commonpb.ObjectPrivilege_PrivilegeHasPartition.String(),
commonpb.ObjectPrivilege_PrivilegeShowPartitions.String(),
commonpb.ObjectPrivilege_PrivilegeShowCollections.String(),
commonpb.ObjectPrivilege_PrivilegeListAliases.String(),
commonpb.ObjectPrivilege_PrivilegeListDatabases.String(),
commonpb.ObjectPrivilege_PrivilegeDescribeCollection.String(),
commonpb.ObjectPrivilege_PrivilegeDescribeDatabase.String(),
commonpb.ObjectPrivilege_PrivilegeDescribeAlias.String(),
commonpb.ObjectPrivilege_PrivilegeGetStatistics.String(),
commonpb.ObjectPrivilege_PrivilegeCreateIndex.String(),
commonpb.ObjectPrivilege_PrivilegeDropIndex.String(),
commonpb.ObjectPrivilege_PrivilegeCreateCollection.String(),
commonpb.ObjectPrivilege_PrivilegeDropCollection.String(),
commonpb.ObjectPrivilege_PrivilegeCreatePartition.String(),
commonpb.ObjectPrivilege_PrivilegeDropPartition.String(),
commonpb.ObjectPrivilege_PrivilegeLoad.String(),
commonpb.ObjectPrivilege_PrivilegeRelease.String(),
commonpb.ObjectPrivilege_PrivilegeInsert.String(),
commonpb.ObjectPrivilege_PrivilegeDelete.String(),
commonpb.ObjectPrivilege_PrivilegeUpsert.String(),
commonpb.ObjectPrivilege_PrivilegeImport.String(),
commonpb.ObjectPrivilege_PrivilegeFlush.String(),
commonpb.ObjectPrivilege_PrivilegeCompaction.String(),
commonpb.ObjectPrivilege_PrivilegeLoadBalance.String(),
commonpb.ObjectPrivilege_PrivilegeRenameCollection.String(),
commonpb.ObjectPrivilege_PrivilegeCreateAlias.String(),
commonpb.ObjectPrivilege_PrivilegeDropAlias.String(),
}
AdminPrivilegeGroup = []string{
commonpb.ObjectPrivilege_PrivilegeQuery.String(),
commonpb.ObjectPrivilege_PrivilegeSearch.String(),
commonpb.ObjectPrivilege_PrivilegeIndexDetail.String(),
commonpb.ObjectPrivilege_PrivilegeGetFlushState.String(),
commonpb.ObjectPrivilege_PrivilegeGetLoadState.String(),
commonpb.ObjectPrivilege_PrivilegeGetLoadingProgress.String(),
commonpb.ObjectPrivilege_PrivilegeHasPartition.String(),
commonpb.ObjectPrivilege_PrivilegeShowPartitions.String(),
commonpb.ObjectPrivilege_PrivilegeShowCollections.String(),
commonpb.ObjectPrivilege_PrivilegeListAliases.String(),
commonpb.ObjectPrivilege_PrivilegeListDatabases.String(),
commonpb.ObjectPrivilege_PrivilegeDescribeCollection.String(),
commonpb.ObjectPrivilege_PrivilegeDescribeDatabase.String(),
commonpb.ObjectPrivilege_PrivilegeDescribeAlias.String(),
commonpb.ObjectPrivilege_PrivilegeGetStatistics.String(),
commonpb.ObjectPrivilege_PrivilegeCreateIndex.String(),
commonpb.ObjectPrivilege_PrivilegeDropIndex.String(),
commonpb.ObjectPrivilege_PrivilegeCreateCollection.String(),
commonpb.ObjectPrivilege_PrivilegeDropCollection.String(),
commonpb.ObjectPrivilege_PrivilegeCreatePartition.String(),
commonpb.ObjectPrivilege_PrivilegeDropPartition.String(),
commonpb.ObjectPrivilege_PrivilegeLoad.String(),
commonpb.ObjectPrivilege_PrivilegeRelease.String(),
commonpb.ObjectPrivilege_PrivilegeInsert.String(),
commonpb.ObjectPrivilege_PrivilegeDelete.String(),
commonpb.ObjectPrivilege_PrivilegeUpsert.String(),
commonpb.ObjectPrivilege_PrivilegeImport.String(),
commonpb.ObjectPrivilege_PrivilegeFlush.String(),
commonpb.ObjectPrivilege_PrivilegeCompaction.String(),
commonpb.ObjectPrivilege_PrivilegeLoadBalance.String(),
commonpb.ObjectPrivilege_PrivilegeRenameCollection.String(),
commonpb.ObjectPrivilege_PrivilegeCreateAlias.String(),
commonpb.ObjectPrivilege_PrivilegeDropAlias.String(),
commonpb.ObjectPrivilege_PrivilegeCreateOwnership.String(),
commonpb.ObjectPrivilege_PrivilegeDropOwnership.String(),
commonpb.ObjectPrivilege_PrivilegeSelectOwnership.String(),
commonpb.ObjectPrivilege_PrivilegeManageOwnership.String(),
commonpb.ObjectPrivilege_PrivilegeSelectUser.String(),
commonpb.ObjectPrivilege_PrivilegeUpdateUser.String(),
commonpb.ObjectPrivilege_PrivilegeCreateResourceGroup.String(),
commonpb.ObjectPrivilege_PrivilegeUpdateResourceGroups.String(),
commonpb.ObjectPrivilege_PrivilegeDropResourceGroup.String(),
commonpb.ObjectPrivilege_PrivilegeDescribeResourceGroup.String(),
commonpb.ObjectPrivilege_PrivilegeListResourceGroups.String(),
commonpb.ObjectPrivilege_PrivilegeTransferReplica.String(),
commonpb.ObjectPrivilege_PrivilegeTransferNode.String(),
commonpb.ObjectPrivilege_PrivilegeCreateDatabase.String(),
commonpb.ObjectPrivilege_PrivilegeDropDatabase.String(),
commonpb.ObjectPrivilege_PrivilegeAlterDatabase.String(),
commonpb.ObjectPrivilege_PrivilegeFlush.String(),
}
)

// StringSet convert array to map for conveniently check if the array contains an element
Expand Down
Loading

0 comments on commit a570567

Please sign in to comment.