Skip to content

Commit

Permalink
Merge #94644 #94754
Browse files Browse the repository at this point in the history
94644: tenantcapabilities: introduce TenantCapabilities proto r=ecwall a=arulajmani

This commit adds the skeletal structure for a `TenantCapabilities` proto,
which is intended to encapsulate capabilities for a specific tenant.
Capabilities are intended to be stored in the `system.tenants` table,
in its Info column. To that end, we modify `TenantInfo` to contain
capabilities. However, actually populating this field through SQL is
left to a future commit.

Future commits will also add the infrastructure required to check a
tenant's requests against its capabilities for "privileged" operations.
For now, I've only accounted for the `CanAdminSplit` capability -- this
will likely expand to a fuller set as we introduce other capabilities
in the system.

References #94643

Epic: CRDB-18503
Release note: None

94754: split: init replica lb splitter with global rand r=erikgrinaker a=kvoli

In #93838 we started initializing a new seeded rand for use in the load
based splitter's reservoir sampling algorithm. Previously, the splitter
was initialized using the global rand.

`rand.Source` heap allocates on init approximately 4kb. When initialized
per-replica, this is problematic as nodes scale to large replica counts.

This patch replaces initializing a new random source per-replica with
using the global rand instead.

This removes the heap allocation. This is shown below running 
`kv/splits/nodes=3/quiesce=true` (not at identical replica counts in the test,
 the proportions are the important part).

before:

![image](https://user-images.githubusercontent.com/39606633/210825416-a940904f-47c9-4271-9692-11b40be927f4.png)

after:

![image](https://user-images.githubusercontent.com/39606633/210825742-fc62ea6c-2125-44fe-ac07-984c22986089.png)

resolves: #94752
resolves: #94737

Release note: None

Co-authored-by: Arul Ajmani <[email protected]>
Co-authored-by: Austen McClernon <[email protected]>
  • Loading branch information
3 people committed Jan 5, 2023
3 parents 6488430 + e5a7093 + 891e7a0 commit e7713f1
Show file tree
Hide file tree
Showing 18 changed files with 242 additions and 61 deletions.
1 change: 1 addition & 0 deletions docs/generated/http/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ genrule(
"//pkg/kv/kvserver/liveness/livenesspb:livenesspb_proto",
"//pkg/kv/kvserver/loqrecovery/loqrecoverypb:loqrecoverypb_proto",
"//pkg/kv/kvserver/readsummary/rspb:rspb_proto",
"//pkg/multitenant/tenantcapabilities/tenantcapabilitiespb:tenantcapabilitiespb_proto",
"//pkg/roachpb:roachpb_proto",
"//pkg/server/diagnostics/diagnosticspb:diagnosticspb_proto",
"//pkg/server/serverpb:serverpb_proto",
Expand Down
2 changes: 2 additions & 0 deletions pkg/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -1261,6 +1261,7 @@ GO_TARGETS = [
"//pkg/kv:kv_test",
"//pkg/multitenant/multitenantcpu:multitenantcpu",
"//pkg/multitenant/multitenantio:multitenantio",
"//pkg/multitenant/tenantcapabilities/tenantcapabilitiespb:tenantcapabilitiespb",
"//pkg/multitenant/tenantcostmodel:tenantcostmodel",
"//pkg/multitenant:multitenant",
"//pkg/obs:obs",
Expand Down Expand Up @@ -2592,6 +2593,7 @@ GET_X_DATA_TARGETS = [
"//pkg/multitenant:get_x_data",
"//pkg/multitenant/multitenantcpu:get_x_data",
"//pkg/multitenant/multitenantio:get_x_data",
"//pkg/multitenant/tenantcapabilities/tenantcapabilitiespb:get_x_data",
"//pkg/multitenant/tenantcostmodel:get_x_data",
"//pkg/obs:get_x_data",
"//pkg/obsservice/cmd/obsservice:get_x_data",
Expand Down
124 changes: 106 additions & 18 deletions pkg/ccl/backupccl/backup_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6891,14 +6891,28 @@ func TestBackupRestoreTenant(t *testing.T) {
restoreDB := sqlutils.MakeSQLRunner(restoreTC.Conns[0])

restoreDB.CheckQueryResults(t, `select id, active, name, crdb_internal.pb_to_json('cockroach.sql.sqlbase.TenantInfo', info, true) from system.tenants`, [][]string{
{`1`, `true`, `system`, `{"droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}`},
{`1`,
`true`,
`system`,
`{"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}`,
},
})
restoreDB.Exec(t, `RESTORE TENANT 10 FROM 'nodelocal://1/t10'`)
restoreDB.CheckQueryResults(t,
`SELECT id, active, name, crdb_internal.pb_to_json('cockroach.sql.sqlbase.TenantInfo', info, true) FROM system.tenants`,
[][]string{
{`1`, `true`, `system`, `{"droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}`},
{`10`, `true`, `NULL`, `{"droppedName": "", "id": "10", "name": "", "state": "ACTIVE", "tenantReplicationJobId": "0"}`},
{
`1`,
`true`,
`system`,
`{"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}`,
},
{
`10`,
`true`,
`NULL`,
`{"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "10", "name": "", "state": "ACTIVE", "tenantReplicationJobId": "0"}`,
},
},
)
restoreDB.CheckQueryResults(t,
Expand Down Expand Up @@ -6927,8 +6941,18 @@ func TestBackupRestoreTenant(t *testing.T) {
restoreDB.CheckQueryResults(t,
`select id, active, name, crdb_internal.pb_to_json('cockroach.sql.sqlbase.TenantInfo', info, true) from system.tenants`,
[][]string{
{`1`, `true`, `system`, `{"droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}`},
{`10`, `false`, `NULL`, `{"droppedName": "", "id": "10", "name": "", "state": "DROP", "tenantReplicationJobId": "0"}`},
{
`1`,
`true`,
`system`,
`{"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}`,
},
{
`10`,
`false`,
`NULL`,
`{"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "10", "name": "", "state": "DROP", "tenantReplicationJobId": "0"}`,
},
},
)

Expand All @@ -6952,8 +6976,18 @@ func TestBackupRestoreTenant(t *testing.T) {
restoreDB.CheckQueryResults(t,
`select id, active, name, crdb_internal.pb_to_json('cockroach.sql.sqlbase.TenantInfo', info, true) from system.tenants`,
[][]string{
{`1`, `true`, `system`, `{"droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}`},
{`10`, `true`, `NULL`, `{"droppedName": "", "id": "10", "name": "", "state": "ACTIVE", "tenantReplicationJobId": "0"}`},
{
`1`,
`true`,
`system`,
`{"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}`,
},
{
`10`,
`true`,
`NULL`,
`{"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "10", "name": "", "state": "ACTIVE", "tenantReplicationJobId": "0"}`,
},
},
)

Expand All @@ -6977,14 +7011,29 @@ func TestBackupRestoreTenant(t *testing.T) {
restoreDB.CheckQueryResults(t,
`select id, active, name, crdb_internal.pb_to_json('cockroach.sql.sqlbase.TenantInfo', info, true) from system.tenants`,
[][]string{
{`1`, `true`, `system`, `{"droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}`},
{
`1`,
`true`,
`system`,
`{"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}`,
},
})
restoreDB.Exec(t, `RESTORE TENANT 10 FROM 'nodelocal://1/t10'`)
restoreDB.CheckQueryResults(t,
`select id, active, name, crdb_internal.pb_to_json('cockroach.sql.sqlbase.TenantInfo', info, true) from system.tenants`,
[][]string{
{`1`, `true`, `system`, `{"droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}`},
{`10`, `true`, `NULL`, `{"droppedName": "", "id": "10", "name": "", "state": "ACTIVE", "tenantReplicationJobId": "0"}`},
{
`1`,
`true`,
`system`,
`{"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}`,
},
{
`10`,
`true`,
`NULL`,
`{"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "10", "name": "", "state": "ACTIVE", "tenantReplicationJobId": "0"}`,
},
},
)
})
Expand All @@ -7006,14 +7055,29 @@ func TestBackupRestoreTenant(t *testing.T) {
restoreDB.CheckQueryResults(t,
`select id, active, name, crdb_internal.pb_to_json('cockroach.sql.sqlbase.TenantInfo', info, true) from system.tenants`,
[][]string{
{`1`, `true`, `system`, `{"droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}`},
{
`1`,
`true`,
`system`,
`{"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}`,
},
})
restoreDB.Exec(t, `RESTORE TENANT 10 FROM 'nodelocal://1/clusterwide'`)
restoreDB.CheckQueryResults(t,
`select id, active, name, crdb_internal.pb_to_json('cockroach.sql.sqlbase.TenantInfo', info, true) from system.tenants`,
[][]string{
{`1`, `true`, `system`, `{"droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}`},
{`10`, `true`, `NULL`, `{"droppedName": "", "id": "10", "name": "", "state": "ACTIVE", "tenantReplicationJobId": "0"}`},
{
`1`,
`true`,
`system`,
`{"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}`,
},
{
`10`,
`true`,
`NULL`,
`{"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "10", "name": "", "state": "ACTIVE", "tenantReplicationJobId": "0"}`,
},
},
)

Expand Down Expand Up @@ -7046,16 +7110,40 @@ func TestBackupRestoreTenant(t *testing.T) {
restoreDB.CheckQueryResults(t,
`select id, active, name, crdb_internal.pb_to_json('cockroach.sql.sqlbase.TenantInfo', info, true) from system.tenants`,
[][]string{
{`1`, `true`, `system`, `{"droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}`},
{
`1`,
`true`,
`system`,
`{"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}`,
},
})
restoreDB.Exec(t, `RESTORE FROM 'nodelocal://1/clusterwide'`)
restoreDB.CheckQueryResults(t,
`select id, active, name, crdb_internal.pb_to_json('cockroach.sql.sqlbase.TenantInfo', info, true) from system.tenants`,
[][]string{
{`1`, `true`, `system`, `{"droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}`},
{`10`, `true`, `NULL`, `{"droppedName": "", "id": "10", "name": "", "state": "ACTIVE", "tenantReplicationJobId": "0"}`},
{`11`, `true`, `NULL`, `{"droppedName": "", "id": "11", "name": "", "state": "ACTIVE", "tenantReplicationJobId": "0"}`},
{`20`, `true`, `NULL`, `{"droppedName": "", "id": "20", "name": "", "state": "ACTIVE", "tenantReplicationJobId": "0"}`},
{
`1`,
`true`, `system`,
`{"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}`,
},
{
`10`,
`true`,
`NULL`,
`{"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "10", "name": "", "state": "ACTIVE", "tenantReplicationJobId": "0"}`,
},
{
`11`,
`true`,
`NULL`,
`{"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "11", "name": "", "state": "ACTIVE", "tenantReplicationJobId": "0"}`,
},
{
`20`,
`true`,
`NULL`,
`{"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "20", "name": "", "state": "ACTIVE", "tenantReplicationJobId": "0"}`,
},
},
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,4 +78,3 @@ exec-sql user=testuser expect-error-regex=(only users with the admin role are al
alter backup schedule $fullID set schedule option updates_cluster_last_backup_time_metric = '1';
----
regex matches error

Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,3 @@ query-sql
SELECT * FROM [SHOW SCHEDULES] WHERE label='hello';
----


Original file line number Diff line number Diff line change
Expand Up @@ -584,4 +584,3 @@ foofoo
baz
show_cluster_backup
show_database_backup

20 changes: 10 additions & 10 deletions pkg/ccl/backupccl/testdata/backup-restore/restore-tenants
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@ SELECT crdb_internal.destroy_tenant(5);
query-sql
SELECT id,active,crdb_internal.pb_to_json('cockroach.sql.sqlbase.TenantInfo', info, true) FROM system.tenants;
----
1 true {"droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}
5 false {"droppedName": "", "id": "5", "name": "", "state": "DROP", "tenantReplicationJobId": "0"}
6 true {"droppedName": "", "id": "6", "name": "", "state": "ACTIVE", "tenantReplicationJobId": "0"}
1 true {"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}
5 false {"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "5", "name": "", "state": "DROP", "tenantReplicationJobId": "0"}
6 true {"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "6", "name": "", "state": "ACTIVE", "tenantReplicationJobId": "0"}

exec-sql
BACKUP INTO 'nodelocal://1/cluster'
Expand All @@ -49,9 +49,9 @@ RESTORE FROM LATEST IN 'nodelocal://1/cluster'
query-sql
SELECT id,active,crdb_internal.pb_to_json('cockroach.sql.sqlbase.TenantInfo', info, true) FROM system.tenants;
----
1 true {"droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}
5 false {"droppedName": "", "id": "5", "name": "", "state": "DROP", "tenantReplicationJobId": "0"}
6 true {"droppedName": "", "id": "6", "name": "", "state": "ACTIVE", "tenantReplicationJobId": "0"}
1 true {"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}
5 false {"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "5", "name": "", "state": "DROP", "tenantReplicationJobId": "0"}
6 true {"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "6", "name": "", "state": "ACTIVE", "tenantReplicationJobId": "0"}

exec-sql
RESTORE TENANT 6 FROM LATEST IN 'nodelocal://1/tenant6' WITH tenant = '7';
Expand All @@ -60,7 +60,7 @@ RESTORE TENANT 6 FROM LATEST IN 'nodelocal://1/tenant6' WITH tenant = '7';
query-sql
SELECT id,active,crdb_internal.pb_to_json('cockroach.sql.sqlbase.TenantInfo', info, true) FROM system.tenants;
----
1 true {"droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}
5 false {"droppedName": "", "id": "5", "name": "", "state": "DROP", "tenantReplicationJobId": "0"}
6 true {"droppedName": "", "id": "6", "name": "", "state": "ACTIVE", "tenantReplicationJobId": "0"}
7 true {"droppedName": "", "id": "7", "name": "", "state": "ACTIVE", "tenantReplicationJobId": "0"}
1 true {"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "1", "name": "system", "state": "ACTIVE", "tenantReplicationJobId": "0"}
5 false {"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "5", "name": "", "state": "DROP", "tenantReplicationJobId": "0"}
6 true {"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "6", "name": "", "state": "ACTIVE", "tenantReplicationJobId": "0"}
7 true {"capabilities": {"canAdminSplit": false}, "droppedName": "", "id": "7", "name": "", "state": "ACTIVE", "tenantReplicationJobId": "0"}
1 change: 1 addition & 0 deletions pkg/gen/protobuf.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ PROTOBUF_SRCS = [
"//pkg/kv/kvserver/rangelog/internal/rangelogtestpb:rangelogtestpb_go_proto",
"//pkg/kv/kvserver/readsummary/rspb:rspb_go_proto",
"//pkg/kv/kvserver:kvserver_go_proto",
"//pkg/multitenant/tenantcapabilities/tenantcapabilitiespb:tenantcapabilitiespb_go_proto",
"//pkg/obsservice/obspb/opentelemetry-proto/common/v1:v1_go_proto",
"//pkg/obsservice/obspb/opentelemetry-proto/logs/v1:v1_go_proto",
"//pkg/obsservice/obspb/opentelemetry-proto/resource/v1:v1_go_proto",
Expand Down
4 changes: 1 addition & 3 deletions pkg/kv/kvserver/replica_init.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ package kvserver
import (
"bytes"
"context"
"math/rand"
"time"

"github.com/cockroachdb/cockroach/pkg/keys"
Expand Down Expand Up @@ -96,8 +95,7 @@ func newUnloadedReplica(
r.mu.stateLoader = stateloader.Make(desc.RangeID)
r.mu.quiescent = true
r.mu.conf = store.cfg.DefaultSpanConfig
randSource := rand.New(rand.NewSource(timeutil.Now().UnixNano()))
split.Init(&r.loadBasedSplitter, store.cfg.Settings, randSource, func() float64 {
split.Init(&r.loadBasedSplitter, store.cfg.Settings, split.GlobalRandSource(), func() float64 {
return float64(SplitByLoadQPSThreshold.Get(&store.cfg.Settings.SV))
}, func() time.Duration {
return kvserverbase.SplitByLoadMergeDelay.Get(&store.cfg.Settings.SV)
Expand Down
22 changes: 22 additions & 0 deletions pkg/kv/kvserver/split/decider.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ package split
import (
"context"
"fmt"
"math/rand"
"time"

"github.com/cockroachdb/cockroach/pkg/keys"
Expand Down Expand Up @@ -64,6 +65,27 @@ type RandSource interface {
Intn(n int) int
}

// globalRandSource implements the RandSource interface.
type globalRandSource struct{}

// Float64 returns, as a float64, a pseudo-random number in the half-open
// interval [0.0,1.0) from the RandSource.
func (g globalRandSource) Float64() float64 {
return rand.Float64()
}

// Intn returns, as an int, a non-negative pseudo-random number in the
// half-open interval [0,n).
func (g globalRandSource) Intn(n int) int {
return rand.Intn(n)
}

// GlobalRandSource returns an implementation of the RandSource interface that
// redirects calls to the global rand.
func GlobalRandSource() RandSource {
return globalRandSource{}
}

var enableUnweightedLBSplitFinder = settings.RegisterBoolSetting(
settings.SystemOnly,
"kv.unweighted_lb_split_finder.enabled",
Expand Down
1 change: 1 addition & 0 deletions pkg/multitenant/tenantcapabilities/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
load("//build/bazelutil/unused_checker:unused.bzl", "get_x_data")
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
load("//build/bazelutil/unused_checker:unused.bzl", "get_x_data")
load("@rules_proto//proto:defs.bzl", "proto_library")
load("@io_bazel_rules_go//go:def.bzl", "go_library")
load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")

proto_library(
name = "tenantcapabilitiespb_proto",
srcs = ["capabilities.proto"],
strip_import_prefix = "/pkg",
visibility = ["//visibility:public"],
deps = ["@com_github_gogo_protobuf//gogoproto:gogo_proto"],
)

go_proto_library(
name = "tenantcapabilitiespb_go_proto",
compilers = ["//pkg/cmd/protoc-gen-gogoroach:protoc-gen-gogoroach_compiler"],
importpath = "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities/tenantcapabilitiespb",
proto = ":tenantcapabilitiespb_proto",
visibility = ["//visibility:public"],
deps = ["@com_github_gogo_protobuf//gogoproto"],
)

go_library(
name = "tenantcapabilitiespb",
embed = [":tenantcapabilitiespb_go_proto"],
importpath = "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities/tenantcapabilitiespb",
visibility = ["//visibility:public"],
)

get_x_data(name = "get_x_data")
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
// Copyright 2023 The Cockroach Authors.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.

syntax = "proto3";
package cockroach.multitenant.tenantcapabilities.tenantcapabilitiespb;
option go_package = "tenantcapabilitiespb";

import "gogoproto/gogo.proto";

// TenantCapabilities encapsulates a set of capabilities[1] for a specific
// tenant. Capabilities for a specific tenant are stored in the system.tenants
// table and are checked against in KV when the tenant performs a privileged
// operation.
//
// [1] Certain requests in the system are considered "privileged", and as such,
// tenants are only allowed to perform them if they have the appropriate
// capability. For example, performing an AdminSplit.
message TenantCapabilities {
option (gogoproto.equal) = true;

// CanAdminSplit, if set to true, grants grants the tenant the ability to
// successfully perform `AdminSplit` requests.
bool can_admin_split = 1;
};

Loading

0 comments on commit e7713f1

Please sign in to comment.