-
Notifications
You must be signed in to change notification settings - Fork 3.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
jobsprofiler: store DistSQL diagram of jobs in job info
This change teaches import, cdc, backup and restore to store their DistSQL plans in the job_info table under a timestamped info key. The generation and writing of the plan diagram is done asynchronously so as to not slow down the execution of the job. A new plan will be stored everytime the job sets up its DistSQL flow. Release note: None Epic: CRDB-8964
- Loading branch information
1 parent
83430fc
commit da25b21
Showing
15 changed files
with
279 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| load("//build/bazelutil/unused_checker:unused.bzl", "get_x_data") | ||
| load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") | ||
|
|
||
| go_library( | ||
| name = "jobsprofiler", | ||
| srcs = ["profiler.go"], | ||
| importpath = "github.com/cockroachdb/cockroach/pkg/jobs/jobsprofiler", | ||
| visibility = ["//visibility:public"], | ||
| deps = [ | ||
| "//pkg/jobs", | ||
| "//pkg/jobs/jobspb", | ||
| "//pkg/sql", | ||
| "//pkg/sql/execinfrapb", | ||
| "//pkg/sql/isql", | ||
| "//pkg/util/log", | ||
| "//pkg/util/stop", | ||
| "//pkg/util/timeutil", | ||
| ], | ||
| ) | ||
|
|
||
| go_test( | ||
| name = "jobsprofiler_test", | ||
| srcs = [ | ||
| "main_test.go", | ||
| "profiler_test.go", | ||
| ], | ||
| args = ["-test.timeout=295s"], | ||
| deps = [ | ||
| "//pkg/base", | ||
| "//pkg/ccl", | ||
| "//pkg/jobs", | ||
| "//pkg/jobs/jobspb", | ||
| "//pkg/security/securityassets", | ||
| "//pkg/security/securitytest", | ||
| "//pkg/server", | ||
| "//pkg/sql", | ||
| "//pkg/sql/isql", | ||
| "//pkg/testutils", | ||
| "//pkg/testutils/serverutils", | ||
| "//pkg/testutils/testcluster", | ||
| "//pkg/util/leaktest", | ||
| "//pkg/util/log", | ||
| "//pkg/util/randutil", | ||
| "@com_github_cockroachdb_errors//:errors", | ||
| "@com_github_stretchr_testify//require", | ||
| ], | ||
| ) | ||
|
|
||
| get_x_data(name = "get_x_data") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| // Copyright 2017 The Cockroach Authors. | ||
| // | ||
| // Use of this software is governed by the Business Source License | ||
| // included in the file licenses/BSL.txt. | ||
| // | ||
| // As of the Change Date specified in that file, in accordance with | ||
| // the Business Source License, use of this software will be governed | ||
| // by the Apache License, Version 2.0, included in the file | ||
| // licenses/APL.txt. | ||
|
|
||
| package jobsprofiler_test | ||
|
|
||
| import ( | ||
| "os" | ||
| "testing" | ||
|
|
||
| "github.com/cockroachdb/cockroach/pkg/ccl" | ||
| "github.com/cockroachdb/cockroach/pkg/security/securityassets" | ||
| "github.com/cockroachdb/cockroach/pkg/security/securitytest" | ||
| "github.com/cockroachdb/cockroach/pkg/server" | ||
| "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" | ||
| "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" | ||
| "github.com/cockroachdb/cockroach/pkg/util/randutil" | ||
| ) | ||
|
|
||
| func TestMain(m *testing.M) { | ||
| defer ccl.TestingEnableEnterprise()() | ||
| securityassets.SetLoader(securitytest.EmbeddedAssets) | ||
| randutil.SeedForTests() | ||
| serverutils.InitTestServerFactory(server.TestServerFactory) | ||
| serverutils.InitTestClusterFactory(testcluster.TestClusterFactory) | ||
| os.Exit(m.Run()) | ||
| } | ||
|
|
||
| //go:generate ../../util/leaktest/add-leaktest.sh *_test.go |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| // Copyright 2023 The Cockroach Authors. | ||
| // | ||
| // Use of this software is governed by the Business Source License | ||
| // included in the file licenses/BSL.txt. | ||
| // | ||
| // As of the Change Date specified in that file, in accordance with | ||
| // the Business Source License, use of this software will be governed | ||
| // by the Apache License, Version 2.0, included in the file | ||
| // licenses/APL.txt. | ||
|
|
||
| package jobsprofiler | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
|
|
||
| "github.com/cockroachdb/cockroach/pkg/jobs" | ||
| "github.com/cockroachdb/cockroach/pkg/jobs/jobspb" | ||
| "github.com/cockroachdb/cockroach/pkg/sql" | ||
| "github.com/cockroachdb/cockroach/pkg/sql/execinfrapb" | ||
| "github.com/cockroachdb/cockroach/pkg/sql/isql" | ||
| "github.com/cockroachdb/cockroach/pkg/util/log" | ||
| "github.com/cockroachdb/cockroach/pkg/util/stop" | ||
| "github.com/cockroachdb/cockroach/pkg/util/timeutil" | ||
| ) | ||
|
|
||
| // StorePlanDiagram stores the DistSQL diagram generated from p in the job info | ||
| // table. The generation of the plan diagram and persistence to the info table | ||
| // are done asynchronously and this method does not block on their completion. | ||
| func StorePlanDiagram( | ||
| ctx context.Context, stopper *stop.Stopper, p *sql.PhysicalPlan, db isql.DB, jobID jobspb.JobID, | ||
| ) { | ||
| if err := stopper.RunAsyncTask(ctx, "jobs-store-plan-diagram", func(ctx context.Context) { | ||
| var cancel func() | ||
| ctx, cancel = stopper.WithCancelOnQuiesce(ctx) | ||
| defer cancel() | ||
|
|
||
| err := db.Txn(ctx, func(ctx context.Context, txn isql.Txn) error { | ||
| flowSpecs := p.GenerateFlowSpecs() | ||
| _, diagURL, err := execinfrapb.GeneratePlanDiagramURL(fmt.Sprintf("job:%d", jobID), flowSpecs, execinfrapb.DiagramFlags{}) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| const infoKey = "dsp-diag-url-%d" | ||
| infoStorage := jobs.InfoStorageForJob(txn, jobID) | ||
| return infoStorage.Write(ctx, []byte(fmt.Sprintf(infoKey, timeutil.Now().UnixNano())), | ||
| []byte(diagURL.String())) | ||
| }) | ||
| if err != nil { | ||
| log.Warningf(ctx, "failed to generate and write DistSQL diagram for job %d: %v", | ||
| jobID, err.Error()) | ||
| } | ||
| }); err != nil { | ||
| log.Warningf(ctx, "failed to spawn task to generate DistSQL plan diagram for job %d: %v", | ||
| jobID, err.Error()) | ||
| } | ||
| } |
Oops, something went wrong.