diff --git a/pkg/cmd/dev/build.go b/pkg/cmd/dev/build.go index 04b1aaa2b540..fc366d4089ec 100644 --- a/pkg/cmd/dev/build.go +++ b/pkg/cmd/dev/build.go @@ -14,6 +14,7 @@ import ( "context" "fmt" "log" + "os" "path" "path/filepath" "strings" @@ -24,7 +25,10 @@ import ( "github.com/spf13/cobra" ) -const crossFlag = "cross" +const ( + crossFlag = "cross" + hoistGeneratedCodeFlag = "hoist-generated-code" +) // makeBuildCmd constructs the subcommand used to build the specified binaries. func makeBuildCmd(runE func(cmd *cobra.Command, args []string) error) *cobra.Command { @@ -46,6 +50,7 @@ func makeBuildCmd(runE func(cmd *cobra.Command, args []string) error) *cobra.Com You can optionally set a config, as in --cross=windows. Defaults to linux if not specified. The config should be the name of a build configuration specified in .bazelrc, minus the "cross" prefix.`) + buildCmd.Flags().Bool(hoistGeneratedCodeFlag, false, "hoist generated code out of the Bazel sandbox into the workspace") buildCmd.Flags().Lookup(crossFlag).NoOptDefVal = "linux" return buildCmd } @@ -74,6 +79,7 @@ var buildTargetMapping = map[string]string{ func (d *dev) build(cmd *cobra.Command, targets []string) error { ctx := cmd.Context() cross := mustGetFlagString(cmd, crossFlag) + hoistGeneratedCode := mustGetFlagBool(cmd, hoistGeneratedCodeFlag) args, fullTargets, err := getBasicBuildArgs(targets) if err != nil { @@ -85,7 +91,7 @@ func (d *dev) build(cmd *cobra.Command, targets []string) error { if err := d.exec.CommandContextInheritingStdStreams(ctx, "bazel", args...); err != nil { return err } - return d.symlinkBinaries(ctx, fullTargets) + return d.stageArtifacts(ctx, fullTargets, hoistGeneratedCode) } // Cross-compilation case. cross = "cross" + cross @@ -117,7 +123,7 @@ func (d *dev) build(cmd *cobra.Command, targets []string) error { return nil } -func (d *dev) symlinkBinaries(ctx context.Context, targets []string) error { +func (d *dev) stageArtifacts(ctx context.Context, targets []string, hoistGeneratedCode bool) error { workspace, err := d.getWorkspace(ctx) if err != nil { return err @@ -126,20 +132,21 @@ func (d *dev) symlinkBinaries(ctx context.Context, targets []string) error { if err = d.os.MkdirAll(path.Join(workspace, "bin")); err != nil { return err } + bazelBin, err := d.getBazelBin(ctx) + if err != nil { + return err + } for _, target := range targets { - binaryPath, err := d.getPathToBin(ctx, target) - if err != nil { - return err - } + binaryPath := filepath.Join(bazelBin, bazelutil.OutputOfBinaryRule(target)) base := targetToBinBasename(target) var symlinkPath string // Binaries beginning with the string "cockroach" go right at // the top of the workspace; others go in the `bin` directory. if strings.HasPrefix(base, "cockroach") { - symlinkPath = path.Join(workspace, base) + symlinkPath = filepath.Join(workspace, base) } else { - symlinkPath = path.Join(workspace, "bin", base) + symlinkPath = filepath.Join(workspace, "bin", base) } // Symlink from binaryPath -> symlinkPath @@ -156,6 +163,46 @@ func (d *dev) symlinkBinaries(ctx context.Context, targets []string) error { logSuccessfulBuild(target, rel) } + if hoistGeneratedCode { + goFiles, err := d.os.ListFilesWithSuffix(filepath.Join(bazelBin, "pkg"), ".go") + if err != nil { + return err + } + for _, file := range goFiles { + const cockroachURL = "github.com/cockroachdb/cockroach/" + ind := strings.LastIndex(file, cockroachURL) + if ind > 0 { + // If the cockroach URL was found in the filepath, then we should + // trim everything up to and including the URL to find the path + // where the file should be staged. + loc := file[ind+len(cockroachURL):] + err := d.os.CopyFile(file, filepath.Join(workspace, loc)) + if err != nil { + return err + } + continue + } + pathComponents := strings.Split(file, string(os.PathSeparator)) + var skip bool + for _, component := range pathComponents[:len(pathComponents)-1] { + // Pretty decent heuristic for whether a file needs to be staged. + // When path components contain ., they normally are generated files + // from third-party packages, as in google.golang.org. Meanwhile, + // when path components end in _, that usually represents internal + // stuff that doesn't need to be staged, like + // pkg/cmd/dev/dev_test_/testmain.go. Note that generated proto code + // is handled by the cockroach URL case above. + if len(component) > 0 && (strings.ContainsRune(component, '.') || component[len(component)-1] == '_') { + skip = true + } + } + if !skip { + // Failures here don't mean much. Just ignore them. + _ = d.os.CopyFile(file, filepath.Join(workspace, strings.TrimPrefix(file, bazelBin+"/"))) + } + } + } + return nil } @@ -170,15 +217,6 @@ func targetToBinBasename(target string) string { return base } -func (d *dev) getPathToBin(ctx context.Context, target string) (string, error) { - bazelBin, err := d.getBazelBin(ctx) - if err != nil { - return "", err - } - rel := bazelutil.OutputOfBinaryRule(target) - return filepath.Join(bazelBin, rel), nil -} - // getBasicBuildArgs is for enumerating the arguments to pass to `bazel` in // order to build the given high-level targets. // The first string slice returned is the list of arguments (i.e. to pass to diff --git a/pkg/cmd/dev/generate.go b/pkg/cmd/dev/generate.go index 1b7d877264a7..a1248fe93cc8 100644 --- a/pkg/cmd/dev/generate.go +++ b/pkg/cmd/dev/generate.go @@ -28,10 +28,10 @@ func makeGenerateCmd(runE func(cmd *cobra.Command, args []string) error) *cobra. Short: `Generate the specified files`, Long: `Generate the specified files.`, Example: ` - dev generate - dev generate bazel - dev generate protobuf - dev generate {exec,opt}gen`, + dev generate + dev generate bazel + dev generate docs +`, Args: cobra.MinimumNArgs(0), // TODO(irfansharif): Errors but default just eaten up. Let's wrap these // invocations in something that prints out the appropriate error log @@ -41,10 +41,12 @@ func makeGenerateCmd(runE func(cmd *cobra.Command, args []string) error) *cobra. } func (d *dev) generate(cmd *cobra.Command, targets []string) error { - // TODO(irfansharif): Flesh out the remaining targets. var generatorTargetMapping = map[string]func(cmd *cobra.Command) error{ - "bazel": d.generateBazel, - "docs": d.generateDocs, + "bazel": d.generateBazel, + "docs": d.generateDocs, + "execgen": d.generateUnimplemented, + "optgen": d.generateUnimplemented, + "proto": d.generateUnimplemented, } if len(targets) == 0 { @@ -129,3 +131,7 @@ func (d *dev) generateDocs(cmd *cobra.Command) error { } return nil } + +func (*dev) generateUnimplemented(*cobra.Command) error { + return errors.New("To generate Go code, run `dev build` with the flag `--hoist-generated-code`") +} diff --git a/pkg/cmd/dev/io/os/os.go b/pkg/cmd/dev/io/os/os.go index 8202bb3e2539..b8f46b24d37e 100644 --- a/pkg/cmd/dev/io/os/os.go +++ b/pkg/cmd/dev/io/os/os.go @@ -13,9 +13,12 @@ package os import ( "fmt" "io" + "io/fs" "io/ioutil" "log" "os" + "path/filepath" + "strings" "github.com/cockroachdb/cockroach/pkg/cmd/dev/recording" "github.com/cockroachdb/errors/oserror" @@ -206,6 +209,40 @@ func (o *OS) CopyFile(src, dst string) error { return err } +// ListFilesWithSuffix lists all the files under a directory recursively that +// end in the given suffix. +func (o *OS) ListFilesWithSuffix(root, suffix string) ([]string, error) { + command := fmt.Sprintf("find %s -name *%s", root, suffix) + o.logger.Print(command) + + var ret []string + if o.Recording == nil { + // Do the real thing. + err := filepath.Walk(root, func(path string, info fs.FileInfo, err error) error { + // If there's an error walking the tree, throw it away -- there's nothing + // interesting we can do with it. + if err != nil || info.IsDir() { + //nolint:returnerrcheck + return nil + } + if strings.HasSuffix(path, suffix) { + ret = append(ret, path) + } + return nil + }) + if err != nil { + return nil, err + } + return ret, nil + } + + lines, err := o.replay(command) + if err != nil { + return nil, err + } + return strings.Split(strings.TrimSpace(lines), "\n"), nil +} + // replay replays the specified command, erroring out if it's mismatched with // what the recording plays back next. It returns the recorded output. func (o *OS) replay(command string) (output string, err error) { diff --git a/pkg/cmd/dev/test.go b/pkg/cmd/dev/test.go index 9f6616e8d19d..e13506a754ca 100644 --- a/pkg/cmd/dev/test.go +++ b/pkg/cmd/dev/test.go @@ -12,8 +12,10 @@ package main import ( "fmt" + "path/filepath" "strings" + bazelutil "github.com/cockroachdb/cockroach/pkg/build/util" "github.com/cockroachdb/errors" "github.com/spf13/cobra" ) @@ -99,10 +101,11 @@ func (d *dev) runUnitTest(cmd *cobra.Command, pkgs []string) error { if err != nil { return err } - stressBin, err = d.getPathToBin(ctx, stressTarget) + bazelBin, err := d.getBazelBin(ctx) if err != nil { return err } + stressBin = filepath.Join(bazelBin, bazelutil.OutputOfBinaryRule(stressTarget)) } var args []string diff --git a/pkg/cmd/dev/testdata/build.txt b/pkg/cmd/dev/testdata/build.txt index 712e95f3c419..1e87b2ce761a 100644 --- a/pkg/cmd/dev/testdata/build.txt +++ b/pkg/cmd/dev/testdata/build.txt @@ -62,3 +62,18 @@ mkdir go/src/github.com/cockroachdb/cockroach/bin bazel info bazel-bin --color=no rm go/src/github.com/cockroachdb/cockroach/cockroach-short ln -s /private/var/tmp/_bazel/99e666e4e674209ecdb66b46371278df/execroot/cockroach/bazel-out/darwin-fastbuild/bin/pkg/cmd/cockroach-short/cockroach-short_/cockroach-short go/src/github.com/cockroachdb/cockroach/cockroach-short + +dev build cockroach-short --hoist-generated-code +---- +getenv PATH +which cc +readlink /usr/local/opt/ccache/libexec/cc +export PATH=/usr/local/opt/make/libexec/gnubin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/go/bin:/Library/Apple/usr/bin +bazel build --color=yes --experimental_convenience_symlinks=ignore //pkg/cmd/cockroach-short --config=dev +bazel info workspace --color=no --config=dev +mkdir go/src/github.com/cockroachdb/cockroach/bin +bazel info bazel-bin --color=no --config=dev +rm go/src/github.com/cockroachdb/cockroach/cockroach-short +ln -s /private/var/tmp/_bazel/99e666e4e674209ecdb66b46371278df/execroot/cockroach/bazel-out/darwin-fastbuild/bin/pkg/cmd/cockroach-short/cockroach-short_/cockroach-short go/src/github.com/cockroachdb/cockroach/cockroach-short +find /private/var/tmp/_bazel/99e666e4e674209ecdb66b46371278df/execroot/cockroach/bazel-out/darwin-fastbuild/bin/pkg -name *.go +cp /private/var/tmp/_bazel/99e666e4e674209ecdb66b46371278df/execroot/cockroach/bazel-out/darwin-fastbuild/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go diff --git a/pkg/cmd/dev/testdata/recording/build.txt b/pkg/cmd/dev/testdata/recording/build.txt index bbafb7c14e2f..e320645b97a5 100644 --- a/pkg/cmd/dev/testdata/recording/build.txt +++ b/pkg/cmd/dev/testdata/recording/build.txt @@ -172,3 +172,50 @@ rm go/src/github.com/cockroachdb/cockroach/cockroach-short ln -s /private/var/tmp/_bazel/99e666e4e674209ecdb66b46371278df/execroot/cockroach/bazel-out/darwin-fastbuild/bin/pkg/cmd/cockroach-short/cockroach-short_/cockroach-short go/src/github.com/cockroachdb/cockroach/cockroach-short ---- + +getenv PATH +---- +/usr/local/opt/ccache/libexec:/usr/local/opt/make/libexec/gnubin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/go/bin:/Library/Apple/usr/bin + +which cc +---- +/usr/local/opt/ccache/libexec/cc + +readlink /usr/local/opt/ccache/libexec/cc +---- +../bin/ccache + +export PATH=/usr/local/opt/make/libexec/gnubin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/go/bin:/Library/Apple/usr/bin +---- + +bazel build --color=yes --experimental_convenience_symlinks=ignore //pkg/cmd/cockroach-short --config=dev +---- + +bazel info workspace --color=no --config=dev +---- +go/src/github.com/cockroachdb/cockroach + +mkdir go/src/github.com/cockroachdb/cockroach/bin +---- + +bazel info bazel-bin --color=no --config=dev +---- +/private/var/tmp/_bazel/99e666e4e674209ecdb66b46371278df/execroot/cockroach/bazel-out/darwin-fastbuild/bin + +rm go/src/github.com/cockroachdb/cockroach/cockroach-short +---- + +ln -s /private/var/tmp/_bazel/99e666e4e674209ecdb66b46371278df/execroot/cockroach/bazel-out/darwin-fastbuild/bin/pkg/cmd/cockroach-short/cockroach-short_/cockroach-short go/src/github.com/cockroachdb/cockroach/cockroach-short +---- + +find /private/var/tmp/_bazel/99e666e4e674209ecdb66b46371278df/execroot/cockroach/bazel-out/darwin-fastbuild/bin/pkg -name *.go +---- +---- +/private/var/tmp/_bazel/99e666e4e674209ecdb66b46371278df/execroot/cockroach/bazel-out/darwin-fastbuild/bin/_bazel/bin/pkg/cmd/dev/dev_test_/testmain.go +/private/var/tmp/_bazel/99e666e4e674209ecdb66b46371278df/execroot/cockroach/bazel-out/darwin-fastbuild/bin/pkg/kv/kvclient/rangefeed/mock_rangefeed_gomock_gopath/src/google.golang.org/grpc/preloader.go +/private/var/tmp/_bazel/99e666e4e674209ecdb66b46371278df/execroot/cockroach/bazel-out/darwin-fastbuild/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go +---- +---- + +cp /private/var/tmp/_bazel/99e666e4e674209ecdb66b46371278df/execroot/cockroach/bazel-out/darwin-fastbuild/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go +----