Skip to content

Commit

Permalink
roachprod: stage arm64 binary
Browse files Browse the repository at this point in the history 
Add `--arch` to override binary's architecture and refactor.
As of this change, `roachprod stage` is able to stage both
amd64 and arm64 on linux and darwin.
In conjunction with the previous change [1], roachprod
now uses arm64-based AMI for graviton2/graviton3 machines.

Below is an example of how to create a VM with graviton3,
```
roachprod create -n1 --clouds aws --aws-machine-type m7g.2xlarge --local-ssd=false $CRL_USERNAME-test
roachprod stage --arch arm64 $CRL_USERNAME-test release v23.1.0-rc.2
roachprod start $CRL_USERNAME-test
```

[1] cockroachdb#103236

Epic: none
Release note: None
  • Loading branch information
@srosenberg
srosenberg committed May 18, 2023
1 parent 9ae0d28 commit 1e98906
Show file tree
Hide file tree
Showing 7 changed files with 220 additions and 28 deletions.
8 changes: 7 additions & 1 deletion pkg/cmd/roachprod/flags.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ var (
createVMOpts = vm.DefaultCreateOpts()
startOpts = roachprod.DefaultStartOpts()
stageOS string
stageArch string
stageDir string
logsDir string
logsFilter string
Expand Down Expand Up @@ -106,6 +107,8 @@ func initFlags() {
vm.AllProviderNames()))
createCmd.Flags().BoolVar(&createVMOpts.GeoDistributed,
"geo", false, "Create geo-distributed cluster")
createCmd.Flags().BoolVar(&createVMOpts.EnableFIPS,
"fips", false, "Enable FIPS mode (uses custom AMI)")
// N.B. We set "usage=roachprod" as the default, custom label for billing tracking.
createCmd.Flags().StringToStringVar(&createVMOpts.CustomLabels,
"label", map[string]string{"usage": "roachprod"},
Expand Down Expand Up @@ -217,9 +220,12 @@ Default is "RECURRING '*/15 * * * *' FULL BACKUP '@hourly' WITH SCHEDULE OPTIONS
putCmd.Flags().BoolVar(&useTreeDist, "treedist", useTreeDist, "use treedist copy algorithm")

stageCmd.Flags().StringVar(&stageOS, "os", "", "operating system override for staged binaries")
stageCmd.Flags().StringVar(&stageDir, "dir", "", "destination for staged binaries")
stageCmd.Flags().StringVar(&stageArch, "arch", "", "architecture override for staged binaries [amd64, arm64]")

stageCmd.Flags().StringVar(&stageDir, "dir", "", "destination for staged binaries")
// N.B. stageURLCmd just prints the URL that stageCmd would use.
stageURLCmd.Flags().StringVar(&stageOS, "os", "", "operating system override for staged binaries")
stageURLCmd.Flags().StringVar(&stageArch, "arch", "", "architecture override for staged binaries [amd64, arm64]")

logsCmd.Flags().StringVar(&logsFilter,
"filter", "", "re to filter log messages")
Expand Down
36 changes: 33 additions & 3 deletions pkg/cmd/roachprod/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@ The above commands will create a "local" 3 node cluster, start a cockroach
cluster on these nodes, run a sql command on the 2nd node, stop, wipe and
destroy the cluster.
`,
Version: "details:\n" + build.GetInfo().Long(),
Version: "details:\n" + build.GetInfo().Long(),
PersistentPreRun: validateAndConfigure,
}

// Provide `cobra.Command` functions with a standard return code handler.
Expand Down Expand Up @@ -779,7 +780,7 @@ Currently available application options are:
if len(args) == 2 {
versionArg = args[1]
}
urls, err := roachprod.StageURL(config.Logger, args[0], versionArg, stageOS)
urls, err := roachprod.StageURL(config.Logger, args[0], versionArg, stageOS, stageArch)
if err != nil {
return err
}
Expand Down Expand Up @@ -818,7 +819,7 @@ Some examples of usage:
if len(args) == 3 {
versionArg = args[2]
}
return roachprod.Stage(context.Background(), config.Logger, args[0], stageOS, stageDir, args[1], versionArg)
return roachprod.Stage(context.Background(), config.Logger, args[0], stageOS, stageArch, stageDir, args[1], versionArg)
}),
}

Expand Down Expand Up @@ -1141,6 +1142,35 @@ var fixLongRunningAWSHostnamesCmd = &cobra.Command{
}),
}

// Before executing any command, validate and canonicalize args.
func validateAndConfigure(cmd *cobra.Command, args []string) {
// Skip validation for commands that are self-sufficient.
switch cmd.Name() {
case "help", "version", "list":
return
}

printErrAndExit := func(err error) {
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
os.Exit(1)
}
}

// Validate architecture flag, if set.
if archOpt := cmd.Flags().Lookup("arch"); archOpt != nil && archOpt.Changed {
arch := strings.ToLower(archOpt.Value.String())

if arch != "amd64" && arch != "arm64" {
printErrAndExit(fmt.Errorf("unsupported architecture %q", arch))
}
if arch != archOpt.Value.String() {
// Set the canonical value.
_ = cmd.Flags().Set("arch", arch)
}
}
}

func main() {
_ = roachprod.InitProviders()
providerOptsContainer = vm.CreateProviderOptionsContainer()
Expand Down
3 changes: 2 additions & 1 deletion pkg/cmd/roachtest/cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1769,7 +1769,8 @@ func (c *clusterImpl) Stage(
}
c.status("staging binary")
defer c.status("")
return errors.Wrap(roachprod.Stage(ctx, l, c.MakeNodes(opts...), "" /* stageOS */, dir, application, versionOrSHA), "cluster.Stage")
return errors.Wrap(roachprod.Stage(ctx, l, c.MakeNodes(opts...),
"" /* stageOS */, "" /* stageArch */, dir, application, versionOrSHA), "cluster.Stage")
}

// Get gets files from remote hosts.
Expand Down
54 changes: 41 additions & 13 deletions pkg/roachprod/install/staging.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,20 +44,34 @@ type archInfo struct {
}

var (
linuxArchInfo = archInfo{
linux_x86_64_ArchInfo = archInfo{
DebugArchitecture: "linux-gnu-amd64",
ReleaseArchitecture: "linux-amd64",
LibraryExtension: ".so",
ExecutableExtension: "",
ReleaseArchiveExtension: "tgz",
}
darwinArchInfo = archInfo{
linux_arm64_ArchInfo = archInfo{
DebugArchitecture: "linux-3.7.10-gnu-aarch64",
ReleaseArchitecture: "linux-arm64",
LibraryExtension: ".so",
ExecutableExtension: "",
ReleaseArchiveExtension: "tgz",
}
darwin_x86_64_ArchInfo = archInfo{
DebugArchitecture: "darwin-amd64",
ReleaseArchitecture: "darwin-10.9-amd64",
LibraryExtension: ".dylib",
ExecutableExtension: "",
ReleaseArchiveExtension: "tgz",
}
darwin_arm64_ArchInfo = archInfo{
DebugArchitecture: "darwin-11.0-aarch64",
ReleaseArchitecture: "darwin-11.0-arm64",
LibraryExtension: ".dylib",
ExecutableExtension: "",
ReleaseArchiveExtension: "tgz",
}
windowsArchInfo = archInfo{
DebugArchitecture: "windows-amd64",
ReleaseArchitecture: "windows-6.2-amd64",
Expand All @@ -69,14 +83,23 @@ var (
crdbLibraries = []string{"libgeos", "libgeos_c"}
)

// ArchInfoForOS returns an ArchInfo for the given OS if the OS is
// currently supported.
func archInfoForOS(os string) (archInfo, error) {
// ArchInfoForOS returns an ArchInfo for the given OS and Architecture if currently supported.
func archInfoForOS(os string, arch string) (archInfo, error) {
if arch != "" && arch != "amd64" && arch != "arm64" {
return archInfo{}, errors.Errorf("unsupported architecture %q", arch)
}

switch os {
case "linux":
return linuxArchInfo, nil
if arch == "arm64" {
return linux_arm64_ArchInfo, nil
}
return linux_x86_64_ArchInfo, nil
case "darwin":
return darwinArchInfo, nil
if arch == "arm64" {
return darwin_arm64_ArchInfo, nil
}
return darwin_x86_64_ArchInfo, nil
case "windows":
return windowsArchInfo, nil
default:
Expand Down Expand Up @@ -130,16 +153,17 @@ func StageApplication(
applicationName string,
version string,
os string,
arch string,
destDir string,
) error {
archInfo, err := archInfoForOS(os)
archInfo, err := archInfoForOS(os, arch)
if err != nil {
return err
}

switch applicationName {
case "cockroach":
err := StageRemoteBinary(
err := stageRemoteBinary(
ctx, l, c, applicationName, "cockroach/cockroach", version, archInfo.DebugArchitecture, destDir,
)
if err != nil {
Expand All @@ -164,7 +188,8 @@ func StageApplication(
}
return nil
case "workload":
err := StageRemoteBinary(
// N.B. workload binary is only available for linux amd64: https://github.com/cockroachdb/cockroach/issues/103563
err := stageRemoteBinary(
ctx, l, c, applicationName, "cockroach/workload", version, "" /* arch */, destDir,
)
return err
Expand All @@ -177,8 +202,10 @@ func StageApplication(

// URLsForApplication returns a slice of URLs that should be
// downloaded for the given application.
func URLsForApplication(application string, version string, os string) ([]*url.URL, error) {
archInfo, err := archInfoForOS(os)
func URLsForApplication(
application string, version string, os string, arch string,
) ([]*url.URL, error) {
archInfo, err := archInfoForOS(os, arch)
if err != nil {
return nil, err
}
Expand All @@ -205,6 +232,7 @@ func URLsForApplication(application string, version string, os string) ([]*url.U
}
return urls, nil
case "workload":
// N.B. workload binary is only available for linux amd64: https://github.com/cockroachdb/cockroach/issues/103563
u, err := getEdgeURL("cockroach/workload", version, "" /* arch */, "" /* extension */)
if err != nil {
return nil, err
Expand All @@ -225,7 +253,7 @@ func URLsForApplication(application string, version string, os string) ([]*url.U
// application path to each specified by the cluster to the specified directory.
// If no SHA is specified, the latest build of the binary is used instead.
// Returns the SHA of the resolved binary.
func StageRemoteBinary(
func stageRemoteBinary(
ctx context.Context,
l *logger.Logger,
c *SyncedCluster,
Expand Down
105 changes: 104 additions & 1 deletion pkg/roachprod/install/staging_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ func TestURLsForApplication(t *testing.T) {
application string
version string
os string
arch string
}
tests := []struct {
name string
Expand All @@ -41,6 +42,20 @@ func TestURLsForApplication(t *testing.T) {
"https://storage.googleapis.com/cockroach-edge-artifacts-prod/cockroach/lib/libgeos_c.linux-gnu-amd64.563ea3967c98c67d47ede30d895c82315e4b1a77.so",
},
},
{
name: "cockroach linux arm64 sha",
args: args{
application: "cockroach",
version: "563ea3967c98c67d47ede30d895c82315e4b1a77",
os: "linux",
arch: "arm64",
},
want: []string{
"https://storage.googleapis.com/cockroach-edge-artifacts-prod/cockroach/cockroach.linux-3.7.10-gnu-aarch64.563ea3967c98c67d47ede30d895c82315e4b1a77",
"https://storage.googleapis.com/cockroach-edge-artifacts-prod/cockroach/lib/libgeos.linux-3.7.10-gnu-aarch64.563ea3967c98c67d47ede30d895c82315e4b1a77.so",
"https://storage.googleapis.com/cockroach-edge-artifacts-prod/cockroach/lib/libgeos_c.linux-3.7.10-gnu-aarch64.563ea3967c98c67d47ede30d895c82315e4b1a77.so",
},
},
{
name: "cockroach darwin sha",
args: args{
Expand All @@ -54,6 +69,20 @@ func TestURLsForApplication(t *testing.T) {
"https://storage.googleapis.com/cockroach-edge-artifacts-prod/cockroach/lib/libgeos_c.darwin-amd64.563ea3967c98c67d47ede30d895c82315e4b1a77.dylib",
},
},
{
name: "cockroach darwin arm64 sha",
args: args{
application: "cockroach",
version: "563ea3967c98c67d47ede30d895c82315e4b1a77",
os: "darwin",
arch: "arm64",
},
want: []string{
"https://storage.googleapis.com/cockroach-edge-artifacts-prod/cockroach/cockroach.darwin-11.0-aarch64.563ea3967c98c67d47ede30d895c82315e4b1a77",
"https://storage.googleapis.com/cockroach-edge-artifacts-prod/cockroach/lib/libgeos.darwin-11.0-aarch64.563ea3967c98c67d47ede30d895c82315e4b1a77.dylib",
"https://storage.googleapis.com/cockroach-edge-artifacts-prod/cockroach/lib/libgeos_c.darwin-11.0-aarch64.563ea3967c98c67d47ede30d895c82315e4b1a77.dylib",
},
},
{
name: "cockroach windows sha",
args: args{
Expand Down Expand Up @@ -91,6 +120,20 @@ func TestURLsForApplication(t *testing.T) {
"https://storage.googleapis.com/cockroach-edge-artifacts-prod/cockroach/lib/libgeos_c.linux-gnu-amd64.so.LATEST",
},
},
{
name: "cockroach linux arm64 latest",
args: args{
application: "cockroach",
version: "",
os: "linux",
arch: "arm64",
},
want: []string{
"https://storage.googleapis.com/cockroach-edge-artifacts-prod/cockroach/cockroach.linux-3.7.10-gnu-aarch64.LATEST",
"https://storage.googleapis.com/cockroach-edge-artifacts-prod/cockroach/lib/libgeos.linux-3.7.10-gnu-aarch64.so.LATEST",
"https://storage.googleapis.com/cockroach-edge-artifacts-prod/cockroach/lib/libgeos_c.linux-3.7.10-gnu-aarch64.so.LATEST",
},
},
{
name: "cockroach darwin latest",
args: args{
Expand All @@ -104,6 +147,20 @@ func TestURLsForApplication(t *testing.T) {
"https://storage.googleapis.com/cockroach-edge-artifacts-prod/cockroach/lib/libgeos_c.darwin-amd64.dylib.LATEST",
},
},
{
name: "cockroach darwin arm64 latest",
args: args{
application: "cockroach",
version: "",
os: "darwin",
arch: "arm64",
},
want: []string{
"https://storage.googleapis.com/cockroach-edge-artifacts-prod/cockroach/cockroach.darwin-11.0-aarch64.LATEST",
"https://storage.googleapis.com/cockroach-edge-artifacts-prod/cockroach/lib/libgeos.darwin-11.0-aarch64.dylib.LATEST",
"https://storage.googleapis.com/cockroach-edge-artifacts-prod/cockroach/lib/libgeos_c.darwin-11.0-aarch64.dylib.LATEST",
},
},
{
name: "cockroach windows latest",
args: args{
Expand All @@ -128,6 +185,18 @@ func TestURLsForApplication(t *testing.T) {
"https://storage.googleapis.com/cockroach-release-artifacts-prod/cockroach-v22.1.11.linux-amd64.tgz",
},
},
{
name: "release linux arm64",
args: args{
application: "release",
version: "v22.1.11",
os: "linux",
arch: "arm64",
},
want: []string{
"https://storage.googleapis.com/cockroach-release-artifacts-prod/cockroach-v22.1.11.linux-arm64.tgz",
},
},
{
name: "release darwin",
args: args{
Expand All @@ -139,6 +208,18 @@ func TestURLsForApplication(t *testing.T) {
"https://storage.googleapis.com/cockroach-release-artifacts-prod/cockroach-v22.1.11.darwin-10.9-amd64.tgz",
},
},
{
name: "release darwin arm64",
args: args{
application: "release",
version: "v22.1.11",
os: "darwin",
arch: "arm64",
},
want: []string{
"https://storage.googleapis.com/cockroach-release-artifacts-prod/cockroach-v22.1.11.darwin-11.0-arm64.tgz",
},
},
{
name: "release windows",
args: args{
Expand All @@ -150,6 +231,28 @@ func TestURLsForApplication(t *testing.T) {
"https://storage.googleapis.com/cockroach-release-artifacts-prod/cockroach-v22.1.11.windows-6.2-amd64.zip",
},
},
{
name: "unsupported arch 'arm63'",
args: args{
application: "release",
version: "v22.1.11",
os: "darwin",
arch: "arm63",
},
wantErr: true,
want: nil,
},
{
name: "unsupported arch 'x86'",
args: args{
application: "release",
version: "v22.1.11",
os: "linux",
arch: "x86",
},
wantErr: true,
want: nil,
},
{
name: "something else",
args: args{
Expand All @@ -163,7 +266,7 @@ func TestURLsForApplication(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := URLsForApplication(tt.args.application, tt.args.version, tt.args.os)
got, err := URLsForApplication(tt.args.application, tt.args.version, tt.args.os, tt.args.arch)
if (err != nil) != tt.wantErr {
t.Errorf("URLsForApplication() error = %v, wantErr %v", err, tt.wantErr)
return
Expand Down
Loading

0 comments on commit 1e98906

Please sign in to comment.