From fc6b68fd81232e24a857c14309bbfa1dad0324fd Mon Sep 17 00:00:00 2001 From: Nick Travers Date: Mon, 8 Nov 2021 15:56:05 -0800 Subject: [PATCH] pkg/roachprod: allow multiple-stores to be created on GCP Port an existing flag from the AWS roachprod flags that allows multiple stores to be created. When this flag is enabled, multiple data directories are created and mounted as `/mnt/data{1..N}`. Standardize the existing ext4 disk creation logic in the GCE setup script to match the AWS functionality. Interleave the existing ZFS setup commands based on the `--filesystem` flag. Fix a bug introduced in #54986 that will always create multiple data disks, ignoring the value of the flag. This has the effect of never creating a RAID 0 array, which is the intended default behavior. The ability to create a RAID 0 array on GCE VMs is required for the Pebble write-throughput benchmarks. Release note: None --- pkg/roachprod/vm/aws/support.go | 2 +- pkg/roachprod/vm/gce/gcloud.go | 24 +++--- pkg/roachprod/vm/gce/utils.go | 140 ++++++++++++++++++++------------ 3 files changed, 103 insertions(+), 63 deletions(-) diff --git a/pkg/roachprod/vm/aws/support.go b/pkg/roachprod/vm/aws/support.go index 2da86a40f7eb..570d72fa552f 100644 --- a/pkg/roachprod/vm/aws/support.go +++ b/pkg/roachprod/vm/aws/support.go @@ -64,7 +64,7 @@ if [ "${#disks[@]}" -eq "0" ]; then echo "No disks mounted, creating ${mountpoint}" mkdir -p ${mountpoint} chmod 777 ${mountpoint} -elif [ "${#disks[@]}" -eq "1" ] || [ -n "use_multiple_disks" ]; then +elif [ "${#disks[@]}" -eq "1" ] || [ -n "$use_multiple_disks" ]; then disknum=1 for disk in "${disks[@]}" do diff --git a/pkg/roachprod/vm/gce/gcloud.go b/pkg/roachprod/vm/gce/gcloud.go index 67d98d3f947d..76d707a571e0 100644 --- a/pkg/roachprod/vm/gce/gcloud.go +++ b/pkg/roachprod/vm/gce/gcloud.go @@ -202,15 +202,16 @@ type ProviderOpts struct { // projects represent the GCE projects to operate on. Accessed through // GetProject() or GetProjects() depending on whether the command accepts // multiple projects or a single one. - projects []string - ServiceAccount string - MachineType string - MinCPUPlatform string - Zones []string - Image string - SSDCount int - PDVolumeType string - PDVolumeSize int + projects []string + ServiceAccount string + MachineType string + MinCPUPlatform string + Zones []string + Image string + SSDCount int + PDVolumeType string + PDVolumeSize int + UseMultipleDisks bool // useSharedUser indicates that the shared user rather than the personal // user should be used to ssh into the remote machines. @@ -301,6 +302,9 @@ func (o *ProviderOpts) ConfigureCreateFlags(flags *pflag.FlagSet) { "Type of the persistent disk volume, only used if local-ssd=false") flags.IntVar(&o.PDVolumeSize, ProviderName+"-pd-volume-size", 500, "Size in GB of persistent disk volume, only used if local-ssd=false") + flags.BoolVar(&o.UseMultipleDisks, ProviderName+"-enable-multiple-stores", + o.UseMultipleDisks, "Enable the use of multiple stores by creating one store directory per disk. "+ + "Default is to raid0 stripe all disks.") flags.StringSliceVar(&o.Zones, ProviderName+"-zones", nil, fmt.Sprintf("Zones for cluster. If zones are formatted as AZ:N where N is an integer, the zone\n"+ @@ -466,7 +470,7 @@ func (p *Provider) Create(names []string, opts vm.CreateOpts) error { } // Create GCE startup script file. - filename, err := writeStartupScript(extraMountOpts, opts.SSDOpts.FileSystem) + filename, err := writeStartupScript(extraMountOpts, opts.SSDOpts.FileSystem, p.opts.UseMultipleDisks) if err != nil { return errors.Wrapf(err, "could not write GCE startup script to temp file") } diff --git a/pkg/roachprod/vm/gce/utils.go b/pkg/roachprod/vm/gce/utils.go index 89d24a078d0a..b80b12b237cf 100644 --- a/pkg/roachprod/vm/gce/utils.go +++ b/pkg/roachprod/vm/gce/utils.go @@ -46,57 +46,86 @@ if [ -e /mnt/data1/.roachprod-initialized ]; then exit 0 fi -disknum=0 -# ignore the boot disk: /dev/disk/by-id/google-persistent-disk-0. -{{if .Zfs}} - sudo apt-get update - sudo apt-get install -y zfsutils-linux - - # For zfs, we use the device names under /dev instead of the device - # links under /dev/disk/by-id/google-local* for local ssds, because - # there is an issue where the links for the zfs partitions which are - # created under /dev/disk/by-id/ when we run "zpool create ..." are - # inaccurate. - for d in $(ls /dev/nvme?n? /dev/disk/by-id/google-persistent-disk-[1-9]); do - let "disknum++" - # skip if the zpool was already created. - zpool list -v -P | grep ${d} > /dev/null - if [ $? -ne 0 ]; then - echo "Disk ${disknum}: ${d} not mounted, creating..." - mountpoint="/mnt/data${disknum}" - sudo mkdir -p "${mountpoint}" - sudo zpool create -f data${disknum} -m ${mountpoint} ${d} - sudo chmod 777 ${mountpoint} - else - echo "Disk ${disknum}: ${d} already mounted, skipping..." - fi - done -{{else}} - mount_opts="defaults" - {{if .ExtraMountOpts}}mount_opts="${mount_opts},{{.ExtraMountOpts}}"{{end}} - for d in $(ls /dev/disk/by-id/google-local-* /dev/disk/by-id/google-persistent-disk-[1-9]); do - let "disknum++" - grep -e "${d}" /etc/fstab > /dev/null - if [ $? -ne 0 ]; then - echo "Disk ${disknum}: ${d} not mounted, creating..." - mountpoint="/mnt/data${disknum}" - sudo mkdir -p "${mountpoint}" - sudo mkfs.ext4 -F ${d} - sudo mount -o ${mount_opts} ${d} ${mountpoint} - echo "${d} ${mountpoint} ext4 ${mount_opts} 1 1" | sudo tee -a /etc/fstab - sudo chmod 777 ${mountpoint} - else - echo "Disk ${disknum}: ${d} already mounted, skipping..." - fi - done -{{end}} +{{ if not .Zfs }} +mount_opts="defaults" +{{if .ExtraMountOpts}}mount_opts="${mount_opts},{{.ExtraMountOpts}}"{{end}} +{{ end }} + +use_multiple_disks='{{if .UseMultipleDisks}}true{{end}}' + +disks=() +mount_prefix="/mnt/data" + +{{ if .Zfs }} +apt-get update -q +apt-get install -yq zfsutils-linux + +# For zfs, we use the device names under /dev instead of the device +# links under /dev/disk/by-id/google-local* for local ssds, because +# there is an issue where the links for the zfs partitions which are +# created under /dev/disk/by-id/ when we run "zpool create ..." are +# inaccurate. +for d in $(ls /dev/nvme?n? /dev/disk/by-id/google-persistent-disk-[1-9]); do + zpool list -v -P | grep ${d} > /dev/null + if [ $? -ne 0 ]; then +{{ else }} +for d in $(ls /dev/disk/by-id/google-local-* /dev/disk/by-id/google-persistent-disk-[1-9]); do + if ! mount | grep ${d}; then +{{ end }} + disks+=("${d}") + echo "Disk ${d} not mounted, creating..." + else + echo "Disk ${d} already mounted, skipping..." + fi +done -if [ "${disknum}" -eq "0" ]; then - echo "No disks mounted, creating /mnt/data1" - sudo mkdir -p /mnt/data1 - sudo chmod 777 /mnt/data1 +if [ "${#disks[@]}" -eq "0" ]; then + mountpoint="${mount_prefix}1" + echo "No disks mounted, creating ${mountpoint}" + mkdir -p ${mountpoint} + chmod 777 ${mountpoint} +elif [ "${#disks[@]}" -eq "1" ] || [ -n "$use_multiple_disks" ]; then + disknum=1 + for disk in "${disks[@]}" + do + mountpoint="${mount_prefix}${disknum}" + disknum=$((disknum + 1 )) + echo "Creating ${mountpoint}" + mkdir -p ${mountpoint} +{{ if .Zfs }} + zpool create -f $(basename $mountpoint) -m ${mountpoint} ${disk} + # NOTE: we don't need an /etc/fstab entry for ZFS. It will handle this itself. +{{ else }} + mkfs.ext4 -q -F ${disk} + mount -o ${mount_opts} ${disk} ${mountpoint} + echo "/dev/md0 ${mountpoint} ext4 ${mount_opts} 1 1" | tee -a /etc/fstab +{{ end }} + chmod 777 ${mountpoint} + done +else + mountpoint="${mount_prefix}1" + echo "${#disks[@]} disks mounted, creating ${mountpoint} using RAID 0" + mkdir -p ${mountpoint} +{{ if .Zfs }} + zpool create -f $(basename $mountpoint) -m ${mountpoint} ${disks[@]} + # NOTE: we don't need an /etc/fstab entry for ZFS. It will handle this itself. +{{ else }} + raiddisk="/dev/md0" + mdadm -q --create ${raiddisk} --level=0 --raid-devices=${#disks[@]} "${disks[@]}" + mkfs.ext4 -q -F ${raiddisk} + mount -o ${mount_opts} ${raiddisk} ${mountpoint} + echo "${raiddisk} ${mountpoint} ext4 ${mount_opts} 1 1" | tee -a /etc/fstab +{{ end }} + chmod 777 ${mountpoint} fi +# Print the block device and FS usage output. This is useful for debugging. +lsblk +df -h +{{ if .Zfs }} +zpool list +{{ end }} + # sshguard can prevent frequent ssh connections to the same host. Disable it. systemctl stop sshguard systemctl mask sshguard @@ -191,13 +220,20 @@ sudo touch /mnt/data1/.roachprod-initialized // // extraMountOpts, if not empty, is appended to the default mount options. It is // a comma-separated list of options for the "mount -o" flag. -func writeStartupScript(extraMountOpts string, fileSystem string) (string, error) { +func writeStartupScript( + extraMountOpts string, fileSystem string, useMultiple bool, +) (string, error) { type tmplParams struct { - ExtraMountOpts string - Zfs bool + ExtraMountOpts string + UseMultipleDisks bool + Zfs bool } - args := tmplParams{ExtraMountOpts: extraMountOpts, Zfs: fileSystem == vm.Zfs} + args := tmplParams{ + ExtraMountOpts: extraMountOpts, + UseMultipleDisks: useMultiple, + Zfs: fileSystem == vm.Zfs, + } tmpfile, err := ioutil.TempFile("", "gce-startup-script") if err != nil {