diff --git a/pkg/roachprod/vm/aws/support.go b/pkg/roachprod/vm/aws/support.go index 2da86a40f7eb..570d72fa552f 100644 --- a/pkg/roachprod/vm/aws/support.go +++ b/pkg/roachprod/vm/aws/support.go @@ -64,7 +64,7 @@ if [ "${#disks[@]}" -eq "0" ]; then echo "No disks mounted, creating ${mountpoint}" mkdir -p ${mountpoint} chmod 777 ${mountpoint} -elif [ "${#disks[@]}" -eq "1" ] || [ -n "use_multiple_disks" ]; then +elif [ "${#disks[@]}" -eq "1" ] || [ -n "$use_multiple_disks" ]; then disknum=1 for disk in "${disks[@]}" do diff --git a/pkg/roachprod/vm/gce/gcloud.go b/pkg/roachprod/vm/gce/gcloud.go index 67d98d3f947d..76d707a571e0 100644 --- a/pkg/roachprod/vm/gce/gcloud.go +++ b/pkg/roachprod/vm/gce/gcloud.go @@ -202,15 +202,16 @@ type ProviderOpts struct { // projects represent the GCE projects to operate on. Accessed through // GetProject() or GetProjects() depending on whether the command accepts // multiple projects or a single one. - projects []string - ServiceAccount string - MachineType string - MinCPUPlatform string - Zones []string - Image string - SSDCount int - PDVolumeType string - PDVolumeSize int + projects []string + ServiceAccount string + MachineType string + MinCPUPlatform string + Zones []string + Image string + SSDCount int + PDVolumeType string + PDVolumeSize int + UseMultipleDisks bool // useSharedUser indicates that the shared user rather than the personal // user should be used to ssh into the remote machines. @@ -301,6 +302,9 @@ func (o *ProviderOpts) ConfigureCreateFlags(flags *pflag.FlagSet) { "Type of the persistent disk volume, only used if local-ssd=false") flags.IntVar(&o.PDVolumeSize, ProviderName+"-pd-volume-size", 500, "Size in GB of persistent disk volume, only used if local-ssd=false") + flags.BoolVar(&o.UseMultipleDisks, ProviderName+"-enable-multiple-stores", + o.UseMultipleDisks, "Enable the use of multiple stores by creating one store directory per disk. "+ + "Default is to raid0 stripe all disks.") flags.StringSliceVar(&o.Zones, ProviderName+"-zones", nil, fmt.Sprintf("Zones for cluster. If zones are formatted as AZ:N where N is an integer, the zone\n"+ @@ -466,7 +470,7 @@ func (p *Provider) Create(names []string, opts vm.CreateOpts) error { } // Create GCE startup script file. - filename, err := writeStartupScript(extraMountOpts, opts.SSDOpts.FileSystem) + filename, err := writeStartupScript(extraMountOpts, opts.SSDOpts.FileSystem, p.opts.UseMultipleDisks) if err != nil { return errors.Wrapf(err, "could not write GCE startup script to temp file") } diff --git a/pkg/roachprod/vm/gce/utils.go b/pkg/roachprod/vm/gce/utils.go index 89d24a078d0a..b80b12b237cf 100644 --- a/pkg/roachprod/vm/gce/utils.go +++ b/pkg/roachprod/vm/gce/utils.go @@ -46,57 +46,86 @@ if [ -e /mnt/data1/.roachprod-initialized ]; then exit 0 fi -disknum=0 -# ignore the boot disk: /dev/disk/by-id/google-persistent-disk-0. -{{if .Zfs}} - sudo apt-get update - sudo apt-get install -y zfsutils-linux - - # For zfs, we use the device names under /dev instead of the device - # links under /dev/disk/by-id/google-local* for local ssds, because - # there is an issue where the links for the zfs partitions which are - # created under /dev/disk/by-id/ when we run "zpool create ..." are - # inaccurate. - for d in $(ls /dev/nvme?n? /dev/disk/by-id/google-persistent-disk-[1-9]); do - let "disknum++" - # skip if the zpool was already created. - zpool list -v -P | grep ${d} > /dev/null - if [ $? -ne 0 ]; then - echo "Disk ${disknum}: ${d} not mounted, creating..." - mountpoint="/mnt/data${disknum}" - sudo mkdir -p "${mountpoint}" - sudo zpool create -f data${disknum} -m ${mountpoint} ${d} - sudo chmod 777 ${mountpoint} - else - echo "Disk ${disknum}: ${d} already mounted, skipping..." - fi - done -{{else}} - mount_opts="defaults" - {{if .ExtraMountOpts}}mount_opts="${mount_opts},{{.ExtraMountOpts}}"{{end}} - for d in $(ls /dev/disk/by-id/google-local-* /dev/disk/by-id/google-persistent-disk-[1-9]); do - let "disknum++" - grep -e "${d}" /etc/fstab > /dev/null - if [ $? -ne 0 ]; then - echo "Disk ${disknum}: ${d} not mounted, creating..." - mountpoint="/mnt/data${disknum}" - sudo mkdir -p "${mountpoint}" - sudo mkfs.ext4 -F ${d} - sudo mount -o ${mount_opts} ${d} ${mountpoint} - echo "${d} ${mountpoint} ext4 ${mount_opts} 1 1" | sudo tee -a /etc/fstab - sudo chmod 777 ${mountpoint} - else - echo "Disk ${disknum}: ${d} already mounted, skipping..." - fi - done -{{end}} +{{ if not .Zfs }} +mount_opts="defaults" +{{if .ExtraMountOpts}}mount_opts="${mount_opts},{{.ExtraMountOpts}}"{{end}} +{{ end }} + +use_multiple_disks='{{if .UseMultipleDisks}}true{{end}}' + +disks=() +mount_prefix="/mnt/data" + +{{ if .Zfs }} +apt-get update -q +apt-get install -yq zfsutils-linux + +# For zfs, we use the device names under /dev instead of the device +# links under /dev/disk/by-id/google-local* for local ssds, because +# there is an issue where the links for the zfs partitions which are +# created under /dev/disk/by-id/ when we run "zpool create ..." are +# inaccurate. +for d in $(ls /dev/nvme?n? /dev/disk/by-id/google-persistent-disk-[1-9]); do + zpool list -v -P | grep ${d} > /dev/null + if [ $? -ne 0 ]; then +{{ else }} +for d in $(ls /dev/disk/by-id/google-local-* /dev/disk/by-id/google-persistent-disk-[1-9]); do + if ! mount | grep ${d}; then +{{ end }} + disks+=("${d}") + echo "Disk ${d} not mounted, creating..." + else + echo "Disk ${d} already mounted, skipping..." + fi +done -if [ "${disknum}" -eq "0" ]; then - echo "No disks mounted, creating /mnt/data1" - sudo mkdir -p /mnt/data1 - sudo chmod 777 /mnt/data1 +if [ "${#disks[@]}" -eq "0" ]; then + mountpoint="${mount_prefix}1" + echo "No disks mounted, creating ${mountpoint}" + mkdir -p ${mountpoint} + chmod 777 ${mountpoint} +elif [ "${#disks[@]}" -eq "1" ] || [ -n "$use_multiple_disks" ]; then + disknum=1 + for disk in "${disks[@]}" + do + mountpoint="${mount_prefix}${disknum}" + disknum=$((disknum + 1 )) + echo "Creating ${mountpoint}" + mkdir -p ${mountpoint} +{{ if .Zfs }} + zpool create -f $(basename $mountpoint) -m ${mountpoint} ${disk} + # NOTE: we don't need an /etc/fstab entry for ZFS. It will handle this itself. +{{ else }} + mkfs.ext4 -q -F ${disk} + mount -o ${mount_opts} ${disk} ${mountpoint} + echo "/dev/md0 ${mountpoint} ext4 ${mount_opts} 1 1" | tee -a /etc/fstab +{{ end }} + chmod 777 ${mountpoint} + done +else + mountpoint="${mount_prefix}1" + echo "${#disks[@]} disks mounted, creating ${mountpoint} using RAID 0" + mkdir -p ${mountpoint} +{{ if .Zfs }} + zpool create -f $(basename $mountpoint) -m ${mountpoint} ${disks[@]} + # NOTE: we don't need an /etc/fstab entry for ZFS. It will handle this itself. +{{ else }} + raiddisk="/dev/md0" + mdadm -q --create ${raiddisk} --level=0 --raid-devices=${#disks[@]} "${disks[@]}" + mkfs.ext4 -q -F ${raiddisk} + mount -o ${mount_opts} ${raiddisk} ${mountpoint} + echo "${raiddisk} ${mountpoint} ext4 ${mount_opts} 1 1" | tee -a /etc/fstab +{{ end }} + chmod 777 ${mountpoint} fi +# Print the block device and FS usage output. This is useful for debugging. +lsblk +df -h +{{ if .Zfs }} +zpool list +{{ end }} + # sshguard can prevent frequent ssh connections to the same host. Disable it. systemctl stop sshguard systemctl mask sshguard @@ -191,13 +220,20 @@ sudo touch /mnt/data1/.roachprod-initialized // // extraMountOpts, if not empty, is appended to the default mount options. It is // a comma-separated list of options for the "mount -o" flag. -func writeStartupScript(extraMountOpts string, fileSystem string) (string, error) { +func writeStartupScript( + extraMountOpts string, fileSystem string, useMultiple bool, +) (string, error) { type tmplParams struct { - ExtraMountOpts string - Zfs bool + ExtraMountOpts string + UseMultipleDisks bool + Zfs bool } - args := tmplParams{ExtraMountOpts: extraMountOpts, Zfs: fileSystem == vm.Zfs} + args := tmplParams{ + ExtraMountOpts: extraMountOpts, + UseMultipleDisks: useMultiple, + Zfs: fileSystem == vm.Zfs, + } tmpfile, err := ioutil.TempFile("", "gce-startup-script") if err != nil {