Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

roachprod: Support multiple stores on AWS #54986

Merged
merged 1 commit into from
Oct 6, 2020
+31 −15
Merged

roachprod: Support multiple stores on AWS #54986

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions pkg/cmd/roachprod/vm/aws/aws.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ type providerOpts struct {
EBSVolumeType string
EBSVolumeSize int
EBSProvisionedIOPs int
UseMultipleDisks bool

// Use specified ImageAMI when provisioning.
// Overrides config.json AMI.
Expand Down Expand Up @@ -147,6 +148,8 @@ func (o *providerOpts) ConfigureCreateFlags(flags *pflag.FlagSet) {
"of geo (default [%s])", strings.Join(defaultCreateZones, ",")))
flags.StringVar(&o.ImageAMI, ProviderName+"-image-ami",
"", "Override image AMI to use. See https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/describe-images.html")
flags.BoolVar(&o.UseMultipleDisks, ProviderName+"-enable-multiple-stores",
false, "Enable the use of multiple stores by creating one store directory per disk. Default is to raid0 stripe all disks.")
}

func (o *providerOpts) ConfigureClusterFlags(flags *pflag.FlagSet, _ vm.MultipleProjectsOption) {
Expand Down Expand Up @@ -676,7 +679,7 @@ func (p *Provider) runInstance(name string, zone string, opts vm.CreateOpts) err
extraMountOpts = "nobarrier"
}
}
filename, err := writeStartupScript(extraMountOpts)
filename, err := writeStartupScript(extraMountOpts, p.opts.UseMultipleDisks)
if err != nil {
return errors.Wrapf(err, "could not write AWS startup script to temp file")
}
Expand Down Expand Up @@ -713,10 +716,10 @@ func (p *Provider) runInstance(name string, zone string, opts vm.CreateOpts) err
if !opts.SSDOpts.UseLocalSSD {
var ebsParams string
switch t := p.opts.EBSVolumeType; t {
case "gp2", "io2":
case "gp2":
ebsParams = fmt.Sprintf("{VolumeSize=%d,VolumeType=%s,DeleteOnTermination=true}",
p.opts.EBSVolumeSize, t)
case "io1":
case "io1", "io2":
ebsParams = fmt.Sprintf("{VolumeSize=%d,VolumeType=%s,Iops=%d,DeleteOnTermination=true}",
p.opts.EBSVolumeSize, t, p.opts.EBSProvisionedIOPs)
default:
Expand Down
37 changes: 25 additions & 12 deletions pkg/cmd/roachprod/vm/aws/support.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,11 @@ sudo apt-get install -qy --no-install-recommends mdadm
mount_opts="discard,defaults"
{{if .ExtraMountOpts}}mount_opts="${mount_opts},{{.ExtraMountOpts}}"{{end}}
use_multiple_disks='{{if .UseMultipleDisks}}true{{end}}'
disks=()
mountpoint="/mnt/data1"
mount_prefix="/mnt/data"
# On different machine types, the drives are either called nvme... or xvdd.
for d in $(ls /dev/nvme?n1 /dev/xvdd); do
if ! mount | grep ${d}; then
Expand All @@ -53,19 +56,28 @@ for d in $(ls /dev/nvme?n1 /dev/xvdd); do
echo "Disk ${d} already mounted, skipping..."
fi
done
if [ "${#disks[@]}" -eq "0" ]; then
mountpoint="${mount_prefix}1"
echo "No disks mounted, creating ${mountpoint}"
mkdir -p ${mountpoint}
chmod 777 ${mountpoint}
elif [ "${#disks[@]}" -eq "1" ]; then
echo "One disk mounted, creating ${mountpoint}"
mkdir -p ${mountpoint}
disk=${disks[0]}
mkfs.ext4 -E nodiscard ${disk}
mount -o ${mount_opts} ${disk} ${mountpoint}
chmod 777 ${mountpoint}
echo "${disk} ${mountpoint} ext4 ${mount_opts} 1 1" | tee -a /etc/fstab
elif [ "${#disks[@]}" -eq "1" ] || [ -n "use_multiple_disks" ]; then
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@miretskiy am I reading this correctly that we always enter this block, because -n "use_multiple_disks" always evaluates to true?

disknum=1
for disk in "${disks[@]}"
do
mountpoint="${mount_prefix}${disknum}"
disknum=$((disknum + 1 ))
echo "Creating ${mountpoint}"
mkdir -p ${mountpoint}
mkfs.ext4 -E nodiscard ${disk}
mount -o ${mount_opts} ${disk} ${mountpoint}
chmod 777 ${mountpoint}
echo "${disk} ${mountpoint} ext4 ${mount_opts} 1 1" | tee -a /etc/fstab
done
else
mountpoint="${mount_prefix}1"
echo "${#disks[@]} disks mounted, creating ${mountpoint} using RAID 0"
mkdir -p ${mountpoint}
raiddisk="/dev/md0"
Expand Down Expand Up @@ -125,12 +137,13 @@ sudo touch /mnt/data1/.roachprod-initialized
//
// extraMountOpts, if not empty, is appended to the default mount options. It is
// a comma-separated list of options for the "mount -o" flag.
func writeStartupScript(extraMountOpts string) (string, error) {
func writeStartupScript(extraMountOpts string, useMultiple bool) (string, error) {
type tmplParams struct {
ExtraMountOpts string
ExtraMountOpts string
UseMultipleDisks bool
}

args := tmplParams{ExtraMountOpts: extraMountOpts}
args := tmplParams{ExtraMountOpts: extraMountOpts, UseMultipleDisks: useMultiple}

tmpfile, err := ioutil.TempFile("", "aws-startup-script")
if err != nil {
Expand Down