From 9fbbe5d612326b581b39f3240e4b1917ae8b3283 Mon Sep 17 00:00:00 2001 From: Yuri Weinstein Date: Tue, 12 Sep 2017 14:41:19 -0700 Subject: [PATCH] Move scripts to help manage items in crontab on the teuthology box Signed-off-by: Yuri Weinstein --- roles/teuthology/files/cron_wrapper | 53 ++++++++++++++++++++ roles/teuthology/files/schedule_rados_ovh.sh | 37 ++++++++++++++ roles/teuthology/files/schedule_subset.sh | 45 +++++++++++++++++ roles/teuthology/files/vps.yaml | 14 ++++++ 4 files changed, 149 insertions(+) create mode 100755 roles/teuthology/files/cron_wrapper create mode 100755 roles/teuthology/files/schedule_rados_ovh.sh create mode 100755 roles/teuthology/files/schedule_subset.sh create mode 100644 roles/teuthology/files/vps.yaml diff --git a/roles/teuthology/files/cron_wrapper b/roles/teuthology/files/cron_wrapper new file mode 100755 index 00000000..ced5a32b --- /dev/null +++ b/roles/teuthology/files/cron_wrapper @@ -0,0 +1,53 @@ +#!/bin/bash +# /nightlies/cron_wrapper.sh + +# check for no argument case and stop +if [ -z $1 ]; then + echo "need argument" + exit 1 +fi + +# set permanent $LOG file var +LOG="/var/log/crontab-nightlies-log/crontab.log" +# set $LOG_LOCKED_ERR in case locking failed +LOG_LOCK_ERR="/var/log/crontab-nightlies-log/crontab_lock_problem.$$" + +# temp files to store stdout and stderr +# named with the PID of this script in their name so they'll be unique +STDERR="/var/tmp/stderr.$$" +STDOUT="/var/tmp/stdout.$$" + +# $STDOUT and $STDERR are removed when the script exits for any reason +trap "rm -f $STDOUT $STDERR" 0 + +# run a command from this script's argument +# redirect stdout to $STDOUT file and redirect stderr to $STDERR file + +DATE=$(date) +echo -n "$DATE: " >> $STDOUT +echo "Running command: $@" >> $STDOUT +"$@" > $STDOUT 2> $STDERR + +# get return code from the command run +code=$? + +if [ $code != 0 ] ; then + # echoing to stdout/stderr makes cron send email + echo "stdout:" + cat $STDOUT + echo "stderr:" + cat $STDERR +else + # normal exit: just log stdout + + # lock $LOG with file descriptor 200 + exec 200>>$LOG + # if $LOG is locked by other process - wait for 20 sec + flock -w 20 200 || LOG=$LOG_LOCK_ERR + echo "stdout:" >> $LOG + cat $STDOUT >> $LOG + echo "stderr:" >> $LOG + cat $STDERR >> $LOG + # unlock + flock -u 200 +fi diff --git a/roles/teuthology/files/schedule_rados_ovh.sh b/roles/teuthology/files/schedule_rados_ovh.sh new file mode 100755 index 00000000..afaec747 --- /dev/null +++ b/roles/teuthology/files/schedule_rados_ovh.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# $1 - part +# $2 - branch name +# $3 - machine name +# $4 - email address +# $5 - filter out (this arg is to be at the end of the command line for now) + +## example #1 +## (date +%U) week number +## % 2 - mod 2 (e.g. 0,1,0,1 ...) +## * 7 - multiplied by 7 (e.g. 0,7,0,7...) +## $1 day of the week (0-6) +## /14 for 2 weeks + +## example #2 +## (date +%U) week number +## % 4 - mod 4 (e.g. 0,1,2,3,0,1,2,3 ...) +## * 7 - multiplied by 7 (e.g. 0,7,14,21,0,7,14,21...) +## $1 day of the week (0-6) +## /28 for 4 weeks + +echo "Scheduling " $2 " branch" +if [ $2 = "master" ] ; then + # run master branch with --newest option looking for good sha1 7 builds back + teuthology-suite -v -c $2 -m $3 -k distro -s rados --subset $(echo "(($(date +%U) % 4) * 7) + $1" | bc)/28 --newest 7 -e $4 ~/vps.yaml $5 +elif [ $2 = "hammer" ] ; then + # run hammer branch with less jobs + teuthology-suite -v -c $2 -m $3 -k distro -s rados --subset $(echo "(($(date +%U) % 4) * 7) + $1" | bc)/56 -e $4 ~/vps.yaml $5 +elif [ $2 = "jewel" ] ; then + # run jewel branch with /40 jobs + teuthology-suite -v -c $2 -m $3 -k distro -s rados --subset $(echo "(($(date +%U) % 4) * 7) + $1" | bc)/40 -e $4 ~/vps.yaml $5 +else + # run NON master branches without --newest + teuthology-suite -v -c $2 -m $3 -k distro -s rados --subset $(echo "(($(date +%U) % 4) * 7) + $1" | bc)/28 -e $4 ~/vps.yaml $5 +fi + diff --git a/roles/teuthology/files/schedule_subset.sh b/roles/teuthology/files/schedule_subset.sh new file mode 100755 index 00000000..c26231aa --- /dev/null +++ b/roles/teuthology/files/schedule_subset.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +#command line => CEPH_BRANCH=; MACHINE_NAME=; SUITE_NAME=; ../schedule_subset.sh $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL + +# $1 - part (day of week) +# $2 - branch name +# $3 - machine name +# $4 - suite name +# $5 - email address +# $6 - filter out (this arg is to be at the end of the command line for now) + +## example #1 +## (date +%U) week number +## % 2 - mod 2 (e.g. 0,1,0,1 ...) +## * 7 - multiplied by 7 (e.g. 0,7,0,7...) +## $1 day of the week (0-6) +## /14 for 2 weeks + +## example #2 +## (date +%U) week number +## % 4 - mod 4 (e.g. 0,1,2,3,0,1,2,3 ...) +## * 7 - multiplied by 7 (e.g. 0,7,14,21,0,7,14,21...) +## $1 day of the week (0-6) +## /28 for 4 weeks + +echo "Scheduling " $2 " branch" +if [ $2 = "master" ] ; then + # run master branch with --newest option looking for good sha1 7 builds back with /999 jobs + teuthology-suite -v -c $2 -m $3 -k distro -s $4 --subset $(echo "(($(date +%U) % 4) * 7) + $1" | bc)/999 --newest 7 -e $5 $6 +elif [ $2 = "hammer" ] ; then + # run hammer branch with less jobs + teuthology-suite -v -c $2 -m $3 -k distro -s $4 --subset $(echo "(($(date +%U) % 4) * 7) + $1" | bc)/56 -e $5 $6 +elif [ $2 = "jewel" ] ; then + # run jewel branch with /40 jobs + teuthology-suite -v -c $2 -m $3 -k distro -s $4 --subset $(echo "(($(date +%U) % 4) * 7) + $1" | bc)/40 -e $5 $6 +elif [ $2 = "kraken" ] ; then + # run kraken branch with /999 jobs + teuthology-suite -v -c $2 -m $3 -k distro -s $4 --subset $(echo "(($(date +%U) % 4) * 7) + $1" | bc)/999 -e $5 $6 +elif [ $2 = "luminous" ] ; then + # run luminous branch with /999 jobs + teuthology-suite -v -c $2 -m $3 -k distro -s $4 --subset $(echo "(($(date +%U) % 4) * 7) + $1" | bc)/999 -e $5 $6 +else + # run NON master branches without --newest + teuthology-suite -v -c $2 -m $3 -k distro -s $4 --subset $(echo "(($(date +%U) % 4) * 7) + $1" | bc)/28 -e $5 $6 +fi diff --git a/roles/teuthology/files/vps.yaml b/roles/teuthology/files/vps.yaml new file mode 100644 index 00000000..64a3da47 --- /dev/null +++ b/roles/teuthology/files/vps.yaml @@ -0,0 +1,14 @@ +overrides: + ceph: + conf: + global: + osd heartbeat grace: 100 + # this line to address issue #1017 + mon lease: 15 + mon lease ack timeout: 25 + s3tests: + idle_timeout: 1200 + ceph-fuse: + client.0: + mount_wait: 60 + mount_timeout: 120