Skip to content

Commit

Permalink
Merge Configtools into Pbench
Browse files Browse the repository at this point in the history
This PR deals with merging the configtools functionality into Pbench
instead of using a VM to do so.

Fixes #19 from configtools Repository
  • Loading branch information
riya-17 authored and portante committed Jan 29, 2020
1 parent 5a4201b commit 90aaffa
Show file tree
Hide file tree
Showing 8 changed files with 200 additions and 21 deletions.
3 changes: 1 addition & 2 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ jobs:
install:
# Perl JSON and JSON-XS required
- sudo apt-get install libjson-perl libjson-xs-perl
- pip install 'configtools<0.4.0'
script: ./agent/run-unittests

- stage: "Server Unit Tests"
Expand All @@ -24,7 +23,7 @@ jobs:
before_install:
- sudo apt-get update
install:
- pip install 'configtools<0.4.0' elasticsearch sh boto3
- pip install elasticsearch sh boto3
script: ./server/bin/unittests

- stage: "Dashboard Unit Tests"
Expand Down
9 changes: 7 additions & 2 deletions agent/bench-scripts/unittests
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ export _testdir=${_testroot}/pbench-agent
# Copy bench-scripts execution environment to _testroot
export _testopt=${_testroot}/opt/pbench-agent
res=0
mkdir -p ${_testopt}/bench-scripts/postprocess ${_testopt}/bench-scripts/templates ${_testopt}/util-scripts ${_testopt}/lib ${_testopt}/unittest-scripts
mkdir -p ${_testopt}/bench-scripts/postprocess ${_testopt}/bench-scripts/templates ${_testopt}/util-scripts ${_testopt}/lib ${_testopt}/unittest-scripts ${_testopt}/common/bin ${_testopt}/common/lib
let res=res+${?}
cp ${_tdir}/../base ${_testopt}/
let res=res+${?}
Expand All @@ -42,6 +42,10 @@ for script in ${_tdir}/../util-scripts/pbench-* ; do
cp ${script} ${_testopt}/util-scripts/
let res=res+${?}
done
cp -a $_tdir/../../bin/* $_testopt/common/bin/
let res=res+${?}
cp -a $_tdir/../../lib/* $_testopt/common/lib/
let res=res+${?}
cp -rH ${_tdir}/test-bin/* ${_testopt}/unittest-scripts/
let res=res+${?}
# replace real post-processing scripts with mocked ones
Expand Down Expand Up @@ -72,7 +76,8 @@ function remove_path {
PATH=$(remove_path /opt/pbench-agent/bench-scripts ${PATH})
PATH=$(remove_path /opt/pbench-agent/util-scripts ${PATH})
# Allows us to intercept scp, ssh, rsync, etc.
export PATH=${_testopt}/unittest-scripts:${_testopt}/util-scripts:${_testopt}/bench-scripts:${PATH}
export PATH=${_testopt}/unittest-scripts:${_testopt}/util-scripts:${_testopt}/bench-scripts:${_testopt}/common/bin:${PATH}
export PYTHONPATH=${_testopt}/common/lib:${PYTHONPATH}

# Fixed timestamp output
export _PBENCH_BENCH_TESTS=1
Expand Down
21 changes: 8 additions & 13 deletions agent/profile
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
# -*- mode: shell-script -*-
prefix=/opt/pbench-agent

if [[ ! -f $prefix/profile ]]
if [[ ! -f ${prefix}/profile ]]
then
if [[ -f $PWD/profile ]]
if [[ -f ${PWD}/profile ]]
then
prefix=$PWD
else
Expand All @@ -18,24 +18,19 @@ pathins() {
fi
}

if [[ ! -f $prefix/util-scripts/pbench-register-tool-set ]]
if [[ ! -f ${prefix}/util-scripts/pbench-register-tool-set ]]
then
echo "*** WARNING *** Something wrong in the pbench base profile:" >&2
echo " Setup has failed - could not determine the proper environment prefix: \"$prefix\"" >&2
echo " You will need to fix it before working with pbench." >&2
else
if [[ -f /opt/configtools/bin/getconf.py ]]
then
pathins /opt/configtools/bin
# no need to set PYTHONPATH if configtools is installed
else
pathins $prefix/configtools/bin
fi
pathins ${prefix}/common/bin

export CONFIG=$prefix/config/pbench-agent.cfg
export PYTHONPATH=${prefix}/common/lib:${PYTHONPATH}
export CONFIG=${prefix}/config/pbench-agent.cfg

pathins $prefix/bench-scripts
pathins $prefix/util-scripts
pathins ${prefix}/bench-scripts
pathins ${prefix}/util-scripts
fi

unset prefix
Expand Down
9 changes: 7 additions & 2 deletions agent/util-scripts/unittests
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,16 @@ export _testdir=${_testroot}/pbench
# Copy util-scripts execution environment to _testroot
_testopt=${_testroot}/opt/pbench-agent
res=0
mkdir -p ${_testopt}/config ${_testopt}/util-scripts ${_testopt}/tool-scripts ${_testopt}/lib ${_testopt}/unittest-scripts
mkdir -p ${_testopt}/config ${_testopt}/util-scripts ${_testopt}/tool-scripts ${_testopt}/lib ${_testopt}/unittest-scripts ${_testopt}/common/bin ${_testopt}/common/lib
let res=res+${?}
cp -L ${_tdir}/../base ${_testopt}/
let res=res+${?}
cp -rL ${_tdir}/../config/* ${_testopt}/config/
let res=res+${?}
cp -a $_tdir/../../bin/* $_testopt/common/bin
let res=res+${?}
cp -a $_tdir/../../lib/* $_testopt/common/lib/
let res=res+${?}
scripts="${_tdir}/pbench-*"
for script in ${scripts}; do
if [[ -d "${script}" ]]; then
Expand Down Expand Up @@ -86,7 +90,8 @@ function remove_path {
PATH=$(remove_path /opt/pbench-agent/bench-scripts ${PATH})
PATH=$(remove_path /opt/pbench-agent/util-scripts ${PATH})
# Allows us to intercept scp, ssh, rsync, etc.
export PATH=${_testopt}/unittest-scripts:${_testopt}/util-scripts:${_testopt}/tool-scripts:${PATH}
export PATH=${_testopt}/unittest-scripts:${_testopt}/util-scripts:${_testopt}/tool-scripts:${_testopt}/common/bin:${PATH}
export PYTHONPATH=${_testopt}/common/lib:${PYTHONPATH}

# Fixed timestamp output
export _PBENCH_BENCH_TESTS=1
Expand Down
57 changes: 57 additions & 0 deletions bin/getconf.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
#! /usr/bin/env python3
from __future__ import print_function

import sys
import os
import configtools
from optparse import make_option

def main(conf, args, opts):
if opts.all:
for sec in args:
if conf.has_section(sec):
print("[%s]" % (sec))
items = conf.items(sec)
items.sort()
for (n, v) in items:
print("%s = %s" % (n, v))
print()
return 0

sep = ','
if opts.list:
sep = ' '

option = args[0]
for sec in args[1:]:
if conf.has_section(sec):
if conf.has_option(sec, option):
configtools.print_list(configtools.get_list(conf.get(sec, option)), sep)
return 0
return 1

options = [
make_option("-a", "--all", action="store_true", dest="all", help="print all items in section"),
make_option("-d", "--dump", action="store_true", dest="dump", help="print everything and exit"),
make_option("-l", "--list", action="store_true", dest="list", help="print it as a shell list, translating commas to spaces"),
make_option("-L", "--listfiles", action="store_true", dest="listfiles", help="print the list of config files and exit"),
]

if __name__ == '__main__':
opts, args = configtools.parse_args(options, usage='Usage: getconf.py [options] <item>|-a <section> [<section> ...]')
conf, files = configtools.init(opts)
if not conf:
sys.exit(1)
if opts.dump:
conf.write(sys.stdout)
sys.exit(0)

if opts.listfiles:
files.reverse()
print(files)
sys.exit(0)
if args:
status = main(conf, args, opts)
else:
status = 1
sys.exit(status)
109 changes: 109 additions & 0 deletions lib/configtools/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
""" Configtools """
from __future__ import print_function

import os, sys

# python3
from configparser import ConfigParser

from optparse import OptionParser

import logging

def uniq(l):
# uniquify the list without scrambling it
seen = set()
seen_add = seen.add
return [x for x in l if x not in seen and not seen_add(x)]

def file_list(root):
# read the root file, get its [config] section
# and use it to construct the file list.
conf = ConfigParser()
conf.read(root)
try:
dirlist = conf.get("config", "path").replace(' ', '').split(',')
except Exception:
dirlist = []
try:
files = conf.get("config", "files").replace(' ', '').split(',')
except Exception:
files = []

root = os.path.abspath(root)
# all relative pathnames will be relative to the rootdir
rootdir = os.path.dirname(root)
flist = [root]
dirlist = [os.path.abspath("%s/%s" % (rootdir, x)) if not os.path.isabs(x) else os.path.abspath(x) for x in dirlist]
# insert the directory of the root file at the beginning
dirlist.insert(0, rootdir)

# import pdb; pdb.set_trace()
for d in dirlist:
for f in files:
fnm = "%s/%s" % (d, f)
if fnm in flist:
continue
if os.access(fnm, os.F_OK):
fnmlist = file_list(fnm)
flist += fnmlist
return uniq(flist)

def init(opts):
"""init"""
# config file
conf = ConfigParser()
if opts.filename:
conf_file = opts.filename
elif 'CONFIG' in os.environ:
conf_file= os.environ['CONFIG']
else:
return (None, [])

conffiles = file_list(conf_file)
conffiles.reverse()
files = conf.read(conffiles)

return (conf, files)

def parse_args(options=[], usage=None):
"""parse_args"""
if usage:
parser = OptionParser(usage=usage)
else:
parser = OptionParser()
# standard options
parser.add_option("-C", "--config", dest="filename",
help="config FILE", metavar="FILE")
parser.add_option("-D", "--debug", action="store_true", dest="debug",
help="commands logged but not executed")
# specific options
for o in options:
parser.add_option(o)

return parser.parse_args()

def get_list(s):
"""get_list"""
if not s:
return []
l = [x.strip().strip('\\\n') for x in s.split(',')]
try:
nl = []
for x in l:
nl += expand_range(x)
return nl
except Exception:
return l

def get(conf, option, sections):
"""get option from section list"""
for s in sections:
try:
return conf.get(s, option)
except Exception:
pass
return None

def print_list(l, sep):
print(sep.join([str(x) for x in l]))
2 changes: 1 addition & 1 deletion server/bin/pbench-trampoline
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ else
printf -- "${_basename}: pbench-trampoline needs python3, either directly or through SCL\n" >&2
exit 1
fi
export PYTHONPATH=$(dirname ${_dirname})/lib
export PYTHONPATH=$(dirname ${_dirname})/lib:$(dirname ${_dirname})/common/bin:$(dirname ${_dirname})/common/lib:$PYTHONPATH
if [[ -e ${_dirname}/${_basename}.py ]]; then
# We don't use pbench-base.py to invoke python programs, they can use the
# PbenchConfig class directly.
Expand Down
11 changes: 10 additions & 1 deletion server/bin/unittests
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,10 @@ function _setup_state {
let res=res+$?
mkdir -p $_testopt_sat/bin
let res=res+$?
mkdir -p $_testopt/common/lib
let res=res+$?
mkdir -p $_testopt/common/bin
let res=res+$?
cp -a $_tdir/{job_pool.sh,unittests,pbench*} $_testopt/bin
let res=res+$?
mkdir -p $_testopt/html/static/{js,css}/v0.{2,3}
Expand All @@ -255,6 +259,10 @@ function _setup_state {
let res=res+$?
cp -a $_tdir/{job_pool.sh,unittests,pbench*} $_testopt_sat/bin
let res=res+$?
cp -a $_tdir/../../lib/* $_testopt/common/lib
let res=res+$?
cp -a $_tdir/../../bin/* $_testopt/common/bin
let res=res+$?
cp -a $_tdir/../lib $_testopt
let res=res+$?
cp -a $_tdir/../lib $_testopt_sat
Expand Down Expand Up @@ -299,7 +307,8 @@ function _setup_state {
# All the "real" scripts are found at $_testopt/bin, the mock scripts
# are found in $_testopt/unittest-scripts.
_orig_PATH=$PATH
export PATH=$_testopt/unittest-scripts:$_testopt/bin:$PATH
export PATH=$_testopt/unittest-scripts:$_testopt/bin:$_testopt/common/bin:$PATH
export PYTHONPATH=$_testopt/common/lib:$PYTHONPATH

# Expected location of the final configuration files
export CONFIG=$_testopt/lib/config/pbench-server.cfg
Expand Down

0 comments on commit 90aaffa

Please sign in to comment.