diff --git a/cueadmin/common/common.py b/cueadmin/common/common.py index 4aae68573..353312db8 100644 --- a/cueadmin/common/common.py +++ b/cueadmin/common/common.py @@ -14,61 +14,63 @@ import argparse -import sys import logging +import sys import time import traceback import output -from util import * - +import util from Manifest import Cue3 TEST_SERVERS = [] + class __NullHandler(logging.Handler): def emit(self, record): pass + + logger = logging.getLogger("cue3.tools") logger.addHandler(__NullHandler()) -__ALL__=["testServer", - "handleCommonArgs", - "handleParserException", - "handFloatCriterion", - "getCommonParser", - "setCommonQueryArgs", - "handleCommonQueryArgs", - "resolveJobNames", - "resolveHostNames", - "resolveShowNames", - "confirm", - "formatTime", - "formatDuration", - "formatLongDuration", - "formatMem", - "cutoff", - "ActionUtil", - "DependUtil", - "Convert", - "AllocUtil"] - -epilog =''' - -''' +__ALL__ = ["testServer", + "handleCommonArgs", + "handleParserException", + "handFloatCriterion", + "getCommonParser", + "setCommonQueryArgs", + "handleCommonQueryArgs", + "resolveJobNames", + "resolveHostNames", + "resolveShowNames", + "confirm", + "formatTime", + "formatDuration", + "formatLongDuration", + "formatMem", + "cutoff", + "ActionUtil", + "DependUtil", + "Convert", + "AllocUtil"] + +epilog = '\n\n' + def testServers(): return TEST_SERVERS + def handleCommonArgs(args): logger.debug(args) if args.unit_test: - enableDebugLogging() + util.enableDebugLogging() Cue3.Cuebot.setHosts(testServers()) logger.info("running unit tests") logger.info("setting up test servers") if args.verbose: - enableDebugLogging() + util.enableDebugLogging() if args.server: logger.debug("setting cue3 host servers to %s" % args.server) Cue3.Cuebot.setHosts(args.server) @@ -78,6 +80,7 @@ def handleCommonArgs(args): if args.force: pass + def handleParserException(args, e): try: if args.verbose: @@ -87,7 +90,8 @@ def handleParserException(args, e): except ValueError, ex: print >>sys.stderr, "Error: %s. Try the -verbose or -h flags for more info." % ex except Exception, ex: - print >>sys.stderr, "Error: %s." %ex + print >> sys.stderr, "Error: %s." % ex + def getCommonParser(**options): parser = argparse.ArgumentParser(**options) @@ -98,66 +102,71 @@ def getCommonParser(**options): parser.epilog = epilog general = parser.add_argument_group("General Options") - general.add_argument("-server",action='store', nargs="+", metavar='HOSTNAME', + general.add_argument("-server", action='store', nargs="+", metavar='HOSTNAME', help='Specify cuebot addres(s).') - general.add_argument("-facility",action='store', metavar='CODE', + general.add_argument("-facility", action='store', metavar='CODE', help='Specify the facility code.') - general.add_argument("-verbose","-v",action='store_true', + general.add_argument("-verbose", "-v", action='store_true', help='Turn on verbose logging.') - general.add_argument("-force",action='store_true', + general.add_argument("-force", action='store_true', help='Force operations that usually require confirmation.') general.add_argument("-unit-test", action="store_true", help=argparse.SUPPRESS) return parser + class QueryAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): if option_string == '-lh': namespace.lh = True namespace.query = values - elif option_string in ('-lj','-laj'): + elif option_string in ('-lj', '-laj'): namespace.lj = True namespace.query = values elif option_string == '-lji': namespace.lji = True namespace.query = values + def setCommonQueryArgs(parser): query = parser.add_argument_group("Query Options") - query.add_argument("-lj","-laj",action=QueryAction,nargs="*", metavar="SUBSTR", + query.add_argument("-lj", "-laj", action=QueryAction, nargs="*", metavar="SUBSTR", help="List jobs with optional name substring match.") - query.add_argument("-lji",action=QueryAction, nargs="*", metavar="SUBSTR", + query.add_argument("-lji", action=QueryAction, nargs="*", metavar="SUBSTR", help="List job info with optional name substring match.") - query.add_argument("-lh",action=QueryAction,nargs="*",metavar="SUBSTR", + query.add_argument("-lh", action=QueryAction, nargs="*", metavar="SUBSTR", help="List hosts with optional name substring match.") - query.add_argument("-ls",action="store_true", help="List shows.") - query.add_argument("-la",action="store_true", help="List allocations.") - query.add_argument("-lb",action="store", nargs="+", help="List subscriptions.", metavar="SHOW") - query.add_argument("-query","-q",nargs="+",action="store",default=[], help=argparse.SUPPRESS) + query.add_argument("-ls", action="store_true", help="List shows.") + query.add_argument("-la", action="store_true", help="List allocations.") + query.add_argument("-lb", action="store", nargs="+", help="List subscriptions.", metavar="SHOW") + query.add_argument("-query", "-q", nargs="+", action="store", default=[], + help=argparse.SUPPRESS) return query + def handleCommonQueryArgs(args): if args.lh: - output.displayHosts(Cue3.HostSearch.byMatch(args.query)) + output.displayHosts(Cue3.search.HostSearch.byMatch(args.query)) return True elif args.lj: - for job in Cue3.JobSearch.byMatch(args.query): + for job in Cue3.search.JobSearch.byMatch(args.query): print job.data.name return True elif args.lji: - output.displayJobs(Cue3.JobSearch.byMatch(args.query)) + output.displayJobs(Cue3.search.JobSearch.byMatch(args.query)) return True elif args.la: - output.displayAllocations(Cue3.getAllocations()) + output.displayAllocations(Cue3.api.getAllocations()) return True elif args.lb: for show in resolveShowNames(args.lb): - output.displaySubscriptions(show.proxy.getSubscriptions(),show.data.name) + output.displaySubscriptions(show.getSubscriptions(), show.data.name) return True elif args.ls: - output.displayShows(Cue3.getShows()) + output.displayShows(Cue3.api.getShows()) return True return False + def handleFloatCriterion(mixed, convert=None): """handleFloatCriterion returns the proper subclass of FloatSearchCriterion based on @@ -174,30 +183,39 @@ def _convert(val): return float(val) return float(convert(float(val))) + criterions = [ + Cue3.api.criterion_pb2.GreaterThanFloatSearchCriterion, + Cue3.api.criterion_pb2.LessThanFloatSearchCriterion, + Cue3.api.criterion_pb2.InRangeFloatSearchCriterion] + if isinstance(mixed, (float, int)): - result = Cue3.SpiIce.GreaterThanFloatSearchCriterion(_convert(mixed)) - elif isinstance(mixed,str): + result = Cue3.api.criterion_pb2.GreaterThanFloatSearchCriterion(value=_convert(mixed)) + elif isinstance(mixed, str): if mixed.startswith("gt"): - result = Cue3.SpiIce.GreaterThanFloatSearchCriterion(_convert(mixed[2:])) + result = Cue3.api.criterion_pb2.GreaterThanFloatSearchCriterion( + value=_convert(mixed[2:])) elif mixed.startswith("lt"): - result = Cue3.SpiIce.LessThanFloatSearchCriterion(_convert(mixed[2:])) + result = Cue3.api.criterion_pb2.LessThanFloatSearchCriterion(value=_convert(mixed[2:])) elif mixed.find("-") > -1: - min,max = mixed.split("-",1) - result = Cue3.SpiIce.InRangeFloatSearchCriterion(_convert(min),_convert(max)) + min_value, max_value = mixed.split("-", 1) + result = Cue3.api.criterion_pb2.InRangeFloatSearchCriterion(min=_convert(min_value), + max=_convert(max_value)) else: try: - result = Cue3.SpiIce.GreaterThanFloatSearchCriterion(_convert(mixed)) + result = Cue3.api.criterion_pb2.GreaterThanFloatSearchCriterion( + value=_convert(mixed)) except ValueError: raise Exception("invalid float search input value: " + str(mixed)) - elif issubclass(mixed.__class__,Cue3.SpiIce.FloatSearchCriterion): + elif any([isinstance(mixed.__class__, crit_cls) for crit_cls in criterions]): result = mixed elif not mixed: return [] else: - raise Exception("invalid float search input value: " + str(mixed)) + raise Exception("invalid float search input value: " + str(mixed)) return [result] + def handleIntCriterion(mixed, convert=None): """handleIntCriterion returns the proper subclass of IntSearchCriterion based on @@ -214,112 +232,138 @@ def _convert(val): return int(val) return int(convert(float(val))) + criterions = [ + Cue3.api.criterion_pb2.GreaterThanIntegerSearchCriterion, + Cue3.api.criterion_pb2.LessThanIntegerSearchCriterion, + Cue3.api.criterion_pb2.InRangeIntegerSearchCriterion] + if isinstance(mixed, (float, int)): - result = Cue3.SpiIce.GreaterThanIntegerSearchCriterion(_convert(mixed)) - elif isinstance(mixed,str): + result = Cue3.api.criterion_pb2.GreaterThanIntegerSearchCriterion(value=_convert(mixed)) + elif isinstance(mixed, str): if mixed.startswith("gt"): - result = Cue3.SpiIce.GreaterThanIntegerSearchCriterion(_convert(mixed[2:])) + result = Cue3.api.criterion_pb2.GreaterThanIntegerSearchCriterion( + value=_convert(mixed[2:])) elif mixed.startswith("lt"): - result = Cue3.SpiIce.LessThanIntegerSearchCriterion(_convert(mixed[2:])) + result = Cue3.api.criterion_pb2.LessThanIntegerSearchCriterion( + value=_convert(mixed[2:])) elif mixed.find("-") > -1: - min,max = mixed.split("-",1) - result = Cue3.SpiIce.InRangeIntegerSearchCriterion(_convert(min),_convert(max)) + min_value, max_value = mixed.split("-", 1) + result = Cue3.api.criterion_pb2.InRangeIntegerSearchCriterion( + min=_convert(min_value), max=_convert(max_value)) else: try: - result = Cue3.SpiIce.GreaterThanIntegerSearchCriterion(_convert(mixed)) + result = Cue3.api.criterion_pb2.GreaterThanIntegerSearchCriterion( + value=_convert(mixed)) except ValueError: raise Exception("invalid int search input value: " + str(mixed)) - elif issubclass(mixed.__class__,Cue3.SpiIce.IntegerSearchCriterion): + elif any([isinstance(mixed.__class__, crit_cls) for crit_cls in criterions]): result = mixed elif not mixed: return [] else: - raise Exception("invalid float search input value: " + str(mixed)) + raise Exception("invalid float search input value: " + str(mixed)) return [result] + def resolveJobNames(names): - items = Cue3.JobSearch.byName(names); + items = Cue3.search.JobSearch.byName(names) logger.debug("found %d of %d supplied jobs" % (len(items), len(names))) if len(names) != len(items) and len(items): - logger.warn("Unable to match all job names with running jobs on the cue."); - logger.warn("Operations executed for %s" % set(names).intersection([i.data.name for i in items])) - logger.warn("Operations NOT executed for %s" % set(names).difference([i.data.name for i in items])) + logger.warn("Unable to match all job names with running jobs on the cue.") + logger.warn("Operations executed for %s" % set(names).intersection( + [i.data.name for i in items])) + logger.warn("Operations NOT executed for %s" % set(names).difference( + [i.data.name for i in items])) if not items: raise ValueError("no valid jobs") return items + def resolveHostNames(names=None, substr=None): + items = [] if names: - items = Cue3.HostSearch.byName(names); + items = Cue3.search.HostSearch.byName(names) logger.debug("found %d of %d supplied hosts" % (len(items), len(names))) if len(names) != len(items) and len(items): - logger.warn("Unable to match all host names with valid hosts on the cue."); - logger.warn("Operations executed for %s" % set(names).intersection([i.data.name for i in items])) - logger.warn("Operations NOT executed for %s" % set(names).difference([i.data.name for i in items])) + logger.warn("Unable to match all host names with valid hosts on the cue.") + logger.warn("Operations executed for %s" % set(names).intersection( + [i.data.name for i in items])) + logger.warn("Operations NOT executed for %s" % set(names).difference([ + i.data.name for i in items])) elif substr: - items = Cue3.HostSearch.byMatch(substr) + items = Cue3.search.HostSearch.byMatch(substr) logger.debug("matched %d hosts using patterns %s" % (len(items), substr)) if not items: raise ValueError("no valid hosts") return items + def resolveShowNames(names): items = [] try: for name in names: - items.append(Cue3.findShow(name)) - except: + items.append(Cue3.api.findShow(name)) + except Cue3.CueException: pass logger.debug("found %d of %d supplied shows" % (len(items), len(names))) if len(names) != len(items) and len(items): - logger.warn("Unable to match all show names with active shows."); - logger.warn("Operations executed for %s" % set(names).intersection([i.data.name for i in items])) - logger.warn("Operations NOT executed for %s" % set(names).difference([i.data.name for i in items])) + logger.warn("Unable to match all show names with active shows.") + logger.warn("Operations executed for %s" % set(names).intersection( + [i.data.name for i in items])) + logger.warn("Operations NOT executed for %s" % set(names).difference( + [i.data.name for i in items])) if not items: raise ValueError("no valid shows") return items + def confirm(msg, force, func, *args, **kwargs): - if promptYesNo("Please confirm. %s?" % msg, force): - logger.debug("%s [forced %s]" % (msg,force)) + if util.promptYesNo("Please confirm. %s?" % msg, force): + logger.debug("%s [forced %s]" % (msg, force)) return func(*args, **kwargs) -def formatTime(epoch, format="%m/%d %H:%M", default="--/-- --:--"): + +def formatTime(epoch, time_format="%m/%d %H:%M", default="--/-- --:--"): """Formats time using time formatting standards see: http://docs.python.org/library/time.html""" if not epoch: return default - return time.strftime(format,time.localtime(epoch)) + return time.strftime(time_format, time.localtime(epoch)) + def findDuration(start, stop): if stop < 1: stop = int(time.time()) return stop - start + def formatDuration(sec): - def splitTime(sec): - min, sec = divmod(sec, 60) - hour, min = divmod(min, 60) - return (hour, min, sec) + def splitTime(seconds): + minutes, seconds = divmod(seconds, 60) + hour, minutes = divmod(minutes, 60) + return hour, minutes, seconds return "%02d:%02d:%02d" % splitTime(sec) + def formatLongDuration(sec): - def splitTime(sec): - min, sec = divmod(sec, 60) - hour, min = divmod(min, 60) - days, hour = divmod(hour, 24) - return (days, hour) + def splitTime(seconds): + minutes, seconds = divmod(seconds, 60) + hour, minutes = divmod(minutes, 60) + days, hour = divmod(hour, 24) + return days, hour return "%02d:%02d" % splitTime(sec) -def formatMem(kmem, unit = None): + +def formatMem(kmem, unit=None): k = 1024 if unit == "K" or not unit and kmem < k: return "%dK" % kmem - if unit == "M" or not unit and kmem < pow(k,2): + if unit == "M" or not unit and kmem < pow(k, 2): return "%dM" % (kmem / k) - if unit == "G" or not unit and kmem < pow(k,3): - return "%.01fG" % (float(kmem) / pow(k,2)) + if unit == "G" or not unit and kmem < pow(k, 3): + return "%.01fG" % (float(kmem) / pow(k, 2)) + def cutoff(s, length): if len(s) < length-2: @@ -327,38 +371,51 @@ def cutoff(s, length): else: return "%s.." % s[0:length-2] + # These static utility methods are implementations that may or may not # need to be moved to the server, but should be someplace common # -class AllocUtil: +class AllocUtil(object): + def __init__(self): + pass + @staticmethod def transferHosts(src, dst): - hosts = [host.proxy for host in Cue3.proxy(src).getHosts()] - logger.debug("transfering %d hosts from %s to %s" % + hosts = Cue3.proxy(src).getHosts() + logger.debug("transferring %d hosts from %s to %s" % (len(hosts), Cue3.rep(src), Cue3.rep(dst))) dst.proxy.reparentHosts(hosts) -class DependUtil: + +class DependUtil(object): + + def __init__(self): + pass + @staticmethod def dropAllDepends(job, layer=None, frame=None): if frame: logger.debug("dropping all depends on: %s/%04d-%s" % (job, layer, frame)) - depend_er_frame = Cue3.findFrame(job, layer, frame) - for depend in depend_er_frame.proxy.getWhatThisDependsOn(): + depend_er_frame = Cue3.api.findFrame(job, layer, frame) + for depend in depend_er_frame.getWhatThisDependsOn(): depend.proxy.satisfy() elif layer: logger.debug("dropping all depends on: %s/%s" % (job, layer)) - depend_er_layer = Cue3.findLayer(job, layer) - for depend in depend_er_layer.proxy.getWhatThisDependsOn(): + depend_er_layer = Cue3.api.findLayer(job, layer) + for depend in depend_er_layer.getWhatThisDependsOn(): depend.proxy.satisfy() else: logger.debug("dropping all depends on: %s" % job) - depend_er_job = Cue3.findJob(job) - for depend in depend_er_job.proxy.getWhatThisDependsOn(): + depend_er_job = Cue3.api.findJob(job) + for depend in depend_er_job.getWhatThisDependsOn(): logger.debug("dropping depend %s %s" % (depend.data.type, Cue3.id(depend))) depend.proxy.satisfy() -class Convert: + +class Convert(object): + + def __init__(self): + pass @staticmethod def gigsToKB(val): @@ -370,106 +427,108 @@ def hoursToSeconds(val): @staticmethod def stringToBoolean(val): - if val.lower() in ("yes","on","enabled","true"): + if val.lower() in ("yes", "on", "enabled", "true"): return True return False @staticmethod def strToMatchSubject(val): try: - return Cue3.MatchSubject.__getattribute__(Cue3.MatchSubject, str(val)) - except: - raise ValueError("invalid match subject: %s" % val) + return getattr(Cue3.api.filter_pb2, str(val).upper()) + except Exception: + raise ValueError("invalid match subject: %s" % val.upper()) @staticmethod def strToMatchType(val): try: - return Cue3.MatchType.__getattribute__(Cue3.MatchType, str(val)) - except: - raise ValueError("invalid match type: %s" % val) + return getattr(Cue3.api.filter_pb2, str(val).upper()) + except Exception: + raise ValueError("invalid match type: %s" % val.upper()) @staticmethod def strToActionType(val): try: - return Cue3.ActionType.__getattribute__(Cue3.ActionType, str(val)) - except: - raise ValueError("invalid action type: %s" % value) + return getattr(Cue3.api.filter_pb2, str(val).upper()) + except Exception: + raise ValueError("invalid action type: %s" % val.upper()) @staticmethod def strToFrameState(val): try: - return Cue3.FrameState.__getattribute__(Cue3.FrameState, str(val)) - except: - raise ValueError("invalid frame state: %s" % val) + return getattr(Cue3.api.job_pb2, str(val).upper()) + except Exception: + raise ValueError("invalid frame state: %s" % val.upper()) @staticmethod def strToHardwareState(val): try: - return Cue3.HardwareState.__getattribute__(Cue3.HardwareState, str(val.capitalize())) - except: - raise ValueError("invalid hardware state: %s" % val.capitalize()) + return getattr(Cue3.api.host_pb2, str(val.upper())) + except Exception: + raise ValueError("invalid hardware state: %s" % val.upper()) @staticmethod def strToThreadMode(val): - """Converts the given value to Cue3.Threadmode enumerated value.""" + """Converts the given value to Cue3.api.host_pb2.ThreadMode enumerated value.""" try: - return Cue3.ThreadMode.__getattribute__(Cue3.ThreadMode, str(val.capitalize())) - except: - raise ValueError("invalid thread mode: %s" % val.capitalize()) + return getattr(Cue3.api.host_pb2, str(val.upper())) + except Exception: + raise ValueError("invalid thread mode: %s" % val.upper()) + -class ActionUtil: +class ActionUtil(object): + + def __init__(self): + pass @staticmethod def factory(actionType, value): - a = Cue3.Entity.ActionData() - a.type = convert.strToActionType(actionType) - action.setValue(a, value) + a = Cue3.api.filter_pb2.Action() + a.type = Convert.strToActionType(actionType) + ActionUtil.setValue(a, value) return a @staticmethod def getValue(a): - valueType = str(a.data.valueType) + valueType = str(a.data.value_type) if valueType == "GroupType": - return a.data.groupValue + return a.data.group_value elif valueType == "StringType": - return a.data.stringValue + return a.data.string_value elif valueType == "IntegerType": - return a.data.integerValue + return a.data.integer_value elif valueType == "FloatType": - return a.data.floatValue + return a.data.float_value elif valueType == "BooleanType": - return a.data.booleanValue + return a.data.boolean_value else: return None @staticmethod def setValue(act, value): - if act.type in (Cue3.ActionType.MoveJobToGroup,): + if act.type == Cue3.api.filter_pb2.MOVE_JOB_TO_GROUP: act.groupValue = Cue3.proxy(value) - act.valueType = Cue3.ActionValueType.GroupType + act.valueType = Cue3.api.filter_pb2.GROUP_TYPE - elif act.type in (Cue3.ActionType.PauseJob,): + elif act.type == Cue3.api.filter_pb2.PAUSE_JOB: act.booleanValue = value - act.valueType = Cue3.ActionValueType.BooleanType + act.valueType = Cue3.api.filter_pb2.BOOLEAN_TYPE - elif act.type in (Cue3.ActionType.SetJobPriority, - Cue3.ActionType.SetAllRenderLayerMemory): + elif act.type in (Cue3.api.filter_pb2.SET_JOB_PRIORITY, + Cue3.api.filter_pb2.SET_ALL_RENDER_LAYER_MEMORY): act.integerValue = int(value) - act.valueType = Cue3.ActionValueType.IntegerType + act.valueType = Cue3.api.filter_pb2.INTEGER_TYPE - elif act.type in (Cue3.ActionType.SetJobMinCores, - Cue3.ActionType.SetJobMaxCores, - Cue3.ActionType.SetAllRenderLayerCores): + elif act.type in (Cue3.api.filter_pb2.SET_JOB_MIN_CORES, + Cue3.api.filter_pb2.SET_JOB_MAX_CORES, + Cue3.api.filter_pb2.SET_ALL_RENDER_LAYER_CORES): act.floatValue = float(value) - act.valueType = Cue3.ActionValueType.FloatType - elif act.type in (Cue3.ActionType.SetAllRenderLayerTags,): + act.valueType = Cue3.api.filter_pb2.FLOAT_TYPE + + elif act.type == Cue3.api.filter_pb2.SET_ALL_RENDER_LAYER_TAGS: act.stringValue = value - act.valueType = Cue3.ActionValueType.StringType + act.valueType = Cue3.api.filter_pb2.STRING_TYPE - elif act.type in (ActionType.StopProcessing,): - act.valueType = Cue3.ActionValueType.NoneType + elif act.type == Cue3.api.filter_pb2.STOP_PROCESSING: + act.valueType = Cue3.api.filter_pb2.NONE_TYPE else: - raise TypeError("invalid action type: %s" % actionType) - - - + raise TypeError("invalid action type: %s" % act.type) diff --git a/cueadmin/common/output.py b/cueadmin/common/output.py index f5f744ed2..5c6d1526c 100644 --- a/cueadmin/common/output.py +++ b/cueadmin/common/output.py @@ -14,116 +14,132 @@ import time -import common +import common from Manifest import Cue3 + def displayProcs(procs): - format = "%-10s %-7s %-24s %-30s / %-30s %-12s %-12s " - print format % ("Host","Cores", "Memory", "Job","Frame","Start","Runtime") + """Displays the proc information on one line each. + @type procs: list + @param procs: Procs to display information about + """ + proc_format = "%-10s %-7s %-24s %-30s / %-30s %-12s %-12s " + print proc_format % ("Host", "Cores", "Memory", "Job", "Frame", "Start", "Runtime") for proc in procs: - print format % (proc.data.name.split("/")[0], - "%0.2f" % proc.data.reservedCores, - "%s of %s (%0.2f%%)" % (common.formatMem(proc.data.usedMemory), - common.formatMem(proc.data.reservedMemory), - (proc.data.usedMemory / float(proc.data.reservedMemory) * 100)), - common.cutoff(proc.data.jobName,30), - common.cutoff(proc.data.frameName,30), - common.formatTime(proc.data.dispatchTime), - common.formatDuration(time.time() - proc.data.dispatchTime)) + print proc_format % (proc.data.name.split("/")[0], + "%0.2f" % proc.data.reserved_cores, + "%s of %s (%0.2f%%)" % ( + common.formatMem(proc.data.used_memory), + common.formatMem(proc.data.reserved_memory), + (proc.data.used_memory / float(proc.data.reserved_memory) * 100)), + common.cutoff(proc.data.job_name, 30), + common.cutoff(proc.data.frame_name, 30), + common.formatTime(proc.data.dispatch_time), + common.formatDuration(time.time() - proc.data.dispatch_time)) def displayHosts(hosts): """Displays the host information on one line each. - @type hosts: list - @param hosts: Hosts to display information about""" - now = int(time.time()) - format = "%-15s %-4s %-5s %-8s %-8s %-9s %-5s %-5s %-16s %-8s %-8s %-6s %-9s %-10s %-7s" - print format % ("Host", "Load","NIMBY", - "freeMem", "freeSwap", "freeMcp", - "Cores", "Mem", "Idle", "Os", "Uptime", "State", "Locked", "Alloc","Thread") - for host in sorted(hosts, key=lambda v:v.data.name): - print format % (host.data.name, host.data.load, - host.data.nimbyEnabled, - common.formatMem(host.data.freeMemory), - common.formatMem(host.data.freeSwap), - common.formatMem(host.data.freeMcp), - host.data.cores, - common.formatMem(host.data.memory), - "[ %0.2f / %s ]" % (host.data.idleCores, - common.formatMem(host.data.idleMemory)), - host.data.os, - common.formatLongDuration(int(time.time()) - host.data.bootTime), - host.data.state, - host.data.lockState, - host.data.allocName, - host.data.threadMode) + @type hosts: list + @param hosts: Hosts to display information about + """ + host_format = "%-15s %-4s %-5s %-8s %-8s %-9s %-5s %-5s %-16s %-8s %-8s %-6s %-9s %-10s %-7s" + print host_format % ("Host", "Load", "NIMBY", "freeMem", "freeSwap", "freeMcp", "Cores", "Mem", + "Idle", "Os", "Uptime", "State", "Locked", "Alloc", "Thread") + for host in sorted(hosts, key=lambda v: v.data.name): + print host_format % (host.data.name, host.data.load, + host.data.nimby_enabled, + common.formatMem(host.data.free_memory), + common.formatMem(host.data.free_swap), + common.formatMem(host.data.free_mcp), + host.data.cores, + common.formatMem(host.data.memory), + "[ %0.2f / %s ]" % (host.data.idle_cores, + common.formatMem(host.data.idle_memory)), + host.data.os, + common.formatLongDuration(int(time.time()) - host.data.boot_time), + host.data.state, + host.data.lock_state, + host.data.alloc_name, + host.data.thread_mode) + def displayShows(shows): """Displays information about a list of shows - @type objs: list - @param objs: A list of show objects""" - format = "%-8s %-6s %15s %15s %15s %15s" - print format % ("Show", "Active", "ReservedCores", "RunningFrames", "PendingFrames", "PendingJobs") + @type shows: list + @param shows: A list of show objects + """ + show_format = "%-8s %-6s %15s %15s %15s %15s" + print show_format % ("Show", "Active", "ReservedCores", "RunningFrames", "PendingFrames", + "PendingJobs") for show in shows: - print format % (show.data.name, - show.data.active, - "%0.2f" % (show.stats.reservedCores), - show.stats.runningFrames, - show.stats.pendingFrames, - show.stats.pendingJobs) + print show_format % (show.data.name, + show.data.active, + "%0.2f" % show.data.show_stats.reserved_cores, + show.data.show_stats.running_frames, + show.data.show_stats.pending_frames, + show.data.show_stats.pending_jobs) + def displayServices(services): """Displays an array of services. - @type objs: list - @param objs: A list of Server objects""" - format = "%-20s %-12s %-20s %-15s %-36s" - print format % ("Name", "Can Thread", "Min Cores Units", "Min Memory", "Tags") + @type services: list + @param services: A list of Server objects + """ + service_format = "%-20s %-12s %-20s %-15s %-36s" + print service_format % ("Name", "Can Thread", "Min Cores Units", "Min Memory", "Tags") for srv in services: - print format % (srv.data.name, - srv.data.threadable, - srv.data.minCores, - "%s MB" % srv.data.minMemory, - " | ".join(srv.data.tags)) + print service_format % (srv.data.name, + srv.data.threadable, + srv.data.min_cores, + "%s MB" % srv.data.min_memory, + " | ".join(srv.data.tags)) + def displayAllocations(allocations): """Displays information about a list of allocations - @type objs: list - @param objs: A list of allocation objects""" - format = "%-25s %15s %8s %8s %8s %8s %8s %8s %-8s" - print format % ("Name", "Tag", "Running", "Avail", "Cores", "Hosts", "Locked", "Down", "Billable") - for alloc in sorted(allocations, key=lambda v:v.data.facility): - print format % (alloc.data.name, - alloc.data.tag, - "%0.2f" % alloc.stats.runningCores, - "%0.2f" % alloc.stats.availableCores, - alloc.stats.cores, - alloc.stats.hosts, - alloc.stats.lockedHosts, - alloc.stats.downHosts, - alloc.data.billable) + @type allocations: list + @param allocations: A list of allocation objects + """ + alloc_format = "%-25s %15s %8s %8s %8s %8s %8s %8s %-8s" + print alloc_format % ("Name", "Tag", "Running", "Avail", "Cores", "Hosts", "Locked", "Down", + "Billable") + for alloc in sorted(allocations, key=lambda v: v.data.facility): + print alloc_format % (alloc.data.name, + alloc.data.tag, + "%0.2f" % alloc.data.stats.running_cores, + "%0.2f" % alloc.data.stats.available_cores, + alloc.data.stats.cores, + alloc.data.stats.hosts, + alloc.data.stats.locked_hosts, + alloc.data.stats.down_hosts, + alloc.data.billable) + def displaySubscriptions(subscriptions, show): """Displays information about a list of subscriptions - @type objs: list - @param objs: A list of subscription objects""" + @type subscriptions: list + @param subscriptions: A list of subscription objects + @type show: str + @param show: show name + """ print "Subscriptions for %s" % show - format = "%-30s %-12s %6s %8s %8s %8s" - print format % ("Allocation", "Show", "Size", "Burst", "Run", "Used") + sub_format = "%-30s %-12s %6s %8s %8s %8s" + print sub_format % ("Allocation", "Show", "Size", "Burst", "Run", "Used") for s in subscriptions: if s.data.size: - perc = (s.data.reservedCores / s.data.size) * 100.0 + perc = (s.data.reserved_cores / s.data.size) * 100.0 else: - perc = (s.data.reservedCores * 100.0) + perc = (s.data.reserved_cores * 100.0) - print format % (s.data.allocationName, - s.data.showName, - s.data.size, - s.data.burst, - "%0.2f" % s.data.reservedCores, - "%0.2f%%" % perc) + print sub_format % (s.data.allocation_name, + s.data.show_name, + s.data.size, + s.data.burst, + "%0.2f" % s.data.reserved_cores, + "%0.2f%%" % perc) - print def displayDepend(depend): print "-" @@ -131,29 +147,32 @@ def displayDepend(depend): print "Type: %s" % depend.data.type print "Internal: %s" % depend.data.target print "Active: %s" % depend.data.active - print "AnyFrame: %s" % depend.data.anyFrame - - print "Depending Job: %s" % depend.data.dependErJob - if depend.data.dependErLayer: - print "Depending Layer: %s" % depend.data.dependErLayer - if depend.data.dependErFrame: - print "Depending Frame: %s" % depend.data.dependErFrame - if depend.data.dependOnJob != depend.data.dependErJob: - print "Depend On Job: %s" % depend.data.dependOnJob - if depend.data.dependOnLayer: - print "Depend On Layer: %s" % depend.data.dependOnLayer - if depend.data.dependOnFrame: - print "Depending Frame: %s" % depend.data.dependOnFrame + print "AnyFrame: %s" % depend.data.any_frame + + print "Depending Job: %s" % depend.data.depend_er_job + if depend.data.depend_er_layer: + print "Depending Layer: %s" % depend.data.depend_er_layer + if depend.data.depend_er_frame: + print "Depending Frame: %s" % depend.data.depend_er_frame + if depend.data.depend_on_job != depend.data.depend_er_job: + print "Depend On Job: %s" % depend.data.depend_on_job + if depend.data.depend_on_layer: + print "Depend On Layer: %s" % depend.data.depend_on_layer + if depend.data.depend_on_frame: + print "Depending Frame: %s" % depend.data.depend_on_frame + def displayDepends(depends, active_only=False): for depend in depends: if (depend.data.active and active_only) or not active_only: displayDepend(depend) + def displayGroups(show): print "Groups for %s" % Cue3.rep(show) - format = "%-32s %-12s %8s %8s %8s %8s %8s %8s %8s %6s" - print format % ("Name","Dept","DefPri","DefMin","DefMax","MaxC","MinC","Booked","Cores","Jobs") + grp_format = "%-32s %-12s %8s %8s %8s %8s %8s %8s %8s %6s" + print grp_format % ("Name", "Dept", "DefPri", "DefMin", "DefMax", "MaxC", "MinC", "Booked", + "Cores", "Jobs") def enabled(v): if v < 0: @@ -161,176 +180,196 @@ def enabled(v): return v def printGroup(group): - name = "|%s+%-20s" % ("".join([" " for i in range(0,int(group.data.level))]),group.data.name) - print format % (name, - group.data.department, - enabled(group.data.defaultJobPriority), - enabled(group.data.defaultJobMinCores), - enabled(group.data.defaultJobMaxCores), - enabled(group.data.maxCores), - group.data.minCores, - group.stats.runningFrames, - "%0.2f" % group.stats.reservedCores, - group.stats.pendingJobs) + name = "|%s+%-20s" % ( + "".join([" " for i in range(0, int(group.data.level))]), group.data.name) + print grp_format % (name, + group.data.department, + enabled(group.data.default_job_priority), + enabled(group.data.default_job_min_cores), + enabled(group.data.default_job_max_cores), + enabled(group.data.max_cores), + group.data.min_cores, + group.data.group_stats.running_frames, + "%0.2f" % group.data.group_stats.reserved_cores, + group.data.group_stats.pending_jobs) + def printGroups(item): printGroup(item) - for group in item.proxy.getGroups(): + for group in item.getGroups(): printGroups(group) - printGroups(show.proxy.getRootGroup()) + + printGroups(show.getRootGroup()) + def displayFilters(show, filters): print "Filters for %s" % show - print "%-32s %-10s %-5s" % ("Name","Type","Enabled") - for filter in filters: - print "%-32s %-10s %-5s" % (filter.data.name, filter.data.type, filter.data.enabled) + print "%-32s %-10s %-5s" % ("Name", "Type", "Enabled") + for filter_ in filters: + print "%-32s %-10s %-5s" % (filter_.data.name, filter_.data.type, filter_.data.enabled) + def displayMatchers(matchers): - print "%-6s %-16s %-16s %-32s" % ("Order","Subject","Type","Query") + print "%-6s %-16s %-16s %-32s" % ("Order", "Subject", "Type", "Query") print "-------------------------------------------------------" order = 0 for matcher in matchers: order = order + 1 - print "%06d %-16s %-16s %-32s" % (order,matcher.data.subject, matcher.data.type, matcher.data.input) + print "%06d %-16s %-16s %-32s" % (order, matcher.data.subject, matcher.data.type, + matcher.data.input) print "------------------------------------------------------" + def displayActions(actions): - print "%-6s %-24s %-16s" % ("Order","Type","Value") + print "%-6s %-24s %-16s" % ("Order", "Type", "Value") num = 0 for action in actions: - num+=1 - print "%06d %-24s %-16s" % (num, action.data.type, common.actionUtil.getValue(action)) + num += 1 + print "%06d %-24s %-16s" % (num, action.data.type, common.ActionUtil.getValue(action)) -def displayFilter(filter): + +def displayFilter(filter_): print "Filter: " - print "Name: %s " % filter.data.name - print "Type: %s " % filter.data.type - print "Enabled: %s " % filter.data.enabled - print "Order: %d " % filter.data.order - print - displayMatchers(filter.proxy.getMatchers()) - print + print "Name: %s " % filter_.data.name + print "Type: %s " % filter_.data.type + print "Enabled: %s " % filter_.data.enabled + print "Order: %d " % filter_.data.order + displayMatchers(filter_.getMatchers()) print "Actions: " print "-------------------------------------------------------" - displayActions(filter.proxy.getActions()) + displayActions(filter_.getActions()) + def displayStrings(strings): """Print all of the strings in a list. - @type objs: list - @param objs: A list of strings""" + @type strings: list + @param strings: A list of strings""" for string in strings: print string + def displayNames(items): """Displays the .name of every object in the list. - @type objs: list<> - @param objs: All objects must have a .name parameter""" + @type items: list<> + @param items: All objects must have a .name parameter""" for item in items: print Cue3.rep(item) + def displayLayers(job, layers): """Displays information about the layers in the list. - @type objs: list - @param objs: List of layers""" - print - print "Job: %s " % (job.data.name) + @type job: Job + @param job: Job object + @type layers: list + @param layers: List of layers""" + print "Job: %s " % job.data.name print "--" for layer in layers: - print "Layer - %s (type: %s) - Tagged: %s - Threadable: %s" % (layer.data.name, - layer.data.type, - layer.data.tags, - layer.data.isThreadable) - print "Minimum Resources - Cores: %0.2f Memory: %s" % (layer.data.minCores, - common.formatMem(layer.data.minMemory)) - print "Frames - Total: %3d Running: %3d Pending: %3d " % (layer.stats.totalFrames, - layer.stats.runningFrames, - layer.stats.pendingFrames) + print "Layer - %s (type: %s) - Tagged: %s - Threadable: %s" % ( + layer.data.name, + layer.data.type, + layer.data.tags, + layer.data.is_threadable) + print "Minimum Resources - Cores: %0.2f Memory: %s" % ( + layer.data.min_cores, + common.formatMem(layer.data.min_memory)) + print "Frames - Total: %3d Running: %3d Pending: %3d " % ( + layer.data.layer_stats.total_frames, + layer.data.layer_stats.running_frames, + layer.data.layer_stats.pending_frames) print "--" - def displayJobs(jobs): """Displays job priority information. - @type objs: list<> - @param objs: All objects must have a .name parameter""" - format = "%-56s %-15s %5s %7s %8s %5s %8s %8s" - print format % ("Job","Group","Booked","Cores","Wait","Pri","MinCores","MaxCores") + @type jobs: list + @param jobs: All objects must have a .name parameter""" + job_format = "%-56s %-15s %5s %7s %8s %5s %8s %8s" + print job_format % ("Job", "Group", "Booked", "Cores", "Wait", "Pri", "MinCores", "MaxCores") for job in jobs: p = "" - if job.data.isPaused: - p=" [paused]" + if job.data.is_paused: + p = " [paused]" name = job.data.name + p - print format % (common.cutoff(name,52), - common.cutoff(job.data.group,15), - job.stats.runningFrames, - "%0.2f" % (job.stats.reservedCores), - job.stats.waitingFrames, - job.data.priority, - "%0.2f" % (job.data.minCores), - "%0.2f" % (job.data.maxCores)) + print job_format % (common.cutoff(name, 52), + common.cutoff(job.data.group, 15), + job.stats.running_frames, + "%0.2f" % job.stats.reserved_cores, + job.stats.waiting_frames, + job.data.priority, + "%0.2f" % job.data.min_cores, + "%0.2f" % job.data.max_cores) + def displayJobInfo(job): """Displays the job's information in cueman format. - @type jobObj: Job - @param jobObj: Job to display""" + @type job: Job + @param job: Job to display""" print "-"*60 print "job: %s\n" % job.data.name - print "%13s: %s" % ("start time", common.formatTime(job.data.startTime)) - if job.data.isPaused: + print "%13s: %s" % ("start time", common.formatTime(job.data.start_time)) + if job.data.is_paused: print "%13s: %s" % ("state", "PAUSED") else: print "%13s: %s" % ("state", job.data.state) print "%13s: %s" % ("type", "N/A") print "%13s: %s" % ("architecture", "N/A") print "%13s: %s" % ("services", "N/A") - print "%13s: %0.2f / %0.2f" % ("Min/Max cores", job.data.minCores, job.data.maxCores) + print "%13s: %0.2f / %0.2f" % ("Min/Max cores", job.data.min_cores, job.data.max_cores) print "" - print "%22s: %s" % ("total number of frames", job.stats.totalFrames) - print "%22s: %s" % ("done", job.stats.succeededFrames) - print "%22s: %s" % ("running", job.stats.runningFrames) - print "%22s: %s" % ("waiting (ready)", job.stats.waitingFrames) - print "%22s: %s" % ("waiting (depend)", job.stats.dependFrames) - print "%22s: %s" % ("failed", job.stats.deadFrames) + print "%22s: %s" % ("total number of frames", job.data.job_stats.total_frames) + print "%22s: %s" % ("done", job.data.job_stats.succeeded_frames) + print "%22s: %s" % ("running", job.data.job_stats.running_frames) + print "%22s: %s" % ("waiting (ready)", job.data.job_stats.waiting_frames) + print "%22s: %s" % ("waiting (depend)", job.data.job_stats.depend_frames) + print "%22s: %s" % ("failed", job.data.job_stats.deadFrames) print "%22s: %s\n" % ("total frame retries", "N/A") - layers = job.proxy.getLayers() + layers = job.getLayers() print "this is a cuerun3 job with %d layers\n" % len(layers) for layer in layers: - print "%s (%d frames, %d done)" % (layer.data.name, layer.stats.totalFrames, layer.stats.succeededFrames) + print "%s (%d frames, %d done)" % (layer.data.name, layer.data.job_stats.total_frames, + layer.data.job_stats.succeeded_frames) print " average frame time: %s" % "N/A" print " average ram usage: %s" % "N/A" print " tags: %s\n" % layer.data.tags + def displayFrames(frames): """Displays the supplied list of frames @type frames: list @param frames: List of frames to display""" header = "%-35s %-10s %-15s %-13s %-12s %-9s %5s %2s %2s" % \ - ("Frame","Staus","Host","Start","End","Runtime","Mem ","R"," Exit") + ("Frame", "Staus", "Host", "Start", "End", "Runtime", "Mem ", "R", " Exit") print header, "\n", "-" * len(header) for frame in frames: dependencies = "" - startTime = common.formatTime(frame.data.startTime) - stopTime = common.formatTime(frame.data.stopTime) + startTime = common.formatTime(frame.data.start_time) + stopTime = common.formatTime(frame.data.stop_time) - if frame.data.startTime: - duration = common.formatDuration(common.findDuration(frame.data.startTime, - frame.data.stopTime)) + if frame.data.start_time: + duration = common.formatDuration(common.findDuration(frame.data.start_time, + frame.data.stop_time)) else: duration = "" - memory = common.formatMem(frame.data.maxRss) - exit = frame.data.exitStatus - - print "%-35s %-10s %-15s %-13s %-12s %-9s %4s %2s %-4s %s" % \ - (common.cutoff(frame.data.name,35), frame.data.state, frame.data.lastResource, - startTime, - stopTime, - duration, - memory, frame.data.retryCount, exit, dependencies) + memory = common.formatMem(frame.data.max_rss) + exitStatus = frame.data.exit_status + + print "%-35s %-10s %-15s %-13s %-12s %-9s %4s %2s %-4s %s" % ( + common.cutoff(frame.data.name, 35), + frame.data.state, + frame.data.last_resource, + startTime, + stopTime, + duration, + memory, + frame.data.retry_count, + exitStatus, + dependencies) if len(frames) == 1000: - print "Warning: Only showing first 1000 matches. See frame query options to \ - limit your results." + print "Warning: Only showing first 1000 matches. See frame query options to " \ + "limit your results." diff --git a/cueadmin/common/util.py b/cueadmin/common/util.py index 6287a6e1c..cc37abef1 100644 --- a/cueadmin/common/util.py +++ b/cueadmin/common/util.py @@ -14,9 +14,9 @@ """Utility functions for Cue3Tools""" +import logging import sys import time -import logging from Manifest import Cue3 @@ -24,6 +24,7 @@ "promptYesNo", "waitOnJobName"] + def enableDebugLogging(): """enables debug logging for cue3 and cue3 tools""" logger = logging.getLogger("cue3") @@ -33,24 +34,26 @@ def enableDebugLogging(): logger.addHandler(console) logger.setLevel(logging.DEBUG) -def promptYesNo(prompt, force = False): + +def promptYesNo(prompt, force=False): """Asks the user the supplied question and returns with a boolean to indicate the users input. - @type option: string - @param option: The question that the user can see + @type prompt: string + @param prompt: The question that the user can see @type force: boolean @param force: (Optional) If true, skips the prompt and returns true @rtype: bool @return: The users response""" try: - result = force or raw_input("%s [y/n] " %prompt) in ("y", "Y") + result = force or raw_input("%s [y/n] " % prompt) in ("y", "Y") except KeyboardInterrupt: - print raise - if not result: print "Canceled" + if not result: + print "Canceled" return result -def waitOnJobName(jobName, maxWaitForLaunch = None): + +def waitOnJobName(jobName, maxWaitForLaunch=None): """Waits on the given job name to enter and then leave the queue. @type jobName: str @param jobName: Full name of the job @@ -67,7 +70,7 @@ def waitOnJobName(jobName, maxWaitForLaunch = None): time.sleep(4) while True: try: - isPending = Cue3.isJobPending(jobName.lower()) + isPending = Cue3.api.isJobPending(jobName.lower()) isLocated = isLocated or isPending if isLocated: @@ -79,10 +82,9 @@ def waitOnJobName(jobName, maxWaitForLaunch = None): waited += delay if maxWaitForLaunch and waited >= maxWaitForLaunch: return False - except Cue3.ConnectionRefusedException, e: + except Cue3.CueException, e: print >>sys.stderr, "Error: %s" % e except Exception, e: print >>sys.stderr, "Error: %s" % e time.sleep(delay) - diff --git a/cueadmin/main.py b/cueadmin/main.py index e8753fa31..735ec88fe 100644 --- a/cueadmin/main.py +++ b/cueadmin/main.py @@ -14,6 +14,8 @@ import logging +import sys + import common Cue3 = common.Cue3 @@ -22,6 +24,7 @@ OS_LIST = [] + def main(argv): parser = common.getCommonParser(description="Cueadmin Cue3 Administrator Tool", @@ -29,165 +32,178 @@ def main(argv): conflict_handler='resolve') query = common.setCommonQueryArgs(parser) - query.add_argument("-lp","-lap",action="store", nargs="*", - metavar="[SHOW ...] [-host HOST ...] [-alloc ...] [-job JOB ...] [-memory ...] [-limit ...]", - help="List running procs. Optionally filter by show, show, memory, alloc. Use \ - -limit to limit the results to N procs.") + query.add_argument("-lp", "-lap", action="store", nargs="*", + metavar="[SHOW ...] [-host HOST ...] [-alloc ...] [-job JOB ...] " + "[-memory ...] [-limit ...]", + help="List running procs. Optionally filter by show, show, memory, alloc. " + "Use -limit to limit the results to N procs.") - query.add_argument("-ll","-lal",action="store", nargs="*", - metavar="[SHOW ...] [-host HOST ...] [-alloc ...] [-job JOB ...] [-memory ...] [-limit ...]", - help="List running frame log paths. Optionally filter by show, show, memory, alloc. Use \ - -limit to limit the results to N logs.") + query.add_argument("-ll", "-lal", action="store", nargs="*", + metavar="[SHOW ...] [-host HOST ...] [-alloc ...] [-job JOB ...] " + "[-memory ...] [-limit ...]", + help="List running frame log paths. Optionally filter by show, show, memory" + ", alloc. Use -limit to limit the results to N logs.") - query.add_argument("-lh",action=common.QueryAction,nargs="*",metavar="[SUBSTR ...] [-state STATE] [-alloc ALLOC]", + query.add_argument("-lh", action=common.QueryAction, nargs="*", + metavar="[SUBSTR ...] [-state STATE] [-alloc ALLOC]", help="List hosts with optional name substring match.") query.add_argument("-lv", action="store", nargs="*", metavar="[SHOW]", help="List default services.") - query.add_argument("-lba",action="store", metavar="ALLOC", + query.add_argument("-lba", action="store", metavar="ALLOC", help="List all subscriptions to a specified allocation.") - query.add_argument("-state", nargs="+",metavar="STATE", action="store",default=[], - choices=["up","down","repair","Up","Down","Repair"], + query.add_argument("-state", nargs="+", metavar="STATE", action="store", default=[], + choices=["up", "down", "repair", "Up", "Down", "Repair"], help="Filter host search by hardware state, up or down.") - - - filter = parser.add_argument_group("Filter Options") - - filter.add_argument("-job", nargs="+",metavar="JOB", action="store",default=[], - help="Filter proc or log search by job") - - filter.add_argument("-alloc", nargs="+",metavar="ALLOC", action="store",default=[], - help="Filter host or proc search by allocation") - - filter.add_argument("-memory", action="store", - help="Filters a list of procs by the amount of reserved memory. \ - Memory can be specified in one of 3 ways. As a range, -. Less than, \ - lt. Greater than, gt. Values should be specified in GB.") - - filter.add_argument("-duration", action="store", - help="Show procs that have been running longer than the specified \ - number of hours or within a specific time frame. Ex. -time 1.2 or \ - -time 3.5-4.5. Waiting frames are automatically filtered out.") - - filter.add_argument("-limit", action="store", default=0, - help="Limit the result of a proc search to N rows") + # + # Filter + # + filter_grp = parser.add_argument_group("Filter Options") + filter_grp.add_argument("-job", nargs="+", metavar="JOB", action="store", default=[], + help="Filter proc or log search by job") + + filter_grp.add_argument("-alloc", nargs="+", metavar="ALLOC", action="store", default=[], + help="Filter host or proc search by allocation") + + filter_grp.add_argument("-memory", action="store", + help="Filters a list of procs by the amount of reserved memory. " + "Memory can be specified in one of 3 ways. As a range, " + "-. Less than, lt. Greater than, gt. " + "Values should be specified in GB.") + + filter_grp.add_argument("-duration", action="store", + help="Show procs that have been running longer than the specified " + "number of hours or within a specific time frame. Ex. -time 1.2 " + "or -time 3.5-4.5. Waiting frames are automatically filtered " + "out.") + + filter_grp.add_argument("-limit", action="store", default=0, + help="Limit the result of a proc search to N rows") # # Show # show = parser.add_argument_group("Show Options") - show.add_argument("-create-show",action="store", metavar="SHOW", + show.add_argument("-create-show", action="store", metavar="SHOW", help="create a new show") - show.add_argument("-delete-show",action="store", metavar="SHOW", + show.add_argument("-delete-show", action="store", metavar="SHOW", help="delete specified show") - show.add_argument("-disable-show",action="store", metavar="SHOW", + show.add_argument("-disable-show", action="store", metavar="SHOW", help="Disable the specified show") - show.add_argument("-enable-show",action="store", metavar="SHOW", + show.add_argument("-enable-show", action="store", metavar="SHOW", help="Enable the specified show") - show.add_argument("-dispatching",action="store", nargs=2, metavar="SHOW ON|OFF", + show.add_argument("-dispatching", action="store", nargs=2, metavar="SHOW ON|OFF", help="Enables frame dispatching on the specified show.") - show.add_argument("-booking",action="store", nargs=2, metavar="SHOW ON|OFF", - help="Booking is new proc assignment. If booking is disabled \ - procs will continue to run on new jobs but no new jobs will \ - be booked.") + show.add_argument("-booking", action="store", nargs=2, metavar="SHOW ON|OFF", + help="Booking is new proc assignment. If booking is disabled " + "procs will continue to run on new jobs but no new jobs will " + "be booked.") - show.add_argument("-default-min-cores",action="store", nargs=2, metavar="SHOW CORES", - help="The default min core value for all jobs before \ - any min core filers are applied.") + show.add_argument("-default-min-cores", action="store", nargs=2, metavar="SHOW CORES", + help="The default min core value for all jobs before " + "any min core filers are applied.") - show.add_argument("-default-max-cores",action="store", nargs=2, metavar="SHOW CORES", - help="The default min core value for all jobs before \ - any max core filters are applied.") + show.add_argument("-default-max-cores", action="store", nargs=2, metavar="SHOW CORES", + help="The default min core value for all jobs before " + "any max core filters are applied.") # # Allocation # alloc = parser.add_argument_group("Allocation Options") - alloc.add_argument("-create-alloc",action="store", nargs=3, metavar="FACILITY ALLOC TAG", + alloc.add_argument("-create-alloc", action="store", nargs=3, metavar="FACILITY ALLOC TAG", help="Create a new allocation.") - alloc.add_argument("-delete-alloc",action="store", metavar="NAME", + alloc.add_argument("-delete-alloc", action="store", metavar="NAME", help="Delete an allocation. It must be empty.") - alloc.add_argument("-rename-alloc",action="store",nargs=2, metavar="OLD NEW", + alloc.add_argument("-rename-alloc", action="store", nargs=2, metavar="OLD NEW", help="Rename allocation. New name must not contain facility prefix.") - alloc.add_argument("-transfer",action="store",nargs=2, metavar="OLD NEW", + alloc.add_argument("-transfer", action="store", nargs=2, metavar="OLD NEW", help="Move all hosts from src alloc to dest alloc") - alloc.add_argument("-tag-alloc",action="store", nargs=2, metavar="ALLOC TAG", + alloc.add_argument("-tag-alloc", action="store", nargs=2, metavar="ALLOC TAG", help="Tag allocation.") # # Subscription # sub = parser.add_argument_group("Subscription Options") - sub.add_argument("-create-sub",action="store", nargs=4, - help="Create new subcription.",metavar="SHOW ALLOC SIZE BURST") + sub.add_argument("-create-sub", action="store", nargs=4, + help="Create new subcription.", metavar="SHOW ALLOC SIZE BURST") sub.add_argument("-delete-sub", action="store", nargs=2, metavar="SHOW ALLOC", help="Delete subscription") sub.add_argument("-size", action="store", nargs=3, metavar="SHOW ALLOC SIZE", - help="Set the gauranteed number of cores.") + help="Set the guaranteed number of cores.") sub.add_argument("-burst", action="store", nargs=3, metavar="SHOW ALLOC BURST", - help="Set the number of burst cores. Use the percent sign to indicate a \ - percentage of the subscription size instead of a hard size.") + help="Set the number of burst cores. Use the percent sign to indicate a " + "percentage of the subscription size instead of a hard size.") # # Host # host = parser.add_argument_group("Host Options") - host.add_argument("-host",action="store", nargs="+",metavar="HOSTNAME", - help="Specify the host names to operate on") - host.add_argument("-hostmatch","-hm",action="store", nargs="+", metavar="SUBSTR", - help="Specify a list of substring matches to match groups of hosts.") + host.add_argument("-host", action="store", nargs="+", metavar="HOSTNAME", + help="Specify the host names to operate on") + host.add_argument("-hostmatch", "-hm", action="store", nargs="+", metavar="SUBSTR", + help="Specify a list of substring matches to match groups of hosts.") host.add_argument("-lock", action="store_true", help="lock hosts") host.add_argument("-unlock", action="store_true", help="unlock hosts") host.add_argument("-move", action="store", metavar="ALLOC", help="move hosts into a new allocation") - host.add_argument("-delete-host",action="store_true", help="delete hosts") - host.add_argument("-safe-reboot",action="store_true", help="lock and reboot hosts when idle") - host.add_argument("-repair", action="store_true", help="Sets hosts into the repair state."); - host.add_argument("-fixed", action="store_true", help="Sets hosts into Up state."); + host.add_argument("-delete-host", action="store_true", help="delete hosts") + host.add_argument("-safe-reboot", action="store_true", help="lock and reboot hosts when idle") + host.add_argument("-repair", action="store_true", help="Sets hosts into the repair state.") + host.add_argument("-fixed", action="store_true", help="Sets hosts into Up state.") host.add_argument("-thread", action="store", help="Set the host's thread mode.", - choices=[mode.lower() for mode in dir(Cue3.ThreadMode) if not mode.startswith('__')]) + choices=[mode.lower() for mode in Cue3.api.host_pb2.ThreadMode.keys()]) host.add_argument("-os", action="store", help="Set the host's operating system.", choices=OS_LIST) try: args = parser.parse_args() + except common.argparse.ArgumentError, exc: + print >>sys.stderr, "Could not parse arguments, check command line flags." + raise exc + + try: common.handleCommonArgs(args) if args.unit_test: runTests(parser) else: handleArgs(args) - except Exception,e: + except Exception, e: common.handleParserException(args, e) + def runTests(parser): import tests tests.run(parser) + def handleArgs(args): # # Query # - if isinstance(args.lp,list) or isinstance(args.ll,list): - if isinstance(args.ll,list): + if isinstance(args.lp, list) or isinstance(args.ll, list): + if isinstance(args.ll, list): args.lp = args.ll if not args.host: args.host = [] - procs = Cue3.ProcSearch().byOptions(show=args.lp, - host=args.host, - limit=args.limit, - alloc=args.alloc, - job=args.job, - memory=common.handleIntCriterion(args.memory, - common.Convert.gigsToKB), - duration=common.handleIntCriterion(args.duration, - common.Convert.hoursToSeconds)) - if isinstance(args.ll,list): + procs = Cue3.search.ProcSearch().byOptions( + show=args.lp, + host=args.host, + limit=args.limit, + alloc=args.alloc, + job=args.job, + memory=common.handleIntCriterion(args.memory, + common.Convert.gigsToKB), + duration=common.handleIntCriterion(args.duration, + common.Convert.hoursToSeconds)) + if isinstance(args.ll, list): print "\n".join([l.data.logPath for l in procs]) else: common.output.displayProcs(procs) @@ -195,21 +211,21 @@ def handleArgs(args): elif args.lh: states = [common.Convert.strToHardwareState(s) for s in args.state] - common.output.displayHosts(Cue3.getHosts(match=args.query, - state=states,alloc=args.alloc)) + common.output.displayHosts(Cue3.api.getHosts(match=args.query, state=states, + alloc=args.alloc)) return - elif args.lba: - common.output.displaySubscriptions( \ - Cue3.findAllocation(args.lba).proxy.getSubscriptions(),"All Shows") + allocation = Cue3.api.findAllocation(args.lba) + common.output.displaySubscriptions(allocation.getSubscriptions(), "All Shows") return elif isinstance(args.lv, (list,)): if args.lv: - common.output.displayServices(Cue3.findShow(args.lv[0]).proxy.getServiceOverrides()) + show = Cue3.api.findShow(args.lv[0]) + common.output.displayServices(show.getServiceOverrides()) else: - common.output.displayServices(Cue3.getDefaultServices()) + common.output.displayServices(Cue3.api.getDefaultServices()) return # # Gather hosts if -host - hostmatch was specified @@ -217,7 +233,7 @@ def handleArgs(args): host_error_msg = "No valid hosts selected, see the -host/-hostmatch options" hosts = None if args.host or args.hostmatch: - hosts = common.resolveHostNames(args.host,args.hostmatch) + hosts = common.resolveHostNames(args.host, args.hostmatch) # # Allocations @@ -225,79 +241,80 @@ def handleArgs(args): if args.create_alloc: fac, name, tag = args.create_alloc common.confirm("Create new allocation %s.%s, with tag %s" % (fac, name, tag), - args.force, createAllocation, fac, name, tag) + args.force, createAllocation, fac, name, tag) elif args.delete_alloc: - common.confirm("Delete allocation %s" % args.delete_alloc, - args.force, Cue3.findAllocation(args.delete_alloc).proxy.delete) + allocation = Cue3.api.findAllocation(args.delete_alloc) + common.confirm("Delete allocation %s" % args.delete_alloc, args.force, + allocation.delete) elif args.rename_alloc: old, new = args.rename_alloc try: new = new.split(".", 2)[1] - except: + except Exception: msg = "Allocation names must be in the form 'facility.name'" raise Exception(msg) - common.confirm("Rename allocation from %s to %s" % (old,new), - args.force, Cue3.findAllocation(old).proxy.setName, new) + common.confirm("Rename allocation from %s to %s" % (old, new), + args.force, Cue3.api.findAllocation(old).setName, new) elif args.transfer: - src = Cue3.findAllocation(args.transfer[0]) - dst = Cue3.findAllocation(args.transfer[1]) + src = Cue3.api.findAllocation(args.transfer[0]) + dst = Cue3.api.findAllocation(args.transfer[1]) common.confirm("Transfer hosts from from %s to %s" % (src.data.name, dst.data.name), - args.force, common.AllocUtil.transferHosts, src, dst) + args.force, common.AllocUtil.transferHosts, src, dst) elif args.tag_alloc: alloc, tag = args.tag_alloc common.confirm("Re-tag allocation %s with %s" % (alloc, tag), - args.force, Cue3.findAllocation(alloc).proxy.setTag,tag) + args.force, Cue3.api.findAllocation(alloc).setTag, tag) # # Shows # elif args.create_show: common.confirm("Create new show %s" % args.create_show, - args.force, Cue3.createShow, args.create_show) + args.force, Cue3.api.createShow, args.create_show) elif args.delete_show: common.confirm("Delete show %s" % args.delete_show, - args.force, Cue3.findShow(args.delete_show).proxy.delete) + args.force, Cue3.api.findShow(args.delete_show).delete) elif args.disable_show: common.confirm("Disable show %s" % args.disable_show, - args.force, Cue3.findShow(args.disable_show).proxy.setActive, False); + args.force, Cue3.api.findShow(args.disable_show).setActive, False) elif args.enable_show: common.confirm("Enable show %s" % args.enable_show, - args.force, Cue3.findShow(args.enable_show).proxy.setActive, True); + args.force, Cue3.api.findShow(args.enable_show).setActive, True) elif args.dispatching: - show = Cue3.findShow(args.dispatching[0]) + show = Cue3.api.findShow(args.dispatching[0]) enabled = common.Convert.stringToBoolean(args.dispatching[1]) if not enabled: common.confirm("Disable dispatching on %s" % Cue3.rep(show), - args.force, show.proxy.enableDispatching, enabled) + args.force, show.enableDispatching, enabled) else: - show.proxy.enableDispatching(True) + show.enableDispatching(True) elif args.booking: - show = Cue3.findShow(args.booking[0]) + show = Cue3.api.findShow(args.booking[0]) enabled = common.Convert.stringToBoolean(args.booking[1]) if not enabled: common.confirm("Disable booking on %s" % Cue3.rep(show), - args.force, show.proxy.enableBooking, False) + args.force, show.enableBooking, False) else: - show.proxy.enableBooking(True) + show.enableBooking(True) elif args.default_min_cores: - common.confirm("Set the default min cores to: %s" % + common.confirm("Set the default min cores to: %s" % args.default_min_cores[1], args.force, - Cue3.findShow(args.default_min_cores[0]).proxy.setDefaultMinCores, + Cue3.api.findShow(args.default_min_cores[0]).setDefaultMinCores, float(int(args.default_min_cores[1]))) elif args.default_max_cores: common.confirm("Set the default max cores to: %s" % args.default_max_cores[1], args.force, - Cue3.findShow(args.default_max_cores[0]).proxy.setDefaultMaxCores, + Cue3.api.findShow(args.default_max_cores[0]).setDefaultMaxCores, float(int(args.default_max_cores[1]))) # # Hosts are handled a bit differently than the rest @@ -311,37 +328,41 @@ def handleArgs(args): raise ValueError(host_error_msg) for host in hosts: logger.debug("locking host: %s" % Cue3.rep(host)) - host.proxy.lock() + host.lock() elif args.unlock: if not hosts: raise ValueError(host_error_msg) for host in hosts: logger.debug("unlocking host: %s" % Cue3.rep(host)) - host.proxy.unlock() + host.unlock() elif args.move: if not hosts: raise ValueError(host_error_msg) - def moveHosts(hosts, dst): - for host in hosts: - logger.debug("moving %s to %s" % (Cue3.rep(host), Cue3.rep(dst))) - host.proxy.setAllocation(dst.proxy) + + def moveHosts(hosts_, dst_): + for host_ in hosts_: + logger.debug("moving %s to %s" % (Cue3.rep(host_), Cue3.rep(dst_))) + host.setAllocation(dst_.data) + common.confirm("Move %d hosts to %s" % (len(hosts), args.move), - args.force, moveHosts, hosts, Cue3.findAllocation(args.move)) + args.force, moveHosts, hosts, Cue3.api.findAllocation(args.move)) # No Test coverage, takes up to a minute for # a host to report back in. elif args.delete_host: if not hosts: raise ValueError(host_error_msg) - def deleteHosts(hosts): - for host in hosts: - logger.debug("deleting host: %s" % host) + + def deleteHosts(hosts_): + for host_ in hosts_: + logger.debug("deleting host: %s" % host_) try: - host.proxy.delete() + host_.delete() except Exception, e: - print "Failed to delete %s due to %s" % (host, e) + print "Failed to delete %s due to %s" % (host_, e) + common.confirm("Delete %s hosts" % len(hosts), args.force, deleteHosts, hosts) @@ -350,94 +371,111 @@ def deleteHosts(hosts): elif args.safe_reboot: if not hosts: raise ValueError(host_error_msg) - def safeReboot(hosts): - for host in hosts: - logger.debug("locking host and rebooting when idle %s" % Cue3.rep(host)) - host.proxy.rebootWhenIdle() + + def safeReboot(hosts_): + for host_ in hosts_: + logger.debug("locking host and rebooting when idle %s" % Cue3.rep(host_)) + host_.rebootWhenIdle() + common.confirm("Lock and reboot %d hosts when idle" % len(hosts), args.force, safeReboot, hosts) elif args.thread: if not hosts: raise ValueError(host_error_msg) - def setThreadMode(hosts, mode): - for host in hosts: - logger.debug("setting host %s to thread mode %s" % (host.data.name, mode)) - host.proxy.setThreadMode(common.Convert.strToThreadMode(mode)) - common.confirm("Set %d hosts to thread mode %s" % (len(hosts),args.thread), args.force, + + def setThreadMode(hosts_, mode): + for host_ in hosts_: + logger.debug("setting host %s to thread mode %s" % (host_.data.name, mode)) + host_.setThreadMode(common.Convert.strToThreadMode(mode)) + + common.confirm("Set %d hosts to thread mode %s" % (len(hosts), args.thread), args.force, setThreadMode, hosts, args.thread) elif args.os: if not hosts: raise ValueError(host_error_msg) - def setHostOs(hosts, os): - for host in hosts: - logger.debug("setting host %s to OS %s" % (host.data.name, os)) - host.proxy.setOs(os) + + def setHostOs(hosts_, os): + for host_ in hosts_: + logger.debug("setting host %s to OS %s" % (host_.data.name, os)) + host_.setOs(os) + common.confirm("Set %d hosts to OS %s" % (len(hosts), args.os), args.force, setHostOs, hosts, args.os) + elif args.repair: if not hosts: raise ValueError(host_error_msg) - def setRepairState(hosts): - for host in hosts: - logger.debug("setting host into the repair state %s" % host.data.name) - host.proxy.setHardwareState(Cue3.HardwareState.Repair) + + def setRepairState(hosts_): + for host_ in hosts_: + logger.debug("setting host into the repair state %s" % host_.data.name) + host_.setHardwareState(Cue3.api.host_pb2.REPAIR) + common.confirm("Set %d hosts into the Repair state?" % len(hosts), args.force, setRepairState, hosts) elif args.fixed: if not hosts: raise ValueError(host_error_msg) - def setUpState(hosts): - for host in hosts: - logger.debug("setting host into the repair state %s" % host.data.name) - host.proxy.setHardwareState(Cue3.HardwareState.Up) + + def setUpState(hosts_): + for host_ in hosts_: + logger.debug("setting host into the repair state %s" % host_.data.name) + host_.setHardwareState(Cue3.api.host_pb2.UP) + common.confirm("Set %d hosts into the Up state?" % len(hosts), args.force, setUpState, hosts) elif args.create_sub: - show = Cue3.findShow(args.create_sub[0]); - alloc = Cue3.findAllocation(args.create_sub[1]); - common.confirm("Create subscripton for %s on %s" % (Cue3.rep(show), Cue3.rep(alloc)), - args.force, show.proxy.createSubscription, - alloc.proxy, float(args.create_sub[2]), float(args.create_sub[3])) + show = Cue3.api.findShow(args.create_sub[0]) + alloc = Cue3.api.findAllocation(args.create_sub[1]) + common.confirm("Create subscription for %s on %s" % (Cue3.rep(show), Cue3.rep(alloc)), + args.force, show.createSubscription, + alloc.data, float(args.create_sub[2]), float(args.create_sub[3])) + elif args.delete_sub: sub_name = "%s.%s" % (args.delete_sub[1], args.delete_sub[0]) common.confirm("Delete %s's subscription to %s" % (args.delete_sub[0], args.delete_sub[1]), - args.force, Cue3.findSubscription(sub_name).proxy.delete) + args.force, Cue3.api.findSubscription(sub_name).delete) + elif args.size: sub_name = "%s.%s" % (args.size[1], args.size[0]) - Cue3.findSubscription(sub_name).proxy.setSize(float(args.size[2])) + Cue3.api.findSubscription(sub_name).setSize(float(args.size[2])) + elif args.burst: sub_name = "%s.%s" % (args.burst[1], args.burst[0]) - sub = Cue3.findSubscription(sub_name) + sub = Cue3.api.findSubscription(sub_name) burst = args.burst[2] if burst.find("%") !=-1: burst = int(sub.data.size + (sub.data.size * (int(burst[0:-1]) / 100.0))) - sub.proxy.setBurst(float(burst)) + sub.setBurst(float(burst)) + else: common.handleCommonQueryArgs(args) + def createAllocation(fac, name, tag): """Create a new allocation with the given name and tag.""" - facprx = Cue3.getFacility(fac).proxy - alloc = facprx.createAllocation(name, tag); + facility = Cue3.api.getFacility(fac) + alloc = Cue3.api.createAllocation(name, tag, facility) print "Created allocation: %s" % alloc.data.name + def displayJobs(jobs): """Displays job priority information. - @type objs: list<> - @param objs: All objects must have a .name parameter""" - format = "%-48s %-15s %5s %7s %8s %5s %8s %8s" - print format % ("Job","Group","Run","Cores","Wait","Pri","MinCores","MaxCores") + @type jobs: list<> + @param jobs: All objects must have a .name parameter""" + jobFormat = "%-48s %-15s %5s %7s %8s %5s %8s %8s" + print jobFormat % ("Job", "Group", "Run", "Cores", "Wait", "Pri", "MinCores", "MaxCores") for job in jobs: - print format % (job.data.name, - job.data.group, - job.stats.runningFrames, - job.stats.reservedCores, - job.stats.waitingFrames, - job.data.priority, - job.data.minCores, - job.data.maxCores) + print jobFormat % (job.data.name, + job.data.group, + job.stats.runningFrames, + job.stats.reservedCores, + job.stats.waitingFrames, + job.data.priority, + job.data.minCores, + job.data.maxCores) diff --git a/cueadmin/tests.py b/cueadmin/tests.py index 9a7ff9557..d645fc89d 100644 --- a/cueadmin/tests.py +++ b/cueadmin/tests.py @@ -13,12 +13,12 @@ # limitations under the License. -import unittest import logging +import unittest +import common import main -import common Cue3 = common.Cue3 logger = logging.getLogger("cue3.cuetools") @@ -29,79 +29,80 @@ TEST_FAC = "" TEST_SHOW = "" + class CueadminTests(unittest.TestCase): def testCreateShow(self): show = "test_show" - args = Parser.parse_args(["-create-show",show,"-force"]) + args = Parser.parse_args(["-create-show", show, "-force"]) try: - s = Cue3.findShow(show) - s.proxy.delete() - except Cue3.EntityNotFoundException,e: + s = Cue3.api.findShow(show) + s.delete() + except Cue3.EntityNotFoundException: pass main.handleArgs(args) - s = Cue3.findShow(show) - self.assertEqual(s.data.name,show) - s.proxy.delete() + s = Cue3.api.findShow(show) + self.assertEqual(s.data.name, show) + s.delete() def testDeleteShow(self): show = "test_show" - args = Parser.parse_args(["-delete-show",show,"-force"]) + args = Parser.parse_args(["-delete-show", show, "-force"]) try: - s = Cue3.findShow(show) - except Cue3.EntityNotFoundException,e: - s = Cue3.createShow(show) + s = Cue3.api.findShow(show) + except Cue3.EntityNotFoundException: + s = Cue3.api.createShow(show) main.handleArgs(args) try: - s = Cue3.findShow(show) + s = Cue3.api.findShow(show) assert False - except Cue3.EntityNotFoundException,e: + except Cue3.EntityNotFoundException: assert True def testEnableBooking(self): show = TEST_SHOW - args = Parser.parse_args(["-booking",show,"off","-force"]) + args = Parser.parse_args(["-booking", show, "off", "-force"]) main.handleArgs(args) - s = Cue3.findShow(show) - self.assertFalse(s.data.bookingEnabled) - args = Parser.parse_args(["-booking",show,"on","-force"]) + s = Cue3.api.findShow(show) + self.assertFalse(s.data.booking_enabled) + args = Parser.parse_args(["-booking", show, "on", "-force"]) main.handleArgs(args) - s = Cue3.findShow(show) - self.assertTrue(s.data.bookingEnabled) + s = Cue3.api.findShow(show) + self.assertTrue(s.data.booking_enabled) def testEnableDispatch(self): show = TEST_SHOW - args = Parser.parse_args(["-dispatching",show,"off","-force"]) + args = Parser.parse_args(["-dispatching", show, "off", "-force"]) main.handleArgs(args) - s = Cue3.findShow(show) - self.assertFalse(s.data.dispatchEnabled) - args = Parser.parse_args(["-dispatching",show,"on","-force"]) + s = Cue3.api.findShow(show) + self.assertFalse(s.data.dispatch_enabled) + args = Parser.parse_args(["-dispatching", show, "on", "-force"]) main.handleArgs(args) - s = Cue3.findShow(show) - self.assertTrue(s.data.dispatchEnabled) + s = Cue3.api.findShow(show) + self.assertTrue(s.data.dispatch_enabled) def testDefaultMinCores(self): show = TEST_SHOW - args = Parser.parse_args(["-default-min-cores",show,"100","-force"]) + args = Parser.parse_args(["-default-min-cores", show, "100", "-force"]) main.handleArgs(args) - s = Cue3.findShow(show) - self.assertEquals(100, s.data.defaultMinCores) - args = Parser.parse_args(["-default-min-cores",show,"1","-force"]) + s = Cue3.api.findShow(show) + self.assertEquals(100, s.data.default_min_cores) + args = Parser.parse_args(["-default-min-cores", show, "1", "-force"]) main.handleArgs(args) - s = Cue3.findShow(show) - self.assertEquals(1, s.data.defaultMinCores) + s = Cue3.api.findShow(show) + self.assertEquals(1, s.data.default_min_cores) def testDefaultMaxCores(self): show = TEST_SHOW - args = Parser.parse_args(["-default-max-cores",show,"100","-force"]) + args = Parser.parse_args(["-default-max-cores", show, "100", "-force"]) main.handleArgs(args) - s = Cue3.findShow(show) - self.assertEquals(100, s.data.defaultMaxCores) - args = Parser.parse_args(["-default-max-cores",show,"200","-force"]) + s = Cue3.api.findShow(show) + self.assertEquals(100, s.data.default_max_cores) + args = Parser.parse_args(["-default-max-cores", show, "200", "-force"]) main.handleArgs(args) - s = Cue3.findShow(show) - self.assertEquals(200, s.data.defaultMaxCores) + s = Cue3.api.findShow(show) + self.assertEquals(200, s.data.default_max_cores) def testCreateAlloc(self): fac = TEST_FAC @@ -109,35 +110,34 @@ def testCreateAlloc(self): entity = "%s.%s" % (fac, name) args = Parser.parse_args(["-create-alloc", fac, name, "tag", "-force"]) try: - s = Cue3.findAllocation(entity) - s.proxy.delete() - except Cue3.EntityNotFoundException,e: + s = Cue3.api.findAllocation(entity) + s.delete() + except Cue3.EntityNotFoundException: pass main.handleArgs(args) - s = Cue3.findAllocation(entity) + s = Cue3.api.findAllocation(entity) self.assertEqual(s.data.name, entity) - s.proxy.delete() + s.delete() def testDeleteAlloc(self): entity = "{0}.test_alloc".format(TEST_FAC) args = Parser.parse_args(["-delete-alloc", entity, "-force"]) try: - s = Cue3.findAllocation(entity) - except Cue3.EntityNotFoundException,e: - f = Cue3.getFacility(TEST_FAC) + s = Cue3.api.findAllocation(entity) + except Cue3.EntityNotFoundException: + f = Cue3.api.getFacility(TEST_FAC) f.proxy.createAllocation("test_alloc", "tulip") main.handleArgs(args) try: - Cue3.findAllocation(entity) + Cue3.api.findAllocation(entity) assert False - except Cue3.EntityNotFoundException,e: + except Cue3.EntityNotFoundException: assert True def testRenameAlloc(self): - facprx = Cue3.getFacility(TEST_FAC).proxy + facility = Cue3.api.getFacility(TEST_FAC) entity1 = "{0}.test_alloc".format(TEST_FAC) - new_name = "other_alloc" entity2 = "{0}.other_alloc".format(TEST_FAC) args = Parser.parse_args(["-rename-alloc", entity1, entity2, "-force"]) @@ -145,41 +145,41 @@ def testRenameAlloc(self): deleteAlloc(entity1) deleteAlloc(entity2) - facprx.createAllocation("test_alloc", "tulip") + facility.createAllocation("test_alloc", "tulip") main.handleArgs(args) - s = Cue3.findAllocation(entity2) + s = Cue3.api.findAllocation(entity2) self.assertEqual(s.data.name, entity2) - s.proxy.delete() + s.delete() def testTagAlloc(self): - fprx = Cue3.getFacility(TEST_FAC).proxy + facility = Cue3.api.getFacility(TEST_FAC) entity = "{0}.test_alloc".format(TEST_FAC) new_tag = "new_tag" - args = Parser.parse_args(["-tag-alloc", entity ,new_tag, "-force"]) + args = Parser.parse_args(["-tag-alloc", entity, new_tag, "-force"]) deleteAlloc(entity) - fprx.createAllocation("test_alloc", entity) + facility.createAllocation("test_alloc", entity) main.handleArgs(args) - s = Cue3.findAllocation(entity) + s = Cue3.api.findAllocation(entity) self.assertEqual(s.data.tag, new_tag) - s.proxy.delete() + s.delete() def testTransferAlloc(self): - fprx = Cue3.getFacility(TEST_FAC).proxy + facility = Cue3.api.getFacility(TEST_FAC) e1 = "{0}.talloc1".format(TEST_FAC) - e2 = "{0}.talloc2";.format(TEST_FAC) - args = Parser.parse_args(["-transfer", e1 ,e2, "-force"]) + e2 = "{0}.talloc2".format(TEST_FAC) + args = Parser.parse_args(["-transfer", e1, e2, "-force"]) deleteAlloc(e1) deleteAlloc(e2) - fprx.createAllocation("talloc1", e1) - fprx.createAllocation("talloc2", e2) + facility.createAllocation("talloc1", e1) + facility.createAllocation("talloc2", e2) main.handleArgs(args) - Cue3.findAllocation(e1).proxy.delete() - Cue3.findAllocation(e2).proxy.delete() + Cue3.api.findAllocation(e1).delete() + Cue3.api.findAllocation(e2).delete() # Need to make this test better to sure hosts are # actually being transfered on the server. @@ -189,71 +189,71 @@ def testSetRepairStare(self): args = Parser.parse_args(["-repair", "-host", e, "-force"]) main.handleArgs(args) - self.assertEquals(Cue3.findHost(e).data.state,Cue3.HardwareState.Repair) - Cue3.findHost(e).proxy.setHardwareState(Cue3.HardwareState.Up) + self.assertEquals(Cue3.api.findHost(e).data.state, Cue3.api.host_pb2.REPAIR) + Cue3.api.findHost(e).setHardwareState(Cue3.api.host_pb2.UP) def testLockHost(self): e = TEST_HOST args = Parser.parse_args(["-lock", "-host", e, "-force"]) main.handleArgs(args) - self.assertEquals(Cue3.findHost(e).data.lockState,Cue3.LockState.Locked) - Cue3.findHost(e).proxy.unlock() + self.assertEquals(Cue3.api.findHost(e).data.lock_state, Cue3.api.host_pb2.LOCKED) + Cue3.api.findHost(e).unlock() def testUnlockHost(self): e = TEST_HOST args = Parser.parse_args(["-unlock", "-host", e, "-force"]) main.handleArgs(args) - self.assertEquals(Cue3.findHost(e).data.lockState,Cue3.LockState.Open) - Cue3.findHost(e).proxy.unlock() + self.assertEquals(Cue3.api.findHost(e).data.lock_state, Cue3.api.host_pb2.OPEN) + Cue3.api.findHost(e).unlock() def testMovesHost(self): e = TEST_HOST dst = "{0}.unassigned".format(TEST_FAC) - back = Cue3.findHost(e).data.allocName + back = Cue3.api.findHost(e).data.alloc_name args = Parser.parse_args(["-move", dst, "-host", e, "-force"]) main.handleArgs(args) - self.assertEquals(Cue3.findHost(e).data.allocName, dst) + self.assertEquals(Cue3.api.findHost(e).data.alloc_name, dst) args = Parser.parse_args(["-move", back, "-host", e, "-force"]) main.handleArgs(args) - self.assertEquals(Cue3.findHost(e).data.allocName, back) + self.assertEquals(Cue3.api.findHost(e).data.alloc_name, back) def testCreateSub(self): a = "{0}.unassigned".format(TEST_FAC) h = TEST_SHOW - r = "%s.%s" % (a,h) + r = "%s.%s" % (a, h) deleteSub(r) - args = Parser.parse_args(["-create-sub", h, a, "100","110", "-force"]) + args = Parser.parse_args(["-create-sub", h, a, "100", "110", "-force"]) main.handleArgs(args) - s = Cue3.findSubscription(r) - self.assertEquals(s.data.showName,h) - self.assertEquals(s.data.allocationName,a) + s = Cue3.api.findSubscription(r) + self.assertEquals(s.data.show_name, h) + self.assertEquals(s.data.allocation_name, a) self.assertEquals(s.data.name, r) self.assertEquals(s.data.size, float(100)) self.assertEquals(s.data.burst, float(110)) - s.proxy.delete() + s.delete() def testDeleteSub(self): a = "{0}.unassigned".format(TEST_FAC) h = TEST_SHOW - r = "%s.%s" % (a,h) + r = "%s.%s" % (a, h) deleteSub(r) - show = Cue3.findShow(h) - show.proxy.createSubscription(Cue3.findAllocation(a).proxy, 100.0, 110.0) + show = Cue3.api.findShow(h) + show.createSubscription(Cue3.api.findAllocation(a).data, 100.0, 110.0) args = Parser.parse_args(["-delete-sub", h, a, "-force"]) main.handleArgs(args) try: - Cue3.findSubscription(r) + Cue3.api.findSubscription(r) raise Exception("subscription should have been deleted") - except: + except Exception: pass def testSetSize(self): @@ -262,14 +262,14 @@ def testSetSize(self): r = "%s.%s" % (a, h) deleteSub(r) - show = Cue3.findShow(h) - show.proxy.createSubscription(Cue3.findAllocation(a).proxy, 100.0, 110.0) + show = Cue3.api.findShow(h) + show.createSubscription(Cue3.api.findAllocation(a).data, 100.0, 110.0) - args = Parser.parse_args(["-size", h, a, "200","-force"]) + args = Parser.parse_args(["-size", h, a, "200", "-force"]) main.handleArgs(args) - s = Cue3.findSubscription(r) - self.assertEquals(s.data.size,200.0) + s = Cue3.api.findSubscription(r) + self.assertEquals(s.data.size, 200.0) deleteSub(r) def testSetBurst (self): @@ -278,14 +278,14 @@ def testSetBurst (self): r = "%s.%s" % (a, h) deleteSub(r) - show = Cue3.findShow(h) - show.proxy.createSubscription(Cue3.findAllocation(a).proxy, 100.0, 110.0) + show = Cue3.api.findShow(h) + show.createSubscription(Cue3.api.findAllocation(a).data, 100.0, 110.0) - args = Parser.parse_args(["-burst", h, a, "200","-force"]) + args = Parser.parse_args(["-burst", h, a, "200", "-force"]) main.handleArgs(args) - s = Cue3.findSubscription(r) - self.assertEquals(s.data.burst,200.0) + s = Cue3.api.findSubscription(r) + self.assertEquals(s.data.burst, 200.0) deleteSub(r) def testSetBurstPercentage (self): @@ -294,29 +294,32 @@ def testSetBurstPercentage (self): r = "%s.%s" % (a, h) deleteSub(r) - show = Cue3.findShow(h) - show.proxy.createSubscription(Cue3.findAllocation(a).proxy, 100.0, 110.0) + show = Cue3.api.findShow(h) + show.createSubscription(Cue3.api.findAllocation(a).data, 100.0, 110.0) args = Parser.parse_args(["-burst", h, a, "20%","-force"]) main.handleArgs(args) - s = Cue3.findSubscription(r) + s = Cue3.api.findSubscription(r) self.assertEquals(s.data.burst, 120.0) deleteSub(r) + def deleteSub(name): try: - Cue3.findSubscription(name).proxy.delete() - except Cue3.EntityNotFoundException,e: + Cue3.api.findSubscription(name).delete() + except Cue3.EntityNotFoundException: pass + def deleteAlloc(name): try: - s = Cue3.findAllocation(name) - s.proxy.delete() - except Cue3.EntityNotFoundException,e: + s = Cue3.api.findAllocation(name) + s.delete() + except Cue3.EntityNotFoundException: pass + def run(parser): if not TEST_FAC or not TEST_HOST or not TEST_SHOW: print "Please set TEST_FAC, TEST_HOST and TEST_SHOW before running tests." diff --git a/spi_cue/Cue3/wrappers/allocation.py b/spi_cue/Cue3/wrappers/allocation.py index 03d91d6ef..e8cf8d9ca 100644 --- a/spi_cue/Cue3/wrappers/allocation.py +++ b/spi_cue/Cue3/wrappers/allocation.py @@ -18,6 +18,9 @@ allocation module """ +import host +import subscription + from Cue3.compiled_proto import facility_pb2 from Cue3.compiled_proto import host_pb2 from Cue3.cuebot import Cuebot @@ -29,23 +32,6 @@ def __init__(self, allocation=None): self.data = allocation self.stub = Cuebot.getStub('allocation') - def setName(self, name): - """Sets a new name for the allocation. - @type name: str - @param name: the new name""" - self.stub.SetName( - facility_pb2.AllocSetNameRequest(allocation=self.data, name=name), - timeout=Cuebot.Timeout) - - def setTag(self, tag): - """Sets a new tag for the allocation. - @type name: str - @param name: the new tag""" - self.stub.SetTag( - facility_pb2.AllocSetTagRequest(allocation=self.data, tag=tag), - timeout=Cuebot.Timeout - ) - def delete(self): """Delete the record of the allocation from the cuebot""" self.stub.Delete( @@ -53,10 +39,30 @@ def delete(self): timeout=Cuebot.Timeout ) + def getHosts(self): + """Returns the list of hosts for this allocation. + @rtype: list + @return: list of hosts + """ + hostSeq = self.stub.GetHosts(facility_pb2.AllocGetHostsRequest(allocation=self.data), + timeout=Cuebot.Timeout).hosts + return [host.Host(h) for h in hostSeq.hosts] + + def getSubscriptions(self): + """Get the subscriptions of this allocation. + @rtype: list + @return: a list of subscriptions + """ + subscriptionSeq = self.stub.GetSubscriptions( + facility_pb2.AllocGetSubscriptionsRequest(allocation=self.data), + timeout=Cuebot.Timeout).subscriptions + return [subscription.Subscription(sub) for sub in subscriptionSeq.subscriptions] + def reparentHosts(self, hosts): """Moves the given hosts to the allocation @type hosts: list - @param hosts: The hosts to move to this allocation""" + @param hosts: The hosts to move to this allocation + """ hostSeq = host_pb2.HostSeq() hostSeq.hosts.extend(hosts) self.stub.ReparentHosts( @@ -64,48 +70,74 @@ def reparentHosts(self, hosts): timeout=Cuebot.Timeout ) + def setName(self, name): + """Sets a new name for the allocation. + @type name: str + @param name: the new name + """ + self.stub.SetName( + facility_pb2.AllocSetNameRequest(allocation=self.data, name=name), + timeout=Cuebot.Timeout) + + def setTag(self, tag): + """Sets a new tag for the allocation. + @type name: str + @param name: the new tag + """ + self.stub.SetTag( + facility_pb2.AllocSetTagRequest(allocation=self.data, tag=tag), + timeout=Cuebot.Timeout + ) + def id(self): """Returns the id of the allocation @rtype: str - @return: Allocation uuid""" + @return: Allocation uuid + """ return self.data.id def name(self): """Returns the name of the allocation @rtype: str - @return: Allocation name""" + @return: Allocation name + """ return self.data.name def tag(self): """Returns the allocation tag @rtype: str - @return: Allocation tag""" + @return: Allocation tag + """ return self.data.tag def totalCores(self): """Returns the total number of cores in the allocation. @rtype: float - @return: Total number of cores in the allocation""" + @return: Total number of cores in the allocation + """ return self.data.stats.cores def totalAvailableCores(self): """Returns the total number of cores available for booking in the allocation. @rtype: float - @return: Total number of cores in the allocation""" + @return: Total number of cores in the allocation + """ return self.data.stats.available_cores def totalIdleCores(self): """Returns the total number of idle cores in the allocation. @rtype: float - @return: Total number of idle cores in the allocation""" + @return: Total number of idle cores in the allocation + """ return self.data.stats.idle_cores def totalRunningCores(self): """Returns the total number of running cores in the allocation. Each 100 returned is the same as 1 physical core. @rtype: float - @return: Total number of running cores in the allocation""" + @return: Total number of running cores in the allocation + """ # All core reserved return self.data.stats.running_cores @@ -113,13 +145,15 @@ def totalLockedCores(self): """Returns the total number of locked cores in the allocation. Each 100 returned is the same as 1 physical core. @rtype: float - @return: Total number of locked cores in the allocation""" + @return: Total number of locked cores in the allocation + """ return self.data.stats.locked_cores def totalHosts(self): """Returns the total number of hosts in the allocation @rtype: int - @return: Total number of hosts in the allocation""" + @return: Total number of hosts in the allocation + """ return self.data.stats.hosts def totalLockedHosts(self): @@ -131,5 +165,6 @@ def totalLockedHosts(self): def totalDownHosts(self): """Returns the total number of down hosts in the allocation @rtype: int - @return: Total number of down hosts in the allocation""" + @return: Total number of down hosts in the allocation + """ return self.data.stats.down_hosts diff --git a/spi_cue/Cue3/wrappers/host.py b/spi_cue/Cue3/wrappers/host.py index 85b09847a..1935e4a04 100644 --- a/spi_cue/Cue3/wrappers/host.py +++ b/spi_cue/Cue3/wrappers/host.py @@ -140,6 +140,29 @@ def getComments(self): timeout=Cuebot.Timeout) return [comment.Comment(c) for c in response.comments] + def setHardwareState(self, state): + """Sets the host's hardware state + @type state: host_pb2.HardwareState + @param state: state to set host to""" + self.stub.SetHardwareState( + host_pb2.HostSetHardwareStateRequest(host=self.data, state=state), + timeout=Cuebot.Timeout) + + def setOs(self, os): + """Sets the host os + @type os: string + @param os: os value to set host to""" + self.stub.SetOs(host_pb2.HostSetOsRequest(host=self.data, os=os), + timeout=Cuebot.Timeout) + + def setThreadMode(self, mode): + """Set the thread mode to mode + @type mode: ThreadMode + @param mode: ThreadMode to set host to + """ + self.stub.SetThreadMode(host_pb2.HostSetThreadModeRequest(host=self.data, mode=mode), + timeout=Cuebot.Timeout) + def id(self): """Returns the id of the host @rtype: str diff --git a/spi_cue/Cue3/wrappers/show.py b/spi_cue/Cue3/wrappers/show.py index 8d9fdaa7a..0abce9062 100644 --- a/spi_cue/Cue3/wrappers/show.py +++ b/spi_cue/Cue3/wrappers/show.py @@ -62,6 +62,20 @@ def createSubscription(self, allocation, size, burst): timeout=Cuebot.Timeout) return subscription.Subscription(response.subscription) + def delete(self): + """Delete this show""" + self.stub.Delete(show_pb2.ShowDeleteRequest(show=self.data), timeout=Cuebot.Timeout) + + def getServiceOverrides(self): + """Returns a list of service overrides on the show. + @rtype: list + @return: a list of service override objects + """ + serviceOverrideSeq = self.stub.GetServiceOverrides( + show_pb2.ShowGetServiceOverridesRequest(show=self.data), + timeout=Cuebot.Timeout).service_overrides + return serviceOverrideSeq.service_overrides + def getSubscriptions(self): """Returns a list of all subscriptions @rtype: list @@ -91,6 +105,14 @@ def getFilters(self): timeout=Cuebot.Timeout) return [filter.Filter(filter) for filter in response.filters] + def setActive(self, value): + """Set the active state of this show to value + @type value: bool + @param value: boolean value to set active state to + """ + self.stub.SetActive(show_pb2.ShowSetActiveRequest(show=self.data, value=value), + timeout=Cuebot.Timeout) + def setDefaultMaxCores(self, maxcores): """Sets the default maximum number of cores that new jobs are launched with.