From ef67c56046625d8b7ae01f414d3e32fc314f26a0 Mon Sep 17 00:00:00 2001 From: Lars van der Bijl <285658+larsbijl@users.noreply.github.com> Date: Thu, 10 Dec 2020 20:42:46 +0000 Subject: [PATCH 001/277] Add booking status bar into Monitor Cue widget. (#837) --- cuegui/cuegui/CueJobMonitorTree.py | 4 ++-- cuegui/cuegui/DarkPalette.py | 3 +++ cuegui/cuegui/ItemDelegate.py | 23 ++++++++++++----------- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/cuegui/cuegui/CueJobMonitorTree.py b/cuegui/cuegui/CueJobMonitorTree.py index 4c5c03504..68c34486b 100644 --- a/cuegui/cuegui/CueJobMonitorTree.py +++ b/cuegui/cuegui/CueJobMonitorTree.py @@ -103,8 +103,8 @@ def __init__(self, parent): data=lambda job: job.data.job_stats.total_frames, sort=lambda job: job.data.job_stats.total_frames, tip="The total number of frames.") -# self.addColumn("_Booking Bar", 150, id=8, default=False, -# delegate=JobBookingBarDelegate) + self.addColumn("_Booking Bar", 150, id=8, + delegate=cuegui.ItemDelegate.JobBookingBarDelegate) self.addColumn("Min", 38, id=9, data=lambda job: "%.0f" % job.data.min_cores, sort=lambda job: job.data.min_cores, diff --git a/cuegui/cuegui/DarkPalette.py b/cuegui/cuegui/DarkPalette.py index 554426d64..224424598 100644 --- a/cuegui/cuegui/DarkPalette.py +++ b/cuegui/cuegui/DarkPalette.py @@ -126,3 +126,6 @@ def ColorF(r, g, b): COLOR_SHOW_BACKGROUND = GreyF(0.13) COLOR_SHOW_FOREGROUND = GreyF(0.79) COLOR_JOB_FOREGROUND = GreyF(0.79) + +KILL_ICON_COLOUR = QtGui.QColor(224, 52, 52) +PAUSE_ICON_COLOUR = QtGui.QColor(88, 163, 209) \ No newline at end of file diff --git a/cuegui/cuegui/ItemDelegate.py b/cuegui/cuegui/ItemDelegate.py index 0ec671aa6..5ea6c7888 100644 --- a/cuegui/cuegui/ItemDelegate.py +++ b/cuegui/cuegui/ItemDelegate.py @@ -165,34 +165,37 @@ def paint(self, painter, option, index): rect = option.rect.adjusted(12, 6, -12, -6) - jobMin = int(job.data.minCores * 100) - jobMax = int(job.data.maxCores * 100) - jobRunning = job.data.runningFrames - jobWaiting = job.data.waitingFrames - painter.save() try: self._drawBackground(painter, option, index) try: + jobRunning = job.data.job_stats.running_frames + jobWaiting = job.data.job_stats.waiting_frames + try: + cores_per_frame = float(job.data.job_stats.reserved_cores / jobRunning) + except: + cores_per_frame = float(6 / 1) + jobMin = int(job.data.min_cores / cores_per_frame) + jobMax = int(job.data.max_cores / cores_per_frame) ratio = rect.width() / float(jobRunning + jobWaiting) if jobWaiting: painter.fillRect( rect.adjusted(0, 2, 0, -2), - RGB_FRAME_STATE[opencue.api.job_pb2.FrameState.Waiting]) + RGB_FRAME_STATE[opencue.api.job_pb2.WAITING]) if jobRunning: painter.fillRect( rect.adjusted(0, 0, -int(ceil(ratio * jobWaiting)), 0), - RGB_FRAME_STATE[opencue.api.job_pb2.FrameState.Running]) + RGB_FRAME_STATE[opencue.api.job_pb2.RUNNING]) - painter.setPen(QtCore.Qt.blue) + painter.setPen(cuegui.Style.ColorTheme.PAUSE_ICON_COLOUR) x = min(rect.x() + ratio * jobMin, option.rect.right() - 9) painter.drawLine(x, option.rect.y(), x, option.rect.y() + option.rect.height()) - painter.setPen(QtCore.Qt.red) + painter.setPen(cuegui.Style.ColorTheme.KILL_ICON_COLOUR) x = min(rect.x() + ratio * jobMax, option.rect.right() - 6) painter.drawLine(x, option.rect.y(), x, option.rect.y() + option.rect.height()) @@ -200,8 +203,6 @@ def paint(self, painter, option, index): except ZeroDivisionError: pass - if option.state & QtWidgets.QStyle.State_Selected: - self._drawSelectionOverlay(painter, option) finally: painter.restore() del painter From d0af3213ad14551104f8fb4081bf58a6dc5b93a0 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Thu, 10 Dec 2020 21:06:40 +0000 Subject: [PATCH 002/277] Notes from Dec 9 TSC meeting. (#858) --- tsc/meetings/2020-12-09.md | 66 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 tsc/meetings/2020-12-09.md diff --git a/tsc/meetings/2020-12-09.md b/tsc/meetings/2020-12-09.md new file mode 100644 index 000000000..3e92e8091 --- /dev/null +++ b/tsc/meetings/2020-12-09.md @@ -0,0 +1,66 @@ +# OpenCue TSC Meeting Notes 9 Dec 2020 + +Secretary: Brian Cipriano + +TSC Attendees: + +* [ ] Matt Chambers +* [x] Brian Cipriano +* [x] Greg Denton +* [x] Ben Dines +* [ ] Dave Fellows +* [x] Daniel Neilson +* [ ] Alex Schworer +* [ ] Erik Strauss + +Committer Attendees: + +* [ ] Christian Smith +* [x] Diego Tavares +* [x] Lars van der Bijl + +Agenda/Notes: + +* User survey + * Still coming soon, ASWF wrapping things up. Will delay our project graduation proposal as we + want to include these results as part of the proposal. +* CII badge updates + * Cuebot -Werror done. + * Working on Python linting: + * PR out for pyoutline linting. Will move on to other components next. + * Working on resolving SonarCloud issues: + * A few PRs out for some initial issues. +* Python packaging + * On hold while Brian works on completing the CII badge. +* Check on SPI gRPC issues + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/821 + * Hasn't been an issue for SPI, seems to be resolved. + * We should upgrade gRPC as high as we can while maintaining Python 2 support. We need to do + this anyway for Python 3.8 support. +* SPI database performance issues affecting DispatchQuery + * Seeing db load problems with more users. + * When many active jobs are waiting, DispatchQuery takes a long time. + * Found problems with the query when looking for gpu in range zero-zero. + * HostReport also takes longer when db under load. + * Changed how host reports are handled. Was a simple thread pool, now will just process the + last report from each host. + * PRs with these fixes coming soon. +* Batch of new GUI bugs from SPI + * Some are actually just GUI bugs, others are due to API methods needing to be implemented, + mostly in Python API but some on Cuebot side as well. +* Check on dropping Oracle support + * SPI has no plans to move back to Oracle, good to drop it. + * Let's do a final check with mailing lists before deleting. +* Daniel: Had a very difficult time setting up an OpenCue deployment, found docs severely lacking + among other issues. + * Docs issues + * Ran into issues with port reservations. Needed guide on config options to avoid digging + through source code. + * Guide overall was just too complex and difficult to follow. + * We need to rename demo_data.sql, it's not clear that this is needed for a functioning system. + * Docker compose sandbox can't be recreated, needs some work. + * Agreed we should prioritize a docs refresh in the new year. Packaging is part of this but not + all. + * Brian to start a brainstorming email. +* Lars: we need someone to test stylesheet changes on windows +* Dec 23 meeting canceled, will resume meetings in January. From 459ce4aefd69efc9aeed06861e03f782a7f35f44 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Fri, 11 Dec 2020 08:50:52 -0800 Subject: [PATCH 003/277] Bug fixes for creating Service at show level. (#818) --- cuegui/cuegui/MenuActions.py | 2 +- cuegui/cuegui/ServiceDialog.py | 29 ++++++++++++----------------- pycue/opencue/wrappers/show.py | 22 ++++++++++++++++++++++ 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/cuegui/cuegui/MenuActions.py b/cuegui/cuegui/MenuActions.py index 8202adef7..6a59ebac5 100644 --- a/cuegui/cuegui/MenuActions.py +++ b/cuegui/cuegui/MenuActions.py @@ -1085,7 +1085,7 @@ def taskProperties(self, rpcObjects=None): opencue.api.findShow(rootgroup.data.name), self._caller).show() - serviceProperties_info = ["Service Properies...", None, "view"] + serviceProperties_info = ["Service Properties...", None, "view"] def serviceProperties(self, rpcObjects=None): for rootgroup in self._getOnlyRootGroupObjects(rpcObjects): cuegui.ServiceDialog.ServiceDialog( diff --git a/cuegui/cuegui/ServiceDialog.py b/cuegui/cuegui/ServiceDialog.py index 3b07e3ee8..13837e624 100644 --- a/cuegui/cuegui/ServiceDialog.py +++ b/cuegui/cuegui/ServiceDialog.py @@ -108,16 +108,13 @@ def setService(self, service): Update the form with data from the given service. """ self.__buttons.setDisabled(False) - self.__service = service.data - - self.name.setText(self.__service.name) - self.threadable.setChecked(self.__service.threadable) - self.min_cores.setValue(self.__service.min_cores) - self.max_cores.setValue(self.__service.max_cores) - self.min_memory.setValue(self.__service.min_memory // 1024) - self.min_gpu.setValue(self.__service.min_gpu // 1024) - - self._tags_w.set_tags(self.__service.tags) + self.name.setText(service.data.name) + self.threadable.setChecked(service.data.threadable) + self.min_cores.setValue(service.data.min_cores) + self.max_cores.setValue(service.data.max_cores) + self.min_memory.setValue(service.data.min_memory // 1024) + self.min_gpu.setValue(service.data.min_gpu // 1024) + self._tags_w.set_tags(service.data.tags) def new(self): """ @@ -257,17 +254,15 @@ def refresh(self): self.__service_list.selectedItems()] self.__service_list.clear() - try: - if not self.__show: - self.__services = opencue.api.getDefaultServices() - else: - self.__services = self.__show.getServiceOverrides() - except Exception: - return + if not self.__show: + self.__services = opencue.api.getDefaultServices() + else: + self.__services = self.__show.getServiceOverrides() for service in self.__services: item = QtWidgets.QListWidgetItem(service.name()) self.__service_list.addItem(item) + if service.name() in selected: item.setSelected(True) diff --git a/pycue/opencue/wrappers/show.py b/pycue/opencue/wrappers/show.py index 005bd4172..8ab4fce15 100644 --- a/pycue/opencue/wrappers/show.py +++ b/pycue/opencue/wrappers/show.py @@ -68,6 +68,28 @@ def delete(self): """Delete this show""" self.stub.Delete(show_pb2.ShowDeleteRequest(show=self.data), timeout=Cuebot.Timeout) + def createServiceOverride(self, data): + """Creates a Service Override at the show level. + + :type data: opencue.wrapper.service.Service + :param data: Service.data object + """ + self.stub.CreateServiceOverride(show_pb2.ShowCreateServiceOverrideRequest( + show=self.data, service=data), + timeout=Cuebot.Timeout) + + def getServiceOverride(self, serviceName): + """ + Returns a service override for a show + + :param serviceName: name of the service for the show + :return: service override object + """ + serviceOverride = self.stub.GetServiceOverride(show_pb2.ShowGetServiceOverrideRequest( + show=self.data, name=serviceName), + timeout=Cuebot.Timeout).service_override + return serviceOverride + def getServiceOverrides(self): """Returns a list of service overrides on the show. From 71d9790756293e4ce9ac7b89e0874e60af4e56d7 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Fri, 11 Dec 2020 08:51:38 -0800 Subject: [PATCH 004/277] Increase max render memory in FilterDialog. (#816) --- cuegui/cuegui/FilterDialog.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cuegui/cuegui/FilterDialog.py b/cuegui/cuegui/FilterDialog.py index 6ee9b0406..b121be897 100644 --- a/cuegui/cuegui/FilterDialog.py +++ b/cuegui/cuegui/FilterDialog.py @@ -51,6 +51,7 @@ FILTERTYPE = opencue.compiled_proto.filter_pb2.FilterType.keys() PAUSETYPE = ["Pause", "Unpause"] MEMOPTTYPE = ["Enabled", "Disabled"] +MAX_RENDER_MEM = 251.0 class FilterDialog(QtWidgets.QDialog): @@ -446,7 +447,7 @@ def createAction(self): "How much memory (in GB) should each render layer require?", 4.0, 0.1, - 47.0, + MAX_RENDER_MEM, 2) value = int(value * 1048576) @@ -671,6 +672,7 @@ def __setValue(self, value = None): value = widget.value() elif self.rpcObject.type() in (opencue.api.filter_pb2.SET_ALL_RENDER_LAYER_MEMORY,): + widget.setMaximum(MAX_RENDER_MEM) value = int(widget.value() * 1048576) elif self.rpcObject.type() in (opencue.api.filter_pb2.SET_JOB_MAX_CORES, @@ -715,6 +717,7 @@ def updateWidgets(self): widget = NoWheelDoubleSpinBox(self.parent()) widget.setDecimals(2) widget.setSingleStep(.10) + widget.setMaximum(MAX_RENDER_MEM) widget.editingFinished.connect(self.__setValue) elif self.rpcObject.type() in (opencue.api.filter_pb2.SET_JOB_MAX_CORES, From 806488da56eef2f2b4953ec18826b79208f2f72e Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Fri, 11 Dec 2020 08:52:17 -0800 Subject: [PATCH 005/277] Fix FrameSearch pagination. (#781) --- cuegui/cuegui/FrameMonitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuegui/cuegui/FrameMonitor.py b/cuegui/cuegui/FrameMonitor.py index 3147675e5..09b42c114 100644 --- a/cuegui/cuegui/FrameMonitor.py +++ b/cuegui/cuegui/FrameMonitor.py @@ -250,7 +250,7 @@ def _pageButtonsHandle(self, offset): frameSearch @type offset: int''' self.page += offset - self.frameMonitorTree.frameSearch.page = self.page + self.frameMonitorTree.frameSearch.options['page'] = self.page self.frameMonitorTree.updateRequest() self._updatePageButtonState() From 70810acdb7a8864804809b05e5b69c7040233dc8 Mon Sep 17 00:00:00 2001 From: Lars van der Bijl <285658+larsbijl@users.noreply.github.com> Date: Sat, 12 Dec 2020 16:23:40 +0000 Subject: [PATCH 006/277] Add subscription graph (#839) Display a job's usage's using the bar chart. Closes #838 --- cuegui/cuegui/DarkPalette.py | 2 +- cuegui/cuegui/ItemDelegate.py | 50 ++++ cuegui/cuegui/SubscriptionGraphWidget.py | 214 ++++++++++++++++++ .../plugins/SubscriptionsGraphPlugin.py | 29 +++ 4 files changed, 294 insertions(+), 1 deletion(-) create mode 100644 cuegui/cuegui/SubscriptionGraphWidget.py create mode 100644 cuegui/cuegui/plugins/SubscriptionsGraphPlugin.py diff --git a/cuegui/cuegui/DarkPalette.py b/cuegui/cuegui/DarkPalette.py index 224424598..0dbbf912c 100644 --- a/cuegui/cuegui/DarkPalette.py +++ b/cuegui/cuegui/DarkPalette.py @@ -127,5 +127,5 @@ def ColorF(r, g, b): COLOR_SHOW_FOREGROUND = GreyF(0.79) COLOR_JOB_FOREGROUND = GreyF(0.79) -KILL_ICON_COLOUR = QtGui.QColor(224, 52, 52) +KILL_ICON_COLOUR = QtGui.QColor(224, 52, 52) PAUSE_ICON_COLOUR = QtGui.QColor(88, 163, 209) \ No newline at end of file diff --git a/cuegui/cuegui/ItemDelegate.py b/cuegui/cuegui/ItemDelegate.py index 5ea6c7888..fec57d66d 100644 --- a/cuegui/cuegui/ItemDelegate.py +++ b/cuegui/cuegui/ItemDelegate.py @@ -210,6 +210,56 @@ def paint(self, painter, option, index): AbstractDelegate.paint(self, painter, option, index) +class SubBookingBarDelegate(AbstractDelegate): + def __init__(self, parent, *args): + AbstractDelegate.__init__(self, parent, *args) + + def paint(self, painter, option, index): + # This itemFromIndex could cause problems + sub = self.parent().itemFromIndex(index).rpcObject + + # The total allocation core count x 100 to match sub value + alloc_obj = index.data(QtCore.Qt.DisplayRole) + alloc_size = (alloc_obj.data.stats.cores) * 100 + + rect = option.rect.adjusted(12, 6, -12, -6) + painter.save() + try: + self._drawBackground(painter, option, index) + try: + subRunning = sub.data.reserved_cores + subMin = int(sub.data.size) + subMax = int(sub.data.burst) + ratio = rect.width() / alloc_size + + if alloc_size: + painter.fillRect( + rect.adjusted(0, 2, 0, -2), + RGB_FRAME_STATE[opencue.api.job_pb2.WAITING]) + + if subRunning: + painter.fillRect( + rect.adjusted(0, 0, -int(ceil(ratio * (alloc_size - subRunning))), 0), + RGB_FRAME_STATE[opencue.api.job_pb2.RUNNING]) + + painter.setPen(cuegui.Style.ColorTheme.PAUSE_ICON_COLOUR) + x = min(rect.x() + ratio * subMin, option.rect.right() - 9) + painter.drawLine(x, option.rect.y(), x, + option.rect.y() + option.rect.height()) + + painter.setPen(cuegui.Style.ColorTheme.KILL_ICON_COLOUR) + x = min(rect.x() + ratio * subMax, option.rect.right() - 6) + painter.drawLine(x, option.rect.y(), x, + option.rect.y() + option.rect.height()) + + except ZeroDivisionError: + pass + + finally: + painter.restore() + del painter + + class JobThinProgressBarDelegate(AbstractDelegate): def __init__(self, parent, *args): AbstractDelegate.__init__(self, parent, *args) diff --git a/cuegui/cuegui/SubscriptionGraphWidget.py b/cuegui/cuegui/SubscriptionGraphWidget.py new file mode 100644 index 000000000..25981bf31 --- /dev/null +++ b/cuegui/cuegui/SubscriptionGraphWidget.py @@ -0,0 +1,214 @@ +from builtins import str + +import opencue + +from PySide2 import QtCore +from PySide2 import QtGui +from PySide2 import QtWidgets + +import cuegui.AbstractTreeWidget +import cuegui.AbstractWidgetItem +import cuegui.Constants +import cuegui.CreatorDialog +import cuegui.ItemDelegate +import cuegui.MenuActions +import cuegui.ShowDialog +import cuegui.Utils + + +class SubscriptionGraphWidget(QtWidgets.QWidget): + def __init__(self, parent): + QtWidgets.QWidget.__init__(self, parent) + + self.__shows = [] + self.__showMenuActions = {} + self.__subBars = [] + self.__timer = QtCore.QTimer(self) + self.__timer.timeout.connect(self.update_data) + self.__timer.setInterval(1000 * 5) + + widget = QtWidgets.QWidget() + self.mainLayout = QtWidgets.QHBoxLayout(widget) + self.mainLayout.setContentsMargins(0, 0, 0, 0) + scroll = QtWidgets.QScrollArea() + scroll.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff) + scroll.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOn) + scroll.setWidgetResizable(True) + scroll.setWidget(widget) + + showMenuBtn = QtWidgets.QPushButton(" Shows") + showMenuBtn.setFixedWidth(100) + showMenuBtn.pressed.connect(self.__showMenuCheck) + + self.__showMenu = QtWidgets.QMenu(self) + showMenuBtn.setMenu(self.__showMenu) + self.__showMenu.setStyleSheet("QMenu { menu-scrollable: 1; }") + + showMenuBtn.setFocusPolicy(QtCore.Qt.NoFocus) + self.__showMenu.setFont(cuegui.Constants.STANDARD_FONT) + self.__showMenu.triggered.connect(self.__showMenuHandle) + + layout = QtWidgets.QVBoxLayout(self) + layout.addWidget(showMenuBtn) + layout.addWidget(scroll) + + def create_widgets(self): + self.clearLayout(self.mainLayout) + for show in self.__shows: + widget = QtWidgets.QWidget() + layout = QtWidgets.QVBoxLayout(widget) + + show_label = QtWidgets.QLabel(show) + layout.addWidget(show_label) + + sub_bar = SubGraphTreeWidget(self) + sub_bar.setShow(show) + layout.addWidget(sub_bar) + self.__subBars.append(sub_bar) + + self.mainLayout.addWidget(widget) + + self.__timer.start() + + def clearLayout(self, layout): + while layout.count() > 0: + item = layout.takeAt(0) + if not item: + continue + w = item.widget() + if w: + w.deleteLater() + + def __showMenuHandle(self, action): + if action.text() == 'All Shows': + try: + self.__shows = sorted(set([job.show() for job in opencue.api.getJobs(include_finished=True)])) + except Exception as e: + self.__shows = [] + self.__showMenuUpdate() + elif action.text() == 'Clear': + self.__shows = [] + self.__showMenuUpdate() + elif action.isChecked(): + self.__shows.append(action.text()) + self.__showMenuUpdate() + else: + self.__shows.remove(action.text()) + self.__showMenuUpdate() + + def __showMenuUpdate(self): + self.__showMenu.clear() + self.__showMenuActions = {} + + # add all shows menu item + action = QtWidgets.QAction('All Shows', self.__showMenu) + self.__showMenu.addAction(action) + self.__showMenuActions['All Shows'] = action + action = QtWidgets.QAction('Clear', self.__showMenu) + self.__showMenu.addAction(action) + self.__showMenuActions['Clear'] = action + self.__showMenu.addSeparator() + + try: + shows = sorted(set([job.show() for job in opencue.api.getJobs(include_finished=True)])) + except Exception as e: + shows = [] + + for show in shows: + action = QtWidgets.QAction(show, self.__showMenu) + action.setCheckable(True) + if show in self.__shows: + action.setChecked(True) + self.__showMenu.addAction(action) + self.__showMenuActions[show] = action + + self.create_widgets() + + def __showMenuCheck(self): + """Populate the list of shows if it is empty""" + if not self.__showMenuActions: + self.__showMenuUpdate() + + def update_data(self): + for sub_bar in self.__subBars: + sub_bar._getUpdate() + + +class SubGraphTreeWidget(cuegui.AbstractTreeWidget.AbstractTreeWidget): + def __init__(self, parent): + self.startColumnsForType(cuegui.Constants.TYPE_SUB) + self.addColumn("_Name", 110, id=0, + data=lambda sub: sub.data.allocation_name) + self.addColumn("_Booking Bar", 125, id=1, + delegate=cuegui.ItemDelegate.SubBookingBarDelegate, + data=lambda sub: opencue.api.findAllocation(sub.data.allocation_name)) + cuegui.AbstractTreeWidget.AbstractTreeWidget.__init__(self, parent) + + self.header().hide() + self.__show = None + + # Used to build right click context menus + self.__menuActions = cuegui.MenuActions.MenuActions( + self, self.updateSoon, self.selectedObjects, self.getShow) + + self.setMinimumSize(240, 80) + + # self.setUpdateInterval(30) + + def setShow(self, show=None): + self._itemsLock.lockForWrite() + try: + if not show: + self.__show = None + elif cuegui.Utils.isShow(show): + self.__show = show + elif isinstance(show, str): + try: + self.__show = opencue.api.findShow(show) + except: + pass + self._update() + finally: + self._itemsLock.unlock() + + def getShow(self): + return self.__show + + def _createItem(self, object): + """Creates and returns the proper item""" + return SubscriptionWidgetItem(object, self) + + def _getUpdate(self): + """Returns the proper data from the cuebot""" + self._itemsLock.lockForWrite() + try: + if self.__show: + return self.__show.getSubscriptions() + return [] + finally: + self._itemsLock.unlock() + + def contextMenuEvent(self, e): + """When right clicking on an item, this raises a context menu""" + + menu = QtWidgets.QMenu() + self.__menuActions.subscriptions().addAction(menu, "editSize") + self.__menuActions.subscriptions().addAction(menu, "editBurst") + menu.addSeparator() + self.__menuActions.subscriptions().addAction(menu, "delete") + menu.addSeparator() + if self.__show: + new_action = QtWidgets.QAction('Add new subscription', self) + new_action.triggered.connect(self.createSubscription) + menu.addAction(new_action) + menu.exec_(QtCore.QPoint(e.globalX(),e.globalY())) + + def createSubscription(self): + d = cuegui.CreatorDialog.SubscriptionCreatorDialog(show=self.__show) + d.exec_() + + +class SubscriptionWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): + def __init__(self, object, parent): + cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( + self, cuegui.Constants.TYPE_SUB, object, parent) \ No newline at end of file diff --git a/cuegui/cuegui/plugins/SubscriptionsGraphPlugin.py b/cuegui/cuegui/plugins/SubscriptionsGraphPlugin.py new file mode 100644 index 000000000..9e895aaf5 --- /dev/null +++ b/cuegui/cuegui/plugins/SubscriptionsGraphPlugin.py @@ -0,0 +1,29 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from PySide2 import QtCore +from PySide2 import QtWidgets + +import cuegui.AbstractDockWidget +import cuegui.SubscriptionGraphWidget + +PLUGIN_NAME = 'Subscription Graphs' +PLUGIN_CATEGORY = "Cuecommander" +PLUGIN_DESCRIPTION = "An administrator interface to subscriptions" +PLUGIN_REQUIRES = "CueCommander" +PLUGIN_PROVIDES = 'SubscriptionGraphDockWidget' + + +class SubscriptionGraphDockWidget(cuegui.AbstractDockWidget.AbstractDockWidget): + """This builds what is displayed on the dock widget""" + def __init__(self, parent): + cuegui.AbstractDockWidget.AbstractDockWidget.__init__(self, parent, PLUGIN_NAME) + + self.__splitter = QtWidgets.QSplitter(QtCore.Qt.Horizontal) + self.layout().addWidget(self.__splitter) + + self.__subgraph_widget = cuegui.SubscriptionGraphWidget.SubscriptionGraphWidget(self) + self.__splitter.addWidget(self.__subgraph_widget) + + self.setMinimumSize(500, 190) From 0ec41362f06919455081558088caef1515f00dca Mon Sep 17 00:00:00 2001 From: Lars van der Bijl <285658+larsbijl@users.noreply.github.com> Date: Tue, 15 Dec 2020 02:26:13 +0000 Subject: [PATCH 007/277] Add automatic color highlighting to the LogView plugin. (#831) --- cuegui/cuegui/Constants.py | 4 ++ cuegui/cuegui/DarkPalette.py | 9 +++- cuegui/cuegui/plugins/LogViewPlugin.py | 74 ++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 1 deletion(-) diff --git a/cuegui/cuegui/Constants.py b/cuegui/cuegui/Constants.py index 273da00ee..27503f363 100644 --- a/cuegui/cuegui/Constants.py +++ b/cuegui/cuegui/Constants.py @@ -128,3 +128,7 @@ COLOR_USER_4 = QtGui.QColor(50, 30, 0) QT_MAX_INT = 2147483647 + +LOG_HIGHLIGHT_ERROR = ['error', 'aborted', 'fatal', 'failed', 'killed', 'command not found', 'no licenses could be found', 'killMessage'] +LOG_HIGHLIGHT_WARN = ['warning', 'not found'] +LOG_HIGHLIGHT_INFO = ['info:', 'rqd cmd:'] diff --git a/cuegui/cuegui/DarkPalette.py b/cuegui/cuegui/DarkPalette.py index 0dbbf912c..304f334a0 100644 --- a/cuegui/cuegui/DarkPalette.py +++ b/cuegui/cuegui/DarkPalette.py @@ -127,5 +127,12 @@ def ColorF(r, g, b): COLOR_SHOW_FOREGROUND = GreyF(0.79) COLOR_JOB_FOREGROUND = GreyF(0.79) +#Log file Colors +LOG_TIME = QtGui.QColor(170, 149, 171) +LOG_ERROR = QtGui.QColor(224, 52, 52) +LOG_WARNING = QtGui.QColor(255, 201, 25) +LOG_INFO = QtGui.QColor(111, 140, 255) +LOG_COMPLETE = QtGui.QColor(132, 201, 12) + KILL_ICON_COLOUR = QtGui.QColor(224, 52, 52) -PAUSE_ICON_COLOUR = QtGui.QColor(88, 163, 209) \ No newline at end of file +PAUSE_ICON_COLOUR = QtGui.QColor(88, 163, 209) diff --git a/cuegui/cuegui/plugins/LogViewPlugin.py b/cuegui/cuegui/plugins/LogViewPlugin.py index edb775a18..2db14b47b 100644 --- a/cuegui/cuegui/plugins/LogViewPlugin.py +++ b/cuegui/cuegui/plugins/LogViewPlugin.py @@ -28,6 +28,7 @@ from PySide2 import QtCore from PySide2 import QtWidgets +import cuegui.Constants import cuegui.AbstractDockWidget @@ -362,6 +363,8 @@ def __init__(self, parent=None): self._content_box.moveCursor(QtGui.QTextCursor.End) self._content_box.ensureCursorVisible() + self.highlighter = Highlighter(self._content_box.document()) + # Search search_top_widget = QtWidgets.QWidget(self) search_top_layout = QtWidgets.QVBoxLayout(search_top_widget) @@ -376,6 +379,8 @@ def __init__(self, parent=None): self._case_stv_checkbox.stateChanged.connect(self._move_to_search_box) self._search_box = QtWidgets.QLineEdit('', self) + self._search_box.setClearButtonEnabled(True) + self._search_box.setPlaceholderText('Search log...') search_layout.addWidget(self._search_box) self._search_box.show() self._search_box.editingFinished.connect(self._find_text) @@ -862,3 +867,72 @@ def __init__(self, parent=None): self, parent, PLUGIN_NAME, QtCore.Qt.RightDockWidgetArea) self.logview_widget = LogViewWidget(self) self.layout().addWidget(self.logview_widget) + + +class Highlighter(QtGui.QSyntaxHighlighter): + def __init__(self, parent=None): + super(Highlighter, self).__init__(parent) + + self.on = True + + self.timeFormat = QtGui.QTextCharFormat() + self.timeFormat.setFontWeight(QtGui.QFont.Bold) + self.timeFormat.setForeground(cuegui.Style.ColorTheme.LOG_TIME) + + self.errorFormat = QtGui.QTextCharFormat() + self.errorFormat.setFontWeight(QtGui.QFont.Bold) + self.errorFormat.setForeground(cuegui.Style.ColorTheme.LOG_ERROR) + + self.warnFormat = QtGui.QTextCharFormat() + self.warnFormat.setFontWeight(QtGui.QFont.Bold) + self.warnFormat.setForeground(cuegui.Style.ColorTheme.LOG_WARNING) + + self.infoFormat = QtGui.QTextCharFormat() + self.infoFormat.setFontWeight(QtGui.QFont.Bold) + self.infoFormat.setForeground(cuegui.Style.ColorTheme.LOG_INFO) + + self.completeFormat = QtGui.QTextCharFormat() + self.completeFormat.setFontWeight(QtGui.QFont.Bold) + self.completeFormat.setForeground(cuegui.Style.ColorTheme.LOG_COMPLETE) + + + def highlightBlock(self, text): + if not self.on: + return + + line = text.lower() + done = False + + for error in cuegui.Constants.LOG_HIGHLIGHT_ERROR: + if error in line: + self.setFormat(0, len(text), self.errorFormat) + done = True + break + + if not done: + for warn in cuegui.Constants.LOG_HIGHLIGHT_WARN: + if warn in line: + self.setFormat(0, len(text), self.warnFormat) + done = True + break + + if not done: + for info in cuegui.Constants.LOG_HIGHLIGHT_INFO: + if info in line: + self.setFormat(0, len(text), self.infoFormat) + done = True + break + + if 'alf_progress' in line: + sidx = line.index('alf_progress') + eidx = line.index('%') + self.setFormat(sidx, eidx + 1, self.infoFormat) + + if ' | ' in line: + idx = line.index(' | ') + self.setFormat(0, idx, self.timeFormat) + + if 'render job complete' in line: + self.setFormat(0, len(text), self.completeFormat) + + self.setCurrentBlockState(0) From 4c549f674ae75748b02593afef17b93b5263687c Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Mon, 14 Dec 2020 18:35:46 -0800 Subject: [PATCH 008/277] Set reproducible builds for Cuebot JAR. (#846) --- cuebot/build.gradle | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cuebot/build.gradle b/cuebot/build.gradle index 0a7c114a8..3839055f0 100644 --- a/cuebot/build.gradle +++ b/cuebot/build.gradle @@ -162,3 +162,8 @@ sonarqube { property "sonar.issue.ignore.multicriteria.j1.resourceKey", "**/*.java" } } + +tasks.withType(AbstractArchiveTask) { + preserveFileTimestamps = false + reproducibleFileOrder = true +} From 41b3b80dd46d1ad7eeac8c8813fb1a3d56c4e7e7 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Tue, 15 Dec 2020 09:48:28 -0800 Subject: [PATCH 009/277] Add RQD_BECOME_JOB_USER config setting to disable user switching. (#847) --- rqd/rqd/__main__.py | 3 ++- rqd/rqd/rqconstants.py | 3 +++ rqd/rqd/rqcore.py | 7 +++++-- rqd/rqd/rqutil.py | 8 +++++--- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/rqd/rqd/__main__.py b/rqd/rqd/__main__.py index 29adfd063..81130fdc1 100755 --- a/rqd/rqd/__main__.py +++ b/rqd/rqd/__main__.py @@ -101,7 +101,8 @@ def usage(): def main(): setupLogging() - if platform.system() == 'Linux' and os.getuid() != 0: + if platform.system() == 'Linux' and os.getuid() != 0 and \ + rqd.rqconstants.RQD_BECOME_JOB_USER: logging.critical("Please run launch as root") sys.exit(1) diff --git a/rqd/rqd/rqconstants.py b/rqd/rqd/rqconstants.py index fb854e400..ed2c2f361 100644 --- a/rqd/rqd/rqconstants.py +++ b/rqd/rqd/rqconstants.py @@ -66,6 +66,7 @@ RQD_RETRY_STARTUP_CONNECT_DELAY = 30 RQD_RETRY_CRITICAL_REPORT_DELAY = 30 RQD_USE_IP_AS_HOSTNAME = True +RQD_BECOME_JOB_USER = True RQD_CREATE_USER_IF_NOT_EXISTS = True KILL_SIGNAL = 9 @@ -182,6 +183,8 @@ LOAD_MODIFIER = config.getint(__section, "LOAD_MODIFIER") if config.has_option(__section, "RQD_USE_IP_AS_HOSTNAME"): RQD_USE_IP_AS_HOSTNAME = config.getboolean(__section, "RQD_USE_IP_AS_HOSTNAME") + if config.has_option(__section, "RQD_BECOME_JOB_USER"): + RQD_BECOME_JOB_USER = config.getboolean(__section, "RQD_BECOME_JOB_USER") if config.has_option(__section, "DEFAULT_FACILITY"): DEFAULT_FACILITY = config.get(__section, "DEFAULT_FACILITY") if config.has_option(__section, "LAUNCH_FRAME_USER_GID"): diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index dcd0daa3e..f0f34a63f 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -263,8 +263,11 @@ def runLinux(self): rqd.rqutil.permissionsHigh() try: - tempCommand += ["/bin/su", runFrame.user_name, rqd.rqconstants.SU_ARGUEMENT, - '"' + self._createCommandFile(runFrame.command) + '"'] + if rqd.rqconstants.RQD_BECOME_JOB_USER: + tempCommand += ["/bin/su", runFrame.user_name, rqd.rqconstants.SU_ARGUEMENT, + '"' + self._createCommandFile(runFrame.command) + '"'] + else: + tempCommand += [self._createCommandFile(runFrame.command)] # Actual cwd is set by /shots/SHOW/home/perl/etc/qwrap.cuerun frameInfo.forkedCommand = subprocess.Popen(tempCommand, diff --git a/rqd/rqd/rqutil.py b/rqd/rqd/rqutil.py index 5fb7e1560..aa84e5eb9 100644 --- a/rqd/rqd/rqutil.py +++ b/rqd/rqd/rqutil.py @@ -73,7 +73,7 @@ def cacheGet(self, cache, key, func): def permissionsHigh(): """Sets the effective gid/uid to processes original values (root)""" - if platform.system() == "Windows": + if platform.system() == "Windows" or not rqd.rqconstants.RQD_BECOME_JOB_USER: return PERMISSIONS.acquire() os.setegid(os.getgid()) @@ -87,7 +87,7 @@ def permissionsHigh(): def permissionsLow(): """Sets the effective gid/uid to one with less permissions: RQD_GID and RQD_UID""" - if platform.system() in ('Windows', 'Darwin'): + if platform.system() in ('Windows', 'Darwin') or not rqd.rqconstants.RQD_BECOME_JOB_USER: return if os.getegid() != rqd.rqconstants.RQD_GID or os.geteuid() != rqd.rqconstants.RQD_UID: __becomeRoot() @@ -100,7 +100,7 @@ def permissionsLow(): def permissionsUser(uid, gid): """Sets the effective gid/uid to supplied values""" - if platform.system() in ('Windows', 'Darwin'): + if platform.system() in ('Windows', 'Darwin') or not rqd.rqconstants.RQD_BECOME_JOB_USER: return PERMISSIONS.acquire() __becomeRoot() @@ -128,6 +128,8 @@ def __becomeRoot(): def checkAndCreateUser(username): """Check to see if the provided user exists, if not attempt to create it.""" # TODO(gregdenton): Add Windows and Mac support here. (Issue #61) + if not rqd.rqconstants.RQD_BECOME_JOB_USER: + return try: pwd.getpwnam(username) return From ec4ec16541091544ad9ee4a81a86c8f6e174b0ea Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 15 Dec 2020 18:11:04 +0000 Subject: [PATCH 010/277] Add note to PR template on adding tests for new code. (#855) --- .github/PULL_REQUEST_TEMPLATE.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 61eb8799b..12b68a1b9 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -12,4 +12,8 @@ It's also helpful to describe **why** you're making this change. From b3add2c568c8b578c13a4b7433004d742a76c7e6 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 15 Dec 2020 18:19:03 +0000 Subject: [PATCH 011/277] Fix or ignore a few issues flagged by SonarCloud. (#857) --- cuebot/build.gradle | 8 +++++++- .../main/java/com/imageworks/spcue/Entity.java | 6 ++++-- .../java/com/imageworks/spcue/SortableShow.java | 16 +++++++++------- .../spcue/service/DepartmentManagerService.java | 6 +++++- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/cuebot/build.gradle b/cuebot/build.gradle index 3839055f0..2af5f04d4 100644 --- a/cuebot/build.gradle +++ b/cuebot/build.gradle @@ -154,12 +154,18 @@ sonarqube { // NOTE: sonar.login must be provided manually, like: // ./gradlew sonarqube -Dsonar.login= - property "sonar.issue.ignore.multicriteria", "j1" + property "sonar.issue.ignore.multicriteria", "j1,j2" // Incompatible @Transactional requirements // It's claimed this causes runtime exceptions but we don't see such issues. property "sonar.issue.ignore.multicriteria.j1.ruleKey", "java:S2229" property "sonar.issue.ignore.multicriteria.j1.resourceKey", "**/*.java" + + // Null pointers + // DAO classes make heavy use of getJdbcTemplate() which triggers this rule, but is + // never null in practice. + property "sonar.issue.ignore.multicriteria.j2.ruleKey", "java:S2259" + property "sonar.issue.ignore.multicriteria.j2.resourceKey", "src/main/java/com/imageworks/spcue/dao/**/*.java" } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/Entity.java b/cuebot/src/main/java/com/imageworks/spcue/Entity.java index d79d772b4..d45b37ab2 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/Entity.java +++ b/cuebot/src/main/java/com/imageworks/spcue/Entity.java @@ -63,8 +63,10 @@ public int hashCode() { @Override public boolean equals(Object other) { - return this.toString().equals( - other.toString()); + if (other == null) { + return false; + } + return this.toString().equals(other.toString()); } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/SortableShow.java b/cuebot/src/main/java/com/imageworks/spcue/SortableShow.java index 1b22a8adb..6e8e8c90c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/SortableShow.java +++ b/cuebot/src/main/java/com/imageworks/spcue/SortableShow.java @@ -104,13 +104,15 @@ public int hashCode() { @Override public boolean equals(Object other) { - try { - SortableShow that = (SortableShow) other; - return that.getShowId().equals(this.getShowId()); - } - catch (ClassCastException e) { - return false; - } + if (other == null) { + return false; + } + try { + SortableShow that = (SortableShow) other; + return that.getShowId().equals(this.getShowId()); + } catch (ClassCastException e) { + return false; + } } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/DepartmentManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/DepartmentManagerService.java index b00af37cd..0d09ca44c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/DepartmentManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/DepartmentManagerService.java @@ -206,7 +206,11 @@ public void updateManagedTasks(PointInterface pd) { * the current data set. Tasks with a 0 minCores value will * be deleted. */ - float normalizedRawPoints = p.cores / totalRawPoints; + float normalizedRawPoints = 0f; + if (totalRawPoints != 0) { + normalizedRawPoints = p.cores / totalRawPoints; + } + for (TrackitTaskDetail task : tasks) { TaskEntity td = new TaskEntity(); From 2228f164b2dafcc430fd50787ead19db0766cb85 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Tue, 15 Dec 2020 10:40:50 -0800 Subject: [PATCH 012/277] Add Host.os to pycue wrapper. (#852) --- pycue/opencue/wrappers/host.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pycue/opencue/wrappers/host.py b/pycue/opencue/wrappers/host.py index fe4dc893b..8a98222c0 100644 --- a/pycue/opencue/wrappers/host.py +++ b/pycue/opencue/wrappers/host.py @@ -403,6 +403,12 @@ def lockState(self): """ return self.data.lock_state + def os(self): + """ + :rtype: str + :return: the operating system of the host + """ + return self.data.os class NestedHost(Host): """This class contains information and actions related to a nested host.""" From 96cefe2de55d0f8328928b885fd6f7e495a3c852 Mon Sep 17 00:00:00 2001 From: Lars van der Bijl <285658+larsbijl@users.noreply.github.com> Date: Tue, 15 Dec 2020 23:33:01 +0000 Subject: [PATCH 013/277] Add menu option to CueGUI to reset to the default layout. (#833) --- cuegui/cuegui/Main.py | 8 ++++++-- cuegui/cuegui/MainWindow.py | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/cuegui/cuegui/Main.py b/cuegui/cuegui/Main.py index a68d9492e..12575c82b 100644 --- a/cuegui/cuegui/Main.py +++ b/cuegui/cuegui/Main.py @@ -90,12 +90,16 @@ def startup(app_name, app_version, argv): config_path = "/.%s/config" % app_name.lower() settings = QtCore.QSettings(QtCore.QSettings.IniFormat, QtCore.QSettings.UserScope, config_path) + local = settings.fileName() + # If the user has chose to revert the layout. delete the file and copy the default back. + if settings.value('RevertLayout'): + os.remove(local) + QtGui.qApp.settings = settings cuegui.Style.init() - # If the config file does not exist, copy over the default - local = settings.fileName() + # If the config file does not exist, copy over the default if not os.path.exists(local): default = os.path.join(cuegui.Constants.DEFAULT_INI_PATH, "%s.ini" % app_name.lower()) logger.warning('Not found: %s\nCopying: %s' % (local, default)) diff --git a/cuegui/cuegui/MainWindow.py b/cuegui/cuegui/MainWindow.py index bac8cedf8..50565e8c6 100644 --- a/cuegui/cuegui/MainWindow.py +++ b/cuegui/cuegui/MainWindow.py @@ -240,6 +240,11 @@ def __windowMenuSetup(self, menu): saveWindowSettings.triggered.connect(self.__saveSettings) menu.addAction(saveWindowSettings) + # Menu Bar: Window -> Revert To Default Window Layout + revertWindowSettings = QtWidgets.QAction("Revert To Default Window Layout", self) + revertWindowSettings.triggered.connect(self.__revertLayout) + menu.addAction(revertWindowSettings) + menu.addSeparator() # Load list of window titles @@ -422,3 +427,15 @@ def __saveSettings(self): self.size()) self.settings.setValue("%s/Position" % self.name, self.pos()) + + def __revertLayout(self): + """Revert back to default window layout""" + result = QtWidgets.QMessageBox.question( + self, + "Restart required ", + "You must restart for this action to take effect, close window?: ", + QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No) + + if result == QtWidgets.QMessageBox.Yes: + self.settings.setValue("RevertLayout", True) + self.__windowCloseApplication() From 518d3c112f9e5e7d7b800671f5c91c927983bdbb Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Tue, 15 Dec 2020 15:33:42 -0800 Subject: [PATCH 014/277] Fix population of JobDetail startTime and stopTime. (#853) --- .../java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java index 59c578576..a3dab8f4e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java @@ -21,6 +21,7 @@ import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -125,6 +126,10 @@ public JobDetail mapRow(ResultSet rs, int rowNum) throws SQLException { job.email = rs.getString("str_email"); job.totalFrames = rs.getInt("int_frame_count"); job.totalLayers = rs.getInt("int_layer_count"); + Timestamp startTime = rs.getTimestamp("ts_started"); + job.startTime = startTime != null ? (int) (startTime.getTime() / 1000) : 0; + Timestamp stopTime = rs.getTimestamp("ts_stopped"); + job.stopTime = stopTime != null ? (int) (stopTime.getTime() / 1000) : 0; job.isPaused = rs.getBoolean("b_paused"); job.maxRetries = rs.getInt("int_max_retries"); job.showName = rs.getString("show_name"); @@ -207,6 +212,8 @@ public boolean isJobComplete(JobInterface job) { "job.str_email,"+ "job.int_frame_count,"+ "job.int_layer_count,"+ + "job.ts_started,"+ + "job.ts_stopped,"+ "job.b_paused,"+ "job.int_max_retries,"+ "job_resource.int_max_cores,"+ From 18efedd82a90f8f72ac93d53e71963653582af28 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Tue, 15 Dec 2020 15:34:20 -0800 Subject: [PATCH 015/277] Properly handle "proc not found" errors on host report. (#867) --- .../spcue/dispatcher/HostReportHandler.java | 44 +++++++++++-------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java index a84814957..83d7342de 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java @@ -423,18 +423,19 @@ private void handleMemoryReservations(final DispatchHost host, final HostReport for (final RunningFrameInfo f: report.getFramesList()) { - VirtualProc p = hostManager.getVirtualProc(f.getResourceId()); + VirtualProc proc = null; + try { + proc = hostManager.getVirtualProc(f.getResourceId()); + + // TODO: handle memory management for local dispatches + // Skip local dispatches for now. + if (proc.isLocalDispatch) { + continue; + } - // TODO: handle memory management for local dispatches - // Skip local dispatches for now. - if (p.isLocalDispatch) { - continue; - } - try { if (f.getRss() > host.memory) { try{ - VirtualProc proc = hostManager.findVirtualProc(p); logger.info("Killing frame " + f.getJobName() + "/" + f.getFrameName() + ", " + proc.getName() + " was OOM"); try { @@ -442,17 +443,17 @@ private void handleMemoryReservations(final DispatchHost host, final HostReport CueUtil.KbToMb(f.getRss()) + "MB but the machine only has " + CueUtil.KbToMb(host.memory), rqdClient)); } catch (TaskRejectedException e) { - logger.warn("Unable to queue RQD kill, task rejected, " + e); + logger.warn("Unable to queue RQD kill, task rejected, " + e); } DispatchSupport.killedOomProcs.incrementAndGet(); } catch (Exception e) { - logger.info("failed to kill frame on " + p.getName() + + logger.info("failed to kill frame on " + proc.getName() + "," + e); } } - if (dispatchSupport.increaseReservedMemory(p, f.getRss())) { - p.memoryReserved = f.getRss(); + if (dispatchSupport.increaseReservedMemory(proc, f.getRss())) { + proc.memoryReserved = f.getRss(); logger.info("frame " + f.getFrameName() + " on job " + f.getJobName() + " increased its reserved memory to " + CueUtil.KbToMb(f.getRss())); @@ -460,23 +461,28 @@ private void handleMemoryReservations(final DispatchHost host, final HostReport } catch (ResourceReservationFailureException e) { - long memNeeded = f.getRss() - p.memoryReserved; + long memNeeded = f.getRss() - proc.memoryReserved; logger.info("frame " + f.getFrameName() + " on job " + f.getJobName() + "was unable to reserve an additional " + CueUtil.KbToMb(memNeeded) - + "on proc " + p.getName() + ", " + e); + + "on proc " + proc.getName() + ", " + e); try { - if (dispatchSupport.balanceReservedMemory(p, memNeeded)) { - p.memoryReserved = f.getRss(); - logger.info("was able to balance host: " + p.getName()); + if (dispatchSupport.balanceReservedMemory(proc, memNeeded)) { + proc.memoryReserved = f.getRss(); + logger.info("was able to balance host: " + proc.getName()); } else { - logger.info("failed to balance host: " + p.getName()); + logger.info("failed to balance host: " + proc.getName()); } } catch (Exception ex) { - logger.warn("failed to balance host: " + p.getName() + ", " + e); + logger.warn("failed to balance host: " + proc.getName() + ", " + e); } + } catch (EmptyResultDataAccessException e) { + logger.info("HostReportHandler: frame " + f.getFrameName() + + " on job " + f.getJobName() + + " was unable be processed" + + " because the proc could not be found"); } } From 7c58cca248af084174c472b858ab89fcab8a1203 Mon Sep 17 00:00:00 2001 From: Lars van der Bijl <285658+larsbijl@users.noreply.github.com> Date: Wed, 16 Dec 2020 20:22:23 +0000 Subject: [PATCH 016/277] Display key information in the job's progressbar. (#835) --- cuegui/cuegui/ItemDelegate.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/cuegui/cuegui/ItemDelegate.py b/cuegui/cuegui/ItemDelegate.py index fec57d66d..2d399a108 100644 --- a/cuegui/cuegui/ItemDelegate.py +++ b/cuegui/cuegui/ItemDelegate.py @@ -293,6 +293,12 @@ def paint(self, painter, option, index): if index.data(QtCore.Qt.UserRole) == cuegui.Constants.TYPE_JOB: # This is a lot of data calls to build this one item frameStateTotals = index.data(QtCore.Qt.UserRole + 1) + + complete_tasks = frameStateTotals[opencue.api.job_pb2.SUCCEEDED] + total_tasks = sum([i for i in frameStateTotals.values()]) + proc = float(complete_tasks) * 100 / float(total_tasks) + line = "{0:d} % ({1:d}/{2:d})".format(int(proc), int(complete_tasks), int(total_tasks)) + state = index.data(QtCore.Qt.UserRole + 2) paused = index.data(QtCore.Qt.UserRole + 3) @@ -302,15 +308,11 @@ def paint(self, painter, option, index): self._drawProgressBar(painter, option.rect.adjusted(0, 2, 0, -2), frameStateTotals) - if state == opencue.api.job_pb2.FINISHED: - painter.setPen(QtCore.Qt.black) - painter.drawText(option.rect, 0, "Finished") - elif paused: - painter.setPen(QtCore.Qt.blue) - painter.drawText(option.rect, 0, "Paused") + painter.setPen(QtCore.Qt.black) + painter.drawText(option.rect, QtCore.Qt.AlignCenter, line) except Exception as e: painter.setPen(QtCore.Qt.red) - painter.drawText(option.rect, 0, "Gui Error") + painter.drawText(option.rect, QtCore.Qt.AlignCenter, "Gui Error") finally: painter.restore() del painter From f160bfd9f867788e66c8d6ebc5442d89af586a92 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Thu, 17 Dec 2020 16:07:00 -0800 Subject: [PATCH 017/277] Fix integer overflow on getProc query (#865) * Update dispatchQuery to use min_cores Sorting jobs only by priority causes a situation where low priority jobs can get starved by a constant flow of high priority jobs. The new formula adds a modifier to the sorting rank to take into account the number of cores the job is requesting and also the number of days the job is waiting on the queue. Priorities numbers over 200 will mostly override the formula and work as a priority only based scheduling. sort = priority + (100 * (1 - (job.cores/job.int_min_cores))) + (age in days) Besides that, also take layer_int_cores_min into account when filtering folder_resourse limitations to avoid allocating more cores than the folder limits. (cherry picked from commit 566411aeeddc60983a30eabe121fd03263d05525) * Revert "Update dispatchQuery to use min_cores" This reverts commit 2eb4936c * Fix integer overflow on getProc query (cherry picked from commit 2b5a4eb3b6c4acc5c711c2ff70644d957572166b) --- .../com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java index 985601cd4..be8643b7e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java @@ -276,12 +276,12 @@ public VirtualProc mapRow(ResultSet rs, int rowNum) throws SQLException { proc.allocationId = rs.getString("pk_alloc"); proc.facilityId = rs.getString("pk_facility"); proc.coresReserved =rs.getInt("int_cores_reserved"); - proc.memoryReserved = rs.getInt("int_mem_reserved"); - proc.memoryMax = rs.getInt("int_mem_max_used"); - proc.gpuReserved = rs.getInt("int_gpu_reserved"); + proc.memoryReserved = rs.getLong("int_mem_reserved"); + proc.memoryMax = rs.getLong("int_mem_max_used"); + proc.gpuReserved = rs.getLong("int_gpu_reserved"); proc.virtualMemoryMax = rs.getLong("int_virt_max_used"); proc.virtualMemoryUsed = rs.getLong("int_virt_used"); - proc.memoryUsed = rs.getInt("int_mem_used"); + proc.memoryUsed = rs.getLong("int_mem_used"); proc.unbooked = rs.getBoolean("b_unbooked"); proc.isLocalDispatch = rs.getBoolean("b_local"); proc.os = rs.getString("str_os"); From e20693ddb6ebb6f243fecdaf50931dcb3b136dee Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Thu, 17 Dec 2020 16:07:18 -0800 Subject: [PATCH 018/277] Fix copy functionality in log view plugi (#864) * Update dispatchQuery to use min_cores Sorting jobs only by priority causes a situation where low priority jobs can get starved by a constant flow of high priority jobs. The new formula adds a modifier to the sorting rank to take into account the number of cores the job is requesting and also the number of days the job is waiting on the queue. Priorities numbers over 200 will mostly override the formula and work as a priority only based scheduling. sort = priority + (100 * (1 - (job.cores/job.int_min_cores))) + (age in days) Besides that, also take layer_int_cores_min into account when filtering folder_resourse limitations to avoid allocating more cores than the folder limits. (cherry picked from commit 566411aeeddc60983a30eabe121fd03263d05525) * Revert "Update dispatchQuery to use min_cores" This reverts commit 2eb4936c * Fix copy functionality in log view plugin Treatment of connecting a signal to a slot with default params in PySide differs a bit from PyQt. Fixed it by explicitly indicating the datatype for the trigger signal (cherry picked from commit a387e041fd051c722537b886c011f25d839c5f14) Co-authored-by: Fermi Perumal --- cuegui/cuegui/plugins/LogViewPlugin.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cuegui/cuegui/plugins/LogViewPlugin.py b/cuegui/cuegui/plugins/LogViewPlugin.py index 2db14b47b..f49957538 100644 --- a/cuegui/cuegui/plugins/LogViewPlugin.py +++ b/cuegui/cuegui/plugins/LogViewPlugin.py @@ -108,8 +108,8 @@ def __init__(self, parent): self.copy_action = QtWidgets.QAction('Copy', self) self.copy_action.setStatusTip('Copy Selection') self.copy_action.setShortcut('Ctrl+C') - self.copy_action.triggered.connect(lambda triggered, i=0: - self.copy_selection(0)) + self.copy_action.triggered[bool].connect(lambda triggered: + self.copy_selection(QtGui.QClipboard.Clipboard)) self.addAction(self.copy_action) def context_menu(self): @@ -159,7 +159,6 @@ def copy_selection(self, mode): QtGui.QClipboard.Selection = Selection (middle-mouse)) @type mode: int """ - selection = self.textCursor().selection() QtWidgets.QApplication.clipboard().setText('', mode) QtWidgets.QApplication.clipboard().setText(selection.toPlainText(), mode) From 639f9a9d22e38ff15d4a16db837f0f0b959e90be Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Thu, 17 Dec 2020 16:07:48 -0800 Subject: [PATCH 019/277] Add try/catch to setupJobData calls (#862) * Update dispatchQuery to use min_cores Sorting jobs only by priority causes a situation where low priority jobs can get starved by a constant flow of high priority jobs. The new formula adds a modifier to the sorting rank to take into account the number of cores the job is requesting and also the number of days the job is waiting on the queue. Priorities numbers over 200 will mostly override the formula and work as a priority only based scheduling. sort = priority + (100 * (1 - (job.cores/job.int_min_cores))) + (age in days) Besides that, also take layer_int_cores_min into account when filtering folder_resourse limitations to avoid allocating more cores than the folder limits. (cherry picked from commit 566411aeeddc60983a30eabe121fd03263d05525) * Revert "Update dispatchQuery to use min_cores" This reverts commit 2eb4936c * Add try/catch to setupJobData calls Calls with jobs that were already removed from the database were flooding the logs and also return exception traces down to the API. (cherry picked from commit 9e4a075413eb816b0f00f096a2783469a200dd5c) --- .../imageworks/spcue/servant/ManageJob.java | 604 ++++++++++++------ 1 file changed, 412 insertions(+), 192 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java index 4f0756c75..dd00476cd 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java @@ -223,31 +223,52 @@ public void isJobPending(JobIsJobPendingRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - FrameSeq frameSeq = whiteboard.getFrames(frameSearchFactory.create(job, request.getReq())); - responseObserver.onNext(JobGetFramesResponse.newBuilder() - .setFrames(frameSeq) - .build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + FrameSeq frameSeq = whiteboard.getFrames(frameSearchFactory.create(job, request.getReq())); + responseObserver.onNext(JobGetFramesResponse.newBuilder() + .setFrames(frameSeq) + .build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void getLayers(JobGetLayersRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - LayerSeq layerSeq = whiteboard.getLayers(job); - responseObserver.onNext(JobGetLayersResponse.newBuilder() - .setLayers(layerSeq) - .build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + LayerSeq layerSeq = whiteboard.getLayers(job); + responseObserver.onNext(JobGetLayersResponse.newBuilder() + .setLayers(layerSeq) + .build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void kill(JobKillRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - manageQueue.execute(new DispatchJobComplete(job, - new Source(request.toString()), true, jobManagerSupport)); - responseObserver.onNext(JobKillResponse.newBuilder().build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + manageQueue.execute(new DispatchJobComplete(job, + new Source(request.toString()), true, jobManagerSupport)); + responseObserver.onNext(JobKillResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override @@ -297,303 +318,502 @@ public void launchSpec(JobLaunchSpecRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - jobManager.setJobPaused(job, true); - responseObserver.onNext(JobPauseResponse.newBuilder().build()); - responseObserver.onCompleted(); + try{ + setupJobData(request.getJob()); + jobManager.setJobPaused(job, true); + responseObserver.onNext(JobPauseResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void resume(JobResumeRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - jobManager.setJobPaused(job, false); - responseObserver.onNext(JobResumeResponse.newBuilder().build()); - responseObserver.onCompleted(); + try{ + setupJobData(request.getJob()); + jobManager.setJobPaused(job, false); + responseObserver.onNext(JobResumeResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void setMaxCores(JobSetMaxCoresRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - jobDao.updateMaxCores(job, Convert.coresToWholeCoreUnits(request.getVal())); - responseObserver.onNext(JobSetMaxCoresResponse.newBuilder().build()); - responseObserver.onCompleted(); + try{ + setupJobData(request.getJob()); + jobDao.updateMaxCores(job, Convert.coresToWholeCoreUnits(request.getVal())); + responseObserver.onNext(JobSetMaxCoresResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void setMinCores(JobSetMinCoresRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - jobDao.updateMinCores(job, Convert.coresToWholeCoreUnits(request.getVal())); - responseObserver.onNext(JobSetMinCoresResponse.newBuilder().build()); - responseObserver.onCompleted(); + try{ + setupJobData(request.getJob()); + jobDao.updateMinCores(job, Convert.coresToWholeCoreUnits(request.getVal())); + responseObserver.onNext(JobSetMinCoresResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void setPriority(JobSetPriorityRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - jobDao.updatePriority(job, request.getVal()); - responseObserver.onNext(JobSetPriorityResponse.newBuilder().build()); - responseObserver.onCompleted(); + try{ + setupJobData(request.getJob()); + jobDao.updatePriority(job, request.getVal()); + responseObserver.onNext(JobSetPriorityResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void getCurrent(JobGetCurrentRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - Job currentJob = whiteboard.getJob(job.getId()); - responseObserver.onNext(JobGetCurrentResponse.newBuilder() - .setJob(currentJob) - .build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + Job currentJob = whiteboard.getJob(job.getId()); + responseObserver.onNext(JobGetCurrentResponse.newBuilder() + .setJob(currentJob) + .build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void eatFrames(JobEatFramesRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - manageQueue.execute( - new DispatchEatFrames( - frameSearchFactory.create(job, request.getReq()), - new Source(request.toString()), - jobManagerSupport)); - responseObserver.onNext(JobEatFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + manageQueue.execute( + new DispatchEatFrames( + frameSearchFactory.create(job, request.getReq()), + new Source(request.toString()), + jobManagerSupport)); + responseObserver.onNext(JobEatFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void killFrames(JobKillFramesRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - manageQueue.execute( - new DispatchKillFrames( - frameSearchFactory.create(job, request.getReq()), - new Source(request.toString()), - jobManagerSupport)); - responseObserver.onNext(JobKillFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + manageQueue.execute( + new DispatchKillFrames( + frameSearchFactory.create(job, request.getReq()), + new Source(request.toString()), + jobManagerSupport)); + responseObserver.onNext(JobKillFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void markDoneFrames(JobMarkDoneFramesRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - manageQueue.execute( - new DispatchSatisfyDepends( - frameSearchFactory.create(job, request.getReq()), jobManagerSupport)); - responseObserver.onNext(JobMarkDoneFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + try{ + setupJobData(request.getJob()); + manageQueue.execute( + new DispatchSatisfyDepends( + frameSearchFactory.create(job, request.getReq()), jobManagerSupport)); + responseObserver.onNext(JobMarkDoneFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void retryFrames(JobRetryFramesRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - manageQueue.execute( - new DispatchRetryFrames( - frameSearchFactory.create(job, request.getReq()), - new Source(request.toString()), - jobManagerSupport)); - responseObserver.onNext(JobRetryFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + manageQueue.execute( + new DispatchRetryFrames( + frameSearchFactory.create(job, request.getReq()), + new Source(request.toString()), + jobManagerSupport)); + responseObserver.onNext(JobRetryFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void setAutoEat(JobSetAutoEatRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - jobDao.updateAutoEat(job, request.getValue()); - responseObserver.onNext(JobSetAutoEatResponse.newBuilder().build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + jobDao.updateAutoEat(job, request.getValue()); + responseObserver.onNext(JobSetAutoEatResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void createDependencyOnFrame(JobCreateDependencyOnFrameRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - JobOnFrame depend = new JobOnFrame(job, - jobManager.getFrameDetail(request.getFrame().getId())); - dependManager.createDepend(depend); - responseObserver.onNext(JobCreateDependencyOnFrameResponse.newBuilder() - .setDepend(whiteboard.getDepend(depend)) - .build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + JobOnFrame depend = new JobOnFrame(job, + jobManager.getFrameDetail(request.getFrame().getId())); + dependManager.createDepend(depend); + responseObserver.onNext(JobCreateDependencyOnFrameResponse.newBuilder() + .setDepend(whiteboard.getDepend(depend)) + .build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void createDependencyOnJob(JobCreateDependencyOnJobRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - JobOnJob depend = new JobOnJob(job, - jobManager.getJobDetail(request.getOnJob().getId())); - dependManager.createDepend(depend); - responseObserver.onNext(JobCreateDependencyOnJobResponse.newBuilder() - .setDepend(whiteboard.getDepend(depend)) - .build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + JobOnJob depend = new JobOnJob(job, + jobManager.getJobDetail(request.getOnJob().getId())); + dependManager.createDepend(depend); + responseObserver.onNext(JobCreateDependencyOnJobResponse.newBuilder() + .setDepend(whiteboard.getDepend(depend)) + .build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void createDependencyOnLayer(JobCreateDependencyOnLayerRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - JobOnLayer depend = new JobOnLayer(job, - jobManager.getLayerDetail(request.getLayer().getId())); - dependManager.createDepend(depend); - responseObserver.onNext(JobCreateDependencyOnLayerResponse.newBuilder() - .setDepend(whiteboard.getDepend(depend)) - .build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + JobOnLayer depend = new JobOnLayer(job, + jobManager.getLayerDetail(request.getLayer().getId())); + dependManager.createDepend(depend); + responseObserver.onNext(JobCreateDependencyOnLayerResponse.newBuilder() + .setDepend(whiteboard.getDepend(depend)) + .build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void getWhatDependsOnThis(JobGetWhatDependsOnThisRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - responseObserver.onNext(JobGetWhatDependsOnThisResponse.newBuilder() - .setDepends(whiteboard.getWhatDependsOnThis(job)) - .build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + responseObserver.onNext(JobGetWhatDependsOnThisResponse.newBuilder() + .setDepends(whiteboard.getWhatDependsOnThis(job)) + .build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void getWhatThisDependsOn(JobGetWhatThisDependsOnRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - responseObserver.onNext(JobGetWhatThisDependsOnResponse.newBuilder() - .setDepends(whiteboard.getWhatThisDependsOn(job)) - .build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + responseObserver.onNext(JobGetWhatThisDependsOnResponse.newBuilder() + .setDepends(whiteboard.getWhatThisDependsOn(job)) + .build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void getDepends(JobGetDependsRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - responseObserver.onNext(JobGetDependsResponse.newBuilder() - .setDepends(whiteboard.getDepends(job)) - .build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + responseObserver.onNext(JobGetDependsResponse.newBuilder() + .setDepends(whiteboard.getDepends(job)) + .build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void getUpdatedFrames(JobGetUpdatedFramesRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - UpdatedFrameCheckResult result = whiteboard.getUpdatedFrames(job, - ServantUtil.convertLayerFilterList(request.getLayerFilter()), request.getLastCheck()); - responseObserver.onNext(JobGetUpdatedFramesResponse.newBuilder() - .setUpdatedFrames(result.getUpdatedFrames()) - .setServerTime(result.getServerTime()) - .setState(result.getState()) - .build()); - responseObserver.onCompleted(); + try{ + setupJobData(request.getJob()); + UpdatedFrameCheckResult result = whiteboard.getUpdatedFrames(job, + ServantUtil.convertLayerFilterList(request.getLayerFilter()), request.getLastCheck()); + responseObserver.onNext(JobGetUpdatedFramesResponse.newBuilder() + .setUpdatedFrames(result.getUpdatedFrames()) + .setServerTime(result.getServerTime()) + .setState(result.getState()) + .build()); + responseObserver.onCompleted(); + + } catch (java.lang.IllegalArgumentException e) { + System.out.println(e); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void setMaxRetries(JobSetMaxRetriesRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - jobDao.updateMaxFrameRetries(job, request.getMaxRetries()); - responseObserver.onNext(JobSetMaxRetriesResponse.newBuilder().build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + jobDao.updateMaxFrameRetries(job, request.getMaxRetries()); + responseObserver.onNext(JobSetMaxRetriesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void addComment(JobAddCommentRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - Comment newComment = request.getNewComment(); - CommentDetail c = new CommentDetail(); - c.message = newComment.getMessage(); - c.subject = newComment.getSubject(); - c.user = newComment.getUser(); - c.timestamp = null; - commentManager.addComment(job, c); - responseObserver.onNext(JobAddCommentResponse.newBuilder().build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + Comment newComment = request.getNewComment(); + CommentDetail c = new CommentDetail(); + c.message = newComment.getMessage(); + c.subject = newComment.getSubject(); + c.user = newComment.getUser(); + c.timestamp = null; + commentManager.addComment(job, c); + responseObserver.onNext(JobAddCommentResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void getComments(JobGetCommentsRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - responseObserver.onNext(JobGetCommentsResponse.newBuilder() - .setComments(whiteboard.getComments(job)) - .build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + responseObserver.onNext(JobGetCommentsResponse.newBuilder() + .setComments(whiteboard.getComments(job)) + .build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void dropDepends(JobDropDependsRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - manageQueue.execute(new DispatchDropDepends(job, request.getTarget(), dependManager)); - responseObserver.onNext(JobDropDependsResponse.newBuilder().build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + manageQueue.execute(new DispatchDropDepends(job, request.getTarget(), dependManager)); + responseObserver.onNext(JobDropDependsResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void setGroup(JobSetGroupRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - jobDao.updateParent(job, groupManager.getGroupDetail(request.getGroupId())); - responseObserver.onNext(JobSetGroupResponse.newBuilder().build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + jobDao.updateParent(job, groupManager.getGroupDetail(request.getGroupId())); + responseObserver.onNext(JobSetGroupResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void markAsWaiting(JobMarkAsWaitingRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - jobManagerSupport.markFramesAsWaiting( - frameSearchFactory.create(job, request.getReq()), new Source(request.toString())); - responseObserver.onNext(JobMarkAsWaitingResponse.newBuilder().build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + jobManagerSupport.markFramesAsWaiting( + frameSearchFactory.create(job, request.getReq()), new Source(request.toString())); + responseObserver.onNext(JobMarkAsWaitingResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void reorderFrames(JobReorderFramesRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - manageQueue.execute(new DispatchReorderFrames(job, - new FrameSet(request.getRange()), request.getOrder(), jobManagerSupport)); - responseObserver.onNext(JobReorderFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + manageQueue.execute(new DispatchReorderFrames(job, + new FrameSet(request.getRange()), request.getOrder(), jobManagerSupport)); + responseObserver.onNext(JobReorderFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void staggerFrames(JobStaggerFramesRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - manageQueue.execute( - new DispatchStaggerFrames(job, request.getRange(), request.getStagger(), jobManagerSupport)); - responseObserver.onNext(JobStaggerFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + manageQueue.execute( + new DispatchStaggerFrames(job, request.getRange(), request.getStagger(), jobManagerSupport)); + responseObserver.onNext(JobStaggerFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } @Override public void addRenderPartition(JobAddRenderPartRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - LocalHostAssignment lha = new LocalHostAssignment(); - lha.setJobId(job.getId()); - lha.setThreads(request.getThreads()); - lha.setMaxCoreUnits(request.getMaxCores() * 100); - lha.setMaxMemory(request.getMaxMemory()); - lha.setMaxGpu(request.getMaxGpu()); - lha.setType(RenderPartitionType.JOB_PARTITION); - - if (localBookingSupport.bookLocal(job, request.getHost(), request.getUsername(), lha)) { - try { - RenderPartition renderPart = whiteboard.getRenderPartition(lha); - responseObserver.onNext(JobAddRenderPartResponse.newBuilder() - .setRenderPartition(renderPart) - .build()); - responseObserver.onCompleted(); - } catch (EmptyResultDataAccessException e) { + try { + setupJobData(request.getJob()); + LocalHostAssignment lha = new LocalHostAssignment(); + lha.setJobId(job.getId()); + lha.setThreads(request.getThreads()); + lha.setMaxCoreUnits(request.getMaxCores() * 100); + lha.setMaxMemory(request.getMaxMemory()); + lha.setMaxGpu(request.getMaxGpu()); + lha.setType(RenderPartitionType.JOB_PARTITION); + + if (localBookingSupport.bookLocal(job, request.getHost(), request.getUsername(), lha)) { + try { + RenderPartition renderPart = whiteboard.getRenderPartition(lha); + responseObserver.onNext(JobAddRenderPartResponse.newBuilder() + .setRenderPartition(renderPart) + .build()); + responseObserver.onCompleted(); + } catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to allocate render partition to host.") + .asRuntimeException()); + } + } else { responseObserver.onError(Status.INTERNAL - .withDescription("Failed to allocate render partition to host.") + .withDescription("Failed to find suitable frames.") .asRuntimeException()); } - } else { + } + catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL - .withDescription("Failed to find suitable frames.") + .withDescription("Failed to find job data") .asRuntimeException()); } } @Override public void runFilters(JobRunFiltersRequest request, StreamObserver responseObserver) { - setupJobData(request.getJob()); - JobDetail jobDetail = jobManager.getJobDetail(job.getJobId()); - filterManager.runFiltersOnJob(jobDetail); - responseObserver.onNext(JobRunFiltersResponse.newBuilder().build()); - responseObserver.onCompleted(); + try { + setupJobData(request.getJob()); + JobDetail jobDetail = jobManager.getJobDetail(job.getJobId()); + filterManager.runFiltersOnJob(jobDetail); + responseObserver.onNext(JobRunFiltersResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } } public JobManager getJobManager() { From 97cc88c046c2add6b88ae49f5d84d2a139809126 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Thu, 17 Dec 2020 16:33:55 -0800 Subject: [PATCH 020/277] Fix bugs in Redirect wrapper. (#860) --- cuegui/cuegui/Redirect.py | 41 +++++++++++++++++----------------- pycue/opencue/wrappers/host.py | 12 ++++++++++ 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/cuegui/cuegui/Redirect.py b/cuegui/cuegui/Redirect.py index a6db64980..88c6f1738 100644 --- a/cuegui/cuegui/Redirect.py +++ b/cuegui/cuegui/Redirect.py @@ -494,11 +494,11 @@ def __is_burst_safe(self, alloc, procs, show): show_obj = opencue.api.findShow(show) show_subs = dict((s.data.name.rstrip('.%s' % show), s) for s in show_obj.getSubscriptions() - if s.data.allocationName in alloc) + if s.data.allocation_name in alloc) try: procs_to_burst = (show_subs.get(alloc).data.burst - - show_subs.get(alloc).data.reservedCores) - procs_to_redirect = int(sum([p.data.reservedCores + show_subs.get(alloc).data.reserved_cores) + procs_to_redirect = int(sum([p.data.reserved_cores for p in procs])) wasted_cores = int(procs_to_redirect - procs_to_burst) if wasted_cores <= wc_ok: @@ -571,6 +571,7 @@ def redirect(self): host = entry["host"] host.redirectToJob(procs, job) except Exception as e: + print(e) errors.append(str(e)) item.setIcon(QtGui.QIcon(QtGui.QPixmap(":retry.png"))) item.setEnabled(False) @@ -605,7 +606,7 @@ def update(self): show = self.__controls.getShow() alloc = self.__controls.getAllocFilter() - procs = opencue.api.getProcs(show=show.data.name, alloc=alloc.getSelected()) + procs = opencue.api.getProcs(show=[str(show.data.name)], alloc=alloc.getSelected()) progress = QtWidgets.QProgressDialog("Searching","Cancel", 0, self.__controls.getLimit(), self) @@ -616,21 +617,21 @@ def update(self): break # Stick with the target show - if proc.data.showName != show.data.name: + if proc.data.show_name != show.data.name: continue - if proc.data.jobName == str(self.__controls.getJob()): + if proc.data.job_name == str(self.__controls.getJob()): continue # Skip over already redirected procs - if proc.data.redirectTarget: + if proc.data.redirect_target: continue if ok >= self.__controls.getLimit(): break if job_regex: - if re.match(job_regex, proc.data.jobName): + if re.match(job_regex, proc.data.job_name): continue if service_filter: @@ -638,7 +639,7 @@ def update(self): continue if group_filter: - if proc.data.groupName not in group_filter: + if proc.data.group_name not in group_filter: continue name = proc.data.name.split("/")[0] @@ -647,20 +648,20 @@ def update(self): hosts[name] = { "host": cue_host, "procs":[], - "mem": cue_host.data.idleMemory, - "cores": int(cue_host.data.idleCores), + "mem": cue_host.data.idle_memory, + "cores": int(cue_host.data.idle_cores), "time": 0, "ok": False, - 'alloc': cue_host.data.allocName} + 'alloc': cue_host.data.alloc_name} host = hosts[name] if host["ok"]: continue host["procs"].append(proc) - host["mem"] = host["mem"] + proc.data.reservedMemory - host["cores"] = host["cores"] + proc.data.reservedCores - host["time"] = host["time"] + (int(time.time()) - proc.data.dispatchTime); + host["mem"] = host["mem"] + proc.data.reserved_memory + host["cores"] = host["cores"] + proc.data.reserved_cores + host["time"] = host["time"] + (int(time.time()) - proc.data.dispatch_time); if host["cores"] >= self.__controls.getCores() and \ host["mem"] >= self.__controls.getMemory() and \ @@ -689,12 +690,12 @@ def __addHost(self, entry): QtGui.QStandardItem(cuegui.Utils.secondsToHHMMSS(rtime))]) for proc in procs: - checkbox.appendRow([QtGui.QStandardItem(proc.data.jobName), - QtGui.QStandardItem(str(proc.data.reservedCores)), - QtGui.QStandardItem("%0.2fGB" % (proc.data.reservedMemory / 1048576.0)), + checkbox.appendRow([QtGui.QStandardItem(proc.data.job_name), + QtGui.QStandardItem(str(proc.data.reserved_cores)), + QtGui.QStandardItem("%0.2fGB" % (proc.data.reserved_memory / 1048576.0)), QtGui.QStandardItem(cuegui.Utils.secondsToHHMMSS(time.time() - - proc.data.dispatchTime)), - QtGui.QStandardItem(proc.data.groupName), + proc.data.dispatch_time)), + QtGui.QStandardItem(proc.data.group_name), QtGui.QStandardItem(",".join(proc.data.services))]) self.__tree.setExpanded(self.__model.indexFromItem(checkbox), True) diff --git a/pycue/opencue/wrappers/host.py b/pycue/opencue/wrappers/host.py index 8a98222c0..50128f35a 100644 --- a/pycue/opencue/wrappers/host.py +++ b/pycue/opencue/wrappers/host.py @@ -88,6 +88,18 @@ def getProcs(self): timeout=Cuebot.Timeout) return [opencue.wrappers.proc.Proc(p) for p in response.procs.procs] + def redirectToJob(self, procs, job): + """Unbooks and redirects the proc to the specified job. Optionally + kills the proc immediately. + + :param procs: list + :param job: job id + """ + self.stub.RedirectToJob( + host_pb2.HostRedirectToJobRequest(host=self.data, + proc_names=[proc.data.id for proc in procs], + job_id=job.data.id), timeout=Cuebot.Timeout) + def getRenderPartitions(self): """Returns a list of render partitions associated with this host From 078978abcb1ac8acacf9165ab3ec0920ffc4ee0d Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Thu, 17 Dec 2020 16:34:30 -0800 Subject: [PATCH 021/277] Introduce OVERRIDE_HOSTNAME in RQD for manually settings a hostname. (#849) --- rqd/rqd/rqconstants.py | 3 +++ rqd/rqd/rqutil.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/rqd/rqd/rqconstants.py b/rqd/rqd/rqconstants.py index ed2c2f361..1860041bb 100644 --- a/rqd/rqd/rqconstants.py +++ b/rqd/rqd/rqconstants.py @@ -114,6 +114,7 @@ OVERRIDE_PROCS = None # number of physical cpus. ex: None or 2 OVERRIDE_MEMORY = None # in Kb OVERRIDE_NIMBY = None # True to turn on, False to turn off +OVERRIDE_HOSTNAME = None # Force to use this hostname ALLOW_GPU = False ALLOW_PLAYBLAST = False LOAD_MODIFIER = 0 # amount to add/subtract from load @@ -175,6 +176,8 @@ CUEBOT_HOSTNAME = config.get(__section, "OVERRIDE_CUEBOT") if config.has_option(__section, "OVERRIDE_NIMBY"): OVERRIDE_NIMBY = config.getboolean(__section, "OVERRIDE_NIMBY") + if config.has_option(__section, "OVERRIDE_HOSTNAME"): + OVERRIDE_HOSTNAME = config.get(__section, "OVERRIDE_HOSTNAME") if config.has_option(__section, "GPU"): ALLOW_GPU = config.getboolean(__section, "GPU") if config.has_option(__section, "PLAYBLAST"): diff --git a/rqd/rqd/rqutil.py b/rqd/rqd/rqutil.py index aa84e5eb9..6bdf3e5ba 100644 --- a/rqd/rqd/rqutil.py +++ b/rqd/rqd/rqutil.py @@ -149,7 +149,9 @@ def getHostIp(): def getHostname(): """Returns the machine's fully qualified domain name""" try: - if rqd.rqconstants.RQD_USE_IP_AS_HOSTNAME: + if rqd.rqconstants.OVERRIDE_HOSTNAME: + return rqd.rqconstants.OVERRIDE_HOSTNAME + elif rqd.rqconstants.RQD_USE_IP_AS_HOSTNAME: return getHostIp() else: return socket.gethostbyaddr(socket.gethostname())[0].split('.')[0] From 6da325b3aaee18d2fa1dba04f41d88c0d9e401e5 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 11 Jan 2021 18:55:56 +0000 Subject: [PATCH 022/277] Make explicit casts to long. (#869) --- .../imageworks/spcue/dao/oracle/WhiteboardDaoJdbc.java | 5 +++-- .../spcue/dao/postgres/WhiteboardDaoJdbc.java | 5 +++-- .../main/java/com/imageworks/spcue/util/CueUtil.java | 10 +++++----- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/WhiteboardDaoJdbc.java index 965a91406..97ddcb927 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/WhiteboardDaoJdbc.java @@ -1214,7 +1214,8 @@ public static JobStats mapJobStats(ResultSet rs) throws SQLException { (int) (rs.getLong("int_clock_time_success") / statsBuilder.getRenderedFrameCount())); statsBuilder.setAvgCoreSec( (int) (statsBuilder.getRenderedCoreSec() / statsBuilder.getRenderedFrameCount())); - statsBuilder.setRemainingCoreSec(statsBuilder.getPendingFrames() * statsBuilder.getAvgCoreSec()); + statsBuilder.setRemainingCoreSec( + (long) statsBuilder.getPendingFrames() * statsBuilder.getAvgCoreSec()); } else { statsBuilder.setAvgFrameSec(0); @@ -1274,7 +1275,7 @@ public Layer mapRow(ResultSet rs, int rowNum) throws SQLException { statsBuilder.setAvgCoreSec( (int) (statsBuilder.getRenderedCoreSec() / statsBuilder.getRenderedFrameCount())); statsBuilder.setRemainingCoreSec( - statsBuilder.getPendingFrames() * statsBuilder.getAvgCoreSec()); + (long) statsBuilder.getPendingFrames() * statsBuilder.getAvgCoreSec()); } else { statsBuilder.setAvgFrameSec(0); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index ad859b050..71ab45af1 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -1214,7 +1214,8 @@ public static JobStats mapJobStats(ResultSet rs) throws SQLException { (int) (rs.getLong("int_clock_time_success") / statsBuilder.getRenderedFrameCount())); statsBuilder.setAvgCoreSec( (int) (statsBuilder.getRenderedCoreSec() / statsBuilder.getRenderedFrameCount())); - statsBuilder.setRemainingCoreSec(statsBuilder.getPendingFrames() * statsBuilder.getAvgCoreSec()); + statsBuilder.setRemainingCoreSec( + (long) statsBuilder.getPendingFrames() * statsBuilder.getAvgCoreSec()); } else { statsBuilder.setAvgFrameSec(0); @@ -1274,7 +1275,7 @@ public Layer mapRow(ResultSet rs, int rowNum) throws SQLException { statsBuilder.setAvgCoreSec( (int) (statsBuilder.getRenderedCoreSec() / statsBuilder.getRenderedFrameCount())); statsBuilder.setRemainingCoreSec( - statsBuilder.getPendingFrames() * statsBuilder.getAvgCoreSec()); + (long) statsBuilder.getPendingFrames() * statsBuilder.getAvgCoreSec()); } else { statsBuilder.setAvgFrameSec(0); diff --git a/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java b/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java index be1f17d91..a7d89e7ee 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java +++ b/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java @@ -65,11 +65,11 @@ public final class CueUtil { public static final long MB256 = 262144; public static final long MB512 = 524288; public static final long GB = 1048576; - public static final long GB2 = 1048576 * 2; - public static final long GB4 = 1048576 * 4; - public static final long GB8 = 1048576 * 8; - public static final long GB16 = 1048576 * 16; - public static final long GB32 = 1048576 * 32; + public static final long GB2 = 1048576L * 2; + public static final long GB4 = 1048576L * 4; + public static final long GB8 = 1048576L * 8; + public static final long GB16 = 1048576L * 16; + public static final long GB32 = 1048576L * 32; /** * Features that relay on an integer greated than 0 to work From 3133eb18dfb267ba77fa083a76a345b43b324cdf Mon Sep 17 00:00:00 2001 From: Lars van der Bijl <285658+larsbijl@users.noreply.github.com> Date: Sat, 16 Jan 2021 11:45:11 +0000 Subject: [PATCH 023/277] Example of exporting cuebot metric's (#844) Export cuebot metric's into a prometheus format. Closes #843 --- connectors/prometheus_metrics/Dockerfile | 52 +++++ connectors/prometheus_metrics/metrics | 182 ++++++++++++++++++ .../requirements_metrics.txt | 1 + sandbox/docker-compose.yml | 13 ++ 4 files changed, 248 insertions(+) create mode 100644 connectors/prometheus_metrics/Dockerfile create mode 100755 connectors/prometheus_metrics/metrics create mode 100644 connectors/prometheus_metrics/requirements_metrics.txt diff --git a/connectors/prometheus_metrics/Dockerfile b/connectors/prometheus_metrics/Dockerfile new file mode 100644 index 000000000..47d697226 --- /dev/null +++ b/connectors/prometheus_metrics/Dockerfile @@ -0,0 +1,52 @@ +FROM centos:7 +ENV PYTHONUNBUFFERED 1 + +WORKDIR /opt/opencue + +RUN yum -y install \ + epel-release \ + gcc \ + python-devel \ + time + +RUN yum -y install \ + python-pip \ + python36 \ + python36-devel \ + python36-pip + +RUN python -m pip install --upgrade pip +RUN python3.6 -m pip install --upgrade pip + +RUN python -m pip install --upgrade setuptools +RUN python3.6 -m pip install --upgrade setuptools + +COPY LICENSE ./ +COPY requirements.txt ./ +COPY connectors/prometheus_metrics/requirements_metrics.txt ./ + +RUN python -m pip install -r requirements.txt -r requirements_metrics.txt +RUN python3.6 -m pip install -r requirements.txt -r requirements_metrics.txt + +COPY connectors/prometheus_metrics/metrics ./metrics +COPY proto/ ./proto +COPY pycue/README.md ./pycue/ +COPY pycue/setup.py ./pycue/ +COPY pycue/FileSequence ./pycue/FileSequence +COPY pycue/opencue ./pycue/opencue + +RUN python -m grpc_tools.protoc \ + -I=./proto \ + --python_out=./pycue/opencue/compiled_proto \ + --grpc_python_out=./pycue/opencue/compiled_proto \ + ./proto/*.proto + +# Fix imports to work in both Python 2 and 3. See +# for more info. +RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py + +RUN cd pycue && python setup.py install + +RUN cd pycue && python3.6 setup.py install + +ENTRYPOINT ["python3", "/opt/opencue/metrics"] diff --git a/connectors/prometheus_metrics/metrics b/connectors/prometheus_metrics/metrics new file mode 100755 index 000000000..960132938 --- /dev/null +++ b/connectors/prometheus_metrics/metrics @@ -0,0 +1,182 @@ +#!/usr/bin/env python +import time + +import opencue + +from prometheus_client import start_http_server +from prometheus_client import Gauge + + +CLUE_HOSTS_HARDWARE = Gauge( + 'cue_hosts_hardware_total', 'hosts hardware status', ['status']) +CLUE_HOSTS_LOCK = Gauge('cue_hosts_lock_total', 'hosts lock status', ['status']) +CLUE_PROCS = Gauge('cue_procs_total', 'number of Procs') +CLUE_PROCS_USABLE = Gauge('cue_procs_usable_total', 'number of usable Procs') +CLUE_PROCS_USED = Gauge('cue_procs_used_total', 'number of Procs currently in use') + +CLUE_FRAMES = Gauge('cue_frames', 'number of frames ', ['status', 'show']) +CLUE_REMAIN = Gauge('cue_remain', 'remaining core seconds (estimated) ', ['show']) + +MANAGE_WAITING = Gauge('cue_manage_waiting_total', '') +MANAGE_REMAINING_CAPACITY = Gauge('cue_manage_remaining_capacity_total', '') +MANAGE_THREADS = Gauge('cue_manage_threads_total', '') +MANAGE_EXECUTED = Gauge('cue_manage_executed_total', '') +MANAGE_REJECTED = Gauge('cue_manage_rejected_total', '') +DISPATCH_WAITING = Gauge('cue_dispatch_waiting_total', '') +DISPATCH_REMAINING_CAPACITY = Gauge('cue_dispatch_remaining_capacity_total', '') +DISPATCH_THREADS = Gauge('cue_dispatch_threads_total', '') +DISPATCH_EXECUTED = Gauge('cue_dispatch_executed_total', '') +DISPATCH_REJECTED = Gauge('cue_dispatch_rejected_total', '') +REPORT_WAITING = Gauge('cue_report_waiting_total', '') +REPORT_REMAINING_CAPACITY = Gauge('cue_report_remaining_capacity_total', '') +REPORT_THREADS = Gauge('cue_report_threads_total', '') +REPORT_EXECUTED = Gauge('cue_report_executed_total', '') +REPORT_REJECTED = Gauge('cue_report_rejected_total', '') +BOOKING_WAITING = Gauge('cue_booking_waiting_total', '') +BOOKING_REMAINING_CAPACITY = Gauge('cue_booking_remaining_capacity_total', '') +BOOKING_THREADS = Gauge('cue_booking_threads_total', '') +BOOKING_SLEEP_MILLIS = Gauge('cue_booking_sleep_millis_total', '') +BOOKING_EXECUTED = Gauge('cue_booking_executed_total', '') +BOOKING_REJECTED = Gauge('cue_booking_rejected_total', '') +HOST_BALANCE_SUCCESS = Gauge('cue_host_balance_success_total', '') +HOST_BALANCE_FAILED = Gauge('cue_host_balance_failed_total', '') +KILLED_OFFENDER_PROCS = Gauge('cue_killed_offender_procs_total', '') +KILLED_OOM_PROCS = Gauge('cue_killed_oom_procs_total', '') +CLEARED_PROCS = Gauge('cue_cleared_procs_total', '') +BOOKING_ERRORS = Gauge('cue_booking_errors_total', '') +BOOKING_RETRIES = Gauge('cue_booking_retries_total', '') +BOOKED_PROCS = Gauge('cue_booked_procs_total', '') +REQ_FOR_DATA = Gauge('cue_req_for_data_total', '') +REQ_FOR_FUNCTION = Gauge('cue_req_for_function_total', '') +REQ_ERRORS = Gauge('cue_req_errors_total', '') +UNBOOKED_PROCS = Gauge('cue_unbooked_procs_total', '') +PICKED_UP_CORES = Gauge('cue_picked_up_cores_total', '') +STRANDED_CORES = Gauge('cue_stranded_cores_total', '') + + +def main(): + while True: + jobs = opencue.api.getJobs() + shows = {} + shows_remaining = {} + + for job in jobs: + show = job.show() + if show not in shows: + shows[show] = {'pending': 0, 'dead': 0, + 'eaten': 0, 'succeeded': 0, 'running': 0} + + if show not in shows_remaining: + shows_remaining[show] = 0 + + shows[show]['pending'] += job.pendingFrames() + shows[show]['dead'] += job.deadFrames() + shows[show]['eaten'] += job.eatenFrames() + shows[show]['succeeded'] += job.succeededFrames() + shows[show]['running'] += job.runningFrames() + + shows_remaining[show] += job.coreSecondsRemaining() + + for show in shows: + for k, v in shows[show].items(): + CLUE_FRAMES.labels(status=k, show=show).set(v) + + for show in shows_remaining: + CLUE_REMAIN.labels(show=show).set(shows_remaining[show]) + + # Handle the Host information + hosts = opencue.api.getHosts() + down_hosts = up_hosts = 0 + open_hosts = locked_hosts = nimby_locked_hosts = 0 + repair_hosts = rebooting_hosts = reboot_when_idle_hosts = shutdown_when_idle_hosts = 0 + total_procs = used_procs = usable_procs = 0 + + for host in hosts: + lstate = host.lockState() + if lstate == 0: + open_hosts += 1 + elif lstate == 1: + locked_hosts += 1 + elif lstate == 2: + nimby_locked_hosts += 1 + + state = host.state() + if state == 5: + repair_hosts += 1 + elif state == 4: + shutdown_when_idle_hosts += 1 + elif state == 3: + reboot_when_idle_hosts += 1 + elif state == 2: + rebooting_hosts += 1 + + if host.isUp(): + up_hosts += 1 + if not host.isLocked(): + usable_procs += host.cores() + used_procs += host.cores() - host.coresIdle() + else: + down_hosts += 1 + + total_procs += host.cores() + + CLUE_HOSTS_LOCK.labels(status='open').set(open_hosts) + CLUE_HOSTS_LOCK.labels(status='locked').set(locked_hosts) + CLUE_HOSTS_LOCK.labels(status='nimby_locked').set(nimby_locked_hosts) + + CLUE_HOSTS_HARDWARE.labels(status='up').set(up_hosts) + CLUE_HOSTS_HARDWARE.labels(status='down').set(down_hosts) + CLUE_HOSTS_HARDWARE.labels(status='repair').set(repair_hosts) + CLUE_HOSTS_HARDWARE.labels(status='rebooting').set(rebooting_hosts) + CLUE_HOSTS_HARDWARE.labels(status='reboot_when_idle').set(reboot_when_idle_hosts) + CLUE_HOSTS_HARDWARE.labels(status='shutdown_when_idle').set(shutdown_when_idle_hosts) + + CLUE_PROCS_USABLE.set(usable_procs) + CLUE_PROCS_USED.set(used_procs) + CLUE_PROCS.set(total_procs) + + # Apply the scheduler system stats. + system_stats = opencue.api.getSystemStats() + + MANAGE_WAITING.set(system_stats.manage_waiting) + MANAGE_REMAINING_CAPACITY.set(system_stats.manage_remaining_capacity) + MANAGE_THREADS.set(system_stats.manage_threads) + MANAGE_EXECUTED.set(system_stats.manage_executed) + MANAGE_REJECTED.set(system_stats.manage_rejected) + DISPATCH_WAITING.set(system_stats.dispatch_waiting) + DISPATCH_REMAINING_CAPACITY.set(system_stats.dispatch_remaining_capacity) + DISPATCH_THREADS.set(system_stats.dispatch_threads) + DISPATCH_EXECUTED.set(system_stats.dispatch_executed) + DISPATCH_REJECTED.set(system_stats.dispatch_rejected) + REPORT_WAITING.set(system_stats.report_waiting) + REPORT_REMAINING_CAPACITY.set(system_stats.report_remaining_capacity) + REPORT_THREADS.set(system_stats.report_threads) + REPORT_EXECUTED.set(system_stats.report_executed) + REPORT_REJECTED.set(system_stats.report_rejected) + BOOKING_WAITING.set(system_stats.booking_waiting) + BOOKING_REMAINING_CAPACITY.set(system_stats.booking_remaining_capacity) + BOOKING_THREADS.set(system_stats.booking_threads) + BOOKING_SLEEP_MILLIS.set(system_stats.booking_sleep_millis) + BOOKING_EXECUTED.set(system_stats.booking_executed) + BOOKING_REJECTED.set(system_stats.booking_rejected) + HOST_BALANCE_SUCCESS.set(system_stats.host_balance_success) + HOST_BALANCE_FAILED.set(system_stats.host_balance_failed) + KILLED_OFFENDER_PROCS.set(system_stats.killed_offender_procs) + KILLED_OOM_PROCS.set(system_stats.killed_oom_procs) + CLEARED_PROCS.set(system_stats.cleared_procs) + BOOKING_ERRORS.set(system_stats.booking_errors) + BOOKING_RETRIES.set(system_stats.booking_retries) + BOOKED_PROCS.set(system_stats.booked_procs) + REQ_FOR_DATA.set(system_stats.req_for_data) + REQ_FOR_FUNCTION.set(system_stats.req_for_function) + REQ_ERRORS.set(system_stats.req_errors) + UNBOOKED_PROCS.set(system_stats.unbooked_procs) + PICKED_UP_CORES.set(system_stats.picked_up_cores) + STRANDED_CORES.set(system_stats.stranded_cores) + + time.sleep(30) + + +if __name__ == '__main__': + start_http_server(8302) + main() diff --git a/connectors/prometheus_metrics/requirements_metrics.txt b/connectors/prometheus_metrics/requirements_metrics.txt new file mode 100644 index 000000000..7db4e22e9 --- /dev/null +++ b/connectors/prometheus_metrics/requirements_metrics.txt @@ -0,0 +1 @@ +prometheus-client==0.9.0 \ No newline at end of file diff --git a/sandbox/docker-compose.yml b/sandbox/docker-compose.yml index 0b2e120ac..f4fde39ec 100644 --- a/sandbox/docker-compose.yml +++ b/sandbox/docker-compose.yml @@ -56,3 +56,16 @@ services: volumes: - /tmp/rqd/logs:/tmp/rqd/logs - /tmp/rqd/shots:/tmp/rqd/shots + + metrics: + build: + context: ./ + dockerfile: ./connectors/prometheus_metrics/Dockerfile + environment: + - CUEBOT_HOSTS=cuebot + depends_on: + - cuebot + links: + - cuebot + ports: + - "8302:8302" \ No newline at end of file From abd902fd5b44d113e6b48ac7e92c106734842a8b Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 19 Jan 2021 21:06:26 +0000 Subject: [PATCH 024/277] Reinterrupt current thread after InterruptedException. (#870) --- .../main/java/com/imageworks/spcue/dispatcher/BookingQueue.java | 1 + .../main/java/com/imageworks/spcue/dispatcher/DispatchQueue.java | 1 + .../java/com/imageworks/spcue/dispatcher/HostReportQueue.java | 1 + .../java/com/imageworks/spcue/service/JobManagerSupport.java | 1 + 4 files changed, 4 insertions(+) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java index cd6f457d0..ace595ea2 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java @@ -93,6 +93,7 @@ protected void beforeExecute(Thread t, Runnable r) { Thread.sleep(sleepTime()); } catch (InterruptedException e) { logger.info("booking queue was interrupted."); + Thread.currentThread().interrupt(); } } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchQueue.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchQueue.java index d084704ea..81366dedc 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchQueue.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchQueue.java @@ -105,6 +105,7 @@ public void shutdown() { } Thread.sleep(250); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); break; } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportQueue.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportQueue.java index 9d0b36b91..34e8d13e6 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportQueue.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportQueue.java @@ -73,6 +73,7 @@ public void shutdown() { } Thread.sleep(250); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); break; } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerSupport.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerSupport.java index 23a013d5a..e6c03c221 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerSupport.java @@ -100,6 +100,7 @@ public boolean shutdownJob(JobInterface job, Source source, boolean isManualKill } catch (InterruptedException e1) { logger.info(job.getName() + "/" + job.getId() + " shutdown thread was interrupted."); + Thread.currentThread().interrupt(); } FrameSearchInterface search = frameSearchFactory.create(job); From 25698f6d93dc78d0870ae0fdf9e6be9418a041fb Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 19 Jan 2021 21:06:44 +0000 Subject: [PATCH 025/277] Remove Transactional annotations from private methods. (#871) --- .../com/imageworks/spcue/service/DependManagerService.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/DependManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/DependManagerService.java index b40e68801..187e81331 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/DependManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/DependManagerService.java @@ -495,7 +495,6 @@ public void createDepend(BuildableDependency depend) { } } - @Transactional(propagation=Propagation.SUPPORTS) private void updateDependCount(LayerInterface layer) { FrameSearchInterface r = frameSearchFactory.create(layer); for (FrameInterface f: frameDao.findFrames(r)) { @@ -503,7 +502,6 @@ private void updateDependCount(LayerInterface layer) { } } - @Transactional(propagation=Propagation.SUPPORTS) private void updateDependCount(JobInterface job) { FrameSearchInterface r = frameSearchFactory.create(job); for (FrameInterface f: frameDao.findFrames(r)) { @@ -511,7 +509,6 @@ private void updateDependCount(JobInterface job) { } } - @Transactional(propagation = Propagation.SUPPORTS) private void updateDependCounts(FrameInterface f) { dependDao.incrementDependCount(f); } From d4d4c225dc295f45cb3f9198fe0d5f3a5816aa92 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 19 Jan 2021 21:07:01 +0000 Subject: [PATCH 026/277] Fix another few potential NullPointerExceptions. (#872) --- .../dispatcher/commands/DispatchBookHost.java | 16 +++++++++------- .../spcue/service/RedirectService.java | 2 +- .../spcue/servlet/JobLaunchServlet.java | 3 ++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java index bd9147075..c1ec337a2 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java @@ -103,13 +103,15 @@ public int hashCode() { @Override public boolean equals(Object other) { - try { - DispatchBookHost that = (DispatchBookHost) other; - return that.host.name.equals(host.name); - } - catch (ClassCastException e) { - return false; - } + if (other == null) { + return false; + } + try { + DispatchBookHost that = (DispatchBookHost) other; + return that.host.name.equals(host.name); + } catch (ClassCastException e) { + return false; + } }; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/RedirectService.java b/cuebot/src/main/java/com/imageworks/spcue/service/RedirectService.java index 1da7cbc97..23f164c53 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/RedirectService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/RedirectService.java @@ -105,7 +105,7 @@ public void put(String key, Redirect r) { continue; } catch (DuplicateKeyException e) { - if (e.getMessage().contains("C_REDIRECT_PK")) { + if (e.getMessage() != null && e.getMessage().contains("C_REDIRECT_PK")) { // MERGE statement race lost; try again. txManager.rollback(status); continue; diff --git a/cuebot/src/main/java/com/imageworks/spcue/servlet/JobLaunchServlet.java b/cuebot/src/main/java/com/imageworks/spcue/servlet/JobLaunchServlet.java index f3dae875c..284b6739a 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servlet/JobLaunchServlet.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servlet/JobLaunchServlet.java @@ -20,6 +20,7 @@ package com.imageworks.spcue.servlet; import java.io.IOException; +import java.util.Objects; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -45,7 +46,7 @@ public class JobLaunchServlet extends FrameworkServlet { @Override public void initFrameworkServlet() throws ServletException { jobLauncher = (JobLauncher) - this.getWebApplicationContext().getBean("jobLauncher"); + Objects.requireNonNull(this.getWebApplicationContext()).getBean("jobLauncher"); } @Override From 50908af28cc7a44b00e558f73cf2d4f31dd123a6 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 19 Jan 2021 21:11:04 +0000 Subject: [PATCH 027/277] Fix username string comparison. (#874) --- cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java index 55aad9923..d309fa264 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java @@ -202,7 +202,7 @@ private void handleSpecTag() { uid = Optional.ofNullable(rootElement.getChildTextTrim("uid")).map(Integer::parseInt); email = rootElement.getChildTextTrim("email"); - if (user == "root" || uid.equals(Optional.of(0))) { + if (user.equals("root") || uid.equals(Optional.of(0))) { throw new SpecBuilderException("Cannot launch jobs as root."); } } From fa9b644fca74087e19b6b89a9b8628fcc7c9504e Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 19 Jan 2021 21:11:27 +0000 Subject: [PATCH 028/277] Added a few new PR checks to help ensure version compatibility. (#877) --- .github/workflows/testing-pipeline.yml | 30 ++++++++++++++ ci/check_changed_files.py | 37 ++++++++++++++++++ ci/check_database_migrations.py | 54 ++++++++++++++++++++++++++ ci/check_version_bump.py | 51 ++++++++++++++++++++++++ 4 files changed, 172 insertions(+) create mode 100755 ci/check_changed_files.py create mode 100755 ci/check_database_migrations.py create mode 100755 ci/check_version_bump.py diff --git a/.github/workflows/testing-pipeline.yml b/.github/workflows/testing-pipeline.yml index bb5d8526d..68072e56c 100644 --- a/.github/workflows/testing-pipeline.yml +++ b/.github/workflows/testing-pipeline.yml @@ -58,3 +58,33 @@ jobs: - uses: actions/checkout@v2 - name: Run Sphinx build run: ci/build_sphinx_docs.sh + + check_changed_files: + name: Check Changed Files + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Get Changed Files + id: get_changed_files + uses: jitterbit/get-changed-files@v1 + - name: Check for Version Change + run: ci/check_changed_files.py ${{ steps.get_changed_files.outputs.modified }} ${{ steps.get_changed_files.outputs.removed }} + + check_migration_files: + name: Check Database Migration Files + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Check Migration Files + run: ci/check_database_migrations.py + + check_for_version_bump: + name: Check for Version Bump + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Get Changed Files + id: get_changed_files + uses: jitterbit/get-changed-files@v1 + - name: Check for Version Change + run: ci/check_version_bump.py ${{ steps.get_changed_files.outputs.all }} diff --git a/ci/check_changed_files.py b/ci/check_changed_files.py new file mode 100755 index 000000000..5b676ae09 --- /dev/null +++ b/ci/check_changed_files.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +"""Script that checks if files were changed which should not be. + +This script is meant to be run in the context or GitHub Actions, and expects all changed files +to be passed via commandline, like: + + $ ci/check_changed_files.py file/that/was/changed.txt other/file/changed.py +""" + +import fnmatch +import sys + + +FILES_THAT_SHOULD_NOT_BE_CHANGED = [ + ('cuebot/src/main/resources/conf/ddl/postgres/migrations/*.sql', 'add a new migration file'), + ('cuebot/src/main/resources/public/dtd/*.dtd', 'add a new schema version'), +] + + +def main(): + changed_files = sys.argv[1:] + + violating_files = False + + for changed_file in changed_files: + for glob_to_check, suggestion in FILES_THAT_SHOULD_NOT_BE_CHANGED: + if fnmatch.fnmatch(changed_file, glob_to_check): + violating_files = True + print('File %s should not be modified. Suggestion: %s' % (changed_file, suggestion)) + + if violating_files: + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/ci/check_database_migrations.py b/ci/check_database_migrations.py new file mode 100755 index 000000000..ab6dcaf01 --- /dev/null +++ b/ci/check_database_migrations.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python + +"""Script that checks database migration files for validity. + +Ensures migration filenames are correct and that each database version has one and only one +file associated with it. +""" + +import glob +import os +import re +import sys + + +MIGRATIONS_DIR = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + 'cuebot/src/main/resources/conf/ddl/postgres/migrations') + +MIGRATION_FILENAME_REGEX = re.compile(r'^V(?P[\d]+)__.*\.sql$') + + +def main(): + if not os.path.exists(MIGRATIONS_DIR): + print('Migrations directory was not found, expected at %s' % MIGRATIONS_DIR) + sys.exit(1) + + version_numbers_seen = {} + + for migration_file in glob.glob(os.path.join(MIGRATIONS_DIR, '*.sql')): + version_match = MIGRATION_FILENAME_REGEX.match(os.path.basename(migration_file)) + if not version_match: + print( + 'Migration file %s did not match expected format, should be ' + '"V__.sql"') + sys.exit(1) + if version_match.group('version_number') not in version_numbers_seen: + version_numbers_seen[version_match.group('version_number')] = [] + version_numbers_seen[version_match.group('version_number')].append( + os.path.basename(migration_file)) + + violation_found = False + for version_number, filenames in list(version_numbers_seen.items()): + if len(filenames) > 1: + violation_found = True + print( + 'Migration version %s has multiple files associated. Conflicting files: %s' % ( + version_number, ', '.join(filenames))) + + if violation_found: + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/ci/check_version_bump.py b/ci/check_version_bump.py new file mode 100755 index 000000000..795df74c0 --- /dev/null +++ b/ci/check_version_bump.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +"""Script that checks if a version bump is needed, and whether it has been made. + +Checks are based on which files have been changed, indicating potential incompatibility +between versions. + +This script is meant to be run in the context or GitHub Actions, and expects all changed files +to be passed via commandline, like: + + $ ci/check_version_bump.py file/that/was/changed.txt other/file/changed.py +""" + +import fnmatch +import sys + + +FILES_CAUSING_INCOMPATIBILITY = [ + 'cuebot/src/main/resources/conf/ddl/postgres/migrations/*.sql', + 'cuebot/src/main/resources/public/dtd/*.dtd', + 'proto/*.proto', +] + +VERSION_FILE = 'VERSION.in' + + +def main(): + changed_files = sys.argv[1:] + + violating_files = [] + version_file_updated = False + + for changed_file in changed_files: + if changed_file == VERSION_FILE: + version_file_updated = True + + for glob_to_check in FILES_CAUSING_INCOMPATIBILITY: + if fnmatch.fnmatch(changed_file, glob_to_check): + violating_files.append(changed_file) + + if violating_files and not version_file_updated: + print( + 'Files were added or changed which indicate version incompatibility, but the OpenCue ' + 'minor version number was not updated.') + print('Please update VERSION.in and increment the minor version number.') + print('Violating files: \n %s' % '\n '.join(violating_files)) + sys.exit(1) + + +if __name__ == '__main__': + main() From a40e641218f723bb8f060b04433e1172eb39fb32 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 19 Jan 2021 22:13:06 +0000 Subject: [PATCH 029/277] Add type tests to equals() before casting. (#873) --- .../src/main/java/com/imageworks/spcue/SortableShow.java | 7 +++---- .../spcue/dispatcher/commands/DispatchBookHost.java | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/SortableShow.java b/cuebot/src/main/java/com/imageworks/spcue/SortableShow.java index 6e8e8c90c..855b95857 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/SortableShow.java +++ b/cuebot/src/main/java/com/imageworks/spcue/SortableShow.java @@ -107,12 +107,11 @@ public boolean equals(Object other) { if (other == null) { return false; } - try { - SortableShow that = (SortableShow) other; - return that.getShowId().equals(this.getShowId()); - } catch (ClassCastException e) { + if (this.getClass() != other.getClass()) { return false; } + SortableShow that = (SortableShow) other; + return that.getShowId().equals(this.getShowId()); } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java index c1ec337a2..6ddaa9a3b 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java @@ -106,12 +106,11 @@ public boolean equals(Object other) { if (other == null) { return false; } - try { - DispatchBookHost that = (DispatchBookHost) other; - return that.host.name.equals(host.name); - } catch (ClassCastException e) { + if (this.getClass() != other.getClass()) { return false; } + DispatchBookHost that = (DispatchBookHost) other; + return that.host.name.equals(host.name); }; } From bf128e42ffbe940e47dcecb98d571036cb88ee3e Mon Sep 17 00:00:00 2001 From: Lars van der Bijl <285658+larsbijl@users.noreply.github.com> Date: Thu, 21 Jan 2021 19:55:18 +0000 Subject: [PATCH 030/277] Add dockerignore so that the sandbox env can be run mutliple times. (#879) --- .dockerignore | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..567708f05 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +ci +images +samples +sandbox/db-data +tsc +venv \ No newline at end of file From 7341f8231873b370b5e84495cb236935e331d879 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Fri, 22 Jan 2021 14:48:43 -0800 Subject: [PATCH 031/277] Introduce RQD_TAGS for setting tags to RQD host. (#875) --- rqd/rqd/rqconstants.py | 3 +++ rqd/rqd/rqmachine.py | 4 ++++ rqd/tests/rqmachine_tests.py | 7 +++++++ 3 files changed, 14 insertions(+) diff --git a/rqd/rqd/rqconstants.py b/rqd/rqd/rqconstants.py index 1860041bb..9ffd10273 100644 --- a/rqd/rqd/rqconstants.py +++ b/rqd/rqd/rqconstants.py @@ -68,6 +68,7 @@ RQD_USE_IP_AS_HOSTNAME = True RQD_BECOME_JOB_USER = True RQD_CREATE_USER_IF_NOT_EXISTS = True +RQD_TAGS = '' KILL_SIGNAL = 9 if platform.system() == 'Linux': @@ -188,6 +189,8 @@ RQD_USE_IP_AS_HOSTNAME = config.getboolean(__section, "RQD_USE_IP_AS_HOSTNAME") if config.has_option(__section, "RQD_BECOME_JOB_USER"): RQD_BECOME_JOB_USER = config.getboolean(__section, "RQD_BECOME_JOB_USER") + if config.has_option(__section, "RQD_TAGS"): + RQD_TAGS = config.getint(__section, "RQD_TAGS") if config.has_option(__section, "DEFAULT_FACILITY"): DEFAULT_FACILITY = config.get(__section, "DEFAULT_FACILITY") if config.has_option(__section, "LAUNCH_FRAME_USER_GID"): diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 6df3a0809..dae6f3752 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -385,6 +385,10 @@ def __initMachineTags(self): """Sets the hosts tags""" self.__renderHost.tags.append("rqdv-%s" % rqd.rqconstants.VERSION) + if rqd.rqconstants.RQD_TAGS: + for tag in rqd.rqconstants.RQD_TAGS.split(): + self.__renderHost.tags.append(tag) + # Tag with desktop if it is a desktop if self.isDesktop(): self.__renderHost.tags.append("desktop") diff --git a/rqd/tests/rqmachine_tests.py b/rqd/tests/rqmachine_tests.py index 1710423b7..eadf63f54 100644 --- a/rqd/tests/rqmachine_tests.py +++ b/rqd/tests/rqmachine_tests.py @@ -430,6 +430,13 @@ def test_reserveHT(self): self.assertEqual({0, 1, 2, 3, 4, 5, 6, 7}, self.machine._Machine__tasksets) + def test_tags(self): + tags = ["test1", "test2", "test3"] + rqd.rqconstants.RQD_TAGS = " ".join(tags) + + machine = rqd.rqmachine.Machine(self.rqCore, self.coreDetail) + + self.assertTrue(all(tag in machine.__dict__['_Machine__renderHost'].tags for tag in tags)) class CpuinfoTests(unittest.TestCase): From 68f7c1c555ea7e875008fedeb3e08e0f9a132f10 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Fri, 22 Jan 2021 14:53:04 -0800 Subject: [PATCH 032/277] Propagate layer environment variables. (#876) --- pyoutline/outline/backend/cue.py | 5 +++++ pyoutline/outline/layer.py | 6 +++++- pyoutline/tests/json/shell.outline | 1 + pyoutline/tests/json_test.py | 19 ++++++++++++++++++- 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/pyoutline/outline/backend/cue.py b/pyoutline/outline/backend/cue.py index 0a6dfc7bd..f3a27e940 100644 --- a/pyoutline/outline/backend/cue.py +++ b/pyoutline/outline/backend/cue.py @@ -294,6 +294,11 @@ def _serialize(launcher, use_pycuerun): limit = Et.SubElement(limits, "limit") limit.text = limit_name + layer_env = Et.SubElement(spec_layer, "env") + for env_k, env_v in layer.get_envs().items(): + pair = Et.SubElement(layer_env, "key", {"name": "{}".format(env_k)}) + pair.text = env_v + services = Et.SubElement(spec_layer, "services") service = Et.SubElement(services, "service") try: diff --git a/pyoutline/outline/layer.py b/pyoutline/outline/layer.py index e7220053a..d3ce07dfd 100644 --- a/pyoutline/outline/layer.py +++ b/pyoutline/outline/layer.py @@ -304,7 +304,11 @@ def set_env(self, key, value): def get_env(self, key, default=None): """Get the value of the env var that will be set before execute.""" - self.__env.get(key, default) + return self.__env.get(key, default) + + def get_envs(self): + """Return all env.""" + return self.__env def get_name(self): """Return the layer name.""" diff --git a/pyoutline/tests/json/shell.outline b/pyoutline/tests/json/shell.outline index 5d6b4e2cb..7e0e859d1 100644 --- a/pyoutline/tests/json/shell.outline +++ b/pyoutline/tests/json/shell.outline @@ -5,6 +5,7 @@ { "name": "shell_layer", "module": "outline.modules.shell.Shell", + "env": {"LAYER_KEY": "LAYER_VALUE"}, "command": ["/bin/ls"] } ] diff --git a/pyoutline/tests/json_test.py b/pyoutline/tests/json_test.py index a9fa7fef4..6a576431f 100644 --- a/pyoutline/tests/json_test.py +++ b/pyoutline/tests/json_test.py @@ -22,7 +22,9 @@ import mock import os import unittest +from xml.etree import ElementTree as Et +import outline from outline import load_json from . import test_utils @@ -40,6 +42,7 @@ def testJson(self): '"layers": [{' '"name": "layer_1", ' '"module": "outline.modules.shell.Shell", ' + '"env": {"LAYER_KEY1": "LAYER_VALUE1"}, ' '"command": ["/bin/ls"]' '}]' '}') @@ -47,17 +50,31 @@ def testJson(self): ol = load_json(s) self.assertEqual('test_job', ol.get_name()) self.assertEqual('1-10', ol.get_frame_range()) + self.assertEqual('LAYER_VALUE1', ol.get_layer('layer_1').get_env('LAYER_KEY1')) + + ol.get_layer('layer_1').set_env('LAYER_KEY2', 'LAYER_VALUE2') + + l = outline.cuerun.OutlineLauncher(ol) + root = Et.fromstring(l.serialize()) + env1 = root.find('job/layers/layer/env/key[@name="LAYER_KEY1"]') + self.assertEqual('LAYER_VALUE1', env1.text) + env2 = root.find('job/layers/layer/env/key[@name="LAYER_KEY2"]') + self.assertEqual('LAYER_VALUE2', env2.text) @mock.patch('outline.layer.Layer.system') + @mock.patch.dict(os.environ, {}, clear=True) def testJsonFile(self, systemMock): """Load JSON from a file""" with open(os.path.join(JSON_DIR, 'shell.outline')) as fp: ol = load_json(fp.read()) with test_utils.TemporarySessionDirectory(): ol.setup() - ol.get_layer('shell_layer').execute('1000') + layer = ol.get_layer('shell_layer') + self.assertEqual('LAYER_VALUE', layer.get_env('LAYER_KEY')) + layer.execute('1000') systemMock.assert_has_calls([mock.call(['/bin/ls'], frame=1000)]) + self.assertEqual('LAYER_VALUE', os.environ['LAYER_KEY']) if __name__ == '__main__': From dff882c7753a6f2f25c38ade2c3cb2122ba24a61 Mon Sep 17 00:00:00 2001 From: Lars van der Bijl <285658+larsbijl@users.noreply.github.com> Date: Sat, 23 Jan 2021 11:29:51 +0000 Subject: [PATCH 033/277] feat: Add timeout and LLU timeout (#761) Add support for layers to have timeout. If a frame goes past it's hard timeout it get's killed. LLU timeout is usually a lower value that check when the last log update has happend. if no update happens in the LLU window it's also killed. Closes #462 --- VERSION.in | 2 +- .../com/imageworks/spcue/FrameDetail.java | 1 + .../com/imageworks/spcue/LayerDetail.java | 18 ++++ .../com/imageworks/spcue/ServiceEntity.java | 4 + .../com/imageworks/spcue/dao/FrameDao.java | 13 +++ .../com/imageworks/spcue/dao/LayerDao.java | 18 ++++ .../spcue/dao/oracle/FrameDaoJdbc.java | 16 ++++ .../spcue/dao/oracle/LayerDaoJdbc.java | 14 +++ .../spcue/dao/postgres/FrameDaoJdbc.java | 23 ++++- .../spcue/dao/postgres/LayerDaoJdbc.java | 25 ++++- .../spcue/dao/postgres/ServiceDaoJdbc.java | 42 ++++++--- .../spcue/dao/postgres/WhiteboardDaoJdbc.java | 19 +++- .../spcue/dispatcher/DispatchSupport.java | 8 ++ .../dispatcher/DispatchSupportService.java | 16 ++++ .../dispatcher/FrameCompleteHandler.java | 21 +++-- .../spcue/dispatcher/HostReportHandler.java | 76 ++++++++++++++- .../imageworks/spcue/servant/ManageLayer.java | 20 ++++ .../spcue/servant/ManageService.java | 4 + .../spcue/servant/ManageServiceOverride.java | 2 + .../com/imageworks/spcue/service/JobSpec.java | 11 +++ .../V7__Add_layer_service_timeout.sql | 13 +++ .../main/resources/public/dtd/cjsl-1.10.dtd | 94 +++++++++++++++++++ .../test/dao/postgres/ServiceDaoTests.java | 18 ++++ .../test/dao/postgres/WhiteboardDaoTests.java | 2 + .../test/service/ServiceManagerTests.java | 6 ++ .../src/test/resources/conf/dtd/cjsl-1.10.dtd | 94 +++++++++++++++++++ .../src/test/resources/conf/dtd/cjsl-1.9.dtd | 92 ++++++++++++++++++ cuegui/cuegui/LayerDialog.py | 43 ++++++++- cuegui/cuegui/LayerMonitorTree.py | 9 +- cuegui/cuegui/ServiceDialog.py | 22 ++++- proto/job.proto | 27 ++++++ proto/report.proto | 1 + proto/service.proto | 2 + pycue/opencue/wrappers/layer.py | 16 ++++ pyoutline/outline/backend/cue.py | 8 +- rqd/rqd/rqmachine.py | 4 + rqd/rqd/rqnetwork.py | 5 +- 37 files changed, 774 insertions(+), 35 deletions(-) create mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V7__Add_layer_service_timeout.sql create mode 100644 cuebot/src/main/resources/public/dtd/cjsl-1.10.dtd create mode 100644 cuebot/src/test/resources/conf/dtd/cjsl-1.10.dtd create mode 100644 cuebot/src/test/resources/conf/dtd/cjsl-1.9.dtd diff --git a/VERSION.in b/VERSION.in index bd73f4707..2eb3c4fe4 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.4 +0.5 diff --git a/cuebot/src/main/java/com/imageworks/spcue/FrameDetail.java b/cuebot/src/main/java/com/imageworks/spcue/FrameDetail.java index 7d33618bf..60c07a030 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/FrameDetail.java +++ b/cuebot/src/main/java/com/imageworks/spcue/FrameDetail.java @@ -37,5 +37,6 @@ public class FrameDetail extends FrameEntity implements FrameInterface { public Timestamp dateStarted; public Timestamp dateStopped; public Timestamp dateUpdated; + public Timestamp dateLLU; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/LayerDetail.java b/cuebot/src/main/java/com/imageworks/spcue/LayerDetail.java index bb964c93f..3b473f8c1 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/LayerDetail.java +++ b/cuebot/src/main/java/com/imageworks/spcue/LayerDetail.java @@ -36,6 +36,8 @@ public class LayerDetail extends LayerEntity implements LayerInterface { public long minimumMemory; public long minimumGpu; public int chunkSize; + public int timeout; + public int timeout_llu; public int dispatchOrder; public int totalFrameCount; @@ -90,6 +92,22 @@ public void setThreadable(boolean isThreadable) { this.isThreadable = isThreadable; } + public int getTimeout() { + return timeout; + } + + public void setTimeout(int timeout) { + this.timeout = timeout; + } + + public int getTimeoutLLU() { + return timeout; + } + + public void setTimeoutLLU(int timeout_llu) { + this.timeout_llu = timeout_llu; + } + public long getMinimumMemory() { return minimumMemory; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/ServiceEntity.java b/cuebot/src/main/java/com/imageworks/spcue/ServiceEntity.java index 3c84525c8..373877e69 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/ServiceEntity.java +++ b/cuebot/src/main/java/com/imageworks/spcue/ServiceEntity.java @@ -55,5 +55,9 @@ public class ServiceEntity extends Entity { */ public LinkedHashSet tags = new LinkedHashSet(); + public int timeout = 0; + + public int timeout_llu = 0; + } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java index 4a6853c25..e576a0f59 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java @@ -315,6 +315,19 @@ boolean updateFrameStopped(FrameInterface frame, FrameState state, int exitStatu */ ResourceUsage getResourceUsage(FrameInterface f); + /** + * Update Frame usage values for the given frame. The + * frame must be in the Running state. If the frame + * is locked by another thread, the process is aborted because + * we'll most likely get a new update one minute later. + * + * @param f + * @param lluTime + * @throws FrameReservationException if the frame is locked + * by another thread. + */ + void updateFrameUsage(FrameInterface f, long lluTime); + /** * Update memory usage values for the given frame. The * frame must be in the Running state. If the frame diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/LayerDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/LayerDao.java index e7eb7162f..7843d8e8d 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/LayerDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/LayerDao.java @@ -281,6 +281,24 @@ public interface LayerDao { */ void updateThreadable(LayerInterface layer, boolean threadable); + /** + * Update a layer's timeout value, which limits how + * much the frame can run on a host. + * + * @param job + * @param timeout + */ + void updateTimeout(LayerInterface layer, int timeout); + + /** + * Update a layer's LLU timeout value, which limits how + * much the frame can run on a host without updates in the log file. + * + * @param job + * @param timeout + */ + void updateTimeoutLLU(LayerInterface layer, int timeout_llu); + /** * Lowers the minimum memory on a layer if the layer * is using less memory and the currnet min memory is diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/FrameDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/FrameDaoJdbc.java index ac36b49ab..fd3a421d1 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/FrameDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/FrameDaoJdbc.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.EnumSet; import java.util.List; +import java.sql.Timestamp; import java.util.Optional; import org.springframework.jdbc.core.RowMapper; @@ -990,6 +991,21 @@ public ResourceUsage getResourceUsage(FrameInterface f) { "pk_frame = ?", RESOURCE_USAGE_MAPPER, f.getFrameId()); } + private static final String UPDATE_FRAME_IO_USAGE = + "UPDATE " + + "frame " + + "SET " + + "ts_updated = current_timestamp," + + "ts_llu = ? " + + "WHERE " + + "pk_frame = ? "; + + @Override + public void updateFrameUsage(FrameInterface f, long lluTime) { + getJdbcTemplate().update(UPDATE_FRAME_IO_USAGE, + new Timestamp(lluTime * 1000l), f.getFrameId()); + } + private static final String UPDATE_FRAME_MEMORY_USAGE = "UPDATE " + "frame " + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/LayerDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/LayerDaoJdbc.java index 04cb1b14f..f189af1d9 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/LayerDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/LayerDaoJdbc.java @@ -624,6 +624,20 @@ public void updateThreadable(LayerInterface layer, boolean threadable) { threadable, layer.getLayerId()); } + @Override + public void updateTimeout(LayerInterface layer, int timeout){ + getJdbcTemplate().update( + "UPDATE layer SET int_timeout=? WHERE pk_layer=?", + timeout, layer.getLayerId()); + } + + @Override + public void updateTimeoutLLU(LayerInterface layer, int timeout_llu){ + getJdbcTemplate().update( + "UPDATE layer SET int_timeout_llu=? WHERE pk_layer=?", + timeout_llu, layer.getLayerId()); + } + @Override public void enableMemoryOptimizer(LayerInterface layer, boolean value) { getJdbcTemplate().update( diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java index fb85cdf10..5c043e995 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.EnumSet; import java.util.List; +import java.sql.Timestamp; import java.util.Optional; import org.springframework.jdbc.core.RowMapper; @@ -397,6 +398,7 @@ public FrameDetail mapRow(ResultSet rs, int rowNum) throws SQLException { frame.dateStarted = rs.getTimestamp("ts_started"); frame.dateStopped = rs.getTimestamp("ts_stopped"); frame.dateUpdated = rs.getTimestamp("ts_updated"); + frame.dateLLU = rs.getTimestamp("ts_llu"); frame.version = rs.getInt("int_version"); if (rs.getString("str_host") != null) { @@ -472,9 +474,10 @@ public boolean isOrphan(FrameInterface frame) { "int_number, " + "int_dispatch_order, " + "int_layer_order, "+ - "ts_updated "+ + "ts_updated, "+ + "ts_llu "+ ") " + - "VALUES (?,?,?,?,?,?,?,?,current_timestamp)"; + "VALUES (?,?,?,?,?,?,?,?,current_timestamp,current_timestamp)"; @Override public void insertFrames(LayerDetail layer, List frames) { @@ -692,6 +695,7 @@ public boolean updateFrameState(FrameInterface frame, FrameState state) { "SET " + "str_state=?, " + "ts_updated = current_timestamp, " + + "ts_llu = current_timestamp, " + "int_depend_count = 0, " + "int_version = int_version + 1 " + "WHERE " + @@ -965,6 +969,21 @@ public ResourceUsage getResourceUsage(FrameInterface f) { "pk_frame = ?", RESOURCE_USAGE_MAPPER, f.getFrameId()); } + private static final String UPDATE_FRAME_IO_USAGE = + "UPDATE " + + "frame " + + "SET " + + "ts_updated = current_timestamp," + + "ts_llu = ? " + + "WHERE " + + "pk_frame = ? "; + + @Override + public void updateFrameUsage(FrameInterface f, long lluTime) { + getJdbcTemplate().update(UPDATE_FRAME_IO_USAGE, + new Timestamp(lluTime * 1000l), f.getFrameId()); + } + private static final String UPDATE_FRAME_MEMORY_USAGE = "UPDATE " + "frame " + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/LayerDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/LayerDaoJdbc.java index c2be2cc1a..26654f392 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/LayerDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/LayerDaoJdbc.java @@ -211,6 +211,8 @@ public LayerDetail mapRow(ResultSet rs, int rowNum) throws SQLException { rs.getString("str_tags").replaceAll(" ", "").split("\\|")); layer.services.addAll( Lists.newArrayList(rs.getString("str_services").split(","))); + layer.timeout = rs.getInt("int_timeout"); + layer.timeout_llu = rs.getInt("int_timeout_llu"); return layer; } }; @@ -310,9 +312,11 @@ public LayerInterface getLayer(String id) { "b_threadable, " + "int_mem_min, " + "int_gpu_min, " + - "str_services " + + "str_services, " + + "int_timeout," + + "int_timeout_llu " + ") " + - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; @Override public void insertLayerDetail(LayerDetail l) { @@ -322,7 +326,8 @@ public void insertLayerDetail(LayerDetail l) { l.range, l.chunkSize, l.dispatchOrder, StringUtils.join(l.tags," | "), l.type.toString(), l.minimumCores, l.maximumCores, l.isThreadable, - l.minimumMemory, l.minimumGpu, StringUtils.join(l.services,",")); + l.minimumMemory, l.minimumGpu, StringUtils.join(l.services,","), + l.timeout, l.timeout_llu); } @Override @@ -623,6 +628,20 @@ public void updateThreadable(LayerInterface layer, boolean threadable) { threadable, layer.getLayerId()); } + @Override + public void updateTimeout(LayerInterface layer, int timeout){ + getJdbcTemplate().update( + "UPDATE layer SET int_timeout=? WHERE pk_layer=?", + timeout, layer.getLayerId()); + } + + @Override + public void updateTimeoutLLU(LayerInterface layer, int timeout_llu){ + getJdbcTemplate().update( + "UPDATE layer SET int_timeout_llu=? WHERE pk_layer=?", + timeout_llu, layer.getLayerId()); + } + @Override public void enableMemoryOptimizer(LayerInterface layer, boolean value) { getJdbcTemplate().update( diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ServiceDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ServiceDaoJdbc.java index bc41fb038..b31d9ade0 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ServiceDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ServiceDaoJdbc.java @@ -63,6 +63,8 @@ public ServiceEntity mapRow(ResultSet rs, int rowNum) throws SQLException { s.minGpu = rs.getLong("int_gpu_min"); s.threadable = rs.getBoolean("b_threadable"); s.tags = splitTags(rs.getString("str_tags")); + s.timeout = rs.getInt("int_timeout"); + s.timeout_llu = rs.getInt("int_timeout_llu"); return s; } }; @@ -81,6 +83,8 @@ public ServiceOverrideEntity mapRow(ResultSet rs, int rowNum) s.threadable = rs.getBoolean("b_threadable"); s.tags = splitTags(rs.getString("str_tags")); s.showId = rs.getString("pk_show"); + s.timeout = rs.getInt("int_timeout"); + s.timeout_llu = rs.getInt("int_timeout_llu"); return s; } }; @@ -94,7 +98,9 @@ public ServiceOverrideEntity mapRow(ResultSet rs, int rowNum) "service.int_cores_max," + "service.int_mem_min," + "service.int_gpu_min," + - "service.str_tags " + + "service.str_tags, " + + "service.int_timeout, " + + "service.int_timeout_llu " + "FROM " + "service "; @@ -114,7 +120,9 @@ public ServiceEntity get(String id) { "show_service.int_cores_max, "+ "show_service.int_mem_min," + "show_service.int_gpu_min," + - "show_service.str_tags, " + + "show_service.str_tags," + + "show_service.int_timeout," + + "show_service.int_timeout_llu," + "show.pk_show " + "FROM " + "show_service," + @@ -160,8 +168,10 @@ public boolean isOverridden(String service, String show) { "int_cores_max, "+ "int_mem_min," + "int_gpu_min," + - "str_tags" + - ") VALUES (?,?,?,?,?,?,?,?)"; + "str_tags," + + "int_timeout," + + "int_timeout_llu " + + ") VALUES (?,?,?,?,?,?,?,?,?,?)"; @Override public void insert(ServiceEntity service) { @@ -169,7 +179,8 @@ public void insert(ServiceEntity service) { getJdbcTemplate().update(INSERT_SERVICE, service.id, service.name, service.threadable, service.minCores, service.maxCores, service.minMemory, service.minGpu, - StringUtils.join(service.tags.toArray(), " | ")); + StringUtils.join(service.tags.toArray(), " | "), + service.timeout, service.timeout_llu); } private static final String INSERT_SERVICE_WITH_SHOW = @@ -184,8 +195,10 @@ public void insert(ServiceEntity service) { "int_cores_max," + "int_mem_min," + "int_gpu_min," + - "str_tags " + - ") VALUES (?,?,?,?,?,?,?,?,?)"; + "str_tags," + + "int_timeout," + + "int_timeout_llu " + + ") VALUES (?,?,?,?,?,?,?,?,?,?,?)"; @Override public void insert(ServiceOverrideEntity service) { @@ -193,7 +206,8 @@ public void insert(ServiceOverrideEntity service) { getJdbcTemplate().update(INSERT_SERVICE_WITH_SHOW, service.id, service.showId, service.name, service.threadable, service.minCores, service.maxCores, service.minMemory, - service.minGpu, joinTags(service.tags)); + service.minGpu, joinTags(service.tags), + service.timeout, service.timeout_llu); } private static final String UPDATE_SERVICE = @@ -206,7 +220,9 @@ public void insert(ServiceOverrideEntity service) { "int_cores_max=?,"+ "int_mem_min=?," + "int_gpu_min=?," + - "str_tags=? " + + "str_tags=?," + + "int_timeout=?," + + "int_timeout_llu=? " + "WHERE " + "pk_service = ?"; @@ -215,7 +231,7 @@ public void update(ServiceEntity service) { getJdbcTemplate().update(UPDATE_SERVICE, service.name, service.threadable, service.minCores, service.maxCores, service.minMemory, service.minGpu, joinTags(service.tags), - service.getId()); + service.timeout, service.timeout_llu, service.getId()); } private static final String UPDATE_SERVICE_WITH_SHOW = @@ -228,7 +244,9 @@ service.minMemory, service.minGpu, joinTags(service.tags), "int_cores_max=?," + "int_mem_min=?," + "int_gpu_min=?," + - "str_tags=? " + + "str_tags=?," + + "int_timeout=?," + + "int_timeout_llu=? " + "WHERE " + "pk_show_service = ?"; @@ -237,7 +255,7 @@ public void update(ServiceOverrideEntity service) { getJdbcTemplate().update(UPDATE_SERVICE_WITH_SHOW, service.name, service.threadable, service.minCores, service.maxCores, service.minMemory, service.minGpu, joinTags(service.tags), - service.getId()); + service.timeout, service.timeout_llu, service.getId()); } @Override diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index 71ab45af1..687e91ecd 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -1246,7 +1246,9 @@ public Layer mapRow(ResultSet rs, int rowNum) throws SQLException { replaceAll(" ","").split("\\|"))) .addAllServices(Arrays.asList(SqlUtil.getString(rs,"str_services").split(","))) .addAllLimits(Arrays.asList(SqlUtil.getString(rs,"str_limit_names").split(","))) - .setMemoryOptimizerEnabled(rs.getBoolean("b_optimize")); + .setMemoryOptimizerEnabled(rs.getBoolean("b_optimize")) + .setTimeout(rs.getInt("int_timeout")) + .setTimeoutLlu(rs.getInt("int_timeout_llu")); LayerStats.Builder statsBuilder = LayerStats.newBuilder() .setReservedCores(Convert.coreUnitsToCores(rs.getInt("int_cores"))) @@ -1410,6 +1412,8 @@ public Service mapRow(ResultSet rs, int rowNum) throws SQLException { .setMinGpu(rs.getInt("int_gpu_min")) .addAllTags(Lists.newArrayList(ServiceDaoJdbc.splitTags( SqlUtil.getString(rs,"str_tags")))) + .setTimeout(rs.getInt("int_timeout")) + .setTimeoutLlu(rs.getInt("int_timeout_llu")) .build(); } }; @@ -1427,6 +1431,8 @@ public ServiceOverride mapRow(ResultSet rs, int rowNum) throws SQLException { .setMinGpu(rs.getInt("int_gpu_min")) .addAllTags(Lists.newArrayList(ServiceDaoJdbc.splitTags( SqlUtil.getString(rs,"str_tags")))) + .setTimeout(rs.getInt("int_timeout")) + .setTimeoutLlu(rs.getInt("int_timeout_llu")) .build(); return ServiceOverride.newBuilder() .setId(SqlUtil.getString(rs,"pk_show_service")) @@ -1505,6 +1511,7 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "frame.int_dispatch_order,"+ "frame.ts_started,"+ "frame.ts_stopped,"+ + "frame.ts_llu,"+ "frame.int_retries,"+ "frame.str_state,"+ "frame.str_host,"+ @@ -1589,6 +1596,7 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "frame.str_state,"+ "frame.str_host,"+ "frame.int_cores,"+ + "frame.ts_llu,"+ "COALESCE(proc.int_mem_max_used, frame.int_mem_max_used) AS int_mem_max_used," + "COALESCE(proc.int_mem_used, frame.int_mem_used) AS int_mem_used " + "FROM "+ @@ -1988,7 +1996,9 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "service.int_cores_max," + "service.int_mem_min," + "service.int_gpu_min," + - "service.str_tags " + + "service.str_tags," + + "service.int_timeout," + + "service.int_timeout_llu " + "FROM "+ "service "; @@ -2001,7 +2011,9 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "show_service.int_cores_max," + "show_service.int_mem_min," + "show_service.int_gpu_min," + - "show_service.str_tags " + + "show_service.str_tags," + + "show_service.int_timeout," + + "show_service.int_timeout_llu " + "FROM "+ "show_service, " + "show " + @@ -2118,6 +2130,7 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "frame.int_dispatch_order,"+ "frame.ts_started,"+ "frame.ts_stopped,"+ + "frame.ts_llu,"+ "frame.int_retries,"+ "frame.str_state,"+ "frame.str_host,"+ diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java index 533066881..ebdd5082d 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java @@ -400,6 +400,14 @@ List findNextDispatchFrames(LayerInterface layer, VirtualProc pro */ void clearFrame(DispatchFrame frame); + /** + * Update usage data for the given frame. + * + * @param frame + * @param lluTime + */ + void updateFrameUsage(FrameInterface frame, long lluTime); + /** * Update memory usage data for the given frame. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java index 739afc468..3e3d82b2f 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java @@ -532,6 +532,22 @@ public void updateProcMemoryUsage(FrameInterface frame, long rss, long maxRss, procDao.updateProcMemoryUsage(frame, rss, maxRss, vsize, maxVsize); } + @Override + @Transactional(propagation = Propagation.REQUIRED) + public void updateFrameUsage(FrameInterface frame, long lluTime) { + + try { + frameDao.updateFrameUsage(frame, lluTime); + } + catch (FrameReservationException ex) { + // Eat this, the frame was not in the correct state or + // was locked by another thread. The only reason it would + // be locked by another thread would be if the state is + // changing. + logger.warn("failed to update io stats for frame: " + frame); + } + } + @Override @Transactional(propagation = Propagation.REQUIRED) public void updateFrameMemoryUsage(FrameInterface frame, long rss, long maxRss) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java index 206181fb6..fe2482720 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java @@ -19,6 +19,7 @@ package com.imageworks.spcue.dispatcher; +import java.sql.Timestamp; import java.util.EnumSet; import java.util.Random; import java.util.concurrent.atomic.AtomicLong; @@ -30,6 +31,7 @@ import com.imageworks.spcue.DispatchHost; import com.imageworks.spcue.DispatchJob; import com.imageworks.spcue.JobDetail; +import com.imageworks.spcue.LayerDetail; import com.imageworks.spcue.LayerInterface; import com.imageworks.spcue.Source; import com.imageworks.spcue.VirtualProc; @@ -132,10 +134,9 @@ public void handleFrameCompleteReport(final FrameCompleteReport report) { } final DispatchJob job = jobManager.getDispatchJob(proc.getJobId()); - final DispatchFrame frame = jobManager.getDispatchFrame( - report.getFrame().getFrameId()); - final FrameState newFrameState = determineFrameState(job, - frame, report); + final LayerDetail layer = jobManager.getLayerDetail(report.getFrame().getLayerId()); + final DispatchFrame frame = jobManager.getDispatchFrame(report.getFrame().getFrameId()); + final FrameState newFrameState = determineFrameState(job, layer, frame, report); if (dispatchSupport.stopFrame(frame, newFrameState, report.getExitStatus(), report.getFrame().getMaxRss())) { @@ -228,7 +229,7 @@ public void handlePostFrameCompleteOperations(VirtualProc proc, try { /* - * The default behavior is to keep the proc on the same. + * The default behavior is to keep the proc on the same job. */ boolean unbookProc = proc.unbooked; @@ -513,7 +514,7 @@ else if (report.getHost().getNimbyLocked()) { * @param report * @return */ - public static final FrameState determineFrameState(DispatchJob job, DispatchFrame frame, FrameCompleteReport report) { + public static final FrameState determineFrameState(DispatchJob job, LayerDetail layer, DispatchFrame frame, FrameCompleteReport report) { if (EnumSet.of(FrameState.WAITING, FrameState.EATEN).contains( frame.state)) { @@ -528,6 +529,9 @@ else if (frame.state.equals(FrameState.DEAD)) { } } else if (report.getExitStatus() != 0) { + long r = System.currentTimeMillis() / 1000; + long lastUpdate = (r - report.getFrame().getLluTime()) / 60; + FrameState newState = FrameState.WAITING; if (report.getExitStatus() == FrameExitStatus.SKIP_RETRY_VALUE || (job.maxRetries != 0 && report.getExitSignal() == 119)) { @@ -535,6 +539,11 @@ else if (frame.state.equals(FrameState.DEAD)) { newState = FrameState.WAITING; } else if (job.autoEat) { newState = FrameState.EATEN; + // ETC Time out and LLU timeout + } else if (layer.timeout_llu != 0 && report.getFrame().getLluTime() != 0 && lastUpdate > (layer.timeout_llu -1)) { + newState = FrameState.DEAD; + } else if (layer.timeout != 0 && report.getRunTime() > layer.timeout * 60) { + newState = FrameState.DEAD; } else if (report.getRunTime() > Dispatcher.FRAME_TIME_NO_RETRY) { newState = FrameState.DEAD; } else if (frame.retries >= job.maxRetries) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java index 83d7342de..815689bb4 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java @@ -36,6 +36,7 @@ import com.imageworks.spcue.FrameInterface; import com.imageworks.spcue.JobEntity; import com.imageworks.spcue.LayerEntity; +import com.imageworks.spcue.LayerDetail; import com.imageworks.spcue.LocalHostAssignment; import com.imageworks.spcue.Source; import com.imageworks.spcue.VirtualProc; @@ -203,6 +204,17 @@ public void handleHostReport(HostReport report, boolean isBoot) { */ updateMemoryUsage(report.getFramesList()); + /* + * Updates usage for the proc, frames, + * jobs, and layers. + */ + updateFrameUsage(report.getFramesList()); + + /* + * kill frames that have over run. + */ + killTimedOutFrames(report); + /* * Increase/decreased reserved memory. */ @@ -505,6 +517,68 @@ private void handleMemoryReservations(final DispatchHost host, final HostReport } } + /** + * Kill frames that over run. + * + * @param rFrames + */ + private void killTimedOutFrames(HostReport report) { + + final Map layers = new HashMap(5); + + for (RunningFrameInfo frame: report.getFramesList()) { + String layerId = frame.getLayerId(); + LayerDetail layer = layerDao.getLayerDetail(layerId); + long runtimeMinutes = ((System.currentTimeMillis() - frame.getStartTime()) / 1000l) / 60; + + if (layer.timeout != 0 && runtimeMinutes > layer.timeout){ + try { + killQueue.execute(new DispatchRqdKillFrame(report.getHost().getName(), + frame.getFrameId(), + "This frame has reached it timeout.", + rqdClient)); + } catch (TaskRejectedException e) { + logger.warn("Unable to queue RQD kill, task rejected, " + e); + } + } + + if (layer.timeout_llu == 0){ + continue; + } + + if (frame.getLluTime() == 0){ + continue; + } + + long r = System.currentTimeMillis() / 1000; + long lastUpdate = (r - frame.getLluTime()) / 60; + + if (layer.timeout_llu != 0 && lastUpdate > (layer.timeout_llu -1)){ + try { + killQueue.execute(new DispatchRqdKillFrame(report.getHost().getName(), + frame.getFrameId(), + "This frame has reached it LLU timeout.", + rqdClient)); + } catch (TaskRejectedException e) { + logger.warn("Unable to queue RQD kill, task rejected, " + e); + } + } + } + } + + /** + * Update IO usage for the given list of frames. + * + * @param rFrames + */ + private void updateFrameUsage(List rFrames) { + + for (RunningFrameInfo rf: rFrames) { + FrameInterface frame = jobManager.getFrame(rf.getFrameId()); + dispatchSupport.updateFrameUsage(frame, rf.getLluTime()); + } + } + /** * Update memory usage for the given list of frames. * @@ -708,7 +782,7 @@ public void verifyRunningFrameInfo(HostReport report) { } catch (Exception e) { CueExceptionUtil.logStackTrace("failed", e); logger.warn("failed to verify " + - runningFrame.getJobName() +"/" + + runningFrame.getJobName() + "/" + runningFrame.getFrameName() + " was running but the frame was " + " unable to be killed, " + e); diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageLayer.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageLayer.java index 9a2a73c97..126639a1b 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageLayer.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageLayer.java @@ -102,6 +102,10 @@ import com.imageworks.spcue.grpc.job.LayerSetTagsResponse; import com.imageworks.spcue.grpc.job.LayerSetThreadableRequest; import com.imageworks.spcue.grpc.job.LayerSetThreadableResponse; +import com.imageworks.spcue.grpc.job.LayerSetTimeoutRequest; +import com.imageworks.spcue.grpc.job.LayerSetTimeoutResponse; +import com.imageworks.spcue.grpc.job.LayerSetTimeoutLLURequest; +import com.imageworks.spcue.grpc.job.LayerSetTimeoutLLUResponse; import com.imageworks.spcue.grpc.job.LayerStaggerFramesRequest; import com.imageworks.spcue.grpc.job.LayerStaggerFramesResponse; import com.imageworks.spcue.grpc.limit.Limit; @@ -351,6 +355,22 @@ public void setThreadable(LayerSetThreadableRequest request, StreamObserver responseObserver) { + updateLayer(request.getLayer()); + layerDao.updateTimeout(layer, request.getTimeout()); + responseObserver.onNext(LayerSetTimeoutResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + + @Override + public void setTimeoutLLU(LayerSetTimeoutLLURequest request, StreamObserver responseObserver) { + updateLayer(request.getLayer()); + layerDao.updateTimeoutLLU(layer, request.getTimeoutLlu()); + responseObserver.onNext(LayerSetTimeoutLLUResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + @Override public void addLimit(LayerAddLimitRequest request, StreamObserver responseObserver) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageService.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageService.java index 6e70fc5e9..eae767006 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageService.java @@ -58,6 +58,8 @@ public void createService(ServiceCreateServiceRequest request, service.minGpu = request.getData().getMinGpu(); service.tags = Sets.newLinkedHashSet(request.getData().getTagsList()); service.threadable = request.getData().getThreadable(); + service.timeout = request.getData().getTimeout(); + service.timeout_llu = request.getData().getTimeoutLlu(); serviceManager.createService(service); responseObserver.onNext(ServiceCreateServiceResponse.newBuilder() .setService(whiteboard.getService(service.getId())) @@ -130,6 +132,8 @@ private ServiceEntity toServiceEntity(Service service) { entity.minGpu = service.getMinGpu(); entity.tags = new LinkedHashSet<> (service.getTagsList()); entity.threadable = service.getThreadable(); + entity.timeout = service.getTimeout(); + entity.timeout_llu = service.getTimeoutLlu(); return entity; } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageServiceOverride.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageServiceOverride.java index e2de68220..bd90575b5 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageServiceOverride.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageServiceOverride.java @@ -70,6 +70,8 @@ private ServiceEntity toServiceEntity(Service service) { entity.minGpu = service.getMinGpu(); entity.tags = new LinkedHashSet<>(service.getTagsList()); entity.threadable = service.getThreadable(); + entity.timeout = service.getTimeout(); + entity.timeout_llu = service.getTimeoutLlu(); return entity; } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java index d309fa264..d553456fd 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java @@ -403,6 +403,15 @@ private void handleLayerTags(BuildableJob buildableJob, Element jobTag) { buildableLayer); determineMinimumGpu(buildableJob, layerTag, layer); + // set a timeout value on the layer + if (layerTag.getChildTextTrim("timeout") != null) { + layer.timeout = Integer.parseInt(layerTag.getChildTextTrim("timeout")); + } + + if (layerTag.getChildTextTrim("timeout_llu") != null) { + layer.timeout_llu = Integer.parseInt(layerTag.getChildTextTrim("timeout_llu")); + } + /* * Handle the layer environment */ @@ -671,6 +680,8 @@ private void determineResourceDefaults(Element layerTag, layer.tags.addAll(primaryService.tags); layer.services.addAll(services); layer.limits.addAll(limits); + layer.timeout = primaryService.timeout; + layer.timeout_llu = primaryService.timeout_llu; } /** diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V7__Add_layer_service_timeout.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V7__Add_layer_service_timeout.sql new file mode 100644 index 000000000..4e0733996 --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V7__Add_layer_service_timeout.sql @@ -0,0 +1,13 @@ + +-- Add timeout + +ALTER TABLE show_service ADD COLUMN int_timeout INT DEFAULT 0 NOT NULL; +ALTER TABLE service ADD COLUMN int_timeout INT DEFAULT 0 NOT NULL; +ALTER TABLE layer ADD COLUMN int_timeout INT DEFAULT 0 NOT NULL; + +-- Add LLU timeout + +ALTER TABLE frame ADD COLUMN ts_llu TIMESTAMP (6) WITH TIME ZONE; +ALTER TABLE show_service ADD COLUMN int_timeout_llu INT DEFAULT 0 NOT NULL; +ALTER TABLE service ADD COLUMN int_timeout_llu INT DEFAULT 0 NOT NULL; +ALTER TABLE layer ADD COLUMN int_timeout_llu INT DEFAULT 0 NOT NULL; \ No newline at end of file diff --git a/cuebot/src/main/resources/public/dtd/cjsl-1.10.dtd b/cuebot/src/main/resources/public/dtd/cjsl-1.10.dtd new file mode 100644 index 000000000..19a5ece29 --- /dev/null +++ b/cuebot/src/main/resources/public/dtd/cjsl-1.10.dtd @@ -0,0 +1,94 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ServiceDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ServiceDaoTests.java index 621ec1504..fee824fc1 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ServiceDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ServiceDaoTests.java @@ -67,6 +67,8 @@ public void testInsertService() { ServiceEntity s = new ServiceEntity(); s.name = "dillweed"; s.minCores = 100; + s.timeout = 0; + s.timeout_llu = 0; s.minMemory = CueUtil.GB4; s.minGpu = CueUtil.GB; s.threadable = false; @@ -83,6 +85,8 @@ public void testUpdateService() { ServiceEntity s = new ServiceEntity(); s.name = "dillweed"; s.minCores = 100; + s.timeout = 0; + s.timeout_llu = 0; s.minMemory = CueUtil.GB4; s.minGpu = CueUtil.GB; s.threadable = false; @@ -93,6 +97,8 @@ public void testUpdateService() { s.name = "smacktest"; s.minCores = 200; + s.timeout = 0; + s.timeout_llu = 0; s.minMemory = CueUtil.GB8; s.minGpu = CueUtil.GB2; s.threadable = true; @@ -116,6 +122,8 @@ public void testDeleteService() { ServiceEntity s = new ServiceEntity(); s.name = "dillweed"; s.minCores = 100; + s.timeout = 0; + s.timeout_llu = 0; s.minMemory = CueUtil.GB4; s.minGpu = CueUtil.GB; s.threadable = false; @@ -138,6 +146,8 @@ public void testInsertServiceOverride() { ServiceOverrideEntity s = new ServiceOverrideEntity(); s.name = "dillweed"; s.minCores = 100; + s.timeout = 0; + s.timeout_llu = 0; s.minMemory = CueUtil.GB4; s.minGpu = CueUtil.GB; s.threadable = false; @@ -155,6 +165,8 @@ public void testUpdateServiceOverride() { ServiceOverrideEntity s = new ServiceOverrideEntity(); s.name = "dillweed"; s.minCores = 100; + s.timeout = 0; + s.timeout_llu = 0; s.minMemory = CueUtil.GB4; s.minGpu = CueUtil.GB2; s.threadable = false; @@ -167,6 +179,8 @@ public void testUpdateServiceOverride() { s.name = "smacktest"; s.minCores = 200; + s.timeout = 10; + s.timeout_llu = 10; s.minMemory = CueUtil.GB8; s.minGpu = CueUtil.GB4; s.threadable = true; @@ -178,6 +192,8 @@ public void testUpdateServiceOverride() { assertEquals(s.name, s1.name); assertEquals(s.minCores, s1.minCores); + assertEquals(s.timeout, s1.timeout); + assertEquals(s.timeout_llu, s1.timeout_llu); assertEquals(s.minMemory, s1.minMemory); assertEquals(s.minGpu, s1.minGpu); assertEquals(s.threadable, s1.threadable); @@ -191,6 +207,8 @@ public void testDeleteServiceOverride() { ServiceOverrideEntity s = new ServiceOverrideEntity(); s.name = "dillweed"; s.minCores = 100; + s.timeout = 0; + s.timeout_llu = 0; s.minMemory = CueUtil.GB4; s.minGpu = CueUtil.GB; s.threadable = false; diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java index fe5514efb..99449337b 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java @@ -324,6 +324,8 @@ public void getServiceOverride() { ServiceOverrideEntity s = new ServiceOverrideEntity(); s.name = "test"; s.minCores = 100; + s.timeout = 0; + s.timeout_llu = 0; s.minMemory = 320000; s.tags.add("general"); s.threadable = false; diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/ServiceManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/ServiceManagerTests.java index 7aeeadb33..3573cbe59 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/ServiceManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/ServiceManagerTests.java @@ -86,6 +86,8 @@ public void testCreateService() { s.minMemory = CueUtil.GB4; s.minGpu = CueUtil.GB2; s.threadable = false; + s.timeout = 0; + s.timeout_llu = 0; s.tags.addAll(Sets.newHashSet("general")); serviceManager.createService(s); @@ -100,6 +102,8 @@ public void testOverrideExistingService() { ServiceOverrideEntity s = new ServiceOverrideEntity(); s.name = "arnold"; s.minCores = 400; + s.timeout = 10; + s.timeout_llu = 10; s.minMemory = CueUtil.GB8; s.minGpu = CueUtil.GB2; s.threadable = false; @@ -111,6 +115,8 @@ public void testOverrideExistingService() { ServiceEntity newService = serviceManager.getService("arnold", s.showId); assertEquals(s, newService); assertEquals(400, newService.minCores); + assertEquals(10, newService.timeout); + assertEquals(10, newService.timeout_llu); assertEquals(CueUtil.GB8, newService.minMemory); assertEquals(CueUtil.GB2, newService.minGpu); assertFalse(newService.threadable); diff --git a/cuebot/src/test/resources/conf/dtd/cjsl-1.10.dtd b/cuebot/src/test/resources/conf/dtd/cjsl-1.10.dtd new file mode 100644 index 000000000..4be16aa98 --- /dev/null +++ b/cuebot/src/test/resources/conf/dtd/cjsl-1.10.dtd @@ -0,0 +1,94 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cuebot/src/test/resources/conf/dtd/cjsl-1.9.dtd b/cuebot/src/test/resources/conf/dtd/cjsl-1.9.dtd new file mode 100644 index 000000000..c945f948f --- /dev/null +++ b/cuebot/src/test/resources/conf/dtd/cjsl-1.9.dtd @@ -0,0 +1,92 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cuegui/cuegui/LayerDialog.py b/cuegui/cuegui/LayerDialog.py index 8364dda01..a091cdba2 100644 --- a/cuegui/cuegui/LayerDialog.py +++ b/cuegui/cuegui/LayerDialog.py @@ -154,6 +154,20 @@ def __init__(self, layers, parent=None): self.__thread = QtWidgets.QCheckBox(self) self.__thread.setChecked(self.getThreading()) + # Timeout + self.__timeout = QtWidgets.QSpinBox(self) + self.__timeout.setRange(0, 4320) + self.__timeout.setSingleStep(1) + self.__timeout.setSuffix(" minutes") + self.__timeout.setSpecialValueText("No timeout") + + # Timeout LLU + self.__timeout_llu = QtWidgets.QSpinBox(self) + self.__timeout_llu.setRange(0, 4320) + self.__timeout_llu.setSingleStep(1) + self.__timeout_llu.setSuffix(" minutes") + self.__timeout_llu.setSpecialValueText("No timeout") + # Memory Optimizer self.__mem_opt = QtWidgets.QCheckBox() self.__mem_opt.setChecked(self.getMemoryOptSetting()) @@ -194,6 +208,8 @@ def __init__(self, layers, parent=None): self.__gpu.slider.setValue(self.getMaxGpu()) self.__core.setValue(self.getMinCores()) self.__max_cores.setValue(self.getMaxCores()) + self.__timeout.setValue(self.getTimeout()) + self.__timeout_llu.setValue(self.getTimeoutLLU()) QtWidgets.QVBoxLayout(self) @@ -222,6 +238,14 @@ def __init__(self, layers, parent=None): self.__gpu, False), multiSelect)) + layout.addWidget(EnableableItem(LayerPropertiesItem("Timeout:", + self.__timeout, + False), + multiSelect)) + layout.addWidget(EnableableItem(LayerPropertiesItem("Timeout LLU:", + self.__timeout_llu, + False), + multiSelect)) layout.addStretch() self.__group.setLayout(layout) @@ -277,7 +301,10 @@ def apply(self): layer.setThreadable(self.__thread.isChecked()) if self.__gpu.isEnabled(): layer.setMinGpu(self.__gpu.slider.value() * self.gpu_tick_kb) - + if self.__timeout.isEnabled(): + layer.setTimeout(self.__timeout.value()) + if self.__timeout_llu.isEnabled(): + layer.setTimeoutLLU(self.__timeout_llu.value()) if self.__tags.isEnabled(): self.__tags.apply() if self.__limits.isEnabled(): @@ -316,6 +343,20 @@ def getThreading(self): break return result + def getTimeout(self): + result = 0 + for layer in self.__layers: + if layer.data.timeout > result: + result = layer.data.timeout + return result + + def getTimeoutLLU(self): + result = 0 + for layer in self.__layers: + if layer.data.timeout_llu > result: + result = layer.data.timeout_llu + return result + def getMemoryOptSetting(self): result = False for layer in self.__layers: diff --git a/cuegui/cuegui/LayerMonitorTree.py b/cuegui/cuegui/LayerMonitorTree.py index 09c012334..19f21b78e 100644 --- a/cuegui/cuegui/LayerMonitorTree.py +++ b/cuegui/cuegui/LayerMonitorTree.py @@ -126,7 +126,14 @@ def __init__(self, parent): data=lambda layer: layer.percentCompleted(), sort=lambda layer: layer.percentCompleted(), tip="Progress for the Layer") - + self.addColumn("Timeout", 45, id=20, + data=lambda layer: cuegui.Utils.secondsToHHHMM(layer.data.timeout*60), + sort=lambda layer: layer.data.timeout, + tip="Timeout for the frames, Hours:Minutes") + self.addColumn("Timeout LLU", 45, id=21, + data=lambda layer: cuegui.Utils.secondsToHHHMM(layer.data.timeout_llu*60), + sort=lambda layer: layer.data.timeout_llu, + tip="Timeout for a frames\' LLU, Hours:Minutes") cuegui.AbstractTreeWidget.AbstractTreeWidget.__init__(self, parent) self.itemDoubleClicked.connect(self.__itemDoubleClickedFilterLayer) diff --git a/cuegui/cuegui/ServiceDialog.py b/cuegui/cuegui/ServiceDialog.py index 13837e624..2eccc6d70 100644 --- a/cuegui/cuegui/ServiceDialog.py +++ b/cuegui/cuegui/ServiceDialog.py @@ -65,6 +65,12 @@ def __init__(self, parent=None): self.min_gpu.setValue(0) self.min_gpu.setSingleStep(self.gpu_tick_mb) self.min_gpu.setSuffix(" MB") + self.timeout = QtWidgets.QSpinBox(self) + self.timeout.setRange(0, 4320) + self.timeout.setValue(0) + self.timeout_llu = QtWidgets.QSpinBox(self) + self.timeout_llu.setRange(0, 4320) + self.timeout_llu.setValue(0) layout = QtWidgets.QGridLayout(self) layout.addWidget(QtWidgets.QLabel("Name:", self), 0, 0) layout.addWidget(self.name, 0, 1) @@ -78,18 +84,22 @@ def __init__(self, parent=None): layout.addWidget(self.min_memory, 4, 1) layout.addWidget(QtWidgets.QLabel("Min Gpu Memory MB:", self), 5, 0) layout.addWidget(self.min_gpu, 5, 1) + layout.addWidget(QtWidgets.QLabel("Timeout (in minutes):", self), 6, 0) + layout.addWidget(self.timeout, 6, 1) + layout.addWidget(QtWidgets.QLabel("Timeout LLU (in minutes):", self), 7, 0) + layout.addWidget(self.timeout_llu, 7, 1) + self._tags_w = cuegui.TagsWidget.TagsWidget(allowed_tags=cuegui.Constants.ALLOWED_TAGS) + layout.addWidget(self._tags_w, 8, 0, 1, 2) self.__buttons = QtWidgets.QDialogButtonBox(QtWidgets.QDialogButtonBox.Save, QtCore.Qt.Horizontal, self) self.__buttons.setDisabled(True) - layout.addWidget(self.__buttons, 8, 1) + layout.addWidget(self.__buttons, 9, 1) self.__buttons.accepted.connect(self.save) - self._tags_w = cuegui.TagsWidget.TagsWidget(allowed_tags=cuegui.Constants.ALLOWED_TAGS) - layout.addWidget(self._tags_w, 6, 0, 1, 2) def _cfg(self): """ @@ -115,6 +125,8 @@ def setService(self, service): self.min_memory.setValue(service.data.min_memory // 1024) self.min_gpu.setValue(service.data.min_gpu // 1024) self._tags_w.set_tags(service.data.tags) + self.timeout.setValue(service.data.timeout) + self.timeout_llu.setValue(service.data.timeout_llu) def new(self): """ @@ -129,6 +141,8 @@ def new(self): self.max_cores.setValue(100) self.min_memory.setValue(3276) self.min_gpu.setValue(0) + self.timeout.setValue(0) + self.timeout_llu.setValue(0) self._tags_w.set_tags(['general']) def save(self): @@ -154,6 +168,8 @@ def save(self): service.setMaxCores(self.max_cores.value()) service.setMinMemory(self.min_memory.value() * 1024) service.setMinGpu(self.min_gpu.value() * 1024) + service.setTimeout(self.timeout.value()) + service.setTimeoutLLU(self.timeout_llu.value()) service.setTags(self._tags_w.get_tags()) self.saved.emit(service) diff --git a/proto/job.proto b/proto/job.proto index 4b33c015d..4a74f3aa8 100644 --- a/proto/job.proto +++ b/proto/job.proto @@ -343,6 +343,12 @@ service LayerInterface { // Set whether the layer is threadable or not rpc SetThreadable(LayerSetThreadableRequest) returns (LayerSetThreadableResponse); + // Set whether the timeout for frames in the layer + rpc SetTimeout(LayerSetTimeoutRequest) returns (LayerSetTimeoutResponse); + + // Set whether the LLU timeout for frames in the layer + rpc SetTimeoutLLU(LayerSetTimeoutLLURequest) returns (LayerSetTimeoutLLUResponse); + // Staggers the specified frame range. rpc StaggerFrames(LayerStaggerFramesRequest) returns (LayerStaggerFramesResponse); } @@ -449,6 +455,7 @@ message Frame { CheckpointState checkpoint_state = 16; int32 checkpoint_count = 17; int32 total_core_time = 18; + int32 llu_time = 19; } // Object for frame searching @@ -491,6 +498,7 @@ message UpdatedFrame { int64 max_rss = 7; int64 used_memory = 8; string last_resource = 9; + int32 llu_time = 10; } message UpdatedFrameSeq { @@ -621,6 +629,8 @@ message Layer { LayerStats layer_stats = 15; string parent_id = 16; repeated string limits = 17; + int32 timeout = 18; + int32 timeout_llu = 19; } message LayerSeq { @@ -1567,6 +1577,23 @@ message LayerSetThreadableRequest { message LayerSetThreadableResponse {} // Empty +// SetTimeout +message LayerSetTimeoutRequest { + Layer layer = 1; + int32 timeout = 2; +} + +message LayerSetTimeoutResponse {} // Empty + +// SetTimeoutLLU +message LayerSetTimeoutLLURequest { + Layer layer = 1; + int32 timeout_llu = 2; +} + +message LayerSetTimeoutLLUResponse {} // Empty + + // StaggerFrames message LayerStaggerFramesRequest { Layer layer = 1; diff --git a/proto/report.proto b/proto/report.proto index e7477df47..7a1fffdb2 100644 --- a/proto/report.proto +++ b/proto/report.proto @@ -86,6 +86,7 @@ message RunningFrameInfo { int64 max_vsize = 11; // kB int64 vsize = 12; // kB map attributes = 13; //additional data can be provided about the running frame + int64 llu_time = 14; }; diff --git a/proto/service.proto b/proto/service.proto index fac6722c1..8b554b388 100644 --- a/proto/service.proto +++ b/proto/service.proto @@ -49,6 +49,8 @@ message Service { int32 min_memory = 6; int32 min_gpu = 7; repeated string tags = 8; + int32 timeout = 9; + int32 timeout_llu = 10; } message ServiceSeq { diff --git a/pycue/opencue/wrappers/layer.py b/pycue/opencue/wrappers/layer.py index b02a5ea7e..dc33bc57a 100644 --- a/pycue/opencue/wrappers/layer.py +++ b/pycue/opencue/wrappers/layer.py @@ -162,6 +162,22 @@ def setThreadable(self, threadable): layer=self.data, threadable=threadable), timeout=Cuebot.Timeout) + def setTimeout(self, timeout): + """Set time out to the value. + :type timeout: int + :param timeout: value for timeout in minutes""" + return self.stub.SetTimeout(job_pb2.LayerSetTimeoutRequest( + layer=self.data, timeout=timeout), + timeout=Cuebot.Timeout) + + def setTimeoutLLU(self, timeout_llu): + """Set LLU time out to the value. + :type timeout: int + :param timeout: value for timeout in minutes""" + return self.stub.SetTimeoutLLU(job_pb2.LayerSetTimeoutLLURequest( + layer=self.data, timeout_llu=timeout_llu), + timeout=Cuebot.Timeout) + def addRenderPartition(self, hostname, threads, max_cores, num_mem, max_gpu): """Add a render partition to the layer. @type hostname: str diff --git a/pyoutline/outline/backend/cue.py b/pyoutline/outline/backend/cue.py index f3a27e940..61d38e2ff 100644 --- a/pyoutline/outline/backend/cue.py +++ b/pyoutline/outline/backend/cue.py @@ -281,6 +281,12 @@ def _serialize(launcher, use_pycuerun): if layer.get_arg("memory"): sub_element(spec_layer, "memory", "%s" % (layer.get_arg("memory"))) + if layer.get_arg("timeout"): + sub_element(spec_layer, "timeout", "%s" % (layer.get_arg("timeout"))) + + if layer.get_arg("timeout_llu"): + sub_element(spec_layer, "timeout_llu", "%s" % (layer.get_arg("timeout_llu"))) + if os.environ.get("OL_TAG_OVERRIDE", False): sub_element(spec_layer, "tags", scrub_tags(os.environ["OL_TAG_OVERRIDE"])) @@ -319,7 +325,7 @@ def _serialize(launcher, use_pycuerun): xml = [ '', '', + '"http://localhost:8080/spcue/dtd/cjsl-1.10.dtd">', Et.tostring(root).decode() ] diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index dae6f3752..e2cc365a0 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -262,6 +262,10 @@ def rssUpdate(self, frames): frame.rss = rss frame.maxRss = max(rss, frame.maxRss) + if os.path.exists(frame.runFrame.log_dir_file): + stat = os.stat(frame.runFrame.log_dir_file).st_mtime + frame.lluTime = int(stat) + frame.vsize = vsize frame.maxVsize = max(vsize, frame.maxVsize) diff --git a/rqd/rqd/rqnetwork.py b/rqd/rqd/rqnetwork.py index e65124d43..0639a5943 100644 --- a/rqd/rqd/rqnetwork.py +++ b/rqd/rqd/rqnetwork.py @@ -67,6 +67,8 @@ def __init__(self, rqCore, runFrame): self.utime = 0 self.stime = 0 + self.lluTime = 0 + def runningFrameInfo(self): """Returns the RunningFrameInfo object""" runningFrameInfo = rqd.compiled_proto.report_pb2.RunningFrameInfo( @@ -82,7 +84,8 @@ def runningFrameInfo(self): rss=self.rss, max_vsize=self.maxVsize, vsize=self.vsize, - attributes=self.runFrame.attributes + attributes=self.runFrame.attributes, + llu_time=self.lluTime ) return runningFrameInfo From 92a8670146fede8aae1375f7b3778711b0a6700f Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sat, 23 Jan 2021 03:32:46 -0800 Subject: [PATCH 034/277] Add IPv6 support (#845) * Introduce RQD_USE_IPV6_AS_HOSTNAME to get IPv6 address * Increase hostname length to 45 * Fix IPv4-mapped IPv6 ddress handling in getHostNameFromFQDN * Add tests for IPv6 address * Bump version to 0.5 for the database schema change --- .../spcue/dao/postgres/HostDaoJdbc.java | 4 ++ .../V7__Increase_hostname_length.sql | 4 ++ .../spcue/test/dao/postgres/HostDaoTests.java | 39 +++++++++++++++++++ rqd/rqd/rqconstants.py | 3 ++ rqd/rqd/rqutil.py | 8 +++- 5 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V7__Increase_hostname_length.sql diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java index adee7843a..bf42f0ecb 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java @@ -658,6 +658,10 @@ private String getHostNameFromFQDN(String fqdn, Boolean useLongNames) { if (ipMatcher.matches()){ hostName = fqdn; } + else if (fqdn.contains(":")) { + // looks like IPv6 address. + hostName = fqdn; + } else if (useLongNames) { hostName = fqdn; Pattern domainPattern = Pattern.compile( diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V7__Increase_hostname_length.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V7__Increase_hostname_length.sql new file mode 100644 index 000000000..0dce0f2fd --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V7__Increase_hostname_length.sql @@ -0,0 +1,4 @@ +-- Increase the length of hostnames for IPv6 address. + +ALTER TABLE "host" ALTER COLUMN "str_name" TYPE VARCHAR(45); +ALTER TABLE "host_tag" ALTER COLUMN "str_tag" TYPE VARCHAR(45); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java index e24620323..9327fc8dd 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java @@ -203,6 +203,45 @@ public void testInsertHostFQDN4() { assertEquals(TEST_HOST_NEW, hostDetail.name); } + @Test + @Transactional + @Rollback(true) + public void testInsertHostIPv61() { + String TEST_HOST_NEW = "::1"; + hostDao.insertRenderHost(buildRenderHost(TEST_HOST_NEW), + hostManager.getDefaultAllocationDetail(), + false); + + HostEntity hostDetail = hostDao.findHostDetail(TEST_HOST_NEW); + assertEquals(TEST_HOST_NEW, hostDetail.name); + } + + @Test + @Transactional + @Rollback(true) + public void testInsertHostIPv62() { + String TEST_HOST_NEW = "ABCD:ABCD:ABCD:ABCD:ABCD:ABCD:ABCD:ABCD"; + hostDao.insertRenderHost(buildRenderHost(TEST_HOST_NEW), + hostManager.getDefaultAllocationDetail(), + false); + + HostEntity hostDetail = hostDao.findHostDetail(TEST_HOST_NEW); + assertEquals(TEST_HOST_NEW, hostDetail.name); + } + + @Test + @Transactional + @Rollback(true) + public void testInsertHostIPv63() { + String TEST_HOST_NEW = "ABCD:ABCD:ABCD:ABCD:ABCD:ABCD:192.168.100.180"; + hostDao.insertRenderHost(buildRenderHost(TEST_HOST_NEW), + hostManager.getDefaultAllocationDetail(), + false); + + HostEntity hostDetail = hostDao.findHostDetail(TEST_HOST_NEW); + assertEquals(TEST_HOST_NEW, hostDetail.name); + } + @Test @Transactional @Rollback(true) diff --git a/rqd/rqd/rqconstants.py b/rqd/rqd/rqconstants.py index 9ffd10273..0a1508e45 100644 --- a/rqd/rqd/rqconstants.py +++ b/rqd/rqd/rqconstants.py @@ -66,6 +66,7 @@ RQD_RETRY_STARTUP_CONNECT_DELAY = 30 RQD_RETRY_CRITICAL_REPORT_DELAY = 30 RQD_USE_IP_AS_HOSTNAME = True +RQD_USE_IPV6_AS_HOSTNAME = False RQD_BECOME_JOB_USER = True RQD_CREATE_USER_IF_NOT_EXISTS = True RQD_TAGS = '' @@ -187,6 +188,8 @@ LOAD_MODIFIER = config.getint(__section, "LOAD_MODIFIER") if config.has_option(__section, "RQD_USE_IP_AS_HOSTNAME"): RQD_USE_IP_AS_HOSTNAME = config.getboolean(__section, "RQD_USE_IP_AS_HOSTNAME") + if config.has_option(__section, "RQD_USE_IPV6_AS_HOSTNAME"): + RQD_USE_IPV6_AS_HOSTNAME = config.getboolean(__section, "RQD_USE_IPV6_AS_HOSTNAME") if config.has_option(__section, "RQD_BECOME_JOB_USER"): RQD_BECOME_JOB_USER = config.getboolean(__section, "RQD_BECOME_JOB_USER") if config.has_option(__section, "RQD_TAGS"): diff --git a/rqd/rqd/rqutil.py b/rqd/rqd/rqutil.py index 6bdf3e5ba..67a50abe0 100644 --- a/rqd/rqd/rqutil.py +++ b/rqd/rqd/rqutil.py @@ -143,7 +143,10 @@ def checkAndCreateUser(username): def getHostIp(): """Returns the machine's local ip address""" - return socket.gethostbyname(socket.gethostname()) + if rqd.rqconstants.RQD_USE_IPV6_AS_HOSTNAME: + return socket.getaddrinfo(socket.gethostname(), None, socket.AF_INET6)[0][4][0] + else: + return socket.gethostbyname(socket.gethostname()) def getHostname(): @@ -151,7 +154,8 @@ def getHostname(): try: if rqd.rqconstants.OVERRIDE_HOSTNAME: return rqd.rqconstants.OVERRIDE_HOSTNAME - elif rqd.rqconstants.RQD_USE_IP_AS_HOSTNAME: + elif rqd.rqconstants.RQD_USE_IP_AS_HOSTNAME or \ + rqd.rqconstants.RQD_USE_IPV6_AS_HOSTNAME: return getHostIp() else: return socket.gethostbyaddr(socket.gethostname())[0].split('.')[0] From 9795283eb97bb5bb84bb9ed7bd3062e3bff19ed1 Mon Sep 17 00:00:00 2001 From: ianianda Date: Sat, 23 Jan 2021 03:39:25 -0800 Subject: [PATCH 035/277] modified subprocess.check_output() and cuewho file dir path (#768) Co-authored-by: ianianda --- cuegui/cuegui/Utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cuegui/cuegui/Utils.py b/cuegui/cuegui/Utils.py index a2d1259d3..f41076f09 100644 --- a/cuegui/cuegui/Utils.py +++ b/cuegui/cuegui/Utils.py @@ -195,7 +195,7 @@ def getCuewho(show): @return: The username who is cuewho for the show @rtype: string""" try: - file = open("cuewho.who" % show, "r") + file = open("/shots/%s/home/cue/cuewho.who" % show, "r") return file.read() except Exception as e: logger.warning("Failed to update cuewho: %s\n%s" % (show, e)) @@ -214,7 +214,7 @@ def getExtension(username): try: # TODO: Replace this with a direct call to the phone util that the # phone widget uses once code is stable - results = subprocess.check_output('phone %s' % username) + results = subprocess.check_output(['phone', username]) for line in results.splitlines(): if line.find('Extension') != -1 and len(line.split()) == 2: From 62b07263e50637d6ea4d10df6c4042333bc568af Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sun, 24 Jan 2021 03:10:17 -0800 Subject: [PATCH 036/277] Fix cuebot (#884) * Fix migration version conflicts * Bump minor version number --- VERSION.in | 2 +- ...ase_hostname_length.sql => V8__Increase_hostname_length.sql} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename cuebot/src/main/resources/conf/ddl/postgres/migrations/{V7__Increase_hostname_length.sql => V8__Increase_hostname_length.sql} (100%) diff --git a/VERSION.in b/VERSION.in index 2eb3c4fe4..5a2a5806d 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.5 +0.6 diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V7__Increase_hostname_length.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V8__Increase_hostname_length.sql similarity index 100% rename from cuebot/src/main/resources/conf/ddl/postgres/migrations/V7__Increase_hostname_length.sql rename to cuebot/src/main/resources/conf/ddl/postgres/migrations/V8__Increase_hostname_length.sql From 174579af2496247cf223a3f63225b04c95ed156e Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 27 Jan 2021 20:45:09 +0000 Subject: [PATCH 037/277] Configure pylint for pyoutline. (#854) --- .github/workflows/testing-pipeline.yml | 2 +- ci/pylintrc_main | 526 +++++++++++++++++++++++++ ci/pylintrc_test | 520 ++++++++++++++++++++++++ ci/run_python_tests.sh | 6 + cuesubmit/cuesubmit/Submission.py | 23 +- pyoutline/Dockerfile | 4 - pyoutline/outline/__init__.py | 13 +- pyoutline/outline/backend/__init__.py | 15 + pyoutline/outline/backend/cue.py | 106 +++-- pyoutline/outline/backend/local.py | 91 +++-- pyoutline/outline/config.py | 4 + pyoutline/outline/cuerun.py | 27 +- pyoutline/outline/depend.py | 14 +- pyoutline/outline/event.py | 22 +- pyoutline/outline/exception.py | 5 - pyoutline/outline/executor.py | 13 +- pyoutline/outline/io.py | 35 +- pyoutline/outline/layer.py | 291 +++++++------- pyoutline/outline/loader.py | 165 ++++---- pyoutline/outline/modules/__init__.py | 6 +- pyoutline/outline/modules/shell.py | 58 ++- pyoutline/outline/plugins/__init__.py | 1 + pyoutline/outline/plugins/local.py | 78 ++-- pyoutline/outline/plugins/manager.py | 29 +- pyoutline/outline/session.py | 95 +++-- pyoutline/outline/versions/__init__.py | 7 +- pyoutline/outline/versions/main.py | 15 +- pyoutline/outline/versions/session.py | 62 ++- pyoutline/tests/__init__.py | 7 - pyoutline/tests/backend/__init__.py | 8 - pyoutline/tests/backend/cue_test.py | 21 +- pyoutline/tests/backend/local_test.py | 7 +- pyoutline/tests/depend_test.py | 27 +- pyoutline/tests/executor_test.py | 11 +- pyoutline/tests/json_test.py | 13 +- pyoutline/tests/layer_test.py | 77 ++-- pyoutline/tests/loader_test.py | 16 +- pyoutline/tests/modules/shell_test.py | 25 +- pyoutline/tests/session_test.py | 38 +- pyoutline/tests/test_utils.py | 6 + requirements.txt | 1 + 41 files changed, 1843 insertions(+), 647 deletions(-) create mode 100644 ci/pylintrc_main create mode 100644 ci/pylintrc_test diff --git a/.github/workflows/testing-pipeline.yml b/.github/workflows/testing-pipeline.yml index 68072e56c..5d8acd88a 100644 --- a/.github/workflows/testing-pipeline.yml +++ b/.github/workflows/testing-pipeline.yml @@ -35,7 +35,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Run Python Tests - run: ci/run_python_tests.sh + run: ci/run_python_tests.sh --lint test_cuebot_2020: name: Build Cuebot and Run Unit Tests (CY2020) diff --git a/ci/pylintrc_main b/ci/pylintrc_main new file mode 100644 index 000000000..f154d667b --- /dev/null +++ b/ci/pylintrc_main @@ -0,0 +1,526 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-whitelist= + +# Specify a score threshold to be exceeded before program exits with error. +fail-under=10.0 + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=duplicate-code, + fixme, + invalid-name, + locally-disabled, + raise-missing-from, + super-with-arguments, + too-few-public-methods, + too-many-arguments, + too-many-branches, + too-many-function-args, + too-many-instance-attributes, + too-many-lines, + too-many-locals, + too-many-public-methods, + too-many-return-statements, + too-many-statements, + useless-object-inheritance + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +# enable= + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'error', 'warning', 'refactor', and 'convention' +# which contain the number of messages in each category, as well as 'statement' +# which is the total number of statements analyzed. This score is used by the +# global evaluation report (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it work, +# install the python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +#notes-rgx= + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. +#class-attribute-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. +#variable-rgx= + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled). +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled). +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library=future,past + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + + +[DESIGN] + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "BaseException, Exception". +overgeneral-exceptions=BaseException, + Exception diff --git a/ci/pylintrc_test b/ci/pylintrc_test new file mode 100644 index 000000000..7c252cde0 --- /dev/null +++ b/ci/pylintrc_test @@ -0,0 +1,520 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-whitelist= + +# Specify a score threshold to be exceeded before program exits with error. +fail-under=10.0 + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=duplicate-code, + fixme, + invalid-name, + locally-disabled, + missing-class-docstring, + missing-function-docstring, + no-self-use, + protected-access, + raise-missing-from, + too-many-public-methods, + useless-object-inheritance + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +# enable= + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'error', 'warning', 'refactor', and 'convention' +# which contain the number of messages in each category, as well as 'statement' +# which is the total number of statements analyzed. This score is used by the +# global evaluation report (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it work, +# install the python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +#notes-rgx= + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. +#class-attribute-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. +#variable-rgx= + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled). +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled). +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + + +[DESIGN] + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "BaseException, Exception". +overgeneral-exceptions=BaseException, + Exception diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index 90b2c4c5e..bd51296db 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -19,3 +19,9 @@ PYTHONPATH=pycue python cueadmin/setup.py test PYTHONPATH=pycue xvfb-run -d python cuegui/setup.py test PYTHONPATH=pycue:pyoutline python cuesubmit/setup.py test python rqd/setup.py test + +# Some environments don't have pylint available, for ones that do they should pass this flag. +if [[ "$1" == "--lint" ]]; then + cd pyoutline && PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main outline && cd .. + cd pyoutline && PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests && cd .. +fi diff --git a/cuesubmit/cuesubmit/Submission.py b/cuesubmit/cuesubmit/Submission.py index b3ab583a3..e6d66402c 100644 --- a/cuesubmit/cuesubmit/Submission.py +++ b/cuesubmit/cuesubmit/Submission.py @@ -18,11 +18,12 @@ from __future__ import absolute_import from builtins import str -from outline import Outline, cuerun -from outline.modules.shell import Shell from cuesubmit import Constants from cuesubmit import JobTypes +import outline +import outline.cuerun +import outline.modules.shell def buildMayaCmd(layerData): @@ -52,6 +53,7 @@ def buildNukeCmd(layerData): renderCommand += '-x {}'.format(nukeFile) return renderCommand + def buildBlenderCmd(layerData): """From a layer, build a Blender render command.""" blenderFile = layerData.cmd.get('blenderFile') @@ -84,9 +86,9 @@ def buildLayer(layerData, command, lastLayer=None): threadable = True else: threadable = False - layer = Shell(layerData.name, command=command.split(), chunk=layerData.chunk, - threads=float(layerData.cores), range=str(layerData.layerRange), - threadable=threadable) + layer = outline.modules.shell.Shell( + layerData.name, command=command.split(), chunk=layerData.chunk, + threads=float(layerData.cores), range=str(layerData.layerRange), threadable=threadable) if layerData.services: layer.set_service(layerData.services[0]) if layerData.limits: @@ -113,14 +115,15 @@ def buildBlenderLayer(layerData, lastLayer): blenderCmd = buildBlenderCmd(layerData) return buildLayer(layerData, blenderCmd, lastLayer) + def buildShellLayer(layerData, lastLayer): return buildLayer(layerData, layerData.cmd['commandTextBox'], lastLayer) def submitJob(jobData): """Submit the job using the PyOutline API.""" - outline = Outline(jobData['name'], shot=jobData['shot'], show=jobData['show'], - user=jobData['username']) + ol = outline.Outline( + jobData['name'], shot=jobData['shot'], show=jobData['show'], user=jobData['username']) lastLayer = None for layerData in jobData['layers']: if layerData.layerType == JobTypes.JobTypes.MAYA: @@ -133,10 +136,10 @@ def submitJob(jobData): layer = buildBlenderLayer(layerData, lastLayer) else: raise ValueError('unrecognized layer type %s' % layerData.layerType) - outline.add_layer(layer) + ol.add_layer(layer) lastLayer = layer if 'facility' in jobData: - outline.set_facility(jobData['facility']) + ol.set_facility(jobData['facility']) - return cuerun.launch(outline, use_pycuerun=False) + return outline.cuerun.launch(ol, use_pycuerun=False) diff --git a/pyoutline/Dockerfile b/pyoutline/Dockerfile index 42b285110..60d04f725 100644 --- a/pyoutline/Dockerfile +++ b/pyoutline/Dockerfile @@ -59,13 +59,9 @@ COPY VERSION.in VERSIO[N] ./ RUN test -e VERSION || echo "$(cat VERSION.in)-custom" | tee VERSION RUN cd pycue && python setup.py install - RUN cd pycue && python3.6 setup.py install -# TODO(bcipriano) Lint the code here. (Issue #78) - RUN cd pyoutline && python setup.py test - RUN cd pyoutline && python3.6 setup.py test RUN cp LICENSE requirements.txt VERSION pyoutline/ diff --git a/pyoutline/outline/__init__.py b/pyoutline/outline/__init__.py index 83ea3c7ae..8982d6cc3 100644 --- a/pyoutline/outline/__init__.py +++ b/pyoutline/outline/__init__.py @@ -13,20 +13,18 @@ # limitations under the License. -"""Outline is a library for scripting shell commands to -be executed over a frame range. Typically these shell -commands would be executed in parallel on a render farm. """ - - -# TODO(bcipriano) Clean up this file - get rid of wildcard imports -# and don't collapse everything into the toplevel namespace like this. (Issue #151) +Outline is a library for scripting shell commands to be executed over a frame range. Typically +these shell commands would be executed in parallel on a render farm. +""" from __future__ import absolute_import from __future__ import print_function from __future__ import division +# pylint: disable=cyclic-import,redefined-builtin + from .config import config from .exception import * from .loader import * @@ -36,4 +34,3 @@ from .layer import * from . import cuerun from .plugins import PluginManager - diff --git a/pyoutline/outline/backend/__init__.py b/pyoutline/outline/backend/__init__.py index d9c80f13d..78922bf40 100644 --- a/pyoutline/outline/backend/__init__.py +++ b/pyoutline/outline/backend/__init__.py @@ -11,3 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +""" +PyOutline output modules. + +PyOutline can be thought of as separate from OpenCue proper -- it is just a job specification +after all, and could be used in any number of contexts. + +To this end, PyOutline supports launching jobs on any number of "backends", i.e. the system +responsible for processing the job -- launching the job / frames, storing job state, etc. + +The main backend of course is OpenCue (`outline.backend.cue`), which launches the job on OpenCue. +However this can be extended to support any job management system. We also include a "local" +backend (`outline.backend.local`) which just runs the job on the current machine, using a SQLite +database for storing state. +""" diff --git a/pyoutline/outline/backend/cue.py b/pyoutline/outline/backend/cue.py index 61d38e2ff..83e57400d 100644 --- a/pyoutline/outline/backend/cue.py +++ b/pyoutline/outline/backend/cue.py @@ -12,9 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +OpenCue backend module. -"""OpenCue integration module.""" +Uses the OpenCue Python API to submit the given job to OpenCue for processing. +See outline.backend.__init__.py for a description of the PyOutline backend system. +""" from __future__ import print_function from __future__ import division @@ -33,8 +37,11 @@ import FileSequence import opencue -from outline import config, util, versions, OutlineException -from outline.depend import DependType +import outline +import outline.depend +import outline.exception +import outline.util +import outline.versions.main logger = logging.getLogger("outline.backend.cue") @@ -48,9 +55,9 @@ def build_command(launcher, layer): """ - Build and return a pycurun shell command for the given layer + Build and return a pycuerun shell command for the given layer. - :type launcher : OutlineLauncher + :type launcher : outline.cuerun.OutlineLauncher :param launcher : The outline launcher. :type layer : Layer @@ -58,7 +65,7 @@ def build_command(launcher, layer): :rtype: list :return: The shell command to run for a the given layer. - """ + """ command = [] if layer.get_arg("strace"): @@ -74,17 +81,18 @@ def build_command(launcher, layer): if layer.get_arg("wrapper"): wrapper = layer.get_arg("wrapper") elif layer.get_arg("setshot", True): - wrapper = "%s/opencue_wrap_frame" % config.get("outline", "wrapper_dir") + wrapper = "%s/opencue_wrap_frame" % outline.config.get("outline", "wrapper_dir") else: - wrapper = "%s/opencue_wrap_frame_no_ss" % config.get("outline", "wrapper_dir") + wrapper = "%s/opencue_wrap_frame_no_ss" % outline.config.get( + "outline", "wrapper_dir") command.append(wrapper) - command.append(config.get("outline", "user_dir")) - command.append("%s/pycuerun" % config.get("outline", "bin_dir")) + command.append(outline.config.get("outline", "user_dir")) + command.append("%s/pycuerun" % outline.config.get("outline", "bin_dir")) command.append("%s -e #IFRAME#-%s" % (launcher.get_outline().get_path(), layer.get_name())) - command.append("--version %s" % versions.get_version("outline")) - command.append("--repos %s" % versions.get_repos()) + command.append("--version %s" % outline.versions.get_version("outline")) + command.append("--repos %s" % outline.versions.get_repos()) command.append("--debug") if launcher.get("dev"): @@ -111,7 +119,7 @@ def launch(launcher, use_pycuerun=True): if launcher.get("server"): opencue.Cuebot.setHosts([launcher.get("server")]) - logger.info("cuebot host set to: %s" % launcher.get("server")) + logger.info("cuebot host set to: %s", launcher.get("server")) jobs = opencue.api.launchSpecAndWait(launcher.serialize(use_pycuerun=use_pycuerun)) @@ -133,7 +141,7 @@ def test(job): :param job: The job to test. """ logging.basicConfig(level=logging.DEBUG) - logger.info("Entering test mode for job: %s" % job.data.name) + logger.info("Entering test mode for job: %s", job.data.name) # Unpause the job. job.resume() @@ -143,16 +151,16 @@ def test(job): try: job = opencue.api.getJob(job.name()) if job.data.job_stats.dead_frames + job.data.job_stats.eaten_frames > 0: - msg = "Job test failed, dead or eaten frames on: %s" - raise OutlineException(msg % job.data.name) + raise outline.exception.OutlineException( + "Job test failed, dead or eaten frames on: %s" % job.data.name) if job.data.state == opencue.api.job_pb2.FINISHED: break - msg = "waiting on %s job to complete: %d/%d" - logger.debug(msg % (job.data.name, job.data.job_stats.succeeded_frames, - job.data.job_stats.total_frames)) + logger.debug( + "waiting on %s job to complete: %d/%d", job.data.name, + job.data.job_stats.succeeded_frames, job.data.job_stats.total_frames) except opencue.CueException as ie: - raise OutlineException("test for job %s failed: %s" % - (job.data.name, ie)) + raise outline.exception.OutlineException( + "test for job %s failed: %s" % (job.data.name, ie)) time.sleep(5) finally: job.kill() @@ -169,23 +177,42 @@ def wait(job): try: if not opencue.api.isJobPending(job.data.name): break - msg = "waiting on %s job to complete: %d/%d" - logger.debug(msg % (job.data.name, job.data.job_stats.succeeded_frames, - job.data.job_stats.total_frames)) + logger.debug( + "waiting on %s job to complete: %d/%d", job.data.name, + job.data.job_stats.succeeded_frames, job.data.job_stats.total_frames) except opencue.CueException as ie: - msg = "opencue error waiting on job: %s, %s. Will continue to wait." - print(msg % (job.data.name, ie), file=sys.stderr) - except Exception as e: - msg = "opencue error waiting on job: %s, %s. Will continue to wait." - print(msg % (job.data.name, e), file=sys.stderr) + print( + "opencue error waiting on job: %s, %s. Will continue to wait." % ( + job.data.name, ie), + file=sys.stderr) time.sleep(JOB_WAIT_PERIOD_SEC) def serialize(launcher): + """ + Serialize the outline part of the given L{OutlineLauncher} into an OpenCue job specification, + using pycuerun to wrap the job commands. + + :type launcher: L{OutlineLauncher} + :param launcher: The outline launcher being used to launch the job. + + :rtype: str + :return: A opencue job specification. + """ return _serialize(launcher, use_pycuerun=True) def serialize_simple(launcher): + """ + Serialize the outline part of the given L{OutlineLauncher} into an OpenCue job specification, + skipping the pycuerun wrapper in favor of launching the job commands directly. + + :type launcher: L{OutlineLauncher} + :param launcher: The outline launcher being used to launch the job. + + :rtype: str + :return: A opencue job specification. + """ return _serialize(launcher, use_pycuerun=False) @@ -210,12 +237,12 @@ def _serialize(launcher, use_pycuerun): sub_element(root, "shot", launcher.get("shot")) user = launcher.get_flag("user") if not user: - user = util.get_user() + user = outline.util.get_user() sub_element(root, "user", user) if not launcher.get("nomail"): sub_element(root, "email", "%s@%s" % (user, - config.get("outline", "domain"))) - sub_element(root, "uid", str(util.get_uid())) + outline.config.get("outline", "domain"))) + sub_element(root, "uid", str(outline.util.get_uid())) j = Et.SubElement(root, "job", {"name": ol.get_name()}) sub_element(j, "paused", str(launcher.get("pause"))) @@ -255,8 +282,7 @@ def _serialize(launcher, use_pycuerun): frame_range = layer.get_frame_range() if not frame_range: logger.info("Skipping layer %s, its range (%s) does not intersect " - "with ol range %s" % (layer, layer.get_arg("range"), - ol.get_frame_range())) + "with ol range %s", layer, layer.get_arg("range"), ol.get_frame_range()) continue spec_layer = Et.SubElement(layers, "layer", @@ -309,15 +335,15 @@ def _serialize(launcher, use_pycuerun): service = Et.SubElement(services, "service") try: service.text = layer.get_service().split(",")[0].strip() - except Exception: + except (AttributeError, IndexError): service.text = "default" build_dependencies(ol, layer, depends) - if not len(layers): - raise OutlineException("Failed to launch job. There are no layers with frame " - "ranges that intersect the job's frame range: %s" - % ol.get_frame_range()) + if not layers: + raise outline.exception.OutlineException( + "Failed to launch job. There are no layers with frame " + "ranges that intersect the job's frame range: %s" % ol.get_frame_range()) # Dependencies go after all of the layers root.append(depends) @@ -365,7 +391,7 @@ def build_dependencies(ol, layer, all_depends): type=dep.get_type(), anyframe=bool_to_str(dep.is_any_frame())) - if dep.get_type() == DependType.LayerOnSimFrame: + if dep.get_type() == outline.depend.DependType.LayerOnSimFrame: frame_range = dep.get_depend_on_layer().get_frame_range() first_frame = FileSequence.FrameSet(frame_range)[0] diff --git a/pyoutline/outline/backend/local.py b/pyoutline/outline/backend/local.py index 2acee5e08..7aa90595a 100644 --- a/pyoutline/outline/backend/local.py +++ b/pyoutline/outline/backend/local.py @@ -12,21 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Local backend module. + +Runs the given job on the local machine, using a SQLite database to store state. + +See outline.backend.__init__.py for a description of the PyOutline backend system. +""" from __future__ import print_function from __future__ import division from __future__ import absolute_import from builtins import object -from builtins import range import subprocess import sqlite3 import FileSequence -from outline import config -from outline import versions +import outline +import outline.versions def build_command(ol, layer, frame): @@ -40,32 +46,34 @@ def build_command(ol, layer, frame): :return: The shell command to run for a the given layer. """ command = [] - command.append("%s/local_wrap_frame" % config.get("outline","wrapper_dir")) - command.append(config.get("outline", "user_dir")) + command.append("%s/local_wrap_frame" % outline.config.get("outline","wrapper_dir")) + command.append(outline.config.get("outline", "user_dir")) command.append(ol.get_show()) command.append(ol.get_shot()) - command.append("%s/pycuerun" % config.get("outline", "bin_dir")) + command.append("%s/pycuerun" % outline.config.get("outline", "bin_dir")) command.append("%s -e %d-%s" % (ol.get_path(), frame, layer.get_name())) - command.append(" -v %s" % versions.get_version("outline")) - command.append(" -r %s" % versions.get_repos()) + command.append(" -v %s" % outline.versions.get_version("outline")) + command.append(" -r %s" % outline.versions.get_repos()) command.append("-D") return command -def launch(launcher): +def launch(launcher, use_pycuerun=None): """ Start the local dispatcher. """ + # pycuerun is not used in this backend, but we keep it as a parameter for compatibility + # with other backends. + del use_pycuerun + dispatcher = serialize(launcher) dispatcher.dispatch() - return None def serialize(launcher): """ Create a local dispatcher object. - """ return Dispatcher(launcher.get_outline()) @@ -73,7 +81,6 @@ def serialize(launcher): def serialize_simple(launcher): """ For local we can call the regular serialize function. - """ return serialize(launcher) @@ -89,44 +96,58 @@ def build_frame_range(frame_range, chunk_size): frames.append(frame_set.__getitem__(0)) else: unique_frames = list(set(frame_set)) - for i in range(0, len(unique_frames)): + for i, unique_frame in enumerate(unique_frames): if i % chunk_size == 0: - frames.append(unique_frames[i]) + frames.append(unique_frame) else: frames = list(FileSequence.FrameSet(frame_range)) return frames +class LocalFrameError(Exception): + """ + Raised when a locally run frame has failed. + """ + + class Dispatcher(object): + """ + Local version of a job dispatcher, responsible for launching each frame, monitoring the + result, and updating the local state. + """ + def __init__(self, ol): self.__ol = ol self.__conn = sqlite3.connect(":memory:") self.__create_dispatch_list() - def dispatch(self): + """ + Run the frames of the job in sequence and record the result. + """ try: while True: l, f = self.__get_next_frame() + if l is None and f is None: + break + layer = self.__ol.get_layer(l) command = build_command(self.__ol, layer, f) try: retcode = subprocess.call(command, shell=False) if retcode != 0: - raise Exception("frame failed") - except Exception as e: + raise LocalFrameError("frame failed") + except LocalFrameError: # Failed to run frame # Set frame to dead - c = self.__conn.cursor() - c.execute("UPDATE frames SET state=? WHERE layer=? AND frame=?", - ('DEAD', l, f)) - self.__conn.commit() - - except Exception as e: - print("Job is done: %s" % e) + c = self.__conn.cursor() + c.execute("UPDATE frames SET state=? WHERE layer=? AND frame=?", + ('DEAD', l, f)) + self.__conn.commit() finally: + print("Job is done") self.__conn.close() def __create_dispatch_list(self): @@ -134,25 +155,26 @@ def __create_dispatch_list(self): Creates a list of dispatchable frames. """ c = self.__conn.cursor() - c.execute('''create table frames (layer text, frame int, state string, layer_order int, frame_order int)''') + c.execute( + 'create table frames ' + '(layer text, frame int, state string, layer_order int, frame_order int)') for layer in self.__ol.get_layers(): frames = build_frame_range(layer.get_frame_range(), layer.get_chunk_size()) for frame in frames: - c.execute('insert into frames values (?,?,?,?,?)', - (layer.get_name(), - frame, - 'WAITING', - self.__ol.get_layers().index(layer), - int(frame))) + c.execute( + 'insert into frames values (?,?,?,?,?)', + (layer.get_name(), frame, 'WAITING', self.__ol.get_layers().index(layer), + int(frame))) self.__conn.commit() def __get_next_frame(self): - c = self.__conn.cursor() - c.execute("SELECT layer, frame, state FROM frames WHERE state='WAITING' ORDER BY frame_order,layer_order LIMIT 1") + c.execute( + "SELECT layer, frame, state FROM frames WHERE state='WAITING' " + "ORDER BY frame_order,layer_order LIMIT 1") result = c.fetchone() if result: @@ -160,4 +182,5 @@ def __get_next_frame(self): ('RUNNING', result[0], result[1])) self.__conn.commit() return result[0], result[1] - raise Exception("Frame not found") + + return None, None diff --git a/pyoutline/outline/config.py b/pyoutline/outline/config.py index f06186d56..97fa13ee9 100644 --- a/pyoutline/outline/config.py +++ b/pyoutline/outline/config.py @@ -21,8 +21,12 @@ from __future__ import absolute_import from builtins import str + +# pylint: disable=wrong-import-position from future import standard_library standard_library.install_aliases() +# pylint: enable=wrong-import-position + import getpass import os import pathlib diff --git a/pyoutline/outline/cuerun.py b/pyoutline/outline/cuerun.py index 9eb116cd6..d1392dd4a 100644 --- a/pyoutline/outline/cuerun.py +++ b/pyoutline/outline/cuerun.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. - -"""Outline launching and frame execution utlities.""" - +""" +Outline launching and frame execution utilities. +""" from __future__ import absolute_import from __future__ import print_function @@ -24,6 +24,7 @@ import logging import os import re +# pylint: disable=deprecated-module from optparse import OptionParser, OptionGroup import FileSequence @@ -54,7 +55,7 @@ def import_backend_module(name): """ Imports the specified backend queuing system module, """ - logger.info("importing [%s] backend module." % name) + logger.info("importing [%s] backend module.", name) return __import__("outline.backend.%s" % name, globals(), locals(), [name]) @@ -182,7 +183,7 @@ def setup(self): # Frames dont' have a range by default. if isinstance(layer, Frame): continue - elif not layer.get_arg("range"): + if not layer.get_arg("range"): fully_baked = False break if not fully_baked: @@ -225,11 +226,10 @@ def serialize(self, use_pycuerun=True): self.setup() if use_pycuerun: return self.__get_backend_module().serialize(self) - else: - return self.__get_backend_module().serialize_simple(self) + return self.__get_backend_module().serialize_simple(self) def __get_backend_module(self): - if self.__backend == None: + if self.__backend is None: self.__backend = import_backend_module(self.get("backend")) return self.__backend @@ -315,18 +315,20 @@ def handle_standard_options(self, options, args): """ Handle standard options common to all cuerun based scripts. """ - logger.debug("Options: %s" % options) - logger.debug("Args: %s" % args) + logger.debug("Options: %s", options) + logger.debug("Args: %s", args) self.setup_frame_range(options, options.range) - def setup_frame_range(self, options, frange=None): + @staticmethod + def setup_frame_range(options, frange=None): """ Setup the frame range for the given job. """ range_default = False if not frange: if os.environ.get("FR"): + # pylint: disable=no-member if FileSequence.FrameSet.isSequence(os.environ.get("FR")): frange = os.environ.get("FR") range_default = True @@ -343,7 +345,8 @@ def add_plugin_option(self, *args, **kwargs): self.add_option_group(self.__plugin_grp) self.__plugin_grp.add_option(*args, **kwargs) - def options_to_args(self, options): + @staticmethod + def options_to_args(options): """ Convert an OptionParser namespace into a dictionary. """ diff --git a/pyoutline/outline/depend.py b/pyoutline/outline/depend.py index e1c538ed6..c2b83d59a 100644 --- a/pyoutline/outline/depend.py +++ b/pyoutline/outline/depend.py @@ -46,7 +46,7 @@ class DependType(object): PreviousFrame = "PREVIOUS_FRAME" LayerOnSimFrame = "LAYER_ON_SIM_FRAME" LayerOnAny = "LAYER_ON_ANY" - + # Short depend types used in require strings. all = LayerOnLayer any = LayerOnAny @@ -55,15 +55,15 @@ class DependType(object): def parse_require_str(require): """ - Parse a require string and returns its components. - A require string is short hand for defining dependencies - which contains the layer_name:depend_type. + Parse a require string and returns its components. + + A require string is short hand for defining dependencies which contains the + layer_name:depend_type. """ parts = str(require).split(":") if len(parts) == 1: return (parts[0], DependType.FrameByFrame) - else: - return (parts[0], getattr(DependType, parts[1])) + return (parts[0], getattr(DependType, parts[1])) class Depend(object): """A dependency""" @@ -78,7 +78,7 @@ def __init__(self, depend_er, depend_on, self.__type = depend_type self.__propigate = propigate self.__any_frame = any_frame - + def get_dependant_layer(self): """ Return the dependant layer. diff --git a/pyoutline/outline/event.py b/pyoutline/outline/event.py index 457352b28..39301bf1a 100644 --- a/pyoutline/outline/event.py +++ b/pyoutline/outline/event.py @@ -31,9 +31,6 @@ logger = logging.getLogger("outline.event") -# -# A List of event types. - EVENT_TYPES = ("AFTER_INIT", "AFTER_PARENTED", "SETUP", @@ -43,9 +40,9 @@ "BEFORE_LAUNCH") AFTER_INIT = EVENT_TYPES[0] -AFTER_PARENTED = EVENT_TYPES[1] -SETUP = EVENT_TYPES[2] -BEFORE_EXECUTE = EVENT_TYPES[3] +AFTER_PARENTED = EVENT_TYPES[1] +SETUP = EVENT_TYPES[2] +BEFORE_EXECUTE = EVENT_TYPES[3] AFTER_EXECUTE = EVENT_TYPES[4] AFTER_LAUNCH = EVENT_TYPES[5] BEFORE_LAUNCH = EVENT_TYPES[6] @@ -64,21 +61,23 @@ def add_event_listener(self, event_type, callback): Adds an event listener for the given type and callback function. """ - logger.debug("adding event listener %s" % event_type) + logger.debug("adding event listener %s", event_type) if event_type not in self.__listeners: self.__listeners[event_type] = [ ] self.__listeners[event_type].append(callback) + # pylint: disable=broad-except def emit(self, event): - logger.debug("fire event %s" % event) + """Fires an event, calling any registered listeners.""" + logger.debug("fire event %s", event) for callback in self.__listeners.get(event.type, []): try: callback(event) except FailImmediately as fi: - logger.debug("FailImmediately exception thrown, %s, %s" % (event.type, fi)) + logger.debug("FailImmediately exception thrown, %s, %s", event.type, fi) raise fi except Exception as e: - logger.debug("failed to execute event %s, %s" % (event.type, e)) + logger.debug("failed to execute event %s, %s", event.type, e) def get_event_listeners(self, event_type): """ @@ -90,6 +89,7 @@ def get_event_listeners(self, event_type): except KeyError: return list() + class LaunchEvent(object): """ A job launch event type. @@ -102,6 +102,7 @@ def __init__(self, event_type, cuerun, **args): def __str__(self): return str(self.__dict__) + class LayerEvent(object): """ A LayerEvent occurs within a layer. @@ -113,4 +114,3 @@ def __init__(self, event_type, layer, **args): def __str__(self): return str(self.__dict__) - diff --git a/pyoutline/outline/exception.py b/pyoutline/outline/exception.py index c7be1b35f..1408d9519 100644 --- a/pyoutline/outline/exception.py +++ b/pyoutline/outline/exception.py @@ -31,7 +31,6 @@ class OutlineException(Exception): """A General outline base exception.""" - pass class LayerException(OutlineException): @@ -40,7 +39,6 @@ class LayerException(OutlineException): This exception is raised when there is a problem setting up or executing a layer entity. """ - pass class ShellCommandFailureException(OutlineException): @@ -61,7 +59,6 @@ class SessionException(OutlineException): This exception is raised when there is a problem reading/writing/finding the job session. """ - pass class FailImmediately(OutlineException): @@ -69,11 +66,9 @@ class FailImmediately(OutlineException): Throwing this exception will fail the frame immediately, even from within a plugin """ - pass class FileSpecException(Exception): """ An exception to describe issues with the io.FileSpec class. """ - pass diff --git a/pyoutline/outline/executor.py b/pyoutline/outline/executor.py index 7d64d2abf..bd7c6bff7 100644 --- a/pyoutline/outline/executor.py +++ b/pyoutline/outline/executor.py @@ -20,13 +20,15 @@ from __future__ import division from __future__ import absolute_import +# pylint: disable=wrong-import-position from future import standard_library standard_library.install_aliases() + from builtins import range from builtins import object -import queue -import threading import logging +import threading +import queue __all__ = ["TaskExecutor"] @@ -35,11 +37,13 @@ class TaskExecutor(object): + """Simple thread pool.""" + def __init__(self, threads): self.__queue = queue.Queue() for i in range(0, threads): - logger.debug("executor creating thread #%d" % i) + logger.debug("executor creating thread #%d", i) t = threading.Thread(target=self.worker) t.setDaemon(True) t.start() @@ -69,6 +73,7 @@ def worker(self): item[0](*item[1]) else: item[0]() + # pylint: disable=broad-except except Exception as e: - logger.warn("Worker thread exception: %s" % e) + logger.warning("Worker thread exception: %s", e) self.__queue.task_done() diff --git a/pyoutline/outline/io.py b/pyoutline/outline/io.py index ff3c2923b..e5d7a5551 100644 --- a/pyoutline/outline/io.py +++ b/pyoutline/outline/io.py @@ -40,7 +40,7 @@ logger = logging.getLogger("outline.io") # Used to match version number in paths -VERSION_REGEX = re.compile("_v([\d+])") +VERSION_REGEX = re.compile(r'_v([\d+])') def prep_shell_command(cmd, frame=None): @@ -65,6 +65,7 @@ def prep_shell_command(cmd, frame=None): new_cmd.append(word) return new_cmd + def system(cmd, ignore_error=False, frame=None): """ Shell out to the given command and wait for it to finish. @@ -79,7 +80,7 @@ def system(cmd, ignore_error=False, frame=None): str_cmd = " ".join(map(str, cmd)) try: - logger.info("About to run: %s" % str_cmd) + logger.info("About to run: %s", str_cmd) retcode = subprocess.call(cmd, shell=False) if retcode != 0 and not ignore_error: msg = "shell out to '%s' failed, exit status %d" % (str_cmd, retcode) @@ -92,6 +93,7 @@ def system(cmd, ignore_error=False, frame=None): logger.critical(msg) raise ShellCommandFailureException(msg, 16) + def resolve(path): """ Resolve a realtive path or shot tree URI to a full path. @@ -156,6 +158,10 @@ def exists(self, frame_set=None): :rtype: boolean :return: true if path exists. """ + # frame_set isn't used here, but child classes need it, so we keep it here to keep + # args consistent + del frame_set + return os.path.exists(self.__path) def get_basename(self): @@ -177,8 +183,7 @@ def get_dirname(self): """ if os.path.isdir(self.__path): return self.__path - else: - return os.path.dirname(self.__path) + return os.path.dirname(self.__path) def get_ext(self): """ @@ -197,11 +202,15 @@ def get_path(self): :return: full path to this file """ return self.__path - - def get_size(self): + + def get_size(self, frame_set=None): """ Return the size of the file or path. """ + # frame_set isn't used here, but child classes need it, so we keep it here to keep + # args consistent + del frame_set + return os.path.getsize(self.__path) def __str__(self): @@ -210,18 +219,20 @@ def __str__(self): def __eq__(self, other): return str(self) == str(other) + class FileSpec(Path): """ A path to an image or sequence of images. Path must be a standard image path. """ + # pylint: disable=no-member def __init__(self, path, **args): Path.__init__(self, path, **args) try: self.__fs = FileSequence.FileSequence(self.get_path()) except ValueError as e: - logger.critical("Failed to parse spec: %s." % self.get_path()) + logger.critical("Failed to parse spec: %s.", self.get_path()) raise e if "mkdir" not in args: @@ -236,13 +247,13 @@ def exists(self, frame_set=None): :return: true if image(s) exist """ def exists(path): - logger.info("checking for existance of path: %s" % path) + logger.info("checking for existance of path: %s", path) if not os.path.exists(path): return False if os.path.getsize(path) == 0: return False return True - + if frame_set: for f in frame_set: path = self.get_frame_path(f) @@ -275,14 +286,14 @@ def get_size(self, frame_set=None): try: size += os.path.getsize(path) except OSError as e: - logger.warn("Failed to find the size of: %s, %s" % (path, e)) + logger.warning("Failed to find the size of: %s, %s", path, e) else: for path in self.__fs: try: size += os.path.getsize(path) except OSError as e: - logger.warn("Failed to find the size of: %s, %s" % (path, e)) - + logger.warning("Failed to find the size of: %s, %s", path, e) + return size def get_basename(self): diff --git a/pyoutline/outline/layer.py b/pyoutline/outline/layer.py index d3ce07dfd..b4e74a833 100644 --- a/pyoutline/outline/layer.py +++ b/pyoutline/outline/layer.py @@ -34,16 +34,13 @@ import FileSequence -from .config import config -from . import constants -from .depend import Depend -from .depend import DependType -from . import event -from .exception import LayerException -from .exception import SessionException -from . import io -from .loader import current_outline -from . import util +import outline +import outline.constants +import outline.depend +import outline.event +import outline.exception +import outline.io +import outline.util __all__ = ["Layer", @@ -64,23 +61,26 @@ class LayerType(type): """ def __call__(cls, *args, **kwargs): r = super(LayerType, cls).__call__(*args, **kwargs) - if current_outline() and r.get_arg("register"): - current_outline().add_layer(r) + if outline.current_outline() and r.get_arg("register"): + outline.current_outline().add_layer(r) # Initialize with plugin system. This is imported # here to get past a circular dependency. + # pylint: disable=import-outside-toplevel from outline.plugins import PluginManager for plugin in PluginManager.get_plugins(): try: plugin.init(r) - except AttributeError as e: + except AttributeError: pass return r + class Layer(with_metaclass(LayerType, object)): """The base class for all outline modules.""" def __init__(self, name, **args): + # pylint: disable=non-parent-init-called object.__init__(self) self.__name = name @@ -91,7 +91,7 @@ def __init__(self, name, **args): # Default the layer type to the Render type as # defined in the constants module self.__type = None - self.set_type(args.get("type", constants.LAYER_TYPES[0])) + self.set_type(args.get("type", outline.constants.LAYER_TYPES[0])) # A set of arguments that is required before # the Layer can be launched. @@ -120,7 +120,7 @@ def __init__(self, name, **args): # The default name of the service. self.__service = self.__args.get("service", "shell") - + # The layer limits. self.__limits = self.__args.get("limits") @@ -131,19 +131,19 @@ def __init__(self, name, **args): self.__outline = None # Register an event handler. - self.__evh = event.EventHandler(self) + self.__evh = outline.event.EventHandler(self) # Keep an array of all pre-process frames. self.__preprocess_layers = [] - logger.debug("module %s loaded from %s" % (self.__class__.__name__, - os.path.realpath(__file__))) + logger.debug( + "module %s loaded from %s", self.__class__.__name__, os.path.realpath(__file__)) + def _after_init(self, ol): """ - This method should be implmented by a subclass. Executed + This method should be implemented by a subclass. Executed after a layer has been initialized and added to an outline. """ - pass def after_init(self, ol): """ @@ -151,14 +151,13 @@ def after_init(self, ol): Emits an event.AFTER_INIT signal. """ self._after_init(ol) - self.__evh.emit(event.LayerEvent(event.AFTER_INIT, self)) + self.__evh.emit(outline.event.LayerEvent(outline.event.AFTER_INIT, self)) def _after_parented(self, parent): """ - This method should be implmented by a subclass. Executed after a + This method should be implemented by a subclass. Executed after a layer has been initialized and added as a child to another layer. """ - pass def after_parented(self, parent): """ @@ -166,21 +165,20 @@ def after_parented(self, parent): called if this Layer instance is parented to another layer instance. """ self._after_parented(parent) - self.__evh.emit(event.LayerEvent(event.AFTER_PARENTED, self)) + self.__evh.emit(outline.event.LayerEvent(outline.event.AFTER_PARENTED, self)) def _before_execute(self): """ - This method should be implemened by a subclass. Executed before + This method should be implemented by a subclass. Executed before all execute checks are started. """ - pass def before_execute(self): """ Executed before all execute checks are started. """ self._before_execute() - self.__evh.emit(event.LayerEvent(event.BEFORE_EXECUTE, self)) + self.__evh.emit(outline.event.LayerEvent(outline.event.BEFORE_EXECUTE, self)) def _after_execute(self): """ @@ -189,7 +187,6 @@ def _after_execute(self): Used for doing cleanup operations that should run even after a frame failure. """ - pass def after_execute(self): """ @@ -199,12 +196,13 @@ def after_execute(self): """ self._after_execute() frames = self.get_local_frame_set(self.__frame) - self.__evh.emit(event.LayerEvent(event.AFTER_EXECUTE, self, - frames=frames)) + self.__evh.emit(outline.event.LayerEvent( + outline.event.AFTER_EXECUTE, self, frames=frames)) - def system(self, cmd, ignore_error=False, frame=None): + @staticmethod + def system(cmd, ignore_error=False, frame=None): """ - A convinience method for calling io.system(). Shell out + A convenience method for calling io.system(). Shell out to the given command and wait for it to finish. @see: L{io.system} @@ -215,7 +213,7 @@ def system(self, cmd, ignore_error=False, frame=None): :type ignore_error: boolean :param ignore_error: Ignore any L{OSError} or shell command failures. """ - io.system(cmd, ignore_error, frame) + outline.io.system(cmd, ignore_error, frame) def get_default_args(self, merge=None): """ @@ -244,8 +242,8 @@ def get_default_args(self, merge=None): # Now apply any settings found in the configuration file. # This settings override the procedural defaults set in # the layer constructur using default_arg method. - if config.has_section(self.__class__.__name__): - for key, value in config.items(self.__class__.__name__): + if outline.config.has_section(self.__class__.__name__): + for key, value in outline.config.items(self.__class__.__name__): defaults[key] = value # Now apply user supplied arguments. These arguments override @@ -263,7 +261,7 @@ def set_parent(self, layer): """Set the parent layer.""" if not isinstance(layer, (Layer)): - raise LayerException("Parent instance must derive from Layer.") + raise outline.exception.LayerException("Parent instance must derive from Layer.") self.__parent = layer @@ -273,7 +271,7 @@ def add_child(self, layer): executed after the parent layer. """ if not isinstance(layer, (Layer)): - raise LayerException("Child instances must derive from Layer.") + raise outline.exception.LayerException("Child instances must derive from Layer.") layer.set_outline(self.get_outline()) layer.set_parent(self) @@ -281,8 +279,8 @@ def add_child(self, layer): self.__children.append(layer) layer.after_parented(self) - def add_event_listener(self, event_type, callback): + """Add an event listener to the layer's event handler.""" self.__evh.add_event_listener(event_type, callback) def get_event_handler(self): @@ -298,8 +296,8 @@ def get_children(self): def set_env(self, key, value): """Set an env var to be set before execute.""" if key in self.__env: - logger.warn("Overwriting outline env var: %s, from %s to %s", - key, self.__env[key], value) + logger.warning( + "Overwriting outline env var: %s, from %s to %s", key, self.__env[key], value) self.__env[str(key)] = str(value) def get_env(self, key, default=None): @@ -325,7 +323,7 @@ def set_name(self, name): """ if self.__outline and self.__outline.get_mode() > 1: msg = "Layer names may only be changed in outline init mode." - raise LayerException(msg) + raise outline.exception.LayerException(msg) self.__name = name def get_type(self): @@ -343,21 +341,20 @@ def set_type(self, t): """ Sets the general scope/purpose of this layer. """ - if t not in constants.LAYER_TYPES: - raise LayerException("%s is not a valid layer type: %s" % ( - t, constants.LAYER_TYPES)) + if t not in outline.constants.LAYER_TYPES: + raise outline.exception.LayerException("%s is not a valid layer type: %s" % ( + t, outline.constants.LAYER_TYPES)) self.__type = t def get_outline(self): """Return the parent outline object.""" if self.__parent: return self.__parent.get_outline() - else: - return self.__outline + return self.__outline - def set_outline(self, outline): + def set_outline(self, new_outline): """Set this layer's parent outline to the given outline object.""" - self.__outline = outline + self.__outline = new_outline def setup(self): """Setup is run once before the job is launched @@ -371,15 +368,13 @@ def setup(self): child.setup() # Emit the setup event. - self.__evh.emit(event.LayerEvent(event.SETUP, self)) + self.__evh.emit(outline.event.LayerEvent(outline.event.SETUP, self)) def _setup(self): """This method should be implemented by a subclass.""" - pass - def _execute(self, frame_set): + def _execute(self, frames): """This method should be implemented by a subclass.""" - pass def execute(self, frame): """ @@ -434,23 +429,23 @@ def execute(self, frame): logger.info("Setting post-set shot environement var: %s %s", env_k, env_v) os.environ[str(env_k)] = str(env_v) - except AttributeError as e: + except AttributeError: pass - logger.info("Layer %s executing local frame set %s" - % (self.get_name(), frames)) + logger.info("Layer %s executing local frame set %s", self.get_name(), frames) # Run the subclasses execute method and all child execute methods self._execute(frames) for child in self.__children: child.execute(frame) - # Run the subclasse's _post_execute method + # Run the subclass's _post_execute method self.after_execute() - ## Check the existance of required output + # Check the existance of required output self.check_output(frames) + # pylint: disable=broad-except,no-member def setup_args_override(self): """ Load the args_override data from the session and set them as the args. @@ -468,11 +463,11 @@ def setup_args_override(self): # This was necessary because plugins/s3d.py uses get_creator() if hasattr(self, 'get_creator') and self.get_creator(): self.get_creator().set_arg(key, value) - logger.warn('Replaced arg %s with %s' % (key, value)) - except SessionException as e: + logger.warning('Replaced arg %s with %s', key, value) + except outline.exception.SessionException: logger.debug('args_override not found in session (This is normal)') except Exception as e: - logger.debug('Not loading args_override from session due to %s' % e) + logger.debug('Not loading args_override from session due to %s', e) def set_default_arg(self, key, value): """ @@ -506,11 +501,10 @@ def set_arg(self, key, value): if not isinstance(value, rtype): msg = "The arg %s for the %s module must be a %s" - raise LayerException(msg % (arg, + raise outline.exception.LayerException(msg % (arg, self.__class__.__name__, rtype)) - else: - break + break self.__args[key] = value def is_arg_set(self, key): @@ -594,7 +588,7 @@ def get_data(self, key): """ return self.__outline.get_session().get_data(key, self) - def sym_file(self, src, layer=None, rename=None): + def sym_file(self, src, rename=None): """ Symlink the given file into the layer's session path. If the optional rename argument is set, the file will be @@ -664,10 +658,12 @@ def require_arg(self, key, rtype=None): def set_frame_range(self, frame_range): """Set the layer's frame range.""" - logger.debug("layer %s changing range from %s to %s" % - (self.get_name(), self.__args["range"], frame_range)) + logger.debug( + "layer %s changing range from %s to %s", self.get_name(), self.__args["range"], + frame_range) self.__args["range"] = str(frame_range) + # pylint: disable=inconsistent-return-statements def get_frame_range(self): """ Return the layer's frame range. If the layer and its @@ -699,31 +695,30 @@ def get_frame_range(self): ol_rng = FileSequence.FrameSet(self.__outline.get_frame_range()) ly_rng = FileSequence.FrameSet(rng) - intersect = util.intersect_frame_set(ol_rng, ly_rng, normalize=False) + intersect = outline.util.intersect_frame_set(ol_rng, ly_rng, normalize=False) if not intersect: return None - # If nomalizing does not change the order of frames, return normalized + # If normalizing does not change the order of frames, return normalized normalized = FileSequence.FrameSet(str(intersect)) normalized.normalize() if list(intersect) == list(normalized): return str(normalized) return str(intersect) - elif rng: + if rng: return rng - elif not rng and self.__outline.get_frame_range(): + if not rng and self.__outline.get_frame_range(): return self.__outline.get_frame_range() - elif not rng and not self.__outline.get_frame_range(): + if not rng and not self.__outline.get_frame_range(): return DEFAULT_FRAME_RANGE else: # There is no parent outline if rng: return rng - else: - return DEFAULT_FRAME_RANGE + return DEFAULT_FRAME_RANGE def get_local_frame_set(self, start_frame): """ @@ -738,33 +733,32 @@ def get_local_frame_set(self, start_frame): chunk = self.get_chunk_size() if chunk == 1: - return util.make_frame_set([int(start_frame)]) - else: - local_frame_set = [] - # - # Remove the duplicates out of our frame range. - # - frame_range = FileSequence.FrameSet(self.get_frame_range()) - frame_set = util.deaggregate_frame_set(frame_range) - - # - # Now find the index for the current frame and start - # frame there. Find all of frames this instance - # is responsible for. - # - idx = frame_set.index(int(start_frame)) - for i in range(idx, idx + chunk): - try: - if frame_set[i] in local_frame_set: - continue - local_frame_set.append(frame_set[i]) - except IndexError: - break - if not local_frame_set: - raise LayerException("Frame %d is outside of the frame range." - % start_frame) - else: - return util.make_frame_set(local_frame_set) + return outline.util.make_frame_set([int(start_frame)]) + + local_frame_set = [] + # + # Remove the duplicates out of our frame range. + # + frame_range = FileSequence.FrameSet(self.get_frame_range()) + frame_set = outline.util.deaggregate_frame_set(frame_range) + + # + # Now find the index for the current frame and start + # frame there. Find all of frames this instance + # is responsible for. + # + idx = frame_set.index(int(start_frame)) + for i in range(idx, idx + chunk): + try: + if frame_set[i] in local_frame_set: + continue + local_frame_set.append(frame_set[i]) + except IndexError: + break + if not local_frame_set: + raise outline.exception.LayerException( + "Frame %d is outside of the frame range." % start_frame) + return outline.util.make_frame_set(local_frame_set) def set_chunk_size(self, size): """ @@ -792,7 +786,7 @@ def depend_previous(self, on_layer): :type on_layer: L{Layer} :param on_layer: The L{Layer} to depend on. """ - self.depend_on(on_layer, DependType.PreviousFrame) + self.depend_on(on_layer, outline.depend.DependType.PreviousFrame) def depend_all(self, on_layer, propigate=False, any_frame=False): """ @@ -810,9 +804,9 @@ def depend_all(self, on_layer, propigate=False, any_frame=False): Default to False. """ self.depend_on(self.__resolve_layer_name(on_layer), - DependType.LayerOnLayer, propigate, any_frame) + outline.depend.DependType.LayerOnLayer, propigate, any_frame) - def depend_on(self, on_layer, depend_type=DependType.FrameByFrame, + def depend_on(self, on_layer, depend_type=outline.depend.DependType.FrameByFrame, propigate=False, any_frame=False): """ Setup a frame by frame on layer depend on the given layer. @@ -831,48 +825,47 @@ def depend_on(self, on_layer, depend_type=DependType.FrameByFrame, # Check for duplicates. for depend in self.__depends: if depend.get_depend_on_layer() == on_layer: - logger.info("Skipping duplicated depend %s on %s" % - (self, on_layer)) + logger.info("Skipping duplicated depend %s on %s", self, on_layer) return if str(self) == str(on_layer): - logger.info("Skipping setting up dependency on self %s" % self) + logger.info("Skipping setting up dependency on self %s", self) return try: on_layer = self.__resolve_layer_name(on_layer) - except LayerException as e: - logger.warn("%s layer does not exist, depend failed" % on_layer) + except outline.exception.LayerException: + logger.warning("%s layer does not exist, depend failed", on_layer) return - logger.info("adding depend %s on %s" % (self, on_layer)) + logger.info("adding depend %s on %s", self, on_layer) # # Handle the depend any bullshit # - if any_frame or depend_type == DependType.LayerOnAny: + if any_frame or depend_type == outline.depend.DependType.LayerOnAny: if isinstance(self, LayerPreProcess): - depend_type = DependType.LayerOnLayer + depend_type = outline.depend.DependType.LayerOnLayer else: - depend_type = DependType.FrameByFrame + depend_type = outline.depend.DependType.FrameByFrame any_frame = False for pre in self.get_preprocess_layers(): pre.depend_on(on_layer, - DependType.LayerOnLayer, + outline.depend.DependType.LayerOnLayer, any_frame=True) - depend = Depend(self, on_layer, depend_type, propigate, any_frame) + depend = outline.depend.Depend(self, on_layer, depend_type, propigate, any_frame) self.__depends.append(depend) # Setup pre-process dependencies for my_preprocess in self.get_preprocess_layers(): for on_preprocess in on_layer.get_preprocess_layers(): # Depend on the layer's pre-process - my_preprocess.depend_on(on_preprocess, DependType.LayerOnLayer) + my_preprocess.depend_on(on_preprocess, outline.depend.DependType.LayerOnLayer) # - # Handle depend propigation. + # Handle depend propagation. # - # Propigation occurs when a layer A depends on layer B, and + # Propagation occurs when a layer A depends on layer B, and # layer C depends on layer D, but Layer A also depends on Layer # C, which means layer D must now also depend on layer B. # @@ -883,8 +876,8 @@ def depend_on(self, on_layer, depend_type=DependType.FrameByFrame, if depend.is_propigated(): for my_depend in self.get_depends(): dependant = my_depend.get_depend_on_layer() - logger.info("propigating dependency %s -> %s" % - (dependant, depend.get_depend_on_layer())) + logger.info( + "propagating dependency %s -> %s", dependant, depend.get_depend_on_layer()) dependant.depend_all(depend.get_depend_on_layer(), propigate=False) @@ -896,12 +889,12 @@ def undepend(self, depend): try: self.__depends.remove(depend) except Exception as e: - logger.warn("failed to remove dependency %s, %s" % (depend, e)) + logger.warning("failed to remove dependency %s, %s", depend, e) def get_depends(self): """Return a tuple of dependencies this layer depends on.""" # Do not let people muck with the real list. - return (self.__depends) + return self.__depends def get_dependents(self): """Return a list of dependencies that depend on this layer.""" @@ -922,7 +915,7 @@ def check_input(self, frame_set=None): continue if not inpt.exists(frame_set): msg = "Check input failed (%s), the path %s does not exist." - raise LayerException(msg % (name, inpt.get_path())) + raise outline.exception.LayerException(msg % (name, inpt.get_path())) def check_output(self, frame_set=None): """ @@ -936,7 +929,7 @@ def check_output(self, frame_set=None): continue if not output.exists(frame_set): msg = "Check output failed (%s), the path %s does not exist." - raise LayerException(msg % (name, output.get_path())) + raise outline.exception.LayerException(msg % (name, output.get_path())) def add_input(self, name, inpt): """ @@ -947,10 +940,10 @@ def add_input(self, name, inpt): name = str(name) if name in self.__input: msg = "An input with the name %s has already been created." - raise LayerException(msg % name) + raise outline.exception.LayerException(msg % name) - if not isinstance(inpt, io.Path): - inpt = io.Path(inpt) + if not isinstance(inpt, outline.io.Path): + inpt = outline.io.Path(inpt) self.__input[name] = inpt @@ -963,10 +956,10 @@ def add_output(self, name, output): name = str(name) if name in self.__output: msg = "An output with the name %s has already been created." - raise LayerException(msg % name) + raise outline.exception.LayerException(msg % name) - if not isinstance(output, io.Path): - output = io.Path(output) + if not isinstance(output, outline.io.Path): + output = outline.io.Path(output) self.__output[name] = output @@ -998,8 +991,9 @@ def get_input(self, name): try: return self.__input[name] except: - raise LayerException("An input by the name %s does not exist." - % name) + raise outline.exception.LayerException( + "An input by the name %s does not exist." % name) + def get_output(self, name): """ Return the named output. @@ -1010,14 +1004,14 @@ def get_output(self, name): try: return self.__output[name] except: - raise LayerException("An output by the name %s does not exist." - % name) + raise outline.exception.LayerException( + "An output by the name %s does not exist." % name) def set_output_attribute(self, name, value): """ Set the given attribute on all registered output. """ - logger.debug("Setting output attribute: %s = %s" % (name, value)) + logger.debug("Setting output attribute: %s = %s", name, value) for output in self.__output.values(): output.set_attribute(name, value) @@ -1025,11 +1019,12 @@ def set_input_attribute(self, name, value): """ Set the given attribute on all registered input. """ - logger.debug("Setting input attribute: %s = %s" % (name, value)) + logger.debug("Setting input attribute: %s = %s", name, value) for output in self.__input.values(): output.set_attribute(name, value) - def get_temp_dir(self): + @staticmethod + def get_temp_dir(): """ Return the path to the current temporary directory. """ @@ -1043,11 +1038,11 @@ def check_required_args(self): for key, rtype in self.__req_args: if key not in self.__args: msg = "The %s layer requires the %s property to be set." - raise LayerException(msg % (self, key)) + raise outline.exception.LayerException(msg % (self, key)) if rtype: if not isinstance(self.__args[key], rtype): msg = "The %s layer requires %s to be of the type %s" - raise LayerException(msg % (self, key, rtype)) + raise outline.exception.LayerException(msg % (self, key, rtype)) def get_preprocess_layers(self): """ @@ -1078,9 +1073,9 @@ def __set_python_path(self): This method is for testing python libraries before release.. """ if self.get_arg("python_path"): - logger.warn("WARNING: PYTHON PATH HAS BEEN ADJUSTED") + logger.warning("WARNING: PYTHON PATH HAS BEEN ADJUSTED") if not isinstance(self.get_arg("python_path"), list): - logger.warn("python_path should be a list.") + logger.warning("python_path should be a list.") return sys.path = self.get_arg("python_path") + sys.path @@ -1137,14 +1132,12 @@ def get_frame_range(self): if self.get_outline().get_frame_range(): seq = FileSequence.FrameSet(self.get_outline().get_frame_range()) return str(seq[0]) - else: - return DEFAULT_FRAME_RANGE + return DEFAULT_FRAME_RANGE def set_frame_range(self, frame_range): """ Calling this method does nothing. """ - pass class LayerPreProcess(Frame): @@ -1160,7 +1153,7 @@ def __init__(self, creator, **args): args.get("suffix","preprocess")), **args) self.__creator = creator - self.__creator.depend_on(self, DependType.LayerOnLayer, propigate=False) + self.__creator.depend_on(self, outline.depend.DependType.LayerOnLayer, propigate=False) self.__creator.add_preprocess_layer(self) self.set_type("Util") @@ -1175,7 +1168,7 @@ def execute(self, frame): Perform pre-propcess execute methods and call the super class's exceute method. """ - super(LayerPreProcess, self).execute(frame) + super().execute(frame) self.__save_outputs() def get_frame_range(self): @@ -1203,14 +1196,14 @@ def __save_outputs(self): load_outputs before execution which will load in all output data saved out by this method. """ - if not len(self.get_outputs()): + if not self.get_outputs(): return - logger.info("Saving %d outputs to ol:outputs" - % len(self.get_outputs())) + logger.info("Saving %d outputs to ol:outputs", len(self.get_outputs())) self.get_creator().put_data("ol:outputs", self.get_outputs(), force=True) + class LayerPostProcess(Frame): """ A subclass of Frame which always runs after the @@ -1222,7 +1215,7 @@ def __init__(self, creator, propigate=True, **args): Frame.__init__(self, "%s_postprocess" % creator.get_name(), **args) self.__creator = creator - self.depend_on(creator, DependType.LayerOnLayer, propigate=propigate) + self.depend_on(creator, outline.depend.DependType.LayerOnLayer, propigate=propigate) self.set_type("Util") diff --git a/pyoutline/outline/loader.py b/pyoutline/outline/loader.py index 8bb66e988..f58ad46ea 100644 --- a/pyoutline/outline/loader.py +++ b/pyoutline/outline/loader.py @@ -20,12 +20,12 @@ from __future__ import print_function from __future__ import division -from past.builtins import execfile from builtins import str from builtins import object import os import logging import json +from past.builtins import execfile import time import uuid import yaml @@ -34,13 +34,12 @@ import FileSequence -from . import constants -from .depend import parse_require_str -from .exception import OutlineException -from .exception import SessionException -from .session import is_session_path -from .session import Session -from . import util +import outline.constants +import outline.depend +import outline.exception +# pylint: disable=cyclic-import +import outline.session +import outline.util logger = logging.getLogger("outline.loader") @@ -64,15 +63,17 @@ def load_outline(path): outline script. :rtype: Outline - :return: The resutling Outline object. + :return: The resulting Outline object. """ - logger.info("loading outline: %s" % path) + logger.info("loading outline: %s", path) # The wan cores may not see this file right away # The avere cache will hold a negative cache for 30 seconds, extending every time it is checked # The local cache will cache for 30 seconds as well if path and not os.path.exists(path): - logger.info('Outline file does not exist, sleeping 35 seconds before checking again due to possible file cache latency.') + logger.info( + 'Outline file does not exist, sleeping 35 seconds before checking again due to ' + 'possible file cache latency.') time.sleep(35) ext = os.path.splitext(path) @@ -81,18 +82,19 @@ def load_outline(path): ol = yaml.load(file_object, Loader=yaml.FullLoader) Outline.current = ol if not isinstance(ol, Outline): - raise OutlineException("The file %s did not produce " + raise outline.exception.OutlineException("The file %s did not produce " "an Outline object." % path) else: # If the file is not .yaml, assume its a python script ol = Outline(path=path, current=True) # If the script is inside of a session - if is_session_path(path): + if outline.session.is_session_path(path): ol.load_session() return ol + def load_json(json_str): """ Parse a json repesentation of an outline file. @@ -110,7 +112,7 @@ def decode_layer(layer): arguments in class constructors. For this reason, they must be converted to byte strings. """ - result = { } + result = {} for k, v in layer.items(): result[str(k)] = v del result["module"] @@ -126,6 +128,7 @@ def decode_layer(layer): ol.set_frame_range(data["range"]) for layer in data["layers"]: + s_class = None try: # Get the fully qualified module name, foo.bar.blah s_module = ".".join(layer["module"].split(".")[0:-1]) @@ -134,16 +137,16 @@ def decode_layer(layer): # Import the module and instantiate the class. module = __import__(s_module, globals(), locals(), [s_class]) - cls = getattr(module, s_class) + cls = getattr(module, s_class) cls(layer["name"], **decode_layer(layer)) except KeyError: error = "Json error, layer missing 'name' or 'module' definition" - raise OutlineException(error) + raise outline.exception.OutlineException(error) except Exception as e: msg = "Failed to load plugin: %s , %s" - raise OutlineException(msg % (s_class, e)) + raise outline.exception.OutlineException(msg % (s_class, e)) return ol @@ -163,12 +166,12 @@ def parse_outline_script(path): :param path: The path to the outline file. """ try: - logger.info("parsing outline file %s" % path) + logger.info("parsing outline file %s", path) execfile(path, {}) except Exception as exp: - logger.warn("failed to parse as python file, %s" % exp) - raise OutlineException("failed to parse outline file: %s, %s" % - (path, exp)) + logger.warning("failed to parse as python file, %s", exp) + raise outline.exception.OutlineException( + "failed to parse outline file: %s, %s" % (path, exp)) def current_outline(): @@ -203,6 +206,7 @@ def quick_outline(layer): ol.add_layer(layer) return ol + class Outline(object): """The outline class represents a single outline script.""" @@ -294,7 +298,7 @@ def __init__(self, name=None, frame_range=None, path=None, # # See contsants for the description of outline modes # - self.__mode = constants.OUTLINE_MODE_INIT + self.__mode = outline.constants.OUTLINE_MODE_INIT # # Stores the array of layers for this outline. To @@ -371,11 +375,11 @@ def load_session(self): """ Reloads the session """ - if is_session_path(self.get_path()): - self.__session = Session(self) + if outline.session.is_session_path(self.get_path()): + self.__session = outline.session.Session(self) else: msg = "failed to load outline %s, not part of a session." - raise OutlineException(msg % self.get_path()) + raise outline.exception.OutlineException(msg % self.get_path()) def setup(self): """ @@ -391,13 +395,13 @@ def setup(self): - Sets the outline state to READY. """ - if self.__mode >= constants.OUTLINE_MODE_SETUP: - raise OutlineException("This outline is already setup.") + if self.__mode >= outline.constants.OUTLINE_MODE_SETUP: + raise outline.exception.OutlineException("This outline is already setup.") self.setup_depends() - self.__mode = constants.OUTLINE_MODE_SETUP - self.__session = Session(self) + self.__mode = outline.constants.OUTLINE_MODE_SETUP + self.__session = outline.session.Session(self) # Run setup() for every layer assuming the frame range # can be determined. If there is no frame range, the layer @@ -415,7 +419,7 @@ def setup(self): "outline.yaml") # Set a new path before serialzing the outline file. - logger.info("setting new outline path: %s" % yaml_file) + logger.info("setting new outline path: %s", yaml_file) self.set_path(yaml_file) # Copy the session over to a local variable and unset @@ -436,12 +440,11 @@ def setup(self): logger.info("copying outline script to session path.") self.__session.put_file(self.get_path(), None, "script.outline") else: - raise OutlineException("Failed to serialize outline, " - "Procedural outlines must always " - "use serialization.") + raise outline.exception.OutlineException( + "Failed to serialize outline, Procedural outlines must always use serialization.") # Set our new mode and save. - self.set_mode(constants.OUTLINE_MODE_READY) + self.set_mode(outline.constants.OUTLINE_MODE_READY) self.__session.save() def setup_depends(self): @@ -454,20 +457,20 @@ def setup_depends(self): # Setup dependencies passed in via the layer's require argument. if layer.get_arg("require", False): if not isinstance(layer.get_arg("require"), (tuple, list, set)): - require, dtype = parse_require_str(layer.get_arg("require")) + require, dtype = outline.depend.parse_require_str(layer.get_arg("require")) try: layer.depend_on(self.get_layer(require), dtype) - except OutlineException as e: - logger.warn("Invalid layer in depend %s, skipping" % require) + except outline.exception.OutlineException: + logger.warning("Invalid layer in depend %s, skipping", require) continue else: # Process the require argument. for require in layer.get_arg("require"): - require, dtype = parse_require_str(require) + require, dtype = outline.depend.parse_require_str(require) try: layer.depend_on(self.get_layer(str(require)), dtype) - except OutlineException as e: - logger.warn("Invalid layer in depend %s, skipping" % require) + except outline.exception.OutlineException: + logger.warning("Invalid layer in depend %s, skipping", require) continue def add_layer(self, layer): @@ -477,13 +480,12 @@ def add_layer(self, layer): return if layer in self.__layers: - logger.info("The layer %s was already added to this outline." % - layer.get_name()) + logger.info("The layer %s was already added to this outline.", layer.get_name()) return if self.is_layer(layer.get_name()): - raise OutlineException("The layer %s already exists" - % layer.get_name()) + raise outline.exception.OutlineException( + "The layer %s already exists" % layer.get_name()) self.__layers.append(layer) layer.set_outline(self) @@ -498,17 +500,17 @@ def add_layer(self, layer): pass # If we're in setup mode, run setup ASAP - if self.__mode == constants.OUTLINE_MODE_SETUP: + if self.__mode == outline.constants.OUTLINE_MODE_SETUP: layer.setup() - logger.info("adding layer: %s" % layer.get_name()) + logger.info("adding layer: %s", layer.get_name()) def remove_layer(self, layer): """Remove an existing layer.""" - if self.__mode >= constants.OUTLINE_MODE_SETUP: + if self.__mode >= outline.constants.OUTLINE_MODE_SETUP: msg = "Cannot remove layers to an outline not in init mode." - raise OutlineException(msg) + raise outline.exception.OutlineException(msg) if layer in self.__layers: self.__layers.remove(layer) @@ -516,11 +518,11 @@ def remove_layer(self, layer): def get_layer(self, name): """Return an later by name.""" - layer_map = dict([(evt.get_name(), evt) for evt in self.__layers]) + layer_map = {evt.get_name(): evt for evt in self.__layers} try: return layer_map[name] except Exception as e: - raise OutlineException("invalid layer name: %s, %s" % (name, e)) + raise outline.exception.OutlineException("invalid layer name: %s, %s" % (name, e)) def get_layers(self): """Return the outline's layers @@ -532,7 +534,7 @@ def get_layers(self): def is_layer(self, name): """Return true if a layer exists with the specified name.""" - layer_map = dict([(evt.get_name(), evt) for evt in self.__layers]) + layer_map = {evt.get_name(): evt for evt in self.__layers} return name in layer_map def get_path(self): @@ -567,18 +569,16 @@ def get_full_name(self): """ if self.__session: return self.get_session().get_name().split("/")[0] - else: - return "%s-%s-%s_%s" % (self.get_show(), - self.get_shot(), - self.get_user(), - self.get_name()) + return "%s-%s-%s_%s" % (self.get_show(), + self.get_shot(), + self.get_user(), + self.get_name()) def get_shot(self): """Return the shot for this outline.""" if self.__shot is None: - return util.get_shot() - else: - return self.__shot + return outline.util.get_shot() + return self.__shot def set_shot(self, shot): """Set the shot name for this outline instance. @@ -591,9 +591,8 @@ def set_shot(self, shot): def get_show(self): """Return the show for this outline.""" if self.__show is None: - return util.get_show() - else: - return self.__show + return outline.util.get_show() + return self.__show def set_show(self, show): """Set the show name for this outline instance. @@ -606,9 +605,8 @@ def set_show(self, show): def get_user(self): """Return the user for this outline.""" if self.__user is None: - return util.get_user() - else: - return self.__user + return outline.util.get_user() + return self.__user def set_user(self, user): """Set the user name for this outline instance. @@ -633,7 +631,7 @@ def set_facility(self, facility): def get_mode(self): """Return the current mode of this outline object. - See constants for a list of possible modes. The mode + See outline.constants for a list of possible modes. The mode cannot be set from outside of the module. """ return self.__mode @@ -641,7 +639,7 @@ def get_mode(self): def set_mode(self, mode): """Set the current mode of the outline.""" if mode < self.__mode: - raise OutlineException("You cannot go back to previous modes.") + raise outline.exception.OutlineException("You cannot go back to previous modes.") self.__mode = mode def get_session(self): @@ -653,7 +651,7 @@ def get_session(self): :return: The outline's session object. """ if not self.__session: - raise SessionException("A session has not been created yet.") + raise outline.exception.SessionException("A session has not been created yet.") return self.__session def set_frame_range(self, frame_range): @@ -699,16 +697,16 @@ def set_env(self, key, value, pre=False): """ if key in self.__env: - logger.warn("Overwriting outline env var: %s, from %s to %s", - key, self.__env[key], value) + logger.warning( + "Overwriting outline env var: %s, from %s to %s", key, self.__env[key], value) if not isinstance(key, six.string_types): - raise OutlineException("Invalid key type for env var: %s", - type(key)) + raise outline.exception.OutlineException( + "Invalid key type for env var: %s" % type(key)) if not isinstance(value, six.string_types): - raise OutlineException("Invalid value type for env var: %s", - type(value)) + raise outline.exception.OutlineException( + "Invalid value type for env var: %s" % type(value)) self.__env[key] = (value, pre) @@ -722,8 +720,7 @@ def get_env(self, key=None): """ if key: return self.__env[key][0] - else: - return dict(self.__env) + return dict(self.__env) def get_args(self): """ @@ -746,8 +743,8 @@ def set_arg(self, key, value): :param value: Value to associate with the given key. """ if key in self.__args: - logger.warn("Overwriting outline argument: %s, from %s to %s", - key, self.__args[key], value) + logger.warning( + "Overwriting outline argument: %s, from %s to %s", key, self.__args[key], value) self.__args[key] = value def get_arg(self, key, default=None): @@ -766,15 +763,14 @@ def get_arg(self, key, default=None): :return: The value associated with the given key. """ try: - if default == None: + if default is None: return self.__args.get(key) - else: - return self.__args.get(key, default) + return self.__args.get(key, default) except KeyError: - raise OutlineException("No arg mapping exists for the value %s", - key) + raise outline.exception.OutlineException( + "No arg mapping exists for the value %s" % key) - def put_file(self, src, rename=None, symlink=False): + def put_file(self, src, rename=None): """ Copy the given file into the job's session path. If the optional rename argument is set, the file will be @@ -834,4 +830,3 @@ def get_data(self, key): :param key: the name that was used to store the value. """ return self.get_session().get_data(key) - diff --git a/pyoutline/outline/modules/__init__.py b/pyoutline/outline/modules/__init__.py index 1b24c5114..79740da2b 100644 --- a/pyoutline/outline/modules/__init__.py +++ b/pyoutline/outline/modules/__init__.py @@ -12,5 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +PyOutline "modules" help to translate between a set of job options and an OpenCue job outline. -"""Dev-Group supported modules.""" +For example, the Shell module translates the given job options into a job outline containing +shell commands to be run on render hosts. +""" diff --git a/pyoutline/outline/modules/shell.py b/pyoutline/outline/modules/shell.py index 007846b01..e564e2162 100644 --- a/pyoutline/outline/modules/shell.py +++ b/pyoutline/outline/modules/shell.py @@ -20,12 +20,12 @@ from __future__ import division from __future__ import absolute_import -from past.builtins import execfile import logging import os +from past.builtins import execfile -from outline import util -from outline.layer import Layer, Frame +import outline.layer +import outline.util logger = logging.getLogger("outline.modules.shell") @@ -38,12 +38,13 @@ "PyEval"] -class PyEval(Layer): +class PyEval(outline.layer.Layer): """ Arbitrary python code execution. """ def __init__(self, name, code, **args): - Layer.__init__(self, name, **args) + super(PyEval, self).__init__(name, **args) + self.__code = code def _setup(self): @@ -58,79 +59,72 @@ def _execute(self, frames): execfile(self.get_file("script")) -class Shell(Layer): +class Shell(outline.layer.Layer): """ Provides a method of executing a shell command over an arbitrary frame range. """ def __init__(self, name, **args): - Layer.__init__(self, name, **args) + super(Shell, self).__init__(name, **args) - ## require the cmd argument self.require_arg("command") self.set_arg("proxy_enable", False) - def _execute(self, frame_set): + def _execute(self, frames): """Execute the shell command.""" - for frame in frame_set: + for frame in frames: self.system(self.get_arg("command"), frame=frame) -class ShellSequence(Layer): +class ShellSequence(outline.layer.Layer): """ A module for executing an array of shell commands. """ def __init__(self, name, **args): - Layer.__init__(self, name, **args) + super(ShellSequence, self).__init__(name, **args) + self.require_arg("commands") self.set_frame_range("1-%d" % len(self.get_arg("commands"))) self.set_arg("proxy_enable", False) def _execute(self, frames): - for cmd in util.get_slice(self.get_frame_range(), - frames, self.get_arg("commands")): + for cmd in outline.util.get_slice(self.get_frame_range(), frames, self.get_arg("commands")): self.system(cmd) -class ShellCommand(Frame): +class ShellCommand(outline.layer.Frame): """ Provides a method of executing a single shell command. All instances of this class will result in a layer with a single - frame regaurdless of what frame range the outline is launched + frame regardless of what frame range the outline is launched to the cue with. """ def __init__(self, name, **args): - Frame.__init__(self, name, **args) + super(ShellCommand, self).__init__(name, **args) - ## require the cmd argument self.require_arg("command") self.set_arg("proxy_enable", False) - def _execute(self, frame_set): + def _execute(self, frames): """Execute the shell command.""" - self.system(self.get_arg("command"), frame=frame_set[0]) + self.system(self.get_arg("command"), frame=frames[0]) -class ShellScript(Frame): - """ - Copies the given script into frame's session - folder and executes it as a frame. - """ + +class ShellScript(outline.layer.Frame): + """Copies the given script into frame's session folder and executes it as a frame.""" def __init__(self, name, **args): - Frame.__init__(self, name, **args) + super(ShellScript, self).__init__(name, **args) self.require_arg("script") def _setup(self): s_path = self.put_file(self.get_arg("script"), "script") - os.chmod(s_path, 0o755) + os.chmod(s_path, 0o755) def _execute(self, frames): self.system(self.get_file("script"), frame=frames[0]) + def shell(name, command, **args): - """ - A factory method for building instances of Shell. - """ + """A factory method for building instances of Shell.""" args["command"] = command return Shell(name, **args) - - diff --git a/pyoutline/outline/plugins/__init__.py b/pyoutline/outline/plugins/__init__.py index 01b71ef25..89f1048ab 100644 --- a/pyoutline/outline/plugins/__init__.py +++ b/pyoutline/outline/plugins/__init__.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Module containing code for the PyOutline plugin system and project-maintained plugins.""" from __future__ import absolute_import from __future__ import print_function diff --git a/pyoutline/outline/plugins/local.py b/pyoutline/outline/plugins/local.py index 104349b48..c0bc1fd17 100644 --- a/pyoutline/outline/plugins/local.py +++ b/pyoutline/outline/plugins/local.py @@ -19,12 +19,14 @@ from __future__ import print_function from __future__ import division from __future__ import absolute_import + from builtins import str import os import logging from socket import gethostname import subprocess +import opencue from outline import event @@ -33,27 +35,39 @@ USE_LOCAL_CORES = 1 USE_LOCAL_THREADS = 0 + def init_cuerun_plugin(cuerun): - cuerun.get_parser().add_plugin_option("-L", "--run-local", - action="callback", - type="int", - nargs=1, - help="Run local cores", - callback=setup_event, - callback_args=(cuerun,)) - - cuerun.get_parser().add_plugin_option("-T", "--run-local-threads", - action="callback", - type="int", - nargs=1, - help="Set number of threads for local cores to run per frame.", - callback=setup_local_threads, - callback_args=(cuerun,)) + """Initialize the cuerun plugin.""" + + cuerun.get_parser().add_plugin_option( + "-L", "--run-local", + action="callback", + type="int", + nargs=1, + help="Run local cores", + callback=setup_event, + callback_args=(cuerun,)) + + cuerun.get_parser().add_plugin_option( + "-T", "--run-local-threads", + action="callback", + type="int", + nargs=1, + help="Set number of threads for local cores to run per frame.", + callback=setup_local_threads, + callback_args=(cuerun,)) def setup_event(option, opt, value, parser, *args, **kwargs): + """Main callback for the plugin, to set up the event listener.""" + del option + del opt + del parser + del kwargs + # Don't have a way to pass this to the event but # a global will do. + # pylint: disable=global-statement global USE_LOCAL_CORES USE_LOCAL_CORES = value @@ -61,38 +75,52 @@ def setup_event(option, opt, value, parser, *args, **kwargs): def setup_local_threads(option, opt, value, parser, *args, **kwargs): + """Callback for the --run-local-threads option.""" + + del option + del opt + del parser + del args + del kwargs + + # pylint: disable=global-statement global USE_LOCAL_THREADS USE_LOCAL_THREADS = value + def deed_local_machine(): - import opencue + """Deed the local machine to the current user.""" user = os.environ.get("USER") - show = opencue.findShow(os.environ.get("SHOW", "pipe")) + show = opencue.api.findShow(os.environ.get("SHOW", "pipe")) try: - owner = opencue.getOwner(user) - except opencue.CueException as e: + owner = opencue.api.getOwner(user) + except opencue.CueException: owner = show.createOwner(user) owner.takeOwnership(gethostname()) + def setup_local_cores(e): + """Configure the outline to use the local machine.""" deed_local_machine() threads = USE_LOCAL_THREADS or USE_LOCAL_CORES outline = e.outline - outline.set_arg("localbook", { "host": gethostname(), - "cores": str(USE_LOCAL_CORES), - "memory": get_half_host_memory(), - "threads": str(threads), - "gpu": str(0) }) + outline.set_arg("localbook", {"host": gethostname(), + "cores": str(USE_LOCAL_CORES), + "memory": get_half_host_memory(), + "threads": str(threads), + "gpu": str(0)}) + def get_half_host_memory(): + """Returns half of the amount of RAM the local machine has.""" + pipe = subprocess.Popen("vmstat -s", shell=True, bufsize=1000, stdout=subprocess.PIPE).stdout data = pipe.read().strip() data = int(data[0:data.find(" ")]) data = data / 2 return str(data) - diff --git a/pyoutline/outline/plugins/manager.py b/pyoutline/outline/plugins/manager.py index 367b2bce9..40b0c4e78 100644 --- a/pyoutline/outline/plugins/manager.py +++ b/pyoutline/outline/plugins/manager.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Module for setting up the PyOutline plugin system. +""" from __future__ import print_function from __future__ import division @@ -28,20 +31,25 @@ class PluginManager(object): + """ + Class responsible for instantiating the plugin system and loading desired plugins. + """ registered_plugins = [] @classmethod def init_cuerun_plugins(cls, cuerun): + """Initialize all registered plugins.""" for plugin in cls.registered_plugins: try: plugin.init_cuerun_plugin(cuerun) - except AttributeError as e: + except AttributeError: pass @classmethod def load_plugin(cls, module_name): - logger.debug("importing [%s] outline plugin." % module_name) + """Load a single plugin and register it with the plugin manager.""" + logger.debug("importing [%s] outline plugin.", module_name) try: module = __import__(module_name, globals(), @@ -49,7 +57,7 @@ def load_plugin(cls, module_name): [module_name]) try: module.loaded() - except AttributeError as e: + except AttributeError as e: pass cls.registered_plugins.append(module) @@ -58,11 +66,9 @@ def load_plugin(cls, module_name): @classmethod def init_plugin(cls, module_name, layer): - """ - Initialize a plugin on the given layer. - """ + """Initialize a plugin on the given layer.""" try: - logger.debug("importing [%s] outline plugin." % module_name) + logger.debug("importing [%s] outline plugin.", module_name) plugin = __import__(module_name, globals(), locals(), [module_name]) try: plugin.init(layer) @@ -73,21 +79,22 @@ def init_plugin(cls, module_name, layer): @classmethod def load_all_plugins(cls): + """Load and register all plugins listed in the config file.""" def section_priority(section_needing_key): priority_option = "priority" if config.has_option(section_needing_key, priority_option): return config.getint(section_needing_key, priority_option) return 0 - + sections = sorted(config.sections(), key=section_priority) - + for section in sections: if section.startswith("plugin:"): if config.getint(section, "enable"): - logger.debug("Loading plugin '%s'" % section) + logger.debug("Loading plugin '%s'", section) cls.load_plugin(config.get(section, "module")) @classmethod def get_plugins(cls): + """Gets a list of all registered plugins.""" return cls.registered_plugins - diff --git a/pyoutline/outline/session.py b/pyoutline/outline/session.py index 2350eeaa2..ad8df3f64 100644 --- a/pyoutline/outline/session.py +++ b/pyoutline/outline/session.py @@ -28,8 +28,9 @@ import uuid import yaml -from .config import config -from .exception import SessionException +import outline +import outline.exception +import outline.layer __all__ = ["is_session_path", @@ -52,7 +53,7 @@ def is_session_path(folder): return False if not os.path.isdir(folder): folder = os.path.abspath(os.path.dirname(folder)) - if not os.path.exists("%s/session" % folder): + if not os.path.exists(os.path.join(folder, 'session')): logger.info("script is not part of an existing session.") return False return True @@ -65,8 +66,8 @@ class Session(object): may be in a different location based on L{outline.config}. Using put_file, get_file you can copy files into this location which - can be used by all frames. Useing put_data, get_data you can - serialize and unserialize data into this session than can be used + can be used by all frames. Using put_data, get_data you can + serialize and deserialize data into this session than can be used by all frames. """ @@ -104,17 +105,17 @@ def __create_session(self): # The base dir is where we copy the outline and # store session data. - base_path = config.get("outline", "session_dir") + base_path = outline.config.get("outline", "session_dir") base_path = base_path.format( HOME=os.path.expanduser("~"), SHOW=self.__outline.get_show(), SHOT=self.__outline.get_shot()) - base_path = os.path.join(base_path, self.__name) + base_path = os.path.join(base_path, self.__name) # If the base dir doesn't exist, create it. Be sure # to make a directory to store the layer data. if not os.path.exists(base_path): - logger.info("creating session path: %s" % base_path) + logger.info("creating session path: %s", base_path) old_mask = os.umask(0) try: os.makedirs(base_path, 0o777) @@ -128,7 +129,7 @@ def __create_session(self): except OSError as exp: msg = "Failed to create session path: %s, reason: %s" - raise SessionException(msg % (self.get_path(), exp)) + raise outline.exception.SessionException(msg % (self.get_path(), exp)) def __load_session(self): """Loads an existing session based on the current path.""" @@ -136,18 +137,18 @@ def __load_session(self): session = "%s/session" % (os.path.dirname(self.__outline.get_path())) if not os.path.exists(session): msg = "Invalid session %s, session file does not exist." - raise SessionException(msg % session) + raise outline.exception.SessionException(msg % session) - logger.info("loading session: %s" % session) + logger.info("loading session: %s", session) with open(session) as file_object: try: data = yaml.load(file_object, Loader=yaml.FullLoader) except Exception as exp: msg = "failed to load session from %s, %s" - raise SessionException(msg % (session, exp)) + raise outline.exception.SessionException(msg % (session, exp)) self.__name = data self.__path = os.path.dirname(self.__outline.get_path()) - logger.info("session loaded: %s" % self.__name) + logger.info("session loaded: %s", self.__name) def get_name(self): """ @@ -168,26 +169,25 @@ def save(self): finally: fp.close() - def __layer_name(self, layer): + @staticmethod + def __layer_name(layer): """ - Helper funtion to conform an layer object or name + Helper function to conform an layer object or name into a string. :rtype: str :return: the name of the given layer. """ - from outline import Layer - if isinstance(layer, (Layer)): + if isinstance(layer, outline.layer.Layer): return layer.get_name() - else: - return str(layer) + return str(layer) def sym_file(self, src, layer=None, rename=None): """ Symlink a file into the session and return the path of the symlink. If the destination file already exists it will be deleted first. - + :type src: str :param src: The source path for the file to symlink. @@ -207,15 +207,15 @@ def sym_file(self, src, layer=None, rename=None): dst.append(os.path.basename(src)) dst_path = "/".join(dst) - logger.info("creating session link %s" % dst_path) + logger.info("creating session link %s", dst_path) try: os.unlink(dst_path) - except Exception as e: + except (OSError, FileNotFoundError): pass os.symlink(os.path.abspath(src), dst_path) - + return dst_path - + def put_file(self, src, layer=None, rename=None): """ Copy a file into the session and return the new path. This @@ -251,7 +251,7 @@ def put_file(self, src, layer=None, rename=None): dst_path = "/".join(dst) - logger.info("creating session file %s" % dst_path) + logger.info("creating session file %s", dst_path) shutil.copy(os.path.abspath(src), dst_path) return dst_path @@ -290,11 +290,11 @@ def get_file(self, name, layer=None, check=True, new=False): if check: if not os.path.exists(path): - raise SessionException("The path %s does not exist." % path) + raise outline.exception.SessionException("The path %s does not exist." % path) if new: if os.path.exists(path): - raise SessionException("The path %s already exists " % path) + raise outline.exception.SessionException("The path %s already exists " % path) return path @@ -320,7 +320,7 @@ def put_data(self, name, data, layer=None, force=False): path = "%s/%s" % (self.get_path(layer), name) if os.path.exists(path) and not force: - raise SessionException("There is already data being \ + raise outline.exception.SessionException("There is already data being \ stored under this name.") fp = open(path, "w") @@ -345,16 +345,16 @@ def get_data(self, name, layer=None): path = "%s/%s" % (self.get_path(layer), name) if not os.path.exists(path): - raise SessionException("There is not data in the session \ + raise outline.exception.SessionException("There is not data in the session \ stored under that name.") - logger.debug("opening data path for %s : %s" % (name, path)) + logger.debug("opening data path for %s : %s", name, path) with open(path) as file_object: try: return yaml.load(file_object, Loader=yaml.FullLoader) except Exception as exp: msg = "failed to load yaml data from %s, %s" - raise SessionException(msg % (path, exp)) + raise outline.exception.SessionException(msg % (path, exp)) def get_path(self, layer=None): """ @@ -370,7 +370,7 @@ def get_path(self, layer=None): or it may not exist if you construct the path on your own. :type layer: L{Layer} or str - :param layer: Specifiy the layer to get its sesstion path. [Optional] + :param layer: Specify the layer to get its session path. [Optional] :rtype: str :return: The path where session data is stored. @@ -378,25 +378,24 @@ def get_path(self, layer=None): """ if not self.__name: msg = "error, the session does not have a name." - raise SessionException(msg) + raise outline.exception.SessionException(msg) if not layer: return "%s" % self.__path - else: - layer_name = self.__layer_name(layer) - path = "%s/layers/%s" % (self.__path, - layer_name) - if os.path.exists(path): - return path - old_mask = os.umask(0) - try: - os.mkdir(path, 0o777) - except OSError: - # Just eat this. If it did actually fail - # the whole process will fail pretty soon. - pass - finally: - os.umask(old_mask) + layer_name = self.__layer_name(layer) + path = "%s/layers/%s" % (self.__path, + layer_name) + if os.path.exists(path): return path + old_mask = os.umask(0) + try: + os.mkdir(path, 0o777) + except OSError: + # Just eat this. If it did actually fail + # the whole process will fail pretty soon. + pass + finally: + os.umask(old_mask) + return path diff --git a/pyoutline/outline/versions/__init__.py b/pyoutline/outline/versions/__init__.py index b21608596..7829171fb 100644 --- a/pyoutline/outline/versions/__init__.py +++ b/pyoutline/outline/versions/__init__.py @@ -12,9 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. - -from __future__ import absolute_import -from __future__ import print_function -from __future__ import division +""" +outline.versions is a module for versioning external Python modules. +""" from .main import * diff --git a/pyoutline/outline/versions/main.py b/pyoutline/outline/versions/main.py index 80a9a81fb..0f1c22165 100644 --- a/pyoutline/outline/versions/main.py +++ b/pyoutline/outline/versions/main.py @@ -13,7 +13,9 @@ # limitations under the License. -"""Versions is a method for versioning python modules.""" +""" +Main interface functions for interacting with outline.versions. +""" from __future__ import absolute_import @@ -35,7 +37,9 @@ class VersionsException(Exception): - pass + """ + Raised when there was a problem configuring the repository path. + """ def require(module, version="latest"): @@ -47,14 +51,14 @@ def require(module, version="latest"): def unrequire(module): """ - Removes the given module from the repository + Removes the given module from the repository. """ return get_session().unrequire(module) def get_version(module, default="latest"): """ - Returns the currently loaded version of the given module + Returns the currently loaded version of the given module. """ return get_session().get_version(module, default) @@ -83,8 +87,7 @@ def set_repos(path): if os.path.exists(path): Settings.repos_path = path return True - else: - raise VersionsException("Cannot set the repos path to a non existent directory. %s" % path) + raise VersionsException("Cannot set the repos path to a non existent directory. %s" % path) def set_module_repos(module, path): diff --git a/pyoutline/outline/versions/session.py b/pyoutline/outline/versions/session.py index b2d357fe3..2d68c4bed 100644 --- a/pyoutline/outline/versions/session.py +++ b/pyoutline/outline/versions/session.py @@ -12,45 +12,57 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Classes responsible for setting up an outline.versions session. +""" from __future__ import print_function from __future__ import division from __future__ import absolute_import +import atexit from builtins import str from builtins import object -import atexit -from future.utils import with_metaclass import logging import os import shutil import sys import tempfile +from future.utils import with_metaclass + logger = logging.getLogger("versions") class Settings(object): + """ + Local session settings. + """ # Path containing different outline library versions repos_path = "" module_repos = {} class Singleton(type): - def __init__(self, name, bases, namespace): - self._obj = type(name, bases, namespace)() - atexit.register(self._obj.clean, shutil_ptr=shutil) + """ + Basic singleton implementation. + """ + # pylint: disable=super-init-not-called + def __init__(cls, name, bases, namespace): + cls._obj = type(name, bases, namespace)() + atexit.register(cls._obj.clean, shutil_ptr=shutil) - def __call__(self): - return self._obj + def __call__(cls): + return cls._obj class Session(with_metaclass(Singleton, object)): - """ The Session class handles creation of the versioning session. """ + + # pylint: disable=non-parent-init-called def __init__(self): object.__init__(self) self.__modules = {} @@ -62,12 +74,15 @@ def require(self, module, version): # Bail out if we've already loaded the module. if self.is_module_loaded(module): if str(version) != self.__modules[module]: - logger.warn("Can't load %s-%s, version %s is already loaded." - % (module, version, self.__modules[module])) + logger.warning( + "Can't load %s-%s, version %s is already loaded.", + module, + version, + self.__modules[module]) return False if not Settings.module_repos and not Settings.repos_path: - logger.warn("No repo paths were configured, not requiring a version.") + logger.warning("No repo paths were configured, not requiring a version.") return False # Find the destination. @@ -90,7 +105,7 @@ def require(self, module, version): # If the src dir doesn't exist in this repos, move on. if not os.path.exists(src): - logger.warn("The source dir '%s' does not exist." % src) + logger.warning("The source dir '%s' does not exist.", src) return False # Not a python module, so, we don't even add @@ -101,13 +116,13 @@ def require(self, module, version): self.__link_version(src, dst) self.__lock_module(module, version) return True - elif os.path.exists("%s/manifest.py" % src): + + if os.path.exists("%s/manifest.py" % src): self.__run_manifest(src) self.__lock_module(module, version) return True - else: - logger.warn("Unabled to load %s, not a module or manifest." % module) + logger.warning("Unable to load %s, not a module or manifest.", module) return False def unrequire(self, module): @@ -120,10 +135,11 @@ def unrequire(self, module): del self.__modules[module] os.unlink(path) return True - except KeyError as kerr: - logger.warn("Module %s was not loaded." % module) + except KeyError: + logger.warning("Module %s was not loaded.", module) else: - logger.warn("Failed to remove symlink '%s', does not exist." % path) + logger.warning("Failed to remove symlink '%s', does not exist.", path) + return False def is_module_loaded(self, name): @@ -154,17 +170,21 @@ def get_version(self, module, default="latest"): return self.__modules.get(module, default) def get_ver_str(self): + """Gets a string representation of all loaded modules and their versions.""" return ",".join(["%s:%s" % (mod, ver) for mod, ver in self.__modules.items()]) - def __link_version(self, src, dst): + @staticmethod + def __link_version(src, dst): os.symlink(src, dst) - def __run_manifest(self, path): + # pylint: disable=broad-except,import-outside-toplevel + @staticmethod + def __run_manifest(path): import imp try: fob, path, desc = imp.find_module('manifest', [path]) - mob = imp.load_module("manifest", fob, path, desc) + imp.load_module("manifest", fob, path, desc) fob.close() except Exception as e: print("Failed to execute manifest file: %s" % e) diff --git a/pyoutline/tests/__init__.py b/pyoutline/tests/__init__.py index 5fc7fabf6..d9c80f13d 100644 --- a/pyoutline/tests/__init__.py +++ b/pyoutline/tests/__init__.py @@ -11,10 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - -from __future__ import absolute_import -from __future__ import print_function -from __future__ import division - -from . import test_utils diff --git a/pyoutline/tests/backend/__init__.py b/pyoutline/tests/backend/__init__.py index ceb352598..d9c80f13d 100644 --- a/pyoutline/tests/backend/__init__.py +++ b/pyoutline/tests/backend/__init__.py @@ -11,11 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - -from __future__ import absolute_import -from __future__ import print_function -from __future__ import division - -from . import cue_test -from . import local_test diff --git a/pyoutline/tests/backend/cue_test.py b/pyoutline/tests/backend/cue_test.py index 53658ae29..029a1ec3f 100644 --- a/pyoutline/tests/backend/cue_test.py +++ b/pyoutline/tests/backend/cue_test.py @@ -14,21 +14,26 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Tests for the outline.backend.cue module. +""" from __future__ import print_function from __future__ import division from __future__ import absolute_import -import mock import os import unittest import xml.etree.ElementTree as ET +import mock + import opencue.compiled_proto.job_pb2 import opencue.wrappers.job import outline import outline.backend.cue +import outline.cuerun from .. import test_utils @@ -39,6 +44,7 @@ class SerializeTest(unittest.TestCase): def testSerializeShellOutline(self): path = os.path.join(SCRIPTS_DIR, 'shell.outline') + outline.config.set('outline', 'home', '/opencue/outline') outline.config.set('outline', 'user_dir', '/tmp/opencue/user') ol = outline.load_outline(path) @@ -147,10 +153,10 @@ def setUp(self): def tearDown(self): outline.backend.cue.JOB_WAIT_PERIOD_SEC = self.job_wait_period_original - @mock.patch('opencue.cuebot.Cuebot.getStub') + @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) @mock.patch('opencue.Cuebot.setHosts') @mock.patch('opencue.api.launchSpecAndWait') - def testLaunch(self, launchSpecAndWaitMock, setHostsMock, getStubMock): + def testLaunch(self, launchSpecAndWaitMock, setHostsMock): launchSpecAndWaitMock.return_value = [opencue.wrappers.job.Job()] serverName = 'foo-server' path = os.path.join(SCRIPTS_DIR, 'shell.outline') @@ -164,10 +170,10 @@ def testLaunch(self, launchSpecAndWaitMock, setHostsMock, getStubMock): launchSpecAndWaitMock.assert_called_with(serializedXml) setHostsMock.assert_called_with([serverName]) + @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) @mock.patch('opencue.api.isJobPending') - @mock.patch('opencue.cuebot.Cuebot.getStub') @mock.patch('opencue.api.launchSpecAndWait') - def testLaunchAndWait(self, launchSpecAndWaitMock, getStubMock, isJobPendingMock): + def testLaunchAndWait(self, launchSpecAndWaitMock, isJobPendingMock): jobName = 'some-job' launchSpecAndWaitMock.return_value = [ opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(name=jobName))] @@ -184,10 +190,10 @@ def testLaunchAndWait(self, launchSpecAndWaitMock, getStubMock, isJobPendingMock launchSpecAndWaitMock.assert_called_with(serializedXml) isJobPendingMock.assert_has_calls([mock.call(jobName), mock.call(jobName)]) + @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) @mock.patch('opencue.api.getJob') - @mock.patch('opencue.cuebot.Cuebot.getStub') @mock.patch('opencue.api.launchSpecAndWait') - def testLaunchAndTest(self, launchSpecAndWaitMock, getStubMock, getJobMock): + def testLaunchAndTest(self, launchSpecAndWaitMock, getJobMock): jobName = 'another-job' launchSpecAndWaitMock.return_value = [ opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(name=jobName))] @@ -207,4 +213,3 @@ def testLaunchAndTest(self, launchSpecAndWaitMock, getStubMock, getJobMock): if __name__ == '__main__': unittest.main() - diff --git a/pyoutline/tests/backend/local_test.py b/pyoutline/tests/backend/local_test.py index 1095a0306..03c40a826 100644 --- a/pyoutline/tests/backend/local_test.py +++ b/pyoutline/tests/backend/local_test.py @@ -14,17 +14,22 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Tests for the outline.backend.local module. +""" from __future__ import print_function from __future__ import division from __future__ import absolute_import import os -import mock import unittest +import mock + import outline import outline.backend.local +import outline.cuerun SCRIPTS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'scripts')) diff --git a/pyoutline/tests/depend_test.py b/pyoutline/tests/depend_test.py index 002304d3c..d2c6d104d 100644 --- a/pyoutline/tests/depend_test.py +++ b/pyoutline/tests/depend_test.py @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Tests for the outline.depend module. +""" from __future__ import absolute_import from __future__ import print_function @@ -22,8 +25,8 @@ import unittest import outline -from outline.depend import DependType -from outline.modules.shell import Shell +import outline.depend +import outline.modules.shell from . import test_utils @@ -36,8 +39,8 @@ def testShell(self): """Test a simple shell command.""" layer1Name = 'bah1' layer2Name = 'bah2' - layer1 = Shell(layer1Name, command=['/bin/ls']) - layer2 = Shell(layer2Name, command=['/bin/ls']) + layer1 = outline.modules.shell.Shell(layer1Name, command=['/bin/ls']) + layer2 = outline.modules.shell.Shell(layer2Name, command=['/bin/ls']) ol = outline.Outline(name='depend_test_v1') ol.add_layer(layer1) @@ -46,15 +49,16 @@ def testShell(self): depends = ol.get_layer(layer1Name).get_depends() self.assertEqual(1, len(depends)) - self.assertEqual(DependType.LayerOnLayer, depends[0].get_type()) + self.assertEqual(outline.depend.DependType.LayerOnLayer, depends[0].get_type()) self.assertEqual(layer2, depends[0].get_depend_on_layer()) def testShortHandDepend(self): with test_utils.TemporarySessionDirectory(): layer1Name = 'bah1' layer2Name = 'bah2' - layer1 = Shell(layer1Name, command=['/bin/ls']) - layer2 = Shell(layer2Name, command=['/bin/ls'], require=['%s:all' % layer1Name]) + layer1 = outline.modules.shell.Shell(layer1Name, command=['/bin/ls']) + layer2 = outline.modules.shell.Shell( + layer2Name, command=['/bin/ls'], require=['%s:all' % layer1Name]) ol = outline.Outline(name='depend_test_v2') ol.add_layer(layer1) @@ -63,15 +67,15 @@ def testShortHandDepend(self): depends = ol.get_layer(layer2Name).get_depends() self.assertEqual(1, len(depends)) - self.assertEqual(DependType.LayerOnLayer, depends[0].get_type()) + self.assertEqual(outline.depend.DependType.LayerOnLayer, depends[0].get_type()) self.assertEqual(layer1, depends[0].get_depend_on_layer()) def testAnyFrameDepend(self): with test_utils.TemporarySessionDirectory(): layer1Name = 'bah1' layer2Name = 'bah2' - layer1 = Shell(layer1Name, command=['/bin/ls'], range='1-2') - layer2 = Shell(layer2Name, command=['/bin/ls'], range='1-1', + layer1 = outline.modules.shell.Shell(layer1Name, command=['/bin/ls'], range='1-2') + layer2 = outline.modules.shell.Shell(layer2Name, command=['/bin/ls'], range='1-1', require=['%s:any' % layer1Name]) ol = outline.Outline(name='depend_test_any_frame') @@ -81,10 +85,9 @@ def testAnyFrameDepend(self): depends = ol.get_layer(layer2Name).get_depends() self.assertEqual(1, len(depends)) - self.assertEqual(DependType.FrameByFrame, depends[0].get_type()) + self.assertEqual(outline.depend.DependType.FrameByFrame, depends[0].get_type()) self.assertEqual(layer1, depends[0].get_depend_on_layer()) if __name__ == '__main__': unittest.main() - diff --git a/pyoutline/tests/executor_test.py b/pyoutline/tests/executor_test.py index bfa0b51d8..a63e1434d 100644 --- a/pyoutline/tests/executor_test.py +++ b/pyoutline/tests/executor_test.py @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Tests for the outline.executor module. +""" from __future__ import print_function from __future__ import division @@ -22,13 +25,13 @@ import time import unittest -from outline import TaskExecutor +import outline class TaskExecutorTest(unittest.TestCase): def test_simple_threading(self): - e = TaskExecutor(5) + e = outline.TaskExecutor(5) e.execute(self.print_, "hello thread 1") e.execute(self.print_, "hello thread 2") e.execute(self.print_, "hello thread 3") @@ -36,11 +39,11 @@ def test_simple_threading(self): e.execute(self.print_, "hello thread 5") e.wait() - def print_(self, msg): + @staticmethod + def print_(msg): print("Test Message: %s" % msg) time.sleep(1) if __name__ == '__main__': unittest.main() - diff --git a/pyoutline/tests/json_test.py b/pyoutline/tests/json_test.py index 6a576431f..dc0fe48fa 100644 --- a/pyoutline/tests/json_test.py +++ b/pyoutline/tests/json_test.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - # Copyright Contributors to the OpenCue Project # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,18 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Tests for the outline.json module. +""" from __future__ import absolute_import from __future__ import print_function from __future__ import division -import mock import os import unittest from xml.etree import ElementTree as Et +import mock + import outline -from outline import load_json from . import test_utils @@ -47,7 +48,7 @@ def testJson(self): '}]' '}') - ol = load_json(s) + ol = outline.load_json(s) self.assertEqual('test_job', ol.get_name()) self.assertEqual('1-10', ol.get_frame_range()) self.assertEqual('LAYER_VALUE1', ol.get_layer('layer_1').get_env('LAYER_KEY1')) @@ -66,7 +67,7 @@ def testJson(self): def testJsonFile(self, systemMock): """Load JSON from a file""" with open(os.path.join(JSON_DIR, 'shell.outline')) as fp: - ol = load_json(fp.read()) + ol = outline.load_json(fp.read()) with test_utils.TemporarySessionDirectory(): ol.setup() layer = ol.get_layer('shell_layer') diff --git a/pyoutline/tests/layer_test.py b/pyoutline/tests/layer_test.py index dd7f73885..567f4aa32 100644 --- a/pyoutline/tests/layer_test.py +++ b/pyoutline/tests/layer_test.py @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Tests for the outline.layer module. +""" from __future__ import absolute_import from __future__ import print_function @@ -22,14 +25,17 @@ # WARNING: Do not import builtins.str here as we do elsewhere in the code. Unit tests on Python 2 # need to preserve the existing Python 2 string type. from builtins import range -import future.types -import mock import os import sys import unittest +import future.types +import mock + import outline -from outline.modules.shell import Shell +import outline.io +import outline.modules.shell + from . import test_utils @@ -47,13 +53,13 @@ def setUp(self): self.layer = outline.Layer("composite") self.ol.add_layer(self.layer) - self.layer.add_child(Shell("blah1", command=["ls","-l"])) - self.layer.add_child(Shell("blah2", command=["ls","-1"])) - self.layer.add_child(Shell("blah3", command=["ls"])) + self.layer.add_child(outline.modules.shell.Shell("blah1", command=["ls", "-l"])) + self.layer.add_child(outline.modules.shell.Shell("blah2", command=["ls", "-1"])) + self.layer.add_child(outline.modules.shell.Shell("blah3", command=["ls"])) self.event = self.ol.get_layer("composite") - @mock.patch('outline.layer.Layer.system') + @mock.patch('outline.Layer.system') def test_execute(self, systemMock): """Run the execute method.""" with test_utils.TemporarySessionDirectory(): @@ -102,36 +108,36 @@ def setUp(self): def test_no_layer_range_no_job_range(self): # No layer range, no outline range defaults to a single frame. - self.assertEquals('1000-1000', self.ol.get_layer('cmd').get_frame_range()) - self.assertEquals(None, self.ol.get_frame_range()) + self.assertEqual('1000-1000', self.ol.get_layer('cmd').get_frame_range()) + self.assertEqual(None, self.ol.get_frame_range()) def test_no_layer_range_job_range(self): self.ol.set_frame_range('1000-2000') # No layer range, no outline range defaults to a single frame. - self.assertEquals('1000-2000', self.ol.get_layer('cmd').get_frame_range()) - self.assertEquals('1000-2000', self.ol.get_frame_range()) + self.assertEqual('1000-2000', self.ol.get_layer('cmd').get_frame_range()) + self.assertEqual('1000-2000', self.ol.get_frame_range()) def test_layer_range_no_job_range(self): self.ol.get_layer('cmd').set_frame_range('1000-2000') - self.assertEquals('1000-2000', self.ol.get_layer('cmd').get_frame_range()) - self.assertEquals(None, self.ol.get_frame_range()) + self.assertEqual('1000-2000', self.ol.get_layer('cmd').get_frame_range()) + self.assertEqual(None, self.ol.get_frame_range()) def test_layer_range_job_range(self): self.ol.set_frame_range('1000-2000') self.ol.get_layer('cmd').set_frame_range('1000-2000') expectedFrameStr = ','.join([str(i) for i in range(1000, 2001)]) - self.assertEquals(expectedFrameStr, self.ol.get_layer('cmd').get_frame_range()) - self.assertEquals('1000-2000', self.ol.get_frame_range()) + self.assertEqual(expectedFrameStr, self.ol.get_layer('cmd').get_frame_range()) + self.assertEqual('1000-2000', self.ol.get_frame_range()) def test_intersecting_range(self): self.ol.set_frame_range('1000-2000x8') self.ol.get_layer('cmd').set_frame_range('1000-2000') expectedFrameStr = ','.join([str(i) for i in range(1000, 2001, 8)]) - self.assertEquals(expectedFrameStr, self.ol.get_layer('cmd').get_frame_range()) - self.assertEquals('1000-2000x8', self.ol.get_frame_range()) + self.assertEqual(expectedFrameStr, self.ol.get_layer('cmd').get_frame_range()) + self.assertEqual('1000-2000x8', self.ol.get_frame_range()) def test_intersecting_failure(self): self.ol.set_frame_range('1000-1010') @@ -188,20 +194,20 @@ def test_invalid_type_args(self): intArgName = 'some-int-arg' self.layer.require_arg(intArgName, int) self.assertRaises( - outline.layer.LayerException, self.layer.set_arg, intArgName, 'some-string-val') + outline.LayerException, self.layer.set_arg, intArgName, 'some-string-val') self.layer.set_arg(intArgName, 872) if sys.version_info[0] >= 3: strArgName = 'some-str-arg' self.layer.require_arg(strArgName, str) self.assertRaises( - outline.layer.LayerException, self.layer.set_arg, strArgName, dict()) + outline.LayerException, self.layer.set_arg, strArgName, dict()) self.layer.set_arg(strArgName, 'py3-string') else: strArgName = 'some-str-arg' self.layer.require_arg(strArgName, str) self.assertRaises( - outline.layer.LayerException, self.layer.set_arg, strArgName, dict()) + outline.LayerException, self.layer.set_arg, strArgName, dict()) self.layer.set_arg(strArgName, 'standard-py2-string') self.layer.set_arg(strArgName, u'py2-unicode') self.layer.set_arg(strArgName, future.types.newstr('py3-string-backport')) @@ -209,7 +215,7 @@ def test_invalid_type_args(self): newstrArgName = 'some-newstr-arg' self.layer.require_arg(newstrArgName, future.types.newstr) self.assertRaises( - outline.layer.LayerException, self.layer.set_arg, newstrArgName, dict()) + outline.LayerException, self.layer.set_arg, newstrArgName, dict()) self.layer.set_arg(newstrArgName, 'standard-py2-string') self.layer.set_arg(newstrArgName, u'py2-unicode') self.layer.set_arg(newstrArgName, future.types.newstr('py3-string-backport')) @@ -221,7 +227,7 @@ def test_require_arg(self): setup() is run. """ self.layer.require_arg('bobofet') - self.assertRaises(outline.layer.LayerException, self.layer.check_required_args) + self.assertRaises(outline.LayerException, self.layer.check_required_args) self.layer.set_arg('bobofet', 1) self.layer.check_required_args() @@ -255,7 +261,7 @@ def test_setup(self): with test_utils.TemporarySessionDirectory(): self.layer.setup() - @mock.patch('outline.layer.Layer.system') + @mock.patch('outline.Layer.system') def test_execute(self, systemMock): """Test execution of a frame.""" os.environ = {} @@ -265,10 +271,10 @@ def test_execute(self, systemMock): self.layer.execute(1) systemMock.assert_has_calls([mock.call(['ps', 'aux'], frame=1)]) - self.assertTrue('foo', os.environ['cue_test_01']) - self.assertTrue('bar', os.environ['cue_test_02']) - self.assertTrue('layer-env-a', os.environ['cue_layer_01']) - self.assertTrue('layer-env-b', os.environ['cue_layer_02']) + self.assertEqual('foo', os.environ['cue_test_01']) + self.assertEqual('bar', os.environ['cue_test_02']) + self.assertEqual('layer-env-a', os.environ['cue_layer_01']) + self.assertEqual('layer-env-b', os.environ['cue_layer_02']) def test_get_set_frame_range(self): """Test getting/setting the frame range. If the frame @@ -355,7 +361,8 @@ def test_set_name(self): with test_utils.TemporarySessionDirectory(): self.ol.setup() - self.assertRaises(outline.layer.LayerException, self.layer.set_name, 'this-should-fail') + self.assertRaises( + outline.LayerException, self.layer.set_name, 'this-should-fail') class OutputRegistrationTest(unittest.TestCase): @@ -377,9 +384,10 @@ def disabled__test_output_passing(self): layer1 = TestA("test1") # the preprocess - prelayer = outline.layer.LayerPreProcess(layer1) - prelayer._execute = lambda frames: prelayer.add_output("test", - outline.io.FileSpec("/tmp/foo.#.exr")) + prelayer = outline.LayerPreProcess(layer1) + prelayer._execute = lambda frames: prelayer.add_output( + "test", outline.io.FileSpec("/tmp/foo.#.exr")) + # Add both to the outline ol.add_layer(layer1) ol.add_layer(prelayer) @@ -402,22 +410,27 @@ def disabled__test_output_passing(self): class TestAfterInit(outline.Layer): def __init__(self, name, **args): outline.Layer.__init__(self, name, **args) + def after_init(self, ol): self.set_arg("after_init", True) + class TestA(outline.Layer): def __init__(self, name, **args): outline.Layer.__init__(self, name, **args) + def setup(self): self.get_outline().add_layer(TestB("test_b")) + class TestB(outline.Layer): def __init__(self, name, **args): outline.Layer.__init__(self, name, **args) + self.is_setup = False + def setup(self): self.is_setup = True if __name__ == '__main__': unittest.main() - diff --git a/pyoutline/tests/loader_test.py b/pyoutline/tests/loader_test.py index b9d94cba4..989c085c2 100644 --- a/pyoutline/tests/loader_test.py +++ b/pyoutline/tests/loader_test.py @@ -1,5 +1,3 @@ -#!/bin/env python2.5 - # Copyright Contributors to the OpenCue Project # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Tests for the outline.loader module. +""" from __future__ import absolute_import from __future__ import print_function @@ -26,7 +27,8 @@ import FileSequence import outline -from outline.modules.shell import Shell +import outline.cuerun +import outline.modules.shell from . import test_utils @@ -98,10 +100,10 @@ def test_get_set_env(self): def test_add_get_remove_layer(self): with test_utils.TemporarySessionDirectory(): ol = outline.load_outline(self.path) - ol.add_layer(Shell("shell_command", cmd=["/bin/ls"])) + ol.add_layer(outline.modules.shell.Shell("shell_command", cmd=["/bin/ls"])) self.assertEqual(2, len(ol.get_layers())) - self.assertTrue(isinstance(ol.get_layer("shell_command"), Shell)) + self.assertTrue(isinstance(ol.get_layer("shell_command"), outline.modules.shell.Shell)) ol.remove_layer(ol.get_layer("shell_command")) @@ -256,12 +258,10 @@ def test_ol_os_env(self): ol = outline.load_outline(self.script) l = outline.cuerun.OutlineLauncher(ol) root = Et.fromstring(l.serialize()) - self.assertEqual("radical", - root.find("job/os").text) + self.assertEqual("radical", root.find("job/os").text) finally: del os.environ["OL_OS"] if __name__ == '__main__': unittest.main() - diff --git a/pyoutline/tests/modules/shell_test.py b/pyoutline/tests/modules/shell_test.py index ace44b1b4..0efea1680 100644 --- a/pyoutline/tests/modules/shell_test.py +++ b/pyoutline/tests/modules/shell_test.py @@ -12,23 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Tests for the outline.modules.shell module. +""" from __future__ import print_function from __future__ import division from __future__ import absolute_import from builtins import range -import mock + import tempfile import unittest -from FileSequence import FrameSet +import mock +from FileSequence import FrameSet import outline -from outline.loader import Outline -from outline.modules.shell import Shell -from outline.modules.shell import ShellSequence -from outline.modules.shell import ShellScript +import outline.modules.shell from .. import test_utils @@ -45,7 +46,7 @@ def testShell(self, systemMock): command = ['/bin/ls'] - shell = Shell('bah', command=command) + shell = outline.modules.shell.Shell('bah', command=command) shell._execute(FrameSet('5-6')) systemMock.assert_has_calls([ @@ -60,7 +61,8 @@ def testShellSequence(self, systemMock): commandCount = 10 commands = ['/bin/echo %d' % (frame+1) for frame in range(commandCount)] - shellSeq = ShellSequence('bah', commands=commands, cores=10, memory='512m') + shellSeq = outline.modules.shell.ShellSequence( + 'bah', commands=commands, cores=10, memory='512m') shellSeq._execute(FrameSet('5-6')) self.assertEqual('1-%d' % commandCount, shellSeq.get_frame_range()) @@ -85,12 +87,12 @@ def testShellScript(self, systemMock): with open(scriptFile.name, 'w') as fp: fp.write(scriptContents) - outln = Outline() + outln = outline.Outline() outln.setup() expectedSessionPath = outln.get_session().put_file( scriptFile.name, layer=layerName, rename='script') - shellScript = ShellScript(layerName, script=scriptFile.name) + shellScript = outline.modules.shell.ShellScript(layerName, script=scriptFile.name) shellScript.set_outline(outln) shellScript._setup() shellScript._execute(FrameSet('5-6')) @@ -107,7 +109,7 @@ def testShellToString(self, systemMock): command = '/bin/ls -l ./' - shell = Shell('bah', command=command) + shell = outline.modules.shell.Shell('bah', command=command) shell._execute(FrameSet('5-6')) systemMock.assert_has_calls([ @@ -118,4 +120,3 @@ def testShellToString(self, systemMock): if __name__ == '__main__': unittest.main() - diff --git a/pyoutline/tests/session_test.py b/pyoutline/tests/session_test.py index b30bb65cd..726118a1e 100644 --- a/pyoutline/tests/session_test.py +++ b/pyoutline/tests/session_test.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Tests for the outline.session module. +""" from __future__ import print_function from __future__ import division @@ -28,14 +31,12 @@ class SessionTest(unittest.TestCase): - """Tests for outline.session""" - def setUp(self): self.script_path = os.path.join(SCRIPTS_DIR, 'shell.outline') self.ol = outline.load_outline(self.script_path) self.ol.set_frame_range("1-10") self.ol.setup() - self.session = self.ol.get_session() + self.session = self.ol.get_session() def test_put_file(self): """Testing get/put file into outline.""" @@ -64,7 +65,7 @@ def test_get_new_file_from_layer(self): # Put file into layer layer = self.ol.get_layer("cmd") - + # Getting a file that doesn't exist should raise SessionException # Unless the new flag is passed in. self.assertRaises(outline.SessionException, layer.get_file, "foo.bar") @@ -74,30 +75,29 @@ def test_get_new_file_from_layer(self): # If the file already exists, new should throw an exception layer.put_file(self.script_path) - self.assertRaises(outline.SessionException, layer.get_file, "shell.outline", new=True) + self.assertRaises( + outline.SessionException, layer.get_file, "shell.outline", new=True) def test_get_unchecked_file(self): """Tests the new option for the get_file method. """ - layer = self.ol.get_layer("cmd") + self.ol.get_layer("cmd") def test_put_data(self): - """Test get/set data""" + """Test get/set data""" - # Serialize an array of ints into the session - # and then retrieve it. - value = [100,200,300,400,500] - self.session.put_data("foo",value) - self.assertEqual(value, self.session.get_data("foo")) + # Serialize an array of ints into the session and then retrieve it. + value = [100, 200, 300, 400, 500] + self.session.put_data("foo", value) + self.assertEqual(value, self.session.get_data("foo")) def test_put_data_to_layer(self): - """Test get/set layer data.""" + """Test get/set layer data.""" + + layer = self.ol.get_layer("cmd") + value = [100, 200, 300, 400, 500] + layer.put_data("foo", value) + self.assertEqual(value, layer.get_data("foo")) - layer = self.ol.get_layer("cmd") - value = [100,200,300,400,500] - layer.put_data("foo", value) - self.assertEqual(value, layer.get_data("foo")) - if __name__ == '__main__': unittest.main() - diff --git a/pyoutline/tests/test_utils.py b/pyoutline/tests/test_utils.py index 3b670d1f1..9fe3bc480 100644 --- a/pyoutline/tests/test_utils.py +++ b/pyoutline/tests/test_utils.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Common classes and functions used throughout the outline module test code. +""" from __future__ import print_function from __future__ import division @@ -25,6 +28,9 @@ class TemporarySessionDirectory(object): + def __init__(self): + self.originalSessionDir = None + self.tempSessionDir = None def __enter__(self): self.originalSessionDir = outline.config.get('outline', 'session_dir') diff --git a/requirements.txt b/requirements.txt index 722434859..1778660a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,5 +9,6 @@ pathlib==1.0.1;python_version<"3.4" pexpect==4.6.0 psutil==5.6.6 pyfakefs==3.6 +pylint==2.6.0;python_version>="3.7" PyYAML==5.1 six==1.11.0 From 8d874a392bb420ee26a70dc240b42a65366fea3d Mon Sep 17 00:00:00 2001 From: Lars van der Bijl <285658+larsbijl@users.noreply.github.com> Date: Wed, 27 Jan 2021 20:56:25 +0000 Subject: [PATCH 038/277] Add IdrisMiles to code owners (#887) As discussed during the last meeting, Add IdrisMiles to code owners. --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 8b7d19639..ff91076cd 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,2 +1,2 @@ -* @bcipriano @gregdenton @jrray @smith1511 @larsbijl @DiegoTavares +* @bcipriano @gregdenton @jrray @smith1511 @larsbijl @DiegoTavares @IdrisMiles /tsc/gsoc @bcipriano @gregdenton @shiva-kannan From 8b3e458bd18c26322361401800180315476f22f3 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Fri, 29 Jan 2021 09:16:07 +0000 Subject: [PATCH 039/277] Notes from Jan 20 TSC meeting. (#893) --- tsc/meetings/2021-01-20.md | 86 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 tsc/meetings/2021-01-20.md diff --git a/tsc/meetings/2021-01-20.md b/tsc/meetings/2021-01-20.md new file mode 100644 index 000000000..615de1900 --- /dev/null +++ b/tsc/meetings/2021-01-20.md @@ -0,0 +1,86 @@ +# OpenCue TSC Meeting Notes 20 Jan 2021 + +Secretary: Brian Cipriano + +TSC Attendees: + +* [ ] Matt Chambers +* [x] Brian Cipriano +* [ ] Greg Denton +* [x] Ben Dines +* [ ] Dave Fellows +* [x] Daniel Neilson +* [ ] Alex Schworer +* [ ] Erik Strauss + +Committer Attendees: + +* [ ] Christian Smith +* [x] Diego Tavares +* [x] Lars van der Bijl + +Agenda/Notes: + +* Goals for 2021 + * User survey + * Completes Feb 1, should have response data soon after that. + * Let's use that data to revisit these priorities once we have it. + * Docs refresh + * Is the focus here really on docs or is it really on new user UX? Docs are needed, but new + user UX is the most important thing. Let's focus on that first (see next section). + * Need API reference docs + * Brian needs to check on this. + * Will also want IDE autocompletion — need to work on cleaning up Python headers. + * This may mostly work already. Help with environment setup may help with this, can we + publish IDE example settings? + * More broadly, make it easier to set up a development environment. + * Need examples on how to use the API, pyoutline examples. + * New user UX + * demo_data.sql is actually required, this trips people up. Needs some work. + * Split into "bare minimum data" vs "actual demo data". + * Highlight this in Quickstart docs. + * Keep docker compose? + * Could use Kubernetes / Helm maybe. + * Let's use survey feedback to help inform this. + * Cloud provider docs for easy deployment. + * Can be useful as it provides a pathway to a production deployment. + * Should consider whether users really want to run Cuebot on the cloud or not. Many + users running Cuebot locally with burst to cloud. Could maybe provide an option. + * Can Kubernetes help with this? + * Any issue running Cuebot in a Kubernetes cluster? For Cuebot itself, no — can run + multiple instances coordinating via the database. One problem, RQD needs to be + responsive to its list of Cuebot hosts changing, this is currently static until + RQD restarts. + * Hosted SQL, hit some recent issues. Brian/Lars to check on Github issue for this. + Might want to consider whether we avoid hosted SQL. + * Terraform a likely candidate to help with this. We could potentially build it in a + base that is mostly cloud-agnostic. + * Brian to talk to folks at Google, let's try to start discussion cross-cloud to create + a base we can build on. + * ASWF graduation + * Brian still working on CII badge. Getting close. + * Also waiting on survey to wrap up. + * Optimize how Cuebot handles booking / dispatching. + * SPI has a PR in progress for this. + * Removing Oracle + * Got final confirmation we are ready to proceed. + * Hopefully relatively simple, just tear out the code. + * Proper GPU support + * ETC has PR in progress. This is probably Linux-specific but should give us a base to build + on. + * UI for adding projects + * This came up recently as something that you currently need to use the API for. + * Expand DCC plugins + * Blender is in progress. Many others we could work on. + * Logging solution + * Build on the Prometheus export example. + * Grafana is a good next step for building on this. + * Look at Loki, which integrates with Grafana too. + * Let's revisit our goals list after the survey finishes. +* TSC members / committers + * Lars van der Bijl unanimously approved via email to join the OpenCue TSC. + * Idris Miles unanimously approved to be added as an OpenCue committer. +* Other current work updates + * Lars + * Rebased timeout / llu timeout PR, ready to merge. + * Added dockerignore to fix issue with sandbox restarting. From e2eef795e3e6f46eb35ee6e36d5382d0d4f0ed23 Mon Sep 17 00:00:00 2001 From: Idris Miles Date: Fri, 29 Jan 2021 09:27:55 +0000 Subject: [PATCH 040/277] fix: Pin pip and setuptools to compatible versions (#883) This fixes building the sandbox environment. --- connectors/prometheus_metrics/Dockerfile | 4 ++-- cueadmin/Dockerfile | 4 ++-- cuegui/Dockerfile | 4 ++-- cuesubmit/Dockerfile | 4 ++-- pycue/Dockerfile | 4 ++-- pyoutline/Dockerfile | 4 ++-- rqd/Dockerfile | 4 ++-- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/connectors/prometheus_metrics/Dockerfile b/connectors/prometheus_metrics/Dockerfile index 47d697226..9b8039568 100644 --- a/connectors/prometheus_metrics/Dockerfile +++ b/connectors/prometheus_metrics/Dockerfile @@ -15,10 +15,10 @@ RUN yum -y install \ python36-devel \ python36-pip -RUN python -m pip install --upgrade pip +RUN python -m pip install --upgrade 'pip<21' RUN python3.6 -m pip install --upgrade pip -RUN python -m pip install --upgrade setuptools +RUN python -m pip install --upgrade 'setuptools<45' RUN python3.6 -m pip install --upgrade setuptools COPY LICENSE ./ diff --git a/cueadmin/Dockerfile b/cueadmin/Dockerfile index bdbcbc746..37caf1d5f 100644 --- a/cueadmin/Dockerfile +++ b/cueadmin/Dockerfile @@ -19,10 +19,10 @@ RUN yum -y install \ python36-devel \ python36-pip -RUN python -m pip install --upgrade pip +RUN python -m pip install --upgrade 'pip<21' RUN python3.6 -m pip install --upgrade pip -RUN python -m pip install --upgrade setuptools +RUN python -m pip install --upgrade 'setuptools<45' RUN python3.6 -m pip install --upgrade setuptools COPY LICENSE ./ diff --git a/cuegui/Dockerfile b/cuegui/Dockerfile index c7a1881fe..4804478f6 100644 --- a/cuegui/Dockerfile +++ b/cuegui/Dockerfile @@ -26,10 +26,10 @@ RUN yum -y install \ python36-devel \ python36-pip -RUN python -m pip install --upgrade pip +RUN python -m pip install --upgrade 'pip<21' RUN python3.6 -m pip install --upgrade pip -RUN python -m pip install --upgrade setuptools +RUN python -m pip install --upgrade 'setuptools<45' RUN python3.6 -m pip install --upgrade setuptools RUN dbus-uuidgen > /etc/machine-id diff --git a/cuesubmit/Dockerfile b/cuesubmit/Dockerfile index 71543f618..252936fa9 100644 --- a/cuesubmit/Dockerfile +++ b/cuesubmit/Dockerfile @@ -20,10 +20,10 @@ RUN yum -y install \ python36-devel \ python36-pip -RUN python -m pip install --upgrade pip +RUN python -m pip install --upgrade 'pip<21' RUN python3.6 -m pip install --upgrade pip -RUN python -m pip install --upgrade setuptools +RUN python -m pip install --upgrade 'setuptools<45' RUN python3.6 -m pip install --upgrade setuptools COPY LICENSE ./ diff --git a/pycue/Dockerfile b/pycue/Dockerfile index 6b0fb6533..54f8a2dc2 100644 --- a/pycue/Dockerfile +++ b/pycue/Dockerfile @@ -19,10 +19,10 @@ RUN yum -y install \ python36-devel \ python36-pip -RUN python -m pip install --upgrade pip +RUN python -m pip install --upgrade 'pip<21' RUN python3.6 -m pip install --upgrade pip -RUN python -m pip install --upgrade setuptools +RUN python -m pip install --upgrade 'setuptools<45' RUN python3.6 -m pip install --upgrade setuptools COPY LICENSE ./ diff --git a/pyoutline/Dockerfile b/pyoutline/Dockerfile index 60d04f725..fb24d1136 100644 --- a/pyoutline/Dockerfile +++ b/pyoutline/Dockerfile @@ -19,10 +19,10 @@ RUN yum -y install \ python36-devel \ python36-pip -RUN python -m pip install --upgrade pip +RUN python -m pip install --upgrade 'pip<21' RUN python3.6 -m pip install --upgrade pip -RUN python -m pip install --upgrade setuptools +RUN python -m pip install --upgrade 'setuptools<45' RUN python3.6 -m pip install --upgrade setuptools COPY LICENSE ./ diff --git a/rqd/Dockerfile b/rqd/Dockerfile index b6c715e4d..fd772c59e 100644 --- a/rqd/Dockerfile +++ b/rqd/Dockerfile @@ -12,10 +12,10 @@ RUN yum -y install \ python36-devel \ python36-pip -RUN python -m pip install --upgrade pip +RUN python -m pip install --upgrade 'pip<21' RUN python3.6 -m pip install --upgrade pip -RUN python -m pip install --upgrade setuptools +RUN python -m pip install --upgrade 'setuptools<45' RUN python3.6 -m pip install --upgrade setuptools WORKDIR /opt/opencue From f03c58763b277cdd1689bca8ec3910028aa074c4 Mon Sep 17 00:00:00 2001 From: Lars van der Bijl <285658+larsbijl@users.noreply.github.com> Date: Fri, 29 Jan 2021 09:28:35 +0000 Subject: [PATCH 041/277] Fix column id count (#886) Increment the column id's. Add extra empty column value for group and root group types Closes #885 --- cuegui/cuegui/CueJobMonitorTree.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/cuegui/cuegui/CueJobMonitorTree.py b/cuegui/cuegui/CueJobMonitorTree.py index 68c34486b..fec7d9759 100644 --- a/cuegui/cuegui/CueJobMonitorTree.py +++ b/cuegui/cuegui/CueJobMonitorTree.py @@ -83,60 +83,60 @@ def __init__(self, parent): tip="If the job has auto eating enabled, a pac-man icon\n" "will appear here and all frames that become dead will\n" "automatically be eaten.") - self.addColumn("Run", 38, id=3, + self.addColumn("Run", 38, id=4, data=lambda job: job.data.job_stats.running_frames, sort=lambda job: job.data.job_stats.running_frames, tip="The number of running frames.") - self.addColumn("Cores", 55, id=4, + self.addColumn("Cores", 55, id=5, data=lambda job: "%.02f" % job.data.job_stats.reserved_cores, sort=lambda job: job.data.job_stats.reserved_cores, tip="The number of reserved cores.") - self.addColumn("Wait", 45, id=5, + self.addColumn("Wait", 45, id=6, data=lambda job: job.data.job_stats.waiting_frames, sort=lambda job: job.data.job_stats.waiting_frames, tip="The number of waiting frames.") - self.addColumn("Depend", 55, id=6, + self.addColumn("Depend", 55, id=7, data=lambda job: job.data.job_stats.depend_frames, sort=lambda job: job.data.job_stats.depend_frames, tip="The number of dependent frames.") - self.addColumn("Total", 50, id=7, + self.addColumn("Total", 50, id=8, data=lambda job: job.data.job_stats.total_frames, sort=lambda job: job.data.job_stats.total_frames, tip="The total number of frames.") - self.addColumn("_Booking Bar", 150, id=8, + self.addColumn("_Booking Bar", 150, id=9, delegate=cuegui.ItemDelegate.JobBookingBarDelegate) - self.addColumn("Min", 38, id=9, + self.addColumn("Min", 38, id=10, data=lambda job: "%.0f" % job.data.min_cores, sort=lambda job: job.data.min_cores, tip="The minimum number of running cores that the cuebot\n" "will try to maintain.") - self.addColumn("Max", 38, id=10, + self.addColumn("Max", 38, id=11, data=lambda job: "%.0f" % job.data.max_cores, sort=lambda job: job.data.max_cores, tip="The maximum number of running cores that the cuebot\n" "will allow.") - self.addColumn("Age", 50, id=11, + self.addColumn("Age", 50, id=12, data=lambda job: cuegui.Utils.secondsToHHHMM(self.currtime - job.data.start_time), sort=lambda job: self.currtime - job.data.start_time, tip="The HOURS:MINUTES since the job was launched.") - self.addColumn("Pri", 30, id=12, + self.addColumn("Pri", 30, id=13, data=lambda job: job.data.priority, sort=lambda job: job.data.priority, tip="The job priority. The cuebot uses this as a suggestion\n" "to determine what job needs the next available matching\n" "resource.") - self.addColumn("ETA", 65, id=13, + self.addColumn("ETA", 65, id=14, data=lambda job: "", tip="(Inacurate and disabled until a better solution exists)\n" "A very rough estimate of the number of HOURS:MINUTES\n" "it will be before the entire job is done.") - self.addColumn("MaxRss", 60, id=14, + self.addColumn("MaxRss", 60, id=15, data=lambda job: cuegui.Utils.memoryToString(job.data.job_stats.max_rss), sort=lambda job: job.data.job_stats.max_rss, tip="The most memory used at one time by any single frame.") - self.addColumn("_Blank", 20, id=15, + self.addColumn("_Blank", 20, id=16, tip="Spacer") - self.addColumn("Progress", 0, id=16, + self.addColumn("Progress", 0, id=17, delegate=cuegui.ItemDelegate.JobThinProgressBarDelegate, tip="A visual overview of the job progress.\n" "Green \t is succeeded\n" @@ -171,6 +171,7 @@ def __init__(self, parent): self.addColumn("", 0, id=16, data=lambda group: (group.data.department != "Unknown" and group.data.department or "")) + self.addColumn("", 0, id=17) cuegui.AbstractTreeWidget.AbstractTreeWidget.__init__(self, parent) From 70e4776fd184062147ab648650f72ba2d00864db Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Fri, 29 Jan 2021 13:10:59 -0800 Subject: [PATCH 042/277] Use non-deprecated ConfigParser in Python 3. (#889) --- pyoutline/outline/config.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pyoutline/outline/config.py b/pyoutline/outline/config.py index 97fa13ee9..371793a3c 100644 --- a/pyoutline/outline/config.py +++ b/pyoutline/outline/config.py @@ -32,7 +32,13 @@ import pathlib import tempfile -from six.moves.configparser import SafeConfigParser +import six + +from six.moves import configparser +if six.PY2: + ConfigParser = configparser.SafeConfigParser +else: + ConfigParser = configparser.ConfigParser __all__ = ["config"] @@ -41,7 +47,7 @@ PYOUTLINE_ROOT_DIR = __file_path__.parent.parent DEFAULT_USER_DIR = pathlib.Path(tempfile.gettempdir()) / 'opencue' / 'outline' / getpass.getuser() -config = SafeConfigParser() +config = ConfigParser() default_config_paths = [__file_path__.parent.parent.parent / 'etc' / 'outline.cfg', __file_path__.parent.parent / 'etc' / 'outline.cfg'] From 10b4086defd7195518f2835c5a66c067fc93c757 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Fri, 29 Jan 2021 23:13:47 +0000 Subject: [PATCH 043/277] Lint the CueAdmin code. (#894) --- ci/pylintrc_test | 2 + ci/run_python_tests.sh | 4 ++ cueadmin/cueadmin/__main__.py | 6 ++ cueadmin/cueadmin/common.py | 121 ++++++++++++++++++++------------- cueadmin/cueadmin/format.py | 61 +++++++++++++++-- cueadmin/cueadmin/output.py | 11 ++- cueadmin/cueadmin/util.py | 16 ++--- cueadmin/tests/common_tests.py | 64 +++++++++-------- cueadmin/tests/output_tests.py | 8 +++ 9 files changed, 199 insertions(+), 94 deletions(-) diff --git a/ci/pylintrc_test b/ci/pylintrc_test index 7c252cde0..1e71a142b 100644 --- a/ci/pylintrc_test +++ b/ci/pylintrc_test @@ -70,6 +70,8 @@ disable=duplicate-code, protected-access, raise-missing-from, too-many-public-methods, + unused-argument, + unused-variable, useless-object-inheritance # Enable the message, report, category or checker with the given id(s). You can diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index bd51296db..8e3b72e7e 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -24,4 +24,8 @@ python rqd/setup.py test if [[ "$1" == "--lint" ]]; then cd pyoutline && PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main outline && cd .. cd pyoutline && PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests && cd .. + + cd cueadmin && PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main cueadmin && cd .. + cd cueadmin && PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests && cd .. fi + diff --git a/cueadmin/cueadmin/__main__.py b/cueadmin/cueadmin/__main__.py index 43e16fac3..2a354f3a2 100644 --- a/cueadmin/cueadmin/__main__.py +++ b/cueadmin/cueadmin/__main__.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Entrypoint for CueAdmin tool.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -25,7 +28,10 @@ logger = logging.getLogger("opencue.tools.cueadmin") +# pylint: disable=broad-except def main(): + """Starts the CueAdmin tool.""" + parser = cueadmin.common.getParser() args = parser.parse_args() diff --git a/cueadmin/cueadmin/common.py b/cueadmin/cueadmin/common.py index 850b29b3f..5141b0ed9 100644 --- a/cueadmin/cueadmin/common.py +++ b/cueadmin/cueadmin/common.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Main CueAdmin code.""" + + from __future__ import print_function from __future__ import absolute_import from __future__ import division @@ -21,10 +24,11 @@ from builtins import object import argparse import logging -import six import sys import traceback +import six + import opencue import opencue.wrappers.job import opencue.wrappers.proc @@ -57,18 +61,21 @@ "AllocUtil"] +# pylint: disable=broad-except def handleParserException(args, e): + """Custom argument parser error handling.""" try: if args.verbose: traceback.print_exc(file=sys.stderr) raise e except ValueError as ex: - print("Error: %s. Try the -verbose or -h flags for more info." % ex, file=sys.stderr) + print("Error: %s. Try the -verbose or -h flags for more info." % ex, file=sys.stderr) except Exception as ex: print("Error: %s." % ex, file=sys.stderr) def getParser(): + """Constructs and returns the CueAdmin argument parser.""" parser = argparse.ArgumentParser(description="CueAdmin OpenCue Administrator Tool", formatter_class=argparse.RawDescriptionHelpFormatter) @@ -225,12 +232,14 @@ def getParser(): host.add_argument("-repair", action="store_true", help="Sets hosts into the repair state.") host.add_argument("-fixed", action="store_true", help="Sets hosts into Up state.") host.add_argument("-thread", action="store", help="Set the host's thread mode.", - choices=[mode.lower() for mode in list(opencue.api.host_pb2.ThreadMode.keys())]) + choices=[ + mode.lower() for mode in list(opencue.api.host_pb2.ThreadMode.keys())]) return parser class QueryAction(argparse.Action): + """Sets various query modes if arguments are detected.""" def __call__(self, parser, namespace, values, option_string=None): if option_string == '-lh': namespace.lh = True @@ -338,46 +347,51 @@ def _convert(val): def resolveHostNames(names=None, substr=None): + """Returns a list of hosts using their names.""" items = [] if names: items = opencue.search.HostSearch.byName(names) - logger.debug("found %d of %d supplied hosts" % (len(items), len(names))) - if len(names) != len(items) and len(items): - logger.warn("Unable to match all host names with valid hosts on the cue.") - logger.warn("Operations executed for %s" % set(names).intersection( - [i.data.name for i in items])) - logger.warn("Operations NOT executed for %s" % set(names).difference([ - i.data.name for i in items])) + logger.debug("found %d of %d supplied hosts", len(items), len(names)) + if len(names) != len(items) and items: + logger.warning("Unable to match all host names with valid hosts on the cue.") + logger.warning( + "Operations executed for %s", set(names).intersection([i.data.name for i in items])) + logger.warning( + "Operations NOT executed for %s", set(names).difference( + [i.data.name for i in items])) elif substr: items = opencue.search.HostSearch.byMatch(substr) - logger.debug("matched %d hosts using patterns %s" % (len(items), substr)) + logger.debug("matched %d hosts using patterns %s", len(items), substr) if not items: raise ValueError("no valid hosts") return items def resolveShowNames(names): + """Returns a list of shows using their names.""" items = [] try: for name in names: items.append(opencue.api.findShow(name)) except opencue.CueException: pass - logger.debug("found %d of %d supplied shows" % (len(items), len(names))) - if len(names) != len(items) and len(items): - logger.warn("Unable to match all show names with active shows.") - logger.warn("Operations executed for %s" % set(names).intersection( + logger.debug("found %d of %d supplied shows", len(items), len(names)) + if len(names) != len(items) and items: + logger.warning("Unable to match all show names with active shows.") + logger.warning("Operations executed for %s", set(names).intersection( [i.data.name for i in items])) - logger.warn("Operations NOT executed for %s" % set(names).difference( + logger.warning("Operations NOT executed for %s", set(names).difference( [i.data.name for i in items])) if not items: raise ValueError("no valid shows") return items +# pylint: disable=inconsistent-return-statements def confirm(msg, force, func, *args, **kwargs): + """Prompts user for confirmation the given function should be executed.""" if cueadmin.util.promptYesNo("Please confirm. %s?" % msg, force): - logger.debug("%s [forced %s]" % (msg, force)) + logger.debug("%s [forced %s]", msg, force) return func(*args, **kwargs) @@ -385,45 +399,52 @@ def confirm(msg, force, func, *args, **kwargs): # need to be moved to the server, but should be someplace common # class DependUtil(object): + """Utility class for working with depends.""" @staticmethod def dropAllDepends(job, layer=None, frame=None): + """Drops all depends on the given object.""" if frame: - logger.debug("dropping all depends on: %s/%04d-%s" % (job, layer, frame)) + logger.debug("dropping all depends on: %s/%04d-%s", job, layer, frame) depend_er_frame = opencue.api.findFrame(job, layer, frame) for depend in depend_er_frame.getWhatThisDependsOn(): depend.proxy.satisfy() elif layer: - logger.debug("dropping all depends on: %s/%s" % (job, layer)) + logger.debug("dropping all depends on: %s/%s", job, layer) depend_er_layer = opencue.api.findLayer(job, layer) for depend in depend_er_layer.getWhatThisDependsOn(): depend.proxy.satisfy() else: - logger.debug("dropping all depends on: %s" % job) + logger.debug("dropping all depends on: %s", job) depend_er_job = opencue.api.findJob(job) for depend in depend_er_job.getWhatThisDependsOn(): - logger.debug("dropping depend %s %s" % (depend.data.type, opencue.id(depend))) + logger.debug("dropping depend %s %s", depend.data.type, opencue.id(depend)) depend.proxy.satisfy() class Convert(object): + """Utility class for converting between units.""" @staticmethod def gigsToKB(val): + """Converts gigabytes to kilobytes.""" return int(1048576 * val) @staticmethod def hoursToSeconds(val): + """Converts hours to seconds.""" return int(3600 * val) @staticmethod def stringToBoolean(val): + """Converts a string to a boolean, see code for values accepted as True.""" if val.lower() in ("yes", "on", "enabled", "true"): return True return False @staticmethod def strToMatchSubject(val): + """Converts a string to a MatchSubject.""" try: return getattr(opencue.api.filter_pb2, str(val).upper()) except Exception: @@ -431,6 +452,7 @@ def strToMatchSubject(val): @staticmethod def strToMatchType(val): + """Converts a string to a MatchType.""" try: return getattr(opencue.api.filter_pb2, str(val).upper()) except Exception: @@ -438,6 +460,7 @@ def strToMatchType(val): @staticmethod def strToActionType(val): + """Converts a string ActionType.""" try: return getattr(opencue.api.filter_pb2, str(val).upper()) except Exception: @@ -445,6 +468,7 @@ def strToActionType(val): @staticmethod def strToFrameState(val): + """Converts a string to a FrameState.""" try: return getattr(opencue.api.job_pb2, str(val).upper()) except Exception: @@ -452,6 +476,7 @@ def strToFrameState(val): @staticmethod def strToHardwareState(val): + """Converts a string to a HardwareState.""" try: return opencue.api.host_pb2.HardwareState.Value(str(val.upper())) except ValueError: @@ -467,9 +492,11 @@ def strToThreadMode(val): class ActionUtil(object): + """Utility class for interacting with Actions.""" @staticmethod def factory(actionType, value): + """Creates an Action.""" a = opencue.api.filter_pb2.Action() a.type = Convert.strToActionType(actionType) ActionUtil.setValue(a, value) @@ -477,24 +504,25 @@ def factory(actionType, value): @staticmethod def getValue(a): + """Gets an action's value.""" valueType = str(a.data.value_type) if valueType == "GroupType": return a.data.group_value - elif valueType == "StringType": + if valueType == "StringType": return a.data.string_value - elif valueType == "IntegerType": + if valueType == "IntegerType": return a.data.integer_value - elif valueType == "FloatType": + if valueType == "FloatType": return a.data.float_value - elif valueType == "BooleanType": + if valueType == "BooleanType": return a.data.boolean_value - else: - return None + return None @staticmethod def setValue(act, value): + """Sets an action's value.""" if act.type == opencue.api.filter_pb2.MOVE_JOB_TO_GROUP: - act.groupValue = opencue.proxy(value) + act.groupValue = opencue.proxy(value, 'Group') act.valueType = opencue.api.filter_pb2.GROUP_TYPE elif act.type == opencue.api.filter_pb2.PAUSE_JOB: @@ -523,16 +551,17 @@ def setValue(act, value): def handleArgs(args): + """Process the given arguments and execute the function described by them.""" if args.verbose: cueadmin.util.enableDebugLogging() if args.server: - logger.debug("setting opencue host servers to %s" % args.server) + logger.debug("setting opencue host servers to %s", args.server) opencue.Cuebot.setHosts(args.server) if args.facility: - logger.debug("setting facility to %s" % args.facility) + logger.debug("setting facility to %s", args.facility) opencue.Cuebot.setFacility(args.facility) # @@ -560,18 +589,18 @@ def handleArgs(args): [opencue.wrappers.proc.Proc(proc) for proc in result.procs.procs]) return - elif args.lh: + if args.lh: states = [Convert.strToHardwareState(s) for s in args.state] cueadmin.output.displayHosts( opencue.api.getHosts(match=args.query, state=states, alloc=args.alloc)) return - elif args.lba: + if args.lba: allocation = opencue.api.findAllocation(args.lba) cueadmin.output.displaySubscriptions(allocation.getSubscriptions(), "All Shows") return - elif args.lv is not None: + if args.lv is not None: if args.lv: show = opencue.api.findShow(args.lv[0]) cueadmin.output.displayServices(show.getServiceOverrides()) @@ -579,27 +608,27 @@ def handleArgs(args): cueadmin.output.displayServices(opencue.api.getDefaultServices()) return - elif args.lj: + if args.lj: for job in opencue.search.JobSearch.byMatch(args.query).jobs.jobs: print(job.name) return - elif args.lji: + if args.lji: cueadmin.output.displayJobs( [opencue.wrappers.job.Job(job) for job in opencue.search.JobSearch.byMatch(args.query).jobs.jobs]) return - elif args.la: + if args.la: cueadmin.output.displayAllocations(opencue.api.getAllocations()) return - elif args.lb: + if args.lb: for show in resolveShowNames(args.lb): cueadmin.output.displaySubscriptions(show.getSubscriptions(), show.data.name) return - elif args.ls: + if args.ls: cueadmin.output.displayShows(opencue.api.getShows()) return @@ -704,14 +733,14 @@ def handleArgs(args): if not hosts: raise ValueError(host_error_msg) for host in hosts: - logger.debug("locking host: %s" % opencue.rep(host)) + logger.debug("locking host: %s", opencue.rep(host)) host.lock() elif args.unlock: if not hosts: raise ValueError(host_error_msg) for host in hosts: - logger.debug("unlocking host: %s" % opencue.rep(host)) + logger.debug("unlocking host: %s", opencue.rep(host)) host.unlock() elif args.move: @@ -720,7 +749,7 @@ def handleArgs(args): def moveHosts(hosts_, dst_): for host_ in hosts_: - logger.debug("moving %s to %s" % (opencue.rep(host_), opencue.rep(dst_))) + logger.debug("moving %s to %s", opencue.rep(host_), opencue.rep(dst_)) host_.setAllocation(dst_) confirm("Move %d hosts to %s" % (len(hosts), args.move), @@ -732,7 +761,7 @@ def moveHosts(hosts_, dst_): def deleteHosts(hosts_): for host_ in hosts_: - logger.debug("deleting host: %s" % host_) + logger.debug("deleting host: %s", host_) host_.delete() confirm("Delete %s hosts" % len(hosts), args.force, deleteHosts, hosts) @@ -743,7 +772,7 @@ def deleteHosts(hosts_): def safeReboot(hosts_): for host_ in hosts_: - logger.debug("locking host and rebooting when idle %s" % opencue.rep(host_)) + logger.debug("locking host and rebooting when idle %s", opencue.rep(host_)) host_.rebootWhenIdle() confirm("Lock and reboot %d hosts when idle" % len(hosts), @@ -755,7 +784,7 @@ def safeReboot(hosts_): def setThreadMode(hosts_, mode): for host_ in hosts_: - logger.debug("setting host %s to thread mode %s" % (host_.data.name, mode)) + logger.debug("setting host %s to thread mode %s", host_.data.name, mode) host_.setThreadMode(Convert.strToThreadMode(mode)) confirm("Set %d hosts to thread mode %s" % (len(hosts), args.thread), args.force, @@ -767,7 +796,7 @@ def setThreadMode(hosts_, mode): def setRepairState(hosts_): for host_ in hosts_: - logger.debug("setting host into the repair state %s" % host_.data.name) + logger.debug("setting host into the repair state %s", host_.data.name) host_.setHardwareState(opencue.api.host_pb2.REPAIR) confirm("Set %d hosts into the Repair state?" % len(hosts), @@ -779,7 +808,7 @@ def setRepairState(hosts_): def setUpState(hosts_): for host_ in hosts_: - logger.debug("setting host into the repair state %s" % host_.data.name) + logger.debug("setting host into the repair state %s", host_.data.name) host_.setHardwareState(opencue.api.host_pb2.UP) confirm("Set %d hosts into the Up state?" % len(hosts), diff --git a/cueadmin/cueadmin/format.py b/cueadmin/cueadmin/format.py index 2e7357488..6321c7c33 100644 --- a/cueadmin/cueadmin/format.py +++ b/cueadmin/cueadmin/format.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Functions for formatting text output.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -21,20 +24,46 @@ def formatTime(epoch, time_format="%m/%d %H:%M", default="--/-- --:--"): - """Formats time using time formatting standards - see: http://docs.python.org/library/time.html""" + """Formats time using time formatting standards. + + See http://docs.python.org/library/time.html + + :type epoch: int + :param epoch: epoch timestamp to be string formatted + :type time_format: str + :param time_format: format the output string should follow, in time.strftime format + :type default: str + :param default: default string to be returned in the event the timestamp is blank + :rtype: str + :return: formatted time string""" if not epoch: return default return time.strftime(time_format, time.localtime(epoch)) def findDuration(start, stop): + """Provides a duration between two timestamps. + + If stop time is blank, current time will be used as a stand-in. + + :type start: int + :param start: start time as an epoch + :type stop: int + :param stop: stop time as an epoch + :rtype: int + :return: duration between the two timestamps""" if stop < 1: stop = int(time.time()) return stop - start def formatDuration(sec): + """Formats a duration in HH:MM:SS format. + + :type sec: int + :param sec: duration in seconds + :rtype: str + :return: duration formatted in HH:MM:SS format.""" def splitTime(seconds): minutes, seconds = divmod(seconds, 60) hour, minutes = divmod(minutes, 60) @@ -43,6 +72,12 @@ def splitTime(seconds): def formatLongDuration(sec): + """Formats a duration in days:hours format, preferable for very long durations. + + :type sec: int + :param sec: duration in seconds + :rtype: str + :return: duration formatted in days:hours format.""" def splitTime(seconds): minutes, seconds = divmod(seconds, 60) hour, minutes = divmod(minutes, 60) @@ -52,17 +87,31 @@ def splitTime(seconds): def formatMem(kmem, unit=None): + """Formats an amount of memory in human-friendly format. + + :type kmem: int + :param kmem: amount of memory in KB + :type unit: str + :param unit: unit to use for formatting, if blank the unit closest in size will be used + :rtype: str + :return: human-friendly formatted string""" k = 1024 if unit == "K" or not unit and kmem < k: return "%dK" % kmem if unit == "M" or not unit and kmem < pow(k, 2): return "%dM" % (kmem / k) - if unit == "G" or not unit and kmem < pow(k, 3): - return "%.01fG" % (float(kmem) / pow(k, 2)) + return "%.01fG" % (float(kmem) / pow(k, 2)) def cutoff(s, length): + """Truncates a string after a certain number of characters. + + :type s: str + :param s: string to be truncated + :type length: int + :param length: max number of characters + :rtype: str + :return: truncated string""" if len(s) < length-2: return s - else: - return "%s.." % s[0:length-2] + return "%s.." % s[0:length-2] diff --git a/cueadmin/cueadmin/output.py b/cueadmin/cueadmin/output.py index 4e4a9a19e..991a32b1f 100644 --- a/cueadmin/cueadmin/output.py +++ b/cueadmin/cueadmin/output.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Functions for displaying output to the terminal.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -23,6 +26,7 @@ import opencue import opencue.compiled_proto.job_pb2 +# pylint: disable=cyclic-import import cueadmin.common import cueadmin.format @@ -66,7 +70,8 @@ def displayHosts(hosts): "[ %0.2f / %s ]" % (host.data.idle_cores, cueadmin.format.formatMem(host.data.idle_memory)), host.data.os, - cueadmin.format.formatLongDuration(int(time.time()) - host.data.boot_time), + cueadmin.format.formatLongDuration( + int(time.time()) - host.data.boot_time), opencue.api.host_pb2.HardwareState.Name(host.data.state), opencue.api.host_pb2.LockState.Name(host.data.lock_state), host.data.alloc_name, @@ -212,8 +217,8 @@ def displayFrames(frames): stopTime = cueadmin.format.formatTime(frame.data.stop_time) if frame.data.start_time: - duration = cueadmin.format.formatDuration(cueadmin.format.findDuration(frame.data.start_time, - frame.data.stop_time)) + duration = cueadmin.format.formatDuration( + cueadmin.format.findDuration(frame.data.start_time, frame.data.stop_time)) else: duration = "" diff --git a/cueadmin/cueadmin/util.py b/cueadmin/cueadmin/util.py index dac80ac8e..bb51e6033 100644 --- a/cueadmin/cueadmin/util.py +++ b/cueadmin/cueadmin/util.py @@ -34,7 +34,7 @@ def enableDebugLogging(): - """enables debug logging for opencue and opencue tools""" + """Enables debug logging for opencue and opencue tools.""" logger = logging.getLogger("opencue") console = logging.StreamHandler() console.setLevel(logging.DEBUG) @@ -44,18 +44,15 @@ def enableDebugLogging(): def promptYesNo(prompt, force=False): - """Asks the user the supplied question and returns with a boolean to - indicate the users input. + """Asks the user to confirm or deny the prompted question. + @type prompt: string @param prompt: The question that the user can see @type force: boolean @param force: (Optional) If true, skips the prompt and returns true @rtype: bool - @return: The users response""" - try: - result = force or input("%s [y/n] " % prompt) in ("y", "Y") - except KeyboardInterrupt: - raise + @return: The user's response""" + result = force or input("%s [y/n] " % prompt) in ("y", "Y") if not result: print("Canceled") return result @@ -63,6 +60,7 @@ def promptYesNo(prompt, force=False): def waitOnJobName(jobName, maxWaitForLaunch=None): """Waits on the given job name to enter and then leave the queue. + @type jobName: str @param jobName: Full name of the job @type maxWaitForLaunch: int @@ -92,7 +90,5 @@ def waitOnJobName(jobName, maxWaitForLaunch=None): return False except opencue.CueException as e: print("Error: %s" % e, file=sys.stderr) - except Exception as e: - print("Error: %s" % e, file=sys.stderr) time.sleep(delay) diff --git a/cueadmin/tests/common_tests.py b/cueadmin/tests/common_tests.py index 6925d3cea..e35acd2f1 100644 --- a/cueadmin/tests/common_tests.py +++ b/cueadmin/tests/common_tests.py @@ -15,14 +15,18 @@ # limitations under the License. +"""Tests for cueadmin.common.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import from builtins import str -import mock import unittest +import mock + import opencue.compiled_proto.facility_pb2 import opencue.compiled_proto.host_pb2 import opencue.compiled_proto.job_pb2 @@ -698,21 +702,22 @@ def testListProcs(self, getStubMock, procSearchMock): args = self.parser.parse_args( ['-lp', TEST_SHOW, '-alloc', TEST_ALLOC, '-duration', '1.5', '-host', TEST_HOST, '-job', TEST_JOB, '-limit', resultsLimit, '-memory', '128']) - procSearchMock.byOptions.return_value = opencue.compiled_proto.host_pb2.ProcGetProcsResponse( - procs=opencue.compiled_proto.host_pb2.ProcSeq( - procs=[ - opencue.compiled_proto.host_pb2.Proc( - name='proc1', - reserved_cores=28, - used_memory=44, - reserved_memory=120, - job_name='mms2oazed2bbcjk60gho_w11licymr63s66bw1b3s', - frame_name='y0ihh3fxrstz6ub7ut2k', - dispatch_time=1556845762 - ) - ] + procSearchMock.byOptions.return_value = \ + opencue.compiled_proto.host_pb2.ProcGetProcsResponse( + procs=opencue.compiled_proto.host_pb2.ProcSeq( + procs=[ + opencue.compiled_proto.host_pb2.Proc( + name='proc1', + reserved_cores=28, + used_memory=44, + reserved_memory=120, + job_name='mms2oazed2bbcjk60gho_w11licymr63s66bw1b3s', + frame_name='y0ihh3fxrstz6ub7ut2k', + dispatch_time=1556845762 + ) + ] + ) ) - ) cueadmin.common.handleArgs(args) @@ -728,21 +733,22 @@ def testListFrameLogPaths(self, getStubMock, procSearchMock): args = self.parser.parse_args( ['-ll', TEST_SHOW, '-alloc', TEST_ALLOC, '-duration', '1.5', '-job', TEST_JOB, '-limit', resultsLimit, '-memory', '128']) - procSearchMock.byOptions.return_value = opencue.compiled_proto.host_pb2.ProcGetProcsResponse( - procs=opencue.compiled_proto.host_pb2.ProcSeq( - procs=[ - opencue.compiled_proto.host_pb2.Proc( - name='proc1', - reserved_cores=28, - used_memory=44, - reserved_memory=120, - job_name='mms2oazed2bbcjk60gho_w11licymr63s66bw1b3s', - frame_name='y0ihh3fxrstz6ub7ut2k', - dispatch_time=1556845762 - ) - ] + procSearchMock.byOptions.return_value = \ + opencue.compiled_proto.host_pb2.ProcGetProcsResponse( + procs=opencue.compiled_proto.host_pb2.ProcSeq( + procs=[ + opencue.compiled_proto.host_pb2.Proc( + name='proc1', + reserved_cores=28, + used_memory=44, + reserved_memory=120, + job_name='mms2oazed2bbcjk60gho_w11licymr63s66bw1b3s', + frame_name='y0ihh3fxrstz6ub7ut2k', + dispatch_time=1556845762 + ) + ] + ) ) - ) cueadmin.common.handleArgs(args) diff --git a/cueadmin/tests/output_tests.py b/cueadmin/tests/output_tests.py index 243c260d3..86305ac74 100644 --- a/cueadmin/tests/output_tests.py +++ b/cueadmin/tests/output_tests.py @@ -15,12 +15,17 @@ # limitations under the License. +"""Tests for cueadmin.output.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import +# pylint: disable=wrong-import-order,wrong-import-position from future import standard_library standard_library.install_aliases() + import contextlib import mock import io @@ -47,6 +52,9 @@ import cueadmin.output +# pylint: disable=line-too-long + + @contextlib.contextmanager def captured_output(): new_out, new_err = io.StringIO(), io.StringIO() From 04fcea370a9a7b36e73d77d3519cbcf349547c64 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 3 Feb 2021 17:04:47 +0000 Subject: [PATCH 044/277] Remove ProgressDelegate secton handling frame progress. (#903) --- cuegui/cuegui/ItemDelegate.py | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/cuegui/cuegui/ItemDelegate.py b/cuegui/cuegui/ItemDelegate.py index 2d399a108..2a554485e 100644 --- a/cuegui/cuegui/ItemDelegate.py +++ b/cuegui/cuegui/ItemDelegate.py @@ -427,28 +427,7 @@ def __init__(self, parent, *args): AbstractDelegate.__init__(self, parent, *args) def paint(self, painter, option, index): - if index.data(QtCore.Qt.UserRole) == cuegui.Constants.TYPE_FRAME: - frame = self.parent().itemFromIndex(index).rpcObject - opts = QtWidgets.QStyleOptionProgressBar() - opts.rect = option.rect - opts.minimum = 1 - opts.maximum = 100 - opts.textVisible = True - - if frame.data.state == opencue.api.job_pb2.SUCCEEDED: - progress = 100 - elif frame.data.state == opencue.api.job_pb2.RUNNING: - progress = int(cuegui.Progress.progress(frame.id())) - else: - progress = 0 - - opts.progress = progress - opts.text = "{0:d} %".format(progress) - opts.textVisible = True - - QtWidgets.QApplication.style().drawControl(QtWidgets.QStyle.CE_ProgressBar, opts, painter) - - elif index.data(QtCore.Qt.UserRole) == cuegui.Constants.TYPE_LAYER: + if index.data(QtCore.Qt.UserRole) == cuegui.Constants.TYPE_LAYER: layer = self.parent().itemFromIndex(index).rpcObject opts = QtWidgets.QStyleOptionProgressBar() opts.rect = option.rect From 6999b9062650eb06e062e4aa3ca09330f83fc2b1 Mon Sep 17 00:00:00 2001 From: Lars van der Bijl <285658+larsbijl@users.noreply.github.com> Date: Wed, 3 Feb 2021 17:51:26 +0000 Subject: [PATCH 045/277] Remove hinv and pexpect (#901) --- cuegui/cuegui/HostMonitorTree.py | 1 - cuegui/cuegui/MenuActions.py | 14 -------------- cuegui/setup.py | 1 - cuegui/tests/MenuActions_tests.py | 14 -------------- requirements.txt | 1 - 5 files changed, 31 deletions(-) diff --git a/cuegui/cuegui/HostMonitorTree.py b/cuegui/cuegui/HostMonitorTree.py index 6e0da13b0..6b88f8241 100644 --- a/cuegui/cuegui/HostMonitorTree.py +++ b/cuegui/cuegui/HostMonitorTree.py @@ -250,7 +250,6 @@ def contextMenuEvent(self, e): menu = QtWidgets.QMenu() self.__menuActions.hosts().addAction(menu, "viewComments") self.__menuActions.hosts().addAction(menu, "viewProc") - self.__menuActions.hosts().addAction(menu, "hinv") self.__menuActions.hosts().addAction(menu, "lock") self.__menuActions.hosts().addAction(menu, "unlock") self.__menuActions.hosts().addAction(menu, "addTags") diff --git a/cuegui/cuegui/MenuActions.py b/cuegui/cuegui/MenuActions.py index 6a59ebac5..8f6fac07f 100644 --- a/cuegui/cuegui/MenuActions.py +++ b/cuegui/cuegui/MenuActions.py @@ -30,7 +30,6 @@ import subprocess import time -import pexpect from PySide2 import QtGui from PySide2 import QtWidgets import six @@ -1212,19 +1211,6 @@ def viewProc(self, rpcObjects=None): if hosts: QtGui.qApp.view_procs.emit(hosts) - hinv_info = ["View Host Information (hinv)", None, "view"] - def hinv(self, rpcObjects=None): - hosts = self._getOnlyHostObjects(rpcObjects) - for host in hosts: - try: - lines = pexpect.run("rsh %s hinv" % host.data.name, timeout=10).splitlines() - QtWidgets.QMessageBox.information(self._caller, - "%s hinv" % host.data.name, - "\n".join(lines), - QtWidgets.QMessageBox.Ok) - except Exception as e: - logger.warning("Failed to get host's hinv: %s" % e) - lock_info = ["Lock Host", None, "lock"] def lock(self, rpcObjects=None): hosts = self._getOnlyHostObjects(rpcObjects) diff --git a/cuegui/setup.py b/cuegui/setup.py index d918a1bf0..eff519b23 100644 --- a/cuegui/setup.py +++ b/cuegui/setup.py @@ -61,7 +61,6 @@ 'future', 'grpcio', 'grpcio-tools', - 'pexpect', 'PySide2', 'PyYAML', ] diff --git a/cuegui/tests/MenuActions_tests.py b/cuegui/tests/MenuActions_tests.py index 8fd225589..a1490da44 100644 --- a/cuegui/tests/MenuActions_tests.py +++ b/cuegui/tests/MenuActions_tests.py @@ -1166,20 +1166,6 @@ def test_viewProc(self, qAppMock): qAppMock.view_procs.emit.assert_called_with([hostName]) - @mock.patch('PySide2.QtWidgets.QMessageBox') - @mock.patch('pexpect.run') - def test_hinv(self, runMock, qMessageBoxMock): - hostName = 'arbitrary-name' - host = opencue.wrappers.host.Host( - opencue.compiled_proto.host_pb2.Host(id='arbitrary-id', name=hostName)) - rshResponse = 'response line one\nanother response line' - runMock.return_value = rshResponse - - self.host_actions.hinv(rpcObjects=[host]) - - qMessageBoxMock.information.assert_called_with( - mock.ANY, '%s hinv' % hostName, rshResponse, mock.ANY) - def test_lock(self): host = opencue.wrappers.host.Host( opencue.compiled_proto.host_pb2.Host(id='arbitrary-id')) diff --git a/requirements.txt b/requirements.txt index 1778660a3..fc1123fad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,6 @@ grpcio==1.16.0 grpcio-tools==1.16.0 mock==2.0.0 pathlib==1.0.1;python_version<"3.4" -pexpect==4.6.0 psutil==5.6.6 pyfakefs==3.6 pylint==2.6.0;python_version>="3.7" From 6d84794b5c8090df5e20f3e9b855e754f47f7b2c Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 3 Feb 2021 21:37:53 +0000 Subject: [PATCH 046/277] Lint the OpenCue Python libraries. (#890) --- ci/pylintrc_test | 1 + ci/run_python_tests.sh | 4 + pycue/FileSequence/FrameRange.py | 49 +- pycue/FileSequence/FrameSet.py | 12 +- pycue/FileSequence/__init__.py | 20 + pycue/opencue/__init__.py | 4 + pycue/opencue/api.py | 59 ++- pycue/opencue/cuebot.py | 19 +- pycue/opencue/exception.py | 8 +- pycue/opencue/search.py | 61 ++- pycue/opencue/util.py | 31 +- pycue/opencue/version.py | 4 + pycue/opencue/wrappers/allocation.py | 81 ++-- pycue/opencue/wrappers/comment.py | 43 +- pycue/opencue/wrappers/deed.py | 92 ++-- pycue/opencue/wrappers/depend.py | 89 +++- pycue/opencue/wrappers/filter.py | 240 +++++++--- pycue/opencue/wrappers/frame.py | 170 ++++--- pycue/opencue/wrappers/group.py | 177 +++++-- pycue/opencue/wrappers/host.py | 249 +++++----- pycue/opencue/wrappers/job.py | 549 +++++++++++++--------- pycue/opencue/wrappers/layer.py | 283 ++++++----- pycue/opencue/wrappers/limit.py | 105 +++-- pycue/opencue/wrappers/owner.py | 70 ++- pycue/opencue/wrappers/proc.py | 104 ++-- pycue/opencue/wrappers/service.py | 94 ++-- pycue/opencue/wrappers/show.py | 230 +++++---- pycue/opencue/wrappers/subscription.py | 73 ++- pycue/opencue/wrappers/task.py | 19 +- pycue/opencue/wrappers/util.py | 63 ++- pycue/tests/api_test.py | 25 +- pycue/tests/file_sequence.py | 6 +- pycue/tests/search_test.py | 8 +- pycue/tests/util_test.py | 19 +- pycue/tests/wrappers/allocation_test.py | 11 +- pycue/tests/wrappers/comment_test.py | 6 +- pycue/tests/wrappers/deed_test.py | 6 +- pycue/tests/wrappers/depend_test.py | 6 +- pycue/tests/wrappers/filter_test.py | 154 +++--- pycue/tests/wrappers/frame_test.py | 8 +- pycue/tests/wrappers/group_test.py | 11 +- pycue/tests/wrappers/host_test.py | 7 +- pycue/tests/wrappers/job_test.py | 33 +- pycue/tests/wrappers/layer_test.py | 24 +- pycue/tests/wrappers/limit_test.py | 36 +- pycue/tests/wrappers/owner_test.py | 7 +- pycue/tests/wrappers/proc_test.py | 6 +- pycue/tests/wrappers/service_test.py | 6 +- pycue/tests/wrappers/show_test.py | 18 +- pycue/tests/wrappers/subscription_test.py | 8 +- pycue/tests/wrappers/task_test.py | 6 +- pycue/tests/wrappers/util_test.py | 2 + 52 files changed, 2074 insertions(+), 1342 deletions(-) diff --git a/ci/pylintrc_test b/ci/pylintrc_test index 1e71a142b..dc175b0b3 100644 --- a/ci/pylintrc_test +++ b/ci/pylintrc_test @@ -69,6 +69,7 @@ disable=duplicate-code, no-self-use, protected-access, raise-missing-from, + too-many-locals, too-many-public-methods, unused-argument, unused-variable, diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index 8e3b72e7e..a4797092f 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -22,6 +22,10 @@ python rqd/setup.py test # Some environments don't have pylint available, for ones that do they should pass this flag. if [[ "$1" == "--lint" ]]; then + cd pycue && python -m pylint --rcfile=../ci/pylintrc_main FileSequence && cd .. + cd pycue && python -m pylint --rcfile=../ci/pylintrc_main opencue --ignore=opencue/compiled_proto && cd .. + cd pycue && python -m pylint --rcfile=../ci/pylintrc_test tests && cd .. + cd pyoutline && PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main outline && cd .. cd pyoutline && PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests && cd .. diff --git a/pycue/FileSequence/FrameRange.py b/pycue/FileSequence/FrameRange.py index 1cce4a39f..6f565f26b 100644 --- a/pycue/FileSequence/FrameRange.py +++ b/pycue/FileSequence/FrameRange.py @@ -12,6 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Helper class for representing a frame range. + +It supports a complex syntax implementing features such as comma-separated frame ranges, +stepped frame ranges and more. See the FrameRange class for more detail. +""" from __future__ import division from __future__ import print_function @@ -25,11 +31,12 @@ class FrameRange(object): - """Represents a sequence of image frames.""" + """Represents a sequence of frame numbers.""" SINGLE_FRAME_PATTERN = re.compile(r'^(-?)\d+$') SIMPLE_FRAME_RANGE_PATTERN = re.compile(r'^(?P(-?)\d+)-(?P(-?)\d+)$') - STEP_PATTERN = re.compile(r'^(?P(-?)\d+)-(?P(-?)\d+)(?P[xy])(?P(-?)\d+)$') + STEP_PATTERN = re.compile( + r'^(?P(-?)\d+)-(?P(-?)\d+)(?P[xy])(?P(-?)\d+)$') INTERLEAVE_PATTERN = re.compile(r'^(?P(-?)\d+)-(?P(-?)\d+):(?P(-?)\d+)$') def __init__(self, frameRange): @@ -103,11 +110,20 @@ def getAll(self): return self.frameList def normalize(self): + """Sorts and deduplicates the sequence.""" self.frameList = list(set(self.frameList)) self.frameList.sort() @classmethod def parseFrameRange(cls, frameRange): + """ + Parse a string representation into a numerical sequence. + + :type frameRange: str + :param frameRange: String representation of the frame range. + :rtype: FrameRange + :return: FrameRange representing the numerical sequence. + """ singleFrameMatcher = re.match(cls.SINGLE_FRAME_PATTERN, frameRange) if singleFrameMatcher: return [int(frameRange)] @@ -116,7 +132,7 @@ def parseFrameRange(cls, frameRange): if simpleRangeMatcher: startFrame = int(simpleRangeMatcher.group('sf')) endFrame = int(simpleRangeMatcher.group('ef')) - return cls.getIntRange(startFrame, endFrame, (1 if endFrame >= startFrame else -1)) + return cls.__getIntRange(startFrame, endFrame, (1 if endFrame >= startFrame else -1)) rangeWithStepMatcher = re.match(cls.STEP_PATTERN, frameRange) if rangeWithStepMatcher: @@ -124,48 +140,49 @@ def parseFrameRange(cls, frameRange): endFrame = int(rangeWithStepMatcher.group('ef')) step = int(rangeWithStepMatcher.group('step')) stepSep = rangeWithStepMatcher.group('stepSep') - return cls.getSteppedRange(startFrame, endFrame, step, stepSep == 'y') + return cls.__getSteppedRange(startFrame, endFrame, step, stepSep == 'y') rangeWithInterleaveMatcher = re.match(cls.INTERLEAVE_PATTERN, frameRange) if rangeWithInterleaveMatcher: startFrame = int(rangeWithInterleaveMatcher.group('sf')) endFrame = int(rangeWithInterleaveMatcher.group('ef')) step = int(rangeWithInterleaveMatcher.group('step')) - return cls.getInterleavedRange(startFrame, endFrame, step) + return cls.__getInterleavedRange(startFrame, endFrame, step) raise ValueError('unrecognized frame range syntax ' + frameRange) @staticmethod - def getIntRange(start, end, step): + def __getIntRange(start, end, step): return list(range(start, end+(step // abs(step)), step)) @classmethod - def getSteppedRange(cls, start, end, step, inverseStep): - cls.validateStepSign(start, end, step) - steppedRange = cls.getIntRange(start, end, step) + def __getSteppedRange(cls, start, end, step, inverseStep): + cls.__validateStepSign(start, end, step) + steppedRange = cls.__getIntRange(start, end, step) if inverseStep: - fullRange = cls.getIntRange(start, end, (-1 if step < 0 else 1)) + fullRange = cls.__getIntRange(start, end, (-1 if step < 0 else 1)) return [frame for frame in fullRange if frame not in steppedRange] return steppedRange @classmethod - def getInterleavedRange(cls, start, end, step): - cls.validateStepSign(start, end, step) + def __getInterleavedRange(cls, start, end, step): + cls.__validateStepSign(start, end, step) interleavedFrames = OrderedDict() incrValue = step // abs(step) while abs(step) > 0: - interleavedFrames.update([(frame, None) for frame in cls.getIntRange(start, end, step)]) + interleavedFrames.update( + [(frame, None) for frame in cls.__getIntRange(start, end, step)]) start += incrValue step = int(step / 2.0) return list(interleavedFrames.keys()) @staticmethod - def validateStepSign(start, end, step): + def __validateStepSign(start, end, step): if step > 1 and end < start: raise ValueError( 'end frame may not be less than start frame when using a positive step') - elif step == 0: + if step == 0: raise ValueError('step cannot be zero') - elif step < 0 and end >= start: + if step < 0 and end >= start: raise ValueError( 'end frame may not be greater than start frame when using a negative step') diff --git a/pycue/FileSequence/FrameSet.py b/pycue/FileSequence/FrameSet.py index b4c45201c..5ff1629e4 100644 --- a/pycue/FileSequence/FrameSet.py +++ b/pycue/FileSequence/FrameSet.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Module for `FileSequence.FrameSet`.""" from __future__ import absolute_import from __future__ import print_function @@ -23,7 +24,7 @@ class FrameSet(object): - """Represents a sequence of FrameRanges.""" + """Represents a sequence of `FileSequence.FrameRange`.""" def __init__(self, frameRange): """Construct a FrameSet object by parsing a spec. @@ -67,11 +68,20 @@ def getAll(self): return self.frameList def normalize(self): + """Sorts and dedeuplicates the sequence.""" self.frameList = list(set(self.frameList)) self.frameList.sort() @staticmethod def parseFrameRange(frameRange): + """ + Parses a string representation of a frame range into a FrameSet. + + :type frameRange: str + :param frameRange: String representation of the frame range. + :rtype: FrameSet + :return: The FrameSet representing the same sequence. + """ frameList = list() for frameRangeSection in frameRange.split(','): frameList.extend(FrameRange.parseFrameRange(frameRangeSection)) diff --git a/pycue/FileSequence/__init__.py b/pycue/FileSequence/__init__.py index 75ee7dd87..711e1dc83 100644 --- a/pycue/FileSequence/__init__.py +++ b/pycue/FileSequence/__init__.py @@ -1,3 +1,23 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Top-level module of FileSequence. + +FileSequence contains helper classes for representing a job's frame range. +""" + from __future__ import absolute_import from __future__ import print_function from __future__ import division diff --git a/pycue/opencue/__init__.py b/pycue/opencue/__init__.py index 3b90531d1..2ceca5f53 100644 --- a/pycue/opencue/__init__.py +++ b/pycue/opencue/__init__.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Top level of the opencue module.""" from __future__ import absolute_import from __future__ import print_function @@ -19,6 +20,7 @@ import logging +# pylint: disable=cyclic-import from .cuebot import Cuebot from . import api from . import wrappers @@ -26,7 +28,9 @@ from .exception import CueException from .exception import EntityNotFoundException +# pylint: disable=redefined-builtin from .util import id +# pylint: enable=redefined-builtin from .util import logPath from .util import proxy from .util import rep diff --git a/pycue/opencue/api.py b/pycue/opencue/api.py index b80ccce73..10dbed4cc 100644 --- a/pycue/opencue/api.py +++ b/pycue/opencue/api.py @@ -12,19 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""The OpenCue static API.""" - -""" -The opencue Static API. This is exported into the package namespace. - -Project: opencue Library -""" from __future__ import absolute_import from __future__ import print_function from __future__ import division -from . import search -from . import util from opencue.compiled_proto import comment_pb2 from opencue.compiled_proto import criterion_pb2 from opencue.compiled_proto import cue_pb2 @@ -42,6 +35,7 @@ from opencue.compiled_proto import subscription_pb2 from opencue.compiled_proto import task_pb2 from .cuebot import Cuebot +# pylint: disable=cyclic-import from .wrappers.allocation import Allocation from .wrappers.comment import Comment from .wrappers.depend import Depend @@ -60,11 +54,13 @@ from .wrappers.show import Show from .wrappers.subscription import Subscription from .wrappers.task import Task +from . import search +from . import util __protobufs = [comment_pb2, criterion_pb2, cue_pb2, department_pb2, depend_pb2, facility_pb2, - filter_pb2, host_pb2, job_pb2, renderPartition_pb2, report_pb2, service_pb2, show_pb2, - subscription_pb2, task_pb2] + filter_pb2, host_pb2, job_pb2, renderPartition_pb2, report_pb2, service_pb2, + show_pb2, subscription_pb2, task_pb2] __wrappers = [Action, Allocation, Comment, Depend, Filter, Frame, Group, Host, Job, Layer, Matcher, NestedHost, Proc, Show, Subscription, Task] @@ -319,7 +315,7 @@ def getJobs(**options): For example:: # returns only pipe jobs. - getJobs(show=["pipe"]) + getJobs(show=["pipe"]) Possible args: - job: job names - list @@ -391,8 +387,6 @@ def getJobNames(**options): """Returns a list of job names that match the search parameters. See getJobs for the job query options. - :type options: dict - :param options: a variable list of search criteria :rtype: list :return: List of matching str job names""" criteria = search.JobSearch.criteriaFromOptions(**options) @@ -557,15 +551,15 @@ def getHost(uniq): # Owners # @util.grpcExceptionParser -def getOwner(id): +def getOwner(owner_id): """Return an Owner object from the ID or name. - :type id: str - :param id: a unique owner identifier or name + :type owner_id: str + :param owner_id: a unique owner identifier or name :rtype: Owner :return: An Owner object""" return Owner(Cuebot.getStub('owner').GetOwner( - host_pb2.OwnerGetOwnerRequest(name=id), timeout=Cuebot.Timeout).owner) + host_pb2.OwnerGetOwnerRequest(name=owner_id), timeout=Cuebot.Timeout).owner) # # Filters @@ -640,12 +634,27 @@ def getAllocation(allocId): @util.grpcExceptionParser def deleteAllocation(alloc): + """Deletes an allocation. + + :type alloc: facility_pb2.Allocation + :param alloc: allocation to delete + :rtype: facility_pb2.AllocDeleteResponse + :return: empty response""" return Cuebot.getStub('allocation').Delete( facility_pb2.AllocDeleteRequest(allocation=alloc), timeout=Cuebot.Timeout) @util.grpcExceptionParser def allocSetBillable(alloc, is_billable): + """Sets an allocation billable or not. + + :type alloc: facility_pb2.Allocation + :param alloc: allocation to set + :type is_billable: bool + :param is_billable: whether alloc should be billable or not + :rtype: facility_pb2.AllocSetBillableResponse + :return: empty response + """ return Cuebot.getStub('allocation').SetBillable( facility_pb2.AllocSetBillableRequest(allocation=alloc, value=is_billable), timeout=Cuebot.Timeout) @@ -653,12 +662,28 @@ def allocSetBillable(alloc, is_billable): @util.grpcExceptionParser def allocSetName(alloc, name): + """Sets an allocation name. + + :type alloc: facility_pb2.Allocation + :param alloc: allocation to set + :type name: str + :param name: new name for the allocation + :rtype: facility_pb2.AllocSetNameResponse + :return: empty response""" return Cuebot.getStub('allocation').SetName( facility_pb2.AllocSetNameRequest(allocation=alloc, name=name), timeout=Cuebot.Timeout) @util.grpcExceptionParser def allocSetTag(alloc, tag): + """Sets an allocation tag. + + :type alloc: facility_pb2.Allocation + :param alloc: allocation to tag + :type tag: str + :param tag: new tag + :rtype: facility_pb2.AllocSetTagResponse + :return: empty response""" return Cuebot.getStub('allocation').SetTag( facility_pb2.AllocSetTagRequest(allocation=alloc, tag=tag), timeout=Cuebot.Timeout) diff --git a/pycue/opencue/cuebot.py b/pycue/opencue/cuebot.py index f3e97bb02..1211c9fe4 100644 --- a/pycue/opencue/cuebot.py +++ b/pycue/opencue/cuebot.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Module for communicating with the Cuebot server(s).""" from __future__ import print_function from __future__ import division @@ -20,11 +21,12 @@ from builtins import object from random import shuffle import atexit -import grpc import logging import os import yaml +import grpc + from opencue.compiled_proto import comment_pb2 from opencue.compiled_proto import comment_pb2_grpc from opencue.compiled_proto import criterion_pb2 @@ -175,8 +177,9 @@ def setChannel(): # Test the connection Cuebot.getStub('cue').GetSystemStats( cue_pb2.CueGetSystemStatsRequest(), timeout=Cuebot.Timeout) + # pylint: disable=broad-except except Exception: - logger.warning('Could not establish grpc channel with {}.'.format(connectStr)) + logger.warning('Could not establish grpc channel with %s', connectStr) continue atexit.register(Cuebot.closeChannel) return None @@ -206,10 +209,9 @@ def setFacility(facility): :param facility: a facility named in the config file""" if facility not in list(config.get("cuebot.facility").keys()): default = config.get("cuebot.facility_default") - logger.warning("The facility '%s' does not exist, defaulting to %s"% - (facility, default)) + logger.warning("The facility '%s' does not exist, defaulting to %s", facility, default) facility = default - logger.debug("setting facility to: %s" % facility) + logger.debug("setting facility to: %s", facility) hosts = config.get("cuebot.facility")[facility] Cuebot.setHosts(hosts) @@ -221,7 +223,7 @@ def setHosts(hosts): :type hosts: list or str""" if isinstance(hosts, str): hosts = [hosts] - logger.debug("setting new server hosts to: %s" % hosts) + logger.debug("setting new server hosts to: %s", hosts) Cuebot.Hosts = hosts Cuebot.resetChannel() @@ -232,11 +234,12 @@ def setTimeout(timeout): :param timeout: The network connection timeout in millis. :type timeout: int """ - logger.debug("setting new server timeout to: %d" % timeout) + logger.debug("setting new server timeout to: %d", timeout) Cuebot.Timeout = timeout @classmethod def getProto(cls, name): + """Returns a proto class for the given name.""" proto = cls.PROTO_MAP.get(name) if proto is None: raise ValueError("Could not find proto for {}.".format(name)) @@ -244,6 +247,7 @@ def getProto(cls, name): @classmethod def getService(cls, name): + """Returns the service for the given name.""" service = cls.SERVICE_MAP.get(name) if service is None: raise ValueError("Could not find stub interface for {}.".format(name)) @@ -264,4 +268,5 @@ def getStub(cls, name): @staticmethod def getConfig(): + """Gets the Cuebot config object, originally read in from the config file on disk.""" return config diff --git a/pycue/opencue/exception.py b/pycue/opencue/exception.py index 5c31a6441..cc15a6902 100644 --- a/pycue/opencue/exception.py +++ b/pycue/opencue/exception.py @@ -12,12 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Custom exception classes for API error handling.""" -""" -Project: opencue Library - -Module: exception.py - Provides opencue access to exceptions -""" from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -58,6 +54,7 @@ class CueInternalErrorException(CueException): failMsg = 'Server caught an internal exception. {details}' retryMsg = 'Server caught an internal exception, checking again...' + class ConnectionException(CueException): """Raised when unable to connect to grpc server.""" failMsg = 'Unable to contact grpc server. {details}' @@ -78,4 +75,3 @@ def getRetryCount(): grpc.StatusCode.INTERNAL: CueInternalErrorException, grpc.StatusCode.UNAVAILABLE: ConnectionException } - diff --git a/pycue/opencue/search.py b/pycue/opencue/search.py index 9e4776724..995cf6e51 100644 --- a/pycue/opencue/search.py +++ b/pycue/opencue/search.py @@ -12,11 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Client side implementation of search criteria. -""" -Client side implementation of search criteria. -This module provides some easy factory -methods to do common search operations. It also exposes +This module provides some easy factory methods to do common search operations. It also exposes lower level RPC functionality for procedural searches. ============== @@ -33,7 +31,7 @@ s.shows.append("pipe") s.users.append("chambers") jobs = s.find() - + A procedural example searching by regular expression:: s = JobSearch() @@ -48,15 +46,16 @@ job.proxy.kill() """ + from __future__ import absolute_import from __future__ import print_function from __future__ import division - from builtins import object import logging import six +# pylint: disable=cyclic-import from opencue.compiled_proto import criterion_pb2 from opencue.compiled_proto import host_pb2 from opencue.compiled_proto import job_pb2 @@ -73,50 +72,56 @@ class BaseSearch(object): + """Base class for searching.""" + def __init__(self, **options): self.options = options def search(self): + """Executes the search using the options provided at initiation.""" return self.byOptions(**self.options) @classmethod def byOptions(cls, **options): + """Executes the search using the provided options.""" raise NotImplementedError class ProcSearch(BaseSearch): - """See: help(opencue.getProcs)""" - def __init__(self, **options): - super(ProcSearch, self).__init__(**options) + """Class for searching for procs. + + See: help(opencue.getProcs)""" @staticmethod def criteriaFromOptions(**options): + """Constructs a search criteria object for the given options.""" return _setOptions(host_pb2.ProcSearchCriteria(), options) @classmethod def byOptions(cls, **options): + """Executes the search using the given options.""" criteria = cls.criteriaFromOptions(**options) return Cuebot.getStub('proc').GetProcs( host_pb2.ProcGetProcsRequest(r=criteria), timeout=Cuebot.Timeout) class FrameSearch(BaseSearch): + """Class for searching for frames.""" page = 1 limit = 1000 change_date = 0 - def __init__(self, **options): - super(FrameSearch, self).__init__(**options) - @classmethod def criteriaFromOptions(cls, **options): + """Constructs a search criteria object for the given options.""" criteria = _setOptions(job_pb2.FrameSearchCriteria(), options) criteria.page = options.get('page', cls.page) criteria.limit = options.get('limit', cls.limit) criteria.change_date = options.get('change_date', cls.change_date) return criteria + # pylint: disable=arguments-differ @classmethod def byOptions(cls, job, **options): criteria = cls.criteriaFromOptions(**options) @@ -125,15 +130,16 @@ def byOptions(cls, job, **options): @classmethod def byRange(cls, job, val): + """Executes a search by frame range.""" cls.byOptions(job, frame_range=val) class HostSearch(BaseSearch): - def __init__(self, **options): - super(HostSearch, self).__init__(**options) + """Class for searching for hosts.""" @staticmethod def criteriaFromOptions(**options): + """Constructs a search criteria object for the given options.""" return _setOptions(host_pb2.HostSearchCriteria(), options) @classmethod @@ -145,31 +151,36 @@ def byOptions(cls, **options): @classmethod def byName(cls, val): + """Searches for a host by name.""" return cls.byOptions(name=val) @classmethod def byRegex(cls, val): + """Searches for a host by regular expression.""" return cls.byOptions(regex=val) @classmethod def byId(cls, val): + """Searches for a host by id.""" return cls.byOptions(id=val) @classmethod def byMatch(cls, val): + """Searches for a host by substring match.""" return cls.byOptions(substr=val) @classmethod def byAllocation(cls, val): + """Searches for a host by allocation.""" return cls.byOptions(alloc=val) class JobSearch(BaseSearch): - def __init__(self, **options): - super(JobSearch, self).__init__(**options) + """Class for searching for jobs.""" @staticmethod def criteriaFromOptions(**options): + """Constructs a search criteria object for the given options.""" return _setOptions(job_pb2.JobSearchCriteria(), options) @classmethod @@ -180,30 +191,37 @@ def byOptions(cls, **options): @classmethod def byName(cls, val): + """Searches for a job by name.""" return cls.byOptions(job=val) @classmethod def byId(cls, val): + """Searches for a job by id.""" return cls.byOptions(id=val) @classmethod def byRegex(cls, val): + """Searches for a job by regex.""" return cls.byOptions(regex=val) @classmethod def byMatch(cls, val): + """Searches for a job by substring match.""" return cls.byOptions(substr=val) @classmethod def byShow(cls, val): + """Searches for a job by show.""" return cls.byOptions(show=val) @classmethod def byShot(cls, val): + """Searches for a job by shot.""" return cls.byOptions(shots=val) @classmethod def byUser(cls, val): + """Searches for a job by user.""" return cls.byOptions(user=val) @@ -255,15 +273,17 @@ def _convert(val): criterion = getattr(criterion_pb2, "GreaterThan%sSearchCriterion" % searchTypeStr) return criterion(_convert(search[2:])) - elif search.startswith("lt"): + + if search.startswith("lt"): criterion = getattr(criterion_pb2, "LessThan%sSearchCriterion" % searchTypeStr) return criterion(_convert(search[2:])) - elif search.find("-") > -1: + + if search.find("-") > -1: criterion = getattr(criterion_pb2, "InRange%sSearchCriterion" % searchTypeStr) - min, max = search.split("-") - return criterion(_convert(min), _convert(max)) + min_range, max_range = search.split("-") + return criterion(_convert(min_range), _convert(max_range)) raise ValueError("Unable to parse this format: %s" % search) @@ -275,6 +295,7 @@ def _raiseIfNotType(searchOption, value, expectedType): def raiseIfNotList(searchOption, value): + """Raises an exception if the provided value is not a list.""" _raiseIfNotType(searchOption, value, list) diff --git a/pycue/opencue/util.py b/pycue/opencue/util.py index 63ee5fc36..1eb704696 100644 --- a/pycue/opencue/util.py +++ b/pycue/opencue/util.py @@ -12,12 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - -""" -Project: opencue Library -Module: util.py -""" - +"""Utility methods used throughout the opencue module.""" from __future__ import absolute_import from __future__ import print_function @@ -26,12 +21,13 @@ from builtins import str import functools import future.utils -import grpc import logging import os -import six import time +import six +import grpc + import opencue logger = logging.getLogger('opencue') @@ -47,8 +43,10 @@ def _decorator(*args, **kwargs): try: return grpcFunc(*args, **kwargs) except grpc.RpcError as exc: + # pylint: disable=no-member code = exc.code() details = exc.details() or "No details found. Check server logs." + # pylint: enable=no-member exception = opencue.exception.EXCEPTION_MAP.get(code) if exception: if exception.retryable and triesRemaining >= 1: @@ -65,6 +63,7 @@ def _decorator(*args, **kwargs): return functools.wraps(grpcFunc)(_decorator) +# pylint: disable=redefined-builtin def id(value): """extract(entity) extracts a string unique ID from a opencue entity or @@ -73,14 +72,14 @@ def id(value): def _extract(item): try: return item.id() + # pylint: disable=bare-except except: pass return item if isinstance(value, (tuple, list, set)): return [_extract(v) for v in value] - else: - return _extract(value) + return _extract(value) @grpcExceptionParser @@ -99,8 +98,7 @@ def _proxy(idString): requestor = getattr(proto, "{cls}Get{cls}Request".format(cls=cls)) getMethod = getattr(opencue.Cuebot.getStub(cls.lower()), "Get{}".format(cls)) return getMethod(requestor(id=idString)) - else: - raise AttributeError('Could not find a proto for {}'.format(cls)) + raise AttributeError('Could not find a proto for {}'.format(cls)) def _proxies(entities): messages = [] @@ -113,10 +111,9 @@ def _proxies(entities): if hasattr(idOrObject, 'id'): return _proxy(idOrObject.id) - elif isinstance(idOrObject, six.string_types): + if isinstance(idOrObject, six.string_types): return _proxy(idOrObject) - else: - return _proxies(idOrObject) + return _proxies(idOrObject) def rep(entity): @@ -124,6 +121,7 @@ def rep(entity): Extracts a string repesentation of a opencue entity""" try: return entity.name + # pylint: disable=bare-except except: return str(entity) @@ -134,5 +132,4 @@ def logPath(job, frame=None): """ if frame: return os.path.join(job.data.log_dir, "%s.%s.rqlog" % (job.data.name, frame.data.name)) - else: - return job.data.log_dir + return job.data.log_dir diff --git a/pycue/opencue/version.py b/pycue/opencue/version.py index c353f0689..4c7df1636 100644 --- a/pycue/opencue/version.py +++ b/pycue/opencue/version.py @@ -12,5 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Stores the API version.""" + +# TODO(bcipriano): This module can probably be removed and replaced with the newer +# VERSION file system. version = '5.0' diff --git a/pycue/opencue/wrappers/allocation.py b/pycue/opencue/wrappers/allocation.py index 2f7f52ec9..8539dbf4e 100644 --- a/pycue/opencue/wrappers/allocation.py +++ b/pycue/opencue/wrappers/allocation.py @@ -12,11 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - -""" -allocation module -""" - +"""Module for classes related to OpenCue allocations.""" from opencue.compiled_proto import facility_pb2 from opencue.compiled_proto import host_pb2 @@ -33,16 +29,15 @@ def __init__(self, allocation=None): self.stub = Cuebot.getStub('allocation') def delete(self): - """Delete the record of the allocation from the cuebot""" + """Deletes the allocation.""" self.stub.Delete( facility_pb2.AllocDeleteRequest(allocation=self.data), - timeout=Cuebot.Timeout - ) + timeout=Cuebot.Timeout) def getHosts(self): - """Returns the list of hosts for this allocation. + """Returns the list of hosts for the allocation. - :rtype: list + :rtype: list :return: list of hosts """ hostSeq = self.stub.GetHosts(facility_pb2.AllocGetHostsRequest(allocation=self.data), @@ -50,34 +45,36 @@ def getHosts(self): return [opencue.wrappers.host.Host(h) for h in hostSeq.hosts] def getSubscriptions(self): - """Get the subscriptions of this allocation. + """Returns the subscriptions of the allocation. - :rtype: list + :rtype: list :return: a list of subscriptions """ subscriptionSeq = self.stub.GetSubscriptions( facility_pb2.AllocGetSubscriptionsRequest(allocation=self.data), timeout=Cuebot.Timeout).subscriptions - return [opencue.wrappers.subscription.Subscription(sub) for sub in subscriptionSeq.subscriptions] + return [opencue.wrappers.subscription.Subscription(sub) + for sub in subscriptionSeq.subscriptions] def reparentHosts(self, hosts): - """Moves the given hosts to the allocation + """Moves the given hosts into the allocation. :type hosts: list - :param hosts: The hosts to move to this allocation + :param hosts: the hosts to move to this allocation """ hostSeq = host_pb2.HostSeq() + # pylint: disable=no-member hostSeq.hosts.extend([host.data for host in hosts]) + # pylint: enable=no-member self.stub.ReparentHosts( facility_pb2.AllocReparentHostsRequest(allocation=self.data, hosts=hostSeq), - timeout=Cuebot.Timeout - ) - + timeout=Cuebot.Timeout) + def reparentHostIds(self, hostIds): - """Moves the given hosts to the allocation + """Moves the hosts identified by the given host ids into the allocation. :type hostIds: list - :param hostIds: The host ids to move to this allocation + :param hostIds: the host ids to move to this allocation """ hosts = [opencue.wrappers.host.Host(host_pb2.Host(id=hostId)) for hostId in hostIds] self.reparentHosts(hosts) @@ -85,7 +82,7 @@ def reparentHostIds(self, hostIds): def setName(self, name): """Sets a new name for the allocation. - :type name: str + :type name: str :param name: the new name """ self.stub.SetName( @@ -95,35 +92,34 @@ def setName(self, name): def setTag(self, tag): """Sets a new tag for the allocation. - :type name: str - :param name: the new tag + :type tag: str + :param tag: the new tag """ self.stub.SetTag( facility_pb2.AllocSetTagRequest(allocation=self.data, tag=tag), - timeout=Cuebot.Timeout - ) + timeout=Cuebot.Timeout) def id(self): - """Returns the id of the allocation + """Returns the id of the allocation. :rtype: str - :return: Allocation uuid + :return: the id of the allocation """ return self.data.id def name(self): - """Returns the name of the allocation + """Returns the name of the allocation. :rtype: str - :return: Allocation name + :return: the name of the allocation """ return self.data.name def tag(self): - """Returns the allocation tag + """Returns the tag of the allocation. :rtype: str - :return: Allocation tag + :return: the tag of the allocation """ return self.data.tag @@ -131,16 +127,15 @@ def totalCores(self): """Returns the total number of cores in the allocation. :rtype: float - :return: Total number of cores in the allocation + :return: total number of cores in the allocation """ return self.data.stats.cores def totalAvailableCores(self): - """Returns the total number of cores available for - booking in the allocation. + """Returns the total number of cores available for booking in the allocation. :rtype: float - :return: Total number of cores in the allocation + :return: total number of cores in the allocation """ return self.data.stats.available_cores @@ -148,26 +143,27 @@ def totalIdleCores(self): """Returns the total number of idle cores in the allocation. :rtype: float - :return: Total number of idle cores in the allocation + :return: total number of idle cores in the allocation """ return self.data.stats.idle_cores def totalRunningCores(self): """Returns the total number of running cores in the allocation. + Each 100 returned is the same as 1 physical core. :rtype: float - :return: Total number of running cores in the allocation + :return: total number of running cores in the allocation """ - # All core reserved return self.data.stats.running_cores def totalLockedCores(self): """Returns the total number of locked cores in the allocation. + Each 100 returned is the same as 1 physical core. :rtype: float - :return: Total number of locked cores in the allocation + :return: total number of locked cores in the allocation """ return self.data.stats.locked_cores @@ -175,7 +171,7 @@ def totalHosts(self): """Returns the total number of hosts in the allocation. :rtype: int - :return: Total number of hosts in the allocation + :return: total number of hosts in the allocation """ return self.data.stats.hosts @@ -183,13 +179,14 @@ def totalLockedHosts(self): """Returns the total number of locked hosts in the allocation. :rtype: int - :return: Total number of locked hosts in the allocation""" + :return: total number of locked hosts in the allocation + """ return self.data.stats.locked_hosts def totalDownHosts(self): """Returns the total number of down hosts in the allocation. :rtype: int - :return: Total number of down hosts in the allocation + :return: total number of down hosts in the allocation """ return self.data.stats.down_hosts diff --git a/pycue/opencue/wrappers/comment.py b/pycue/opencue/wrappers/comment.py index ad0dd53d6..e229ec283 100644 --- a/pycue/opencue/wrappers/comment.py +++ b/pycue/opencue/wrappers/comment.py @@ -12,14 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - - -""" -Project: opencue - -Module: comment.py - comment object - -""" +"""Module for classes related to comments.""" from opencue.compiled_proto import comment_pb2 from opencue.cuebot import Cuebot @@ -33,52 +26,58 @@ def __init__(self, comment=None): self.stub = Cuebot.getStub('comment') def delete(self): - """Delete this comment""" - self.stub.Delete(comment_pb2.CommentDeleteRequest(comment=self.data), timeout=Cuebot.Timeout) + """Deletes this comment.""" + self.stub.Delete( + comment_pb2.CommentDeleteRequest(comment=self.data), timeout=Cuebot.Timeout) def save(self): - """Saves the current comment values""" + """Saves the current comment values.""" self.stub.Save(comment_pb2.CommentSaveRequest(comment=self.data), timeout=Cuebot.Timeout) def message(self): - """Message of the comment + """Returns the message of the comment. :rtype: str - :return: comment message""" + :return: comment message + """ return self.data.message def subject(self): - """Subject of the comment + """Returns the subject of the comment. :rtype: str - :return: comment subject""" + :return: comment subject + """ return self.data.subject def user(self): """Returns the username of the user who submitted the comment. :rtype: str - :return: Username of submitter""" + :return: username of the commenter + """ return self.data.user def timestamp(self): """Returns the timestamp for the comment as an epoch. :rtype: int - :return: The time the comment was submitted as an epoch""" + :return: the time the comment was submitted as an epoch + """ return self.data.timestamp def setMessage(self, message): - """Set a new message for the comment. + """Sets a new message for the comment. :type message: str - :param message: a new message""" + :param message: a new message + """ self.data.message = message def setSubject(self, subject): - """Set a new subject for the comment. + """Sets a new subject for the comment. :type subject: str - :param subject: a new subject""" + :param subject: a new subject + """ self.data.subject = subject - diff --git a/pycue/opencue/wrappers/deed.py b/pycue/opencue/wrappers/deed.py index dafb587c3..d29ec8854 100644 --- a/pycue/opencue/wrappers/deed.py +++ b/pycue/opencue/wrappers/deed.py @@ -12,45 +12,42 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Module for classes related to deeds.""" - -""" -Project: opencue - -Module: deed.py - deed object - -""" - -import opencue.wrappers.host from opencue.compiled_proto import host_pb2 from opencue.cuebot import Cuebot +# pylint: disable=cyclic-import +import opencue.wrappers.host +import opencue.wrappers.owner class Deed(object): """This class contains the grpc implementation related to a Deed.""" - def __init__(self, comment=None): - self.data = comment + def __init__(self, deed=None): + self.data = deed self.stub = Cuebot.getStub('comment') def delete(self): - """Delete this comment""" + """Deletes the deed.""" self.stub.Delete(host_pb2.DeedDeleteRequest(deed=self.data), timeout=Cuebot.Timeout) def getHost(self): - """Return the host for this deed. + """Returns the host for the deed. - :rtype: opencue.wrappers.host.Host Wrapper - :return: Host associated with this deed""" + :rtype: opencue.wrappers.host.Host + :return: deed host + """ return opencue.wrappers.host.Host( self.stub.GetHost(host_pb2.DeedGetHostRequest(deed=self.data), timeout=Cuebot.Timeout).host) def getOwner(self): - """Returns the owner for these settings. + """Returns the owner of the deed. - :rtype: opencue.wrappers.host.Host - :return: Owner of this deed""" + :rtype: opencue.wrappers.owner.Owner + :return: deed owner + """ return opencue.wrappers.owner.Owner( self.stub.GetOwner(host_pb2.DeedGetOwnerRequest(deed=self.data), timeout=Cuebot.Timeout).owner) @@ -58,43 +55,78 @@ def getOwner(self): def setBlackoutTime(self, startTime, stopTime): """Sets a blackout time for the host. - :type startTime: int - :param startTime: blackout start time - :type stopTime: int - :param stopTime: blackout stop time""" + :type startTime: int + :param startTime: blackout start time as an epoch + :type stopTime: int + :param stopTime: blackout stop time as an epoch + """ self.stub.SetBlackoutTime( - host_pb2.DeedSetBlackoutTimeRequest(deed=self.data, - start_time=startTime, - stop_time=stopTime), + host_pb2.DeedSetBlackoutTimeRequest( + deed=self.data, start_time=startTime, stop_time=stopTime), timeout=Cuebot.Timeout) def setBlackoutTimeEnabled(self, enabled): - """Enable/Disable blackout time without changing the times. + """Enable/disable the host blackout time without changing the times. - :type enabled: bool - :param enabled: enable/disable blackout time""" + :type enabled: bool + :param enabled: enable/disable blackout time + """ self.stub.SetBlackoutTimeEnabled( - host_pb2.DeedSetBlackoutTimeEnabledRequest(deed=self.data, - enabled=enabled), + host_pb2.DeedSetBlackoutTimeEnabledRequest(deed=self.data, enabled=enabled), timeout=Cuebot.Timeout) def id(self): + """Returns the id of the deed. + + :rtype: str + :return: deed id + """ return self.data.id def host(self): + """Returns the name of the host associated with the deed. + + :rtype: str + :return: name of the deed host + """ return self.data.host def owner(self): + """Returns the name of the owner of the deed. + + :rtype: str + :return: name of the deed owner + """ return self.data.owner def show(self): + """Returns the name of the show of the deed. + + :rtype: str + :return: name of the deed show + """ return self.data.show def blackout(self): + """Returns whether the blackout time is enabled. + + :rtype: bool + :return: whether the blackout is enabled + """ return self.data.blackout def blackoutStartTime(self): + """Returns the blackout start time as an epoch. + + :rtype: int + :return: blackout start time as an epoch + """ return self.data.blackout_start_time def blackoutStopTime(self): + """Returns the blackout end time as an epoch. + + :rtype: int + :return: blackout end time as an epoch + """ return self.data.blackout_stop_time diff --git a/pycue/opencue/wrappers/depend.py b/pycue/opencue/wrappers/depend.py index e4e9857e8..535589f5c 100644 --- a/pycue/opencue/wrappers/depend.py +++ b/pycue/opencue/wrappers/depend.py @@ -12,14 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - - -""" -Project: opencue Library - -Module: depend.py - opencue Library implementation of a Depend -""" - +"""Module for classes related to dependencies.""" import enum @@ -31,6 +24,7 @@ class Depend(object): """This class contains the grpc implementation related to a Depend.""" class DependType(enum.IntEnum): + """Enum representing the type of dependency between subject and object.""" JOB_ON_JOB = depend_pb2.JOB_ON_JOB JOB_ON_LAYER = depend_pb2.JOB_ON_LAYER JOB_ON_FRAME = depend_pb2.JOB_ON_FRAME @@ -45,6 +39,7 @@ class DependType(enum.IntEnum): LAYER_ON_SIM_FRAME = depend_pb2.LAYER_ON_SIM_FRAME class DependTarget(enum.IntEnum): + """The type of target represented by this dependency.""" INTERNAL = depend_pb2.INTERNAL EXTERNAL = depend_pb2.EXTERNAL ANY_TARGET = depend_pb2.ANY_TARGET @@ -54,60 +49,116 @@ def __init__(self, depend=None): self.stub = Cuebot.getStub('depend') def satisfy(self): + """Satisfies the dependency. + + This sets any frames waiting on this dependency to the WAITING state. + """ self.stub.Satisfy( depend_pb2.DependSatisfyRequest(depend=self.data), timeout=Cuebot.Timeout) def unsatisfy(self): + """Unsatisfies the dependency. + + This makes the dependency active again and sets matching frames to DEPEND. + """ self.stub.Unsatisfy( depend_pb2.DependUnsatisfyRequest(depend=self.data), timeout=Cuebot.Timeout) def id(self): - """Returns the depdendency's unique id. Dependencies are one of the only - entities without a unique name so the unique ID is exposed to act - as the name. This is mainly to make command line tools easier to use. + """Returns the dependency's unique id. + + Dependencies are one of the only entities without a unique name so the unique ID is + exposed to act as the name. This is mainly to make command line tools easier to use. - :rtype: str - :return: the dependencies unique id""" + :rtype: str + :return: the dependency's unique id""" return self.data.id def isInternal(self): - """Returns true if the dependency is internal to the depender job, false if not. + """Returns whether the dependency is contained within a single job. - :rtype: bool - :returns: true""" + :rtype: bool + :return: whether the dependency is contained within a single job + """ if self.data.depend_er_job == self.data.depend_on_job: return True return False def type(self): + """Returns the type of dependency. + + :rtype: opencue.compiled_proto.depend_pb2.DependType + :return: dependency type + """ return self.data.type def target(self): - return self.data.target + """Returns the target of the dependency, either internal or external. - def chunkSize(self): - return self.data.chunk_size + :rtype: opencue.compiled_proto.depend_pb2.DependTarget + :return: dependency target type + """ + return self.data.target def anyFrame(self): + """Returns whether the depend is an any-frame depend. + + :rtype: bool + :return: whether the depend is an any-frame depend + """ return self.data.any_frame def isActive(self): + """Returns whether the depend is active. + + :rtype: bool + :return: whether the depend is active + """ return self.data.active def dependErJob(self): + """Returns the name of the job that is depending. + + :rtype: str + :return: name of the job that is depending""" return self.data.depend_er_job def dependErLayer(self): + """Returns the name of the layer that is depending. + + :rtype: str + :return: name of the layer that is depending + """ return self.data.depend_er_layer def dependErFrame(self): + """Returns the name of the frame that is depending. + + :rtype: str + :return: name of the frame that is depending + """ return self.data.depend_er_frame def dependOnJob(self): + """Returns the name of the job to depend on. + + :rtype: str + :return: name of the job to depend on + """ return self.data.depend_on_job def dependOnLayer(self): + """Returns the name of the layer to depend on. + + :rtype: str + :return: name of the layer to depend on + """ return self.data.depend_on_layer def dependOnFrame(self): + """Returns the name of the frame to depend on. + + :rtype: str + :return: name of the frame to depend on + """ return self.data.depend_on_frame diff --git a/pycue/opencue/wrappers/filter.py b/pycue/opencue/wrappers/filter.py index 81a6d109a..45e3949ca 100644 --- a/pycue/opencue/wrappers/filter.py +++ b/pycue/opencue/wrappers/filter.py @@ -12,15 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - - -""" -Project: opencue Library - -Module: filter.py - opencue Library implementation of spank filter - -""" - +"""Classes for working with filters.""" import enum @@ -48,11 +40,12 @@ class Filter(object): """This class contains the grpc implementation related to a Filter.""" class FilterType(enum.IntEnum): + """The type of match used to determine if objects pass the filter.""" MATCH_ANY = filter_pb2.MATCH_ANY MATCH_ALL = filter_pb2.MATCH_ALL + # pylint: disable=redefined-builtin def __init__(self, filter=None): - """_Filter class initialization""" self.data = filter self.stub = Cuebot.getStub('filter') @@ -62,20 +55,21 @@ def __eq__(self, other): return self.data == other.data def delete(self): - """Deletes the filter""" + """Deletes the filter.""" self.stub.Delete(filter_pb2.FilterDeleteRequest(filter=self.data), timeout=Cuebot.Timeout) def createMatcher(self, subject, matchType, query): - """Creates a matcher for this filter + """Creates a matcher for the filter. :type subject: filter_pb2.MatchSubject.* - :param subject: The job attribute to match + :param subject: the job attribute to match :type matchType: filter_pb2.MatchType.* - :param matchType: The type of match to perform + :param matchType: the type of match to perform :type query: string - :param query: The value to match + :param query: the value to match :rtype: Matcher - :return: The new matcher object""" + :return: the new matcher object + """ matcher = MatcherData( subject=subject, type=matchType, @@ -86,14 +80,15 @@ def createMatcher(self, subject, matchType, query): timeout=Cuebot.Timeout).matcher) def createAction(self, actionType, value): - """Creates an action for this filter. + """Creates an action for the filter. :type actionType: filter_pb2.ActionType.* - :param actionType: The action to perform - :type value: Group or str, or int or bool - :param value: Value relevant to the type selected - :rtype: Action - :return: The new Action object""" + :param actionType: the action to perform + :type value: opencue.wrapper.group.Group / str / int / bool + :param value: value relevant to the type selected + :rtype: opencue.wrappers.filter.Action + :return: the new action + """ action = ActionData( type=actionType, group_value=None, @@ -126,48 +121,51 @@ def createAction(self, actionType, value): timeout=Cuebot.Timeout).action) def getActions(self): - """Returns the actions in this filter. + """Returns the filter actions. - :rtype: list - :return: A list of the actions in this filter""" + :rtype: list + :return: list of the filter actions + """ response = self.stub.GetActions(filter_pb2.FilterGetActionsRequest(filter=self.data), timeout=Cuebot.Timeout) return [Action(action) for action in response.actions.actions] def getMatchers(self): - """Returns the matchers in this filter. + """Returns the filter matchers. - :rtype: list - :return: A list of the matchers in this filter""" + :rtype: list + :return: list of the filter matchers + """ response = self.stub.GetMatchers(filter_pb2.FilterGetMatchersRequest(filter=self.data), timeout=Cuebot.Timeout) return [Matcher(matcher) for matcher in response.matchers.matchers] def lowerOrder(self): - """Lowers the order of this filter relative to the other filters""" + """Lowers the order of the filter relative to other filters.""" self.stub.LowerOrder(filter_pb2.FilterLowerOrderRequest(filter=self.data), timeout=Cuebot.Timeout) def raiseOrder(self): - """Raises the order of this filter relative to the other filters""" + """Raises the order of the filter relative to other filters.""" self.stub.RaiseOrder(filter_pb2.FilterRaiseOrderRequest(filter=self.data), timeout=Cuebot.Timeout) def orderFirst(self): - """Orders this filter above all the other filters""" + """Orders the filter above all other filters.""" self.stub.OrderFirst(filter_pb2.FilterOrderFirstRequest(filter=self.data), timeout=Cuebot.Timeout) def orderLast(self): - """Orders this filter below all the other filters""" + """Orders the filter below all other filters.""" self.stub.OrderLast(filter_pb2.FilterOrderLastRequest(filter=self.data), timeout=Cuebot.Timeout) def runFilterOnGroup(self, group): """Runs the filter on the group provided. - :type group: list - :param group: The group to run the filter on""" + :type group: opencue.wrapper.group.Group + :param group: group to run the filter on + """ self.stub.RunFilterOnGroup( filter_pb2.FilterRunFilterOnGroupRequest(filter=self.data, group=group.data), timeout=Cuebot.Timeout) @@ -176,7 +174,8 @@ def runFilterOnJobs(self, jobs): """Runs the filter on the list of jobs provided. :type jobs: list - :param jobs: The jobs to run the filter on""" + :param jobs: jobs to run the filter on + """ jobSeq = job_pb2.JobSeq(jobs=[job.data for job in jobs]) self.stub.RunFilterOnJobs( filter_pb2.FilterRunFilterOnJobsRequest(filter=self.data, jobs=jobSeq), @@ -186,47 +185,75 @@ def setEnabled(self, value): """Enables or disables the filter. :type value: bool - :param value: True to enable the filter and false to disable it""" + :param value: true to enable the filter and false to disable it + """ self.stub.SetEnabled(filter_pb2.FilterSetEnabledRequest(filter=self.data, enabled=value), timeout=Cuebot.Timeout) def setName(self, name): - """Sets the name of this filter. + """Sets the name of the filter. :type name: str - :param name: The new name for this filter""" + :param name: new filter name + """ self.stub.SetName(filter_pb2.FilterSetNameRequest(filter=self.data, name=name), timeout=Cuebot.Timeout) def setType(self, filterType): """Changes the filter type. - :type filterType: filter_pb2.FilterType - :param filterType: The new filter type""" + :type filterType: opencue.compiled_proto.filter_pb2.FilterType + :param filterType: the new filter type + """ self.stub.SetType(filter_pb2.FilterSetTypeRequest(filter=self.data, type=filterType), timeout=Cuebot.Timeout) def setOrder(self, order): + """Directly sets the order of the filter. + + :type order: int + :param order: the new filter order + """ self.stub.SetOrder(filter_pb2.FilterSetOrderRequest(filter=self.data, order=order), timeout=Cuebot.Timeout) def name(self): + """Returns the filter name. + + :rtype: str + :return: the filter name + """ return self.data.name def type(self): + """Returns the filter type. + + :rtype: opencue.compiled_proto.filter_pb2.FilterType + :return: the filter type + """ return self.data.type def order(self): + """Returns the current position of the filter. + + :rtype: float + :return: the current position of the filter""" return self.data.order def isEnabled(self): + """Returns whether the filter is enabled. + + :rtype: bool + :return: whether the filter is enabled + """ return self.data.enabled def id(self): """Returns the id of the filter. :rtype: str - :return: Filter uuid""" + :return: id of the filter + """ return self.data.id @@ -234,6 +261,7 @@ class Action(object): """This class contains the grpc implementation related to an Action.""" class ActionType(enum.IntEnum): + """Enum representing the type of Action to be performed.""" MOVE_JOB_TO_GROUP = filter_pb2.MOVE_JOB_TO_GROUP PAUSE_JOB = filter_pb2.PAUSE_JOB SET_JOB_MIN_CORES = filter_pb2.SET_JOB_MIN_CORES @@ -246,6 +274,7 @@ class ActionType(enum.IntEnum): SET_MEMORY_OPTIMIZER = filter_pb2.SET_MEMORY_OPTIMIZER class ActionValueType(enum.IntEnum): + """Enum representing the type of the action's object.""" GROUP_TYPE = filter_pb2.GROUP_TYPE STRING_TYPE = filter_pb2.STRING_TYPE INTEGER_TYPE = filter_pb2.INTEGER_TYPE @@ -258,49 +287,89 @@ def __init__(self, action=None): self.stub = Cuebot.getStub('action') def getParentFilter(self): + """Returns the filter the action belongs to. + + :rtype: opencue.wrappers.filter.Filter + :return: the filter the action belongs to + """ response = self.stub.GetParentFilter( filter_pb2.ActionGetParentFilterRequest(action=self.data), - timeout=Cuebot.Timeout - ) + timeout=Cuebot.Timeout) return Filter(response.filter) def delete(self): + """Deletes the action.""" self.stub.Delete(filter_pb2.ActionDeleteRequest(action=self.data), timeout=Cuebot.Timeout) def commit(self): + """Commits any changes to the action to the database.""" if self.isNew(): raise Exception( "unable to commit action that has not been created, proxy does not exist") self.stub.Commit(filter_pb2.ActionCommitRequest(action=self.data), timeout=Cuebot.Timeout) def isNew(self): + """Returns whether the action has been initialized yet. + + :rtype: bool + :return: True if the action has been initialized with data from the database + """ return self.data is None def name(self): + """Returns the name of the action. + + :rtype: str + :return: name of the action + """ if self.value() is None: return "%s" % ActionType.Name(self.type()) - else: - return "%s %s" % (ActionType.Name(self.type()), self.value()) + return "%s %s" % (ActionType.Name(self.type()), self.value()) def value(self): + """Returns the value of the action; what will happen if the filter is matched. + + Type of value returned depends on the action's value_type. + + :rtype: str/int/float/bool + :return: value of the action + """ valueType = self.data.value_type if valueType == filter_pb2.GROUP_TYPE: return self.data.group_value - elif valueType == filter_pb2.STRING_TYPE: + if valueType == filter_pb2.STRING_TYPE: return self.data.string_value - elif valueType == filter_pb2.INTEGER_TYPE: + if valueType == filter_pb2.INTEGER_TYPE: return self.data.integer_value - elif valueType == filter_pb2.FLOAT_TYPE: + if valueType == filter_pb2.FLOAT_TYPE: return self.data.float_value - elif valueType == filter_pb2.BOOLEAN_TYPE: + if valueType == filter_pb2.BOOLEAN_TYPE: return self.data.boolean_value - else: - return None + return None def type(self): + """Returns the type of the action. + + An action's type determines what will happen if the action is triggered by its filter. + For example, if the type is GROUP_TYPE, the object which has triggered the filter will + be assigned to the group specified by the action's value. + + :rtype: filter_pb2.ActionValueType + :return: the type of the action + """ return self.data.type def setTypeAndValue(self, actionType, value): + """Sets a new type and value for the action. + + These fields should be set together as the value can be only be properly validated and + stored in the correct database field if the type is also known. + + :type actionType: filter_pb2.ActionValueType + :param actionType: the new type of the action + :type value: str/int/float/bool + :param value: the new value of the action + """ self.data.type = actionType if actionType == filter_pb2.MOVE_JOB_TO_GROUP: if not isinstance(value, job_pb2.Group): @@ -333,7 +402,7 @@ def setTypeAndValue(self, actionType, value): self.data.value_type = filter_pb2.NONE_TYPE else: - raise Exception("invalid action type: %s" % actionType) + raise Exception("invalid action type: %s" % actionType) self.commit() @@ -341,14 +410,20 @@ def id(self): """Returns the id of the action. :rtype: str - :return: Action uuid""" + :return: id of the action + """ return self.data.id class Matcher(object): - """This class contains the grpc implementation related to a Matcher.""" + """This class contains the grpc implementation related to a Matcher. + + Matchers belong to a single filter, and indicate the conditions where a given object will + satisfy that filter, i.e. if it will trigger the actions in that filter. + """ class MatchSubject(enum.IntEnum): + """Enum representing the type of the subject; the thing being matched.""" JOB_NAME = filter_pb2.JOB_NAME SHOW = filter_pb2.SHOW SHOT = filter_pb2.SHOT @@ -359,6 +434,7 @@ class MatchSubject(enum.IntEnum): LAYER_NAME = filter_pb2.LAYER_NAME class MatchType(enum.IntEnum): + """Enum representing the type of matching that will occur.""" CONTAINS = filter_pb2.CONTAINS DOES_NOT_CONTAIN = filter_pb2.DOES_NOT_CONTAIN IS = filter_pb2.IS @@ -372,16 +448,22 @@ def __init__(self, matcher=None): self.stub = Cuebot.getStub('matcher') def getParentFilter(self): + """Returns the filter the matcher belongs to. + + :rtype: opencue.wrappers.filter.Filter + :return: the filter the matcher belongs to + """ response = self.stub.GetParentFilter( filter_pb2.MatcherGetParentFilterRequest(matcher=self.data), - timeout=Cuebot.Timeout - ) + timeout=Cuebot.Timeout) return Filter(response.filter) def delete(self): + """Deletes the matcher.""" self.stub.Delete(filter_pb2.MatcherDeleteRequest(matcher=self.data), timeout=Cuebot.Timeout) def commit(self): + """Commits any changes to the matcher to the database.""" if self.isNew(): raise Exception( "unable to commit matcher that has not been created, proxy does not exist") @@ -389,36 +471,78 @@ def commit(self): self.stub.Commit(filter_pb2.MatcherCommitRequest(matcher=self.data), timeout=Cuebot.Timeout) def isNew(self): + """Returns whether the matcher has been initialized yet with data from the database. + + :rtype: bool + :return: True if the matcher has been initialized + """ return self.data is None def name(self): - return "%s %s %s" % (MatchSubject.Name(self.data.subject), MatchType.Name(self.data.type), self.data.input) + """Returns the name of the matcher. + + :rtype: str + :return: the name of the matcher + """ + return "%s %s %s" % ( + MatchSubject.Name(self.data.subject), MatchType.Name(self.data.type), self.data.input) def subject(self): + """Returns the subject of the matcher; the type of object to be matched. + + :rtype: filter_pb2.MatchSubject + :return: the subject of the matcher + """ return self.data.subject def type(self): + """Returns the type of the matcher; the kind of comparison used to determine a match. + + :rtype: filter_pb2.MatchType + :return: the type of the matcher + """ return self.data.type def input(self): + """Returns the input data of the matcher; the value to be matched against. + + :rtype: str + :return: input data of the matcher + """ return self.data.input def id(self): """Returns the id of the matcher. :rtype: str - :return: Matcher uuid""" + :return: id of the matcher + """ return self.data.id def setSubject(self, value): + """Sets a new subject for the matcher. + + :type value: str + :param value: new subject for the matcher + """ self.data.subject = value self.commit() def setType(self, value): + """Sets a new type for the matcher. + + :type value: filter_pb2.MatchType + :param value: new type for the matcher + """ self.data.type = value self.commit() def setInput(self, value): + """Set new input data for the matcher. + + :type value: str + :param value: new input data for the matcher + """ value = value.replace(" ", "") self.data.input = str(value) self.commit() diff --git a/pycue/opencue/wrappers/frame.py b/pycue/opencue/wrappers/frame.py index 3b79bed65..0455cbd7c 100644 --- a/pycue/opencue/wrappers/frame.py +++ b/pycue/opencue/wrappers/frame.py @@ -12,13 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - -""" -Project: opencue Library - -Module: frame.py - opencue Library implementation of a frame -""" - +"""Module for classes related to frames.""" import enum import time @@ -33,17 +27,20 @@ class Frame(object): """This class contains the grpc implementation related to a Frame.""" class CheckpointState(enum.IntEnum): + """Possible states for a frame's checkpointing status, if it uses that.""" DISABLED = job_pb2.DISABLED ENABLED = job_pb2.ENABLED COPYING = job_pb2.COPYING COMPLETE = job_pb2.COMPLETE class FrameExitStatus(enum.IntEnum): + """Possible frame exit statuses.""" SUCCESS = job_pb2.SUCCESS NO_RETRY = job_pb2.NO_RETRY SKIP_RETRY = job_pb2.SKIP_RETRY class FrameState(enum.IntEnum): + """Possible frame states.""" WAITING = job_pb2.WAITING SETUP = job_pb2.SETUP RUNNING = job_pb2.RUNNING @@ -54,52 +51,54 @@ class FrameState(enum.IntEnum): CHECKPOINT = job_pb2.CHECKPOINT def __init__(self, frame=None): - """_Frame class initialization""" self.data = frame self.stub = Cuebot.getStub('frame') def eat(self): - """Eat frame""" + """Eats the frame.""" if self.data.state != job_pb2.FrameState.Value('EATEN'): self.stub.Eat(job_pb2.FrameEatRequest(frame=self.data), timeout=Cuebot.Timeout) def kill(self): - """Kill frame""" + """Kills the frame.""" if self.data.state == job_pb2.FrameState.Value('RUNNING'): self.stub.Kill(job_pb2.FrameKillRequest(frame=self.data), timeout=Cuebot.Timeout) def retry(self): - """Retry frame""" + """Retries the frame.""" if self.data.state != job_pb2.FrameState.Value('WAITING'): self.stub.Retry(job_pb2.FrameRetryRequest(frame=self.data), timeout=Cuebot.Timeout) def addRenderPartition(self, hostname, threads, max_cores, num_mem, max_gpu): - """Add a render partition to the frame. - @type hostname: str - @param hostname: hostname of the partition - @type threads: int - @param threads: number of threads of the partition - @type max_cores: int - @param max_cores: max cores enabled for the partition - @type num_mem: int - @param num_mem: amount of memory reserved for the partition - @type max_gpu: int - @param max_gpu: max gpu cores enabled for the partition + """Adds a render partition to the frame. + + :type hostname: str + :param hostname: hostname of the partition + :type threads: int + :param threads: number of threads of the partition + :type max_cores: int + :param max_cores: max cores enabled for the partition + :type num_mem: int + :param num_mem: amount of memory reserved for the partition + :type max_gpu: int + :param max_gpu: max gpu cores enabled for the partition """ - response = self.stub.AddRenderPartition( - job_pb2.FrameAddRenderPartitionRequest(frame=self.data, - host=hostname, - threads=threads, - max_cores=max_cores, - max_memory=num_mem, - max_gpu=max_gpu, - username=os.getenv("USER", "unknown"))) + self.stub.AddRenderPartition( + job_pb2.FrameAddRenderPartitionRequest( + frame=self.data, + host=hostname, + threads=threads, + max_cores=max_cores, + max_memory=num_mem, + max_gpu=max_gpu, + username=os.getenv("USER", "unknown"))) def getWhatDependsOnThis(self): """Returns a list of dependencies that depend directly on this frame. - :rtype: list - :return: List of dependencies that depend directly on this frame""" + :rtype: list + :return: list of dependencies that depend directly on this frame + """ response = self.stub.GetWhatDependsOnThis( job_pb2.FrameGetWhatDependsOnThisRequest(frame=self.data), timeout=Cuebot.Timeout) @@ -108,44 +107,48 @@ def getWhatDependsOnThis(self): def getWhatThisDependsOn(self): """Returns a list of dependencies that this frame depends on. - :rtype: list - :return: List of dependencies that this frame depends on""" + :rtype: list + :return: list of dependencies that this frame depends on + """ response = self.stub.GetWhatThisDependsOn( job_pb2.FrameGetWhatThisDependsOnRequest(frame=self.data), timeout=Cuebot.Timeout) return [opencue.wrappers.depend.Depend(dep) for dep in response.depends.depends] def createDependencyOnJob(self, job): - """Create and return a frame on job dependency. + """Creates and returns a frame-on-job dependency. :type job: opencue.wrappers.job.Job :param job: the job you want this frame to depend on :rtype: opencue.wrappers.depend.Depend - :return: The new dependency""" + :return: The new dependency + """ response = self.stub.CreateDependencyOnJob( job_pb2.FrameCreateDependencyOnJobRequest(frame=self.data, job=job.data), timeout=Cuebot.Timeout) return opencue.wrappers.depend.Depend(response.depend) def createDependencyOnLayer(self, layer): - """Create and return a frame on layer dependency. + """Creates and returns a frame-on-layer dependency. - :type layer: opencue.wrappers.layer.Layer + :type layer: opencue.wrappers.layer.Layer :param layer: the layer you want this frame to depend on :rtype: opencue.wrappers.depend.Depend - :return: The new dependency""" + :return: the new dependency + """ response = self.stub.CreateDependencyOnLayer( job_pb2.FrameCreateDependencyOnLayerRequest(frame=self.data, layer=layer.data), timeout=Cuebot.Timeout) return opencue.wrappers.depend.Depend(response.depend) def createDependencyOnFrame(self, frame): - """Create and return a frame on frame dependency. + """Creates and returns a frame-on-frame dependency. - :type frame: opencue.wrappers.frame.Frame + :type frame: opencue.wrappers.frame.Frame :param frame: the frame you want this frame to depend on :rtype: opencue.wrappers.depend.Depend - :return: The new dependency""" + :return: the new dependency + """ response = self.stub.CreateDependencyOnFrame( job_pb2.FrameCreateDependencyOnFrameRequest(frame=self.data, depend_on_frame=frame.data), @@ -159,134 +162,153 @@ def dropDepends(self, target): timeout=Cuebot.Timeout) def markAsWaiting(self): - """Mark the frame as waiting, similar to drop depends. The frame will be - able to run even if the job has an external dependency.""" + """Marks the frame as waiting; ready to run. + + Similar to dropDepends. The frame will be able to run even if the job has an external + dependency.""" self.stub.MarkAsWaiting( job_pb2.FrameMarkAsWaitingRequest(frame=self.data), timeout=Cuebot.Timeout) def setCheckpointState(self, checkPointState): - """Sets the checkPointState of the frame - :param checkPointState: job_pb.CheckpointState(Int) - :return: + """Sets the checkPointState of the frame. + + :type checkPointState: job_pb.CheckpointState + :param checkPointState: the checkpoint state of the frame """ self.stub.SetCheckpointState( - job_pb2.FrameSetCheckpointStateRequest(frame=self.data, state=checkPointState) - ) + job_pb2.FrameSetCheckpointStateRequest(frame=self.data, state=checkPointState)) def id(self): """Returns the id of the frame. + :rtype: str - :return: Frame uuid""" + :return: id of the frame + """ return self.data.id def name(self): """Returns the name of the frame. + :rtype: str - :return: Frame name""" + :return: name of the frame + """ return "%04d-%s" % (self.data.number, self.data.layer_name) def layer(self): """Returns the name of the layer name that the frame belongs to. :rtype: str - :return: Layer name""" + :return: name of the layer + """ return self.data.layer_name def frame(self): - """Returns the frames number as a padded string. + """Returns the frame number as a padded string. :rtype: str - :return: Frame number string""" + :return: frame number padded to 4 digits + """ return "%04d" % self.data.number def number(self): - """Returns the frames number. + """Returns the frame number, unpadded. :rtype: int - :return: Frame number""" + :return: frame number + """ return self.data.number def dispatchOrder(self): - """Returns the frames dispatch order. + """Returns the frame's dispatch order. :rtype: int - :return: Frame dispatch order""" + :return: frame dispatch order + """ return self.data.dispatch_order def startTime(self): """Returns the epoch timestamp of the frame's start time. :rtype: int - :return: Job start time in epoch""" + :return: frame start time as an epoch + """ return self.data.start_time def stopTime(self): """Returns the epoch timestamp of the frame's stop time. :rtype: int - :return: Frame stop time in epoch""" + :return: frame stop time as an epoch + """ return self.data.stop_time def resource(self): """Returns the most recent resource that the frame has started running on. + Ex: vrack999/1.0 = host/proc:cores :rtype: str - :return: Most recent running resource""" + :return: most recent running resource + """ return self.data.last_resource def retries(self): - """Returns the number of retries. + """Returns the number of times the frame has been retried. :rtype: int - :return: Number of retries""" + :return: number of retries + """ return self.data.retry_count def exitStatus(self): - """Returns the frame's exitStatus. + """Returns the frame's exit status. :rtype: int - :return: Frames last exit status""" + :return: last exit status of the frame + """ return self.data.exit_status def maxRss(self): """Returns the frame's maxRss. :rtype: long - :return: Max RSS in Kb""" + :return: max RSS in Kb + """ return self.data.max_rss def memUsed(self): """Returns the frame's currently used memory. :rtype: long - :return: Current used memory in Kb""" + :return: currently used memory in Kb + """ return self.data.used_memory def memReserved(self): """Returns the frame's currently reserved memory. :rtype: long - :return: Current used memory in Kb""" + :return: currently reserved memory in Kb + """ return self.data.reserved_memory def state(self): # call it status? """Returns the state of the frame. - :rtype: opencue.FrameState - :return: Frame state""" + :rtype: job_pb2.FrameState + :return: state of the frame + """ return self.data.state def runTime(self): """Returns the number of seconds that the frame has been (or was) running. :rtype: int - :return: Job runtime in seconds""" + :return: frame runtime in seconds + """ if self.data.start_time == 0: return 0 if self.data.stop_time == 0: return int(time.time() - self.data.start_time) - else: - return self.data.stop_time - self.data.start_time - + return self.data.stop_time - self.data.start_time diff --git a/pycue/opencue/wrappers/group.py b/pycue/opencue/wrappers/group.py index 1b02947ae..59685cc8b 100644 --- a/pycue/opencue/wrappers/group.py +++ b/pycue/opencue/wrappers/group.py @@ -12,11 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - -""" -opencue group module -""" - +"""Modules for classes related to groups.""" from opencue import Cuebot from opencue.compiled_proto import job_pb2 @@ -31,41 +27,82 @@ def __init__(self, group=None): self.stub = Cuebot.getStub('group') def createSubGroup(self, name): + """Creates a subgroup under this group. + + :type name: str + :param name: name of the new subgroup + """ return Group(self.stub.CreateSubGroup( job_pb2.GroupCreateSubGroupRequest(group=self.data, name=name), timeout=Cuebot.Timeout).group) def delete(self): + """Deletes the group.""" self.stub.Delete(job_pb2.GroupDeleteRequest(group=self.data), timeout=Cuebot.Timeout) def setName(self, name): + """Sets the name of the group. + + :type name: str + :param name: new name for the group + """ self.stub.SetName(job_pb2.GroupSetNameRequest(group=self.data, name=name), timeout=Cuebot.Timeout) def setMaxCores(self, value): + """Sets the maximum cores of everything in the group. + + :type value: int + :param value: new maximum number of cores + """ self.stub.SetMaxCores(job_pb2.GroupSetMaxCoresRequest(group=self.data, max_cores=value), timeout=Cuebot.Timeout) def setMinCores(self, value): + """Sets the minimum cores of everything the group. + + :type value: int + :param value: new minimum number of cores + """ self.stub.SetMinCores(job_pb2.GroupSetMinCoresRequest(group=self.data, min_cores=value), timeout=Cuebot.Timeout) def setDefaultJobPriority(self, value): + """Sets the default job priority for everything in the group. + + :type value: int + :param value: new default priority + """ self.stub.SetDefaultJobPriority( job_pb2.GroupSetDefJobPriorityRequest(group=self.data, priority=value), timeout=Cuebot.Timeout) def setDefaultJobMinCores(self, value): + """Sets the default job minimum cores for everything in the group. + + :type value: int + :param value: new default job minimum cores + """ self.stub.SetDefaultJobMinCores( job_pb2.GroupSetDefJobMinCoresRequest(group=self.data, min_cores=value), timeout=Cuebot.Timeout) def setDefaultJobMaxCores(self, value): + """Sets the default job maximum cores for everything in the group. + + :type value: int + :param value: new default job maximum cores + """ self.stub.SetDefaultJobMaxCores( job_pb2.GroupSetDefJobMaxCoresRequest(group=self.data, max_cores=value), timeout=Cuebot.Timeout) def getGroups(self): + """Returns child groups of this group. + + :rtype: list + :return: list of child groups + """ response = self.stub.GetGroups(job_pb2.GroupGetGroupsRequest(group=self.data), timeout=Cuebot.Timeout) return [Group(g) for g in response.groups.groups] @@ -74,7 +111,8 @@ def getJobs(self): """Returns the jobs in this group. :rtype: list - :return: List of jobs in this group""" + :return: list of jobs in this group + """ response = self.stub.GetJobs(job_pb2.GroupGetJobsRequest(group=self.data), timeout=Cuebot.Timeout) return [opencue.wrappers.job.Job(j) for j in response.jobs.jobs] @@ -83,7 +121,8 @@ def reparentJobs(self, jobs): """Moves the given jobs into this group. :type jobs: list - :param jobs: The jobs to add to this group""" + :param jobs: The jobs to add to this group + """ jobsToReparent = [] for job in jobs: if isinstance(job, opencue.wrappers.job.NestedJob): @@ -94,10 +133,11 @@ def reparentJobs(self, jobs): timeout=Cuebot.Timeout) def reparentGroups(self, groups): - """Moves the given groups into this group. + """Moves the given groups to be subgroups of this group. :type groups: list - :param groups: The groups to move into""" + :param groups: the groups to make subgroups of this group + """ groupSeq = job_pb2.GroupSeq(groups=[group.data for group in groups]) self.stub.ReparentGroups( job_pb2.GroupReparentGroupsRequest(group=self.data, groups=groupSeq), @@ -106,95 +146,136 @@ def reparentGroups(self, groups): def reparentGroupIds(self, groupIds): """Moves the given group ids into this group. - :type groups: list - :param groups: The group ids to move into""" - groups = [opencue.wrappers.group.Group(job_pb2.Group(id=groupId)) for groupId in groupIds] + :type groupIds: list + :param groupIds: The group ids to make subgroups of this group + """ + groups = [Group(job_pb2.Group(id=groupId)) for groupId in groupIds] self.reparentGroups(groups) def setDepartment(self, name): - """Sets the group's department to the specified name. The department - name must be one of the allowed department names. All jobs in the group - will inherit the new department name. See AdminStatic for getting a list - of allowed department names. Department names are maintained by the - middle-tier group. - - :type name: string - :param name: a valid department name""" + """Sets the group's department to the specified name. + + The department name must be one of the allowed department names. All jobs in the group + will inherit the new department name. + + :type name: string + :param name: a valid department name + """ self.stub.SetDepartment(job_pb2.GroupSetDeptRequest(group=self.data, dept=name), timeout=Cuebot.Timeout) self.data.department = name def setGroup(self, parentGroup): - """Sets this group's parent to parentGroup. + """Sets this group's parent. :type parentGroup: opencue.wrappers.group.Group - :param parentGroup: Group to parent under""" - self.stub.SetGroup(job_pb2.GroupSetGroupRequest(group=self.data, - parent_group=parentGroup.data), - timeout=Cuebot.Timeout) + :param parentGroup: group to parent under + """ + self.stub.SetGroup( + job_pb2.GroupSetGroupRequest(group=self.data, parent_group=parentGroup.data), + timeout=Cuebot.Timeout) def id(self): """Returns the id of the group. :rtype: str - :return: Group uuid""" + :return: id of the group + """ return self.data.id def name(self): + """Returns the name of the group. + + :rtype: str + :return: name of the group""" return self.data.name def department(self): + """Returns the department of the group. + + :rtype: str + :return: department of the group + """ return self.data.department def defaultJobPriority(self): + """Returns the default job priority of the group. + + :rtype: int + :return: default job priority of the group + """ return self.data.default_job_priority def defaultJobMinCores(self): + """Returns the default job minimum cores of the group. + + :rtype: float + :return: default job min cores + """ return self.data.default_job_min_cores def defaultJobMaxCores(self): + """Returns the default job maximum cores of the group. + + :rtype: float + :return: default job max cores + """ return self.data.default_job_max_cores def maxCores(self): + """Returns the maximum cores of the group. + + :rtype: float + :return: max cores of the group + """ return self.data.max_cores def minCores(self): + """Returns the minimum cores of the group. + + :rtype: float + :return: min cores of the group + """ return self.data.min_cores def reservedCores(self): """Returns the total number of reserved cores for the group. - :rtype: int - :return: total numnber of frames""" + :rtype: float + :return: total number of reserved cores + """ return self.data.group_stats.reserved_cores def totalRunning(self): - """Returns the total number of running frames under this object. + """Returns the total number of running frames under this group. :rtype: int - :return: Total number of running frames""" + :return: total number of running frames + """ return self.data.group_stats.running_frames def totalDead(self): - """Returns the total number of deads frames under this object. + """Returns the total number of dead frames under this group. :rtype: int - :return: Total number of dead frames""" + :return: total number of dead frames + """ return self.data.group_stats.dead_frames def totalPending(self): - """Returns the total number of pending (dependent and waiting) frames - under this object. + """Returns the total number of pending (dependent and waiting) frames under this group. :rtype: int - :return: Total number of pending (dependent and waiting) frames""" + :return: total number of pending (dependent and waiting) frames + """ return self.data.group_stats.pending_frames def pendingJobs(self): """Returns the total number of running jobs. :rtype: int - :return: total number of running jobs""" + :return: total number of running jobs + """ return self.data.group_stats.pending_jobs @@ -207,19 +288,31 @@ def __init__(self, group): self.__children_init = False def createSubGroup(self, name): - """Create a sub group""" + """Creates a subgroup under this nested group. + + :type name: str + :param name: name of the new subgroup + """ return self.asGroup().createSubGroup(name) def children(self): - """returns jobs and groups in a single array""" + """Returns this group's jobs and child groups in a single array. + + :rtype: list + :return: list of all jobs and child groups in this group + """ if not self.__children_init: - self.__children.extend(self.groups) - self.__children.extend(self.jobs) + self.__children.extend(self.data.groups) + self.__children.extend(self.data.jobs) self.__children_init = True return self.__children def asGroup(self): - """returns a Group object from this NestedGroup""" + """Returns a Group object from this NestedGroup. + + :rtype: opencue.wrappers.group.Group + :return: Group version of this NestedGroup + """ return Group(job_pb2.Group( id=self.data.id, name=self.data.name, @@ -236,7 +329,7 @@ def asGroup(self): def hasParent(self): """Whether this NestedGroup has a parent group. - :rtype: bool + :rtype: bool :return: whether the group has a parent group. """ return self.data.HasField('parent') diff --git a/pycue/opencue/wrappers/host.py b/pycue/opencue/wrappers/host.py index 50128f35a..83dac252f 100644 --- a/pycue/opencue/wrappers/host.py +++ b/pycue/opencue/wrappers/host.py @@ -12,14 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - -""" -Project: opencue Library - -Module: host.py - opencue Library implementation of a host - -""" - +"""Module for classes related to hosts.""" import enum import os @@ -29,6 +22,7 @@ from opencue.compiled_proto import comment_pb2 from opencue.compiled_proto import host_pb2 import opencue.wrappers.comment +# pylint: disable=cyclic-import import opencue.wrappers.proc @@ -36,6 +30,7 @@ class Host(object): """This class contains the grpc implementation related to a Host.""" class HardwareState(enum.IntEnum): + """Enum representing the hardware state of the host.""" UP = host_pb2.UP DOWN = host_pb2.DOWN REBOOTING = host_pb2.REBOOTING @@ -43,23 +38,25 @@ class HardwareState(enum.IntEnum): REPAIR = host_pb2.REPAIR class HostTagType(enum.IntEnum): + """Enum representing the type of a host tag.""" MANUAL = host_pb2.MANUAL HARDWARE = host_pb2.HARDWARE ALLOC = host_pb2.ALLOC HOSTNAME = host_pb2.HOSTNAME class LockState(enum.IntEnum): + """Enum representing whether the host is locked.""" OPEN = host_pb2.OPEN LOCKED = host_pb2.LOCKED NIMBY_LOCKED = host_pb2.NIMBY_LOCKED class ThreadMode(enum.IntEnum): + """Enum representing the thread mode of the host.""" AUTO = host_pb2.AUTO ALL = host_pb2.ALL VARIABLE = host_pb2.VARIABLE def __init__(self, host=None): - """Host class initialization""" self.data = host self.__id = host.id self.stub = Cuebot.getStub('host') @@ -69,13 +66,13 @@ def lock(self): self.stub.Lock(host_pb2.HostLockRequest(host=self.data), timeout=Cuebot.Timeout) def unlock(self): - """Unlocks the host and cancels any actions that were waiting for all - running frames to finish. - """ + """Unlocks the host. + + Cancels any actions that were waiting for all running frames to finish.""" self.stub.Unlock(host_pb2.HostUnlockRequest(host=self.data), timeout=Cuebot.Timeout) def delete(self): - """Delete the host from the cuebot""" + """Deletes the host from the cuebot""" self.stub.Delete(host_pb2.HostDeleteRequest(host=self.data), timeout=Cuebot.Timeout) def getProcs(self): @@ -103,8 +100,8 @@ def redirectToJob(self, procs, job): def getRenderPartitions(self): """Returns a list of render partitions associated with this host - :rtype: list - :return: A list of render partitions under this host + :rtype: list + :return: list of render partitions under this host """ response = self.stub.GetRenderPartitions(host_pb2.HostGetRenderPartitionsRequest( host=self.data), timeout=Cuebot.Timeout) @@ -112,9 +109,9 @@ def getRenderPartitions(self): return partitionSeq.render_partitions def rebootWhenIdle(self): - """Causes the host to no longer accept new frames and - when the machine is idle it will reboot. - """ + """Sets the machine to reboot once idle. + + The host will no longer accept new frames.""" self.stub.RebootWhenIdle(host_pb2.HostRebootWhenIdleRequest(host=self.data), timeout=Cuebot.Timeout) @@ -125,16 +122,16 @@ def reboot(self): def addTags(self, tags): """Adds tags to a host. - :type tags: list + :type tags: list :param tags: The tags to add """ self.stub.AddTags(host_pb2.HostAddTagsRequest(host=self.data, tags=tags), timeout=Cuebot.Timeout) def removeTags(self, tags): - """Remove tags from this host. + """Removes tags from this host. - :type tags: list + :type tags: list :param tags: The tags to remove """ self.stub.RemoveTags(host_pb2.HostRemoveTagsRequest(host=self.data, tags=tags), @@ -143,10 +140,10 @@ def removeTags(self, tags): def renameTag(self, oldTag, newTag): """Renames a tag. - :type oldTag: str - :param oldTag: The old tag to rename - :type newTag: str - :param newTag: The new name for the tag + :type oldTag: str + :param oldTag: old tag to rename + :type newTag: str + :param newTag: new name for the tag """ self.stub.RenameTag( host_pb2.HostRenameTagRequest(host=self.data, old_tag=oldTag, new_tag=newTag), @@ -155,19 +152,19 @@ def renameTag(self, oldTag, newTag): def setAllocation(self, allocation): """Sets the host to the given allocation. - :type allocation: opencue.wrappers.allocation.Allocation - :param allocation: An allocation object + :type allocation: opencue.wrappers.allocation.Allocation + :param allocation: allocation to put the host under """ self.stub.SetAllocation( host_pb2.HostSetAllocationRequest(host=self.data, allocation_id=allocation.id()), timeout=Cuebot.Timeout) def addComment(self, subject, message): - """Appends a comment to the hosts's comment list. + """Appends a comment to the host's comment list. - :type subject: str + :type subject: str :param subject: Subject data - :type message: str + :type message: str :param message: Message data """ comment = comment_pb2.Comment( @@ -180,32 +177,39 @@ def addComment(self, subject, message): timeout=Cuebot.Timeout) def getComments(self): - """returns the hosts comments""" + """Returns the host's comment list. + + :rtype: list + :return: the comment list of the host + """ response = self.stub.GetComments(host_pb2.HostGetCommentsRequest(host=self.data), timeout=Cuebot.Timeout) commentSeq = response.comments return [opencue.wrappers.comment.Comment(c) for c in commentSeq.comments] def setHardwareState(self, state): - """Sets the host's hardware state + """Sets the host hardware state. - :type state: host_pb2.HardwareState - :param state: state to set host to""" + :type state: host_pb2.HardwareState + :param state: state to set host to + """ self.stub.SetHardwareState( host_pb2.HostSetHardwareStateRequest(host=self.data, state=state), timeout=Cuebot.Timeout) def setOs(self, osName): """Sets the host operating system. - :type osName: string - :param osName: os value to set host to""" + + :type osName: string + :param osName: os value to set host to + """ self.stub.SetOs(host_pb2.HostSetOsRequest(host=self.data, os=osName), timeout=Cuebot.Timeout) def setThreadMode(self, mode): - """Set the thread mode to mode. + """Sets the host thread mode. - :type mode: host_pb2.ThreadMode + :type mode: host_pb2.ThreadMode :param mode: ThreadMode to set host to """ self.stub.SetThreadMode(host_pb2.HostSetThreadModeRequest(host=self.data, mode=mode), @@ -214,8 +218,8 @@ def setThreadMode(self, mode): def id(self): """Returns the id of the host. - :rtype: str - :return: Host uuid + :rtype: str + :return: id of the host """ if not hasattr(self, "__id"): self.__id = self.data.id @@ -224,23 +228,23 @@ def id(self): def name(self): """Returns the name of the host. - :rtype: str - :return: Host name + :rtype: str + :return: name of the host """ return self.data.name def isNimbyEnabled(self): """Returns true if nimby is enabled. - :rtype: bool + :rtype: bool :return: True if nimby is enabled """ return self.data.nimby_enabled def isUp(self): - """Returns True if the host is up. + """Returns True if the host hardware state indicates the machine is up. - :rtype: bool + :rtype: bool :return: True if the host is up """ return self.data.state == host_pb2.HardwareState.Value('UP') @@ -248,169 +252,191 @@ def isUp(self): def isLocked(self): """Returns True if the host is locked. - :rtype: bool + :rtype: bool :return: True if the host is locked """ return self.data.lock_state == host_pb2.LockState.Value('LOCKED') def isCommented(self): """Returns true if the host has a comment. - - :rtype: bool - :return: If the job has a comment + + :rtype: bool + :return: whether the host has a comment """ return self.data.has_comment def cores(self): - """ + """Returns the total number of cores the host has. + :rtype: float - :return: number of cores + :return: total number of host cores """ return self.data.cores def coresReserved(self): - """ + """Returns the number of cores the host has which are currently reserved. + :rtype: float :return: number of cores reserved """ return self.data.cores - self.data.idle_ores def coresIdle(self): - """ + """Returns the number of cores the host currently has idel. + :rtype: float :return: number of cores idle """ return self.data.idle_cores def mem(self): - """ - :rtype: int - :return: value of memory + """Returns the amount of memory the host has in kb. + + :rtype: int + :return: amount of memory in kb """ return self.data.memory def memReserved(self): - """ - :rtype: int - :return: value of memory reserved + """Returns the amount of memory the host has currently reserved. + + :rtype: int + :return: amount of memory reserved in kb """ return self.data.memory - self.data.idle_memory def memIdle(self): - """ - :rtype: int - :return: value of memory idle + """Returns the amount of memory the host currently has idle. + + :rtype: int + :return: amount of idle memory in kb """ return self.data.idle_memory def memUsed(self): - """ - :rtype: int - :return: value of memory used + """Returns the amount of memory the host currently has in use. + + :rtype: int + :return: amount of in-use memory in kb """ return self.data.total_memory - self.data.free_memory def memTotal(self): - """ - :rtype: int + """Returns the total amount of memory the host has. + + :rtype: int :return: total amount of memory on host """ return self.data.total_memory def memFree(self): - """ - :rtype: int - :return: amount of free memory + """Returns the amount of memory the host currently has free. + + :rtype: int + :return: amount of free memory in kb """ return self.data.free_memory def swapUsed(self): - """ - :rtype: int - :return: amount of swap used + """Returns the amount of swap space the host has in use. + + :rtype: int + :return: amount of swap used in kb """ return self.data.total_swap - self.data.free_swap def swapTotal(self): - """ - :rtype: int - :return: total amount of swap + """Returns the total amount of swap space the host has. + + :rtype: int + :return: total amount of swap space in kb """ return self.data.total_swap def swapFree(self): - """ - :rtype: int - :return: amount of free swap + """Returns the amount of free swap space the host has. + + :rtype: int + :return: amount of free swap space in kb """ return self.data.free_swap def mcpUsed(self): - """ - :rtype: int - :return: amount of mcp used + """Returns the amount of /mcp space the host is using. + + :rtype: int + :return: amount of mcp used in kb """ return self.mcpTotal() - self.mcpFree() def mcpTotal(self): - """ - :rtype: int - :return: total amount of mcp + """Returns the total amount of /mcp space the host has. + + :rtype: int + :return: total amount of mcp in kb """ return self.data.total_mcp def mcpFree(self): - """ - :rtype: int - :return: amount of mcp free + """Returns the amount of free /mcp space the host has. + + :rtype: int + :return: amount of mcp free in kb """ return self.data.free_mcp def load(self): - """Returns the load on the host - :rtype: int - :return: Host load average * 100 + """Returns the host load average. + + :rtype: int + :return: host load average * 100 """ return self.data.load def bootTime(self): - """ - :rtype: int - :return: Boot time epoch + """Returns the time the host was booted. + + :rtype: int + :return: host boot time as an epoch """ return self.data.boot_time def pingTime(self): - """ - :rtype: int - :return: Ping time epoch + """Returns the last time the host sent a status report. + + :rtype: int + :return: last ping time as an epoch """ return self.data.ping_time def pingLast(self): - """ - :rtype: int - :return: Seconds since last ping + """Returns the number of seconds since the last time the host sent a status report. + + :rtype: int + :return: seconds since last ping """ return int(time.time() - self.pingTime()) def tags(self): - """ - :rtype: list - :return: Tags applied to the host + """Returns the tags the host has. + + :rtype: list + :return: list of tags applied to the host """ return self.data.tags def state(self): - """ - :rtype: opencue.HardwareState - :return: the state of the host + """Returns the hardware state of the host. + + :rtype: host_pb2.HardwareState + :return: the hardware state of the host """ return self.data.state def lockState(self): - """ - :rtype: opencue.LockState + """Returns the lock state of the host. + + :rtype: host_pb2.LockState :return: the lock state of the host """ return self.data.lock_state @@ -431,6 +457,7 @@ def __init__(self, host): def children(self): """The procs running on this host. - :rtype: list - :return: The procs running on this host""" - return self.procs + :rtype: host_pb2.NestedProcSeq + :return: the procs running on this host + """ + return self.data.procs diff --git a/pycue/opencue/wrappers/job.py b/pycue/opencue/wrappers/job.py index 10a140ba8..eb977a280 100644 --- a/pycue/opencue/wrappers/job.py +++ b/pycue/opencue/wrappers/job.py @@ -12,14 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - - -""" -Project: opencue Library - -Module: job.py - opencue Library implementation of a job - -""" +"""Module for classes related to jobs.""" import enum import os @@ -39,6 +32,7 @@ class Job(object): """This class contains the ice implementation related to a job.""" class JobState(enum.IntEnum): + """Enum representing the state of a job.""" PENDING = job_pb2.PENDING FINISHED = job_pb2.FINISHED STARTUP = job_pb2.STARTUP @@ -46,27 +40,28 @@ class JobState(enum.IntEnum): POSTED = job_pb2.POSTED def __init__(self, job=None): - """_Job class initialization""" self.data = job self.stub = Cuebot.getStub('job') + self.__frameStateTotals = {} def kill(self): - """Kills the job""" + """Kills the job.""" self.stub.Kill(job_pb2.JobKillRequest(job=self.data), timeout=Cuebot.Timeout) def pause(self): - """Pauses the job""" + """Pauses the job.""" self.stub.Pause(job_pb2.JobPauseRequest(job=self.data), timeout=Cuebot.Timeout) def resume(self): - """Resumes the job""" + """Resumes the job.""" self.stub.Resume(job_pb2.JobResumeRequest(job=self.data), timeout=Cuebot.Timeout) def killFrames(self, **request): """Kills all frames that match the FrameSearch. :type request: Dict - :param request: FrameSearch parameters""" + :param request: FrameSearch parameters + """ criteria = opencue.search.FrameSearch.criteriaFromOptions(**request) self.stub.KillFrames(job_pb2.JobKillFramesRequest(job=self.data, req=criteria), timeout=Cuebot.Timeout) @@ -75,7 +70,8 @@ def eatFrames(self, **request): """Eats all frames that match the FrameSearch. :type request: Dict - :param request: FrameSearch parameters""" + :param request: FrameSearch parameters + """ criteria = opencue.search.FrameSearch.criteriaFromOptions(**request) return self.stub.EatFrames(job_pb2.JobEatFramesRequest(job=self.data, req=criteria), timeout=Cuebot.Timeout) @@ -84,17 +80,18 @@ def retryFrames(self, **request): """Retries all frames that match the FrameSearch. :type request: Dict - :param request: FrameSearch parameters""" + :param request: FrameSearch parameters + """ criteria = opencue.search.FrameSearch.criteriaFromOptions(**request) return self.stub.RetryFrames(job_pb2.JobRetryFramesRequest(job=self.data, req=criteria), timeout=Cuebot.Timeout) def markdoneFrames(self, **request): - """Drops any dependency that requires any frame that matches the - FrameSearch. + """Drops any dependency that requires any frame that matches the FrameSearch. :type request: Dict - :param request: FrameSearch parameters""" + :param request: FrameSearch parameters + """ criteria = opencue.search.FrameSearch.criteriaFromOptions(**request) return self.stub.MarkDoneFrames( job_pb2.JobMarkDoneFramesRequest(job=self.data, req=criteria), @@ -104,33 +101,37 @@ def markAsWaiting(self, **request): """Changes the matching frames from the depend state to the waiting state. :type request: Dict - :param request: FrameSearch parameters""" + :param request: FrameSearch parameters + """ criteria = opencue.search.FrameSearch.criteriaFromOptions(**request) return self.stub.MarkAsWaiting( job_pb2.JobMarkAsWaitingRequest(job=self.data, req=criteria), timeout=Cuebot.Timeout) def setMinCores(self, minCores): - """Sets the minimum procs value. + """Sets the minimum number of cores the job needs. :type minCores: int - :param minCores: New minimum cores value""" + :param minCores: new minimum cores value + """ self.stub.SetMinCores(job_pb2.JobSetMinCoresRequest(job=self.data, val=minCores), timeout=Cuebot.Timeout) def setMaxCores(self, maxCores): - """Sets the maximum procs value. + """Sets the maximum number of cores the job will use. :type maxCores: int - :param maxCores: New maximum cores value""" + :param maxCores: new maximum cores value + """ self.stub.SetMaxCores(job_pb2.JobSetMaxCoresRequest(job=self.data, val=maxCores), timeout=Cuebot.Timeout) def setPriority(self, priority): - """Sets the priority number. + """Sets the job priority. :type priority: int - :param priority: New priority number""" + :param priority: new job priority number + """ self.stub.SetPriority(job_pb2.JobSetPriorityRequest(job=self.data, val=priority), timeout=Cuebot.Timeout) @@ -138,16 +139,18 @@ def setMaxRetries(self, maxRetries): """Sets the number of retries before a frame goes dead. :type maxRetries: int - :param maxRetries: New max retries""" + :param maxRetries: new max retries + """ self.stub.SetMaxRetries( job_pb2.JobSetMaxRetriesRequest(job=self.data, max_retries=maxRetries), timeout=Cuebot.Timeout) def getLayers(self): - """Returns the list of layers. + """Returns the list of layers in the job. - :rtype: list - :return: List of layers""" + :rtype: list + :return: list of layers in the job + """ response = self.stub.GetLayers(job_pb2.JobGetLayersRequest(job=self.data), timeout=Cuebot.Timeout) layerSeq = response.layers @@ -162,8 +165,9 @@ def getFrames(self, **options): frames = job.getFrames(show=["edu","beo"],user="jwelborn") frames = job.getFrames(show="edu",shot="bs.012") - :rtype: list - :return: List of frames""" + :rtype: list + :return: list of frames + """ criteria = opencue.search.FrameSearch.criteriaFromOptions(**options) response = self.stub.GetFrames(job_pb2.JobGetFramesRequest(job=self.data, req=criteria), timeout=Cuebot.Timeout) @@ -171,26 +175,24 @@ def getFrames(self, **options): return [opencue.wrappers.frame.Frame(frm) for frm in frameSeq.frames] def getUpdatedFrames(self, lastCheck, layers=None): - """Returns a list of updated state information for frames that have - changed since the last update time as well as the current state of the - job. If layer proxies are provided in the layers list, only frames from - those layers will be returned. + """Returns a list of state information for frames that have been recently updated. - UpdatedFrameCheckResult:: - - CueIce::JobState state = - int updated = - UpdatedFrameSeq updatedFrames = + This includes any frames that have changed since the last update time as well as the + current state of the job. If layer proxies are provided in the layers list, only frames + from those layers will be returned. :type lastCheck: int - :param lastCheck: Epoch when last updated + :param lastCheck: epoch when last updated :type layers: list - :param layers: List of layers to check, empty list checks all - :rtype: job_pb2.UpdatedFrameCheckResult - :return: Job state and a list of updatedFrames""" + :param layers: list of layers to check, empty list checks all + :rtype: job_pb2.JobGetUpdatedFramesResponse + :return: job state and a list of updated frames + """ if layers is not None: layerSeq = job_pb2.LayerSeq() + # pylint: disable=no-member layerSeq.layers.extend(layers) + # pylint: enable=no-member else: layerSeq = None return self.stub.GetUpdatedFrames( @@ -199,25 +201,29 @@ def getUpdatedFrames(self, lastCheck, layers=None): timeout=Cuebot.Timeout) def setAutoEating(self, value): - """If set to true, any frames that would become dead, will become eaten. + """Sets the job autoeat field. + + If set to true, any frames that would become dead will become eaten. :type value: bool - :param value: State of autoeat""" + :param value: whether job should autoeat + """ self.stub.SetAutoEat(job_pb2.JobSetAutoEatRequest(job=self.data, value=value), timeout=Cuebot.Timeout) def addRenderPartition(self, hostname, threads, max_cores, num_mem, max_gpu): - """Add a render partition to the job. - @type hostname: str - @param hostname: hostname of the partition - @type threads: int - @param threads: number of threads of the partition - @type max_cores: int - @param max_cores: max cores enabled for the partition - @type num_mem: int - @param num_mem: amount of memory reserved for the partition - @type max_gpu: int - @param max_gpu: max gpu cores enabled for the partition + """Adds a render partition to the job. + + :type hostname: str + :param hostname: hostname of the partition + :type threads: int + :param threads: number of threads of the partition + :type max_cores: int + :param max_cores: max cores enabled for the partition + :type num_mem: int + :param num_mem: amount of memory reserved for the partition + :type max_gpu: int + :param max_gpu: max gpu cores enabled for the partition """ self.stub.AddRenderPartition( job_pb2.JobAddRenderPartRequest(job=self.data, @@ -232,7 +238,8 @@ def getWhatDependsOnThis(self): """Returns a list of dependencies that depend directly on this job. :rtype: list - :return: List of dependencies that depend directly on this job""" + :return: list of dependencies that depend directly on this job + """ response = self.stub.GetWhatDependsOnThis( job_pb2.JobGetWhatDependsOnThisRequest(job=self.data), timeout=Cuebot.Timeout) @@ -243,7 +250,8 @@ def getWhatThisDependsOn(self): """Returns a list of dependencies that this job depends on. :rtype: list - :return: dependencies that this job depends on""" + :return: dependencies that this job depends on + """ response = self.stub.GetWhatThisDependsOn( job_pb2.JobGetWhatThisDependsOnRequest(job=self.data), timeout=Cuebot.Timeout) @@ -254,7 +262,8 @@ def getDepends(self): """Returns a list of all depends this job is involved with. :rtype: list - :return: all depends involved with this job""" + :return: all depends involved with this job + """ response = self.stub.GetDepends( job_pb2.JobGetDependsRequest(job=self.data), timeout=Cuebot.Timeout) @@ -262,46 +271,48 @@ def getDepends(self): return [opencue.wrappers.depend.Depend(dep) for dep in dependSeq.depends] def dropDepends(self, target): - """Drops the desired dependency target: - depend_pb2.DependTarget.AnyTarget - depend_pb2.DependTarget.External - depend_pb2.DependTarget.Internal + """Drops the desired dependency target. + :type target: depend_pb2.DependTarget - :param target: The desired dependency target to drop""" + :param target: the desired dependency target to drop + """ return self.stub.DropDepends(job_pb2.JobDropDependsRequest(job=self.data, target=target), timeout=Cuebot.Timeout) def createDependencyOnJob(self, job): - """Create and return a job on job dependency. + """Creates and returns a job-on-job dependency. :type job: opencue.wrappers.job.Job :param job: the job you want this job to depend on :rtype: opencue.wrappers.depend.Depend - :return: The new dependency""" + :return: the new dependency + """ response = self.stub.CreateDependencyOnJob( job_pb2.JobCreateDependencyOnJobRequest(job=self.data, on_job=job.data), timeout=Cuebot.Timeout) return opencue.wrappers.depend.Depend(response.depend) def createDependencyOnLayer(self, layer): - """Create and return a job on layer dependency. + """Create and return a job-on-layer dependency. :type layer: opencue.wrappers.layer.Layer :param layer: the layer you want this job to depend on :rtype: opencue.wrappers.Depend - :return: the new dependency""" + :return: the new dependency + """ response = self.stub.CreateDependencyOnLayer( job_pb2.JobCreateDependencyOnLayerRequest(job=self.data, layer=layer.data), timeout=Cuebot.Timeout) return opencue.wrappers.depend.Depend(response.depend) def createDependencyOnFrame(self, frame): - """Create and return a job on frame dependency. + """Creates and returns a job-on-frame dependency. :type frame: opencue.wrappers.frame.Frame :param frame: the frame you want this job to depend on :rtype: opencue.wrappers.depend.Depend - :return: the new dependency""" + :return: the new dependency + """ response = self.stub.CreateDependencyOnFrame( job_pb2.JobCreateDependencyOnFrameRequest(job=self.data, frame=frame.data), timeout=Cuebot.Timeout) @@ -322,9 +333,10 @@ def addComment(self, subject, message): """Appends a comment to the job's comment list. :type subject: str - :param subject: Subject data + :param subject: comment subject :type message: str - :param message: Message data""" + :param message: comment message body + """ comment = comment_pb2.Comment( user=os.getenv("USER", "unknown"), subject=subject, @@ -334,7 +346,11 @@ def addComment(self, subject, message): timeout=Cuebot.Timeout) def getComments(self): - """returns the jobs comments""" + """Returns the job's comment list. + + :rtype: list + :return: the job's comment list + """ response = self.stub.GetComments(job_pb2.JobGetCommentsRequest(job=self.data), timeout=Cuebot.Timeout) commentSeq = response.comments @@ -344,130 +360,151 @@ def setGroup(self, group): """Sets the job to a new group. :type group: opencue.wrappers.group.Group - :param group: the group you want the job to be in.""" + :param group: the group you want the job to be in + """ self.stub.SetGroup(job_pb2.JobSetGroupRequest(job=self.data, group_id=group.id()), timeout=Cuebot.Timeout) - def reorderFrames(self, range, order): + def reorderFrames(self, frame_range, order): """Reorders the specified frame range on this job. - :type range: string - :param range: The frame range to reorder + :type frame_range: string + :param frame_range: The frame range to reorder :type order: job_pb2.Order - :param order: First, Last or Reverse""" + :param order: First, Last or Reverse + """ self.stub.ReorderFrames( - job_pb2.JobReorderFramesRequest(job=self.data, range=range, order=order), + job_pb2.JobReorderFramesRequest(job=self.data, range=frame_range, order=order), timeout=Cuebot.Timeout) - def staggerFrames(self, range, stagger): + def staggerFrames(self, frame_range, stagger): """Staggers the specified frame range on this job. - :type range: string - :param range: The frame range to stagger + :type frame_range: string + :param frame_range: the frame range to stagger :type stagger: int - :param stagger: The amount to stagger by""" + :param stagger: the amount to stagger by + """ self.stub.StaggerFrames( - job_pb2.JobStaggerFramesRequest(job=self.data, range=range, stagger=stagger), + job_pb2.JobStaggerFramesRequest(job=self.data, range=frame_range, stagger=stagger), timeout=Cuebot.Timeout) def facility(self): - """Returns the facility that the job must run in""" + """Returns the facility that the job must run in. + + :rtype: str + :return: the job facility + """ return self.data.facility def id(self): - """Returns the uuid of the job. + """Returns the id of the job. :rtype: str - :return: Job uuid""" + :return: id of the job + """ return self.data.id def name(self): """Returns the name of the job. :rtype: str - :return: Job name""" + :return: name of the job + """ return self.data.name def show(self): - """Returns the show name. + """Returns the show name of the job. :rtype: str - :return: Show name""" + :return: show name of the job + """ return self.data.show def shot(self): - """Returns the shot name. + """Returns the shot name of the job. :rtype: str - :return: Shot name""" + :return: shot name of the job + """ return self.data.shot def logDir(self): - """Returns the path to the log files. + """Returns the path to the job log files. :rtype: str - :return: Path of log files""" + :return: path of job log files + """ return self.data.log_dir def uid(self): """Returns the uid of the person who owns the job. :rtype: Optional[int] - :return: Uid of job owner""" + :return: uid of job owner + """ return self.data.uid if self.data.HasField("uid") else None def user(self): """Returns the username of the person who owns the job. :rtype: str - :return: Username of job owner""" + :return: username of job owner + """ return self.data.user def username(self): """Returns the username of the person who owns the job. :rtype: str - :return: Username of job owner""" + :return: username of job owner""" return self.user() def state(self): """Returns the job state. - :rtype: JobState - :return: Job state""" + :rtype: job_pb2.JobState + :return: job state + """ return self.data.state def priority(self): """Returns the job priority. :rtype: int - :return: Job priority""" + :return: job priority + """ return self.data.priority def minCores(self): - """Returns the job's minProcs. + """Returns the minimum number of cores the job needs. :rtype: int - :return: Job's minCores""" + :return: job's min cores + """ return self.data.min_cores def maxCores(self): - """Returns the job's maxProcs. + """Returns the maximum number of cores the job will use. :rtype: int - :return: Job's maxProcs""" + :return: job's max cores + """ return self.data.max_cores def os(self): """Returns the job's operating system. - :rtype: str - :return: operating system name of the Job""" + :rtype: str + :return: operating system name of the job + """ return self.data.os + # pylint: disable=redefined-builtin def startTime(self, format=None): - """Returns the job start time in the desired format:: + """Returns the job start time in the desired format. + Examples: None => 1203634027 "%m/%d %H:%M" => 02/21 14:47 "%a %b %d %H:%M:%S %Y" => Thu Feb 21 14:47:07 2008 @@ -476,16 +513,18 @@ def startTime(self, format=None): https://docs.python.org/3/library/time.html :type format: str - :param format: Desired time format - :rtype: int - :return: Job start time in epoch""" + :param format: desired time format + :rtype: int/str + :return: job start time in epoch, or string version of that timestamp if format given""" if not format: return self.data.start_time return time.strftime(format, time.localtime(self.data.start_time)) + # pylint: disable=redefined-builtin def stopTime(self, format=None): - """Returns the job stop time in the desired format:: + """Returns the job stop time in the desired format. + Examples: None => 1203634027 "%m/%d %H:%M" => 02/21 14:47 "%a %b %d %H:%M:%S %Y" => Thu Feb 21 14:47:07 2008 @@ -494,9 +533,9 @@ def stopTime(self, format=None): https://docs.python.org/3/library/time.html :type format: str - :param format: Desired time format - :rtype: int - :return: Job stop time in epoch""" + :param format: desired time format + :rtype: int/str + :return: job stop time in epoch, or string version of that timestamp if format given""" if not format: return self.data.stop_time return time.strftime(format, time.localtime(self.data.stop_time)) @@ -505,271 +544,307 @@ def runTime(self): """Returns the number of seconds that the job has been (or was) running. :rtype: int - :return: Job runtime in seconds""" + :return: job runtime in seconds + """ if self.data.stop_time == 0: return int(time.time() - self.data.start_time) - else: - return self.data.stop_time - self.data.start_time + return self.data.stop_time - self.data.start_time def coreSecondsRemaining(self): - """Returns the estimated number of core seconds reeded to finish all - waiting frames. Does note take into account running frames. + """Returns the estimated number of core seconds needed to finish all waiting frames. + + Does not take into account running frames. :rtype: long - :return: core seconds remaining""" + :return: core seconds remaining + """ return self.data.job_stats.remaining_core_sec def age(self): """Returns the number of seconds since the job was launched. :rtype: int - :return: Seconds since the job was launched""" + :return: seconds since the job was launched + """ return int(time.time() - self.data.start_time) def isPaused(self): - """Returns true if the job is paused + """Returns true if the job is paused. + :rtype: bool - :return: Paused or not paused""" + :return: paused or not paused""" return self.data.is_paused def isAutoEating(self): """Returns true if the job is eating all frames that become dead. :rtype: bool - :return: If the job eating all frames that become dead""" + :return: if the job eating all frames that become dead + """ return self.data.auto_eat def isCommented(self): """Returns true if the job has a comment. :rtype: bool - :return: If the job has a comment""" + :return: if the job has a comment + """ return self.data.has_comment def setAutoEat(self, value): - """Changes the state of autoeating. When frames become eaten instead of dead. + """Sets a new autoeat value for the job. + + Autoeat means job frames, when they would become dead, are eaten instead. :type value: bool - :param value: The new state for autoEat""" + :param value: new state for autoeat + """ self.setAutoEating(value) self.data.auto_eat = value def coresReserved(self): - """Returns the number of reserved cores. + """Returns the number of reserved cores the job has. - :rtype: float - :return: total number of reserved cores""" + :rtype: float + :return: number of reserved cores + """ return self.data.job_stats.reserved_cores def totalFrames(self): - """Returns the total number of frames under this object. + """Returns the total number of frames the job has. :rtype: int - :return: Total number of frames""" + :return: total number of frames + """ return self.data.job_stats.total_frames def totalLayers(self): - """Returns the total number of frames under this object. + """Returns the total number of layers the job has. :rtype: int - :return: Total number of frames""" + :return: total number of layers + """ return self.data.job_stats.total_layers def dependFrames(self): - """Returns the total number of dependent frames under this object. + """Returns the total number of dependent frames the job has. :rtype: int - :return: Total number of dependent frames""" + :return: total number of dependent frames + """ return self.data.job_stats.depend_frames def succeededFrames(self): - """Returns the total number of succeeded frames under this object. + """Returns the total number of succeeded frames the job has. :rtype: int - :return: Total number of succeeded frames""" + :return: total number of succeeded frames + """ return self.data.job_stats.succeeded_frames def runningFrames(self): - """Returns the total number of running frames under this object. + """Returns the total number of running frames the job has. :rtype: int - :return: Total number of running frames""" + :return: total number of running frames + """ return self.data.job_stats.running_frames def deadFrames(self): - """Returns the total number of deads frames under this object. + """Returns the total number of deads frames the job has. :rtype: int - :return: Total number of dead frames""" + :return: total number of dead frames + """ return self.data.job_stats.dead_frames def waitingFrames(self): - """Returns the total number of waiting frames under this object. + """Returns the total number of waiting frames the job has. :rtype: int - :return: Total number of waiting frames""" + :return: total number of waiting frames + """ return self.data.job_stats.waiting_frames def eatenFrames(self): - """Returns the total number of eaten frames under this object. + """Returns the total number of eaten frames the job has. :rtype: int - :return: Total number of eaten frames""" + :return: total number of eaten frames + """ return self.data.job_stats.eaten_frames def pendingFrames(self): - """Returns the total number of pending (dependent and waiting) frames - under this object. + """Returns the total number of pending (dependent and waiting) frames the job has. :rtype: int - :return: Total number of pending (dependent and waiting) frames""" + :return: total number of pending (dependent and waiting) frames + """ return self.data.job_stats.pending_frames def frameStateTotals(self): - """Returns a dictionary of frame states and the number of frames in each - state. The states are available from opencue.FrameState.* + """Returns a dictionary of frame states and the number of frames in each state. + + The states are available from job_pb2.FrameState. - :rtype: dict - :return: total number of frames in each state""" - if not hasattr(self, "__frameStateTotals"): - self.__frameStateTotals = {} + :rtype: dict + :return: total number of frames in each state + """ + if not self.__frameStateTotals: + self.__frameStateTotals.clear() for state in job_pb2.FrameState.keys(): frameCount = getattr(self.data.job_stats, '{}_frames'.format(state.lower()), 0) self.__frameStateTotals[getattr(job_pb2, state)] = frameCount return self.__frameStateTotals def percentCompleted(self): - """Returns the percent that the object's frames are completed. + """Returns the percent that the job's frames are completed. :rtype: float - :return: Percentage of frame completion""" + :return: percentage of frame completion + """ try: return self.data.job_stats.succeeded_frames /\ float(self.data.job_stats.total_frames) * 100.0 - except: + except ZeroDivisionError: return 0 def group(self): """Returns the name of the group that the job is in. :rtype: str - :return: Jobs group name""" + :return: job group name + """ return self.data.group def avgFrameTime(self): """Returns the average completed frame time in seconds. :rtype: int - :return: Average completed frame time in seconds""" + :return: average completed frame time in seconds + """ return self.data.job_stats.avg_frame_sec def averageCoreTime(self): - """Returns the average frame time. + """Returns the average frame core time. :rtype: int - :return: Average frame time for entire job""" + :return: average frame core time for entire job + """ return self.data.job_stats.avg_core_sec def maxRss(self): - """Returns the highest amount of memory that any frame in this job used - in kB. Value is within 5% of the actual highest frame. + """Returns the highest amount of memory that any frame in this job used. + + Value is within 5% of the actual highest frame. :rtype: long - :return: Most memory used by any frame in kB""" + :return: most memory used by any frame in kB""" return self.data.job_stats.max_rss + class NestedJob(Job): """This class contains information and actions related to a nested job.""" def __init__(self, nestedJob=None): super(NestedJob, self).__init__(nestedJob) - ## job children are most likely empty but its possible to - ## populate this with NesterLayer objects. + # job children are most likely empty but its possible to + # populate this with NestedLayer objects. self.__children = [] def children(self): + """Returns all job children.""" return self.__children def kill(self): - """Kills the job""" + """Kills the job.""" self.asJob().kill() def pause(self): - """Pauses the job""" + """Pauses the job.""" self.asJob().pause() def resume(self): - """Resumes the job""" + """Resumes the job.""" self.asJob().resume() def killFrames(self, **request): """Kills all frames that match the FrameSearch. :type request: Dict - :param request: FrameSearch parameters""" + :param request: FrameSearch parameters + """ self.asJob().killFrames(**request) def eatFrames(self, **request): """Eats all frames that match the FrameSearch. :type request: Dict - :param request: FrameSearch parameters""" + :param request: FrameSearch parameters + """ self.asJob().eatFrames(**request) def retryFrames(self, **request): """Retries all frames that match the FrameSearch. :type request: Dict - :param request: FrameSearch parameters""" + :param request: FrameSearch parameters + """ self.asJob().retryFrames(**request) def markdoneFrames(self, **request): - """Drops any dependency that requires any frame that matches the - FrameSearch. + """Drops any dependency that requires any frame that matches the FrameSearch. :type request: Dict - :param request: FrameSearch parameters""" + :param request: FrameSearch parameters + """ self.asJob().markdoneFrames(**request) def markAsWaiting(self, **request): """Changes the matching frames from the depend state to the waiting state. :type request: Dict - :param request: FrameSearch parameters""" + :param request: FrameSearch parameters + """ self.asJob().markAsWaiting(**request) def setMinCores(self, minCores): - """Sets the minimum procs value. + """Sets the minimum cores value. :type minCores: int - :param minCores: New minimum cores value""" + :param minCores: new minimum cores value + """ self.asJob().setMinCores(minCores) def setMaxCores(self, maxCores): - """Sets the maximum procs value. + """Sets the maximum cores value. :type maxCores: int - :param maxCores: New maximum cores value""" + :param maxCores: new maximum cores value + """ self.asJob().setMaxCores(maxCores) def setPriority(self, priority): - """Sets the priority number. + """Sets the job priority. :type priority: int - :param priority: New priority number""" + :param priority: new priority number + """ self.asJob().setPriority(priority) def setMaxRetries(self, maxRetries): """Sets the number of retries before a frame goes dead. :type maxRetries: int - :param maxRetries: New max retries""" + :param maxRetries: new max retries + """ self.asJob().setMaxRetries(maxRetries) def getLayers(self): - """Returns the list of layers. + """Returns the list of job layers. :rtype: list - :return: List of layers""" + :return: list of layers in the job + """ return self.asJob().getLayers() def getFrames(self, **options): @@ -778,137 +853,149 @@ def getFrames(self, **options): frames = job.getFrames(show=["edu","beo"],user="jwelborn") frames = job.getFrames(show="edu",shot="bs.012") Allowed: offset, limit, states+, layers+. frameset, changedate + :rtype: list - :return: List of frames""" + :return: list of matching frames""" return self.asJob().getFrames(**options) def getUpdatedFrames(self, lastCheck, layers=None): - """Returns a list of updated state information for frames that have - changed since the last update time as well as the current state of the - job. If layer proxies are provided in the layers list, only frames from - those layers will be returned. - - UpdatedFrameCheckResult:: + """Returns a list of state information for frames that have been recently updated. - CueIce::JobState state = - int updated = - UpdatedFrameSeq updatedFrames = + This includes any frames that have changed since the last update time as well as the + current state of the job. If layer proxies are provided in the layers list, only frames + from those layers will be returned. :type lastCheck: int - :param lastCheck: Epoch when last updated - :type layers: list - :param layers: List of layers to check, empty list checks all - :rtype: UpdatedFrameCheckResult - :return: Job state and a list of updatedFrames""" + :param lastCheck: epoch when last updated + :type layers: list + :param layers: list of layers to check, empty list checks all + :rtype: job_pb2.JobGetUpdatedFramesResponse + :return: job state and a list of updated frames + """ return self.asJob().getUpdatedFrames(lastCheck, layers) def setAutoEating(self, value): """If set to true, any frames that would become dead, will become eaten. :type value: bool - :param value: State of autoeat""" + :param value: state of autoeat + """ self.asJob().setAutoEating(value) - def getWhatDependsOnThis(self): """Returns a list of dependencies that depend directly on this job. :rtype: list - :return: List of dependencies that depend directly on this job""" + :return: list of dependencies that depend directly on this job + """ return self.asJob().getWhatDependsOnThis() def getWhatThisDependsOn(self): """Returns a list of dependencies that this job depends on. :rtype: list - :return: dependencies that this job depends on""" + :return: dependencies that this job depends on + """ return self.asJob().getWhatThisDependsOn() def getDepends(self): """Returns a list of all depends this job is involved with. :rtype: list - :return: all depends involved with this job""" + :return: all depends involved with this job + """ return self.asJob().getDepends() def dropDepends(self, target): - """Drops the desired dependency target:: + """Drops the desired dependency target. depend_pb2.DependTarget.AnyTarget depend_pb2.DependTarget.External depend_pb2.DependTarget.Internal :type target: depend_pb2.DependTarget - :param target: The desired dependency target to drop""" + :param target: The desired dependency target to drop + """ return self.asJob().dropDepends(target) def createDependencyOnJob(self, job): - """Create and return a job on job dependency. + """Creates and returns a job-on-job dependency. :type job: opencue.wrappers.job.Job :param job: the job you want this job to depend on :rtype: opencue.wrappers.depend.Depend - :return: The new dependency""" - return self.asJob().createDependencyOnJOb(job) + :return: The new dependency + """ + return self.asJob().createDependencyOnJob(job) def createDependencyOnLayer(self, layer): - """Create and return a job on layer dependency. + """Creates and returns a job-on-layer dependency. :type layer: opencue.wrappers.layer.Layer :param layer: the layer you want this job to depend on :rtype: opencue.wrappers.depend.Depend - :return: the new dependency""" + :return: the new dependency + """ return self.asJob().createDependencyOnLayer(layer) def createDependencyOnFrame(self, frame): - """Create and return a job on frame dependency. + """Creates and returns a job-on-frame dependency. :type frame: opencue.wrappers.frame.Frame :param frame: the frame you want this job to depend on :rtype: opencue.wrappers.depend.Depend - :return: the new dependency""" + :return: the new dependency + """ return self.asJob().createDependencyOnFrame(frame) def addComment(self, subject, message): """Appends a comment to the job's comment list. :type subject: str - :param subject: Subject data + :param subject: comment subject :type message: str - :param message: Message data""" + :param message: comment message body + """ self.asJob().addComment(subject, message) def getComments(self): - """Returns the jobs comments.""" + """Returns the job's comment list.""" return self.asJob().getComments() def setGroup(self, group): """Sets the job to a new group. :type group: opencue.wrappers.group.Group - :param group: the group you want the job to be in.""" + :param group: the group you want the job to be in. + """ self.asJob().setGroup(group) - def reorderFrames(self, range, order): + def reorderFrames(self, frame_range, order): """Reorders the specified frame range on this job. - :type range: string - :param range: The frame range to reorder + :type frame_range: string + :param frame_range: The frame range to reorder :type order: job_pb2.Order - :param order: First, Last or Reverse""" - self.asJob().reorderFrames(range, order) + :param order: First, Last or Reverse + """ + self.asJob().reorderFrames(frame_range, order) - def staggerFrames(self, range, stagger): + def staggerFrames(self, frame_range, stagger): """Staggers the specified frame range on this job. - :type range: string - :param range: The frame range to stagger + :type frame_range: string + :param frame_range: the frame range to stagger :type stagger: int - :param stagger: The amount to stagger by""" - self.asJob().staggerFrames(range, stagger) + :param stagger: the amount to stagger by + """ + self.asJob().staggerFrames(frame_range, stagger) def asJob(self): - """Returns a Job object from this NestedJob""" + """Returns a Job object from this NestedJob. + + :rtype: opencue.wrappers.job.Job + :return: Job version of this NestedJob + """ return Job(job_pb2.Job( id=self.data.id, state=self.data.state, diff --git a/pycue/opencue/wrappers/layer.py b/pycue/opencue/wrappers/layer.py index dc33bc57a..7c33b68e7 100644 --- a/pycue/opencue/wrappers/layer.py +++ b/pycue/opencue/wrappers/layer.py @@ -12,35 +12,32 @@ # See the License for the specific language governing permissions and # limitations under the License. - -""" -opencue layer module - -implementation of a layer in opencue -""" +"""Module for classes related to job layers.""" import enum import os +import opencue.api from opencue.compiled_proto import job_pb2 from opencue.cuebot import Cuebot import opencue.search import opencue.wrappers.depend import opencue.wrappers.frame import opencue.wrappers.limit -import opencue.api class Layer(object): """This class contains the grpc implementation related to a Layer.""" class LayerType(enum.IntEnum): + """Represents the type of layer.""" PRE = job_pb2.PRE POST = job_pb2.POST RENDER = job_pb2.RENDER UTIL = job_pb2.UTIL class Order(enum.IntEnum): + """Represents the order of a layer.""" FIRST = job_pb2.FIRST LAST = job_pb2.LAST REVERSE = job_pb2.REVERSE @@ -50,51 +47,54 @@ def __init__(self, layer=None): self.stub = Cuebot.getStub('layer') def kill(self): - """Kill entire layer""" + """Kills the entire layer.""" return self.stub.KillFrames(job_pb2.LayerKillFramesRequest(layer=self.data), timeout=Cuebot.Timeout) def eat(self): - """Eat entire layer""" + """Eats the entire layer.""" return self.stub.EatFrames(job_pb2.LayerEatFramesRequest(layer=self.data), timeout=Cuebot.Timeout) def retry(self): - """Retry entire layer""" + """Retries the entire layer.""" return self.stub.RetryFrames(job_pb2.LayerRetryFramesRequest(layer=self.data), timeout=Cuebot.Timeout) def markdone(self): - """Drops any dependency that requires this layer or requires any frame - in the layer""" + """Drops any dependency that requires this layer or requires any frame in the layer.""" return self.stub.MarkdoneFrames(job_pb2.LayerMarkdoneFramesRequest(layer=self.data), timeout=Cuebot.Timeout) def addLimit(self, limit_id): - """Add a limit to the current layer.""" + """Adds a limit to the current layer.""" return self.stub.AddLimit(job_pb2.LayerAddLimitRequest(layer=self.data, limit_id=limit_id), timeout=Cuebot.Timeout) - + def dropLimit(self, limit_id): - """Remove a limit on the current layer.""" + """Removes a limit on the current layer.""" return self.stub.DropLimit( job_pb2.LayerDropLimitRequest(layer=self.data, limit_id=limit_id), timeout=Cuebot.Timeout) def enableMemoryOptimizer(self, value): - """Set enableMemoryOptimizer to the value. + """Enables or disables the memory optimizer. - :type value: bool - :param value: boolean to enable/disable memory optimizer""" + :type value: bool + :param value: whether memory optimizer is enabled + """ return self.stub.EnableMemoryOptimizer(job_pb2.LayerEnableMemoryOptimizerRequest( layer=self.data, value=value), timeout=Cuebot.Timeout) def getFrames(self, **options): - """Returns the list of up to 1000 frames from within the layer. + """Returns a list of up to 1000 frames from within the layer. + :type options: dict + :param options: FrameSearch options :rtype: list - :return: Sequence of Frame obejcts""" + :return: sequence of matching frames + """ criteria = opencue.search.FrameSearch.criteriaFromOptions(**options) response = self.stub.GetFrames(job_pb2.LayerGetFramesRequest(layer=self.data, s=criteria), timeout=Cuebot.Timeout) @@ -104,15 +104,17 @@ def getOutputPaths(self): """Return the output paths for this layer. :rtype: list - :return: list of output paths""" + :return: list of output paths + """ return self.stub.GetOutputPaths(job_pb2.LayerGetOutputPathsRequest(layer=self.data), timeout=Cuebot.Timeout).output_paths def setTags(self, tags): - """Sets the tags, TODO: update description of tag structure. + """Sets the layer tags. :type tags: list - :param tags: Layer tags""" + :param tags: layer tags + """ return self.stub.SetTags(job_pb2.LayerSetTagsRequest(layer=self.data, tags=tags), timeout=Cuebot.Timeout) @@ -120,17 +122,20 @@ def setMaxCores(self, cores): """Sets the maximum number of cores that this layer requires. :type cores: float - :param cores: Core units, 100 reserves 1 core""" + :param cores: Core units, 100 reserves 1 core + """ return self.stub.SetMaxCores( job_pb2.LayerSetMaxCoresRequest(layer=self.data, cores=cores/100.0), timeout=Cuebot.Timeout) def setMinCores(self, cores): """Sets the minimum number of cores that this layer requires. + Use 100 to reserve 1 core. :type cores: int - :param cores: Core units, 100 reserves 1 core""" + :param cores: core units, 100 reserves 1 core + """ return self.stub.SetMinCores( job_pb2.LayerSetMinCoresRequest(layer=self.data, cores=cores/100.0), timeout=Cuebot.Timeout) @@ -139,25 +144,28 @@ def setMinGpu(self, gpu): """Sets the minimum number of gpu memory that this layer requires. :type gpu: int - :param gpu: gpu value""" + :param gpu: gpu value + """ return self.stub.SetMinGpu( job_pb2.LayerSetMinGpuRequest(layer=self.data, gpu=gpu), timeout=Cuebot.Timeout) def setMinMemory(self, memory): - """Sets the minimum amount of memory that this layer requires. in Kb + """Sets the minimum amount of memory that this layer requires. :type memory: int - :param memory: Minimum Kb memory reserved by each frame""" + :param memory: Minimum Kb memory reserved by each frame + """ return self.stub.SetMinMemory( job_pb2.LayerSetMinMemoryRequest(layer=self.data, memory=memory), timeout=Cuebot.Timeout) def setThreadable(self, threadable): - """Set enableMemoryOptimizer to the value. + """Sets the threadable field. - :type threadable: bool - :param threadable: boolean to enable/disable threadable""" + :type threadable: bool + :param threadable: boolean to enable/disable threadable + """ return self.stub.SetThreadable(job_pb2.LayerSetThreadableRequest( layer=self.data, threadable=threadable), timeout=Cuebot.Timeout) @@ -179,17 +187,18 @@ def setTimeoutLLU(self, timeout_llu): timeout=Cuebot.Timeout) def addRenderPartition(self, hostname, threads, max_cores, num_mem, max_gpu): - """Add a render partition to the layer. - @type hostname: str - @param hostname: hostname of the partition - @type threads: int - @param threads: number of threads of the partition - @type max_cores: int - @param max_cores: max cores enabled for the partition - @type num_mem: int - @param num_mem: amount of memory reserved for the partition - @type max_gpu: int - @param max_gpu: max gpu cores enabled for the partition + """Adds a render partition to the layer. + + :type hostname: str + :param hostname: hostname of the partition + :type threads: int + :param threads: number of threads of the partition + :type max_cores: int + :param max_cores: max cores enabled for the partition + :type num_mem: int + :param num_mem: amount of memory reserved for the partition + :type max_gpu: int + :param max_gpu: max gpu cores enabled for the partition """ self.stub.AddRenderPartition( job_pb2.LayerAddRenderPartitionRequest(layer=self.data, @@ -204,7 +213,8 @@ def getWhatDependsOnThis(self): """Gets a list of dependencies that depend directly on this layer. :rtype: list - :return: List of dependencies that depend directly on this layer""" + :return: list of dependencies that depend directly on this layer + """ response = self.stub.GetWhatDependsOnThis( job_pb2.LayerGetWhatDependsOnThisRequest(layer=self.data), timeout=Cuebot.Timeout) @@ -212,10 +222,11 @@ def getWhatDependsOnThis(self): return [opencue.wrappers.depend.Depend(dep) for dep in dependSeq.depends] def getWhatThisDependsOn(self): - """Get a list of dependencies that this layer depends on. + """Returns a list of dependencies that this layer depends on. :rtype: list - :return: List of dependences that this layer depends on""" + :return: list of dependences that this layer depends on + """ response = self.stub.GetWhatThisDependsOn( job_pb2.LayerGetWhatThisDependsOnRequest(layer=self.data), timeout=Cuebot.Timeout) @@ -223,48 +234,52 @@ def getWhatThisDependsOn(self): return [opencue.wrappers.depend.Depend(dep) for dep in dependSeq.depends] def createDependencyOnJob(self, job): - """Create and return a layer on job dependency. + """Creates and returns a layer-on-job dependency. :type job: opencue.wrappers.job.Job :param job: the job you want this job to depend on :rtype: opencue.wrappers.depend.Depend - :return: the new dependency""" + :return: the new dependency + """ response = self.stub.CreateDependencyOnJob( job_pb2.LayerCreateDependOnJobRequest(layer=self.data, job=job.data), timeout=Cuebot.Timeout) return opencue.wrappers.depend.Depend(response.depend) def createDependencyOnLayer(self, layer): - """Create and return a layer on layer dependency. + """Creates and returns a layer-on-layer dependency. :type layer: opencue.wrappers.layer.Layer :param layer: the layer you want this layer to depend on :rtype: opencue.wrappers.depend.Depend - :return: the new dependency""" + :return: the new dependency + """ response = self.stub.CreateDependencyOnLayer( job_pb2.LayerCreateDependOnLayerRequest(layer=self.data, depend_on_layer=layer.data), timeout=Cuebot.Timeout) return opencue.wrappers.depend.Depend(response.depend) def createDependencyOnFrame(self, frame): - """Create and return a layer on frame dependency. + """Creates and returns a layer-on-frame dependency. :type frame: opencue.wrappers.frame.Frame :param frame: the frame you want this layer to depend on :rtype: opencue.wrappers.depend.Depend - :return: the new dependency""" + :return: the new dependency + """ response = self.stub.CreateDependencyOnFrame( job_pb2.LayerCreateDependOnFrameRequest(layer=self.data, frame=frame.data), timeout=Cuebot.Timeout) return opencue.wrappers.depend.Depend(response.depend) def createFrameByFrameDependency(self, layer): - """Create and return a frame by frame frame dependency. + """Creates and returns a frame-by-frame layer dependency. :param layer: the layer you want this layer to depend on :type layer: opencue.wrappers.layer.Layer :rtype: opencue.wrappers.depend.Depend - :return: the new dependency""" + :return: the new dependency + """ # anyframe is hard coded right now, this option should be moved # to LayerOnLayer for better efficiency. response = self.stub.CreateFrameByFrameDependency( @@ -285,220 +300,254 @@ def createFrameByFrameDependency(self, layer): # self.proxy.unbookProcs([a.proxy for a in subs], number, kill) def registerOutputPath(self, outputPath): - """Register an output with the given layer. The output paths are sent in the opencue email. + """Registers an output path for the layer. - :type outputPath: str - :param outputPath: Output path to register + Output paths are included in OpenCue alert emails. + + :type outputPath: str + :param outputPath: output path to register """ self.stub.RegisterOutputPath( job_pb2.LayerRegisterOutputPathRequest(layer=self.data, spec=outputPath), timeout=Cuebot.Timeout) - def reorderFrames(self, range, order): + def reorderFrames(self, frameRange, order): """Reorders the specified frame range on this layer. - :type range: string - :param range: The frame range to reorder + :type frameRange: string + :param frameRange: the frame range to reorder :type order: opencue.wrapper.layer.Layer.Order - :param order: First, Last or Reverse""" + :param order: First, Last or Reverse + """ self.stub.ReorderFrames( - job_pb2.LayerReorderFramesRequest(layer=self.data, range=range, order=order), + job_pb2.LayerReorderFramesRequest(layer=self.data, range=frameRange, order=order), timeout=Cuebot.Timeout) - def staggerFrames(self, range, stagger): + def staggerFrames(self, frameRange, stagger): """Staggers the specified frame range on this layer. - :type range: string - :param range: The frame range to stagger + :type frameRange: string + :param frameRange: the frame range to stagger :type stagger: int - :param stagger: The amount to stagger by""" + :param stagger: the amount to stagger by + """ self.stub.StaggerFrames( - job_pb2.LayerStaggerFramesRequest(layer=self.data, range=range, stagger=stagger), + job_pb2.LayerStaggerFramesRequest(layer=self.data, range=frameRange, stagger=stagger), timeout=Cuebot.Timeout) - + def getLimitDetails(self): - """Return the Limit objects for the given layer. + """Returns the Limit objects for the given layer. - :rtype: list - :return: The list of limits on this layer.""" + :rtype: list + :return: list of limits on this layer + """ return [opencue.wrappers.limit.Limit(limit) for limit in self.stub.GetLimits( job_pb2.LayerGetLimitsRequest(layer=self.data), timeout=Cuebot.Timeout).limits] def id(self): - """Returns the uuid of the layer. + """Returns the id of the layer. :rtype: str - :return: Layer uuid""" + :return: layer id + """ return self.data.id def name(self): """Returns the name of the layer. :rtype: str - :return: Layer name""" + :return: layer name + """ return self.data.name def range(self): - """Returns the frame range for the layer. + """Returns the frame range of the layer. :rtype: str - :return: Layer frame range""" + :return: layer frame range + """ return self.data.range def chunkSize(self): """Returns the number of frames per task. :rtype: int - :return: the chunks size""" + :return: layer chunk size + """ return self.data.chunk_size def tags(self): - """Returns the tags applied to the layer + """Returns the tags applied to the layer. + TODO: Document syntax :rtype: str - :return: Layer tags""" + :return: layer tags""" return self.data.tags def dispatchOrder(self): - """Returns the layers dispatch order. + """Returns the layer dispatch order. :rtype: int - :return: Layer dispatch order""" + :return: layer dispatch order + """ return self.data.dispatch_order def coresReserved(self): """Returns the number of cores reserved on this layer. :rtype: float - :return: cores reserved""" + :return: cores reserved + """ return self.data.layer_stats.reserved_cores def minCores(self): """Returns the minimum number of cores that frames in this layer require. :rtype: int - :return: Minimum number of cores required""" + :return: minimum number of cores required + """ return self.data.min_cores def minMemory(self): - """Returns the minimum about of memory that frames in this layer require. + """Returns the minimum amount of memory that frames in this layer require. :rtype: int - :return: Minimum Kb memory required by frames in this layer""" + :return: minimum kB of memory required by frames in this layer + """ return self.data.min_memory - + def limits(self): """Returns the limit names for this layer. :rtype: list - :return: Names of the limits on this layer.""" + :return: names of the limits on this layer + """ return self.data.limits def maxRss(self): - """Returns the highest amount of memory that any frame in this layer - used in kB. Value is within 5% of the actual highest frame. + """Returns the highest amount of memory that any frame in this layer used. + + Value is within 5% of the actual highest frame. :rtype: long - :return: Most memory used by any frame in this layer in kB""" + :return: most memory used by any frame in this layer in kB""" return self.data.layer_stats.max_rss def type(self): - """Returns the type of layer. Ex: Pre, Post, Render + """Returns the type of layer. - :rtype: opencue.LayerType - :return: Type of layer""" + Ex: Pre, Post, Render + + :rtype: job_pb2.LayerType + :return: type of layer + """ return self.data.type def totalFrames(self): - """Returns the total number of frames under this object. + """Returns the total number of frames in the layer. :rtype: int - :return: Total number of frames""" + :return: total number of frames + """ return self.data.layer_stats.total_frames def dependFrames(self): - """Returns the total number of dependent frames under this object. + """Returns the total number of dependent frames in the layer. :rtype: int - :return: Total number of dependent frames""" + :return: total number of dependent frames + """ return self.data.layer_stats.depend_frames def succeededFrames(self): - """Returns the total number of succeeded frames under this object. + """Returns the total number of succeeded frames in the layer. :rtype: int - :return: Total number of succeeded frames""" + :return: total number of succeeded frames + """ return self.data.layer_stats.succeeded_frames def runningFrames(self): - """Returns the total number of running frames under this object. + """Returns the total number of running frames in the layer. :rtype: int - :return: Total number of running frames""" + :return: total number of running frames + """ return self.data.layer_stats.running_frames def deadFrames(self): - """Returns the total number of deads frames under this object. + """Returns the total number of dead frames in the layer. :rtype: int - :return: Total number of dead frames""" + :return: total number of dead frames + """ return self.data.layer_stats.dead_frames def waitingFrames(self): - """Returns the total number of waiting frames under this object. + """Returns the total number of waiting frames in the layer. :rtype: int - :return: Total number of waiting frames""" + :return: total number of waiting frames + """ return self.data.layer_stats.waiting_frames def eatenFrames(self): - """Returns the total number of eaten frames under this object. + """Returns the total number of eaten frames in the layer. :rtype: int - :return: Total number of eaten frames""" + :return: total number of eaten frames + """ return self.data.layer_stats.eaten_frames def pendingFrames(self): - """Returns the total number of pending (dependent and waiting) frames - under this object. + """Returns the total number of pending (dependent and waiting) frames in the layer. :rtype: int - :return: Total number of pending (dependent and waiting) frames""" + :return: total number of pending (dependent and waiting) frames + """ return self.data.layer_stats.pending_frames def percentCompleted(self): - """Returns the percent that the object's frames are completed. + """Returns the percent that the layer's frames are completed. :rtype: float - :return: Percentage of frame completion""" + :return: percentage of frame completion + """ try: return self.data.layer_stats.succeeded_frames / \ float(self.data.layer_stats.total_frames) * 100.0 - except: + except ZeroDivisionError: return 0 def avgFrameTimeSeconds(self): """Returns the average frame completion time in seconds. :rtype: int - :return: Average frame completion time in seconds""" + :return: average frame completion time in seconds + """ return self.data.layer_stats.avg_frame_sec def avgCoreSeconds(self): """Returns the average core time used in seconds. :rtype: int - :return: Average core time in seconds""" + :return: average core time in seconds + """ return self.data.layer_stats.avg_core_sec def coreSecondsRemaining(self): - """Returns the estimated core time that is remnainining to complete - all waiting frames. + """Returns the estimated core time that is remaining to complete all waiting frames. :rtype: int - :return: the number of seconds of estimated core time remaining""" + :return: the number of seconds of estimated core time remaining + """ return self.data.layer_stats.remaining_core_sec def parent(self): + """Gets the parent of the layer; its job. + + :rtype: opencue.wrappers.job.Job + :return: the layer's parent job + """ return opencue.api.getJob(self.data.parent_id) diff --git a/pycue/opencue/wrappers/limit.py b/pycue/opencue/wrappers/limit.py index 309261cc6..89bddca35 100644 --- a/pycue/opencue/wrappers/limit.py +++ b/pycue/opencue/wrappers/limit.py @@ -12,13 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - -""" -Project: opencue Library - -Module: limit.py - opencue Library implementation of a limit - -""" +"""Module for classes related to limits.""" from opencue import Cuebot from opencue.compiled_proto import limit_pb2 @@ -28,85 +22,104 @@ class Limit(object): """This class contains the grpc implementation related to a Limit.""" def __init__(self, limit=None): - """Limit class initialization""" self.data = limit self.stub = Cuebot.getStub('limit') def create(self): - """Create a new Limit from the current Limit object. - - :rtype: Limit - :return: The newly created Limit + """Creates a new Limit from the current Limit object. + + :rtype: opencue.wrappers.limit.Limit + :return: the newly created Limit """ return Limit(self.stub.Create( limit_pb2.LimitCreateRequest(name=self.name(), max_value=self.maxValue()), timeout=Cuebot.Timeout)) def delete(self): - """Delete the limit record""" + """Deletes the limit.""" self.stub.Delete(limit_pb2.LimitDeleteRequest(name=self.name()), timeout=Cuebot.Timeout) def find(self, name): - """Find an existing limit by it's name. - - :type name: str - :param name: Name of limit to find. - :rtype: opencue.wrappers.limit.Limit - :return: The limit found by name. + """Finds an existing limit by its name. + + :type name: str + :param name: name of limit to find + :rtype: opencue.wrappers.limit.Limit + :return: the limit found by name """ - return Limit(self.stub.Find(limit_pb2.LimitFindRequest(name=name), timeout=Cuebot.Timeout).limit) - - def get(self, id): - """Return an existing limit by it's id. - - :type id: str - :param id: Name of limit to find. - :rtype: opencue.wrappers.limit.Limit - :return: The limit found by id. + return Limit( + self.stub.Find(limit_pb2.LimitFindRequest(name=name), timeout=Cuebot.Timeout).limit) + + def get(self, limit_id): + """Returns an existing limit by its id. + + :type limit_id: str + :param limit_id: id of limit to find + :rtype: opencue.wrappers.limit.Limit + :return: the limit found by id. """ - return Limit(self.stub.Get(limit_pb2.LimitGetRequest(id=id), timeout=Cuebot.Timeout).limit) + return Limit( + self.stub.Get(limit_pb2.LimitGetRequest(id=limit_id), timeout=Cuebot.Timeout).limit) def rename(self, newName): - """Rename the current limit to the provided newName. - - :type newName: str - :param newName: Name to rename the limit to. + """Renames the limit. + + :type newName: str + :param newName: new limit name """ self.stub.Rename(limit_pb2.LimitRenameRequest(old_name=self.name(), new_name=newName), timeout=Cuebot.Timeout) self._update() def setMaxValue(self, maxValue): - """Set the max value of an existing limit. - - :type maxValue: int - :param maxValue: Max value number to set limit to. + """Sets the maximum value of an existing limit. + + :type maxValue: int + :param maxValue: new limit maximum """ - self.stub.SetMaxValue(limit_pb2.LimitSetMaxValueRequest(name=self.name(), max_value=maxValue), - timeout=Cuebot.Timeout) + self.stub.SetMaxValue( + limit_pb2.LimitSetMaxValueRequest(name=self.name(), max_value=maxValue), + timeout=Cuebot.Timeout) self._update() def _update(self): - """Update the current data object from the DB.""" + """Updates the current data object from the database.""" self.data = self.stub.Get(limit_pb2.LimitGetRequest(id=self.id()), timeout=Cuebot.Timeout) def id(self): + """Returns the limit id. + + :rtype: str + :return: the limit id + """ return self.data.id def name(self): + """Returns the limit name. + + :rtype: str + :return: the limit name + """ if hasattr(self.data, 'name'): return self.data.name - else: - return "" + return "" def maxValue(self): + """Returns the limit maximum. + + :rtype: int + :return: the limit maximum + """ if hasattr(self.data, 'max_value'): return self.data.max_value - else: - return -1 + return -1 def currentRunning(self): + """Returns the current amount of the limit in use. + + :rtype: int + :return: current limit usage + """ if hasattr(self.data, 'current_running'): return self.data.current_running - else: - return -1 + return -1 diff --git a/pycue/opencue/wrappers/owner.py b/pycue/opencue/wrappers/owner.py index f4ab9f21f..c7896d88a 100644 --- a/pycue/opencue/wrappers/owner.py +++ b/pycue/opencue/wrappers/owner.py @@ -12,81 +12,99 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Module for classes related to owners.""" -""" -Project: opencue Library - -Module: owner.py - opencue Library implementation of a owner - -""" - -import opencue.wrappers.deed -import opencue.wrappers.host from opencue import Cuebot from opencue.compiled_proto import host_pb2 +import opencue.wrappers.deed +import opencue.wrappers.host class Owner(object): """This class contains the grpc implementation related to an Owner.""" def __init__(self, owner=None): - """Host class initialization""" self.data = owner self.stub = Cuebot.getStub('owner') def delete(self): - """Delete the owner record""" + """Deletes the owner record.""" self.stub.Delete(host_pb2.OwnerDeleteRequest(owner=self.data), timeout=Cuebot.Timeout) def getDeeds(self): - """Return the list of deeds for the owner. + """Returns the list of deeds for the owner. - :rtype: List - :return: The list of deeds associated with this owner.""" + :rtype: list + :return: the list of deeds associated with this owner + """ response = self.stub.GetDeeds(host_pb2.OwnerGetDeedsRequest(owner=self.data), timeout=Cuebot.Timeout) return [opencue.wrappers.deed.Deed(deed) for deed in response.deeds.deeds] def getHosts(self): - """Get a list of all hosts this owner is responsible for. + """Returns a list of all hosts this owner is responsible for. - :rtype: List - :return: List of hosts the owned by this owner.""" + :rtype: list + :return: list of hosts owned by this owner + """ response = self.stub.GetHosts(host_pb2.OwnerGetHostsRequest(owner=self.data), timeout=Cuebot.Timeout) return [opencue.wrappers.host.Host(host) for host in response.hosts.hosts] def getOwner(self, name): - """Return an owner by name. + """Returns an owner by name. - :type: str - :param: Name of the owner - :rtype: opencue.wrappers.owner.Owner - :return: Owner that matches the specified name""" + :type name: str + :param name: Name of the owner + :rtype: opencue.wrappers.owner.Owner + :return: owner that matches the specified name + """ return Owner(self.stub.GetOwner(host_pb2.OwnerGetOwnerRequest(name=name), timeout=Cuebot.Timeout).owner) def setShow(self, show): - """Set the show for the owner. + """Sets the show for the owner. - :type: str - :param: name of the show""" + :type show: str + :param show: name of the show + """ self.stub.SetShow(host_pb2.OwnerSetShowRequest(owner=self.data, show=show), timeout=Cuebot.Timeout) def takeOwnership(self, host): - """Set the hosts new owner settings.""" + """Sets the hosts for the owner. + + :type host: str + :param host: the name of the host to take ownership of + """ self.stub.TakeOwnership(host_pb2.OwnerTakeOwnershipRequest(owner=self.data, host=host), timeout=Cuebot.Timeout) def hostCount(self): + """Returns the number of hosts owned by this owner. + + :rtype: int + :return: number of hosts owned by this owner + """ return self.data.host_count def id(self): + """Returns the owner id. + + :rtype: str + :return: the owner id""" return self.data.id def name(self): + """Returns the owner name. + + :rtype: str + :return: the owner name""" return self.data.name def show(self): + """Returns the name of the show of the owner. + + :rtype: str + :return: the name of the show of the owner""" return self.data.show diff --git a/pycue/opencue/wrappers/proc.py b/pycue/opencue/wrappers/proc.py index 14e5c44da..da8ab2fae 100644 --- a/pycue/opencue/wrappers/proc.py +++ b/pycue/opencue/wrappers/proc.py @@ -12,14 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - - -""" -Project: opencue Library - -Module: proc.py - opencue Library implementation of a proc - -""" +"""Module for classes related to procs.""" import enum @@ -32,13 +25,18 @@ class Proc(object): - """This class contains the grpc implementation related to a Proc.""" + """This class contains the grpc implementation related to a Proc. + + A proc is a bookable unit of a host. Hosts may contain many procs; each proc can be assigned + a different frame to work on.""" class RedirectType(enum.IntEnum): + """Represents the type of a proc redirect.""" JOB_REDIRECT = host_pb2.JOB_REDIRECT GROUP_REDIRECT = host_pb2.GROUP_REDIRECT class RunState(enum.IntEnum): + """Represents the current state of a proc.""" IDLE = host_pb2.IDLE BOOKED = host_pb2.BOOKED @@ -47,50 +45,56 @@ def __init__(self, proc=None): self.stub = Cuebot.getStub('proc') def kill(self): - """Kill the frame running on this proc""" + """Kills the frame running on this proc.""" response = self.stub.Kill(host_pb2.ProcKillRequest(proc=self.data), timeout=Cuebot.Timeout) return response def unbook(self, kill=False): - """Unbook the current frame. If the value of kill is true, - the frame will be immediately killed. + """Unbooks the current frame from this proc. + + :type kill: bool + :param kill: if true, the frame will be immediately killed """ - response = self.stub.Unbook(host_pb2.ProcUnbookRequest(proc=self.data, kill=kill), - timeout=Cuebot.Timeout) + response = self.stub.Unbook( + host_pb2.ProcUnbookRequest(proc=self.data, kill=kill), timeout=Cuebot.Timeout) return response def getHost(self): - """Return the host this proc is allocated from. + """Returns the host this proc is allocated from. :rtype: opencue.wrappers.host.Host - :return: The host this proc is allocated from.""" + :return: the host this proc is allocated from + """ response = self.stub.GetHost(host_pb2.ProcGetHostRequest(proc=self.data), timeout=Cuebot.Timeout) return opencue.wrappers.host.Host(response.host) def getFrame(self): - """Return the frame this proc is running. + """Returns the frame this proc is running. :rtype: opencue.wrappers.frame.Frame - :return: The fame this proc is running.""" + :return: the frame this proc is running + """ response = self.stub.GetFrame(host_pb2.ProcGetFrameRequest(proc=self.data), timeout=Cuebot.Timeout) return opencue.wrappers.frame.Frame(response.frame) def getLayer(self): - """Return the layer this proc is running. + """Returns the layer this proc is running. :rtype: opencue.wrappers.layer.Layer - :return: The layer this proc is running.""" + :return: the layer this proc is running + """ response = self.stub.GetLayer(host_pb2.ProcGetLayerRequest(proc=self.data), timeout=Cuebot.Timeout) return opencue.wrappers.layer.Layer(response.layer) def getJob(self): - """Return the job this proc is running. + """Returns the job this proc is running. :rtype: opencue.wrappers.job.Job - :return: The job this proc is running.""" + :return: the job this proc is running + """ response = self.stub.GetJob(host_pb2.ProcGetJobRequest(proc=self.data), timeout=Cuebot.Timeout) return opencue.wrappers.job.Job(response.job) @@ -99,72 +103,87 @@ def id(self): """Returns the id of the proc. :rtype: str - :return: Proc uuid""" + :return: id of the proc + """ return self.data.id def name(self): """Returns the name of the proc. :rtype: str - :return: Proc name""" + :return: name of the proc + """ return self.data.name def jobName(self): - """Returns the job name of the frame running on the proc. + """Returns the name of the job of the frame running on the proc. :rtype: str - :return: Job name""" + :return: name of the current job""" return self.data.job_name def frameName(self): """Returns the name of the frame on the proc. :rtype: str - :return: Frame name""" + :return: name of the current frame + """ return self.data.frame_name def showName(self): - """Returns the name of the show whos frame is running on the proc. + """Returns the name of the show of the frame running on the proc. :rtype: str - :return: Frames show name""" + :return: name of the current show + """ return self.data.show_name def coresReserved(self): """The number of cores reserved for this frame. :rtype: float - :return: Cores reserved for the running frame""" + :return: cores reserved for the running frame + """ return self.data.reserved_cores def memReserved(self): """The amount of memory reserved for the running frame. :rtype: int - :return: Kb memory reserved for the running frame""" + :return: memory reserved for the running frame in kB + """ return self.data.reserved_memory def memUsed(self): """The amount of memory used by the running frame. :rtype: int - :return: Kb memory used by the running frame""" + :return: memory used by the running frame in kB + """ return self.data.used_memory - + def bookedTime(self): - """The last time this proc was assigned to a job in epoch seconds. - :rtype: int""" + """The last time this proc was booked to a job. + + :rtype: int + :return: last time booked as an epoch + """ return self.data.booked_time def dispatchTime(self): - """The last time this proc was assigned to a job in epoch seconds. - :rtype: int""" + """The last time this proc was dispatched work. + + :rtype: int + :return: last time dispatched as an epoch + """ return self.data.dispatch_time - + def isUnbooked(self): - """Returns true if this proc is unbooked. + """Returns whether this proc is unbooked. - :rtype: boolean""" + :rtype: bool + :return: whether the proc is unbooked + """ return self.data.unbooked @@ -173,9 +192,10 @@ class NestedProc(Proc): def __init__(self, nestedProc): super(NestedProc, self).__init__(nestedProc) - ## job children are most likely empty but its possible to - ## populate this with NestedLayer objects. + # job children are most likely empty but its possible to + # populate this with NestedLayer objects. self.__children = [] def children(self): + """Returns children of the proc.""" return self.__children diff --git a/pycue/opencue/wrappers/service.py b/pycue/opencue/wrappers/service.py index 35e8c5db6..acacb233a 100644 --- a/pycue/opencue/wrappers/service.py +++ b/pycue/opencue/wrappers/service.py @@ -12,14 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - - -""" -Project: opencue Library - -Module: service.py - opencue Library implementation of a service - -""" +"""Module for classes related to services.""" import grpc @@ -35,18 +28,21 @@ def __init__(self, service=None): self.stub = Cuebot.getStub('service') def create(self): + """Creates a service in the database using the current instance data.""" response = self.stub.CreateService( service_pb2.ServiceCreateServiceRequest(data=self.data), timeout=Cuebot.Timeout) return Service(response.service) def delete(self): + """Deletes the service.""" return self.stub.Delete( service_pb2.ServiceDeleteRequest(service=self.data), timeout=Cuebot.Timeout) @staticmethod def getDefaultServices(): + """Returns the default services.""" response = Cuebot.getStub('service').GetDefaultServices( service_pb2.ServiceGetDefaultServicesRequest(), timeout=Cuebot.Timeout) @@ -54,17 +50,25 @@ def getDefaultServices(): @staticmethod def getService(name): + """Returns a service by name. + + :type name: str + :param name: service name to find + """ try: response = Cuebot.getStub('service').GetService( service_pb2.ServiceGetServiceRequest(name=name), timeout=Cuebot.Timeout) except grpc.RpcError as e: + # pylint: disable=no-member if e.code() == grpc.StatusCode.NOT_FOUND: return None + # pylint: enable=no-member raise e return Service(response.service) def update(self): + """Updates the service database record with the current instance data.""" return self.stub.Update( service_pb2.ServiceUpdateRequest(service=self.data), timeout=Cuebot.Timeout) @@ -73,102 +77,118 @@ def id(self): """Returns the id of the service. :rtype: str - :return: Frame uuid""" + :return: service id + """ return self.data.id def name(self): """Returns the name of the service. :rtype: str - :return: service name""" + :return: service name + """ return self.data.name def setName(self, name): - """Set the name field of the message. + """Sets the service name. :type: string - :param: name to set""" + :param: new service name + """ self.data.name = name def threadable(self): - """Returns if the service is threadable. + """Returns whether the service is threadable. :rtype: bool - :return: is service threadable""" + :return: whether service is threadable + """ return self.data.threadable def setThreadable(self, threadable): - """Set the threadabel field of the message. + """Sets the threadable field of the service. - :type: bool - :param: whether or not the service should be threadable""" + :type: bool + :param: whether or not the service should be threadable + """ self.data.threadable = threadable def minCores(self): - """Returns the min_cores of the service. + """Returns the minimum cores of the service. :rtype: int - :return: min_cores""" + :return: min cores + """ return self.data.min_cores def setMinCores(self, minCores): - """Set the minCores field of the message. + """Sets the minimum cores of the service. :type: int - :param: min_cores""" + :param: new min cores + """ self.data.min_cores = minCores def maxCores(self): - """Returns the max_cores of the service. + """Returns the maximum cores of the service. :rtype: int - :return: max_cores""" + :return: max cores + """ return self.data.max_cores def setMaxCores(self, maxCores): - """Set the maxCores field of the message. + """Sets the maximum cores of the service. :type: int - :param: max_cores""" + :param: new max cores + """ self.data.max_cores = maxCores def minMemory(self): - """Returns the min_memory of the service. + """Returns the minimum memory of the service. :rtype: int - :return: min_memory""" + :return: min memory + """ return self.data.min_memory def setMinMemory(self, minMemory): - """Set the minMemory field of the message. + """Sets the minimum memory field of the service. :type: int - :param: min_memory""" + :param: new min memory + """ self.data.min_memory = minMemory def minGpu(self): - """Returns the min_gpu of the service. + """Returns the minimum gpu of the service. :rtype: int - :return: min_gpu""" + :return: min gpu + """ return self.data.min_gpu def setMinGpu(self, minGpu): - """Set the minGpu field of the message. + """Sets the minimum gpu of the service. :type: int - :param: min_gpu""" + :param: new min gpu + """ self.data.min_gpu = minGpu def tags(self): """Returns the list of tags for the service. :rtype: list - :return: tags""" + :return: list of service tags + """ return self.data.tags def setTags(self, tags): - """Clear and set the tags. - :type: list - :param: list of tags to set""" + """Clears and sets the service tags. + + :type: list + :param: new list of service tags + """ self.data.tags[:] = tags diff --git a/pycue/opencue/wrappers/show.py b/pycue/opencue/wrappers/show.py index 8ab4fce15..750d05645 100644 --- a/pycue/opencue/wrappers/show.py +++ b/pycue/opencue/wrappers/show.py @@ -12,14 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - - -""" -Project: opencue Library - -Module: show.py - opencue Library implementation of a show - -""" +"""Module for classes related to shows.""" from opencue.compiled_proto import show_pb2 from opencue.cuebot import Cuebot @@ -36,28 +29,31 @@ def __init__(self, show=None): self.stub = Cuebot.getStub('show') def createOwner(self, user): - """Creates a new owner. + """Creates a new owner for the show. - :type user: str + :type user: str :param user: user name - :rtype: Owner - :return: The created owner object + :rtype: host_pb2.Owner + :return: the created owner object """ response = self.stub.CreateOwner(show_pb2.ShowCreateOwnerRequest(show=self.data, name=user), timeout=Cuebot.Timeout) return response.owner def createSubscription(self, allocation, size, burst): - """Creates a new subscription. - - :type allocation: opencue.wrappers.allocation.Allocation - :param allocation: Allocation object - :type size: float - :param size: Allocation size - :type burst: float - :param burst: Allocation burst - :rtype: Subscription - :return: The created subscription object + """Creates a new subscription for the show. + + A subscription links a show to an allocation, and determines how many cores the show + can utilize within that allocation. + + :type allocation: opencue.wrappers.allocation.Allocation + :param allocation: allocation to subscribe to + :type size: float + :param size: number of cores the show is allowed to use consistently + :type burst: float + :param burst: number of cores the show is allowed to burst to + :rtype: opencue.wrappers.subscription.Subscription + :return: the created subscription object """ response = self.stub.CreateSubscription(show_pb2.ShowCreateSubscriptionRequest( show=self.data, allocation_id=allocation.id(), size=size, burst=burst), @@ -65,36 +61,36 @@ def createSubscription(self, allocation, size, burst): return opencue.wrappers.subscription.Subscription(response.subscription) def delete(self): - """Delete this show""" + """Deletes this show.""" self.stub.Delete(show_pb2.ShowDeleteRequest(show=self.data), timeout=Cuebot.Timeout) def createServiceOverride(self, data): """Creates a Service Override at the show level. - :type data: opencue.wrapper.service.Service - :param data: Service.data object + :type data: service_pb2.Service + :param data: service data, typically from opencue.wrappers.service.Service.data """ - self.stub.CreateServiceOverride(show_pb2.ShowCreateServiceOverrideRequest( - show=self.data, service=data), - timeout=Cuebot.Timeout) + self.stub.CreateServiceOverride( + show_pb2.ShowCreateServiceOverrideRequest(show=self.data, service=data), + timeout=Cuebot.Timeout) def getServiceOverride(self, serviceName): - """ - Returns a service override for a show + """Returns a service override for a show. + :type serviceName: str :param serviceName: name of the service for the show + :rtype: service_pb2.ServiceOverride :return: service override object """ - serviceOverride = self.stub.GetServiceOverride(show_pb2.ShowGetServiceOverrideRequest( - show=self.data, name=serviceName), - timeout=Cuebot.Timeout).service_override - return serviceOverride + return self.stub.GetServiceOverride( + show_pb2.ShowGetServiceOverrideRequest(show=self.data, name=serviceName), + timeout=Cuebot.Timeout).service_override def getServiceOverrides(self): """Returns a list of service overrides on the show. - :rtype: list - :return: a list of service override objects + :rtype: list + :return: list of service overrides on the show """ serviceOverrideSeq = self.stub.GetServiceOverrides( show_pb2.ShowGetServiceOverridesRequest(show=self.data), @@ -102,22 +98,25 @@ def getServiceOverrides(self): return serviceOverrideSeq.service_overrides def getSubscriptions(self): - """Returns a list of all subscriptions. + """Returns a list of all subscriptions the show has. - :rtype: list - :return: A list of subscription objects + :rtype: list + :return: list of the show's subscriptions """ - response = self.stub.GetSubscriptions(show_pb2.ShowGetSubscriptionRequest( - show=self.data), - timeout=Cuebot.Timeout) + response = self.stub.GetSubscriptions( + show_pb2.ShowGetSubscriptionRequest(show=self.data), timeout=Cuebot.Timeout) subscriptionSeq = response.subscriptions - return [opencue.wrappers.subscription.Subscription(subs) for subs in subscriptionSeq.subscriptions] + return [opencue.wrappers.subscription.Subscription(subs) + for subs in subscriptionSeq.subscriptions] - def findSubscription(self, name): + @staticmethod + def findSubscription(name): """Returns the matching subscription. - :rtype: opencue.wrappers.subscription.Subscription - :return: The matching subscription + :type name: str + :param name: name of subscription to find + :rtype: opencue.wrappers.subscription.Subscription + :return: the matching subscription """ subscriptions = opencue.wrappers.subscription.Subscription() return subscriptions.find(name) @@ -125,31 +124,29 @@ def findSubscription(self, name): def getFilters(self): """Returns the job filters for this show. - :rtype: list - :return: List of Filter wrapper objects for this show. + :rtype: list + :return: list of filters for this show """ - response = self.stub.GetFilters(show_pb2.ShowGetFiltersRequest( - show=self.data), - timeout=Cuebot.Timeout) + response = self.stub.GetFilters( + show_pb2.ShowGetFiltersRequest(show=self.data), timeout=Cuebot.Timeout) filterSeq = response.filters return [opencue.wrappers.filter.Filter(filter) for filter in filterSeq.filters] def setActive(self, value): - """Set the active state of this show to value. + """Sets whether this show is active. - :type value: bool - :param value: boolean value to set active state to + :type value: bool + :param value: whether the show is active """ self.stub.SetActive(show_pb2.ShowSetActiveRequest(show=self.data, value=value), timeout=Cuebot.Timeout) def setDefaultMaxCores(self, maxcores): - """Sets the default maximum number of cores - that new jobs are launched with. + """Sets the default maximum number of cores that new jobs are launched with. - :type: float - :param: value to set maxCores to - :rtype: show_pb2.ShowSetDefaultMaxCoresResponse + :type maxcores: float + :param maxcores: new maximum number of cores for new jobs + :rtype: show_pb2.ShowSetDefaultMaxCoresResponse :return: response is empty """ response = self.stub.SetDefaultMaxCores(show_pb2.ShowSetDefaultMaxCoresRequest( @@ -158,12 +155,11 @@ def setDefaultMaxCores(self, maxcores): return response def setDefaultMinCores(self, mincores): - """Sets the default minimum number of cores - all new jobs are launched with. + """Sets the default minimum number of cores new jobs are launched with. - :type: float - :param: value to set minCores to - :rtype: show_pb2.ShowSetDefaultMinCoresResponse + :type mincores: float + :param mincores: new minimum number of cores for new jobs + :rtype: show_pb2.ShowSetDefaultMinCoresResponse :return: response is empty """ response = self.stub.SetDefaultMinCores(show_pb2.ShowSetDefaultMinCoresRequest( @@ -172,34 +168,34 @@ def setDefaultMinCores(self, mincores): return response def findFilter(self, name): - """Find the filter by name. + """Finds a filter by name. - :type: string - :param: name of filter to find - :rtype: opencue.wrappers.filter.Filter - :return: filter wrapper of found filter + :type name: string + :param name: name of filter to find + :rtype: opencue.wrappers.filter.Filter + :return: matching filter """ response = self.stub.FindFilter(show_pb2.ShowFindFilterRequest( show=self.data, name=name), timeout=Cuebot.Timeout) return opencue.wrappers.filter.Filter(response.filter) def createFilter(self, name): - """Create a filter on the show. + """Creates a filter on the show. - :type: string - :param: Name of the filter to create - :rtype: show_pb2.ShowCreateFilterResponse - :return: response is empty + :type name: str + :param name: name of the filter to create + :rtype: opencue.wrappers.filter.Filter + :return: the new filter object """ response = self.stub.CreateFilter(show_pb2.ShowCreateFilterRequest( show=self.data, name=name), timeout=Cuebot.Timeout) return opencue.wrappers.filter.Filter(response.filter) def getGroups(self): - """Get the groups for this show. + """Gets the groups for the show. - :rtype: list - :return: list of group wrappers for this show + :rtype: list + :return: list of groups for this show """ response = self.stub.GetGroups(show_pb2.ShowGetGroupsRequest( show=self.data), @@ -208,10 +204,10 @@ def getGroups(self): return [opencue.wrappers.group.Group(grp) for grp in groupSeq.groups] def getJobWhiteboard(self): - """Get the whiteboard for the show. + """Gets the whiteboard for the show. - :rtype: NestedGroup - :return: gRPC NestedGroup whiteboard for the show + :rtype: job_pb2.NestedGroup + :return: NestedGroup whiteboard for the show """ response = self.stub.GetJobWhiteboard(show_pb2.ShowGetJobWhiteboardRequest( show=self.data), @@ -219,10 +215,10 @@ def getJobWhiteboard(self): return response.whiteboard def getRootGroup(self): - """Get the root group for the show. + """Gets the root group for the show. - :rtype: opencue.wrappers.group.Group - :return: Group wrapper of the root group + :rtype: opencue.wrappers.group.Group + :return: the root group """ response = self.stub.GetRootGroup(show_pb2.ShowGetRootGroupRequest( show=self.data), @@ -230,12 +226,12 @@ def getRootGroup(self): return opencue.wrappers.group.Group(response.group) def enableBooking(self, value): - """Enable booking on the show. + """Enables or disables booking on the show. - :type: Boolean - :param: Whether or not to enable booking - :rtype: show_pb2.ShowEnableBookingResponse - :return: Response is empty + :type value: bool + :param value: whether to enable booking + :rtype: show_pb2.ShowEnableBookingResponse + :return: response is empty """ response = self.stub.EnableBooking(show_pb2.ShowEnableBookingRequest( show=self.data, @@ -244,12 +240,12 @@ def enableBooking(self, value): return response def enableDispatching(self, value): - """Enable dispatching on the show. + """Enables or disables dispatching on the show. - :type: Boolean - :param: Whether or not to enable booking - :rtype: show_pb2.ShowEnableDispatchingResponse - :return: Response is empty + :type value: bool + :param value: whether to enable booking + :rtype: show_pb2.ShowEnableDispatchingResponse + :return: response is empty """ response = self.stub.EnableDispatching(show_pb2.ShowEnableDispatchingRequest( show=self.data, @@ -258,39 +254,39 @@ def enableDispatching(self, value): return response def id(self): - """Returns the id of the show. + """Returns the show id. - :rtype: str - :return: Frame uuid + :rtype: str + :return: id of the show """ return self.data.id def name(self): - """Returns the name of the show. + """Returns the show name. - :rtype: str - :return: Show name + :rtype: str + :return: name of the show """ return self.data.name def pendingJobs(self): - """Total number of pending jobs. + """Returns the total number of pending jobs on the show. - :rtype: int - :return: the total number of pending jobs + :rtype: int + :return: total number of pending jobs """ return self.data.show_stats.pending_jobs def pendingFrames(self): - """Total number of running frames currently in the queue. + """Returns the total number of running frames currently in the queue. - :rtype: int + :rtype: int :return: the total number of pending frames """ return self.data.show_stats.pending_frames def runningFrames(self): - """Total number of running frames currently in the queue. + """Returns the total number of running frames currently in the queue. :rtype: int :return: the total number of running frames @@ -298,17 +294,17 @@ def runningFrames(self): return self.data.show_stats.running_frames def deadFrames(self): - """Total number of dead frames currently in the queue. + """Returns the total number of dead frames currently in the queue. - :rtype: int + :rtype: int :return: the total number dead frames """ return self.data.show_stats.dead_frames def reservedCores(self): - """Total number of reserved cores by all frames. + """Returns the total number of reserved cores by all frames. - :rtype: float + :rtype: float :return: the total number of reserved cores """ return self.data.show_stats.reserved_cores @@ -316,23 +312,23 @@ def reservedCores(self): def defaultMinProcs(self): """Returns the default minProcs that new jobs are set to. - :rtype: int - :return: Default minProcs value for new jobs + :rtype: int + :return: default minProcs value for new jobs """ return self.data.default_min_procs def defaultMaxProcs(self): """Returns the default maxProcs that new jobs are set to. - :rtype: int - :return: Default maxProcs value for new jobs + :rtype: int + :return: default maxProcs value for new jobs """ return self.data.default_max_procs - + def totalJobsCreated(self): """A running counter of jobs launched. - :rtype: int + :rtype: int :return: total number of jobs created """ return self.data.show_stats.created_job_count @@ -340,9 +336,7 @@ def totalJobsCreated(self): def totalFramesCreated(self): """A running counter of frames launched. - :rtype: int + :rtype: int :return: total number of frames created """ return self.data.show_stats.created_frame_count - - diff --git a/pycue/opencue/wrappers/subscription.py b/pycue/opencue/wrappers/subscription.py index 25aa6c23e..057067e3c 100644 --- a/pycue/opencue/wrappers/subscription.py +++ b/pycue/opencue/wrappers/subscription.py @@ -12,50 +12,73 @@ # See the License for the specific language governing permissions and # limitations under the License. - -""" -Project: opencue Library - -Module: subscription.py - opencue Library implementation of a subscription - -""" +"""Module for classes related to subscriptions.""" from opencue.compiled_proto import subscription_pb2 from opencue.cuebot import Cuebot class Subscription(object): - """This class contains the grpc implementation related to a Subscription.""" + """This class contains the grpc implementation related to a Subscription. + + A subscription associates a show with an allocation. It allows the show to utilize resources + within that allocation and determines the amount of resources the show is allowed to use at + any given time.""" def __init__(self, subscription=None): self.data = subscription self.stub = Cuebot.getStub('subscription') def find(self, name): + """Returns a subscription by name. + + :type name: str + :param name: name of subscription to find + :rtype: opencue.wrappers.subscription.Subscription + :return: the named subscription + """ response = self.stub.Find( subscription_pb2.SubscriptionFindRequest(name=name), timeout=Cuebot.Timeout) return Subscription(response.subscription) - def get(self, id): + def get(self, subscription_id): + """Returns a subscription by id. + + :type subscription_id: str + :param subscription_id: id of subscription to get + :rtype: opencue.wrappers.subscription.Subscription + :return: the subscription of the id + """ response = self.stub.Get( - subscription_pb2.SubscriptionGetRequest(id=id), + subscription_pb2.SubscriptionGetRequest(id=subscription_id), timeout=Cuebot.Timeout) return Subscription(response.subscription) def setSize(self, size): + """Sets subscription size; the number of cores the show is allowed to use consistently. + + :type size: int + :param size: the new subscription size + """ assert (isinstance(size, int)), "size is not expected type: int" self.stub.SetSize( subscription_pb2.SubscriptionSetSizeRequest(subscription=self.data, new_size=size), timeout=Cuebot.Timeout) def setBurst(self, burst): + """Sets subscription burst size; the number of cores the show is allowed to burst to. + + :type burst: int + :param burst: the new burst size + """ assert (isinstance(burst, int)), "burst is not expected type: int" self.stub.SetBurst( subscription_pb2.SubscriptionSetBurstRequest(subscription=self.data, burst=burst), timeout=Cuebot.Timeout) def delete(self): + """Deletes a subscription.""" self.stub.Delete( subscription_pb2.SubscriptionDeleteRequest(subscription=self.data), timeout=Cuebot.Timeout) @@ -64,48 +87,54 @@ def id(self): """Returns the id of the subscription. :rtype: str - :return: Frame uuid""" + :return: id of the subscription + """ return self.data.id def name(self): """Returns the name of the subscription. :rtype: str - :return: Subscription name""" + :return: name of the subscription + """ return self.data.name def size(self): """Returns the subscription size. :rtype: int - :return: Subscription size""" + :return: subscription size + """ return self.data.size def burst(self): - """Returns the subscription burst. + """Returns the subscription burst size. :rtype: int - :return: Allowed burst""" + :return: subscription burst size + """ return self.data.burst def reservedCores(self): - """Returns the current number reserved in this subscription. + """Returns the current number of cores reserved in this subscription. :rtype: float - :return: Total running cores""" + :return: reserved cores + """ return self.data.reserved_cores def show(self): - """Returns the show that this subscription is for. + """Returns the name of the show that this subscription is for. :rtype: str - :return: The show that this subscription is for""" + :return: the name of the show that this subscription is for + """ return self.data.show_name def allocation(self): - """Returns the allocation that this subscription is subscribed to. + """Returns the allocation that this subscription is for. :rtype: str - :return: The allocation subscribed to""" + :return: the allocation subscribed to + """ return self.data.allocation_name - diff --git a/pycue/opencue/wrappers/task.py b/pycue/opencue/wrappers/task.py index c2246f4bc..356e942f3 100644 --- a/pycue/opencue/wrappers/task.py +++ b/pycue/opencue/wrappers/task.py @@ -12,13 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - -""" -Project: opencue Library - -Module: task.py - opencue Library implementation of a task - -""" +"""Module for classes related to tasks.""" from opencue.compiled_proto import task_pb2 from opencue.cuebot import Cuebot @@ -32,18 +26,23 @@ def __init__(self, task=None): self.stub = Cuebot.getStub('task') def id(self): - """Returns the task's unique id""" + """Returns the unique id of the task. + + :rtype: str + :return: task id + """ return self.data.id def setMinCores(self, minCores): """Sets the minimum amount of cores for the task. :type minCores: int - :param minCores: the minimum number of cores the task needs""" + :param minCores: the minimum number of cores the task needs + """ self.stub.SetMinCores( task_pb2.TaskSetMinCoresRequest(task=self.data, new_min_cores=minCores), timeout=Cuebot.Timeout) def delete(self): - """Deletes this task""" + """Deletes the task.""" self.stub.Delete(task_pb2.TaskDeleteRequest(task=self.data), timeout=Cuebot.Timeout) diff --git a/pycue/opencue/wrappers/util.py b/pycue/opencue/wrappers/util.py index 20c8068a9..aa3f164d8 100644 --- a/pycue/opencue/wrappers/util.py +++ b/pycue/opencue/wrappers/util.py @@ -12,43 +12,55 @@ # See the License for the specific language governing permissions and # limitations under the License. - -""" -Project: opencue Library - -Module: util.py - opencue Library utility - -""" +"""Utility methods used by the wrapper classes.""" import time +# pylint: disable=redefined-builtin def format_time(epoch, format="%m/%d %H:%M", default="--/-- --:--"): - """Formats time using time formatting standards - see: https://docs.python.org/3/library/time.html""" + """Formats time using time formatting standards. + + See: https://docs.python.org/3/library/time.html + + :type epoch: int + :param epoch: time as an epoch + :type format: str + :param format: desired format of output string + :type default: str + :param default: the output if the given time is empty + :rtype: str + :return: string-formatted version of the given time + """ if not epoch: return default return time.strftime(format, time.localtime(epoch)) def dateToMMDDHHMM(sec): - """Returns date in the format %m/%d %H:%M + """Returns a time in the format `%m/%d %H:%M`. + :type sec: int + :param sec: time as an epoch :rtype: str - :return: Date in the format %m/%d %H:%M""" + :return: time in the format %m/%d %H:%M + """ if sec == 0: return "--/-- --:--" return time.strftime("%m/%d %H:%M", time.localtime(sec)) def __splitTime(sec): - """Returns time in the format H:MM:SS + """Splits a timestamp into hour, minute, and second components. - :rtype: str - :return: Time in the format H:MM:SS""" - min, sec = divmod(sec, 60) - hour, min = divmod(min, 60) - return (hour, min, sec) + :type sec: int + :param sec: timestamp as an epoch + :rtype: tuple + :return: (hour, min, sec) + """ + minute, sec = divmod(sec, 60) + hour, minute = divmod(minute, 60) + return hour, minute, sec def secondsToHHMMSS(sec): @@ -76,14 +88,15 @@ def secondsToHHHMM(sec): def secondsDiffToHMMSS(secA, secB): - """Returns time difference of arguements in the format H:MM:SS + """Returns time difference of arguments in the format H:MM:SS :type secA: int or float - :param secA: Seconds. 0 will be replaced with current time + :param secA: seconds. 0 will be replaced with current time :type secB: int or float - :param secB: Seconds. 0 will be replaced with current time + :param secB: seconds. 0 will be replaced with current time :rtype: str - :return: Time difference of arguments in the format H:MM:SS""" + :return: Time difference of arguments in the format H:MM:SS + """ if secA == 0: secA = time.time() if secB == 0: @@ -92,6 +105,13 @@ def secondsDiffToHMMSS(secA, secB): def convert_mem(kmem, unit=None): + """Returns an amount of memory in a human-readable string. + + :type kmem: int + :param kmem: amount of memory in kB + :rtype: str + :return: same amount of memory formatted into a human-readable string + """ k = 1024 if unit == 'K' or (unit is None and kmem < k): return '%dK' % kmem @@ -99,3 +119,4 @@ def convert_mem(kmem, unit=None): return '%dM' % (kmem / k) if unit == 'G' or (unit is None and kmem < pow(k, 3)): return '%.01fG' % (float(kmem) / pow(k, 2)) + return str(kmem) diff --git a/pycue/tests/api_test.py b/pycue/tests/api_test.py index 20c130e1d..0d05246c4 100644 --- a/pycue/tests/api_test.py +++ b/pycue/tests/api_test.py @@ -14,14 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for `opencue.api`.""" from __future__ import print_function from __future__ import division from __future__ import absolute_import -import mock import unittest -import opencue +import mock + +import opencue.api from opencue.compiled_proto import cue_pb2 from opencue.compiled_proto import depend_pb2 from opencue.compiled_proto import facility_pb2 @@ -639,12 +641,12 @@ def testFindFilter(self, getStubMock): filter=filter_pb2.Filter(name=filterName)) getStubMock.return_value = stubMock - filter = opencue.api.findFilter(TEST_SHOW_NAME, filterName) + filterReturned = opencue.api.findFilter(TEST_SHOW_NAME, filterName) stubMock.FindFilter.assert_called_with( filter_pb2.FilterFindFilterRequest(show=TEST_SHOW_NAME, name=filterName), timeout=mock.ANY) - self.assertEqual(filterName, filter.name()) + self.assertEqual(filterName, filterReturned.name()) class AllocTests(unittest.TestCase): @@ -668,7 +670,8 @@ def testCreateAlloc(self, getStubMock): def testGetAllocs(self, getStubMock): stubMock = mock.Mock() stubMock.GetAll.return_value = facility_pb2.AllocGetAllResponse( - allocations=facility_pb2.AllocationSeq(allocations=[facility_pb2.Allocation(name=TEST_ALLOC_NAME)])) + allocations=facility_pb2.AllocationSeq( + allocations=[facility_pb2.Allocation(name=TEST_ALLOC_NAME)])) getStubMock.return_value = stubMock allocs = opencue.api.getAllocations() @@ -775,17 +778,17 @@ def testGetProcs(self, getStubMock): self.assertEqual([TEST_PROC_NAME], [proc.name() for proc in procs]) -class Limittests(unittest.TestCase): - +class LimitTests(unittest.TestCase): + @mock.patch('opencue.cuebot.Cuebot.getStub') def testCreateLimit(self, getStubMock): stubMock = mock.Mock() stubMock.Create.return_value = limit_pb2.LimitCreateResponse() getStubMock.return_value = stubMock - + testLimitValue = 42 opencue.api.createLimit(TEST_LIMIT_NAME, testLimitValue) - + stubMock.Create.assert_called_with( limit_pb2.LimitCreateRequest(name=TEST_LIMIT_NAME, max_value=testLimitValue), timeout=mock.ANY) @@ -796,9 +799,9 @@ def testGetLimits(self, getStubMock): stubMock.GetAll.return_value = limit_pb2.LimitGetAllResponse( limits=[limit_pb2.Limit(name=TEST_LIMIT_NAME)]) getStubMock.return_value = stubMock - + limits = opencue.api.getLimits() - + stubMock.GetAll.assert_called_with(limit_pb2.LimitGetAllRequest(), timeout=mock.ANY) self.assertEqual(len(limits), 1) self.assertEqual(limits[0].name(), TEST_LIMIT_NAME) diff --git a/pycue/tests/file_sequence.py b/pycue/tests/file_sequence.py index 070bb5a4a..f03438f9c 100644 --- a/pycue/tests/file_sequence.py +++ b/pycue/tests/file_sequence.py @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for `FileSequence`.""" + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -155,7 +157,9 @@ class FrameSetTests(unittest.TestCase): def testMultipleSegments(self): result = FrameSet('57,1-3,4-2,12-15x2,76-70x-3,5-12y3,1-7:5') - self.assertEqual([57, 1, 2, 3, 4, 3, 2, 12, 14, 76, 73, 70, 6, 7, 9, 10, 12, 1, 6, 2, 4, 3, 5, 7], result.getAll()) + self.assertEqual( + [57, 1, 2, 3, 4, 3, 2, 12, 14, 76, 73, 70, 6, 7, 9, 10, 12, 1, 6, 2, 4, 3, 5, 7], + result.getAll()) def testSize(self): result = FrameSet('1-7') diff --git a/pycue/tests/search_test.py b/pycue/tests/search_test.py index 161a1709e..13e73046f 100644 --- a/pycue/tests/search_test.py +++ b/pycue/tests/search_test.py @@ -14,16 +14,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for `opencue.search`.""" from __future__ import print_function from __future__ import division from __future__ import absolute_import -import mock import unittest -import opencue +import mock + from opencue.compiled_proto import job_pb2 from opencue.compiled_proto import host_pb2 +import opencue.search @mock.patch('opencue.cuebot.Cuebot.getStub') @@ -77,7 +79,7 @@ def testRaiseIfNotList(self, getStubMock): opencue.search.raiseIfNotList('user', 42) with self.assertRaises(TypeError): - opencue.search.raiseIfNotList('user', set(['iamnotalist'])) + opencue.search.raiseIfNotList('user', {'iamnotalist'}) self.assertIsNone(opencue.search.raiseIfNotList('user', ['iamnotalist'])) diff --git a/pycue/tests/util_test.py b/pycue/tests/util_test.py index 667f9bc6f..b76641851 100644 --- a/pycue/tests/util_test.py +++ b/pycue/tests/util_test.py @@ -14,18 +14,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for `opencue.util`.""" from __future__ import print_function from __future__ import division from __future__ import absolute_import from builtins import range -import grpc -import mock import unittest import uuid -import opencue +import grpc +import mock + from opencue.compiled_proto import job_pb2 +import opencue.exception from opencue.wrappers.job import Job @@ -96,15 +98,16 @@ def testUnknownExceptionParser(self): @mock.patch('opencue.cuebot.Cuebot.getStub') def testProxyUniqueId(self, getStubMock): """convert a string and class name to proxy""" - id = 'A0000000-0000-0000-0000-000000000000' + objectId = 'A0000000-0000-0000-0000-000000000000' stubMock = mock.Mock() - stubMock.GetGroup.return_value = job_pb2.GroupGetGroupResponse(group=job_pb2.Group(id=id)) + stubMock.GetGroup.return_value = job_pb2.GroupGetGroupResponse( + group=job_pb2.Group(id=objectId)) getStubMock.return_value = stubMock - proxy = opencue.proxy(id, 'Group') + proxy = opencue.proxy(objectId, 'Group') - stubMock.GetGroup.assert_called_with(job_pb2.GroupGetGroupRequest(id=id)) - self.assertEqual(id, proxy.group.id) + stubMock.GetGroup.assert_called_with(job_pb2.GroupGetGroupRequest(id=objectId)) + self.assertEqual(objectId, proxy.group.id) @mock.patch('opencue.cuebot.Cuebot.getStub') def testProxyUniqueIdArray(self, getStubMock): diff --git a/pycue/tests/wrappers/allocation_test.py b/pycue/tests/wrappers/allocation_test.py index 203cfda6a..844423460 100644 --- a/pycue/tests/wrappers/allocation_test.py +++ b/pycue/tests/wrappers/allocation_test.py @@ -14,17 +14,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for `opencue.wrappers.allocation`.""" from __future__ import print_function from __future__ import division from __future__ import absolute_import -import mock import unittest -import opencue +import mock + from opencue.compiled_proto import facility_pb2 from opencue.compiled_proto import host_pb2 from opencue.compiled_proto import subscription_pb2 +import opencue.wrappers.allocation +import opencue.wrappers.host TEST_ALLOC_NAME = 'test_allocation' @@ -101,13 +104,13 @@ def testReparentHostIds(self, getStubMock): stubMock = mock.Mock() stubMock.ReparentHosts.return_value = facility_pb2.AllocReparentHostsResponse() getStubMock.return_value = stubMock - + alloc = opencue.wrappers.allocation.Allocation( facility_pb2.Allocation(name=TEST_ALLOC_NAME)) hostIds = [TEST_HOST_ID] alloc.reparentHostIds(hostIds) hosts = [host_pb2.Host(id=TEST_HOST_ID)] - + stubMock.ReparentHosts.assert_called_with( facility_pb2.AllocReparentHostsRequest( allocation=alloc.data, hosts=host_pb2.HostSeq(hosts=hosts)), timeout=mock.ANY) diff --git a/pycue/tests/wrappers/comment_test.py b/pycue/tests/wrappers/comment_test.py index 22de54ac7..68481b6c5 100644 --- a/pycue/tests/wrappers/comment_test.py +++ b/pycue/tests/wrappers/comment_test.py @@ -14,15 +14,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for `opencue.wrappers.comment`.""" from __future__ import print_function from __future__ import division from __future__ import absolute_import -import mock import unittest -import opencue +import mock + from opencue.compiled_proto import comment_pb2 +import opencue.wrappers.comment TEST_COMMENT_MESSAGE = "hi, I'm a message" diff --git a/pycue/tests/wrappers/deed_test.py b/pycue/tests/wrappers/deed_test.py index f07734851..8fd0833f6 100644 --- a/pycue/tests/wrappers/deed_test.py +++ b/pycue/tests/wrappers/deed_test.py @@ -14,15 +14,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for `opencue.wrappers.deed`.""" from __future__ import print_function from __future__ import division from __future__ import absolute_import -import mock import unittest -import opencue +import mock + from opencue.compiled_proto import host_pb2 +import opencue.wrappers.deed TEST_DEED_ID = 'ddd-dd-dddd' diff --git a/pycue/tests/wrappers/depend_test.py b/pycue/tests/wrappers/depend_test.py index f86140e0c..27c7c298d 100644 --- a/pycue/tests/wrappers/depend_test.py +++ b/pycue/tests/wrappers/depend_test.py @@ -14,15 +14,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for `opencue.wrappers.depend`.""" from __future__ import print_function from __future__ import division from __future__ import absolute_import -import mock import unittest -import opencue +import mock + from opencue.compiled_proto import depend_pb2 +import opencue.wrappers.depend TEST_DEPEND_ID = 'zzz-aaa-fff' diff --git a/pycue/tests/wrappers/filter_test.py b/pycue/tests/wrappers/filter_test.py index cfc95b611..4aac87877 100644 --- a/pycue/tests/wrappers/filter_test.py +++ b/pycue/tests/wrappers/filter_test.py @@ -14,16 +14,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for `opencue.wrappers.filter`.""" from __future__ import print_function from __future__ import division from __future__ import absolute_import -import mock import unittest -import opencue +import mock + from opencue.compiled_proto import filter_pb2 from opencue.compiled_proto import job_pb2 +import opencue.wrappers.filter +import opencue.wrappers.group +import opencue.wrappers.job TEST_ACTION_ID = 'aaa-aaaa-aaa' @@ -39,12 +43,11 @@ def testDelete(self, getStubMock): stubMock.Delete.return_value = filter_pb2.FilterDeleteResponse() getStubMock.return_value = stubMock - filter = opencue.wrappers.filter.Filter( - filter_pb2.Filter(name=TEST_FILTER_NAME)) - filter.delete() + filterToDelete = opencue.wrappers.filter.Filter(filter_pb2.Filter(name=TEST_FILTER_NAME)) + filterToDelete.delete() stubMock.Delete.assert_called_with( - filter_pb2.FilterDeleteRequest(filter=filter.data), timeout=mock.ANY) + filter_pb2.FilterDeleteRequest(filter=filterToDelete.data), timeout=mock.ANY) def testCreateMatcher(self, getStubMock): matcherId = 'mmm-mmmm-mmm' @@ -56,15 +59,13 @@ def testCreateMatcher(self, getStubMock): queryStr = 'john' subject = filter_pb2.USER matcherType = filter_pb2.IS_NOT - matcherData = opencue.wrappers.filter.MatcherData(subject=subject, - type=matcherType, - input=queryStr) - filter = opencue.wrappers.filter.Filter( - filter_pb2.Filter(name=TEST_FILTER_NAME)) - matcher = filter.createMatcher(subject, matcherType, queryStr) + matcherData = opencue.wrappers.filter.MatcherData( + subject=subject, type=matcherType, input=queryStr) + filterForMatcher = opencue.wrappers.filter.Filter(filter_pb2.Filter(name=TEST_FILTER_NAME)) + matcher = filterForMatcher.createMatcher(subject, matcherType, queryStr) stubMock.CreateMatcher.assert_called_with( - filter_pb2.FilterCreateMatcherRequest(filter=filter.data, data=matcherData), + filter_pb2.FilterCreateMatcherRequest(filter=filterForMatcher.data, data=matcherData), timeout=mock.ANY) self.assertEqual(matcher.id(), matcherId) @@ -77,9 +78,8 @@ def testCreateAction(self, getStubMock): actionType = filter_pb2.PAUSE_JOB value = 10 - filter = opencue.wrappers.filter.Filter( - filter_pb2.Filter(name=TEST_FILTER_NAME)) - action = filter.createAction(actionType, value) + filterForAction = opencue.wrappers.filter.Filter(filter_pb2.Filter(name=TEST_FILTER_NAME)) + action = filterForAction.createAction(actionType, value) actionData = opencue.wrappers.filter.ActionData( type=actionType, value_type=filter_pb2.INTEGER_TYPE, @@ -90,7 +90,7 @@ def testCreateAction(self, getStubMock): boolean_value=False) stubMock.CreateAction.assert_called_with( - filter_pb2.FilterCreateActionRequest(filter=filter.data, data=actionData), + filter_pb2.FilterCreateActionRequest(filter=filterForAction.data, data=actionData), timeout=mock.ANY) self.assertEqual(action.id(), actionId) @@ -101,12 +101,11 @@ def testGetActions(self, getStubMock): actions=filter_pb2.ActionSeq(actions=[filter_pb2.Action(id=actionId)])) getStubMock.return_value = stubMock - filter = opencue.wrappers.filter.Filter( - filter_pb2.Filter(name=TEST_FILTER_NAME)) - actions = filter.getActions() + filterForActions = opencue.wrappers.filter.Filter(filter_pb2.Filter(name=TEST_FILTER_NAME)) + actions = filterForActions.getActions() stubMock.GetActions.assert_called_with( - filter_pb2.FilterGetActionsRequest(filter=filter.data), timeout=mock.ANY) + filter_pb2.FilterGetActionsRequest(filter=filterForActions.data), timeout=mock.ANY) self.assertEqual(len(actions), 1) self.assertEqual(actions[0].id(), actionId) @@ -117,12 +116,11 @@ def testGetMatchers(self, getStubMock): matchers=filter_pb2.MatcherSeq(matchers=[filter_pb2.Matcher(id=matcherId)])) getStubMock.return_value = stubMock - filter = opencue.wrappers.filter.Filter( - filter_pb2.Filter(name=TEST_FILTER_NAME)) - matchers = filter.getMatchers() + filterForMatchers = opencue.wrappers.filter.Filter(filter_pb2.Filter(name=TEST_FILTER_NAME)) + matchers = filterForMatchers.getMatchers() stubMock.GetMatchers.assert_called_with( - filter_pb2.FilterGetMatchersRequest(filter=filter.data), timeout=mock.ANY) + filter_pb2.FilterGetMatchersRequest(filter=filterForMatchers.data), timeout=mock.ANY) self.assertEqual(len(matchers), 1) self.assertEqual(matchers[0].id(), matcherId) @@ -131,74 +129,56 @@ def testLowerOrder(self, getStubMock): stubMock.LowerOrder.return_value = filter_pb2.FilterLowerOrderResponse() getStubMock.return_value = stubMock - filter = opencue.wrappers.filter.Filter( - filter_pb2.Filter(name=TEST_FILTER_NAME)) - filter.lowerOrder() + filterInst = opencue.wrappers.filter.Filter(filter_pb2.Filter(name=TEST_FILTER_NAME)) + filterInst.lowerOrder() stubMock.LowerOrder.assert_called_with( - filter_pb2.FilterLowerOrderRequest(filter=filter.data), timeout=mock.ANY) + filter_pb2.FilterLowerOrderRequest(filter=filterInst.data), timeout=mock.ANY) def testRaiseOrder(self, getStubMock): stubMock = mock.Mock() stubMock.RaiseOrder.return_value = filter_pb2.FilterRaiseOrderResponse() getStubMock.return_value = stubMock - filter = opencue.wrappers.filter.Filter( - filter_pb2.Filter(name=TEST_FILTER_NAME)) - filter.raiseOrder() + filterInst = opencue.wrappers.filter.Filter(filter_pb2.Filter(name=TEST_FILTER_NAME)) + filterInst.raiseOrder() stubMock.RaiseOrder.assert_called_with( - filter_pb2.FilterRaiseOrderRequest(filter=filter.data), timeout=mock.ANY) + filter_pb2.FilterRaiseOrderRequest(filter=filterInst.data), timeout=mock.ANY) def testOrderFirst(self, getStubMock): stubMock = mock.Mock() stubMock.OrderFirst.return_value = filter_pb2.FilterOrderFirstResponse() getStubMock.return_value = stubMock - filter = opencue.wrappers.filter.Filter( - filter_pb2.Filter(name=TEST_FILTER_NAME)) - filter.orderFirst() + filterInst = opencue.wrappers.filter.Filter(filter_pb2.Filter(name=TEST_FILTER_NAME)) + filterInst.orderFirst() stubMock.OrderFirst.assert_called_with( - filter_pb2.FilterOrderFirstRequest(filter=filter.data), timeout=mock.ANY) - - def testOrderLast(self, getStubMock): - stubMock = mock.Mock() - stubMock.OrderLast.return_value = filter_pb2.FilterOrderLastResponse() - getStubMock.return_value = stubMock - - filter = opencue.wrappers.filter.Filter( - filter_pb2.Filter(name=TEST_FILTER_NAME)) - filter.orderLast() - - stubMock.OrderLast.assert_called_with( - filter_pb2.FilterOrderLastRequest(filter=filter.data), timeout=mock.ANY) + filter_pb2.FilterOrderFirstRequest(filter=filterInst.data), timeout=mock.ANY) def testOrderLast(self, getStubMock): stubMock = mock.Mock() stubMock.OrderLast.return_value = filter_pb2.FilterOrderLastResponse() getStubMock.return_value = stubMock - filter = opencue.wrappers.filter.Filter( - filter_pb2.Filter(name=TEST_FILTER_NAME)) - filter.orderLast() + filterInst = opencue.wrappers.filter.Filter(filter_pb2.Filter(name=TEST_FILTER_NAME)) + filterInst.orderLast() stubMock.OrderLast.assert_called_with( - filter_pb2.FilterOrderLastRequest(filter=filter.data), timeout=mock.ANY) + filter_pb2.FilterOrderLastRequest(filter=filterInst.data), timeout=mock.ANY) def testRunFilterOnGroup(self, getStubMock): stubMock = mock.Mock() stubMock.RunFilterOnGroup.return_value = filter_pb2.FilterRunFilterOnGroupResponse() getStubMock.return_value = stubMock - group = opencue.wrappers.group.Group( - job_pb2.Group(name='testGroup')) - filter = opencue.wrappers.filter.Filter( - filter_pb2.Filter(name=TEST_FILTER_NAME)) - filter.runFilterOnGroup(group) + group = opencue.wrappers.group.Group(job_pb2.Group(name='testGroup')) + filterToRun = opencue.wrappers.filter.Filter(filter_pb2.Filter(name=TEST_FILTER_NAME)) + filterToRun.runFilterOnGroup(group) stubMock.RunFilterOnGroup.assert_called_with( - filter_pb2.FilterRunFilterOnGroupRequest(filter=filter.data, group=group.data), + filter_pb2.FilterRunFilterOnGroupRequest(filter=filterToRun.data, group=group.data), timeout=mock.ANY) def testRunFilterOnJobs(self, getStubMock): @@ -208,12 +188,11 @@ def testRunFilterOnJobs(self, getStubMock): jobs = [opencue.wrappers.job.Job(job_pb2.Job(name='testJob'))] jobSeq = job_pb2.JobSeq(jobs=[job.data for job in jobs]) - filter = opencue.wrappers.filter.Filter( - filter_pb2.Filter(name=TEST_FILTER_NAME)) - filter.runFilterOnJobs(jobs) + filterToRun = opencue.wrappers.filter.Filter(filter_pb2.Filter(name=TEST_FILTER_NAME)) + filterToRun.runFilterOnJobs(jobs) stubMock.RunFilterOnJobs.assert_called_with( - filter_pb2.FilterRunFilterOnJobsRequest(filter=filter.data, jobs=jobSeq), + filter_pb2.FilterRunFilterOnJobsRequest(filter=filterToRun.data, jobs=jobSeq), timeout=mock.ANY) def testSetEnabled(self, getStubMock): @@ -222,12 +201,12 @@ def testSetEnabled(self, getStubMock): getStubMock.return_value = stubMock value = True - filter = opencue.wrappers.filter.Filter( - filter_pb2.Filter(name=TEST_FILTER_NAME)) - filter.setEnabled(value) + filterToEnable = opencue.wrappers.filter.Filter(filter_pb2.Filter(name=TEST_FILTER_NAME)) + filterToEnable.setEnabled(value) stubMock.SetEnabled.assert_called_with( - filter_pb2.FilterSetEnabledRequest(filter=filter.data, enabled=value), timeout=mock.ANY) + filter_pb2.FilterSetEnabledRequest( + filter=filterToEnable.data, enabled=value), timeout=mock.ANY) def testSetName(self, getStubMock): stubMock = mock.Mock() @@ -235,12 +214,11 @@ def testSetName(self, getStubMock): getStubMock.return_value = stubMock value = 'newname' - filter = opencue.wrappers.filter.Filter( - filter_pb2.Filter(name=TEST_FILTER_NAME)) - filter.setName(value) + filterToSet = opencue.wrappers.filter.Filter(filter_pb2.Filter(name=TEST_FILTER_NAME)) + filterToSet.setName(value) stubMock.SetName.assert_called_with( - filter_pb2.FilterSetNameRequest(filter=filter.data, name=value), timeout=mock.ANY) + filter_pb2.FilterSetNameRequest(filter=filterToSet.data, name=value), timeout=mock.ANY) def testSetType(self, getStubMock): stubMock = mock.Mock() @@ -248,12 +226,11 @@ def testSetType(self, getStubMock): getStubMock.return_value = stubMock value = filter_pb2.MATCH_ALL - filter = opencue.wrappers.filter.Filter( - filter_pb2.Filter(name=TEST_FILTER_NAME)) - filter.setType(value) + filterToSet = opencue.wrappers.filter.Filter(filter_pb2.Filter(name=TEST_FILTER_NAME)) + filterToSet.setType(value) stubMock.SetType.assert_called_with( - filter_pb2.FilterSetTypeRequest(filter=filter.data, type=value), timeout=mock.ANY) + filter_pb2.FilterSetTypeRequest(filter=filterToSet.data, type=value), timeout=mock.ANY) def testSetOrder(self, getStubMock): stubMock = mock.Mock() @@ -261,12 +238,12 @@ def testSetOrder(self, getStubMock): getStubMock.return_value = stubMock value = 2 - filter = opencue.wrappers.filter.Filter( - filter_pb2.Filter(name=TEST_FILTER_NAME)) - filter.setOrder(value) + filterToSet = opencue.wrappers.filter.Filter(filter_pb2.Filter(name=TEST_FILTER_NAME)) + filterToSet.setOrder(value) stubMock.SetOrder.assert_called_with( - filter_pb2.FilterSetOrderRequest(filter=filter.data, order=value), timeout=mock.ANY) + filter_pb2.FilterSetOrderRequest( + filter=filterToSet.data, order=value), timeout=mock.ANY) @mock.patch('opencue.cuebot.Cuebot.getStub') @@ -278,21 +255,19 @@ def testGetParentFilter(self, getStubMock): filter=filter_pb2.Filter(name=TEST_FILTER_NAME)) getStubMock.return_value = stubMock - action = opencue.wrappers.filter.Action( - filter_pb2.Action(id=TEST_ACTION_ID)) - filter = action.getParentFilter() + action = opencue.wrappers.filter.Action(filter_pb2.Action(id=TEST_ACTION_ID)) + filterReturned = action.getParentFilter() stubMock.GetParentFilter.assert_called_with( filter_pb2.ActionGetParentFilterRequest(action=action.data), timeout=mock.ANY) - self.assertEqual(filter.name(), TEST_FILTER_NAME) + self.assertEqual(filterReturned.name(), TEST_FILTER_NAME) def testDelete(self, getStubMock): stubMock = mock.Mock() stubMock.Delete.return_value = filter_pb2.ActionDeleteResponse() getStubMock.return_value = stubMock - action = opencue.wrappers.filter.Action( - filter_pb2.Action(id=TEST_ACTION_ID)) + action = opencue.wrappers.filter.Action(filter_pb2.Action(id=TEST_ACTION_ID)) action.delete() stubMock.Delete.assert_called_with( @@ -462,13 +437,12 @@ def testGetParentFilter(self, getStubMock): filter=filter_pb2.Filter(name=TEST_FILTER_NAME)) getStubMock.return_value = stubMock - matcher = opencue.wrappers.filter.Matcher( - filter_pb2.Matcher(id=TEST_MATCHER_ID)) - filter = matcher.getParentFilter() + matcher = opencue.wrappers.filter.Matcher(filter_pb2.Matcher(id=TEST_MATCHER_ID)) + filterReturns = matcher.getParentFilter() stubMock.GetParentFilter.assert_called_with( filter_pb2.MatcherGetParentFilterRequest(matcher=matcher.data), timeout=mock.ANY) - self.assertEqual(filter.name(), TEST_FILTER_NAME) + self.assertEqual(filterReturns.name(), TEST_FILTER_NAME) def testDelete(self, getStubMock): stubMock = mock.Mock() @@ -526,7 +500,7 @@ def testActionType(self): self.assertEqual(opencue.api.Action.ActionType.MOVE_JOB_TO_GROUP, opencue.compiled_proto.filter_pb2.MOVE_JOB_TO_GROUP) self.assertEqual(opencue.api.Action.ActionType.MOVE_JOB_TO_GROUP, 0) - + def testActionValueType(self): self.assertEqual(opencue.api.Action.ActionValueType.INTEGER_TYPE, opencue.compiled_proto.filter_pb2.INTEGER_TYPE) diff --git a/pycue/tests/wrappers/frame_test.py b/pycue/tests/wrappers/frame_test.py index 760c63085..8be9ff0fd 100644 --- a/pycue/tests/wrappers/frame_test.py +++ b/pycue/tests/wrappers/frame_test.py @@ -14,17 +14,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for `opencue.wrappers.frame`.""" from __future__ import print_function from __future__ import division from __future__ import absolute_import -import mock import time import unittest -import opencue +import mock + from opencue.compiled_proto import depend_pb2 from opencue.compiled_proto import job_pb2 +import opencue.wrappers.frame +import opencue.wrappers.job +import opencue.wrappers.layer TEST_FRAME_NAME = 'testFrame' diff --git a/pycue/tests/wrappers/group_test.py b/pycue/tests/wrappers/group_test.py index 8a05f266e..615e64afc 100644 --- a/pycue/tests/wrappers/group_test.py +++ b/pycue/tests/wrappers/group_test.py @@ -14,15 +14,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for `opencue.wrappers.group`.""" from __future__ import print_function from __future__ import division from __future__ import absolute_import -import mock import unittest -import opencue +import mock + from opencue.compiled_proto import job_pb2 +import opencue.wrappers.group +import opencue.wrappers.job TEST_GROUP_NAME = 'testGroup' @@ -215,7 +218,7 @@ def testReparentGroupIds(self, getStubMock): stubMock = mock.Mock() stubMock.ReparentGroups.return_value = job_pb2.GroupReparentGroupsResponse() getStubMock.return_value = stubMock - + groupId = 'ggg-gggg-ggg' groupIds = [groupId] groups = [opencue.wrappers.group.Group(job_pb2.Group(id='ggg-gggg-ggg'))] @@ -223,7 +226,7 @@ def testReparentGroupIds(self, getStubMock): group = opencue.wrappers.group.Group( job_pb2.Group(name=TEST_GROUP_NAME)) group.reparentGroupIds(groupIds) - + stubMock.ReparentGroups.assert_called_with( job_pb2.GroupReparentGroupsRequest(group=group.data, groups=groupSeq), timeout=mock.ANY) diff --git a/pycue/tests/wrappers/host_test.py b/pycue/tests/wrappers/host_test.py index bac92d892..74f406490 100644 --- a/pycue/tests/wrappers/host_test.py +++ b/pycue/tests/wrappers/host_test.py @@ -14,19 +14,22 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for `opencue.wrappers.host`.""" from __future__ import print_function from __future__ import division from __future__ import absolute_import import os -import mock import unittest -import opencue +import mock + from opencue.compiled_proto import comment_pb2 from opencue.compiled_proto import facility_pb2 from opencue.compiled_proto import host_pb2 from opencue.compiled_proto import renderPartition_pb2 +import opencue.wrappers.allocation +import opencue.wrappers.host TEST_HOST_NAME = 'testHost' diff --git a/pycue/tests/wrappers/job_test.py b/pycue/tests/wrappers/job_test.py index f8e00fccc..0ad2504de 100644 --- a/pycue/tests/wrappers/job_test.py +++ b/pycue/tests/wrappers/job_test.py @@ -14,18 +14,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for `opencue.wrappers.job`.""" from __future__ import print_function from __future__ import division from __future__ import absolute_import -import mock import os import unittest -import opencue +import mock + from opencue.compiled_proto import comment_pb2 from opencue.compiled_proto import depend_pb2 from opencue.compiled_proto import job_pb2 +import opencue.wrappers.frame +import opencue.wrappers.group +import opencue.wrappers.job +import opencue.wrappers.layer TEST_JOB_NAME = 'testJob' @@ -438,14 +443,14 @@ def testReorderFrames(self, getStubMock): stubMock.ReorderFrames.return_value = job_pb2.JobReorderFramesResponse() getStubMock.return_value = stubMock - range = '1-10' + frameRange = '1-10' order = job_pb2.REVERSE job = opencue.wrappers.job.Job( job_pb2.Job(name=TEST_JOB_NAME)) - job.reorderFrames(range, order) + job.reorderFrames(frameRange, order) stubMock.ReorderFrames.assert_called_with( - job_pb2.JobReorderFramesRequest(job=job.data, range=range, order=order), + job_pb2.JobReorderFramesRequest(job=job.data, range=frameRange, order=order), timeout=mock.ANY) def testStaggerFrames(self, getStubMock): @@ -453,14 +458,14 @@ def testStaggerFrames(self, getStubMock): stubMock.StaggerFrames.return_value = job_pb2.JobStaggerFramesResponse() getStubMock.return_value = stubMock - range = '1-10' + frameRange = '1-10' stagger = 5 job = opencue.wrappers.job.Job( job_pb2.Job(name=TEST_JOB_NAME)) - job.staggerFrames(range, stagger) + job.staggerFrames(frameRange, stagger) stubMock.StaggerFrames.assert_called_with( - job_pb2.JobStaggerFramesRequest(job=job.data, range=range, stagger=stagger), + job_pb2.JobStaggerFramesRequest(job=job.data, range=frameRange, stagger=stagger), timeout=mock.ANY) def testFrameStateTotals(self, getStubMock): @@ -491,7 +496,7 @@ def testFrameStateTotals(self, getStubMock): class NestedJobTests(unittest.TestCase): def testAsJob(self, getStubMock): - id = 'nnn-nnnn-nnn' + jobId = 'nnn-nnnn-nnn' state = job_pb2.FINISHED name = 'testNestedJob' shot = 'shotname' @@ -499,7 +504,7 @@ def testAsJob(self, getStubMock): user = 'username' group = 'groupname' facility = 'facilityname' - os = 'os' + jobOs = 'os' uid = 12345 priority = 14 minCores = 5 @@ -507,13 +512,13 @@ def testAsJob(self, getStubMock): logDir = '/path/to/logs' isPaused = False nestedJob = opencue.wrappers.job.NestedJob( - job_pb2.NestedJob(id=id, state=state, name=name, shot=shot, show=show, user=user, - group=group, facility=facility, os=os, uid=uid, priority=priority, + job_pb2.NestedJob(id=jobId, state=state, name=name, shot=shot, show=show, user=user, + group=group, facility=facility, os=jobOs, uid=uid, priority=priority, min_cores=minCores, max_cores=maxCores, log_dir=logDir, is_paused=isPaused)) job = opencue.wrappers.job.Job( - job_pb2.Job(id=id, state=state, name=name, shot=shot, show=show, user=user, - group=group, facility=facility, os=os, uid=uid, priority=priority, + job_pb2.Job(id=jobId, state=state, name=name, shot=shot, show=show, user=user, + group=group, facility=facility, os=jobOs, uid=uid, priority=priority, min_cores=minCores, max_cores=maxCores, log_dir=logDir, is_paused=isPaused)) diff --git a/pycue/tests/wrappers/layer_test.py b/pycue/tests/wrappers/layer_test.py index 34a699d8a..cf8fb0c33 100644 --- a/pycue/tests/wrappers/layer_test.py +++ b/pycue/tests/wrappers/layer_test.py @@ -14,16 +14,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for `opencue.wrappers.layer`""" from __future__ import print_function from __future__ import division from __future__ import absolute_import -import mock import unittest -import opencue +import mock + from opencue.compiled_proto import depend_pb2 from opencue.compiled_proto import job_pb2 +import opencue.wrappers.frame +import opencue.wrappers.layer +import opencue.wrappers.job TEST_LAYER_NAME = 'testLayer' @@ -32,6 +36,7 @@ @mock.patch('opencue.cuebot.Cuebot.getStub') class LayerTests(unittest.TestCase): + """Tests for `opencue.wrappers.layer.Layer`.""" def testKill(self, getStubMock): stubMock = mock.Mock() @@ -370,14 +375,13 @@ def testReorderFrames(self, getStubMock): stubMock.ReorderFrames.return_value = job_pb2.LayerReorderFramesResponse() getStubMock.return_value = stubMock - range = '1-10' + frameRange = '1-10' order = job_pb2.REVERSE - layer = opencue.wrappers.layer.Layer( - job_pb2.Layer(name=TEST_LAYER_NAME)) - layer.reorderFrames(range, order) + layer = opencue.wrappers.layer.Layer(job_pb2.Layer(name=TEST_LAYER_NAME)) + layer.reorderFrames(frameRange, order) stubMock.ReorderFrames.assert_called_with( - job_pb2.LayerReorderFramesRequest(layer=layer.data, range=range, order=order), + job_pb2.LayerReorderFramesRequest(layer=layer.data, range=frameRange, order=order), timeout=mock.ANY) def testStaggerFrames(self, getStubMock): @@ -385,14 +389,14 @@ def testStaggerFrames(self, getStubMock): stubMock.StaggerFrames.return_value = job_pb2.LayerStaggerFramesResponse() getStubMock.return_value = stubMock - range = '1-10' + frameRange = '1-10' stagger = 4 layer = opencue.wrappers.layer.Layer( job_pb2.Layer(name=TEST_LAYER_NAME)) - layer.staggerFrames(range, stagger) + layer.staggerFrames(frameRange, stagger) stubMock.StaggerFrames.assert_called_with( - job_pb2.LayerStaggerFramesRequest(layer=layer.data, range=range, stagger=stagger), + job_pb2.LayerStaggerFramesRequest(layer=layer.data, range=frameRange, stagger=stagger), timeout=mock.ANY) diff --git a/pycue/tests/wrappers/limit_test.py b/pycue/tests/wrappers/limit_test.py index cbc079a54..ac72c9579 100644 --- a/pycue/tests/wrappers/limit_test.py +++ b/pycue/tests/wrappers/limit_test.py @@ -14,15 +14,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for `opencue.wrappers.limit`.""" from __future__ import print_function from __future__ import division from __future__ import absolute_import -import mock import unittest -import opencue +import mock + from opencue.compiled_proto import limit_pb2 +import opencue.wrappers.limit TEST_LIMIT_ID = 'lll-llll-lll' @@ -32,29 +34,29 @@ @mock.patch('opencue.cuebot.Cuebot.getStub') class LimitTests(unittest.TestCase): - + def testCreate(self, getStubMock): stubMock = mock.Mock() stubMock.Create.return_value = limit_pb2.LimitCreateResponse() getStubMock.return_value = stubMock - + limit = opencue.wrappers.limit.Limit( limit_pb2.Limit(name=TEST_LIMIT_NAME, max_value=TEST_LIMIT_MAX_VALUE)) limit.create() - + stubMock.Create.assert_called_with( limit_pb2.LimitCreateRequest(name=TEST_LIMIT_NAME, max_value=TEST_LIMIT_MAX_VALUE), timeout=mock.ANY) - + def testDelete(self, getStubMock): stubMock = mock.Mock() stubMock.Delete.return_value = limit_pb2.LimitDeleteResponse() getStubMock.return_value = stubMock - + limit = opencue.wrappers.limit.Limit( limit_pb2.Limit(name=TEST_LIMIT_NAME, max_value=TEST_LIMIT_MAX_VALUE)) limit.delete() - + stubMock.Delete.assert_called_with( limit_pb2.LimitDeleteRequest(name=TEST_LIMIT_NAME), timeout=mock.ANY) @@ -63,10 +65,10 @@ def testFind(self, getStubMock): stubMock.Find.return_value = limit_pb2.LimitFindResponse( limit=limit_pb2.Limit(name=TEST_LIMIT_NAME, max_value=TEST_LIMIT_MAX_VALUE)) getStubMock.return_value = stubMock - + limit = opencue.wrappers.limit.Limit() responseLimit = limit.find(TEST_LIMIT_NAME) - + stubMock.Find.assert_called_with( limit_pb2.LimitFindRequest(name=TEST_LIMIT_NAME), timeout=mock.ANY) self.assertEqual(responseLimit.name(), TEST_LIMIT_NAME) @@ -77,10 +79,10 @@ def testGet(self, getStubMock): stubMock.Get.return_value = limit_pb2.LimitGetResponse( limit=limit_pb2.Limit(name=TEST_LIMIT_NAME, max_value=TEST_LIMIT_MAX_VALUE)) getStubMock.return_value = stubMock - + limit = opencue.wrappers.limit.Limit() responseLimit = limit.get(TEST_LIMIT_ID) - + stubMock.Get.assert_called_with( limit_pb2.LimitGetRequest(id=TEST_LIMIT_ID), timeout=mock.ANY) self.assertEqual(responseLimit.name(), TEST_LIMIT_NAME) @@ -91,11 +93,11 @@ def testRename(self, getStubMock): stubMock = mock.Mock() stubMock.Rename.return_value = limit_pb2.LimitRenameResponse() getStubMock.return_value = stubMock - + limit = opencue.wrappers.limit.Limit( limit_pb2.Limit(name=TEST_LIMIT_NAME, max_value=TEST_LIMIT_MAX_VALUE)) limit.rename(test_new_name) - + stubMock.Rename.assert_called_with( limit_pb2.LimitRenameRequest(old_name=TEST_LIMIT_NAME, new_name=test_new_name), timeout=mock.ANY) @@ -105,15 +107,15 @@ def testSetMaxValue(self, getStubMock): stubMock = mock.Mock() stubMock.SetMaxValue.return_value = limit_pb2.LimitSetMaxValueResponse() getStubMock.return_value = stubMock - + limit = opencue.wrappers.limit.Limit( limit_pb2.Limit(name=TEST_LIMIT_NAME, max_value=TEST_LIMIT_MAX_VALUE)) limit.setMaxValue(max_value) - + stubMock.SetMaxValue.assert_called_with( limit_pb2.LimitSetMaxValueRequest(name=TEST_LIMIT_NAME, max_value=max_value), timeout=mock.ANY) if __name__ == '__main__': - unittest.main() + unittest.main() diff --git a/pycue/tests/wrappers/owner_test.py b/pycue/tests/wrappers/owner_test.py index 3bcfc4303..6c31151ee 100644 --- a/pycue/tests/wrappers/owner_test.py +++ b/pycue/tests/wrappers/owner_test.py @@ -14,15 +14,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for `opencue.wrappers.owner`.""" from __future__ import print_function from __future__ import division from __future__ import absolute_import -import mock import unittest -import opencue +import mock + from opencue.compiled_proto import host_pb2 +import opencue.wrappers.owner TEST_DEED_ID = 'ddd-dd-dddd' @@ -32,7 +34,6 @@ TEST_SHOW_NAME = 'testShow' - @mock.patch('opencue.cuebot.Cuebot.getStub') class OwnerTests(unittest.TestCase): diff --git a/pycue/tests/wrappers/proc_test.py b/pycue/tests/wrappers/proc_test.py index 693ac4ab2..82434f15f 100644 --- a/pycue/tests/wrappers/proc_test.py +++ b/pycue/tests/wrappers/proc_test.py @@ -14,16 +14,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for `opencue.wrappers.proc`.""" from __future__ import print_function from __future__ import division from __future__ import absolute_import -import mock import unittest -import opencue +import mock + from opencue.compiled_proto import host_pb2 from opencue.compiled_proto import job_pb2 +import opencue.wrappers.proc TEST_HOST_NAME = 'testHost' diff --git a/pycue/tests/wrappers/service_test.py b/pycue/tests/wrappers/service_test.py index 5e4a20936..92b2e3b2c 100644 --- a/pycue/tests/wrappers/service_test.py +++ b/pycue/tests/wrappers/service_test.py @@ -14,15 +14,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for `opencue.wrappers.service`.""" from __future__ import print_function from __future__ import division from __future__ import absolute_import -import mock import unittest -import opencue +import mock + from opencue.compiled_proto import service_pb2 +import opencue.wrappers.service TEST_SERVICE_NAME = 'testService' diff --git a/pycue/tests/wrappers/show_test.py b/pycue/tests/wrappers/show_test.py index c338d4246..b7cc3dd9a 100644 --- a/pycue/tests/wrappers/show_test.py +++ b/pycue/tests/wrappers/show_test.py @@ -14,14 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for the opencue.wrappers.show module.""" from __future__ import print_function from __future__ import division from __future__ import absolute_import -import mock import unittest -import opencue +import mock + from opencue.compiled_proto import facility_pb2 from opencue.compiled_proto import filter_pb2 from opencue.compiled_proto import host_pb2 @@ -29,6 +30,8 @@ from opencue.compiled_proto import service_pb2 from opencue.compiled_proto import show_pb2 from opencue.compiled_proto import subscription_pb2 +import opencue.wrappers.allocation +import opencue.wrappers.show TEST_ALLOCATION_ID = 'aaa-zzz-fff' @@ -74,7 +77,8 @@ def testCreateSubscription(self, getStubMock): show = opencue.wrappers.show.Show(show_pb2.Show(name=TEST_SHOW_NAME)) allocation = opencue.wrappers.allocation.Allocation( facility_pb2.Allocation(id=TEST_ALLOCATION_ID)) - subscription = show.createSubscription(allocation, TEST_SUBSCRIPTION_SIZE, TEST_SUBSCRIPTION_BURST) + subscription = show.createSubscription( + allocation, TEST_SUBSCRIPTION_SIZE, TEST_SUBSCRIPTION_BURST) stubMock.CreateSubscription.assert_called_with( show_pb2.ShowCreateSubscriptionRequest(show=show.data, @@ -200,12 +204,12 @@ def testFindFilter(self, getStubMock): getStubMock.return_value = stubMock show = opencue.wrappers.show.Show(show_pb2.Show(name=TEST_SHOW_NAME)) - filter = show.findFilter(TEST_FILTER_NAME) + filter_found = show.findFilter(TEST_FILTER_NAME) stubMock.FindFilter.assert_called_with( show_pb2.ShowFindFilterRequest(show=show.data, name=TEST_FILTER_NAME), timeout=mock.ANY) - self.assertEqual(filter.name(), TEST_FILTER_NAME) + self.assertEqual(filter_found.name(), TEST_FILTER_NAME) def testCreateFilter(self, getStubMock): stubMock = mock.Mock() @@ -214,12 +218,12 @@ def testCreateFilter(self, getStubMock): getStubMock.return_value = stubMock show = opencue.wrappers.show.Show(show_pb2.Show(name=TEST_SHOW_NAME)) - filter = show.createFilter(TEST_FILTER_NAME) + filter_created = show.createFilter(TEST_FILTER_NAME) stubMock.CreateFilter.assert_called_with( show_pb2.ShowCreateFilterRequest(show=show.data, name=TEST_FILTER_NAME), timeout=mock.ANY) - self.assertEqual(filter.name(), TEST_FILTER_NAME) + self.assertEqual(filter_created.name(), TEST_FILTER_NAME) def testGetGroups(self, getStubMock): stubMock = mock.Mock() diff --git a/pycue/tests/wrappers/subscription_test.py b/pycue/tests/wrappers/subscription_test.py index c29a875ae..342258487 100644 --- a/pycue/tests/wrappers/subscription_test.py +++ b/pycue/tests/wrappers/subscription_test.py @@ -14,19 +14,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for `opencue.wrappers.subscription`.""" from __future__ import print_function from __future__ import division from __future__ import absolute_import -import mock import unittest -import opencue +import mock + from opencue.compiled_proto import subscription_pb2 +import opencue.wrappers.subscription + TEST_SUBSCRIPTION_ID = 'aaaa-aaa-aaaa' TEST_SUBSCRIPTION_NAME = 'testSubscription' + @mock.patch('opencue.cuebot.Cuebot.getStub') class SubscriptionTests(unittest.TestCase): diff --git a/pycue/tests/wrappers/task_test.py b/pycue/tests/wrappers/task_test.py index cf03578c9..bc1e0e146 100644 --- a/pycue/tests/wrappers/task_test.py +++ b/pycue/tests/wrappers/task_test.py @@ -14,15 +14,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for `opencue.wrappers.task`.""" from __future__ import print_function from __future__ import division from __future__ import absolute_import -import mock import unittest -import opencue +import mock + from opencue.compiled_proto import task_pb2 +import opencue.wrappers.task @mock.patch('opencue.cuebot.Cuebot.getStub') diff --git a/pycue/tests/wrappers/util_test.py b/pycue/tests/wrappers/util_test.py index bb6c8217f..35f26192b 100644 --- a/pycue/tests/wrappers/util_test.py +++ b/pycue/tests/wrappers/util_test.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for `opencue.wrappers.util`.""" from __future__ import print_function from __future__ import division @@ -25,6 +26,7 @@ import opencue.wrappers.util + TEST_SECONDS_A = 1557942905 TEST_SECONDS_B = 2000 TEST_SECONDS_C = 0 From a2a36426e0551bf0d32c2622bbb0ff1d32b81f0a Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Thu, 4 Feb 2021 00:56:19 -0800 Subject: [PATCH 047/277] Redirect DTD http request to the resource file (#850) * Redirect DTD http request to the resource file * Add JobSpec unittests * Remove Cuebot TCP 8080 port from Docker --- .../com/imageworks/spcue/service/JobSpec.java | 30 ++++++- .../src/main/resources/application.properties | 5 +- .../spcue/test/service/JobSpecTests.java | 85 +++++++++++++++++++ .../resources/conf/jobspec/jobspec_1_10.xml | 47 ++++++++++ .../conf/jobspec/jobspec_nonexistent_dtd.xml | 47 ++++++++++ sandbox/docker-compose.yml | 1 - 6 files changed, 209 insertions(+), 6 deletions(-) create mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/service/JobSpecTests.java create mode 100644 cuebot/src/test/resources/conf/jobspec/jobspec_1_10.xml create mode 100644 cuebot/src/test/resources/conf/jobspec/jobspec_nonexistent_dtd.xml diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java index d553456fd..3540c77a5 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java @@ -20,6 +20,8 @@ package com.imageworks.spcue.service; import java.io.File; +import java.io.IOException; +import java.io.InputStream; import java.io.StringReader; import java.util.ArrayList; import java.util.HashSet; @@ -36,6 +38,9 @@ import org.jdom.Element; import org.jdom.input.SAXBuilder; import org.springframework.dao.EmptyResultDataAccessException; +import org.xml.sax.EntityResolver; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; import com.imageworks.spcue.BuildableDependency; import com.imageworks.spcue.BuildableJob; @@ -102,6 +107,8 @@ public class JobSpec { public static final String DEFAULT_SERVICE = "default"; + public static final String SPCUE_DTD_URL = "http://localhost:8080/spcue/dtd/"; + private List jobs = new ArrayList(); private List depends = new ArrayList(); @@ -836,9 +843,30 @@ public void parse(File file) { handleDependsTags(); } + private class DTDRedirector implements EntityResolver { + public InputSource resolveEntity(String publicId, + String systemId) throws SAXException, IOException { + if (systemId.startsWith(SPCUE_DTD_URL)) { + // Redirect to resource file. + try { + String filename = systemId.substring(SPCUE_DTD_URL.length()); + InputStream dtd = getClass().getResourceAsStream("/public/dtd/" + filename); + return new InputSource(dtd); + } catch (Exception e) { + throw new SpecBuilderException("Failed to redirect DTD " + systemId + ", " + e); + } + } else { + // Use default resolver. + return null; + } + } + } + public void parse(String cjsl) { try { - doc = new SAXBuilder(true).build(new StringReader(cjsl)); + SAXBuilder builder = new SAXBuilder(true); + builder.setEntityResolver(new DTDRedirector()); + doc = builder.build(new StringReader(cjsl)); } catch (Exception e) { throw new SpecBuilderException("Failed to parse job spec XML, " + e); diff --git a/cuebot/src/main/resources/application.properties b/cuebot/src/main/resources/application.properties index 79aa81236..cbc51e2fd 100644 --- a/cuebot/src/main/resources/application.properties +++ b/cuebot/src/main/resources/application.properties @@ -1,5 +1,2 @@ -server.contextPath=/spcue -server.port=8080 - spring.main.allow-bean-definition-overriding=true -spring.mvc.static-path-pattern=/spcue/** +spring.main.web-application-type=none diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/JobSpecTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/JobSpecTests.java new file mode 100644 index 000000000..d2aa142d8 --- /dev/null +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/JobSpecTests.java @@ -0,0 +1,85 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.imageworks.spcue.test.service; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import javax.annotation.Resource; + +import org.junit.Test; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; +import org.springframework.test.context.support.AnnotationConfigContextLoader; + +import com.imageworks.spcue.SpecBuilderException; +import com.imageworks.spcue.config.TestAppConfig; +import com.imageworks.spcue.service.JobLauncher; +import com.imageworks.spcue.service.JobSpec; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + + +@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) +public class JobSpecTests extends AbstractTransactionalJUnit4SpringContextTests { + + @Resource + JobLauncher jobLauncher; + + private static String readJobSpec(String name) + { + String path = "src/test/resources/conf/jobspec/" + name; + byte[] encoded = null; + + try { + encoded = Files.readAllBytes(Paths.get(path)); + } catch (IOException e) { + fail("readJobSpec should succeed to read jobspec file"); + } + + return new String(encoded, StandardCharsets.UTF_8); + } + + @Test + public void testParseSuccess() { + String xml = readJobSpec("jobspec_1_10.xml"); + JobSpec spec = jobLauncher.parse(xml); + assertEquals(spec.getDoc().getDocType().getPublicID(), + "SPI Cue Specification Language"); + assertEquals(spec.getDoc().getDocType().getSystemID(), + "http://localhost:8080/spcue/dtd/cjsl-1.10.dtd"); + assertEquals(spec.getJobs().size(), 1); + assertEquals(spec.getJobs().get(0).detail.name, "testing-default-testuser_test"); + } + + @Test + public void testParseNonExistent() { + String xml = readJobSpec("jobspec_nonexistent_dtd.xml"); + try { + jobLauncher.parse(xml); + fail("Expected exception"); + } catch (SpecBuilderException e) { + assertEquals(e.getMessage(), + "Failed to parse job spec XML, java.net.MalformedURLException"); + } + } +} diff --git a/cuebot/src/test/resources/conf/jobspec/jobspec_1_10.xml b/cuebot/src/test/resources/conf/jobspec/jobspec_1_10.xml new file mode 100644 index 000000000..ce42deb16 --- /dev/null +++ b/cuebot/src/test/resources/conf/jobspec/jobspec_1_10.xml @@ -0,0 +1,47 @@ + + + + + + + + + local + testing + default + testuser + 9860 + + + False + 2 + False + + + + echo hello + 1 + 1 + + + shell + + + + + + diff --git a/cuebot/src/test/resources/conf/jobspec/jobspec_nonexistent_dtd.xml b/cuebot/src/test/resources/conf/jobspec/jobspec_nonexistent_dtd.xml new file mode 100644 index 000000000..870cec16d --- /dev/null +++ b/cuebot/src/test/resources/conf/jobspec/jobspec_nonexistent_dtd.xml @@ -0,0 +1,47 @@ + + + + + + + + + local + testing + default + testuser + 9860 + + + False + 2 + False + + + + echo hello + 1 + 1 + + + shell + + + + + + diff --git a/sandbox/docker-compose.yml b/sandbox/docker-compose.yml index f4fde39ec..33938c06f 100644 --- a/sandbox/docker-compose.yml +++ b/sandbox/docker-compose.yml @@ -34,7 +34,6 @@ services: - db ports: - "8443:8443" - - "8080:8080" depends_on: - db - flyway From d9c7fd9d04fa94d8bfe87513ce148e6c171c50cd Mon Sep 17 00:00:00 2001 From: Lars van der Bijl <285658+larsbijl@users.noreply.github.com> Date: Sun, 7 Feb 2021 12:25:19 +0000 Subject: [PATCH 048/277] Add priority to pyoutline cue backend. (#625) * Add priority to pyoutline cue backend. Support new parameter in xml spec. Set priority on job resource after insertion into jobs table Closes #493 * bump version * Use priority only from launcher * build the version number in the header Co-authored-by: Lars van der Bijl --- VERSION.in | 2 +- .../spcue/service/JobManagerService.java | 4 + .../com/imageworks/spcue/service/JobSpec.java | 5 + .../main/resources/public/dtd/cjsl-1.11.dtd | 95 +++++++++++++++++++ .../src/test/resources/conf/dtd/cjsl-1.11.dtd | 95 +++++++++++++++++++ pyoutline/outline/backend/cue.py | 3 +- pyoutline/outline/cuerun.py | 2 + 7 files changed, 204 insertions(+), 2 deletions(-) create mode 100644 cuebot/src/main/resources/public/dtd/cjsl-1.11.dtd create mode 100644 cuebot/src/test/resources/conf/dtd/cjsl-1.11.dtd diff --git a/VERSION.in b/VERSION.in index 5a2a5806d..eb49d7c7f 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.6 +0.7 diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java index e8f241c7a..68821ed64 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java @@ -277,6 +277,10 @@ public JobDetail createJob(BuildableJob buildableJob) { frameDao.insertFrames(layer, frames); } + // The priority of a job is set on it's resource entry. + // To update it we set the priority after it's been inserted. + jobDao.updatePriority(job, job.priority); + /* * Finally, run any filters on the job which may set the job's * priority. diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java index 3540c77a5..57ddcbfdf 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java @@ -307,6 +307,11 @@ private BuildableJob handleJobTag(Element jobTag) { job.maxRetries = FRAME_RETRIES_MIN; } } + + if (jobTag.getChildTextTrim("priority") != null) { + job.priority = Integer.valueOf(jobTag.getChildTextTrim("priority")); + } + handleLayerTags(buildableJob, jobTag); if (buildableJob.getBuildableLayers().size() > MAX_LAYERS) { diff --git a/cuebot/src/main/resources/public/dtd/cjsl-1.11.dtd b/cuebot/src/main/resources/public/dtd/cjsl-1.11.dtd new file mode 100644 index 000000000..b923d6a21 --- /dev/null +++ b/cuebot/src/main/resources/public/dtd/cjsl-1.11.dtd @@ -0,0 +1,95 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cuebot/src/test/resources/conf/dtd/cjsl-1.11.dtd b/cuebot/src/test/resources/conf/dtd/cjsl-1.11.dtd new file mode 100644 index 000000000..0b8d70c5b --- /dev/null +++ b/cuebot/src/test/resources/conf/dtd/cjsl-1.11.dtd @@ -0,0 +1,95 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pyoutline/outline/backend/cue.py b/pyoutline/outline/backend/cue.py index 83e57400d..6808ff7e2 100644 --- a/pyoutline/outline/backend/cue.py +++ b/pyoutline/outline/backend/cue.py @@ -246,6 +246,7 @@ def _serialize(launcher, use_pycuerun): j = Et.SubElement(root, "job", {"name": ol.get_name()}) sub_element(j, "paused", str(launcher.get("pause"))) + sub_element(j, "priority", str(launcher.get("priority"))) sub_element(j, "maxretries", str(launcher.get("maxretries"))) sub_element(j, "autoeat", str(launcher.get("autoeat"))) @@ -351,7 +352,7 @@ def _serialize(launcher, use_pycuerun): xml = [ '', '', + '"http://localhost:8080/spcue/dtd/cjsl-1.11.dtd">', Et.tostring(root).decode() ] diff --git a/pyoutline/outline/cuerun.py b/pyoutline/outline/cuerun.py index d1392dd4a..7f40364f8 100644 --- a/pyoutline/outline/cuerun.py +++ b/pyoutline/outline/cuerun.py @@ -119,6 +119,7 @@ class OutlineLauncher(object): def __init__(self, outline, **args): self.__outline = outline self.__flags = {"pause": False, + "priority": 1, "wait": False, "test": False, "server": False, @@ -355,6 +356,7 @@ def options_to_args(options): "basename": options.basename, "server": options.server, "pause": options.pause, + "priority": options.priority, "wait": options.wait, "test": options.test, "range": options.range, From 014bcefe2793da3c53a7e2a7796fc2c55bff0c35 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sun, 7 Feb 2021 08:43:15 -0800 Subject: [PATCH 049/277] Change virt type from INT to BIGINT (#902) * Change virt type from INT to BIGINT * Bump minor version number * Update VERSION.in Co-authored-by: Lars van der Bijl <285658+larsbijl@users.noreply.github.com> --- VERSION.in | 2 +- .../conf/ddl/postgres/migrations/V9__Change_virt_type.sql | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V9__Change_virt_type.sql diff --git a/VERSION.in b/VERSION.in index eb49d7c7f..ce609caf8 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.7 +0.8 \ No newline at end of file diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V9__Change_virt_type.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V9__Change_virt_type.sql new file mode 100644 index 000000000..dd88ffb20 --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V9__Change_virt_type.sql @@ -0,0 +1,4 @@ +-- Change virt type from INT to BIGINT. + +ALTER TABLE "proc" ALTER COLUMN "int_virt_used" TYPE BIGINT; +ALTER TABLE "proc" ALTER COLUMN "int_virt_max_used" TYPE BIGINT; From 67454c93c38e60a5e6d02ccdd1f2e3aed956d4b8 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sun, 7 Feb 2021 08:52:15 -0800 Subject: [PATCH 050/277] Validate strings (#907) * conformShotName * Unittest for shot * Add non-existent show test * conformShowName * Add show creation tests --- .../spcue/service/AdminManagerService.java | 3 ++ .../com/imageworks/spcue/service/JobSpec.java | 24 ++++++++-- .../spcue/test/service/AdminManagerTests.java | 26 ++++++++++ .../spcue/test/service/JobManagerTests.java | 17 +++++++ .../spcue/test/service/JobSpecTests.java | 13 +++++ .../conf/jobspec/jobspec_invalid_shot.xml | 47 +++++++++++++++++++ .../conf/jobspec/jobspec_nonexistent_show.xml | 41 ++++++++++++++++ 7 files changed, 166 insertions(+), 5 deletions(-) create mode 100644 cuebot/src/test/resources/conf/jobspec/jobspec_invalid_shot.xml create mode 100644 cuebot/src/test/resources/conf/jobspec/jobspec_nonexistent_show.xml diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java index 2889040d8..193de98a5 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java @@ -40,6 +40,7 @@ import com.imageworks.spcue.dao.LimitDao; import com.imageworks.spcue.dao.ShowDao; import com.imageworks.spcue.dao.SubscriptionDao; +import com.imageworks.spcue.service.JobSpec; @Transactional public class AdminManagerService implements AdminManager { @@ -71,6 +72,8 @@ public boolean showExists(String name) { public void createShow(ShowEntity show) { + show.name = JobSpec.conformShowName(show.name); + DepartmentInterface dept = getDefaultDepartment(); showDao.insertShow(show); diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java index 57ddcbfdf..1e498eaf4 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java @@ -157,11 +157,13 @@ public String conformJobName(String name) { return String.format("%s_%s", prefix, suffix).toLowerCase(); } - public String conformLayerName(String name) { + public static String conformName(String type, String name) { + + String lowerType = type.toLowerCase(); if (name.length() < 3) { throw new SpecBuilderException( - "The layer name must be at least 3 characters."); + "The " + lowerType + " name must be at least 3 characters."); } String newName = name; @@ -170,14 +172,26 @@ public String conformLayerName(String name) { Matcher matcher = NAME_PATTERN.matcher(newName); if (!matcher.matches()) { - throw new SpecBuilderException("The layer name: " + newName - + " is not in the proper format. Layer names must be " + throw new SpecBuilderException("The " + lowerType + " name: " + newName + + " is not in the proper format. " + type + " names must be " + "alpha numeric, no dashes or punctuation."); } return newName; } + public static String conformShowName(String name) { + return conformName("Show", name); + } + + public static String conformShotName(String name) { + return conformName("Shot", name); + } + + public static String conformLayerName(String name) { + return conformName("Layer", name); + } + public static final String FRAME_NAME_REGEX = "^([\\d]{4,6})-([\\w]+)$"; public static final Pattern FRAME_NAME_PATTERN = Pattern @@ -204,7 +218,7 @@ private void handleSpecTag() { } show = rootElement.getChildTextTrim("show"); - shot = rootElement.getChildTextTrim("shot"); + shot = conformShotName(rootElement.getChildTextTrim("shot")); user = rootElement.getChildTextTrim("user"); uid = Optional.ofNullable(rootElement.getChildTextTrim("uid")).map(Integer::parseInt); email = rootElement.getChildTextTrim("email"); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/AdminManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/AdminManagerTests.java index 6310d445f..8f1d24ffe 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/AdminManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/AdminManagerTests.java @@ -30,11 +30,16 @@ import com.imageworks.spcue.AllocationEntity; import com.imageworks.spcue.LimitInterface; +import com.imageworks.spcue.SpecBuilderException; import com.imageworks.spcue.ShowEntity; import com.imageworks.spcue.config.TestAppConfig; import com.imageworks.spcue.dao.FacilityDao; +import com.imageworks.spcue.dao.ShowDao; import com.imageworks.spcue.service.AdminManager; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + @Transactional @ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) public class AdminManagerTests extends AbstractTransactionalJUnit4SpringContextTests { @@ -45,6 +50,9 @@ public class AdminManagerTests extends AbstractTransactionalJUnit4SpringContextT @Resource FacilityDao facilityDao; + @Resource + ShowDao showDao; + private static final String TEST_ALLOC_NAME = "testAlloc"; @Test @@ -64,6 +72,24 @@ public void createShow() { ShowEntity show = new ShowEntity(); show.name = "testtest"; adminManager.createShow(show); + ShowEntity result = showDao.findShowDetail(show.name); + assertEquals(result.name, show.name); + } + + @Test + @Transactional + @Rollback(true) + public void createInvalidShow() { + ShowEntity show = new ShowEntity(); + show.name = "test/test"; + try { + adminManager.createShow(show); + fail("Expected exception"); + } catch (SpecBuilderException e) { + assertEquals(e.getMessage(), + "The show name: test/test is not in the proper format. " + + "Show names must be alpha numeric, no dashes or punctuation."); + } } @Test diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/JobManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/JobManagerTests.java index 323abc577..b2446fe20 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/JobManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/JobManagerTests.java @@ -34,6 +34,7 @@ import com.imageworks.spcue.DispatchFrame; import com.imageworks.spcue.DispatchHost; +import com.imageworks.spcue.EntityCreationError; import com.imageworks.spcue.FrameInterface; import com.imageworks.spcue.FrameStateTotals; import com.imageworks.spcue.JobDetail; @@ -70,6 +71,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; @Transactional @@ -280,6 +282,21 @@ public void testMisNamedJob2() { "pipe-dev.cue-testuser_blah_blah_v1"); } + @Test + @Transactional + @Rollback(true) + public void testNonExistentShow() { + JobSpec spec = jobLauncher.parse(new File("src/test/resources/conf/jobspec/jobspec_nonexistent_show.xml")); + try { + jobLauncher.launch(spec); + fail("Expected exception"); + } catch (EntityCreationError e) { + assertEquals(e.getMessage(), + "The nonexistentshow does not exist. Please contact administrator of your " + + "OpenCue deployment to have this show created."); + } + } + @Test @Transactional @Rollback(true) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/JobSpecTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/JobSpecTests.java index d2aa142d8..e68daa551 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/JobSpecTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/JobSpecTests.java @@ -82,4 +82,17 @@ public void testParseNonExistent() { "Failed to parse job spec XML, java.net.MalformedURLException"); } } + + @Test + public void testParseInvalidShot() { + String xml = readJobSpec("jobspec_invalid_shot.xml"); + try { + jobLauncher.parse(xml); + fail("Expected exception"); + } catch (SpecBuilderException e) { + assertEquals(e.getMessage(), + "The shot name: invalid/shot is not in the proper format. " + + "Shot names must be alpha numeric, no dashes or punctuation."); + } + } } diff --git a/cuebot/src/test/resources/conf/jobspec/jobspec_invalid_shot.xml b/cuebot/src/test/resources/conf/jobspec/jobspec_invalid_shot.xml new file mode 100644 index 000000000..fb67b5bfb --- /dev/null +++ b/cuebot/src/test/resources/conf/jobspec/jobspec_invalid_shot.xml @@ -0,0 +1,47 @@ + + + + + + + + + local + testing + invalid/shot + testuser + 9860 + + + False + 2 + False + + + + echo hello + 1 + 1 + + + shell + + + + + + diff --git a/cuebot/src/test/resources/conf/jobspec/jobspec_nonexistent_show.xml b/cuebot/src/test/resources/conf/jobspec/jobspec_nonexistent_show.xml new file mode 100644 index 000000000..12517467b --- /dev/null +++ b/cuebot/src/test/resources/conf/jobspec/jobspec_nonexistent_show.xml @@ -0,0 +1,41 @@ + + + + + + + + + + nonexistentshow + dev.cue + testuser + middle-tier@imageworks.com + 9860 + + + true + 0 + + + echo test + 1 + 1 + + + + From 9e49bc8149d1fb9c81bc5ae127bf15341fd88fca Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 8 Feb 2021 18:52:10 +0000 Subject: [PATCH 051/277] Lint CueGUI code. (#904) --- ci/pylintrc_main | 2 +- ci/pylintrc_test | 6 +- ci/run_python_tests.sh | 3 + cuegui/cuegui/AbstractDialog.py | 22 +- cuegui/cuegui/AbstractDockWidget.py | 14 +- cuegui/cuegui/AbstractTreeWidget.py | 94 ++-- cuegui/cuegui/AbstractWidgetItem.py | 37 +- cuegui/cuegui/Action.py | 51 +- cuegui/cuegui/ApplicationConfig.py | 25 - cuegui/cuegui/BugReportDialog.py | 25 - cuegui/cuegui/Comments.py | 14 +- cuegui/cuegui/ConfirmationDialog.py | 11 +- cuegui/cuegui/Constants.py | 12 +- cuegui/cuegui/CreatorDialog.py | 16 +- cuegui/cuegui/CueJobMonitorTree.py | 164 +++--- cuegui/cuegui/CueStateBarWidget.py | 15 +- cuegui/cuegui/Cuedepend.py | 129 ++--- cuegui/cuegui/DarkPalette.py | 13 +- cuegui/cuegui/DependDialog.py | 21 +- cuegui/cuegui/DependMonitorTree.py | 42 +- cuegui/cuegui/DependWizard.py | 258 +++++---- cuegui/cuegui/EmailDialog.py | 49 +- cuegui/cuegui/FilterDialog.py | 216 +++++--- cuegui/cuegui/FrameMonitor.py | 95 ++-- cuegui/cuegui/FrameMonitorTree.py | 204 ++++--- cuegui/cuegui/FrameRangeSelection.py | 95 ++-- cuegui/cuegui/GarbageCollector.py | 42 +- cuegui/cuegui/GraphSubscriptionsWidget.py | 117 ---- cuegui/cuegui/GroupDialog.py | 61 +- cuegui/cuegui/HostMonitor.py | 78 +-- cuegui/cuegui/HostMonitorTree.py | 55 +- cuegui/cuegui/ItemDelegate.py | 158 +++--- cuegui/cuegui/JobMonitorTree.py | 67 ++- cuegui/cuegui/LayerDialog.py | 80 +-- cuegui/cuegui/LayerMonitorTree.py | 44 +- cuegui/cuegui/LimitSelectionWidget.py | 29 +- cuegui/cuegui/LimitsWidget.py | 222 ++++---- cuegui/cuegui/LocalBooking.py | 80 +-- cuegui/cuegui/Main.py | 31 +- cuegui/cuegui/MainWindow.py | 87 ++- cuegui/cuegui/MenuActions.py | 520 +++++++++++------- cuegui/cuegui/MiscDialog.py | 9 +- cuegui/cuegui/Plugins.py | 128 +++-- cuegui/cuegui/PreviewWidget.py | 40 +- cuegui/cuegui/ProcMonitor.py | 41 +- cuegui/cuegui/ProcMonitorTree.py | 94 ++-- cuegui/cuegui/ProgressDialog.py | 17 +- cuegui/cuegui/Redirect.py | 98 ++-- cuegui/cuegui/ServiceDialog.py | 12 +- cuegui/cuegui/ShowDialog.py | 9 +- cuegui/cuegui/ShowsWidget.py | 23 +- cuegui/cuegui/SplashWindow.py | 11 +- cuegui/cuegui/Style.py | 19 +- cuegui/cuegui/SubscriptionGraphWidget.py | 61 +- cuegui/cuegui/SubscriptionsWidget.py | 55 +- cuegui/cuegui/TagsWidget.py | 16 +- cuegui/cuegui/TasksDialog.py | 54 +- cuegui/cuegui/TextEditDialog.py | 13 +- cuegui/cuegui/ThreadPool.py | 47 +- cuegui/cuegui/UnbookDialog.py | 120 ++-- cuegui/cuegui/Utils.py | 220 ++++---- cuegui/cuegui/__init__.py | 2 +- cuegui/cuegui/__main__.py | 9 +- cuegui/cuegui/eta.py | 328 +++++------ cuegui/cuegui/plugins/AllocationsPlugin.py | 33 +- cuegui/cuegui/plugins/AttributesPlugin.py | 50 +- cuegui/cuegui/plugins/LimitsPlugin.py | 32 +- cuegui/cuegui/plugins/LogViewPlugin.py | 36 +- cuegui/cuegui/plugins/MonitorCuePlugin.py | 41 +- cuegui/cuegui/plugins/MonitorHostsPlugin.py | 6 +- .../cuegui/plugins/MonitorJobDetailsPlugin.py | 13 +- cuegui/cuegui/plugins/MonitorJobsPlugin.py | 68 ++- cuegui/cuegui/plugins/RedirectPlugin.py | 5 + cuegui/cuegui/plugins/ServicePlugin.py | 8 +- cuegui/cuegui/plugins/ShowsPlugin.py | 6 +- .../plugins/SubscriptionsGraphPlugin.py | 22 +- cuegui/cuegui/plugins/SubscriptionsPlugin.py | 6 +- cuegui/tests/CueJobMonitorTree_tests.py | 12 +- cuegui/tests/FilterDialog_tests.py | 27 +- cuegui/tests/FrameMonitorTree_tests.py | 17 +- cuegui/tests/MenuActions_tests.py | 97 ++-- cuegui/tests/Redirect_tests.py | 10 +- cuegui/tests/Utils_tests.py | 6 +- cuegui/tests/images/images_tests.py | 4 + cuegui/tests/plugins/LogViewPlugin_tests.py | 32 +- cuegui/tests/test_utils.py | 4 + pycue/opencue/wrappers/service.py | 16 + 87 files changed, 3094 insertions(+), 2187 deletions(-) delete mode 100644 cuegui/cuegui/ApplicationConfig.py delete mode 100644 cuegui/cuegui/BugReportDialog.py delete mode 100644 cuegui/cuegui/GraphSubscriptionsWidget.py diff --git a/ci/pylintrc_main b/ci/pylintrc_main index f154d667b..b4510f658 100644 --- a/ci/pylintrc_main +++ b/ci/pylintrc_main @@ -3,7 +3,7 @@ # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code. -extension-pkg-whitelist= +extension-pkg-whitelist=PySide2 # Specify a score threshold to be exceeded before program exits with error. fail-under=10.0 diff --git a/ci/pylintrc_test b/ci/pylintrc_test index dc175b0b3..a436236f2 100644 --- a/ci/pylintrc_test +++ b/ci/pylintrc_test @@ -3,7 +3,7 @@ # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code. -extension-pkg-whitelist= +extension-pkg-whitelist=PySide2 # Specify a score threshold to be exceeded before program exits with error. fail-under=10.0 @@ -60,7 +60,8 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use "--disable=all --enable=classes # --disable=W". -disable=duplicate-code, +disable=arguments-differ, + duplicate-code, fixme, invalid-name, locally-disabled, @@ -69,6 +70,7 @@ disable=duplicate-code, no-self-use, protected-access, raise-missing-from, + too-many-lines, too-many-locals, too-many-public-methods, unused-argument, diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index a4797092f..7f0e9c0a4 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -31,5 +31,8 @@ if [[ "$1" == "--lint" ]]; then cd cueadmin && PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main cueadmin && cd .. cd cueadmin && PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests && cd .. + + cd cuegui && PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main cuegui --ignore=cuegui/images,cuegui/images/crystal && cd .. + cd cuegui && PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests && cd .. fi diff --git a/cuegui/cuegui/AbstractDialog.py b/cuegui/cuegui/AbstractDialog.py index 13563a947..613b3d83c 100644 --- a/cuegui/cuegui/AbstractDialog.py +++ b/cuegui/cuegui/AbstractDialog.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Base class for dialog windows.""" + + from __future__ import division from __future__ import print_function from __future__ import absolute_import @@ -23,18 +26,14 @@ class AbstractDialog(QtWidgets.QDialog): + """Base class for dialog windows.""" + def __init__(self, parent=None): QtWidgets.QDialog.__init__(self, parent) - def _newCheckBoxSelectionMatrix(self, - title, - allowedOptions, - checkedOptions, - parent=None): - return CheckBoxSelectionMatrix(title, - allowedOptions, - checkedOptions, - parent) + @staticmethod + def _newCheckBoxSelectionMatrix(title, allowedOptions, checkedOptions, parent=None): + return CheckBoxSelectionMatrix(title, allowedOptions, checkedOptions, parent) def _newDialogButtonBox(self, buttons, orientation=QtCore.Qt.Horizontal): buttonBox = QtWidgets.QDialogButtonBox(buttons, orientation, self) @@ -50,6 +49,8 @@ def _addWidgetRow(self, *widgets): class CheckBoxSelectionMatrix(QtWidgets.QWidget): + """Widget for displaying a matrix of checkboxes.""" + def __init__(self, title, allowedOptions, checkedOptions, parent=None): QtWidgets.QWidget.__init__(self, parent) layout = QtWidgets.QVBoxLayout(self) @@ -70,11 +71,14 @@ def __init__(self, title, allowedOptions, checkedOptions, parent=None): layout.addStretch() def checkedBoxes(self): + """Gets all checked boxes.""" return [cb for cb in self.__checkBoxes if cb.isChecked()] def checkedOptions(self): + """Gets text value of all checked boxes.""" return [str(cb.text()) for cb in self.__checkBoxes if cb.isChecked()] def checkBoxes(self, names): + """Sets checked state for checkboxes representing the named values.""" for box in self.__checkBoxes: box.setChecked(str(box.text()) in names) diff --git a/cuegui/cuegui/AbstractDockWidget.py b/cuegui/cuegui/AbstractDockWidget.py index 457c59e70..15857d5a2 100644 --- a/cuegui/cuegui/AbstractDockWidget.py +++ b/cuegui/cuegui/AbstractDockWidget.py @@ -13,9 +13,9 @@ # limitations under the License. -""" -Extends QDockWidget to provide a standard setup -""" +"""Base class for all CueGUI widgets. + +Extends QDockWidget to provide a standard setup.""" from __future__ import absolute_import @@ -29,6 +29,9 @@ class AbstractDockWidget(cuegui.Plugins.Plugin, QtWidgets.QDockWidget): + """Base class for all CueGUI widgets. + + Extends QDockWidget to provide a standard setup.""" closed = QtCore.Signal(object) enabled = QtCore.Signal() @@ -40,7 +43,8 @@ def __init__(self, parent, name, area = QtCore.Qt.LeftDockWidgetArea): self.parent = parent self.setAllowedAreas(QtCore.Qt.AllDockWidgetAreas) - self.setFeatures(QtWidgets.QDockWidget.DockWidgetClosable | QtWidgets.QDockWidget.DockWidgetMovable) + self.setFeatures( + QtWidgets.QDockWidget.DockWidgetClosable | QtWidgets.QDockWidget.DockWidgetMovable) self.setObjectName(name) parent.addDockWidget(area, self) @@ -53,9 +57,11 @@ def __init__(self, parent, name, area = QtCore.Qt.LeftDockWidgetArea): self.widget().setLayout(self.__layout) def closeEvent(self, event): + del event self.closed.emit(self) def showEvent(self, event): + del event self.enabled.emit() def layout(self): diff --git a/cuegui/cuegui/AbstractTreeWidget.py b/cuegui/cuegui/AbstractTreeWidget.py index 2ce367696..cf74590ef 100644 --- a/cuegui/cuegui/AbstractTreeWidget.py +++ b/cuegui/cuegui/AbstractTreeWidget.py @@ -13,9 +13,9 @@ # limitations under the License. -""" -Provides extended QTreeWidget functionality. -""" +"""Base class for CueGUI tree widgets. + +Provides extended QTreeWidget functionality.""" from __future__ import absolute_import @@ -48,13 +48,15 @@ COLUMN_TOOLTIP = 5 COLUMN_INFO_LENGTH = 6 -DEFAULT_LAMBDA = lambda s:"" +DEFAULT_LAMBDA = lambda s: "" DEFAULT_NAME = "" DEFAULT_WIDTH = 0 class AbstractTreeWidget(QtWidgets.QTreeWidget): - """Forms the basis for all TreeWidgets""" + """Base class for CueGUI tree widgets. + + Provides extended QTreeWidget functionality.""" itemDoubleClicked = QtCore.Signal(QtWidgets.QTreeWidgetItem, int) itemSingleClicked = QtCore.Signal(QtWidgets.QTreeWidgetItem, int) @@ -101,7 +103,9 @@ def __init__(self, parent): self.itemClicked.connect(self.__itemSingleClickedEmitToApp) self.itemDoubleClicked.connect(self.__itemDoubleClickedEmitToApp) self._timer.timeout.connect(self.updateRequest) + # pylint: disable=no-member QtGui.qApp.request_update.connect(self.updateRequest) + # pylint: enable=no-member self.updateRequest() self.setUpdateInterval(10) @@ -117,6 +121,7 @@ def closeEvent(self, event): event.accept() + # pylint: disable=attribute-defined-outside-init def startColumnsForType(self, itemType): """Start column definitions for the given item type. The first call to this defines the primary column type used to populate the column headers. @@ -131,6 +136,7 @@ def startColumnsForType(self, itemType): self.__columnInfoByType[itemType] = [] self.__columnCurrent = itemType + # pylint: disable=redefined-builtin def addColumn(self, name, width, id=0, default=True, data=DEFAULT_LAMBDA, sort=None, delegate=None, tip=""): @@ -178,7 +184,7 @@ def __setupColumns(self): # Setup column delegates if primaryColumnInfo[col][COLUMN_DELEGATE]: - self.setItemDelegateForColumn(col, primaryColumnInfo[col][COLUMN_DELEGATE](self)) + self.setItemDelegateForColumn(col, primaryColumnInfo[col][COLUMN_DELEGATE](self)) # Setup column name list if columnInfo[COLUMN_NAME].startswith("_"): @@ -214,13 +220,14 @@ def startTicksUpdate(self, updateInterval, self.ticksWithoutUpdate = 999 def tickNeedsUpdate(self): + """Gets whether enough time has passed for contents to need an update.""" if self.ticksWithoutUpdate >= self.updateInterval: if self.window().isMinimized(): if self.__maxUpdateInterval is not None and \ self.ticksWithoutUpdate >= self.__maxUpdateInterval: # Sufficient maximum interval return True - elif not self.__updateWhenMinimized: + if not self.__updateWhenMinimized: # Sufficient interval, except minimized return False # Sufficient interval, set to update when minimized @@ -236,6 +243,7 @@ def __tick(self): if not self.ticksLock.tryLock(): return try: + # pylint: disable=broad-except try: self.tick() except Exception as e: @@ -244,6 +252,9 @@ def __tick(self): self.ticksLock.unlock() def tick(self): + """Determines whether an update is needed and initiates updating logic. + + Must be defined by inheriting classes.""" raise NotImplementedError def getColumnInfo(self, columnType = None): @@ -259,16 +270,21 @@ def getColumnInfo(self, columnType = None): return self.__columnInfoByType[columnType] return self.__columnInfoByType[self.__columnPrimaryType] - def __itemSingleClickedEmitToApp(self, item, col): + @staticmethod + def __itemSingleClickedEmitToApp(item, col): """When an item is single clicked on: emits "single_click(PyQt_PyObject)" to the app @type item: QTreeWidgetItem @param item: The item single clicked on @type col: int @param col: Column number single clicked on""" + del col + # pylint: disable=no-member QtGui.qApp.single_click.emit(item.rpcObject) + # pylint: enable=no-member - def __itemDoubleClickedEmitToApp(self, item, col): + @staticmethod + def __itemDoubleClickedEmitToApp(item, col): """Handles when an item is double clicked on. emits "double_click(PyQt_PyObject)" to the app emits "view_object(PyQt_PyObject)" to the app @@ -276,8 +292,11 @@ def __itemDoubleClickedEmitToApp(self, item, col): @param item: The item double clicked on @type col: int @param col: Column number double clicked on""" + del col + # pylint: disable=no-member QtGui.qApp.view_object.emit(item.rpcObject) QtGui.qApp.double_click.emit(item.rpcObject) + # pylint: enable=no-member def addObject(self, rpcObject): """Adds or updates an rpcObject in the list using the _createItem function @@ -326,8 +345,6 @@ def _removeItem(self, item): item.setSelected(False) if item.parent(): - #This allowed more segfaults - #item.parent().removeChild(item) self.invisibleRootItem().removeChild(item) self.takeTopLevelItem(self.indexOfTopLevelItem(item)) objectClass = item.rpcObject.__class__.__name__ @@ -335,6 +352,7 @@ def _removeItem(self, item): del self._items['{}.{}'.format(objectClass, objectId)] def removeAllItems(self): + """Removes all items from the tree.""" self._itemsLock.lockForWrite() try: self._items = {} @@ -368,7 +386,10 @@ def _update(self): updated""" self._lastUpdate = time.time() if hasattr(QtGui.qApp, "threadpool"): - QtGui.qApp.threadpool.queue(self._getUpdate, self._processUpdate, "getting data for %s" % self.__class__) + # pylint: disable=no-member + QtGui.qApp.threadpool.queue( + self._getUpdate, self._processUpdate, "getting data for %s" % self.__class__) + # pylint: enable=no-member else: logger.warning("threadpool not found, doing work in gui thread") self._processUpdate(None, self._getUpdate()) @@ -382,6 +403,7 @@ def _processUpdate(self, work, rpcObjects): @type work: from ThreadPool @param rpcObjects: A list of rpc objects @type rpcObjects: list """ + del work self._itemsLock.lockForWrite() try: updated = [] @@ -416,22 +438,20 @@ def updateSoon(self): def redraw(self): """Forces the displayed items to be redrawn""" if not self.window().isMinimized(): + # pylint: disable=broad-except try: self.scheduleDelayedItemsLayout() - # This setDirtyRegion works but can give this error sometimes: - # "underlying C/C++ object has been deleted" - #self.setDirtyRegion(QtGui.QRegion(self.viewport().rect())) except Exception as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) def getColumnWidths(self): - """Return the column widths + """Gets the column widths. @rtype: list @return: A list of column widths""" return [self.columnWidth(index) for index in range(self.columnCount())] def setColumnWidths(self, widths): - """Set the column widths if thecorrect number are provided, but ignore + """Sets the column widths if the correct number are provided, but ignore the last one since it is stretched to the end. @type widths: list @param widths: The desired width for each column""" @@ -439,9 +459,10 @@ def setColumnWidths(self, widths): for index, width in enumerate(widths[:-1]): self.setColumnWidth(index, width) -################################################################################ -# Optionally limit updates when user scrolls -################################################################################ + ################################################################################ + # Optionally limit updates when user scrolls + ################################################################################ + def _limitUpdatesDuringScrollSetup(self, skipsAllowed = 1, delay = 1.0): """Allows the ability to skip updates when the user is scrolling @type skipsAllowed: int @@ -459,6 +480,7 @@ def _limitUpdatesDuringScrollSetup(self, skipsAllowed = 1, delay = 1.0): def __userScrolled(self, val): """Stores the time when the user last scrolled""" + del val self.__lastScrollTime = time.time() def __limitUpdatesDuringScrollAllowUpdate(self): @@ -469,17 +491,18 @@ def __limitUpdatesDuringScrollAllowUpdate(self): if not hasattr(self, "limitUpdatesDuringScroll"): return True - if time.time() - self.__lastScrollTime > self.__allowedSkipDelay or \ - self.__updateSkipCount >= self.__allowedSkipCount: + if (time.time() - self.__lastScrollTime > self.__allowedSkipDelay or + self.__updateSkipCount >= self.__allowedSkipCount): self.__updateSkipCount = 0 return True - else: - self.__updateSkipCount += 1 - return False -################################################################################ -# Allow the user to show or hide columns -################################################################################ + self.__updateSkipCount += 1 + return False + + ################################################################################ + # Allow the user to show or hide columns + ################################################################################ + def __setupColumnMenu(self): self.__dropdown = QtWidgets.QToolButton(self) self.__dropdown.setFocusPolicy(QtCore.Qt.NoFocus) @@ -518,22 +541,25 @@ def __handleColumnMenu(self, action): self.setColumnWidth(col, width) def getColumnVisibility(self): + """Gets table column visibility.""" settings = [] for col in range(self.columnCount()): settings.append(self.isColumnHidden(col)) return settings def setColumnVisibility(self, settings): + """Sets table column visibility.""" if settings: - for col in range(len(settings)): + for col, setting in enumerate(settings): if col <= self.columnCount(): - self.setColumnHidden(col, settings[col]) + self.setColumnHidden(col, setting) -################################################################################ -# Allow the user to move columns and remember position -################################################################################ + ################################################################################ + # Allow the user to move columns and remember position + ################################################################################ def getColumnOrder(self): + """Gets table column order.""" settings = {} header = self.header() for col in range(header.count()): @@ -541,7 +567,9 @@ def getColumnOrder(self): return settings def setColumnOrder(self, settings): + """Sets table column order.""" header = self.header() + # pylint: disable=unnecessary-lambda cols = sorted(settings.keys(), key=lambda x: int(x)) for col in cols: old_col = header.visualIndex(settings[col]) diff --git a/cuegui/cuegui/AbstractWidgetItem.py b/cuegui/cuegui/AbstractWidgetItem.py index 3ba6604a6..fec1dc5fe 100644 --- a/cuegui/cuegui/AbstractWidgetItem.py +++ b/cuegui/cuegui/AbstractWidgetItem.py @@ -13,9 +13,9 @@ # limitations under the License. -""" -Provides extended QWidgetItem functionality. -""" +"""Base class for CueGUI widget items. + +Provides extended QWidgetItem functionality.""" from __future__ import absolute_import @@ -27,11 +27,13 @@ from PySide2 import QtCore from PySide2 import QtWidgets +import opencue +import opencue.wrappers.job + import cuegui.Constants import cuegui.Logger import cuegui.Style -import opencue logger = cuegui.Logger.getLogger(__file__) @@ -42,17 +44,21 @@ class AbstractWidgetItem(QtWidgets.QTreeWidgetItem): - def __init__(self, itemType, object, parent, source=None): + """Base class for CueGUI widget items. + + Provides extended QWidgetItem functionality.""" + + def __init__(self, itemType, rpcObject, parent, source=None): QtWidgets.QTreeWidgetItem.__init__(self, parent, itemType) self.column_info = self.treeWidget().getColumnInfo(itemType) self._cache = {} self._source = source - self.rpcObject = object + self.rpcObject = rpcObject - def update(self, object=None, parent=None): + def update(self, rpcObject=None, parent=None): """Updates visual representation with latest data - @type object: Object - @param object: The object that contains updated information + @type rpcObject: Object + @param rpcObject: The object that contains updated information @type parent: QTreeWidgetItem @param parent: Changes the current parent to this parent if different""" # Changes parent if needed @@ -60,8 +66,8 @@ def update(self, object=None, parent=None): self.parent().removeChild(self) parent.addChild(self) - if object: - self.rpcObject = object + if rpcObject: + self.rpcObject = rpcObject self._cache = {} def data(self, col, role): @@ -75,12 +81,12 @@ def data(self, col, role): if role == QtCore.Qt.DisplayRole: return self.column_info[col][DISPLAY_LAMBDA](self.rpcObject) - elif role == QtCore.Qt.ForegroundRole: + if role == QtCore.Qt.ForegroundRole: if cuegui.Style.ColorTheme is None: cuegui.Style.init() return cuegui.Style.ColorTheme.COLOR_JOB_FOREGROUND - elif role == QtCore.Qt.UserRole: + if role == QtCore.Qt.UserRole: return self.type() return cuegui.Constants.QVARIANT_NULL @@ -91,8 +97,9 @@ def __lt__(self, other): column = self.treeWidget().sortColumn() if sortLambda and isinstance(other.rpcObject, opencue.wrappers.job.Job): + # pylint: disable=broad-except try: return sortLambda(self.rpcObject) < sortLambda(other.rpcObject) - except: - logger.warning("Sort failed on column {}, using text sort.".format(column)) + except Exception: + logger.warning("Sort failed on column %s, using text sort.", column) return str(self.text(column)) < str(other.text(column)) diff --git a/cuegui/cuegui/Action.py b/cuegui/cuegui/Action.py index e320249e5..853e6a087 100644 --- a/cuegui/cuegui/Action.py +++ b/cuegui/cuegui/Action.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -utility functions for creating QActions -""" +"""Utility functions for creating QActions.""" from __future__ import absolute_import @@ -49,12 +47,12 @@ def create(parent, text, tip, callback=None, icon=None): return a -def createAction(parent, id, text, tip, callback=None, icon=None): - """create(QtWidgets.QWidget, string text, string tip, callable callback=None, string icon=None) - creates a QtGui.QAction and optionally connects it to a slot - """ - if id in Actions: - raise Exception("Action %s has already been created" % (id)) +def createAction(parent, action_id, text, tip, callback=None, icon=None): + """Creates a QtGui.QAction and optionally connects it to a slot. + + create(QtWidgets.QWidget, string text, string tip, callable callback=None, string icon=None)""" + if action_id in Actions: + raise Exception("Action %s has already been created" % (action_id)) a = QtWidgets.QAction(parent) a.setText(text) @@ -64,42 +62,41 @@ def createAction(parent, id, text, tip, callback=None, icon=None): a.setIcon(QtGui.QIcon(":/images/%s.png" % icon)) if callback: connectActionSlot(a,callback) - Actions[id] = a + Actions[action_id] = a return a -def getAction(id): - return Actions[id] +def getAction(action_id): + """Gets an action by ID.""" + return Actions[action_id] -def createActionGroup(parent, id, actions): +def createActionGroup(parent, action_id, actions): + """Creates an action group.""" g = QtWidgets.QActionGroup(parent) for action in actions: g.addAction(action) - Groups[id] = g + Groups[action_id] = g -def getActionGroup(id): - return Groups[id] +def getActionGroup(group_id): + """Gets an action group.""" + return Groups[group_id] -def applyActionGroup(id, menu): - for act in getActionGroup(id).actions(): +def applyActionGroup(group_id, menu): + """Add all actions in a group to the given menu.""" + for act in getActionGroup(group_id).actions(): menu.addAction(act) -def connectActionSlot(action, callable): - """connectActionSlot - connects an action's triggered() signal to a callable object - """ - action.triggered.connect(callable) +def connectActionSlot(action, actionCallable): + """Connects an action's triggered() signal to a callable object.""" + action.triggered.connect(actionCallable) class Refresh(QtWidgets.QAction): - """Refresh - - refresh something - """ + """Refreshes something.""" def __init__(self,callback=None, parent=None): QtWidgets.QAction.__init__(self,parent) diff --git a/cuegui/cuegui/ApplicationConfig.py b/cuegui/cuegui/ApplicationConfig.py deleted file mode 100644 index 3b4638168..000000000 --- a/cuegui/cuegui/ApplicationConfig.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright Contributors to the OpenCue Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from __future__ import print_function -from __future__ import division -from __future__ import absolute_import - -from PySide2 import QtWidgets - - -class ApplicationConfig(QtWidgets.QWidget): - def __init__(self): - pass \ No newline at end of file diff --git a/cuegui/cuegui/BugReportDialog.py b/cuegui/cuegui/BugReportDialog.py deleted file mode 100644 index 6cae58d91..000000000 --- a/cuegui/cuegui/BugReportDialog.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright Contributors to the OpenCue Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from __future__ import print_function -from __future__ import division -from __future__ import absolute_import - -from PySide2 import QtWidgets - - -class BugReportWidget(QtWidgets.QWidget): - def __init__(self, parent=None): - QtWidgets.QWidget.__init__(self) diff --git a/cuegui/cuegui/Comments.py b/cuegui/cuegui/Comments.py index 0af3092e7..dc493433f 100644 --- a/cuegui/cuegui/Comments.py +++ b/cuegui/cuegui/Comments.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Dialog for displaying a comment list.""" + + from __future__ import absolute_import from __future__ import print_function from __future__ import division @@ -37,9 +40,11 @@ class CommentListDialog(QtWidgets.QDialog): - """A dialog to display a comment list""" + """Dialog for displaying a comment list.""" + def __init__(self, source, parent=None): - """Initialize the dialog + """Initialize the dialog. + @type source: Job or Host @param source: The source to get the comments from @type parent: QWidget @@ -103,6 +108,7 @@ def __init__(self, source, parent=None): def __textEdited(self, text=None): """Called when the text boxes are modified, enables the save button""" + del text self.__btnSave.setEnabled(True) def __close(self): @@ -198,8 +204,10 @@ def refreshComments(self): def __macroLoad(self): """Loads the defined comment macros from settings""" + # pylint: disable=no-member self.__macroList = pickle.loads( str(QtGui.qApp.settings.value("Comments", pickle.dumps({})))) + # pylint: enable=no-member self.__macroRefresh() def __macroRefresh(self): @@ -213,7 +221,9 @@ def __macroRefresh(self): def __macroSave(self): """Saves the current comment macros to settings""" + # pylint: disable=no-member QtGui.qApp.settings.setValue("Comments", pickle.dumps(self.__macroList)) + # pylint: enable=no-member def __macroHandle(self, selection): """Called when the comment macro combo box is selected diff --git a/cuegui/cuegui/ConfirmationDialog.py b/cuegui/cuegui/ConfirmationDialog.py index 3d7e8a850..0c71e88e3 100644 --- a/cuegui/cuegui/ConfirmationDialog.py +++ b/cuegui/cuegui/ConfirmationDialog.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -A confirmation dialog -""" +"""Confirmation dialog.""" from __future__ import print_function @@ -28,8 +26,11 @@ class ConfirmationDialog(QtWidgets.QDialog): - def __init__(self, title, text, items = [], parent = None): - """A confirmation dialog + """Confirmation dialog.""" + + # pylint: disable=dangerous-default-value + def __init__(self, title, text, items=[], parent=None): + """ @type title: string @param title: The title for the confirmation dialog @type text: string diff --git a/cuegui/cuegui/Constants.py b/cuegui/cuegui/Constants.py index 27503f363..bb38e9495 100644 --- a/cuegui/cuegui/Constants.py +++ b/cuegui/cuegui/Constants.py @@ -32,7 +32,8 @@ import opencue -possible_version_path = os.path.join(os.path.abspath(os.path.join(__file__ , "../../..")), 'VERSION.in') +possible_version_path = os.path.join( + os.path.abspath(os.path.join(__file__ , "../../..")), 'VERSION.in') if os.path.exists(possible_version_path): with open(possible_version_path) as fp: VERSION = fp.read().strip() @@ -70,9 +71,10 @@ EMAIL_BODY_SUFFIX = "\n\n" EMAIL_DOMAIN = "" +GITHUB_CREATE_ISSUE_URL = 'https://github.com/AcademySoftwareFoundation/OpenCue/issues/new' URL_USERGUIDE = "https://www.opencue.io/docs/" -URL_SUGGESTION = "https://github.com/AcademySoftwareFoundation/OpenCue/issues/new?labels=enhancement&template=enhancement.md" -URL_BUG = "https://github.com/AcademySoftwareFoundation/OpenCue/issues/new?labels=bug&template=bug_report.md" +URL_SUGGESTION = "%s?labels=enhancement&template=enhancement.md" % GITHUB_CREATE_ISSUE_URL +URL_BUG = "%s?labels=bug&template=bug_report.md" % GITHUB_CREATE_ISSUE_URL if platform.system() == "Windows": DEFAULT_EDITOR = "notepad" @@ -129,6 +131,8 @@ QT_MAX_INT = 2147483647 -LOG_HIGHLIGHT_ERROR = ['error', 'aborted', 'fatal', 'failed', 'killed', 'command not found', 'no licenses could be found', 'killMessage'] +LOG_HIGHLIGHT_ERROR = [ + 'error', 'aborted', 'fatal', 'failed', 'killed', 'command not found', + 'no licenses could be found', 'killMessage'] LOG_HIGHLIGHT_WARN = ['warning', 'not found'] LOG_HIGHLIGHT_INFO = ['info:', 'rqd cmd:'] diff --git a/cuegui/cuegui/CreatorDialog.py b/cuegui/cuegui/CreatorDialog.py index 2ed8b96d0..0e436b201 100644 --- a/cuegui/cuegui/CreatorDialog.py +++ b/cuegui/cuegui/CreatorDialog.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Dialog for creating a subscription.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -26,14 +29,13 @@ class SubscriptionCreator(QtWidgets.QWidget): + """Widget for creating a subscription.""" + def __init__(self, show=None, parent=None): QtWidgets.QWidget.__init__(self, parent) show_name = "" if show: - try: - show_name = show.data.name - except Exception: - show_name = str(show) + show_name = show.data.name self.__shows = opencue.api.getShows() self.__allocs = opencue.api.getAllocations() @@ -70,7 +72,7 @@ def create(self): show.createSubscription(alloc, float(self.sizeBox.value()), float(self.burstBox.value())) - except Exception as e: + except opencue.exception.CueException as e: QtWidgets.QMessageBox.warning( self, "Create Subscription", @@ -79,7 +81,11 @@ def create(self): class SubscriptionCreatorDialog(QtWidgets.QDialog): + """Dialog for creating a subscription.""" + def __init__(self, show=None, parent=None): + del parent + QtWidgets.QDialog.__init__(self) self.__creator = SubscriptionCreator(show, self) diff --git a/cuegui/cuegui/CueJobMonitorTree.py b/cuegui/cuegui/CueJobMonitorTree.py index fec7d9759..8d1307771 100644 --- a/cuegui/cuegui/CueJobMonitorTree.py +++ b/cuegui/cuegui/CueJobMonitorTree.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Tree widget for displaying a show/job hierarchy.""" + + from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -26,6 +29,8 @@ from PySide2 import QtWidgets import opencue +import opencue.compiled_proto.job_pb2 +import opencue.wrappers.group import cuegui.AbstractTreeWidget import cuegui.AbstractWidgetItem @@ -47,6 +52,7 @@ def getEta(stats): + """Gets estimated time remaining for a job.""" if stats.runningFrames: remaining = (((stats.pendingFrames - 1) * stats.avgFrameSec) + stats.highFrameSec) if remaining: @@ -55,6 +61,7 @@ def getEta(stats): class CueJobMonitorTree(cuegui.AbstractTreeWidget.AbstractTreeWidget): + """Tree widget for displaying a show/job hierarchy.""" view_object = QtCore.Signal(object) single_click = QtCore.Signal(object) @@ -65,15 +72,15 @@ def __init__(self, parent): self.currtime = time.time() self.startColumnsForType(cuegui.Constants.TYPE_JOB) - self.addColumn("Job", 550, id=1, - data=lambda job: job.data.name, - tip="The name of the job: show-shot-user_uniqueName\n\n" - "The color behind the job will change to:\n" - "Blue \t if it is paused\n" - "Red \t if it has dead frames\n" - "Green \t if it has no running frames with frames waiting\n" - "Purple \t if all remaining frames depend on something\n" - "Yellow \t if the maxRss is over %sKb" % cuegui.Constants.MEMORY_WARNING_LEVEL) + self.addColumn( + "Job", 550, id=1, data=lambda job: job.data.name, + tip="The name of the job: show-shot-user_uniqueName\n\n" + "The color behind the job will change to:\n" + "Blue \t if it is paused\n" + "Red \t if it has dead frames\n" + "Green \t if it has no running frames with frames waiting\n" + "Purple \t if all remaining frames depend on something\n" + "Yellow \t if the maxRss is over %sKb" % cuegui.Constants.MEMORY_WARNING_LEVEL) self.addColumn("_Comment", 20, id=2, sort=lambda job: job.data.has_comment, tip="A comment icon will appear if a job has a comment. You\n" @@ -103,7 +110,7 @@ def __init__(self, parent): data=lambda job: job.data.job_stats.total_frames, sort=lambda job: job.data.job_stats.total_frames, tip="The total number of frames.") - self.addColumn("_Booking Bar", 150, id=9, + self.addColumn("_Booking Bar", 150, id=9, delegate=cuegui.ItemDelegate.JobBookingBarDelegate) self.addColumn("Min", 38, id=10, data=lambda job: "%.0f" % job.data.min_cores, @@ -115,10 +122,11 @@ def __init__(self, parent): sort=lambda job: job.data.max_cores, tip="The maximum number of running cores that the cuebot\n" "will allow.") - self.addColumn("Age", 50, id=12, - data=lambda job: cuegui.Utils.secondsToHHHMM(self.currtime - job.data.start_time), - sort=lambda job: self.currtime - job.data.start_time, - tip="The HOURS:MINUTES since the job was launched.") + self.addColumn( + "Age", 50, id=12, + data=lambda job: cuegui.Utils.secondsToHHHMM(self.currtime - job.data.start_time), + sort=lambda job: self.currtime - job.data.start_time, + tip="The HOURS:MINUTES since the job was launched.") self.addColumn("Pri", 30, id=13, data=lambda job: job.data.priority, sort=lambda job: job.data.priority, @@ -162,7 +170,8 @@ def __init__(self, parent): self.addColumn("", 0, id=9, data=lambda group: (group.data.min_cores or "")) self.addColumn("", 0, id=10, - data=lambda group: (group.data.max_cores > 0 and group.data.max_cores or "")) + data=lambda group: ( + group.data.max_cores > 0 and group.data.max_cores or "")) self.addColumn("", 0, id=11) self.addColumn("", 0, id=12) self.addColumn("", 0, id=13) @@ -185,7 +194,9 @@ def __init__(self, parent): self.__menuActions = cuegui.MenuActions.MenuActions( self, self.updateSoon, self.selectedObjects) + # pylint: disable=no-member QtGui.qApp.facility_changed.connect(self.removeAllShows) + # pylint: enable=no-member self.itemClicked.connect(self.__itemSingleClickedCopy) self.itemClicked.connect(self.__itemSingleClickedComment) @@ -201,9 +212,12 @@ def __itemSingleClickedCopy(self, item, col): @param item: The item clicked on @type col: int @param col: The column clicked on""" + del item + del col selected = [job.data.name for job in self.selectedObjects() if cuegui.Utils.isJob(job)] if selected: - QtWidgets.QApplication.clipboard().setText(" ".join(selected), QtGui.QClipboard.Selection) + QtWidgets.QApplication.clipboard().setText( + " ".join(selected), QtGui.QClipboard.Selection) def __itemSingleClickedComment(self, item, col): """If the comment column is clicked on, and there is a comment on the @@ -250,17 +264,17 @@ def dropEvent(self, event): if item and item.type() in (cuegui.Constants.TYPE_ROOTGROUP, cuegui.Constants.TYPE_GROUP): job_ids = cuegui.Utils.dropEvent(event, "application/x-job-ids") group_ids = cuegui.Utils.dropEvent(event, "application/x-group-ids") - job_names = cuegui.Utils.dropEvent(event, "application/x-job-names") - group_names = cuegui.Utils.dropEvent(event, "application/x-group-names") if job_ids or group_ids: body = "" if group_ids: - body += "Groups:\n" + "\n".join(cuegui.Utils.dropEvent(event, "application/x-group-names")) + body += "Groups:\n" + "\n".join( + cuegui.Utils.dropEvent(event, "application/x-group-names")) if group_ids and job_ids: body += "\n\n" if job_ids: - body += "Jobs:\n" + "\n".join(cuegui.Utils.dropEvent(event, "application/x-job-names")) + body += "Jobs:\n" + "\n".join( + cuegui.Utils.dropEvent(event, "application/x-job-names")) result = QtWidgets.QMessageBox.question( self, @@ -290,8 +304,8 @@ def addShow(self, show, update=True): if show not in self.__shows: try: self.__shows[show] = opencue.api.findShow(show) - except: - logger.warning("This show does not exist: %s" % show) + except opencue.exception.EntityNotFoundException: + logger.warning("This show does not exist: %s", show) if update: self._update() @@ -332,9 +346,9 @@ def __getCollapsed(self): def __setCollapsed(self, collapsed): self.expandAll() - for id in collapsed: - if id in self._items: - self._items[id].setExpanded(False) + for itemId in collapsed: + if itemId in self._items: + self._items[itemId].setExpanded(False) def _getUpdate(self): """Returns a list of NestedGroup from the cuebot for the monitored shows @@ -349,27 +363,28 @@ def _getUpdate(self): for group in groups: nestedGroups.append(opencue.wrappers.group.NestedGroup(group)) allIds.extend(self.__getNestedIds(group)) - except Exception as e: + except opencue.exception.CueException as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) return None return [nestedGroups, allIds] - def _processUpdate(self, work, results): + def _processUpdate(self, work, rpcObjects): """Adds or updates jobs and groups. Removes those that do not get updated @type work: from threadpool @param work: from threadpool - @type results: [list, set(str)] - @param results: List that contains updated nested groups and a set + @type rpcObjects: [list, set(str)] + @param rpcObjects: List that contains updated nested groups and a set of all updated item ids""" - if results is None: + if rpcObjects is None: return self._itemsLock.lockForWrite() + # pylint: disable=broad-except try: current = set(self._items.keys()) - if current == set(results[1]): + if current == set(rpcObjects[1]): # Only updates - self.__processUpdateHandleNested(self.invisibleRootItem(), results[0]) + self.__processUpdateHandleNested(self.invisibleRootItem(), rpcObjects[0]) self.redraw() else: # (Something removed) or (Something added) @@ -378,10 +393,11 @@ def _processUpdate(self, work, results): scrolled = self.verticalScrollBar().value() self._items = {} self.clear() - self.__processUpdateHandleNested(self.invisibleRootItem(), results[0]) + self.__processUpdateHandleNested(self.invisibleRootItem(), rpcObjects[0]) self.__setCollapsed(collapsed) self.verticalScrollBar().setValue(scrolled) - [self._items[id_].setSelected(True) for id_ in selected_ids if id_ in self._items] + list(map(lambda id_: self._items[id_].setSelected(True), + [id_ for id_ in selected_ids if id_ in self._items])) except Exception: logger.warning("Failed to process update.", exc_info=True) finally: @@ -394,17 +410,17 @@ def __getNestedIds(self, group): @rtype: list @return: The list of all child ids""" updated = [] - for group in group.groups.nested_groups: - updated.append(group.id) + for innerGroup in group.groups.nested_groups: + updated.append(innerGroup.id) # If group has groups, recursively call this function - for g in group.groups.nested_groups: + for g in innerGroup.groups.nested_groups: updated_g = self.__getNestedIds(g) if updated_g: updated.extend(updated_g) # If group has jobs, update them - for jobId in group.jobs: + for jobId in innerGroup.jobs: updated.append(jobId) return updated @@ -428,7 +444,9 @@ def __processUpdateHandleNested(self, parent, groups): else: self._items[group.id()] = groupItem = RootGroupWidgetItem(group, parent) - nestedGroups = [opencue.wrappers.group.NestedGroup(nestedGroup) for nestedGroup in group.data.groups.nested_groups] + nestedGroups = [ + opencue.wrappers.group.NestedGroup(nestedGroup) + for nestedGroup in group.data.groups.nested_groups] self.__processUpdateHandleNested(groupItem, nestedGroups) for jobId in group.data.jobs: @@ -439,9 +457,11 @@ def __processUpdateHandleNested(self, parent, groups): else: self._items[job.id()] = JobWidgetItem(job, groupItem) except RuntimeError: - logger.warning("Failed to create tree item. RootView might be closed", exc_info=True) + logger.warning( + "Failed to create tree item. RootView might be closed", exc_info=True) - def mouseDoubleClickEvent(self,event): + def mouseDoubleClickEvent(self, event): + del event objects = self.selectedObjects() if objects: self.view_object.emit(objects[0]) @@ -513,7 +533,6 @@ def contextMenuEvent(self, e): if counts["job"] == 1: self.__menuActions.jobs().addAction(menu, "reorder") self.__menuActions.jobs().addAction(menu, "stagger") - #Broken: self.__menuActions.jobs().addAction(menu, "testCloBook") menu.addSeparator() if jobTypes["unpaused"]: self.__menuActions.jobs().addAction(menu, "pause") @@ -554,9 +573,17 @@ def actionResumeSelectedItems(self): """Resume selected jobs""" self.__menuActions.jobs().resume() + def tick(self): + pass + + class RootGroupWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): + """Widget item representing a single root group.""" + __initialized = False - def __init__(self, object, parent): + + # pylint: disable=protected-access + def __init__(self, rpcObject, parent): if not self.__initialized: if cuegui.Style.ColorTheme is None: cuegui.Style.init() @@ -567,7 +594,7 @@ def __init__(self, object, parent): self.__class__.__type = cuegui.Constants.TYPE_ROOTGROUP cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( - self, cuegui.Constants.TYPE_ROOTGROUP, object, parent) + self, cuegui.Constants.TYPE_ROOTGROUP, rpcObject, parent) def data(self, col, role): """Returns the proper display data for the given column and role @@ -580,16 +607,16 @@ def data(self, col, role): if role == QtCore.Qt.DisplayRole: return self.column_info[col][cuegui.Constants.COLUMN_INFO_DISPLAY](self.rpcObject) - elif role == QtCore.Qt.FontRole: + if role == QtCore.Qt.FontRole: return FONT_BOLD - elif role == QtCore.Qt.ForegroundRole: + if role == QtCore.Qt.ForegroundRole: return self.__foregroundColor - elif role == QtCore.Qt.BackgroundRole: + if role == QtCore.Qt.BackgroundRole: return self.__backgroundColor - elif role == QtCore.Qt.DecorationRole: + if role == QtCore.Qt.DecorationRole: if col == 0: return self.__icon @@ -610,8 +637,11 @@ def __ne__(self, other): class GroupWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): """Represents a group entry in the MonitorCue widget.""" + __initialized = False - def __init__(self, object, parent): + + # pylint: disable=protected-access + def __init__(self, rpcObject, parent): if not self.__initialized: self.__class__.__initialized = True self.__class__.__icon = QtGui.QIcon(":group.png") @@ -620,7 +650,7 @@ def __init__(self, object, parent): self.__class__.__type = cuegui.Constants.TYPE_GROUP cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( - self, cuegui.Constants.TYPE_GROUP, object, parent) + self, cuegui.Constants.TYPE_GROUP, rpcObject, parent) def data(self, col, role): """Returns the proper display data for the given column and role @@ -633,19 +663,19 @@ def data(self, col, role): if role == QtCore.Qt.DisplayRole: return self.column_info[col][cuegui.Constants.COLUMN_INFO_DISPLAY](self.rpcObject) - elif role == QtCore.Qt.FontRole: + if role == QtCore.Qt.FontRole: return FONT_BOLD - elif role == QtCore.Qt.ForegroundRole: + if role == QtCore.Qt.ForegroundRole: return self.__foregroundColor - elif role == QtCore.Qt.BackgroundRole: + if role == QtCore.Qt.BackgroundRole: return self.__backgroundColor - elif role == QtCore.Qt.DecorationRole and col == 0: + if role == QtCore.Qt.DecorationRole and col == 0: return self.__icon - elif role == QtCore.Qt.UserRole: + if role == QtCore.Qt.UserRole: return self.__type return cuegui.Constants.QVARIANT_NULL @@ -659,19 +689,23 @@ def __lt__(self, other): def __ne__(self, other): if hasattr(other, 'rpcObject'): return other.rpcObject != self.rpcObject - else: - return True + return True class JobWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): """Represents a job entry in the MonitorCue widget.""" + __initialized = False - def __init__(self, object, parent): + + # pylint: disable=protected-access + def __init__(self, rpcObject, parent): if not self.__initialized: self.__class__.__initialized = True self.__class__.__commentIcon = QtGui.QIcon(":comment.png") self.__class__.__eatIcon = QtGui.QIcon(":eat.png") + # pylint: disable=no-member self.__class__.__backgroundColor = QtGui.qApp.palette().color(QtGui.QPalette.Base) + # pylint: enable=no-member self.__class__.__foregroundColor = cuegui.Style.ColorTheme.COLOR_JOB_FOREGROUND self.__class__.__pausedColor = cuegui.Style.ColorTheme.COLOR_JOB_PAUSED_BACKGROUND self.__class__.__finishedColor = cuegui.Style.ColorTheme.COLOR_JOB_FINISHED_BACKGROUND @@ -681,10 +715,10 @@ def __init__(self, object, parent): self.__class__.__highMemoryColor = cuegui.Style.ColorTheme.COLOR_JOB_HIGH_MEMORY self.__class__.__type = cuegui.Constants.TYPE_JOB - object.parent = None + rpcObject.parent = None cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( - self, cuegui.Constants.TYPE_JOB, object, parent) + self, cuegui.Constants.TYPE_JOB, rpcObject, parent) def data(self, col, role): """Returns the proper display data for the given column and role @@ -700,13 +734,13 @@ def data(self, col, role): self.column_info[col][cuegui.Constants.COLUMN_INFO_DISPLAY](self.rpcObject) return self._cache.get(col, cuegui.Constants.QVARIANT_NULL) - elif role == QtCore.Qt.ForegroundRole: + if role == QtCore.Qt.ForegroundRole: return self.__foregroundColor - elif role == QtCore.Qt.BackgroundRole: + if role == QtCore.Qt.BackgroundRole: if col == COLUMN_MAXRSS and \ self.rpcObject.data.job_stats.max_rss > cuegui.Constants.MEMORY_WARNING_LEVEL: - return self.__highMemoryColor + return self.__highMemoryColor if self.rpcObject.data.is_paused: return self.__pausedColor if self.rpcObject.data.job_stats.dead_frames: @@ -720,10 +754,10 @@ def data(self, col, role): return self.__noRunningColor return self.__backgroundColor - elif role == QtCore.Qt.DecorationRole: + if role == QtCore.Qt.DecorationRole: if col == COLUMN_COMMENT and self.rpcObject.data.has_comment: return self.__commentIcon - elif col == COLUMN_EAT and self.rpcObject.data.auto_eat: + if col == COLUMN_EAT and self.rpcObject.data.auto_eat: return self.__eatIcon elif role == QtCore.Qt.UserRole: diff --git a/cuegui/cuegui/CueStateBarWidget.py b/cuegui/cuegui/CueStateBarWidget.py index f77a1e5e1..3bf8be62d 100644 --- a/cuegui/cuegui/CueStateBarWidget.py +++ b/cuegui/cuegui/CueStateBarWidget.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Widget that graphically displays the state of all jobs displayed.""" + + from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -32,22 +35,29 @@ class CueStateBarWidget(QtWidgets.QWidget): - """Creates a bar that graphically displays the state of all jobs displayed""" + """Widget that graphically displays the state of all jobs displayed.""" + __colorInvalid = QtGui.QColor() __brushPattern = QtGui.QBrush(QtCore.Qt.Dense4Pattern) - def __init__(self, sourceTree, parent = None): + + def __init__(self, sourceTree, parent=None): """CueStateBar init @type sourceTree: QTreeWidget @param sourceTree: The tree to get the jobs from @type parent: QWidget @param parent: The parent widget""" QtWidgets.QWidget.__init__(self, parent) + + self.__background = None + self.setContentsMargins(8, 1, 1, 1) self.setFixedWidth(22) self.__sourceTree = weakref.proxy(sourceTree) self.__colors = [] + # pylint: disable=no-member self.__baseColor = QtGui.qApp.palette().color(QtGui.QPalette.Base) + # pylint: enable=no-member self.__colorsLock = QtCore.QReadWriteLock() self.__timer = QtCore.QTimer(self) self.__lastUpdate = 0 @@ -84,6 +94,7 @@ def paintEvent(self, event): """Called when the widget is being redrawn @type event: QEvent @param event: The draw event""" + del event assert threading.currentThread().getName() == "MainThread" self.__colorsLock.lockForWrite() try: diff --git a/cuegui/cuegui/Cuedepend.py b/cuegui/cuegui/Cuedepend.py index e16ea56ce..2997513e5 100644 --- a/cuegui/cuegui/Cuedepend.py +++ b/cuegui/cuegui/Cuedepend.py @@ -13,7 +13,8 @@ # limitations under the License. -""" +"""Utility functions for creating depends. + Dependency Types: The long and short version of dependency types are valid. In most cases its not required to actually specify a dependency type. @@ -22,25 +23,6 @@ LayerOnJob / loj LayerOnLayer / lol LayerOnFrame / lof FrameOnJob / foj FrameOnLayer / fol FrameOnFrame / fof FrameByFRame / fbf HardDepend / hd - -Examples: - Create a hard depend between two jobs (all layers) - cuedepend -c -t hd -job pipe-dev.cue-chambers_shell_v1 -on-job pipe-dev.cue-chambers_shell_v2 - - Create a frame by frame dependency between two layers in the same job - cuedepend -t fbf -c pipe-dev.cue-chambers_shell_v1 -layer pass_1 -on-layer pass_2 - - Create a frame by frame dependency between two layers in different jobs - cuedepend -c -t fbf -job pipe-dev.cue-chambers_comp_v1 -layer comp -on-job pipe-dev.cue-chambers_render_v1 -on-layer bty_pass - - Create a depependency between a layer and a frame - cuedepend -c -job pipe-dev.cue-chambers_j1 -layer render -on-layer setup -on-frame 1 - - Drop a dependency using the unique id: - cuedepend -d baafaadc-498c-4100-a74d-42abd2b8e6b9 - - Drop all dependencies a job is waiting for - cuedepend -drop-all -j pipe-dev.cue-chambers_comp_v1 """ @@ -54,29 +36,35 @@ from opencue.compiled_proto import depend_pb2 -logger = logging.getLogger("opencue.tools.cuedepend") +logger = logging.getLogger(__file__) + -ERR_INVALID_ON_JOB = "Error, a dependency of this type requires a valid job name to depend on. See -on-job." -ERR_INVALID_ON_LAYER = "Error, a dependency of this type requires a valid layer name to depend on. See -on-layer." -ERR_INVALID_ON_FRAME = "Error, a dependency of this type requries a valid frame name to depend on. See -on-frame." -ERR_INVALID_ER_JOB = "Error, a dependency of this type requires a valid job name to depend on. See -job." -ERR_INVALID_ER_LAYER = "Error, a dependency of this type requires a valid layer name to depend on. See -layer." -ERR_INVALID_ER_FRAME = "Error, a dependency of this type requries a valid frame name to depend on. See -frame." +ERR_INVALID_ON_JOB = ( + "Error, a dependency of this type requires a valid job name to depend on. See -on-job.") +ERR_INVALID_ON_LAYER = ( + "Error, a dependency of this type requires a valid layer name to depend on. See -on-layer.") +ERR_INVALID_ON_FRAME = ( + "Error, a dependency of this type requries a valid frame name to depend on. See -on-frame.") +ERR_INVALID_ER_JOB = ( + "Error, a dependency of this type requires a valid job name to depend on. See -job.") +ERR_INVALID_ER_LAYER = ( + "Error, a dependency of this type requires a valid layer name to depend on. See -layer.") +ERR_INVALID_ER_FRAME = ( + "Error, a dependency of this type requries a valid frame name to depend on. See -frame.") def __is_valid(value, error): - """A couple sanity checks before. The gRpc library takes - care of everything else""" + """Minor depend validation. The gRPC library takes care of everything else.""" if not value: raise ValueError(error) if isinstance(value, str) and len(value) < 1: - raise ValueError(error) + raise ValueError(error) -def createDepend(type, job, layer, frame, onjob, onlayer, onframe): +def createDepend(depend_type, job, layer, frame, onjob, onlayer, onframe): """Creates a new dependency of the specified type. - @type type: string - @param type: The type of dependency + @type depend_type: string + @param depend_type: The type of dependency @type job: string @param job: The name of the dependant job @type layer: string @@ -93,13 +81,14 @@ def createDepend(type, job, layer, frame, onjob, onlayer, onframe): @return: The newly created dependency""" if not onjob and not onlayer and not onframe: - raise ValueError("You must specify something to depend on, see -on-job, -on-layer, -on-frame") + raise ValueError( + "You must specify something to depend on, see -on-job, -on-layer, -on-frame") if not onjob: logger.debug("assuming internal depend") onjob = job - typeName = depend_pb2.DependType.Name(type) + typeName = depend_pb2.DependType.Name(depend_type) if typeName in ("HARD_DEPEND", "hd"): depend = createHardDepend(job, onjob) elif typeName in ("JOB_ON_JOB", "joj"): @@ -125,10 +114,11 @@ def createDepend(type, job, layer, frame, onjob, onlayer, onframe): elif typeName in ("LAYER_ON_SIM_FRAME", "los"): depend = createLayerOnSimFrameDepend(job, layer, onjob, onlayer, onframe) else: - raise Exception("invalid dependency type: %s" % (type)) + raise Exception("invalid dependency type: %s" % depend_type) return depend + def createHardDepend(job, onjob): """Creates a frame by frame dependency for all non-preprocess/refshow layers (Hard Depend) @@ -144,15 +134,15 @@ def createHardDepend(job, onjob): depends = [] - logger.debug("creating hard depend from %s to %s" % (job, onjob)) + logger.debug("creating hard depend from %s to %s", job, onjob) onLayers = opencue.api.findJob(onjob).getLayers() for depend_er_layer in opencue.api.findJob(job).getLayers(): - for depend_on_layer in onLayers: - if depend_er_layer.data.type == depend_on_layer.data.type: - depends.append(depend_er_layer.createFrameByFrameDependency(depend_on_layer, - False)) + for depend_on_layer in onLayers: + if depend_er_layer.data.type == depend_on_layer.data.type: + depends.append(depend_er_layer.createFrameByFrameDependency(depend_on_layer, False)) return depends + def createJobOnJobDepend(job, onjob): """Creates a job on job dependency. (Soft Depend) @@ -165,10 +155,11 @@ def createJobOnJobDepend(job, onjob): __is_valid(job, ERR_INVALID_ER_JOB) __is_valid(onjob, ERR_INVALID_ON_JOB) - logger.debug("creating joj depend from %s to %s" % (job, onjob)) + logger.debug("creating joj depend from %s to %s", job, onjob) depend_er_job = opencue.api.findJob(job) return depend_er_job.createDependencyOnJob(opencue.api.findJob(onjob)) + def createJobOnLayerDepend(job, onjob, onlayer): """Creates a job on layer dependency @type job: string @@ -183,11 +174,12 @@ def createJobOnLayerDepend(job, onjob, onlayer): __is_valid(onjob, ERR_INVALID_ON_JOB) __is_valid(onlayer, ERR_INVALID_ON_LAYER) - logger.debug("creating jol depend from %s to %s/%s" % (job, onjob, onlayer)) + logger.debug("creating jol depend from %s to %s/%s", job, onjob, onlayer) depend_er_job = opencue.api.findJob(job) depend_on_layer = opencue.api.findLayer(onjob, onlayer) return depend_er_job.createDependencyOnLayer(depend_on_layer) + def createJobOnFrameDepend(job, onjob, onlayer, onframe): """Creates a job on frame dependency @type job: string @@ -205,12 +197,12 @@ def createJobOnFrameDepend(job, onjob, onlayer, onframe): __is_valid(onlayer, ERR_INVALID_ON_LAYER) __is_valid(onframe, ERR_INVALID_ON_FRAME) - logger.debug("creating jof depend from %s to %s/%s-%04d" - % (job, onjob, onlayer, onframe)) + logger.debug("creating jof depend from %s to %s/%s-%04d", job, onjob, onlayer, onframe) depend_er_job = opencue.api.findJob(job) depend_on_frame = opencue.api.findFrame(onjob, onlayer, onframe) return depend_er_job.createDependencyOnFrame(depend_on_frame) + def createLayerOnJobDepend(job, layer, onjob): """Creates a layer on job dependency @type job: string @@ -226,10 +218,11 @@ def createLayerOnJobDepend(job, layer, onjob): __is_valid(layer, ERR_INVALID_ER_LAYER) __is_valid(onjob, ERR_INVALID_ON_JOB) - logger.debug("creating loj depend from %s/%s to %s" % (job, layer, onjob)) + logger.debug("creating loj depend from %s/%s to %s", job, layer, onjob) depend_er_layer = opencue.api.findLayer(job, layer) return depend_er_layer.createDependencyOnJob(opencue.api.findJob(onjob)) + def createLayerOnLayerDepend(job, layer, onjob, onlayer): """Creates a layer on layer dependency @type job: string @@ -248,12 +241,12 @@ def createLayerOnLayerDepend(job, layer, onjob, onlayer): __is_valid(onjob, ERR_INVALID_ON_JOB) __is_valid(onlayer, ERR_INVALID_ON_LAYER) - logger.debug("creating lol depend from %s/%s to %s/%s" - % (job, layer, onjob, onlayer)) + logger.debug("creating lol depend from %s/%s to %s/%s", job, layer, onjob, onlayer) depend_er_layer = opencue.api.findLayer(job,layer) depend_on_layer = opencue.api.findLayer(onjob, onlayer) return depend_er_layer.createDependencyOnLayer(depend_on_layer) + def createLayerOnFrameDepend(job, layer, onjob, onlayer, onframe): """Creates a layer on frame dependency @type job: string @@ -275,12 +268,13 @@ def createLayerOnFrameDepend(job, layer, onjob, onlayer, onframe): __is_valid(onlayer, ERR_INVALID_ON_LAYER) __is_valid(onframe, ERR_INVALID_ON_FRAME) - logger.debug("creating lof depend from %s/%s to %s/%s-%04d" - % (job, layer, onjob, onlayer, onframe)) + logger.debug( + "creating lof depend from %s/%s to %s/%s-%04d", job, layer, onjob, onlayer, onframe) depend_er_layer = opencue.api.findLayer(job,layer) depend_on_frame = opencue.api.findFrame(onjob, onlayer, onframe) return depend_er_layer.createDependencyOnFrame(depend_on_frame) + def createFrameOnJobDepend(job, layer, frame, onjob): """Creates a frame on job dependency @type job: string @@ -299,11 +293,11 @@ def createFrameOnJobDepend(job, layer, frame, onjob): __is_valid(frame, ERR_INVALID_ER_FRAME) __is_valid(onjob, ERR_INVALID_ON_JOB) - logger.debug("creating foj depend from %s/%s-%04d to %s" - % (job, layer, frame, onjob)) + logger.debug("creating foj depend from %s/%s-%04d to %s", job, layer, frame, onjob) depend_er_frame = opencue.api.findFrame(job, layer, frame) return depend_er_frame.createDependencyOnJob(opencue.api.findJob(onjob)) + def createFrameOnLayerDepend(job, layer, frame, onjob, onlayer): """Creates a frame on layer dependency @type job: string @@ -324,12 +318,12 @@ def createFrameOnLayerDepend(job, layer, frame, onjob, onlayer): __is_valid(onjob, ERR_INVALID_ON_JOB) __is_valid(onlayer, ERR_INVALID_ON_LAYER) - logger.debug("creating fol depend from %s/%s-%04d to %s/%s" - % (job, layer, frame, onjob, onlayer)) + logger.debug("creating fol depend from %s/%s-%04d to %s/%s", job, layer, frame, onjob, onlayer) depend_er_frame = opencue.api.findFrame(job, layer, frame) depend_on_layer = opencue.api.findLayer(onjob, onlayer) return depend_er_frame.createDependencyOnLayer(depend_on_layer) + def createFrameOnFrameDepend(job, layer, frame, onjob, onlayer, onframe): """Creates a frame on frame dependency @type job: string @@ -354,12 +348,14 @@ def createFrameOnFrameDepend(job, layer, frame, onjob, onlayer, onframe): __is_valid(onlayer, ERR_INVALID_ON_LAYER) __is_valid(onframe, ERR_INVALID_ON_FRAME) - logger.debug("creating fof depend from %s/%s-%04d to %s/%s-%04d" - % (job, layer, frame, onjob, onlayer,onframe)) + logger.debug( + "creating fof depend from %s/%s-%04d to %s/%s-%04d", + job, layer, frame, onjob, onlayer, onframe) depend_er_frame = opencue.api.findFrame(job, layer, frame) depend_on_frame = opencue.api.findFrame(onjob, onlayer, onframe) return depend_er_frame.createDependencyOnFrame(depend_on_frame) + def createFrameByFrameDepend(job, layer, onjob, onlayer): """Creates a frame by frame dependency @type job: string @@ -378,13 +374,13 @@ def createFrameByFrameDepend(job, layer, onjob, onlayer): __is_valid(onjob, ERR_INVALID_ON_JOB) __is_valid(onlayer, ERR_INVALID_ON_LAYER) - logger.debug("creating fbf depend from %s/%s to %s/%s" - % (job, layer, onjob, onlayer)) + logger.debug("creating fbf depend from %s/%s to %s/%s", job, layer, onjob, onlayer) depend_er_layer = opencue.api.findLayer(job, layer) depend_on_layer = opencue.api.findLayer(onjob, onlayer) return depend_er_layer.createFrameByFrameDependency(depend_on_layer, False) + def createLayerOnSimFrameDepend(job, layer, onjob, onlayer, onframe): """Creates a layer on sim frame dependency @type job: string @@ -406,22 +402,19 @@ def createLayerOnSimFrameDepend(job, layer, onjob, onlayer, onframe): __is_valid(onlayer, ERR_INVALID_ON_LAYER) __is_valid(onframe, ERR_INVALID_ON_FRAME) - logger.debug("creating los depend from %s/%s to %s/%s-%04d" - % (job, layer, onjob, onlayer, onframe)) + logger.debug( + "creating los depend from %s/%s to %s/%s-%04d", job, layer, onjob, onlayer, onframe) depend_er_layer = opencue.api.findLayer(job,layer) depend_on_frame = opencue.api.findFrame(onjob, onlayer, onframe) - # if createSimDependencyOnFrame existed, we would use it here: - #return depend_er_layer.createSimDependencyOnFrame(depend_on_frame) - depends = [] for depend_er_frame in depend_er_layer.getFrames(): depends.append(depend_er_frame.createDependencyOnFrame(depend_on_frame)) return depends -def dropDepend(id): - """deactivates a dependency by GUID - @type id: string - @param id: the GUID of the dependency""" - opencue.api.getDepend(id).satisfy() +def dropDepend(depend_id): + """deactivates a dependency by GUID + @type depend_id: string + @param depend_id: the GUID of the dependency""" + opencue.api.getDepend(depend_id).satisfy() diff --git a/cuegui/cuegui/DarkPalette.py b/cuegui/cuegui/DarkPalette.py index 304f334a0..2b15bd66a 100644 --- a/cuegui/cuegui/DarkPalette.py +++ b/cuegui/cuegui/DarkPalette.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -The dark widget color scheme used by image viewing applications. -""" +"""The dark widget color scheme used by image viewing applications.""" from __future__ import absolute_import @@ -34,6 +32,7 @@ def init(): """Convenience function that takes the QApplication object for the application and configures the palette and style for the Plastique color scheme""" + # pylint: disable=no-member QtGui.qApp.setPalette(DarkPalette()) if platform.system() in ['Darwin', 'Linux']: setDarkStyleSheet() @@ -44,13 +43,13 @@ def init(): def setDarkStyleSheet(): + """Sets the stylesheet.""" + # pylint: disable=no-member QtGui.qApp.setStyleSheet(open(cuegui.Constants.DARK_STYLE_SHEET).read()) def DarkPalette(): - """The dark widget color scheme used by image viewing applications - at Imageworks. - """ + """The dark widget color scheme used by image viewing applications.""" p = QtGui.QPalette() c = GreyF(0.175) @@ -103,12 +102,14 @@ def DarkPalette(): def GreyF(value): + """Creates a grey color.""" c = QtGui.QColor() c.setRgbF(value, value, value) return c def ColorF(r, g, b): + """Creates an RGB color.""" c = QtGui.QColor() c.setRgbF(r, g, b) return c diff --git a/cuegui/cuegui/DependDialog.py b/cuegui/cuegui/DependDialog.py index 26e78e333..ce94f526f 100644 --- a/cuegui/cuegui/DependDialog.py +++ b/cuegui/cuegui/DependDialog.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Dialog displaying a list of dependencies for an object.""" + + from __future__ import absolute_import from __future__ import print_function from __future__ import division @@ -29,7 +32,9 @@ class DependDialog(QtWidgets.QDialog): - def __init__(self, object, parent=None): + """Dialog displaying a list of dependencies for an object.""" + + def __init__(self, rpcOject, parent=None): super(DependDialog, self).__init__(parent) self.setAttribute(QtCore.Qt.WA_DeleteOnClose) self.setSizeGripEnabled(True) @@ -37,18 +42,18 @@ def __init__(self, object, parent=None): self.resize(1000, 600) name = "Dependencies for " - if cuegui.Utils.isJob(object): - name += "Job: %s" % object.data.name - elif cuegui.Utils.isLayer(object): - name += "Layer: %s" % object.data.name - elif cuegui.Utils.isFrame(object): - name += "Frame: %s" % object.data.name + if cuegui.Utils.isJob(rpcOject): + name += "Job: %s" % rpcOject.data.name + elif cuegui.Utils.isLayer(rpcOject): + name += "Layer: %s" % rpcOject.data.name + elif cuegui.Utils.isFrame(rpcOject): + name += "Frame: %s" % rpcOject.data.name self.setWindowTitle(name) self.hlayout = QtWidgets.QHBoxLayout(self) - self._depend = cuegui.DependMonitorTree.DependMonitorTree(self, object) + self._depend = cuegui.DependMonitorTree.DependMonitorTree(self, rpcOject) self.hlayout.addWidget(self._depend) self.setLayout(self.hlayout) diff --git a/cuegui/cuegui/DependMonitorTree.py b/cuegui/cuegui/DependMonitorTree.py index 7516a1347..fed246a36 100644 --- a/cuegui/cuegui/DependMonitorTree.py +++ b/cuegui/cuegui/DependMonitorTree.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Tree for displaying a list of depends.""" + + from __future__ import absolute_import from __future__ import print_function from __future__ import division @@ -22,6 +25,7 @@ from PySide2 import QtWidgets from opencue.compiled_proto import depend_pb2 +import opencue.exception import cuegui.AbstractTreeWidget import cuegui.AbstractWidgetItem @@ -35,40 +39,35 @@ class DependMonitorTree(cuegui.AbstractTreeWidget.AbstractTreeWidget): - def __init__(self, parent, object): + """Tree for displaying a list of depends.""" + + def __init__(self, parent, rpcObject): self.startColumnsForType(cuegui.Constants.TYPE_DEPEND) self.addColumn("Type", 130, id=1, data=lambda depend: depend_pb2.DependType.Name(depend.type())) self.addColumn("Target", 60, id=2, data=lambda depend: depend_pb2.DependTarget.Name(depend.target())) self.addColumn("Active", 50, id=3, - data=lambda depend:(depend.isActive())) -# self.addColumn("Job", 230, id=4, -# data=lambda depend:(depend.dependErJob())) -# self.addColumn("Layer", 50, id=5, -# data=lambda depend:(depend.dependErLayer())) -# self.addColumn("Frame", 100, id=6, -# data=lambda depend:(depend.dependErFrame())) + data=lambda depend: (depend.isActive())) self.addColumn("OnJob", 300, id=7, - data=lambda depend:(depend.dependOnJob())) + data=lambda depend: (depend.dependOnJob())) self.addColumn("OnLayer", 200, id=8, - data=lambda depend:(depend.dependOnLayer())) + data=lambda depend: (depend.dependOnLayer())) self.addColumn("OnFrame", 100, id=9, - data=lambda depend:(depend.dependOnFrame())) + data=lambda depend: (depend.dependOnFrame())) - self.rpcObject = object + self.rpcObject = rpcObject cuegui.AbstractTreeWidget.AbstractTreeWidget.__init__(self, parent) - # Used to build right click context menus self.__menuActions = cuegui.MenuActions.MenuActions( self, self.updateSoon, self.selectedObjects) self.setUpdateInterval(60) - def _createItem(self, object): + def _createItem(self, rpcObject): """Creates and returns the proper item""" - return DependWidgetItem(object, self) + return DependWidgetItem(rpcObject, self) def _getUpdate(self): """Returns the proper data from the cuebot""" @@ -76,7 +75,7 @@ def _getUpdate(self): if hasattr(self.rpcObject, "getDepends"): return self.rpcObject.getDepends() return self.rpcObject.getWhatThisDependsOn() - except Exception as e: + except opencue.exception.CueException as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) return [] @@ -86,13 +85,16 @@ def contextMenuEvent(self, e): menu = QtWidgets.QMenu() self.__menuActions.dependencies().addAction(menu, "satisfy") - #self.__menuActions.dependencies().addAction(menu, "unsatisfy") menu.exec_(e.globalPos()) -################################################################################ + def tick(self): + pass + class DependWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): - def __init__(self, object, parent): + """Widget item for displaying a single depend.""" + + def __init__(self, rpcObject, parent): cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( - self, cuegui.Constants.TYPE_DEPEND, object, parent) \ No newline at end of file + self, cuegui.Constants.TYPE_DEPEND, rpcObject, parent) diff --git a/cuegui/cuegui/DependWizard.py b/cuegui/cuegui/DependWizard.py index 1c31721da..6910ee685 100644 --- a/cuegui/cuegui/DependWizard.py +++ b/cuegui/cuegui/DependWizard.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -Wizard interface to setting up dependencies. -""" +"""Wizard interface for setting up dependencies.""" from __future__ import absolute_import @@ -84,17 +82,27 @@ PROGRESS_TEXT = "Are you sure you want to cancel setting up these dependencies?\n\n" + \ "The dependencies that are already partially setup will still remain." + class DependWizard(QtWidgets.QWizard): - def __init__(self, parent, jobs, layers = [], frames = []): + """Wizard interface for setting up dependencies.""" + + def __init__(self, parent, jobs, layers=None, frames=None): QtWidgets.QWizard.__init__(self, parent) # Only allow jobs from one show jobs = [job for job in jobs if job.data.show == jobs[0].data.show] self.jobs = jobs - self.layers = [layer.data.name for layer in layers] - self.layerOptions = layers - self.frames = [frame.data.name for frame in frames] + if layers is None: + self.layers = [] + self.layerOptions = [] + else: + self.layers = [layer.data.name for layer in layers] + self.layerOptions = layers + if frames is None: + self.frames = [] + else: + self.frames = [frame.data.name for frame in frames] self.dependType = None self.onJobOptions = [] @@ -104,14 +112,14 @@ def __init__(self, parent, jobs, layers = [], frames = []): self.onFrame = [] # Create the pages - self.__pages = {} - self.__pages [PAGE_SELECT_DEPEND_TYPE] = PageDependType(self, jobs, layers, frames) - self.__pages [PAGE_SELECT_JOB_LAYER] = PageSelectLayer(self) - self.__pages [PAGE_SELECT_JOB_FRAME] = PageSelectFrame(self) - self.__pages [PAGE_SELECT_ONJOB] = PageSelectOnJob(self) - self.__pages [PAGE_SELECT_ONLAYER] = PageSelectOnLayer(self) - self.__pages [PAGE_SELECT_ONFRAME] = PageSelectOnFrame(self) - self.__pages [PAGE_CONFIRMATION] = PageConfirmation(self, jobs, layers, frames) + self.__pages = {} + self.__pages[PAGE_SELECT_DEPEND_TYPE] = PageDependType(self, jobs, layers, frames) + self.__pages[PAGE_SELECT_JOB_LAYER] = PageSelectLayer(self) + self.__pages[PAGE_SELECT_JOB_FRAME] = PageSelectFrame(self) + self.__pages[PAGE_SELECT_ONJOB] = PageSelectOnJob(self) + self.__pages[PAGE_SELECT_ONLAYER] = PageSelectOnLayer(self) + self.__pages[PAGE_SELECT_ONFRAME] = PageSelectOnFrame(self) + self.__pages[PAGE_CONFIRMATION] = PageConfirmation(self, jobs, layers, frames) # Add the pages to the wizard for key in self.__pages : @@ -128,27 +136,27 @@ def __init__(self, parent, jobs, layers = [], frames = []): self.show() def _onJobOptionsPopulate(self): - """Populates self.onJobOptions to contain a list of job names for the - given jobs show""" + """Populates self.onJobOptions to contain a list of job names for the given job's show.""" self.onJobOptions = [] try: show = self.jobs[0].data.name.split('-')[0] self.onJobOptions = [name for name in sorted(opencue.api.getJobNames()) if name.startswith(show)] - except Exception as e: + except opencue.exception.CueException as e: logger.critical("Failed getting list of jobs") list(map(logger.critical, cuegui.Utils.exceptionOutput(e))) -################################################################################ class AbstractWizardPage(QtWidgets.QWizardPage): + """Base class for the depend wizard pages.""" + def __init__(self, parent): QtWidgets.QWizardPage.__init__(self, parent) self.setLayout(QtWidgets.QGridLayout(self)) self._widgets = [] - def _addLabel(self, text, row, col, rowSpan = 1, columnSpan = 1, align = QtCore.Qt.AlignLeft): + def _addLabel(self, text, row, col, rowSpan=1, columnSpan=1, align=QtCore.Qt.AlignLeft): """Adds a QLabel to the current WizardPage @type text: str @param text: The text to display in the edit box @@ -170,7 +178,7 @@ def _addLabel(self, text, row, col, rowSpan = 1, columnSpan = 1, align = QtCore. self._widgets.append(label) return label - def _addTextEdit(self, row, col, text, height = None): + def _addTextEdit(self, row, col, text, height=None): """Adds a QTextEdit to the current WizardPage @type row: int @param row: The row to place the widget @@ -205,22 +213,22 @@ def _addLineEdit(self, row, col, text): self._widgets.append(edit) return edit - def _addSpinBox(self, row, col, min, max, value): + def _addSpinBox(self, row, col, min_val, max_val, value): """Adds a line edit box to the current WizardPage @type row: int @param row: The row to place the widget @type col: int @param col: The column to place the widget - @type min: int - @param min: The minimum number to allow - @type max: int - @param max: The maximum number to allow + @type min_val: int + @param min_val: The minimum number to allow + @type max_val: int + @param max_val: The maximum number to allow @type value: int @param value: The value to display initially @rtype: QLineEdit @return: A reference to the new widget""" spin = QtWidgets.QSpinBox(self) - spin.setRange(min, max) + spin.setRange(min_val, max_val) spin.setValue(value) self.layout().addWidget(spin, row, col) self._widgets.append(spin) @@ -239,7 +247,7 @@ def _addCombo(self, row, col): self._widgets.append(combo) return combo - def _addListWidget(self, row, col, selection = None): + def _addListWidget(self, row, col, selection=None): """Adds a QListWidget to the current WizardPage. @type row: int @param row: The row to place the widget @@ -249,14 +257,15 @@ def _addListWidget(self, row, col, selection = None): @param selection: Allowed selection type @rtype: QListWidget @return: A reference to the new widget""" - list = QtWidgets.QListWidget(self) + list_widget = QtWidgets.QListWidget(self) if selection: - list.setSelectionMode(selection) - self.layout().addWidget(list, row, col) - self._widgets.append(list) - return list + list_widget.setSelectionMode(selection) + self.layout().addWidget(list_widget, row, col) + self._widgets.append(list_widget) + return list_widget - def _getNames(self, items): + @staticmethod + def _getNames(items): """Returns a list of names for all items provided. @type items: str, list, list, list or list @param items: Any items to return the names of @@ -266,9 +275,13 @@ def _getNames(self, items): return [] if isinstance(items, str): return [items] - return [item.data.name for item in items if hasattr(item, "data") and hasattr(item.data, "name")] + \ - [str(item) for item in items if not hasattr(item, "data")] + names_of_items_with_data = [ + item.data.name for item in items + if hasattr(item, "data") and hasattr(item.data, "name")] + names_of_items_without_data = [str(item) for item in items if not hasattr(item, "data")] + return names_of_items_with_data + names_of_items_without_data + # pylint: disable=inconsistent-return-statements def _displayItems(self, name, items, row): """Displays a label description and a list of items. If more than one item is given @@ -309,20 +322,26 @@ def _removeAllWidgets(self): self._widgets.remove(widget) widget.hide() -################################################################################ class PageDependType(AbstractWizardPage): - """This page asks the user for the type of dependency to setup + """This page asks the user for the type of dependency to create. + PAGE_SELECT_DEPEND_TYPE""" - def __init__(self, parent, jobs, layers = [], frames = []): + + def __init__(self, parent, jobs, layers=None, frames=None): AbstractWizardPage.__init__(self, parent) self.setTitle("Select Dependency Type") - # this should come from a field self.jobs = jobs - self.layers = layers - self.frames = frames + if layers is None: + self.layers = [] + else: + self.layers = layers + if frames is None: + self.frames = [] + else: + self.frames = frames self._displayItems("Job", jobs, 0) self._displayItems("Layer", layers, 1) @@ -353,13 +372,14 @@ def __init__(self, parent, jobs, layers = [], frames = []): self.layout().addWidget(self.__groupBox, 3, 0, 1, -1) + # pylint: disable=inconsistent-return-statements def __msg(self): for item in [("frame", self.wizard().frames), ("layer", self.wizard().layers), ("job", self.wizard().jobs)]: if len(item[1]) > 1: return "these %ss" % item[0] - elif item[1]: + if item[1]: return "this %s" % item[0] def initializePage(self): @@ -381,27 +401,29 @@ def nextId(self): @rtype: int""" if not self.wizard().dependType: return PAGE_SELECT_DEPEND_TYPE - elif self.frames: + if self.frames: return PAGE_SELECT_ONJOB - elif len(self.layers) == 1 and \ + if len(self.layers) == 1 and \ self.wizard().dependType in (FOJ, FOL, FOF): return PAGE_SELECT_JOB_FRAME - elif self.layers: + if self.layers: return PAGE_SELECT_ONJOB - elif len(self.jobs) == 1 and \ - self.wizard().dependType in (LOJ, LOL, LOF, FOJ, FOL, FOF, FBF, LOS): + if len(self.jobs) == 1 and \ + self.wizard().dependType in (LOJ, LOL, LOF, FOJ, FOL, FOF, FBF, LOS): return PAGE_SELECT_JOB_LAYER - elif self.jobs: + if self.jobs: return PAGE_SELECT_ONJOB - else: - logger.critical("error, no place to go: jobs:%s layers:%s frames:%s type:%s" % (len(self.jobs), len(self.layers), len(self.frames), self.wizard().dependType)) - raise RuntimeError() + logger.critical( + "error, no place to go: jobs:%s layers:%s frames:%s type:%s", + len(self.jobs), len(self.layers), len(self.frames), self.wizard().dependType) + raise RuntimeError() -################################################################################ class PageSelectLayer(AbstractWizardPage): - """This page asks the user for the layer that should depend on something + """This page asks the user for the layer that should depend on something. + PAGE_SELECT_JOB_LAYER""" + def __init__(self, parent): AbstractWizardPage.__init__(self, parent) @@ -418,10 +440,11 @@ def initializePage(self): QtWidgets.QWizardPage.initializePage(self) self.__layerList.clear() - self.__layerList.addItems([layer for layer in self._getNames(self.wizard().layerOptions)]) + self.__layerList.addItems(self._getNames(self.wizard().layerOptions)) for num in range(self.__layerList.count()): - self.__layerList.item(num).setSelected(str(self.__layerList.item(num).text()) in self._getNames(self.wizard().onLayer)) + self.__layerList.item(num).setSelected( + str(self.__layerList.item(num).text()) in self._getNames(self.wizard().onLayer)) def validatePage(self): self.wizard().layers = [] @@ -431,11 +454,10 @@ def validatePage(self): if self.wizard().layers: return True - QtWidgets.QMessageBox.warning(self, - "Warning", - "Please select one or more layers or go back " - "and change the dependency type", - QtWidgets.QMessageBox.Ok) + QtWidgets.QMessageBox.warning( + self, "Warning", + "Please select one or more layers or go back and change the dependency type", + QtWidgets.QMessageBox.Ok) return False def nextId(self): @@ -444,14 +466,14 @@ def nextId(self): @rtype: int""" if self.wizard().dependType in (FOJ, FOL, FOF): return PAGE_SELECT_JOB_FRAME - else: - return PAGE_SELECT_ONJOB + return PAGE_SELECT_ONJOB -################################################################################ class PageSelectFrame(AbstractWizardPage): - """This page asks the user for the frames that should depend on something + """This page asks the user for the frames that should depend on something. + PAGE_SELECT_JOB_FRAME""" + def __init__(self, parent): AbstractWizardPage.__init__(self, parent) @@ -468,6 +490,7 @@ def initializePage(self): def validatePage(self): frames = str(self.field("frame")) if frames: + # pylint: disable=broad-except try: fs = FileSequence.FrameSet(frames) fs.normalize() @@ -483,11 +506,12 @@ def nextId(self): @rtype: int""" return PAGE_SELECT_ONJOB -################################################################################ class PageSelectOnJob(AbstractWizardPage): - """This page asks the user for the job should be depended on + """This page asks the user for the job that should be depended on. + PAGE_SELECT_ONJOB""" + def __init__(self, parent): AbstractWizardPage.__init__(self, parent) @@ -502,14 +526,18 @@ def __init__(self, parent): self.__jobList = self._addListWidget(3, 0) def filterJobs(self, text): - # Exlcude job names that would cause a job to depend on itself + """Pre-filters the list of possible jobs. + + Excludes job names that would cause a job to depend on itself.""" exclude = [] if self.wizard().dependType in (JOJ, LOJ, FOJ, JFBF): for job in self.wizard().jobs: exclude.append(job.data.name) self.__jobList.clear() - self.__jobList.addItems([job for job in self.wizard().onJobOptions if re.search(str(text), job, re.IGNORECASE) and not job in exclude]) + self.__jobList.addItems( + [job for job in self.wizard().onJobOptions + if re.search(str(text), job, re.IGNORECASE) and job not in exclude]) def initializePage(self): # If the filter edit box is empty, populate it with SHOW-SHOT-USER_ @@ -523,7 +551,8 @@ def initializePage(self): self.__jobList.setSelectionMode(QtWidgets.QAbstractItemView.SingleSelection) for num in range(self.__jobList.count()): - self.__jobList.item(num).setSelected(str(self.__jobList.item(num).text()) in self.wizard().onJob) + self.__jobList.item(num).setSelected( + str(self.__jobList.item(num).text()) in self.wizard().onJob) QtWidgets.QWizardPage.initializePage(self) @@ -548,14 +577,14 @@ def nextId(self): @rtype: int""" if self.wizard().dependType in (JOL, JOF, LOL, LOF, FOL, FOF, FBF, LOS): return PAGE_SELECT_ONLAYER - else: - return PAGE_CONFIRMATION + return PAGE_CONFIRMATION -################################################################################ class PageSelectOnLayer(AbstractWizardPage): - """This page asks the user for the layer should be depended on: + """This page asks the user for the layer that should be depended on. + PAGE_SELECT_ONLAYER""" + def __init__(self, parent): AbstractWizardPage.__init__(self, parent) @@ -571,7 +600,11 @@ def initializePage(self): self.wizard().onLayerOptions = opencue.api.findJob(self.wizard().onJob[0]).getLayers() if self.wizard().dependType in (LOS,): - self.wizard().onLayerOptions = [layer for layer in self.wizard().onLayerOptions if 'simulation' in layer.data.services or 'simulationhi' in layer.data.services or 'houdini' in layer.data.services] + self.wizard().onLayerOptions = [ + layer for layer in self.wizard().onLayerOptions + if 'simulation' in layer.data.services or + 'simulationhi' in layer.data.services or + 'houdini' in layer.data.services] if self.wizard().dependType in (JOL, LOL, FOL, FBF, JOF, LOF, FOF): self.__onLayerList.setSelectionMode(QtWidgets.QAbstractItemView.MultiSelection) @@ -579,10 +612,11 @@ def initializePage(self): self.__onLayerList.setSelectionMode(QtWidgets.QAbstractItemView.SingleSelection) self.__onLayerList.clear() - self.__onLayerList.addItems([layer for layer in self._getNames(self.wizard().onLayerOptions)]) + self.__onLayerList.addItems(self._getNames(self.wizard().onLayerOptions)) for num in range(self.__onLayerList.count()): - self.__onLayerList.item(num).setSelected(str(self.__onLayerList.item(num).text()) in self._getNames(self.wizard().onLayer)) + self.__onLayerList.item(num).setSelected( + str(self.__onLayerList.item(num).text()) in self._getNames(self.wizard().onLayer)) def validatePage(self): self.wizard().onLayer = [] @@ -592,11 +626,10 @@ def validatePage(self): if self.wizard().onLayer: return True - QtWidgets.QMessageBox.warning(self, - "Warning", - "Please select one or more layers or go back " - "and change the dependency type", - QtWidgets.QMessageBox.Ok) + QtWidgets.QMessageBox.warning( + self, "Warning", + "Please select one or more layers or go back and change the dependency type", + QtWidgets.QMessageBox.Ok) return False def nextId(self): @@ -605,14 +638,14 @@ def nextId(self): @rtype: int""" if self.wizard().dependType in (JOF, LOF, FOF, LOS): return PAGE_SELECT_ONFRAME - else: - return PAGE_CONFIRMATION + return PAGE_CONFIRMATION -################################################################################ class PageSelectOnFrame(AbstractWizardPage): - """This page asks the user for the frame should be depended on: + """This page asks the user for the frame that should be depended on. + PAGE_SELECT_ONFRAME""" + def __init__(self, parent): AbstractWizardPage.__init__(self, parent) @@ -631,6 +664,7 @@ def initializePage(self): def validatePage(self): frames = str(self.field("onFrame")) if frames: + # pylint: disable=broad-except try: fs = FileSequence.FrameSet(frames) fs.normalize() @@ -646,14 +680,21 @@ def nextId(self): @rtype: int""" return PAGE_CONFIRMATION -################################################################################ class PageConfirmation(AbstractWizardPage): - """ + """Page to collect final confirmation of depend details before creating it. + PAGE_CONFIRMATION""" + def __init__(self, parent, jobs, layers, frames): + del jobs + del layers + del frames + AbstractWizardPage.__init__(self, parent) + self.work = [] + self.setTitle("Confirmation") self.setSubTitle("Are you sure?") @@ -680,6 +721,7 @@ def initializePage(self): if self.wizard().dependType in (JOF, LOF, FOF, LOS): self._displayItems("Frame", self.wizard().onFrame, 9) + # pylint: disable=too-many-nested-blocks def validatePage(self): # Just names: jobs = self._getNames(self.wizard().jobs) @@ -702,16 +744,11 @@ def validatePage(self): self.__addDependWork(layer, onLayer) cuegui.ProgressDialog.ProgressDialog( - "Setting up Hard Depend", - self.__createFrameByFrameDepend, - self.work, - 2, - PROGRESS_TITLE, - PROGRESS_TEXT, - self.parent()) + "Setting up Hard Depend", self.__createFrameByFrameDepend, self.work, 2, + PROGRESS_TITLE, PROGRESS_TEXT, self.parent()) return True - elif frames: + if frames: for onJob in onJobs: for onLayer in onLayers: for framelayer in frames: @@ -721,41 +758,44 @@ def validatePage(self): frame = framelayer layer = layers[0] for onFrame in onFrames: - self.__addDependWork(self.wizard().dependType, jobs[0], layer, int(frame), onJob, onLayer, onFrame) + self.__addDependWork( + self.wizard().dependType, jobs[0], layer, int(frame), + onJob, onLayer, onFrame) elif layers: for onJob in onJobs: for onLayer in onLayers: for layer in layers: for onFrame in onFrames: - self.__addDependWork(self.wizard().dependType, jobs[0], layer, None, onJob, onLayer, onFrame) + self.__addDependWork( + self.wizard().dependType, jobs[0], layer, None, + onJob, onLayer, onFrame) elif jobs: for onJob in onJobs: for onLayer in onLayers: for job in jobs: for onFrame in onFrames: - self.__addDependWork(self.wizard().dependType, job, None, None, onJob, onLayer, onFrame) + self.__addDependWork( + self.wizard().dependType, job, None, None, onJob, onLayer, onFrame) cuegui.ProgressDialog.ProgressDialog( - "Setting up dependencies", - cuegui.Cuedepend.createDepend, - self.work, - 2, - PROGRESS_TITLE, - PROGRESS_TEXT, - self.parent()) + "Setting up dependencies", cuegui.Cuedepend.createDepend, self.work, 2, PROGRESS_TITLE, + PROGRESS_TEXT, self.parent()) return True def __addDependWork(self, *args): - """Adds arguements for a call to Cuedepend.createDepend to a list + """Adds arguments for a call to Cuedepend.createDepend to a list. + @type args: string, string, string, int, string, string, int @param args: The arguements required by Cuedepend.createDepend""" self.work.append(args) - def __createFrameByFrameDepend(self, layer, onLayer): + @staticmethod + def __createFrameByFrameDepend(layer, onLayer): """A function callback provided to the ProgressDialog that sets up a - frame by frame dependency + frame by frame dependency. + @type layer: opencue.wrappers.layer.Layer @param layer: The layer that contains the frames that will have the dependency @type onLayer: opencue.wrappers.layer.Layer diff --git a/cuegui/cuegui/EmailDialog.py b/cuegui/cuegui/EmailDialog.py index e12ee777a..4775553d9 100644 --- a/cuegui/cuegui/EmailDialog.py +++ b/cuegui/cuegui/EmailDialog.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -Displays the email dialog when emailing an artist. -""" +"""Dialog for emailing a job owner.""" from __future__ import absolute_import @@ -55,12 +53,14 @@ class EmailDialog(QtWidgets.QDialog): - def __init__(self, job, format, parent = None): + """Dialog for emailing a job owner.""" + + def __init__(self, job, parent=None): QtWidgets.QDialog.__init__(self, parent) try: self.__frames = job.getFrames(state=[opencue.api.job_pb2.DEAD]) - except: + except opencue.exception.CueException: self.__frames = [] self.setWindowTitle("Email For: %s" % job.data.name) @@ -68,7 +68,7 @@ def __init__(self, job, format, parent = None): self.setSizeGripEnabled(True) self.setFixedSize(1000,600) - self.__email = EmailWidget(job, format, self) + self.__email = EmailWidget(job, self) self.__appendDeadFrameInfo(job) self.__logView = LogViewWidget(job, self.__frames, self) @@ -96,13 +96,13 @@ def __appendDeadFrameInfo(self, job): cuegui.Utils.secondsToHHMMSS(frame.runTime()), frame.retries())) i_total_render_time += frame.retries() * frame.runTime() i_total_retries += frame.retries() - try: - self.__email.appendToBody("\nEstimated Proc Hours: %0.2f\n\n" % \ - ((i_total_render_time / 3600.0))) - except: - pass + self.__email.appendToBody( + "\nEstimated Proc Hours: %0.2f\n\n" % (i_total_render_time / 3600.0)) + class LogViewWidget(QtWidgets.QWidget): + """Widget for displaying a log within the email dialog.""" + def __init__(self, job, frames, parent=None): QtWidgets.QWidget.__init__(self, parent) QtWidgets.QVBoxLayout(self) @@ -133,16 +133,19 @@ def __init__(self, job, frames, parent=None): self.__sel_frames.activated.connect(self.switchLogEvent) self.__txt_find.returnPressed.connect(self.findEvent) + # pylint: disable=inconsistent-return-statements def __getFrame(self, name): for frame in self.__frames: if frame.data.name == name: return frame def switchLogEvent(self, str_frame): + """Displays the log for the given frame.""" + # pylint: disable=broad-except try: self.__txt_log.clear() log_file_path = cuegui.Utils.getFrameLogFile(self.__job, self.__getFrame(str_frame)) - fp = open(log_file_path,"r") + fp = open(log_file_path, "r") if os.path.getsize(log_file_path) > 1242880: fp.seek(0, 2) fp.seek(-1242880, 1) @@ -155,7 +158,7 @@ def switchLogEvent(self, str_frame): except Exception as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) - logger.info("error loading frame: %s, %s" % (str_frame, e)) + logger.info("error loading frame: %s, %s", str_frame, e) def findEvent(self): """attempts to find the text from the find text box, @@ -164,17 +167,19 @@ def findEvent(self): cursor = document.find( str(self.__txt_find.text()).strip(), self.__txt_log.textCursor().position(), - QtWidgets.QTextDocument.FindBackward) + QtGui.QTextDocument.FindBackward) if cursor.position() > 1: self.__txt_log.setTextCursor(cursor) + class EmailWidget(QtWidgets.QWidget): + """Widget for displaying an email form.""" send = QtCore.Signal() cancel = QtCore.Signal() - def __init__(self, job, format, parent=None): - QtWidgets.QWidget.__init__(self) + def __init__(self, job, parent=None): + QtWidgets.QWidget.__init__(self, parent=parent) self.__job = job @@ -210,7 +215,7 @@ def __init__(self, job, format, parent=None): self.__email_bcc = QtWidgets.QLineEdit(__default_bcc, self) self.__email_subject = QtWidgets.QLineEdit(__default_subject, self) - # Main Virtical Layout + # Main Vertical Layout vlayout = QtWidgets.QVBoxLayout(self) # Top Grid Layout @@ -246,6 +251,7 @@ def __init__(self, job, format, parent=None): self.__btnCancel.clicked.connect(self.cancel.emit) def giveFocus(self): + """Initializes widget state when the widget gains focus.""" self.__email_body.setFocus(QtCore.Qt.OtherFocusReason) self.__email_body.moveCursor(QtGui.QTextCursor.Start) self.__email_body.moveCursor(QtGui.QTextCursor.Down) @@ -254,30 +260,39 @@ def giveFocus(self): self.__email_body.moveCursor(QtGui.QTextCursor.Down) def email_from(self): + """Gets the email sender.""" return "%s" % self.__email_from.text() def email_to(self): + """Gets the email recipient.""" return "%s" % self.__email_to.text() def email_cc(self): + """Gets the email CC field.""" return "%s" % self.__email_cc.text() def email_bcc(self): + """Gets the email BCC field.""" return "%s" % self.__email_bcc.text() def email_subject(self): + """Gets the email subject.""" return "%s" % self.__email_subject.text() def email_body(self): + """Get the email body text.""" return "%s" % self.__email_body.toPlainText().toAscii() def appendToBody(self, txt): + """Appends text to the email body.""" self.__email_body.append(txt) def setBody(self, txt): + """Sets the value of the email body.""" self.__email_body.setText(txt) def sendEmail(self): + """Sends the email.""" self.send.emit() msg = MIMEText(self.email_body()) diff --git a/cuegui/cuegui/FilterDialog.py b/cuegui/cuegui/FilterDialog.py index b121be897..f130eb77e 100644 --- a/cuegui/cuegui/FilterDialog.py +++ b/cuegui/cuegui/FilterDialog.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -Handles the dialog to display/modify a show's filters, matchers and actions -""" +"""Dialog to display/modify a show's filters, matchers and actions.""" from __future__ import absolute_import @@ -55,6 +53,8 @@ class FilterDialog(QtWidgets.QDialog): + """Dialog to display/modify a show's filters, matchers and actions.""" + def __init__(self, show, parent=None): """ Creates an instance of the FilterDialog. @@ -130,16 +130,20 @@ def __createFilter(self): def __refresh(self): """Calls update on the widgets""" + # pylint: disable=protected-access self.__filters._update() self.__matchers._update() self.__actions._update() def __itemSingleClicked(self, item, col): - filter = item.rpcObject - self.__matchers.setObject(filter) - self.__actions.setObject(filter) + del col + self.__matchers.setObject(item.rpcObject) + self.__actions.setObject(item.rpcObject) + class FilterMonitorTree(cuegui.AbstractTreeWidget.AbstractTreeWidget): + """Tree displaying a list of filters.""" + def __init__(self, show, parent): self.startColumnsForType(cuegui.Constants.TYPE_FILTER) self.addColumn("Order", 100, id=1, @@ -163,9 +167,9 @@ def __init__(self, show, parent): self, self.updateSoon, self.selectedObjects) self._timer.stop() - def _createItem(self, object): - """Creates and returns the proper item""" - return FilterWidgetItem(object, self) + def _createItem(self, filter_object): + """Creates and returns a widget item for the given filter.""" + return FilterWidgetItem(filter_object, self) def _processUpdate(self, work, rpcObjects): """Adds the feature of forcing the items to be sorted by the first @@ -176,7 +180,7 @@ def _getUpdate(self): """Returns the proper data from the cuebot""" try: return self.__show.getFilters() - except Exception as e: + except opencue.exception.CueException as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) return [] @@ -196,8 +200,14 @@ def contextMenuEvent(self, e): menu.exec_(e.globalPos()) + def tick(self): + pass + + class MatcherMonitorTree(cuegui.AbstractTreeWidget.AbstractTreeWidget): - def __init__(self, filter, parent): + """Tree for displaying a list of filter matchers.""" + + def __init__(self, parent_filter, parent): self.startColumnsForType(cuegui.Constants.TYPE_MATCHER) self.addColumn("Matcher Subject", 130, id=1, data=lambda matcher:(matcher.subject())) @@ -207,7 +217,7 @@ def __init__(self, filter, parent): data=lambda matcher:(matcher.input())) self.addColumn("", 20, id=4) - self.__filter = filter + self.__filter = parent_filter cuegui.AbstractTreeWidget.AbstractTreeWidget.__init__(self, parent) @@ -216,26 +226,25 @@ def __init__(self, filter, parent): self, self.updateSoon, self.selectedObjects) self._timer.stop() - def setObject(self, object): + def setObject(self, matcher_object): """Sets the Matcher object to monitor - @type object: Matcher - @param object: The Matcher object to monitor""" - self.__filter = object + @type matcher_object: Matcher + @param matcher_object: The Matcher object to monitor""" + self.__filter = matcher_object self.sortByColumn(2, QtCore.Qt.AscendingOrder) self._update() - def _createItem(self, object): - """Creates and returns the proper item""" - item = MatcherWidgetItem(object, self) - + def _createItem(self, matcher_object): + """Creates and returns a widget item for the given matcher.""" + item = MatcherWidgetItem(matcher_object, self) return item def _getUpdate(self): - """Returns the proper data from the cuebot""" + """Returns the selected filter's matchers.""" try: if self.__filter: return self.__filter.getMatchers() - except Exception as e: + except opencue.exception.CueException as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) return [] @@ -262,24 +271,28 @@ def createMatcher(self): if not choice: return - (input, choice) = QtWidgets.QInputDialog.getText(self, "Create Matcher", - "Please enter the string to match", - QtWidgets.QLineEdit.Normal, "") + (matchQuery, choice) = QtWidgets.QInputDialog.getText( + self, + "Create Matcher", + "Please enter the string to match", + QtWidgets.QLineEdit.Normal, + "") if not choice: return self.addObject(self.__filter.createMatcher( opencue.compiled_proto.filter_pb2.MatchSubject.Value(str(matchSubject)), opencue.compiled_proto.filter_pb2.MatchType.Value(str(matchType)), - str(input))) + str(matchQuery))) def deleteAllMatchers(self): - """Prompts the user and then deletes all matchers""" + """Deletes all matchers.""" if self.__filter: - result = QtWidgets.QMessageBox.question(self, - "Delete All Matchers?", - "Are you sure you want to delete all matchers?", - QtWidgets.QMessageBox.Yes|QtWidgets.QMessageBox.No) + result = QtWidgets.QMessageBox.question( + self, + "Delete All Matchers?", + "Are you sure you want to delete all matchers?", + QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No) if result == QtWidgets.QMessageBox.Yes: self._itemsLock.lockForWrite() try: @@ -341,20 +354,29 @@ def __bulkAddMatchers(self, title, deleteExisting): self._update() - def __parseShotList(self, text): + @staticmethod + def __parseShotList(text): return [line.split()[0].strip().lower() for line in str(text).splitlines() if line.split()] + def tick(self): + pass + class ActionMonitorTree(cuegui.AbstractTreeWidget.AbstractTreeWidget): - def __init__(self, show, filter, parent): + """Tree for displaying a list of actions.""" + + def __init__(self, show, parent_filter, parent): self.startColumnsForType(cuegui.Constants.TYPE_ACTION) - self.addColumn("Action Type", 210, id=1, - data=lambda action:(opencue.compiled_proto.filter_pb2.ActionType.Name(action.type()))) + self.addColumn( + "Action Type", + 210, + id=1, + data=lambda action: (opencue.compiled_proto.filter_pb2.ActionType.Name(action.type()))) self.addColumn("", 180, id=2) self.addColumn("", 20, id=3) self.__show = show - self.__filter = filter + self.__filter = parent_filter cuegui.AbstractTreeWidget.AbstractTreeWidget.__init__(self, parent) @@ -369,28 +391,28 @@ def __init__(self, show, filter, parent): self, self.updateSoon, self.selectedObjects) self._timer.stop() - def setObject(self, object): + def setObject(self, action_object): """Sets the Action object to monitor - @type object: Action - @param object: The Action object to monitor""" - self.__filter = object + @type action_object: Action + @param action_object: The Action object to monitor""" + self.__filter = action_object self._update() - def _createItem(self, object): - """Creates and returns the proper item""" - return ActionWidgetItem(object, self) + def _createItem(self, action_object): + """Creates and returns the item associated with the given object.""" + return ActionWidgetItem(action_object, self) def _getUpdate(self): """Returns the proper data from the cuebot""" try: if self.__filter: return self.__filter.getActions() - except Exception as e: + except opencue.exception.CueException as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) return [] def contextMenuEvent(self, e): - """When right clicking on an item, this raises a context menu""" + """When right clicking on an item, this raises a context menu.""" menu = QtWidgets.QMenu() menu.addSeparator() @@ -399,10 +421,15 @@ def contextMenuEvent(self, e): menu.exec_(e.globalPos()) def createAction(self): - """Prompts the user to create a new action""" + """Prompts the user to create a new action.""" if self.__filter: (actionType, choice) = QtWidgets.QInputDialog.getItem( - self, "Create Action", "Please select the type of action to add:", ACTIONTYPE, 0, False) + self, + "Create Action", + "Please select the type of action to add:", + ACTIONTYPE, + 0, + False) if choice: value = None actionType = getattr(opencue.api.filter_pb2, str(actionType).replace(" ", "")) @@ -511,32 +538,39 @@ def deleteAllActions(self): self._itemsLock.unlock() self.removeAllItems() -################################################################################ + def tick(self): + pass + class FilterWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): - def __init__(self, object, parent): + """Widget item for displaying a single filter.""" + + def __init__(self, filter_object, parent): self.__widgets = {} cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( - self, cuegui.Constants.TYPE_FILTER, object, parent) + self, cuegui.Constants.TYPE_FILTER, filter_object, parent) self.updateWidgets() - def update(self, object = None, parent = None): + def update(self, rpcObject=None, parent=None): """Adds a call to updateWidgets()""" - cuegui.AbstractWidgetItem.AbstractWidgetItem.update(self, object, parent) + cuegui.AbstractWidgetItem.AbstractWidgetItem.update(self, rpcObject, parent) self.updateWidgets() - def setType(self, filterType): - self.rpcObject.setType(filterType) + def setType(self, filter_type): + """Sets the filter's type.""" + self.rpcObject.setType(filter_type) def setEnabled(self, value): + """Enables or disables the filter.""" self.rpcObject.setEnabled(bool(value)) def delete(self): - result = QtWidgets.QMessageBox.question(self.treeWidget(), - "Delete Filter?", - "Are you sure you want to delete this filter?\n\n%s" % - self.rpcObject.name(), - QtWidgets.QMessageBox.Yes|QtWidgets.QMessageBox.No) + """Deletes the filter.""" + result = QtWidgets.QMessageBox.question( + self.treeWidget(), + "Delete Filter?", + "Are you sure you want to delete this filter?\n\n%s" % self.rpcObject.name(), + QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No) if result == QtWidgets.QMessageBox.Yes: self.rpcObject.delete() QtCore.QTimer.singleShot(0, self.__delete) @@ -545,6 +579,7 @@ def __delete(self): self.treeWidget().removeItem(self) def updateWidgets(self): + """Refreshes the displayed information.""" if not self.__widgets: combo = QtWidgets.QCheckBox(self.parent()) combo.setFocusPolicy(QtCore.Qt.NoFocus) @@ -565,35 +600,43 @@ def updateWidgets(self): state = QtCore.Qt.Unchecked self.__widgets["enabled"].setCheckState(state) + class MatcherWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): - def __init__(self, object, parent): + """Widget item for displaying a single matcher.""" + + def __init__(self, rpcObject, parent): self.__widgets = {} cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( - self, cuegui.Constants.TYPE_MATCHER, object, parent) + self, cuegui.Constants.TYPE_MATCHER, rpcObject, parent) self.updateWidgets() - def update(self, object = None, parent = None): - """Adds a call to updateWidgets()""" - cuegui.AbstractWidgetItem.AbstractWidgetItem.update(self, object, parent) + def update(self, rpcObject=None, parent=None): + """Refreshes the widget display.""" + cuegui.AbstractWidgetItem.AbstractWidgetItem.update(self, rpcObject, parent) self.updateWidgets() def setType(self, matcherType): + """Sets the matcher type.""" self.rpcObject.setType(matcherType) def setSubject(self, matcherSubject): + """Sets the matcher subject.""" self.rpcObject.setSubject(matcherSubject) def setInput(self): + """Sets the matcher input.""" text = str(self.__widgets["input"].text()) if self.rpcObject.input() != text: self.rpcObject.setInput(text) - def delete(self, checked = False): - result = QtWidgets.QMessageBox.question(self.treeWidget(), - "Delete Matcher?", - "Are you sure you want to delete this matcher?\n\n%s" % - self.rpcObject.name(), - QtWidgets.QMessageBox.Yes|QtWidgets.QMessageBox.No) + def delete(self, checked=False): + """Deletes the matcher.""" + del checked + result = QtWidgets.QMessageBox.question( + self.treeWidget(), + "Delete Matcher?", + "Are you sure you want to delete this matcher?\n\n%s" % self.rpcObject.name(), + QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No) if result == QtWidgets.QMessageBox.Yes: self.rpcObject.delete() QtCore.QTimer.singleShot(0, self.__delete) @@ -602,6 +645,7 @@ def __delete(self): self.treeWidget().removeItem(self) def updateWidgets(self): + """Refreshes the widget display.""" if not self.__widgets: parent = self.parent() treeWidget = self.treeWidget() @@ -635,24 +679,29 @@ def updateWidgets(self): not self.__widgets["input"].isModified(): self.__widgets["input"].setText(self.rpcObject.input()) + class ActionWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): - def __init__(self, object, parent): + """Widget item for displaying a single action.""" + + def __init__(self, action_object, parent): self.__widgets = {} cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( - self, cuegui.Constants.TYPE_ACTION, object, parent) + self, cuegui.Constants.TYPE_ACTION, action_object, parent) self.updateWidgets() - def update(self, object = None, parent = None): - """Adds a call to updateWidgets()""" - cuegui.AbstractWidgetItem.AbstractWidgetItem.update(self, object, parent) + def update(self, rpcObject=None, parent=None): + """Updates the displayed content.""" + cuegui.AbstractWidgetItem.AbstractWidgetItem.update(self, rpcObject, parent) self.updateWidgets() - def delete(self, checked = False): - result = QtWidgets.QMessageBox.question(self.treeWidget(), - "Delete Action?", - "Are you sure you want to delete this action?\n\n%s" % - self.rpcObject.name(), - QtWidgets.QMessageBox.Yes|QtWidgets.QMessageBox.No) + def delete(self, checked=False): + """Deletes an action.""" + del checked + result = QtWidgets.QMessageBox.question( + self.treeWidget(), + "Delete Action?", + "Are you sure you want to delete this action?\n\n%s" % self.rpcObject.name(), + QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No) if result == QtWidgets.QMessageBox.Yes: self.rpcObject.delete() QtCore.QTimer.singleShot(0, self.__delete) @@ -660,8 +709,8 @@ def delete(self, checked = False): def __delete(self): self.treeWidget().removeItem(self) - def __setValue(self, value = None): - """Sets the value from the widget""" + def __setValue(self, value=None): + """Sets the action value.""" widget = self.__widgets["ActionValue"] # Get the proper value from the widget @@ -698,6 +747,7 @@ def __setValue(self, value = None): self.rpcObject.setTypeAndValue(self.rpcObject.type(), value) def updateWidgets(self): + """Updates the action display.""" if not self.__widgets: widget = None diff --git a/cuegui/cuegui/FrameMonitor.py b/cuegui/cuegui/FrameMonitor.py index 09b42c114..fc83fe1a4 100644 --- a/cuegui/cuegui/FrameMonitor.py +++ b/cuegui/cuegui/FrameMonitor.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Widget for displaying a list of frames with controls at the top.""" + + from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -28,6 +31,7 @@ from opencue.compiled_proto import job_pb2 import cuegui.FrameMonitorTree +import cuegui.FrameRangeSelection import cuegui.Logger @@ -35,7 +39,7 @@ class FrameMonitor(QtWidgets.QWidget): - """This contains the frame list table with controls at the top""" + """Widget for displaying a list of frames with controls at the top.""" handle_filter_layers_byLayer = QtCore.Signal(list) @@ -60,7 +64,6 @@ def __init__(self, parent): self._selectStatusSetup(hlayout) # Menu to select frames by status self._filterLayersSetup(hlayout) # Menu to filter layers self._filterStatusSetup(hlayout) # Menu to filter frames by status - #For the filter range setup: self._filterRangeSetup(hlayout) hlayout.addStretch() hlayout.addWidget(QtWidgets.QLabel("(Limited to 1000 frames)")) hlayout.addStretch() @@ -72,41 +75,50 @@ def __init__(self, parent): self._frameRangeSelectionFilterSetup(self.layout()) def updateRequest(self): + """Requests an update of the frame list.""" self.frameMonitorTree.updateRequest() def updateChangedRequest(self): + """Updates the frame list if sufficient time has passed since last updated.""" self.frameMonitorTree.updateChangedRequest() def setJob(self, job): + """Sets the current job.""" self.frameMonitorTree.setJob(job) def getColumnWidths(self): + """Gets the table column widths.""" return self.frameMonitorTree.getColumnWidths() def setColumnWidths(self, widths): + """Sets the table column widths.""" self.frameMonitorTree.setColumnWidths(widths) def getColumnVisibility(self): + """Gets the table column visibility.""" return self.frameMonitorTree.getColumnVisibility() def setColumnVisibility(self, settings): + """Sets the table column visibility.""" self.frameMonitorTree.setColumnVisibility(settings) def getColumnOrder(self): + """Gets the table column order.""" return self.frameMonitorTree.getColumnOrder() def setColumnOrder(self, settings): + """Sets the table column order.""" self.frameMonitorTree.setColumnOrder(settings) def filterLayersFromDoubleClick(self, layerNames): + """Event handler for filtering layers.""" self._filterLayersHandleByLayer(layerNames) -# ============================================================================== -# Frame range bar to filter by frame range -# ============================================================================== + # ============================================================================== + # Frame range bar to filter by frame range + # ============================================================================== def _frameRangeSelectionFilterSetup(self, layout): - from .FrameRangeSelection import FrameRangeSelectionWidget - widget = FrameRangeSelectionWidget(self) + widget = cuegui.FrameRangeSelection.FrameRangeSelectionWidget(self) layout.addWidget(widget) widget.selectionChanged.connect(self._frameRangeSelectionFilterHandle) self.frameRangeSelection = widget @@ -146,33 +158,9 @@ def _frameRangeSelectionFilterHandle(self, start, end): self.frameMonitorTree.frameSearch.options['range'] = "%s-%s" % (start, end) self.frameMonitorTree.updateRequest() -# ============================================================================== -# Widgets to filter by frame range -# ============================================================================== - def _filterRangeSetup(self, layout): - btn = QtWidgets.QSpinBox(self) - btn.setValue(1) - layout.addWidget(btn) - self.filter_range_start_box = btn - - btn = QtWidgets.QSpinBox(self) - btn.setValue(1000) - layout.addWidget(btn) - self.filter_range_end_box = btn - - btn = QtWidgets.QPushButton("Set Frame Range") - btn.setFocusPolicy(QtCore.Qt.NoFocus) - layout.addWidget(btn) - self.filter_range_btn = btn - btn.clicked.connect(self._filterRangeHandle) - - def _filterRangeHandle(self): - value = "%s-%s" % (self.filter_range_start_box.value(), self.filter_range_end_box.value()) - self.frameMonitorTree.frameSearch.setOptions(range=value) - -# ============================================================================== -# Button to refresh -# ============================================================================== + # ============================================================================== + # Button to refresh + # ============================================================================== def _refreshButtonSetup(self, layout): """Sets up the refresh button, adds it to the given layout @param layout: The layout to add the button to @@ -192,9 +180,9 @@ def _refreshButtonDisableHandle(self): self.btn_refresh.setEnabled(False) QtCore.QTimer.singleShot(5000, self._refreshButtonEnableHandle) -# ============================================================================== -# Button to clear all filters -# ============================================================================== + # ============================================================================== + # Button to clear all filters + # ============================================================================== def _clearButtonSetup(self, layout): """Sets up the clear button, adds it to the given layout @param layout: The layout to add the button to @@ -212,9 +200,9 @@ def _clearButtonHandle(self): self._frameRangeSelectionFilterUpdate() self.frameMonitorTree.clearFilters() -# ============================================================================== -# Widgets to Load previous/next page -# ============================================================================== + # ============================================================================== + # Widgets to Load previous/next page + # ============================================================================== def _pageButtonSetup(self, layout): '''Sets up the page flipping buttons and the page # label @param layout: The layout to add the buttons & label to @@ -294,9 +282,9 @@ def _updatePageButtonState(self): self.page_label.setText('{0}' .format(page_label_text)) -# ============================================================================== -# Menu to select frames by status -# ============================================================================== + # ============================================================================== + # Menu to select frames by status + # ============================================================================== def _selectStatusSetup(self, layout): """Sets up the select status menu, adds it to the given layout @param layout: The layout to add the menu to @@ -328,9 +316,9 @@ def _selectStatusHandle(self, action): else: self.frameMonitorTree.selectByStatus(action.text()) -# ============================================================================== -# Menu to filter frames by layers -# ============================================================================== + # ============================================================================== + # Menu to filter frames by layers + # ============================================================================== def _filterLayersSetup(self, layout): """Sets up the filter layers menu, adds it to the given layout @param layout: The layout to add the menu to @@ -423,9 +411,9 @@ def _filterLayersHandleByLayer(self, layer_list): self.frameMonitorTree.updateRequest() self._updatePageButtonState() -# ============================================================================== -# Menu to filter frames by status -# ============================================================================== + # ============================================================================== + # Menu to filter frames by status + # ============================================================================== def _filterStatusSetup(self, layout): """Sets up the filter status menu, adds it to the given layout @param layout: The layout to add the menu to @@ -496,9 +484,9 @@ def _filterStatusHandle(self, action): self._updatePageButtonState() self.frameMonitorTree.updateRequest() -# ============================================================================== -# QLabel that displays the job name -# ============================================================================== + # ============================================================================== + # QLabel that displays the job name + # ============================================================================== def _displayJobNameSetup(self, layout): """Sets up the displaying the name of the currently job. @param layout: The layout to add the label to @@ -512,6 +500,7 @@ def _displayJobNameSetup(self, layout): def _displayJobNameUpdate(self): """Updates the display job name label with the name of the current job.""" if self.frameMonitorTree.getJob(): - self._displayJobNameLabel.setText(" %s " % self.frameMonitorTree.getJob().data.name) + self._displayJobNameLabel.setText( + " %s " % self.frameMonitorTree.getJob().data.name) else: self._displayJobNameLabel.clear() diff --git a/cuegui/cuegui/FrameMonitorTree.py b/cuegui/cuegui/FrameMonitorTree.py index 9d5c35857..33ccccad9 100644 --- a/cuegui/cuegui/FrameMonitorTree.py +++ b/cuegui/cuegui/FrameMonitorTree.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -A frame list based on AbstractTreeWidget -""" +"""Tree widget for displaying a list of frames.""" from __future__ import absolute_import @@ -45,6 +43,7 @@ import cuegui.Logger import cuegui.MenuActions import cuegui.Style +import cuegui.ThreadPool import cuegui.Utils @@ -64,6 +63,7 @@ class FrameMonitorTree(cuegui.AbstractTreeWidget.AbstractTreeWidget): + """Tree widget for displaying a list of frames.""" job_changed = QtCore.Signal() handle_filter_layers_byLayer = QtCore.Signal(list) @@ -97,7 +97,7 @@ def __init__(self, parent): "Depend: \t The frame depends on another frame or job.\n" "Dead: \t The frame failed with an error.") self.addColumn("Cores", 55, id=5, - data=lambda job, frame: (self.getCores(frame, True) or ""), + data=lambda job, frame: (self.getCores(frame, format_as_string=True) or ""), sort=lambda job, frame: (self.getCores(frame)), tip="The number of cores a frame is using") self.addColumn("Host", 120, id=6, @@ -151,9 +151,10 @@ def __init__(self, parent): "frame for most types of jobs") self.addColumn("Memory", 60, id=12, - data=lambda job, frame: (frame.data.state == opencue.api.job_pb2.RUNNING and - cuegui.Utils.memoryToString(frame.data.used_memory) or - cuegui.Utils.memoryToString(frame.data.max_rss)), + data=lambda job, frame: ( + frame.data.state == opencue.api.job_pb2.RUNNING and + cuegui.Utils.memoryToString(frame.data.used_memory) or + cuegui.Utils.memoryToString(frame.data.max_rss)), sort=lambda job, frame: (frame.data.state == opencue.api.job_pb2.RUNNING and frame.data.used_memory or frame.data.max_rss), tip="If a frame is running:\n" @@ -195,13 +196,14 @@ def __init__(self, parent): cuegui.AbstractTreeWidget.AbstractTreeWidget.__init__(self, parent) self.__sortByColumnCache = {} + self.ticksWithoutUpdate = 999 + self.__lastUpdateTime = None self.itemClicked.connect(self.__itemSingleClickedCopy) self.itemClicked.connect(self.__itemSingleClickedViewLog) self.itemDoubleClicked.connect(self.__itemDoubleClickedViewLog) self.header().sortIndicatorChanged.connect(self.__sortByColumnSave) - self.__load = None self.startTicksUpdate(20) @@ -219,7 +221,7 @@ def tick(self): self.ticksWithoutUpdate = 0 self._update() return - elif self.ticksWithoutUpdate > self.updateInterval: + if self.ticksWithoutUpdate > self.updateInterval: logger.info("doing changed update") self.ticksWithoutUpdate = 0 self._updateChanged() @@ -233,19 +235,23 @@ def tick(self): if not self.ticksWithoutUpdate % 2: self.redraw() - def getCores(self, frame, format=False): + @staticmethod + def getCores(frame, format_as_string=False): + """Gets the number of cores a frame is using.""" cores = None - m = re.search(".*\/(\d+\.?\d*)", frame.data.last_resource) + m = re.search(r".*\/(\d+\.?\d*)", frame.data.last_resource) if m: cores = float(m.group(1)) - if format: + if format_as_string: cores = "{:.2f}".format(cores) return cores - def getTimeString(self, timestamp): + @staticmethod + def getTimeString(timestamp): + """Gets a timestamp formatted as a string.""" tstring = None if timestamp and timestamp > 0: tstring = datetime.datetime.fromtimestamp(timestamp).strftime("%m/%d %H:%M") @@ -254,6 +260,7 @@ def getTimeString(self, timestamp): def redrawRunning(self): """Forces the running frames to be redrawn with current values""" + # pylint: disable=broad-except try: items = self.findItems("Running", QtCore.Qt.MatchExactly, @@ -276,7 +283,7 @@ def __sortByColumnSave(self, logicalIndex, order): def __sortByColumnLoad(self): """Loads the last used sort column and order for the current job, or uses default ascending dispatch order""" - key = self.__job and cuegui.Utils.getObjectKey(self.__job) or None + key = cuegui.Utils.getObjectKey(self.__job) if self.__job else None settings = self.__sortByColumnCache.get(key, (0, QtCore.Qt.AscendingOrder)) self.sortByColumn(settings[0], settings[1]) @@ -287,7 +294,10 @@ def __itemSingleClickedCopy(self, item, col): @param item: The item single clicked on @type col: int @param col: Column number single clicked on""" - selected = [frame.data.name for frame in self.selectedObjects() if cuegui.Utils.isFrame(frame)] + del item + del col + selected = [ + frame.data.name for frame in self.selectedObjects() if cuegui.Utils.isFrame(frame)] if selected: QtWidgets.QApplication.clipboard().setText(" ".join(selected)) @@ -297,6 +307,7 @@ def __itemSingleClickedViewLog(self, item, col): @param item: The item single clicked on @type col: int @param col: Column number single clicked on""" + del col current_log_file = cuegui.Utils.getFrameLogFile(self.__job, item.rpcObject) try: old_log_files = sorted(glob.glob('%s.*' % current_log_file), @@ -304,7 +315,9 @@ def __itemSingleClickedViewLog(self, item, col): reverse=True) except ValueError: pass + # pylint: disable=no-member QtGui.qApp.display_log_file_content.emit([current_log_file] + old_log_files) + # pylint: enable=no-member def __itemDoubleClickedViewLog(self, item, col): """Called when a frame is double clicked, views the frame log in a popup @@ -312,13 +325,16 @@ def __itemDoubleClickedViewLog(self, item, col): @param item: The item double clicked on @type col: int @param col: Column number double clicked on""" + del col frame = item.rpcObject if frame.data.state == opencue.api.job_pb2.RUNNING: cuegui.Utils.popupFrameTail(self.__job, frame) else: cuegui.Utils.popupFrameView(self.__job, frame) + # pylint: disable=inconsistent-return-statements def setJob(self, job): + """Sets the current job.""" if job is None: return self.__setJob(None) job = cuegui.Utils.findJob(job) @@ -344,13 +360,15 @@ def getJob(self): return self.__job def clearFilters(self): + """Clears any user-defined filters or sorting, restores default state.""" self.clearSelection() self.frameSearch = opencue.search.FrameSearch() self.sortByColumn(0, QtCore.Qt.AscendingOrder) self.updateRequest() def selectByStatus(self, status): - """Selects all frames that match the given status + """Selects all frames that match the given status. + @type status: string @param status: A frame status to match""" items = self.findItems(str(status), @@ -365,18 +383,18 @@ def selectByStatus(self, status): # Scroll to the first item self.scrollToItem(items[0], QtWidgets.QAbstractItemView.PositionAtTop) - def _createItem(self, object): + def _createItem(self, rpcObject): """Creates and returns the proper item""" - return FrameWidgetItem(object, self, self.__job) + return FrameWidgetItem(rpcObject, self, self.__job) -# -# updateRequest -> _update -> _getUpdate -> _processUpdate -# updateChangedRequest -> _updateChanged -> _getUpdateChanged -> _processUpdateChanged -# -# autoUpdate -> updateRequest -# updateAll -> updateRequest -# _updateAll -> _update -# + # + # updateRequest -> _update -> _getUpdate -> _processUpdate + # updateChangedRequest -> _updateChanged -> _getUpdateChanged -> _processUpdateChanged + # + # autoUpdate -> updateRequest + # updateAll -> updateRequest + # _updateAll -> _update + # def updateRequest(self): """Updates the items in the TreeWidget if sufficient time has passed @@ -394,7 +412,10 @@ def _update(self): logger.info("_update") self._lastUpdate = time.time() if hasattr(QtGui.qApp, "threadpool"): - QtGui.qApp.threadpool.queue(self._getUpdate, self._processUpdate, "getting data for %s" % self.__class__) + # pylint: disable=no-member + QtGui.qApp.threadpool.queue( + self._getUpdate, self._processUpdate, "getting data for %s" % self.__class__) + # pylint: enable=no-member else: logger.warning("threadpool not found, doing work in gui thread") self._processUpdate(None, self._getUpdate()) @@ -405,7 +426,11 @@ def _updateChanged(self): logger.info("_updateChanged") self._lastUpdate = time.time() if hasattr(QtGui.qApp, "threadpool"): - QtGui.qApp.threadpool.queue(self._getUpdateChanged, self._processUpdateChanged, "getting data for %s" % self.__class__) + # pylint: disable=no-member + QtGui.qApp.threadpool.queue( + self._getUpdateChanged, self._processUpdateChanged, + "getting data for %s" % self.__class__) + # pylint: enable=no-member else: logger.warning("threadpool not found, doing work in gui thread") self._processUpdateChanged(None, self._getUpdateChanged()) @@ -418,7 +443,7 @@ def _getUpdate(self): self.__lastUpdateTime = int(time.time()) return self.__job.getFrames(**self.frameSearch.options) return [] - except Exception as e: + except opencue.exception.CueException as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) def _getUpdateChanged(self): @@ -430,7 +455,7 @@ def _getUpdateChanged(self): (self.__jobState and self.__jobState == opencue.api.job_pb2.FINISHED): logger.debug("no job or job is finished, bailing") return [] - logger.info(" + Nth update = %s" % self.__class__) + logger.info(" + Nth update = %s", self.__class__) updatedFrames = [] try: updated_data = self.__job.getUpdatedFrames(self.__lastUpdateTime) @@ -438,16 +463,17 @@ def _getUpdateChanged(self): self.__jobState = updated_data.state updatedFrames = updated_data.updated_frames.updated_frames - except opencue.EntityNotFoundException as e: + except opencue.EntityNotFoundException: self.setJobObj(None) - except Exception as e: - if hasattr(e, "message") and e.message.find("timestamp cannot be over a minute off") != -1: - logger.warning("Forcing a full update due to: %s" % e.message) + except opencue.exception.CueException as e: + # pylint: disable=no-member + if hasattr(e, "message") and 'timestamp cannot be over a minute off' in e.message: + logger.warning("Forcing a full update due to: %s", e.message) return None - else: - list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) + # pylint: enable=no-member + list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) - logger.info(" - %s" % self.__class__) + logger.info(" - %s", self.__class__) return updatedFrames def _processUpdate(self, work, rpcObjects): @@ -465,10 +491,11 @@ def _processUpdate(self, work, rpcObjects): self._items = {} if rpcObjects: for rpcObject in rpcObjects: - self._items[cuegui.Utils.getObjectKey(rpcObject)] = self._createItem(rpcObject) + self._items[cuegui.Utils.getObjectKey(rpcObject)] = \ + self._createItem(rpcObject) finally: self._itemsLock.unlock() - except Exception as e: + except opencue.exception.CueException as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) def _processUpdateChanged(self, work, rpcObjects): @@ -478,6 +505,7 @@ def _processUpdateChanged(self, work, rpcObjects): @type work: @param rpcObjects: A list of rpcObjects @type rpcObjects: list """ + del work logger.info("_processUpdateChanged") try: if rpcObjects is None: @@ -496,7 +524,7 @@ def _processUpdateChanged(self, work, rpcObjects): logger.info("_processUpdateChanged calling redraw") self.redraw() - except Exception as e: + except opencue.exception.CueException as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) def _updateFrame(self, updatedFrame): @@ -524,11 +552,17 @@ def _actionFilterSelectedLayers(self): class FrameWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): + """Widget item representing a single frame.""" + __initialized = False - def __init__(self, object, parent, job): + + # pylint: disable=protected-access + def __init__(self, rpcObject, parent, job): if not self.__initialized: self.__class__.__initialized = True + # pylint: disable=no-member self.__class__.__backgroundColor = QtGui.qApp.palette().color(QtGui.QPalette.Base) + # pylint: enable=no-member self.__class__.__foregroundColor = cuegui.Style.ColorTheme.COLOR_JOB_FOREGROUND self.__class__.__foregroundColorBlack = QCOLOR_BLACK self.__class__.__foregroundColorGreen = QCOLOR_GREEN @@ -539,7 +573,7 @@ def __init__(self, object, parent, job): self.__class__.__rgbFrameState[key] = cuegui.Constants.RGB_FRAME_STATE[key] self.__class__.__type = cuegui.Constants.TYPE_FRAME cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( - self, cuegui.Constants.TYPE_FRAME, object, parent, job) + self, cuegui.Constants.TYPE_FRAME, rpcObject, parent, job) self.__show = job.data.show def data(self, col, role): @@ -554,18 +588,17 @@ def data(self, col, role): return self.column_info[col][cuegui.Constants.COLUMN_INFO_DISPLAY]( self._source, self.rpcObject) - elif role == QtCore.Qt.ForegroundRole: + if role == QtCore.Qt.ForegroundRole: if col == STATUS_COLUMN: return self.__foregroundColorBlack - elif col == PROC_COLUMN and self.rpcObject.data.last_resource.startswith(LOCALRESOURCE): + if col == PROC_COLUMN and self.rpcObject.data.last_resource.startswith(LOCALRESOURCE): return self.__foregroundColorGreen - else: - return self.__foregroundColor + return self.__foregroundColor - elif role == QtCore.Qt.BackgroundRole and col == STATUS_COLUMN: + if role == QtCore.Qt.BackgroundRole and col == STATUS_COLUMN: return self.__rgbFrameState[self.rpcObject.data.state] - elif role == QtCore.Qt.DecorationRole and col == CHECKPOINT_COLUMN: + if role == QtCore.Qt.DecorationRole and col == CHECKPOINT_COLUMN: if self.rpcObject.data.checkpoint_state == opencue.api.job_pb2.ENABLED: return QtGui.QIcon(":markdone.png") elif role == QtCore.Qt.TextAlignmentRole: @@ -582,7 +615,8 @@ def data(self, col, role): def __lt__(self, other): """Custom sorting for columns that have a function defined for sorting""" - sortLambda = self.column_info[self.treeWidget().sortColumn()][cuegui.AbstractWidgetItem.SORT_LAMBDA] + sortLambda = self.column_info[ + self.treeWidget().sortColumn()][cuegui.AbstractWidgetItem.SORT_LAMBDA] if sortLambda: return sortLambda(self._source, self.rpcObject) < sortLambda( other._source, other.rpcObject) @@ -607,9 +641,7 @@ class FrameLogDataBuffer(object): # default to 0% if nothing found unless already a previous value def __init__(self): - from .ThreadPool import ThreadPool - - self.__threadPool = ThreadPool(self.maxThreads, self.maxQueue) + self.__threadPool = cuegui.ThreadPool.ThreadPool(self.maxThreads, self.maxQueue) self.__currentJob = None self.__cache = {} self.__queue = {} @@ -626,6 +658,7 @@ def __init__(self): def getLastLineData(self, job, frame): """Returns the last line and LLU of the log file or queues a request to update it""" + # pylint: disable=broad-except try: __now = time.time() jobKey = cuegui.Utils.getObjectKey(job) @@ -635,6 +668,7 @@ def getLastLineData(self, job, frame): self.__queue.clear() self.__currentJob = jobKey + # pylint: disable=protected-access if len(self.__queue) > len(self.__threadPool._q_queue): # Everything is hung up, start over self.__cache.clear() @@ -651,25 +685,27 @@ def getLastLineData(self, job, frame): self.__threadPool.queue(self.__doWork, self.__saveWork, "getting data for %s" % self.__class__) # Return the cached results anyway - return (__cached[self.__LINE], __cached[self.__LLU]) - else: - __path = cuegui.Utils.getFrameLogFile(job, frame) - # Cache a blank entry until it is filled in - self.__cache[frameKey] = [__now + 60, - __path, - self.__defaultLine, - self.__defaultLLU] - # Queue an update - self.__queue[frameKey] = __path - self.__threadPool.queue(self.__doWork, self.__saveWork, - "getting data for %s" % self.__class__) - # Since nothing is updated yet, return an empty string - return (self.__defaultLine, self.__defaultLLU) + return __cached[self.__LINE], __cached[self.__LLU] + + __path = cuegui.Utils.getFrameLogFile(job, frame) + # Cache a blank entry until it is filled in + self.__cache[frameKey] = [__now + 60, + __path, + self.__defaultLine, + self.__defaultLLU] + # Queue an update + self.__queue[frameKey] = __path + self.__threadPool.queue(self.__doWork, self.__saveWork, + "getting data for %s" % self.__class__) + # Since nothing is updated yet, return an empty string + return self.__defaultLine, self.__defaultLLU except Exception as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) + # pylint: disable=inconsistent-return-statements def __doWork(self): """Pops work from the queue and returns the proxy and last log line""" + # pylint: disable=broad-except try: if self.__queue: (proxy, path) = self.__queue.popitem() @@ -677,13 +713,14 @@ def __doWork(self): return (proxy, cuegui.Utils.getLastLine(path), cuegui.Utils.secondsToHHMMSS(time.time() - os.stat(path).st_mtime)) - else: - return None + return None except Exception as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) def __saveWork(self, work, results): """Stores the resulting last log line to the cache with the proxy key""" + del work + # pylint: disable=broad-except try: if results: __cached = self.__cache[results[0]] @@ -699,15 +736,13 @@ def __saveWork(self, work, results): class FrameEtaDataBuffer(object): """A cached and threaded interface to reading the last log line""" + maxCacheTime = 60 maxThreads = 2 maxQueue = 501 - def __init__(self): - from .ThreadPool import ThreadPool - - self.__threadPool = ThreadPool(self.maxThreads, self.maxQueue) + self.__threadPool = cuegui.ThreadPool.ThreadPool(self.maxThreads, self.maxQueue) self.__currentJob = None self.__cache = {} @@ -717,13 +752,16 @@ def __init__(self): self.__ETA = 1 def getEtaFormatted(self, job, frame): + """Gets frame ETA formatted as a string.""" result = self.getEta(job, frame) if result: return cuegui.Utils.secondsToHHMMSS(result) return False def getEta(self, job, frame): + """Gets frame ETA as a number of seconds.""" __now = time.time() + # pylint: disable=broad-except try: jobKey = cuegui.Utils.getObjectKey(job) if self.__currentJob != jobKey: @@ -738,16 +776,18 @@ def getEta(self, job, frame): if __cached[self.__TIME] < __now - self.maxCacheTime: # It is an old cache, queue an update, reset the time until updated self.__cache[frameKey][0] = __now + 60 - self.__threadPool.queue(self.__doWork, self.__saveWork, - "getting data for %s" % self.__class__, frameKey, job, frame) + self.__threadPool.queue( + self.__doWork, self.__saveWork, "getting data for %s" % self.__class__, + frameKey, job, frame) # Return the cached results anyway if __cached[self.__ETA] is not None: return max(__cached[self.__ETA] - __now + __cached[self.__TIME], 0) else: # Queue an update, cache a blank entry until updated self.__cache[frameKey] = [__now + 60, None] - self.__threadPool.queue(self.__doWork, self.__saveWork, - "getting data for %s" % self.__class__, frameKey, job, frame) + self.__threadPool.queue( + self.__doWork, self.__saveWork, "getting data for %s" % self.__class__, + frameKey, job, frame) # Since nothing is updated yet, return a default except Exception as e: self.__cache[frameKey] = [__now, @@ -758,14 +798,17 @@ def getEta(self, job, frame): def __doWork(self, proxy, job, frame): """Pops work from the queue and returns the proxy and last log line""" + # pylint: disable=broad-except try: - return (proxy, cuegui.eta.ETASeconds(job, frame)) + return proxy, cuegui.eta.ETASeconds(job, frame) except Exception as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) - return (proxy, self.__defaultETA) + return proxy, self.__defaultETA def __saveWork(self, work, results): """Stores the resulting last log line to the cache with the proxy key""" + del work + # pylint: disable=broad-except try: if results: __cached = self.__cache[results[0]] @@ -779,6 +822,8 @@ def __saveWork(self, work, results): class FrameContextMenu(QtWidgets.QMenu): + """Context menu for frames.""" + def __init__(self, widget, filterSelectedLayersCallback): super(FrameContextMenu, self).__init__() @@ -801,8 +846,10 @@ def __init__(self, widget, filterSelectedLayersCallback): self.__menuActions.frames().addAction(self, "useLocalCores") + # pylint: disable=no-member if QtGui.qApp.applicationName() == "CueCommander": self.__menuActions.frames().addAction(self, "viewHost") + # pylint: enable=no-member depend_menu = QtWidgets.QMenu("&Dependencies", self) self.__menuActions.frames().addAction(depend_menu, "viewDepends") @@ -828,4 +875,3 @@ def __init__(self, widget, filterSelectedLayersCallback): self.__menuActions.frames().addAction(self, "eat") self.__menuActions.frames().addAction(self, "kill") self.__menuActions.frames().addAction(self, "eatandmarkdone") - diff --git a/cuegui/cuegui/FrameRangeSelection.py b/cuegui/cuegui/FrameRangeSelection.py index 06da309d1..851e2aab2 100644 --- a/cuegui/cuegui/FrameRangeSelection.py +++ b/cuegui/cuegui/FrameRangeSelection.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -From katana -""" +"""Widget for displaying and selecting within a frame range.""" from __future__ import division @@ -33,20 +31,21 @@ class FrameRangeSelectionWidget(QtWidgets.QWidget): - """The Timeline Widget is a rectangular area with ticks to represent units - of frames. A frame range may be selected. - This widget emits the following signals: - * startFrameChanged(int) - user clicks new time or program changes time - * endFrameChanged(int) - user clicks new time or program changes time - * frameRangeChanged(int,int) - program changes viewed frame range - * selectionChanged(int,int) - user releases a drag that selects a frame range - """ + """Widget for displaying and selecting within a frame range. + + A rectangular area with ticks to represent units of frames. + + This widget emits the following signals: + * startFrameChanged(int) - user clicks new time or program changes time + * endFrameChanged(int) - user clicks new time or program changes time + * frameRangeChanged(int,int) - program changes viewed frame range + * selectionChanged(int,int) - user releases a drag that selects a frame range + """ endFrameChanged = QtCore.Signal(int) startFrameChanged = QtCore.Signal(int) frameRangeChanged = QtCore.Signal(int, int) selectionChanged = QtCore.Signal(int, int) - def __init__(self, parent, *args): QtWidgets.QWidget.__init__(self, parent, *args) self.setMinimumWidth(100) @@ -54,12 +53,11 @@ def __init__(self, parent, *args): self.__layout.addStretch(100) self.__right_margin = 25 - self.__layout.setContentsMargins(0,0,0,0) + self.__layout.setContentsMargins(0, 0, 0, 0) toolButton = QtWidgets.QToolButton() toolButton.setArrowType(QtCore.Qt.UpArrow) toolButton.setFocusPolicy(QtCore.Qt.NoFocus) - toolButton.setFixedSize(20,20) - #toolButton.setContentsMargins(0,0,0,0) + toolButton.setFixedSize(20, 20) self.__layout.addWidget(toolButton) self.setLayout(self.__layout) @@ -93,42 +91,47 @@ def __init__(self, parent, *args): self.default_select_size = 1000 def endFrame(self): + """Returns the current end frame displayed in the timeline.""" return self.__endFrame def setEndTime(self, t, final = True): - if (self.__endFrame == t and self.__endFrameFinal == final): return + """Sets the current end frame displayed in the timeline.""" + if self.__endFrame == t and self.__endFrameFinal == final: + return self.__endFrame = int(t) self.__endFrameFinal = final - #mine self.__selectionRange = (self.__startFrame, self.__endFrame) self.update() self.endFrameChanged.emit(int(t)) - # The current time displayed in the timeline. def startFrame(self): + """Returns the current start frame displayed in the timeline.""" return self.__startFrame def setStartTime(self, t, final = True): + """Sets the current start frame displayed in the timeline.""" if self.__startFrame == t and self.__startFrameFinal == final: return self.__startFrame = int(t) self.__startFrameFinal = final if self.__endFrame == self.__startFrame: - self.setEndTime(min(self.__startFrame + self.default_select_size - 1, self.__frameRange[1]), True) + self.setEndTime( + min(self.__startFrame + self.default_select_size - 1, self.__frameRange[1]), True) self.__selectionRange = (self.__startFrame, self.__endFrame) self.update() self.startFrameChanged.emit(int(t)) - # The viewed range of time in the timeline. def frameRange(self): + """Returns the viewed range of time in the timeline.""" return self.__frameRange def setFrameRange(self, frameRange): + """Sets the viewed frame range in the timeline.""" frameRange = tuple(sorted(map(int, frameRange))) self.__floatTime = None @@ -140,11 +143,12 @@ def setFrameRange(self, frameRange): self.frameRangeChanged.emit(self.__frameRange[0], self.__frameRange[1]) self.update() - # The selected (highlighted) range of time in the timeline. def selectedFrameRange(self): + """Returns the selected (highlighted) range of time in the timeline.""" return self.__selectionRange def setSelectedFrameRange(self, selectedRange): + """Sets the selected (highlighted) range of time in the timeline.""" if selectedRange != self.__selectionRange: self.__selectionRange = selectedRange @@ -186,15 +190,19 @@ def mouseReleaseEvent(self, mouseEvent): self.setEndTime(max(hitTime, self.startFrame()), True) self.setStartTime(min(hitTime, self.startFrame()), True) - self.__selectionRange = (min(self.__selectionRange[0], self.__selectionRange[1]), max(self.__selectionRange[0], self.__selectionRange[1])) + self.__selectionRange = ( + min(self.__selectionRange[0], self.__selectionRange[1]), + max(self.__selectionRange[0], self.__selectionRange[1])) self.setSelectedFrameRange(self.__selectionRange) self.update() def mouseDoubleClickEvent(self, mouseEvent): + del mouseEvent self.__double = True def paintEvent(self, paintEvent): + del paintEvent painter = QtGui.QPainter(self) self.__paintBackground(painter) @@ -209,20 +217,26 @@ def paintEvent(self, paintEvent): self.__paintEndTime(painter) def leaveEvent(self, event): + del event self.__floatTime = None self.update() def __getTickAreaExtent(self): - return QtCore.QRect(10, -self.height() // 2, self.width() - self.__right_margin - 20, self.height()) + return QtCore.QRect( + 10, -self.height() // 2, self.width() - self.__right_margin - 20, self.height()) def __getTickArea(self, time): tickArea = self.__getTickAreaExtent() - tickSpacing = float(self.__getTickAreaExtent().width()) / max(1,(self.__frameRange[1] - self.__frameRange[0])) + tickSpacing = ( + float(self.__getTickAreaExtent().width()) / + max(1, (self.__frameRange[1] - self.__frameRange[0]))) return QtCore.QRect(tickArea.left() + tickSpacing * (time - self.__frameRange[0]), tickArea.top(), tickSpacing, tickArea.height()) def __getTimeFromLocalPoint(self, x): - tickSpacing = float(self.__getTickAreaExtent().width()) / max(1,(self.__frameRange[1] - self.__frameRange[0])) + tickSpacing = ( + float(self.__getTickAreaExtent().width()) / + max(1, (self.__frameRange[1] - self.__frameRange[0]))) deltaX = x - self.__getTickAreaExtent().left() hitTime = int(deltaX / tickSpacing + 0.5) + self.__frameRange[0] hitTime = int(max(self.__frameRange[0], min(hitTime, self.__frameRange[1]))) @@ -230,9 +244,9 @@ def __getTimeFromLocalPoint(self, x): def __getLabelPeriod(self): delta = self.__frameRange[1] - self.__frameRange[0] - if (delta < 20): + if delta < 20: return 2 - if (delta < 10000): + if delta < 10000: base = 5 offset = 2 else: @@ -245,7 +259,12 @@ def __paintBackground(self, painter): bgBrush = self.palette().window() painter.fillRect(0, 0, self.width() - self.__right_margin, self.height(), bgBrush) highlightBrush = QtGui.QBrush(QtGui.QColor(75, 75, 75)) - painter.fillRect(0, self.height() // 2, self.width() - self.__right_margin+5, self.height() // 2, highlightBrush) + painter.fillRect( + 0, + self.height() // 2, + self.width() - self.__right_margin + 5, + self.height() // 2, + highlightBrush) def __paintTickmarks(self, painter): tickExtent = self.__getTickAreaExtent() @@ -267,7 +286,8 @@ def __paintLabels(self, painter): tickExtent = self.__getTickAreaExtent() labelHeight = tickExtent.height() // 3 labelPeriod = self.__getLabelPeriod() - if labelPeriod == 0: return + if labelPeriod == 0: + return firstLabel = self.__frameRange[0] + labelPeriod - 1 firstLabel = firstLabel - (firstLabel % labelPeriod) @@ -332,17 +352,19 @@ def __paintEndTime(self, painter): metric = QtGui.QFontMetrics(painter.font()) frameString = str(int(endFrame)) xPos = timeExtent.left() - metric.width(frameString) // 2 - yPos = metric.ascent() + 1 + yPos = metric.ascent() + 1 painter.drawText(xPos, yPos, frameString) painter.setPen(oldPen) def __paintFloatTime(self, painter): - if self.__floatTime == None: return + if self.__floatTime is None: + return timeExtent = self.__getTickArea(self.__floatTime) oldPen = painter.pen() painter.setPen(QtGui.QColor(90, 90, 90)) - painter.drawLine(timeExtent.left(), timeExtent.top(), timeExtent.left(), timeExtent.bottom()) + painter.drawLine( + timeExtent.left(), timeExtent.top(), timeExtent.left(), timeExtent.bottom()) if self.__selectionRange: painter.setPen(QtGui.QColor(255,255,255)) @@ -351,13 +373,16 @@ def __paintFloatTime(self, painter): metric = QtGui.QFontMetrics(painter.font()) frameString = str(self.__floatTime) xPos = timeExtent.left() - metric.width(frameString) // 2 - yPos = timeExtent.top() + metric.ascent() + yPos = timeExtent.top() + metric.ascent() painter.drawText(xPos, yPos, frameString) painter.setPen(oldPen) def __paintSelection(self, painter): - if self.__selectionRange == None: return - selection = (min(self.__selectionRange[0], self.__selectionRange[1]), max(self.__selectionRange[0], self.__selectionRange[1])) + if self.__selectionRange is None: + return + selection = ( + min(self.__selectionRange[0], self.__selectionRange[1]), + max(self.__selectionRange[0], self.__selectionRange[1])) leftExtent = self.__getTickArea(selection[0]) rightExtent = self.__getTickArea(selection[1] - 1) diff --git a/cuegui/cuegui/GarbageCollector.py b/cuegui/cuegui/GarbageCollector.py index 6b3971244..ec0587dbd 100644 --- a/cuegui/cuegui/GarbageCollector.py +++ b/cuegui/cuegui/GarbageCollector.py @@ -1,15 +1,38 @@ -from PySide2 import QtCore +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Custom garbage collector class. + +Disables automatic garbage collection and instead collect manually every INTERVAL milliseconds. + +This is done to ensure that garbage collection only happens in the GUI thread, as otherwise Qt +can crash.""" + + import gc +from PySide2 import QtCore + + class GarbageCollector(QtCore.QObject): + """Custom garbage collector class. - ''' - Disable automatic garbage collection and instead collect manually - every INTERVAL milliseconds. + Disables automatic garbage collection and instead collect manually every INTERVAL milliseconds. - This is done to ensure that garbage collection only happens in the GUI - thread, as otherwise Qt can crash. - ''' + This is done to ensure that garbage collection only happens in the GUI thread, as otherwise Qt + can crash.""" INTERVAL = 5000 @@ -25,7 +48,10 @@ def __init__(self, parent, debug=False): self.timer.start(self.INTERVAL) def check(self): + """Runs the garbage collector. + + This method is run every INTERNAL seconds.""" gc.collect() if self.debug: for obj in gc.garbage: - print (obj, repr(obj), type(obj)) + print(obj, repr(obj), type(obj)) diff --git a/cuegui/cuegui/GraphSubscriptionsWidget.py b/cuegui/cuegui/GraphSubscriptionsWidget.py deleted file mode 100644 index 39aa6f613..000000000 --- a/cuegui/cuegui/GraphSubscriptionsWidget.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright Contributors to the OpenCue Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from __future__ import division -from __future__ import print_function -from __future__ import absolute_import - -from builtins import range - -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets - -import opencue - - -class GraphSubscriptionsWidget(QtWidgets.QWidget): - def __init__(self, parent): - QtWidgets.QWidget.__init__(self, parent) - - self.__color = QtGui.QColor(55,200,55) - self.__brush = QtGui.QBrush() - self.__brush.setColor(self.__color) - self.__brush.setStyle(QtCore.Qt.SolidPattern) - - self.__show = opencue.api.findShow("clo") - self.__history = [0]*100 - self.__line = 575 - self.__max = max(self.__line * 1.2, 80) - - self.__timer = QtCore.QTimer(self) - self.__timer.timeout.connect(self.addNumber) - - self.__timer.start(10000) - - def addNumber(self): - for sub in self.__show.getSubscriptions(): - if sub.name() == "clo.General": - val = sub.runningCores() / 100 - self.__history.append(val) - self.__max = max(self.__max, val + 80) - - if len(self.__history) > 100: - self.__history.pop(0) - - self.update() - - def paintEvent(self, event): - QtWidgets.QWidget.paintEvent(self, event) - - #Skip this if too small, if splitter is all the way over - - painter = QtGui.QPainter(self) - try: - rect = self.contentsRect().adjusted(5, 5, -5, -5) - painter.save() - - painter.fillRect(rect, - QtGui.qApp.palette().color(QtGui.QPalette.Base)) - - - - - if len(self.__history) > 1: - stepWidth = rect.width() / float(len(self.__history) - 1) - ratioHeight = (rect.height() - 2) / float(self.__max) - - # Box outline - painter.setPen(QtCore.Qt.black) - painter.drawRect(rect) - - painter.setPen(self.__color) - points = QtGui.QPolygon(len(self.__history) + 2) - - # Bottom left - #points.setPoint(0, rect.bottomLeft()) - points.setPoint(0, rect.left() + 1, rect.bottom()) - - # All the data points - num = 1 - for i in range(len(self.__history)): - points.setPoint(num, - max(rect.x() + 1, rect.x() - 1 + stepWidth * i), - rect.bottom() - ratioHeight * self.__history[i]) - num += 1 - - # Bottom right - points.setPoint(num, rect.bottomRight()) - - # Draw filled in - painter.setBrush(self.__brush) - painter.drawPolygon(points) - - # Draw subscription size line - painter.setPen(QtCore.Qt.red) - height = rect.bottom() - ratioHeight * self.__line - painter.drawLine(rect.left() + 1, - height, - rect.right() - 1, - height) - - finally: - painter.restore() - painter.end() - del painter diff --git a/cuegui/cuegui/GroupDialog.py b/cuegui/cuegui/GroupDialog.py index 332e51236..2c59f405c 100644 --- a/cuegui/cuegui/GroupDialog.py +++ b/cuegui/cuegui/GroupDialog.py @@ -14,6 +14,9 @@ # limitations under the License. +"""Dialog for creating or modifying a group.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -27,6 +30,8 @@ class GroupDialog(QtWidgets.QDialog): + """Base class for group dialogs.""" + def __init__(self, parentGroup, modifyGroup, defaults, parent): QtWidgets.QDialog.__init__(self, parent) layout = QtWidgets.QGridLayout(self) @@ -38,7 +43,7 @@ def __init__(self, parentGroup, modifyGroup, defaults, parent): self._departments = opencue.api.getDepartmentNames() try: self._departments = opencue.api.getDepartmentNames() - except Exception as e: + except opencue.exception.CueException: self._departments = ["Unknown"] __title = defaults["title"] @@ -88,7 +93,8 @@ def __init__(self, parentGroup, modifyGroup, defaults, parent): self.__createButtons( QtWidgets.QDialogButtonBox.Save | QtWidgets.QDialogButtonBox.Cancel, 8, 3) - def __createToggleDoubleSpinBox(self, text, row, startEnabled = False, currentValue = 0, minValue = 0): + def __createToggleDoubleSpinBox( + self, text, row, startEnabled = False, currentValue = 0, minValue = 0): inputWidget = QtWidgets.QDoubleSpinBox(self) inputWidget.setEnabled(startEnabled) inputWidget.setRange(minValue, 30000) @@ -96,7 +102,8 @@ def __createToggleDoubleSpinBox(self, text, row, startEnabled = False, currentVa inputWidget.setValue(currentValue) return self.__createToggleInput(text, row, inputWidget, startEnabled) - def __createToggleSpinBox(self, text, row, startEnabled = False, currentValue = 0, minValue = 0): + def __createToggleSpinBox( + self, text, row, startEnabled = False, currentValue = 0, minValue = 0): inputWidget = QtWidgets.QSpinBox(self) inputWidget.setEnabled(startEnabled) inputWidget.setRange(minValue, 30000) @@ -164,8 +171,8 @@ def accept(self): self.close() - def __setValue(self, checkBox, setter, newValue, currentValue, disableValue): - result = None + @staticmethod + def __setValue(checkBox, setter, newValue, currentValue, disableValue): if checkBox.isChecked(): result = newValue else: @@ -173,29 +180,37 @@ def __setValue(self, checkBox, setter, newValue, currentValue, disableValue): if result is not None and result != currentValue: setter(result) + class ModifyGroupDialog(GroupDialog): + """Dialog for modifying an existing group.""" + def __init__(self, modifyGroup, parent=None): modifyGroup = opencue.api.getGroup(modifyGroup.id()) - defaults = {"title": "Modify Group: %s" % modifyGroup.data.name, - "message": "Modifying the group %s" % modifyGroup.data.name, - "name": modifyGroup.data.name, - "department": modifyGroup.data.department, - "defaultJobPriority": modifyGroup.data.default_job_priority, - "defaultJobMinCores": modifyGroup.data.default_job_min_cores, - "defaultJobMaxCores": modifyGroup.data.default_job_max_cores, - "minCores": modifyGroup.data.min_cores, - "maxCores": modifyGroup.data.max_cores} + defaults = { + "title": "Modify Group: %s" % modifyGroup.data.name, + "message": "Modifying the group %s" % modifyGroup.data.name, + "name": modifyGroup.data.name, + "department": modifyGroup.data.department, + "defaultJobPriority": modifyGroup.data.default_job_priority, + "defaultJobMinCores": modifyGroup.data.default_job_min_cores, + "defaultJobMaxCores": modifyGroup.data.default_job_max_cores, + "minCores": modifyGroup.data.min_cores, + "maxCores": modifyGroup.data.max_cores} GroupDialog.__init__(self, None, modifyGroup, defaults, parent) + class NewGroupDialog(GroupDialog): + """Dialog for creating a new group.""" + def __init__(self, parentGroup, parent=None): - defaults = {"title": "Create New Group", - "message": "Group to be created as a child of the group %s" % parentGroup.name(), - "name": "", - "department": "Unknown", - "defaultJobPriority": 0, - "defaultJobMinCores": 1.0, - "defaultJobMaxCores": 1.0, - "minCores": 0.0, - "maxCores": 1.0} + defaults = { + "title": "Create New Group", + "message": "Group to be created as a child of the group %s" % parentGroup.name(), + "name": "", + "department": "Unknown", + "defaultJobPriority": 0, + "defaultJobMinCores": 1.0, + "defaultJobMaxCores": 1.0, + "minCores": 0.0, + "maxCores": 1.0} GroupDialog.__init__(self, parentGroup, None, defaults, parent) diff --git a/cuegui/cuegui/HostMonitor.py b/cuegui/cuegui/HostMonitor.py index 3ef2d1eed..0c467d004 100644 --- a/cuegui/cuegui/HostMonitor.py +++ b/cuegui/cuegui/HostMonitor.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Widget for displaying a list of hosts with admin controls.""" + + from __future__ import absolute_import from __future__ import print_function from __future__ import division @@ -36,10 +39,12 @@ class HostMonitor(QtWidgets.QWidget): - """This contains the frame list table with controls at the top""" + """Widget for displaying a list of hosts with admin controls.""" + def __init__(self, parent): QtWidgets.QWidget.__init__(self, parent) + self.__filterByHostNameLastInput = None self.hostMonitorTree = cuegui.HostMonitorTree.HostMonitorTree(self) # Setup main vertical layout @@ -52,40 +57,47 @@ def __init__(self, parent): # This hlayout would contain any filter/control buttons hlayout = QtWidgets.QHBoxLayout() - self.__filterByHostNameSetup(hlayout) # Menu to filter by host name - self.__filterAllocationSetup(hlayout) # Menu to filter by allocation - self.__filterHardwareStateSetup(hlayout) # Menu to filter by hardware state + self.__filterByHostNameSetup(hlayout) + self.__filterAllocationSetup(hlayout) + self.__filterHardwareStateSetup(hlayout) hlayout.addStretch() - self.__refreshToggleCheckBoxSetup(hlayout) # Checkbox to enable/disable auto refresh - self.__refreshButtonSetup(hlayout) # Button to refresh - self.__clearButtonSetup(hlayout) # Button to clear all filters + self.__refreshToggleCheckBoxSetup(hlayout) + self.__refreshButtonSetup(hlayout) + self.__clearButtonSetup(hlayout) self.layout().addLayout(hlayout) self.layout().addWidget(self.hostMonitorTree) - self.__viewHostsSetup() # For view_hosts signal - - if bool(int(QtGui.qApp.settings.value("AutoRefreshMonitorHost", 1))): # For refresh on launch + self.__viewHostsSetup() + + # pylint: disable=no-member + if bool(int(QtGui.qApp.settings.value("AutoRefreshMonitorHost", 1))): self.updateRequest() + # pylint: enable=no-member def updateRequest(self): + """Requests an update of the displayed information.""" self.hostMonitorTree.updateRequest() def getColumnVisibility(self): + """Gets table column visibility.""" return self.hostMonitorTree.getColumnVisibility() def setColumnVisibility(self, settings): + """Sets table column visibility.""" self.hostMonitorTree.setColumnVisibility(settings) def getColumnOrder(self): + """Gets the table column order.""" return self.hostMonitorTree.getColumnOrder() def setColumnOrder(self, settings): + """Sets the table column order.""" self.hostMonitorTree.setColumnOrder(settings) -# ============================================================================== -# Text box to filter by host name -# ============================================================================== + # ============================================================================== + # Text box to filter by host name + # ============================================================================== def __filterByHostNameSetup(self, layout): btn = QtWidgets.QLineEdit(self) btn.setMaximumHeight(FILTER_HEIGHT) @@ -124,9 +136,9 @@ def __filterByHostNameClear(self): self.__filterByHostName.setText("") self.hostMonitorTree.hostSearch.options['regex'] = [] -# ============================================================================== -# Menu to filter by allocation -# ============================================================================== + # ============================================================================== + # Menu to filter by allocation + # ============================================================================== def __filterAllocationSetup(self, layout): self.__filterAllocationList = sorted( [alloc.name() for alloc in opencue.api.getAllocations()]) @@ -188,9 +200,9 @@ def __filterAllocationHandle(self, action): self.hostMonitorTree.updateRequest() -# ============================================================================== -# Menu to filter by hardware state -# ============================================================================== + # ============================================================================== + # Menu to filter by hardware state + # ============================================================================== def __filterHardwareStateSetup(self, layout): self.__filterHardwareStateList = sorted(opencue.api.host_pb2.HardwareState.keys()) @@ -252,9 +264,9 @@ def __filterHardwareStateHandle(self, action): self.hostMonitorTree.updateRequest() -# ============================================================================== -# Checkbox to toggle auto-refresh -# ============================================================================== + # ============================================================================== + # Checkbox to toggle auto-refresh + # ============================================================================== def __refreshToggleCheckBoxSetup(self, layout): checkBox = QtWidgets.QCheckBox("Auto-refresh", self) layout.addWidget(checkBox) @@ -267,11 +279,13 @@ def __refreshToggleCheckBoxSetup(self, layout): def __refreshToggleCheckBoxHandle(self, state): self.hostMonitorTree.enableRefresh = bool(state) + # pylint: disable=no-member QtGui.qApp.settings.setValue("AutoRefreshMonitorHost", int(bool(state))) + # pylint: enable=no-member -# ============================================================================== -# Button to refresh -# ============================================================================== + # ============================================================================== + # Button to refresh + # ============================================================================== def __refreshButtonSetup(self, layout): """Sets up the refresh button, adds it to the given layout @param layout: The layout to add the button to @@ -292,9 +306,9 @@ def __refreshButtonDisableHandle(self): self.btn_refresh.setEnabled(False) QtCore.QTimer.singleShot(5000, self.__refreshButtonEnableHandle) -# ============================================================================== -# Button to clear all filters -# ============================================================================== + # ============================================================================== + # Button to clear all filters + # ============================================================================== def __clearButtonSetup(self, layout): """Sets up the clear button, adds it to the given layout @param layout: The layout to add the button to @@ -314,11 +328,13 @@ def __clearButtonHandle(self): self.__filterByHostNameClear() self.hostMonitorTree.clearFilters() -# ============================================================================== -# Monitors and handles the view_hosts signal -# ============================================================================== + # ============================================================================== + # Monitors and handles the view_hosts signal + # ============================================================================== def __viewHostsSetup(self): + # pylint: disable=no-member QtGui.qApp.view_hosts.connect(self.__viewHostsHandle) + # pylint: enable=no-member def __viewHostsHandle(self, hosts): self.__clearButtonHandle() diff --git a/cuegui/cuegui/HostMonitorTree.py b/cuegui/cuegui/HostMonitorTree.py index 6b88f8241..f9e5a7c90 100644 --- a/cuegui/cuegui/HostMonitorTree.py +++ b/cuegui/cuegui/HostMonitorTree.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -A frame list based on AbstractTreeWidget -""" +"""Tree widget for displaying a list of hosts.""" from __future__ import absolute_import @@ -23,7 +21,6 @@ from __future__ import print_function from builtins import map -from builtins import str import time from PySide2 import QtCore @@ -31,6 +28,9 @@ from PySide2 import QtWidgets import opencue +from opencue.compiled_proto.host_pb2 import HardwareState +from opencue.compiled_proto.host_pb2 import LockState +from opencue.compiled_proto.host_pb2 import ThreadMode import cuegui.AbstractTreeWidget import cuegui.AbstractWidgetItem @@ -41,10 +41,6 @@ import cuegui.Style import cuegui.Utils -from opencue.compiled_proto.host_pb2 import HardwareState -from opencue.compiled_proto.host_pb2 import LockState -from opencue.compiled_proto.host_pb2 import ThreadMode - logger = cuegui.Logger.getLogger(__file__) @@ -52,6 +48,8 @@ class HostMonitorTree(cuegui.AbstractTreeWidget.AbstractTreeWidget): + """Tree widget for displaying a list of hosts.""" + def __init__(self, parent): self.startColumnsForType(cuegui.Constants.TYPE_HOST) @@ -160,12 +158,17 @@ def __init__(self, parent): self.itemClicked.connect(self.__itemSingleClickedComment) # Don't use the standard space bar to refresh + # pylint: disable=no-member QtGui.qApp.request_update.connect(self.updateRequest) + # pylint: enable=no-member self.startTicksUpdate(40) # Don't start refreshing until the user sets a filter or hits refresh self.ticksWithoutUpdate = -1 + + # pylint: disable=no-member self.enableRefresh = bool(int(QtGui.qApp.settings.value("AutoRefreshMonitorHost", 1))) + # pylint: enable=no-member def tick(self): if self.ticksWithoutUpdate >= self.updateInterval and \ @@ -196,6 +199,8 @@ def __itemSingleClickedCopy(self, item, col): @param item: The item clicked on @type col: int @param col: The column clicked on""" + del item + del col selected = [host.data.name for host in self.selectedObjects() if cuegui.Utils.isHost(host)] if selected: QtWidgets.QApplication.clipboard().setText(",".join(selected)) @@ -212,6 +217,7 @@ def __itemSingleClickedComment(self, item, col): self.__menuActions.hosts().viewComments([host]) def clearFilters(self): + """Clears any sorting and filtering, restoring the default view.""" self.clearSelection() self.hostSearch = opencue.search.HostSearch() self.sortByColumn(0, QtCore.Qt.AscendingOrder) @@ -229,21 +235,21 @@ def _getUpdate(self): # Sorting by name here incase that makes displaying it faster hosts.sort(key=lambda host: host.data.name) return hosts - except Exception as e: + except opencue.exception.CueException as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) return [] - def _createItem(self, object, parent=None): + def _createItem(self, rpcObject, parent=None): """Creates and returns the proper item - @type object: Host - @param object: The object for this item + @type rpcObject: Host + @param rpcObject: The object for this item @type parent: QTreeWidgetItem @param parent: Optional parent for this item @rtype: QTreeWidgetItem @return: The created item""" if not parent: parent = self - return HostWidgetItem(object, parent) + return HostWidgetItem(rpcObject, parent) def contextMenuEvent(self, e): """When right clicking on an item, this raises a context menu""" @@ -274,20 +280,25 @@ def dragMoveEvent(self, event): class HostWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): + """Widget item representing a single host.""" + __initialized = False - def __init__(self, object, parent): + # pylint: disable=protected-access + def __init__(self, rpcObject, parent): if not self.__initialized: cuegui.Style.init() self.__class__.__initialized = True self.__class__.__commentIcon = QtGui.QIcon(":comment.png") + # pylint: disable=no-member self.__class__.__backgroundColor = QtGui.qApp.palette().color(QtGui.QPalette.Base) + # pylint: enable=no-member self.__class__.__foregroundColor = cuegui.Style.ColorTheme.COLOR_JOB_FOREGROUND self.__class__.__pausedColor = cuegui.Style.ColorTheme.COLOR_JOB_PAUSED_BACKGROUND self.__class__.__dyingColor = cuegui.Style.ColorTheme.COLOR_JOB_DYING_BACKGROUND self.__class__.__type = cuegui.Constants.TYPE_HOST cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( - self, cuegui.Constants.TYPE_HOST, object, parent) + self, cuegui.Constants.TYPE_HOST, rpcObject, parent) def data(self, col, role): """Returns the proper display data for the given column and role @@ -303,32 +314,32 @@ def data(self, col, role): self.column_info[col][cuegui.Constants.COLUMN_INFO_DISPLAY](self.rpcObject) return self._cache.get(col, cuegui.Constants.QVARIANT_NULL) - elif role == QtCore.Qt.ForegroundRole: + if role == QtCore.Qt.ForegroundRole: return self.__foregroundColor - elif role == QtCore.Qt.BackgroundRole: + if role == QtCore.Qt.BackgroundRole: if not self.rpcObject.data.state == opencue.api.host_pb2.UP: return self.__dyingColor if self.rpcObject.data.lock_state == opencue.api.host_pb2.LOCKED: return self.__pausedColor return self.__backgroundColor - elif role == QtCore.Qt.DecorationRole: + if role == QtCore.Qt.DecorationRole: if col == COMMENT_COLUMN and self.rpcObject.data.has_comment: return self.__commentIcon - elif role == QtCore.Qt.UserRole: + if role == QtCore.Qt.UserRole: return self.__type - elif role == QtCore.Qt.UserRole + 1: + if role == QtCore.Qt.UserRole + 1: return [self.rpcObject.data.total_swap - self.rpcObject.data.free_swap, self.rpcObject.data.total_swap] - elif role == QtCore.Qt.UserRole + 2: + if role == QtCore.Qt.UserRole + 2: return [self.rpcObject.data.total_memory - self.rpcObject.data.free_memory, self.rpcObject.data.total_memory] - elif role == QtCore.Qt.UserRole + 3: + if role == QtCore.Qt.UserRole + 3: return [self.rpcObject.data.total_gpu - self.rpcObject.data.free_gpu, self.rpcObject.data.total_gpu] diff --git a/cuegui/cuegui/ItemDelegate.py b/cuegui/cuegui/ItemDelegate.py index 2a554485e..27aff67ec 100644 --- a/cuegui/cuegui/ItemDelegate.py +++ b/cuegui/cuegui/ItemDelegate.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Custom delegate classes for drawing items in a tree widget.""" + + from __future__ import absolute_import from __future__ import print_function from __future__ import division @@ -28,6 +31,7 @@ import opencue import cuegui.Constants +import cuegui.Style RGB_FRAME_STATE = {opencue.api.job_pb2.SUCCEEDED: QtGui.QColor(55, 200, 55), @@ -50,14 +54,17 @@ class AbstractDelegate(QtWidgets.QItemDelegate): - """Handles drawing of items for the TreeWidget. Provides special handling + """Base delegate class. + + Handles drawing of items for the TreeWidget. Provides special handling for selected jobs in order to still display background color.""" + __colorInvalid = QtGui.QColor() __brushSelected = QtGui.QBrush(QtCore.Qt.Dense4Pattern) __colorUsed = QtGui.QColor(255, 0, 0) __colorFree = QtGui.QColor(0, 255, 0) - def __init__(self, parent, jobProgressBarColumn = None, *args): + def __init__(self, parent, *args): QtWidgets.QItemDelegate.__init__(self, parent, *args) def paint(self, painter, option, index): @@ -89,7 +96,8 @@ def _paintDifferenceBar(self, painter, option, index, used, total): painter.restore() del painter - def _drawProgressBar(self, painter, rect, frameStateTotals): + @staticmethod + def _drawProgressBar(painter, rect, frameStateTotals): """Returns the list that defines the column. @type painter: QPainter @param painter: The painter to draw with @@ -131,7 +139,8 @@ def _paintSelected(self, painter, option, index): painter.restore() del painter - def _drawBackground(self, painter, option, index): + @staticmethod + def _drawBackground(painter, option, index): # Draw the background color painter.setPen(NO_PEN) role = index.data(QtCore.Qt.BackgroundRole) @@ -152,6 +161,8 @@ def _drawSelectionOverlay(self, painter, option): class JobBookingBarDelegate(AbstractDelegate): + """Delegate for the job booking bar.""" + def __init__(self, parent, *args): AbstractDelegate.__init__(self, parent, *args) @@ -159,58 +170,61 @@ def paint(self, painter, option, index): # Only if job if index.data(QtCore.Qt.UserRole) == cuegui.Constants.TYPE_JOB and \ option.rect.width() > 30: - # This itemFromIndex could cause problems - # I need: minCores, maxCores, totalRunning, totalWaiting - job = self.parent().itemFromIndex(index).rpcObject + # This itemFromIndex could cause problems + # I need: minCores, maxCores, totalRunning, totalWaiting + job = self.parent().itemFromIndex(index).rpcObject - rect = option.rect.adjusted(12, 6, -12, -6) + rect = option.rect.adjusted(12, 6, -12, -6) - painter.save() - try: - self._drawBackground(painter, option, index) + painter.save() + try: + self._drawBackground(painter, option, index) + try: + jobRunning = job.data.job_stats.running_frames + jobWaiting = job.data.job_stats.waiting_frames + # pylint: disable=broad-except try: - jobRunning = job.data.job_stats.running_frames - jobWaiting = job.data.job_stats.waiting_frames - try: - cores_per_frame = float(job.data.job_stats.reserved_cores / jobRunning) - except: - cores_per_frame = float(6 / 1) - jobMin = int(job.data.min_cores / cores_per_frame) - jobMax = int(job.data.max_cores / cores_per_frame) - ratio = rect.width() / float(jobRunning + jobWaiting) - - if jobWaiting: - painter.fillRect( - rect.adjusted(0, 2, 0, -2), - RGB_FRAME_STATE[opencue.api.job_pb2.WAITING]) - - if jobRunning: - painter.fillRect( - rect.adjusted(0, 0, -int(ceil(ratio * jobWaiting)), 0), - RGB_FRAME_STATE[opencue.api.job_pb2.RUNNING]) - - painter.setPen(cuegui.Style.ColorTheme.PAUSE_ICON_COLOUR) - x = min(rect.x() + ratio * jobMin, option.rect.right() - 9) - painter.drawLine(x, option.rect.y(), x, - option.rect.y() + option.rect.height()) - - painter.setPen(cuegui.Style.ColorTheme.KILL_ICON_COLOUR) - x = min(rect.x() + ratio * jobMax, option.rect.right() - 6) - painter.drawLine(x, option.rect.y(), x, - option.rect.y() + option.rect.height()) - - except ZeroDivisionError: - pass - - finally: - painter.restore() - del painter + cores_per_frame = float(job.data.job_stats.reserved_cores / jobRunning) + except Exception: + cores_per_frame = float(6 / 1) + jobMin = int(job.data.min_cores / cores_per_frame) + jobMax = int(job.data.max_cores / cores_per_frame) + ratio = rect.width() / float(jobRunning + jobWaiting) + + if jobWaiting: + painter.fillRect( + rect.adjusted(0, 2, 0, -2), + RGB_FRAME_STATE[opencue.api.job_pb2.WAITING]) + + if jobRunning: + painter.fillRect( + rect.adjusted(0, 0, -int(ceil(ratio * jobWaiting)), 0), + RGB_FRAME_STATE[opencue.api.job_pb2.RUNNING]) + + painter.setPen(cuegui.Style.ColorTheme.PAUSE_ICON_COLOUR) + x = min(rect.x() + ratio * jobMin, option.rect.right() - 9) + painter.drawLine(x, option.rect.y(), x, + option.rect.y() + option.rect.height()) + + painter.setPen(cuegui.Style.ColorTheme.KILL_ICON_COLOUR) + x = min(rect.x() + ratio * jobMax, option.rect.right() - 6) + painter.drawLine(x, option.rect.y(), x, + option.rect.y() + option.rect.height()) + + except ZeroDivisionError: + pass + + finally: + painter.restore() + del painter else: AbstractDelegate.paint(self, painter, option, index) class SubBookingBarDelegate(AbstractDelegate): + """Delegate for the subscription booking bar.""" + def __init__(self, parent, *args): AbstractDelegate.__init__(self, parent, *args) @@ -261,6 +275,8 @@ def paint(self, painter, option, index): class JobThinProgressBarDelegate(AbstractDelegate): + """Delegate for a small job progress bar.""" + def __init__(self, parent, *args): AbstractDelegate.__init__(self, parent, *args) @@ -286,6 +302,8 @@ def paint(self, painter, option, index): class JobProgressBarDelegate(AbstractDelegate): + """Delegate for the fullsize job progress bar.""" + def __init__(self, parent, *args): AbstractDelegate.__init__(self, parent, *args) @@ -295,22 +313,20 @@ def paint(self, painter, option, index): frameStateTotals = index.data(QtCore.Qt.UserRole + 1) complete_tasks = frameStateTotals[opencue.api.job_pb2.SUCCEEDED] - total_tasks = sum([i for i in frameStateTotals.values()]) + total_tasks = sum(frameStateTotals.values()) proc = float(complete_tasks) * 100 / float(total_tasks) line = "{0:d} % ({1:d}/{2:d})".format(int(proc), int(complete_tasks), int(total_tasks)) - - state = index.data(QtCore.Qt.UserRole + 2) - paused = index.data(QtCore.Qt.UserRole + 3) painter.save() try: + # pylint: disable=broad-except try: self._drawProgressBar(painter, option.rect.adjusted(0, 2, 0, -2), frameStateTotals) painter.setPen(QtCore.Qt.black) painter.drawText(option.rect, QtCore.Qt.AlignCenter, line) - except Exception as e: + except Exception: painter.setPen(QtCore.Qt.red) painter.drawText(option.rect, QtCore.Qt.AlignCenter, "Gui Error") finally: @@ -321,6 +337,8 @@ def paint(self, painter, option, index): class HostSwapBarDelegate(AbstractDelegate): + """Delegate for the host swap field.""" + def __init__(self, parent, *args): AbstractDelegate.__init__(self, parent, *args) @@ -333,6 +351,8 @@ def paint(self, painter, option, index): class HostMemBarDelegate(AbstractDelegate): + """Delegate for the host memory field.""" + def __init__(self, parent, *args): AbstractDelegate.__init__(self, parent, *args) @@ -345,6 +365,8 @@ def paint(self, painter, option, index): class HostGpuBarDelegate(AbstractDelegate): + """Delegate for the host GPU field.""" + def __init__(self, parent, *args): AbstractDelegate.__init__(self, parent, *args) @@ -357,15 +379,17 @@ def paint(self, painter, option, index): class HostHistoryDelegate(AbstractDelegate): -#To use this delegate, the host item must have this: -#in __init__: -# self.coresHistory = [object.coresReserved()] -# def update(self, object = None, parent = None): -# if object: -# self.coresHistory.append(object.coresReserved()) -# if len(self.coresHistory) > 40: -# self.coresHistory.pop(0) -# AbstractWidgetItem.update(self, object, parent) + """Delegate for the host history field.""" + + # To use this delegate, the host item must have this: + # in __init__: + # self.coresHistory = [object.coresReserved()] + # def update(self, object = None, parent = None): + # if object: + # self.coresHistory.append(object.coresReserved()) + # if len(self.coresHistory) > 40: + # self.coresHistory.pop(0) + # AbstractWidgetItem.update(self, object, parent) def __init__(self, parent, *args): AbstractDelegate.__init__(self, parent, *args) @@ -396,7 +420,9 @@ def paint(self, painter, option, index): points.setPoint(0, option.rect.bottomLeft()) num = 1 for i in range(len(hostItem.coresHistory)): - points.setPoint(num, option.rect.x() + stepWidth * i, option.rect.bottom() - ratioHeight * hostItem.coresHistory[i]) + points.setPoint( + num, option.rect.x() + stepWidth * i, + option.rect.bottom() - ratioHeight * hostItem.coresHistory[i]) num += 1 points.setPoint(num, option.rect.bottomRight()) @@ -414,6 +440,8 @@ def paint(self, painter, option, index): class ItemDelegate(AbstractDelegate): + """Generic item delegate class.""" + def __init__(self, parent, *args): AbstractDelegate.__init__(self, parent, *args) @@ -422,6 +450,7 @@ def paint(self, painter, option, index): class ProgressDelegate(AbstractDelegate): + """Delegate for displaying layer progress.""" def __init__(self, parent, *args): AbstractDelegate.__init__(self, parent, *args) @@ -440,9 +469,12 @@ def paint(self, painter, option, index): opts.text = "{0:d} %".format(progress) opts.textVisible = True - QtWidgets.QApplication.style().drawControl(QtWidgets.QStyle.CE_ProgressBar, opts, painter) + QtWidgets.QApplication.style().drawControl( + QtWidgets.QStyle.CE_ProgressBar, opts, painter) else: AbstractDelegate.paint(self, painter, option, index) def sizeHint(self, option, index): + del option + del index return QtCore.QSize(12, 12) diff --git a/cuegui/cuegui/JobMonitorTree.py b/cuegui/cuegui/JobMonitorTree.py index 45318e77e..58773c0ac 100644 --- a/cuegui/cuegui/JobMonitorTree.py +++ b/cuegui/cuegui/JobMonitorTree.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -A monitored job list based on AbstractTreeWidget -""" +"""Tree widget to display a list of monitored jobs.""" from __future__ import absolute_import @@ -70,10 +68,14 @@ def displayState(job): class JobMonitorTree(cuegui.AbstractTreeWidget.AbstractTreeWidget): + """Tree widget to display a list of monitored jobs.""" + __loadMine = True view_object = QtCore.Signal(object) def __init__(self, parent): + self.ticksWithoutUpdate = 0 + self.startColumnsForType(cuegui.Constants.TYPE_JOB) self.addColumn("Job", 470, id=1, data=lambda job: job.data.name, @@ -87,8 +89,7 @@ def __init__(self, parent): tip="If the job has auto eating enabled, a pac-man icon\n" "will appear here and all frames that become dead will\n" "automatically be eaten.") -# self.addColumn("Group", 70, id=3, -# data=lambda job:("%s [%d]" % (job.group(), job.priority()))) + # pylint: disable=unnecessary-lambda self.addColumn("State", 80, id=4, data=lambda job: displayState(job), tip="The state of each job.\n" @@ -186,9 +187,12 @@ def __itemSingleClickedCopy(self, item, col): @param item: The item clicked on @type col: int @param col: The column clicked on""" - selected = [job.data.name for job in self.selectedObjects() if cuegui.Utils.isJob(job)] + del item + del col + selected = [job.data.name for job in self.selectedObjects() if cuegui.Utils.isJob(job)] if selected: - QtWidgets.QApplication.clipboard().setText(" ".join(selected), QtGui.QClipboard.Selection) + QtWidgets.QApplication.clipboard().setText( + " ".join(selected), QtGui.QClipboard.Selection) def __itemSingleClickedComment(self, item, col): """If the comment column is clicked on, and there is a comment on the @@ -235,13 +239,16 @@ def addJob(self, job): self.ticksLock.unlock() def getJobProxies(self): + """Gets a list of IDs of monitored jobs.""" return list(self._items.keys()) def _removeItem(self, item): """Removes an item from the TreeWidget without locking @param item: A tree widget item @type item: AbstractTreeWidgetItem""" + # pylint: disable=no-member QtGui.qApp.unmonitor.emit(item.rpcObject) + # pylint: enable=no-member cuegui.AbstractTreeWidget.AbstractTreeWidget._removeItem(self, item) self.__jobTimeLoaded.pop(item.rpcObject, "") @@ -249,7 +256,9 @@ def removeAllItems(self): """Notifies the other widgets of each item being unmonitored, then calls the the AbstractTreeWidget.removeAllItems like normal""" for proxy in list(self._items.keys()): + # pylint: disable=no-member QtGui.qApp.unmonitor.emit(proxy) + # pylint: enable=no-member if proxy in self.__jobTimeLoaded: del self.__jobTimeLoaded[proxy] cuegui.AbstractTreeWidget.AbstractTreeWidget.removeAllItems(self) @@ -390,25 +399,26 @@ def _getUpdate(self): objectKey = cuegui.Utils.getObjectKey(job) jobs[objectKey] = job - except Exception as e: + except opencue.exception.CueException as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) return None return jobs - def _processUpdate(self, work, jobObjects): - if jobObjects is None: + def _processUpdate(self, work, rpcObjects): + if rpcObjects is None: return self._itemsLock.lockForWrite() # include rpcObjects from self._items that are not in jobObjects for proxy, item in list(self._items.items()): - if not proxy in jobObjects: - jobObjects[proxy] = item.rpcObject + if not proxy in rpcObjects: + rpcObjects[proxy] = item.rpcObject try: - selectedKeys = [cuegui.Utils.getObjectKey(item.rpcObject) for item in self.selectedItems()] + selectedKeys = [ + cuegui.Utils.getObjectKey(item.rpcObject) for item in self.selectedItems()] scrolled = self.verticalScrollBar().value() # Store the creation time for the current item @@ -418,7 +428,7 @@ def _processUpdate(self, work, jobObjects): self._items = {} self.clear() - for proxy, job in list(jobObjects.items()): + for proxy, job in list(rpcObjects.items()): self._items[proxy] = JobWidgetItem(job, self.invisibleRootItem(), self.__jobTimeLoaded.get(proxy, None)) @@ -426,14 +436,18 @@ def _processUpdate(self, work, jobObjects): self._items[proxy].setUserColor(self.__userColors[proxy]) self.verticalScrollBar().setValue(scrolled) - [self._items[key].setSelected(True) for key in selectedKeys if key in self._items] - except Exception as e: + list(map(lambda key: self._items[key].setSelected(True), + [key for key in selectedKeys if key in self._items])) + + except opencue.exception.CueException as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) finally: self._itemsLock.unlock() + class JobWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): """Represents a job entry in the CueJobTreeWidget.""" + __initialized = False __commentIcon = None __eatIcon = None @@ -447,14 +461,18 @@ class JobWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): __centerAlign = None __type = None __userColor = None - def __init__(self, object, parent, created): + + # pylint: disable=protected-access + def __init__(self, rpcObject, parent, created): if not self.__initialized: if cuegui.Style.ColorTheme is None: cuegui.Style.init() self.__class__.__initialized = True self.__class__.__commentIcon = QtGui.QIcon(":comment.png") self.__class__.__eatIcon = QtGui.QIcon(":eat.png") + # pylint: disable=no-member self.__class__.__backgroundColor = QtGui.qApp.palette().color(QtGui.QPalette.Base) + # pylint: enable=no-member self.__class__.__foregroundColor = cuegui.Style.ColorTheme.COLOR_JOB_FOREGROUND self.__class__.__pausedColor = cuegui.Style.ColorTheme.COLOR_JOB_PAUSED_BACKGROUND self.__class__.__dyingColor = cuegui.Style.ColorTheme.COLOR_JOB_DYING_BACKGROUND @@ -470,33 +488,34 @@ def __init__(self, object, parent, created): self.created = created or time.time() cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( - self, cuegui.Constants.TYPE_JOB, object, parent) + self, cuegui.Constants.TYPE_JOB, rpcObject, parent) def setUserColor(self, color): + """Sets the color scheme.""" self.__userColor = color def data(self, col, role): if role == QtCore.Qt.DisplayRole: return self.column_info[col][cuegui.Constants.COLUMN_INFO_DISPLAY](self.rpcObject) - elif role == QtCore.Qt.ForegroundRole: + if role == QtCore.Qt.ForegroundRole: if col == 0: if self.created > time.time() - 5: return self.__newJobColor return self.__foregroundColor - elif role == QtCore.Qt.BackgroundRole and col == COLUMN_STATE: + if role == QtCore.Qt.BackgroundRole and col == COLUMN_STATE: if self.rpcObject.data.state == opencue.api.job_pb2.FINISHED: return self.__finishedColor - elif self.rpcObject.data.is_paused: + if self.rpcObject.data.is_paused: return self.__pausedColor - elif self.rpcObject.data.job_stats.dead_frames: + if self.rpcObject.data.job_stats.dead_frames: return self.__dyingColor return self.__backgroundColor - elif role == QtCore.Qt.BackgroundRole and self.__userColor: + if role == QtCore.Qt.BackgroundRole and self.__userColor: return self.__userColor - elif role == QtCore.Qt.FontRole and col == COLUMN_NAME: + if role == QtCore.Qt.FontRole and col == COLUMN_NAME: if self.created > time.time() - 5: return self.__newJobFont diff --git a/cuegui/cuegui/LayerDialog.py b/cuegui/cuegui/LayerDialog.py index a091cdba2..989df17cd 100644 --- a/cuegui/cuegui/LayerDialog.py +++ b/cuegui/cuegui/LayerDialog.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Dialog for editing a layer.""" + + from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -30,15 +33,14 @@ def warning(msg, parent=None): - """ - Utility method for poping up a warning. - """ + """Utility method for popping up a warning.""" box = QtWidgets.QMessageBox(parent) box.setText(msg) box.exec_() class EnableableItem(QtWidgets.QWidget): + """General class for widget items which can be enabled and disabled.""" def __init__(self, widget, enable, parent=None): QtWidgets.QWidget.__init__(self, parent) self.__widget = widget @@ -53,21 +55,22 @@ def __init__(self, widget, enable, parent=None): layout.addWidget(self.__widget) def getWidget(self): + """Gets the wrapped widget.""" return self.__widget def isEnabled(self): + """Gets the enabled state.""" return self.__checkbox.isChecked() - def enable(self, b): - self.__checkbox.setChecked(b) - self.__widget.setDisabled(b == False) + def enable(self, is_enabled): + """Sets the enabled state.""" + self.__checkbox.setChecked(is_enabled) + self.__widget.setDisabled(is_enabled is False) class LayerPropertiesItem(QtWidgets.QWidget): - """ - An key/value widget for populating a dialog box. - """ - def __init__(self, label, widget, stretch=True, help=None, parent=None): + """An key/value widget for populating a dialog box.""" + def __init__(self, label, widget, stretch=True, help_widget=None, parent=None): QtWidgets.QWidget.__init__(self, parent) self.__label = label self.__widget = widget @@ -78,14 +81,12 @@ def __init__(self, label, widget, stretch=True, help=None, parent=None): if stretch: layout.addStretch() layout.addWidget(self.__widget) - if help: - layout.addWidget(help) + if help_widget: + layout.addWidget(help_widget) class SlideSpinner(QtWidgets.QWidget): - """ - A QSlider and QSpinBox - """ + """A QSlider and QSpinBox.""" def __init__(self, parent=None): QtWidgets.QWidget.__init__(self, parent) @@ -101,12 +102,11 @@ def __init__(self, parent=None): class LayerPropertiesDialog(QtWidgets.QDialog): - """ - A dialog box for editing a layer. - """ + """Dialog for editing a layer.""" + def __init__(self, layers, parent=None): QtWidgets.QDialog.__init__(self, parent) - self.__layers = [opencue.api.getLayer(opencue.id(l)) for l in layers] + self.__layers = [opencue.api.getLayer(opencue.id(layer)) for layer in layers] self.setWindowTitle("Layer Properties") @@ -126,7 +126,7 @@ def __init__(self, layers, parent=None): self.__group = QtWidgets.QGroupBox("Resource Options", self) - ## Memory + # Memory self.__mem = SlideSpinner(self) self.__mem.slider.setMinimumWidth(200) self.__mem.slider.setRange(self.mem_min_kb, self.mem_max_kb) @@ -135,20 +135,22 @@ def __init__(self, layers, parent=None): self.__mem.spinner.setSuffix(" GB") self.__mem.spinner.setRange(self.mem_min_gb, self.mem_max_gb) - ## Cores + # Cores self.__core = QtWidgets.QDoubleSpinBox(self) self.__core.setDecimals(1) self.__core.setRange(0, int(self._cfg().get('max_cores', 16))) self.__core.setSingleStep(1) - ## Max cores + # Max cores self.__max_cores = QtWidgets.QSpinBox(self) self.__max_cores.setRange(0, int(self._cfg().get('max_cores', 16))) self.__max_cores.setSingleStep(1) - ## Disable this for everything except commander. + # Disable this for everything except commander. + # pylint: disable=no-member if QtGui.qApp.applicationName() != "CueCommander": self.__core.setDisabled(True) + # pylint: enable=no-member # Threads self.__thread = QtWidgets.QCheckBox(self) @@ -174,7 +176,7 @@ def __init__(self, layers, parent=None): # Tags self.__tags = LayerTagsWidget(self.__layers, self) - + # Limits self.__limits = LayerLimitsWidget(self.__layers, self) @@ -255,17 +257,18 @@ def __init__(self, layers, parent=None): self.layout().addWidget(self.__buttons) def _cfg(self): - ''' + """ Loads (if necessary) and returns the config values. Warns and returns an empty dict if there's a problem reading the config @return: The keys & values stored in the config file @rtype: dict - ''' + """ if not hasattr(self, '__config'): self.__config = cuegui.Utils.getResourceConfig() return self.__config + # pylint: disable=inconsistent-return-statements def verify(self): """ Verify the contents of all widgets. @@ -312,6 +315,7 @@ def apply(self): self.close() def getMaxMemory(self): + """Gets the layer max memory.""" result = 0 for layer in self.__layers: if layer.data.min_memory > result: @@ -319,9 +323,11 @@ def getMaxMemory(self): return result def getMaxGpu(self): + """Gets the layer max GPU.""" return max([layer.data.min_gpu // self.gpu_tick_kb for layer in self.__layers]) def getMinCores(self): + """Gets the layer min cores.""" result = 0 for layer in self.__layers: if layer.data.min_cores > result: @@ -329,6 +335,7 @@ def getMinCores(self): return result def getMaxCores(self): + """Gets the layer max cores.""" result = 0 for layer in self.__layers: if layer.data.max_cores > result: @@ -336,6 +343,7 @@ def getMaxCores(self): return result def getThreading(self): + """Gets whether the layer is threadable.""" result = False for layer in self.__layers: if layer.data.is_threadable: @@ -344,6 +352,7 @@ def getThreading(self): return result def getTimeout(self): + """Gets the layer timeout.""" result = 0 for layer in self.__layers: if layer.data.timeout > result: @@ -351,6 +360,7 @@ def getTimeout(self): return result def getTimeoutLLU(self): + """Gets the layer LLU timeout.""" result = 0 for layer in self.__layers: if layer.data.timeout_llu > result: @@ -358,6 +368,7 @@ def getTimeoutLLU(self): return result def getMemoryOptSetting(self): + """Gets whether the layer has memory optimizer enabled.""" result = False for layer in self.__layers: if layer.data.memory_optimizer_enabled: @@ -409,10 +420,10 @@ def apply(self): for layer in self.__layers: layer.setTags(tags) except opencue.CueException as e: - warning = QtWidgets.QMessageBox(self) - warning.setText("Error applying layer tags.") - warning.setDetailedText("%s" % e) - warning.exec_() + warning_dialog = QtWidgets.QMessageBox(self) + warning_dialog.setText("Error applying layer tags.") + warning_dialog.setDetailedText("%s" % e) + warning_dialog.exec_() def verify(self): """ @@ -422,7 +433,7 @@ def verify(self): warning("You must have at least 1 tag selected.") return False return True - + class LayerLimitsWidget(QtWidgets.QWidget): """ @@ -459,12 +470,14 @@ def apply(self): class LayerTagsDialog(QtWidgets.QDialog): + """Dialog for displayer a layer's tags.""" + def __init__(self, layers, parent=None): QtWidgets.QDialog.__init__(self, parent) self._tags_widget = LayerTagsWidget(layers=layers, parent=parent) - self.__warning = QtWidgets.QLabel("Warning: Changing these tags may cause " - "your job to not run any frames") + self.__warning = QtWidgets.QLabel( + 'Warning: Changing these tags may cause your job to not run any frames') self.__buttons = QtWidgets.QDialogButtonBox( QtWidgets.QDialogButtonBox.Save | QtWidgets.QDialogButtonBox.Cancel, QtCore.Qt.Horizontal, @@ -476,4 +489,3 @@ def __init__(self, layers, parent=None): def accept(self): self._tags_widget.apply() self.close() - diff --git a/cuegui/cuegui/LayerMonitorTree.py b/cuegui/cuegui/LayerMonitorTree.py index 19f21b78e..fdf0c249e 100644 --- a/cuegui/cuegui/LayerMonitorTree.py +++ b/cuegui/cuegui/LayerMonitorTree.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -A layer list based on AbstractTreeWidget -""" +"""Tree widget for displaying a list of layers.""" from __future__ import absolute_import @@ -35,12 +33,14 @@ def displayRange(layer): + """Returns a string representation of a layer's frame range.""" if layer.data.chunk_size != 1: return '%s chunked %s' % (layer.data.range, layer.data.chunk_size) return layer.data.range class LayerMonitorTree(cuegui.AbstractTreeWidget.AbstractTreeWidget): + """Tree widget for displaying a list of layers.""" handle_filter_layers_byLayer = QtCore.Signal(list) @@ -58,6 +58,7 @@ def __init__(self, parent): self.addColumn("Limits", 100, id=4, data=lambda layer: ",".join(layer.data.limits), tip="The limits that have been applied to this layer's frames.") + # pylint: disable=unnecessary-lambda self.addColumn("Range", 150, id=5, data=lambda layer: displayRange(layer), tip="The range of frames that the layer should render.") @@ -79,11 +80,12 @@ def __init__(self, parent): tip="The amount of gpu memory each frame in this layer\n" "will reserve for its use. Note that we may not have\n" "machines as much gpu memory as you request.") - self.addColumn("MaxRss", 60, id=9, - data=lambda layer: cuegui.Utils.memoryToString(layer.data.layer_stats.max_rss), - sort=lambda layer: layer.data.layer_stats.max_rss, - tip="Maximum amount of memory used by any frame in\n" - "this layer at any time since the job was launched.") + self.addColumn( + "MaxRss", 60, id=9, + data=lambda layer: cuegui.Utils.memoryToString(layer.data.layer_stats.max_rss), + sort=lambda layer: layer.data.layer_stats.max_rss, + tip="Maximum amount of memory used by any frame in\n" + "this layer at any time since the job was launched.") self.addColumn("Total", 40, id=10, data=lambda layer: layer.data.layer_stats.total_frames, sort=lambda layer: layer.data.layer_stats.total_frames, @@ -112,11 +114,11 @@ def __init__(self, parent): data=lambda layer: layer.data.layer_stats.dead_frames, sort=lambda layer: layer.data.layer_stats.dead_frames, tip="Total number of dead frames in this layer.") - self.addColumn("Avg", 65, id=17, - data=lambda layer: cuegui.Utils.secondsToHHMMSS(layer.data.layer_stats.avg_frame_sec), - sort=lambda layer: layer.data.layer_stats.avg_frame_sec, - tip="Average number of HOURS:MINUTES:SECONDS per frame\n" - "in this layer.") + self.addColumn( + "Avg", 65, id=17, + data=lambda layer: cuegui.Utils.secondsToHHMMSS(layer.data.layer_stats.avg_frame_sec), + sort=lambda layer: layer.data.layer_stats.avg_frame_sec, + tip="Average number of HOURS:MINUTES:SECONDS per frame\nin this layer.") self.addColumn("Tags", 100, id=18, data=lambda layer: " | ".join(layer.data.tags), tip="The tags define what resources may be booked on\n" @@ -147,6 +149,7 @@ def __init__(self, parent): self.__load = None self.startTicksUpdate(20, False, 60*60*24) + # pylint: disable=attribute-defined-outside-init def tick(self): if self.__load: self.__job = self.__load @@ -169,8 +172,10 @@ def updateRequest(self): since last updated""" self.ticksWithoutUpdate = 9999 + # pylint: disable=inconsistent-return-statements def setJob(self, job): - """Sets the current job + """Sets the current job. + @param job: Job can be None, a job object, or a job name. @type job: job, string, None""" if job is None: @@ -188,6 +193,7 @@ def __setJob(self, job): self.removeAllItems() def getJob(self): + """Gets the current job.""" return self.__job def _createItem(self, obj): @@ -221,7 +227,7 @@ def contextMenuEvent(self, e): if len(__selectedObjects) == 1: menu.addSeparator() self.__menuActions.layers().addAction(menu, "useLocalCores") - if len(set([layer.data.range for layer in __selectedObjects])) == 1: + if len({layer.data.range for layer in __selectedObjects}) == 1: self.__menuActions.layers().addAction(menu, "reorder") self.__menuActions.layers().addAction(menu, "stagger") @@ -238,9 +244,13 @@ def contextMenuEvent(self, e): menu.exec_(e.globalPos()) def __itemDoubleClickedFilterLayer(self, item, col): + del col self.handle_filter_layers_byLayer.emit([item.rpcObject.data.name]) + class LayerWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): - def __init__(self, object, parent): + """Widget item for displaying a single layer.""" + + def __init__(self, rpcObject, parent): cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( - self, cuegui.Constants.TYPE_LAYER, object, parent) + self, cuegui.Constants.TYPE_LAYER, rpcObject, parent) diff --git a/cuegui/cuegui/LimitSelectionWidget.py b/cuegui/cuegui/LimitSelectionWidget.py index 84c0c30f4..8a37e370e 100644 --- a/cuegui/cuegui/LimitSelectionWidget.py +++ b/cuegui/cuegui/LimitSelectionWidget.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -A Widget for displaying and editing Limits -""" +"""A widget for displaying and editing limits.""" from __future__ import absolute_import @@ -30,32 +28,28 @@ class LimitSelectionWidget(QtWidgets.QWidget): - """ - A Widget for displaying and editing Limits. Includes checkboxes for the given - list of limit options. - """ + """A widget for displaying and editing limits. + + Includes checkboxes for the given list of limit options.""" + def __init__(self, limits=None, parent=None): """ - A Widget for displaying and editing Limits - @param limits: The list of limits to include as checkboxes. @type limits: list @param parent: The parent widget for this LimitSelectionWidget. Default is None @type parent: QtWidgets.QWidget """ - QtWidgets.QWidget.__init__(self, parent) layout = QtWidgets.QGridLayout(self) - + self.limits = cuegui.AbstractDialog.CheckBoxSelectionMatrix( 'Limits', limits, [], self) layout.addWidget(self.limits, 0, 0, 1, 2) layout.setContentsMargins(0, 0, 0, 0) - + def enable_limits(self, limits_to_enable=None): - """ - Set the limit value based on the given list of limits. - + """Sets the limit value based on the given list of limits. + @param limits_to_enable: The list of limits to enable @type limits_to_enable: iter """ @@ -63,9 +57,8 @@ def enable_limits(self, limits_to_enable=None): self.limits.checkBoxes(limits_to_enable) def get_selected_limits(self): - """ - Returns the list of selected limits. - + """Returns the list of selected limits. + @return: The list of selected limits @rtype: list """ diff --git a/cuegui/cuegui/LimitsWidget.py b/cuegui/cuegui/LimitsWidget.py index 360eaec70..b7eaccd3b 100644 --- a/cuegui/cuegui/LimitsWidget.py +++ b/cuegui/cuegui/LimitsWidget.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Widget for managing limits.""" + + from __future__ import absolute_import from __future__ import print_function from __future__ import division @@ -28,115 +31,134 @@ import cuegui.Constants import cuegui.Logger import cuegui.MenuActions +import cuegui.Utils logger = cuegui.Logger.getLogger(__file__) class LimitsWidget(QtWidgets.QWidget): - def __init__(self, parent): - QtWidgets.QWidget.__init__(self, parent) - - self.__btnRefresh = QtWidgets.QPushButton("Refresh", self) - self.__btnRefresh.setFocusPolicy(QtCore.Qt.NoFocus) - self.__btnAddLimit = QtWidgets.QPushButton("Add Limit", self) - self.__btnAddLimit.setFocusPolicy(QtCore.Qt.NoFocus) - - self.__monitorLimits = LimitsTreeWidget(self) - - layout = QtWidgets.QGridLayout(self) - layout.setContentsMargins(0, 0, 0, 0) - layout.setSpacing(0) - layout.addWidget(self.__btnAddLimit, 0, 3) - layout.addWidget(self.__btnRefresh, 0, 2) - layout.addWidget(self.__monitorLimits, 2, 0, 3, 4) - - self.__btnAddLimit.clicked.connect(self.__addLimit) - self.__btnRefresh.clicked.connect(self.updateSoon) - - self.__menuActions = cuegui.MenuActions.MenuActions(self, self.updateSoon, list) - - def updateSoon(self): - self.__monitorLimits._update() - - def __addLimit(self): - self.__menuActions.limits().create() - self.updateSoon() - - def getColumnVisibility(self): - return self.__monitorLimits.getColumnVisibility() - - def setColumnVisibility(self, settings): - self.__monitorLimits.setColumnVisibility(settings) - - def getColumnOrder(self): - return self.__monitorLimits.getColumnOrder() - - def setColumnOrder(self, settings): - self.__monitorLimits.setColumnOrder(settings) + """Widget for managing limits.""" + + def __init__(self, parent): + QtWidgets.QWidget.__init__(self, parent) + + self.__btnRefresh = QtWidgets.QPushButton("Refresh", self) + self.__btnRefresh.setFocusPolicy(QtCore.Qt.NoFocus) + self.__btnAddLimit = QtWidgets.QPushButton("Add Limit", self) + self.__btnAddLimit.setFocusPolicy(QtCore.Qt.NoFocus) + + self.__monitorLimits = LimitsTreeWidget(self) + + layout = QtWidgets.QGridLayout(self) + layout.setContentsMargins(0, 0, 0, 0) + layout.setSpacing(0) + layout.addWidget(self.__btnAddLimit, 0, 3) + layout.addWidget(self.__btnRefresh, 0, 2) + layout.addWidget(self.__monitorLimits, 2, 0, 3, 4) + + self.__btnAddLimit.clicked.connect(self.__addLimit) + self.__btnRefresh.clicked.connect(self.updateSoon) + + self.__menuActions = cuegui.MenuActions.MenuActions(self, self.updateSoon, list) + + def updateSoon(self): + """Requests a refresh of the limits list.""" + # pylint: disable=protected-access + self.__monitorLimits._update() + + def __addLimit(self): + self.__menuActions.limits().create() + self.updateSoon() + + def getColumnVisibility(self): + """Gets the table column visibility.""" + return self.__monitorLimits.getColumnVisibility() + + def setColumnVisibility(self, settings): + """Sets the table column visibility.""" + self.__monitorLimits.setColumnVisibility(settings) + + def getColumnOrder(self): + """Gets the table column order.""" + return self.__monitorLimits.getColumnOrder() + + def setColumnOrder(self, settings): + """Sets the table column order.""" + self.__monitorLimits.setColumnOrder(settings) class LimitsTreeWidget(cuegui.AbstractTreeWidget.AbstractTreeWidget): - def __init__(self, parent): - self.startColumnsForType(cuegui.Constants.TYPE_LIMIT) - self.addColumn("Limit Name", 90, id=1, - data=lambda limit: limit.name()) - self.addColumn("Max Value", 80, id=2, - data=lambda limit: ("%d" % limit.maxValue()), - sort=lambda limit: limit.maxValue()) - self.addColumn("Current Running", 80, id=2, - data=lambda limit: ("%d" % limit.currentRunning()), - sort=lambda limit: limit.currentRunning()) - - cuegui.AbstractTreeWidget.AbstractTreeWidget.__init__(self, parent) - - # Used to build right click context menus - self.__menuActions = cuegui.MenuActions.MenuActions( - self, self.updateSoon, self.selectedObjects) - - self.itemClicked.connect(self.__itemSingleClickedToDouble) - QtGui.qApp.facility_changed.connect(self.__facilityChanged) - - self.setUpdateInterval(60) - - def __facilityChanged(self): - """Called when the facility is changed""" - self.removeAllItems() - self._update() - - def __itemSingleClickedToDouble(self, item, col): - """Called when an item is clicked on. Causes single clicks to be treated - as double clicks. - @type item: QTreeWidgetItem - @param item: The item single clicked on - @type col: int - @param col: Column number single clicked on""" - self.itemDoubleClicked.emit(item, col) - - def _createItem(self, object): - """Creates and returns the proper item""" - item = LimitWidgetItem(object, self) - return item - - def _getUpdate(self): - """Returns the proper data from the cuebot""" - try: - return opencue.api.getLimits() - except Exception as e: - logger.critical(e) - return [] - - def contextMenuEvent(self, e): - """When right clicking on an item, this raises a context menu""" - menu = QtWidgets.QMenu() - self.__menuActions.limits().addAction(menu, "editMaxValue") - menu.addSeparator() - self.__menuActions.limits().addAction(menu, "delete") - self.__menuActions.limits().addAction(menu, "rename") - menu.exec_(QtCore.QPoint(e.globalX(), e.globalY())) + """Tree widget for displaying a list of limits.""" + + def __init__(self, parent): + self.startColumnsForType(cuegui.Constants.TYPE_LIMIT) + self.addColumn("Limit Name", 90, id=1, + data=lambda limit: limit.name()) + self.addColumn("Max Value", 80, id=2, + data=lambda limit: ("%d" % limit.maxValue()), + sort=lambda limit: limit.maxValue()) + self.addColumn("Current Running", 80, id=2, + data=lambda limit: ("%d" % limit.currentRunning()), + sort=lambda limit: limit.currentRunning()) + + cuegui.AbstractTreeWidget.AbstractTreeWidget.__init__(self, parent) + + # Used to build right click context menus + self.__menuActions = cuegui.MenuActions.MenuActions( + self, self.updateSoon, self.selectedObjects) + + self.itemClicked.connect(self.__itemSingleClickedToDouble) + # pylint: disable=no-member + QtGui.qApp.facility_changed.connect(self.__facilityChanged) + # pylint: enable=no-member + + self.setUpdateInterval(60) + + def __facilityChanged(self): + """Called when the facility is changed""" + self.removeAllItems() + self._update() + + def __itemSingleClickedToDouble(self, item, col): + """Called when an item is clicked on. Causes single clicks to be treated + as double clicks. + @type item: QTreeWidgetItem + @param item: The item single clicked on + @type col: int + @param col: Column number single clicked on""" + self.itemDoubleClicked.emit(item, col) + + def _createItem(self, rpcObject): + """Creates and returns the proper item""" + item = LimitWidgetItem(rpcObject, self) + return item + + # pylint: disable=no-self-use + def _getUpdate(self): + """Returns the proper data from the cuebot""" + try: + return opencue.api.getLimits() + except opencue.exception.CueException as e: + list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) + return [] + + def contextMenuEvent(self, e): + """When right clicking on an item, this raises a context menu""" + menu = QtWidgets.QMenu() + self.__menuActions.limits().addAction(menu, "editMaxValue") + menu.addSeparator() + self.__menuActions.limits().addAction(menu, "delete") + self.__menuActions.limits().addAction(menu, "rename") + menu.exec_(QtCore.QPoint(e.globalX(), e.globalY())) + + def tick(self): + pass class LimitWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): - def __init__(self, object, parent): - cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( - self, cuegui.Constants.TYPE_LIMIT, object, parent) + """Widget item for displaying a single limit.""" + + def __init__(self, rpcObject, parent): + cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( + self, cuegui.Constants.TYPE_LIMIT, rpcObject, parent) diff --git a/cuegui/cuegui/LocalBooking.py b/cuegui/cuegui/LocalBooking.py index 351f049b6..475f0536f 100644 --- a/cuegui/cuegui/LocalBooking.py +++ b/cuegui/cuegui/LocalBooking.py @@ -13,6 +13,9 @@ # limitations under the License. +"""A widget for creating RenderPartitions, otherwise known as local core booking.""" + + from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -37,10 +40,7 @@ class LocalBookingWidget(QtWidgets.QWidget): - """ - A widget for creating opencue RenderParitions, otherwise know - as local core booking. - """ + """A widget for creating RenderPartitions, otherwise known as local core booking.""" hosts_changed = QtCore.Signal() @@ -65,7 +65,7 @@ def __init__(self, target, parent=None): for host in owner.getHosts(): if host.data.lockState != opencue.api.host_pb2.OPEN: self.__select_host.addItem(host.data.name) - except Exception: + except opencue.exception.CueException: pass self.__deed_button = None @@ -82,10 +82,10 @@ def __init__(self, target, parent=None): self.__text_target = QtWidgets.QLabel(self.__target.data.name, self) self.__num_threads = QtWidgets.QSpinBox(self) - self.__num_threads.setValue(1); + self.__num_threads.setValue(1) self.__num_cores = QtWidgets.QLineEdit(self) - self.__num_cores.setText("1"); + self.__num_cores.setText("1") self.__num_cores.setReadOnly(True) self.__num_frames = QtWidgets.QSpinBox(self) @@ -94,7 +94,7 @@ def __init__(self, target, parent=None): self.__frame_warn = QtWidgets.QLabel(self) self.__num_mem = QtWidgets.QSlider(self) - self.__num_mem.setValue(4); + self.__num_mem.setValue(4) self.__num_mem.setOrientation(QtCore.Qt.Horizontal) self.__num_mem.setTickPosition(QtWidgets.QSlider.TicksBelow) self.__num_mem.setTickInterval(1) @@ -124,7 +124,6 @@ def __init__(self, target, parent=None): self.__btn_clear = QtWidgets.QPushButton("Clear", self) - # # Setup the signals. # @@ -182,21 +181,23 @@ def __init__(self, target, parent=None): self.layout().addLayout(self.__stack) - ## Set initial values. + # Set initial values. self.__host_changed(self.__select_host.currentText()) self.resize(400, 400) def getTargetJobName(self): + """Gets the job name of the target.""" + if cuegui.Utils.isJob(self.__target): return self.__target.data.name - elif cuegui.Utils.isLayer(self.__target): + if cuegui.Utils.isLayer(self.__target): return self.__target.name - elif cuegui.Utils.isFrame(self.__target): + if cuegui.Utils.isFrame(self.__target): return self.__parent.getJob().data.name - else: - return '' + return '' def hostAvailable(self): + """Gets whether the host has cores available.""" return self.__select_host.count() > 0 def __host_changed(self, hostname): @@ -206,7 +207,7 @@ def __host_changed(self, hostname): host = opencue.api.findHost(str(hostname)) try: rp = [r for r in host.getRenderPartitions() if r.job == self.jobName] - + if rp: rp = rp[0] self.__stack.setCurrentIndex(1) @@ -223,15 +224,16 @@ def __host_changed(self, hostname): self.__num_threads.setRange(1, host.data.idleCores) self.__num_mem.setRange(1, int(host.data.totalMemory / 1024 / 1024)) self.__num_threads.setRange(1, host.data.idleCores) - except Exception as e: + except opencue.exception.CueException as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) def deedLocalhost(self): + """Deeds the target to the local host.""" show_name = os.environ.get("SHOW", "pipe") try: _show = opencue.api.findShow(show_name) - except Exception as e: + except opencue.exception.CueException as e: msg = QtWidgets.QMessageBox(self) msg.setText("Error %s, please setshot and rerun cuetopia", e) msg.exec_() @@ -243,7 +245,7 @@ def deedLocalhost(self): except opencue.EntityNotFoundException as e: # Owner does not exist owner = _show.createOwner(user) - + hostname = gethostname() try: host = opencue.api.findHost(hostname.rsplit(".",2)[0]) @@ -259,14 +261,14 @@ def deedLocalhost(self): self.__msg_widget = None self.hosts_changed.emit() - except Exception as e: + except opencue.exception.CueException: msg = QtWidgets.QMessageBox(self) msg.setText("Unable to determine your machine's hostname. " + "It is not setup properly for local booking") - msg.exec_() def __calculateCores(self, ignore): + del ignore frames = self.__num_frames.value() threads = self.__num_threads.value() @@ -284,16 +286,18 @@ def __hasError(self): if frames * threads > self.__num_frames.maximum(): return True - elif frames == 0: + if frames == 0: return True - elif cores % threads > 0: + if cores % threads > 0: return True - elif threads > cores: + if threads > cores: return True return False def clearCurrentHost(self): + """Clears the current host.""" + hostname = str(self.__select_host.currentText()) if not hostname: return @@ -305,23 +309,25 @@ def clearCurrentHost(self): rp = [r for r in host.getRenderPartitions() if r.job == self.jobName] if rp: rp = rp[0] - + rp.delete() - ## Wait for hosts to clear out, then switch - ## back to the booking widget - for i in range(0, 10): + # Wait for hosts to clear out, then switch back to the booking widget. + for _ in range(0, 10): + # pylint: disable=broad-except try: rp = [r for r in host.getRenderPartitions() if r.job == self.jobName][0] time.sleep(1) - except: + except Exception: break self.__host_changed(hostname) - except Exception as e: + except opencue.exception.CueException as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) def bookCurrentHost(self): + """Books the current host.""" + if self.__hasError(): return @@ -333,17 +339,14 @@ def bookCurrentHost(self): int(self.__run_mem.value()) * 1024 * 1024, 0) else: - self.__target.addRenderPartition(str(self.__select_host.currentText()), - int(self.__num_threads.value()), - int(self.__num_cores.text()), - int(self.__num_mem.value() * 1048576), - 0) + self.__target.addRenderPartition( + str(self.__select_host.currentText()), int(self.__num_threads.value()), + int(self.__num_cores.text()), int(self.__num_mem.value() * 1048576), 0) class LocalBookingDialog(QtWidgets.QDialog): - """ - A dialog to wrap a LocalBookingWidget. Provides action buttons. - """ + """A dialog to wrap a LocalBookingWidget. Provides action buttons.""" + def __init__(self, target, parent=None): QtWidgets.QDialog.__init__(self, parent) QtWidgets.QVBoxLayout(self) @@ -372,6 +375,8 @@ def __updateOkButtion(self): self.__btn_ok.setDisabled(not self.__booking.hostAvailable()) def doLocalBooking(self): + """Performs the booking of local cores..""" + # pylint: disable=broad-except try: self.__booking.bookCurrentHost() self.close() @@ -382,4 +387,3 @@ def doLocalBooking(self): 'that your job has waiting frames.') msg.setDetailedText(str(e)) msg.exec_() - diff --git a/cuegui/cuegui/Main.py b/cuegui/cuegui/Main.py index 12575c82b..b2c540c74 100644 --- a/cuegui/cuegui/Main.py +++ b/cuegui/cuegui/Main.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -Main entry point for the application. -""" +"""Main entry point for the application.""" from __future__ import absolute_import @@ -23,6 +21,8 @@ from __future__ import division import os +import shutil +import signal from PySide2 import QtCore from PySide2 import QtGui @@ -42,6 +42,7 @@ class CueGuiApplication(QtWidgets.QApplication): + """The CueGUI application.""" # Global signals display_log_file_content = QtCore.Signal(object) @@ -56,19 +57,20 @@ class CueGuiApplication(QtWidgets.QApplication): status = QtCore.Signal() quit = QtCore.Signal() - def __init__(self, *args, **kwargs): - super(CueGuiApplication, self).__init__(*args, **kwargs) - def cuetopia(argv): + """Starts the Cuetopia window.""" startup("Cuetopia", cuegui.Constants.VERSION, argv) def cuecommander(argv): + """Starts the CueCommander window.""" startup("CueCommander", cuegui.Constants.VERSION, argv) def startup(app_name, app_version, argv): + """Starts an application window.""" + app = CueGuiApplication(argv) # Start splash screen @@ -76,7 +78,6 @@ def startup(app_name, app_version, argv): app, app_name, app_version, cuegui.Constants.RESOURCE_PATH) # Allow ctrl-c to kill the application - import signal signal.signal(signal.SIGINT, signal.SIG_DFL) # Load window icon @@ -99,20 +100,21 @@ def startup(app_name, app_version, argv): cuegui.Style.init() - # If the config file does not exist, copy over the default + # If the config file does not exist, copy over the default + # pylint: disable=broad-except if not os.path.exists(local): default = os.path.join(cuegui.Constants.DEFAULT_INI_PATH, "%s.ini" % app_name.lower()) - logger.warning('Not found: %s\nCopying: %s' % (local, default)) + logger.warning('Not found: %s\nCopying: %s', local, default) try: os.mkdir(os.path.dirname(local)) except Exception as e: logger.debug(e) try: - import shutil shutil.copy2(default, local) except Exception as e: logger.debug(e) settings.sync() + # pylint: enable=broad-except mainWindow = cuegui.MainWindow.MainWindow(app_name, app_version, None) mainWindow.displayStartupNotice() @@ -125,10 +127,12 @@ def startup(app_name, app_version, argv): # End splash screen splash.hide() - + # TODO(#609) Refactor the CueGUI classes to make this garbage collector # replacement unnecessary. + # pylint: disable=unused-variable gc = cuegui.GarbageCollector.GarbageCollector(parent=app, debug=False) + # pylint: enable=unused-variable app.aboutToQuit.connect(closingTime) app.exec_() @@ -136,5 +140,8 @@ def startup(app_name, app_version, argv): def closingTime(): """Window close callback.""" logger.info("Closing all threads...") - for thread in QtGui.qApp.threads: + # pylint: disable=no-member + threads = QtGui.qApp.threads + # pylint: enable=no-member + for thread in threads: cuegui.Utils.shutdownThread(thread) diff --git a/cuegui/cuegui/MainWindow.py b/cuegui/cuegui/MainWindow.py index 50565e8c6..35891e2fd 100644 --- a/cuegui/cuegui/MainWindow.py +++ b/cuegui/cuegui/MainWindow.py @@ -13,9 +13,9 @@ # limitations under the License. -""" -All windows are an instance of this MainWindow. -""" +"""The main window of the application. Multiple windows may exist. + +All CueGUI windows are an instance of this MainWindow.""" from __future__ import absolute_import @@ -26,6 +26,7 @@ from builtins import str from builtins import range import sys +import time from PySide2 import QtCore from PySide2 import QtGui @@ -44,6 +45,7 @@ class MainWindow(QtWidgets.QMainWindow): """The main window of the application. Multiple windows may exist.""" + windows = [] windows_names = [] windows_titles = {} @@ -52,9 +54,15 @@ class MainWindow(QtWidgets.QMainWindow): def __init__(self, app_name, app_version, window_name, parent = None): QtWidgets.QMainWindow.__init__(self, parent) - # Setup variables + self.__actions_facility = {} + self.facility_default = None + self.facility_dict = None + self.windowMenu = None + self.qApp = QtGui.qApp + # pylint: disable=no-member self.settings = QtGui.qApp.settings + # pylint: enable=no-member self.windows_names = [app_name] + ["%s_%s" % (app_name, num) for num in range(2, 5)] self.app_name = app_name self.app_version = app_version @@ -80,18 +88,22 @@ def __init__(self, app_name, app_version, window_name, parent = None): self.__createMenus() # Setup plugins + # pylint: disable=no-member self.__plugins = cuegui.Plugins.Plugins(self, self.name) + # pylint: enable=no-member self.__plugins.setupPluginMenu(self.PluginMenu) # Restore saved settings self.__restoreSettings() + # pylint: disable=no-member QtGui.qApp.status.connect(self.showStatusBarMessage) + # pylint: enable=no-member self.showStatusBarMessage("Ready") def displayStartupNotice(self): - import time + """Displays the application startup notice.""" now = int(time.time()) lastView = int(self.settings.value("LastNotice", 0)) if lastView < cuegui.Constants.STARTUP_NOTICE_DATE: @@ -100,26 +112,35 @@ def displayStartupNotice(self): self.settings.setValue("LastNotice", now) def showStatusBarMessage(self, message, delay=5000): + """Shows a message on the status bar.""" self.statusBar().showMessage(str(message), delay) def displayAbout(self): + """Displays about text.""" msg = self.app_name + "\n\nA opencue tool\n\n" msg += "Qt:\n%s\n\n" % QtCore.qVersion() msg += "Python:\n%s\n\n" % sys.version QtWidgets.QMessageBox.about(self, "About", msg) - def openSuggestionPage(self): + @staticmethod + def openSuggestionPage(): + """Opens the suggestion page URL.""" cuegui.Utils.openURL(cuegui.Constants.URL_SUGGESTION) - def openBugPage(self): + @staticmethod + def openBugPage(): + """Opens the bug report page.""" cuegui.Utils.openURL(cuegui.Constants.URL_BUG) - def openUserGuide(self): + @staticmethod + def openUserGuide(): + """Opens the user guide page.""" cuegui.Utils.openURL(cuegui.Constants.URL_USERGUIDE) -################################################################################ -# Handles facility menu -################################################################################ + ################################################################################ + # Handles facility menu + ################################################################################ + def __facilityMenuSetup(self, menu): """Creates the facility menu actions @param menu: The QMenu that the actions should be added to @@ -163,10 +184,12 @@ def __facilityMenuHandle(self, action): for facility in list(self.__actions_facility.values()): if facility.isChecked(): opencue.Cuebot.setFacility(str(facility.text())) + # pylint: disable=no-member QtGui.qApp.facility_changed.emit() + # pylint: enable=no-member return -################################################################################ + ################################################################################ def __createMenus(self): """Creates the menus at the top of the window""" @@ -186,11 +209,11 @@ def __createMenus(self): self.fileMenu.addAction(close) # Menu Bar: File -> Exit Application - exit = QtWidgets.QAction(QtGui.QIcon('icons/exit.png'), 'E&xit Application', self) - exit.setShortcut('Ctrl+Q') - exit.setStatusTip('Exit application') - exit.triggered.connect(self.__windowCloseApplication) - self.fileMenu.addAction(exit) + exitAction = QtWidgets.QAction(QtGui.QIcon('icons/exit.png'), 'E&xit Application', self) + exitAction.setShortcut('Ctrl+Q') + exitAction.setStatusTip('Exit application') + exitAction.triggered.connect(self.__windowCloseApplication) + self.fileMenu.addAction(exitAction) self.__windowMenuSetup(self.windowMenu) @@ -222,9 +245,9 @@ def __createMenus(self): about.triggered.connect(self.displayAbout) self.helpMenu.addAction(about) -################################################################################ -# Handles adding windows -################################################################################ + ################################################################################ + # Handles adding windows + ################################################################################ def __windowMenuSetup(self, menu): """Creates the menu items for dealing with multiple main windows""" @@ -336,19 +359,26 @@ def windowMenuOpenWindow(self, name): def __windowOpened(self): """Called from __init__ on window creation""" + # pylint: disable=no-member self.qApp.quit.connect(self.close) self.windows.append(self) self.qApp.closingApp = False + # pylint: enable=no-member def __windowClosed(self): """Called from closeEvent on window close""" # Disconnect to avoid multiple attempts to close a window + # pylint: disable=no-member self.qApp.quit.connect(self.close) + # pylint: enable=no-member # Save the fact that this window is open or not when the app closed + # pylint: disable=no-member self.settings.setValue("%s/Open" % self.name, self.qApp.closingApp) + # pylint: enable=no-member + # pylint: disable=bare-except try: self.windows.remove(self) except: @@ -362,14 +392,17 @@ def __windowCloseWindow(self): def __windowCloseApplication(self): """Called when the entire application should exit. Signals other windows to exit.""" + # pylint: disable=no-member self.qApp.closingApp = True self.qApp.quit.emit() + # pylint: enable=no-member -################################################################################ + ################################################################################ def __toggleFullscreenSetup(self, menu): # Menu Bar: Window -> Toggle Full-Screen - fullscreen = QtWidgets.QAction(QtGui.QIcon('icons/fullscreen.png'), 'Toggle Full-Screen', self) + fullscreen = QtWidgets.QAction( + QtGui.QIcon('icons/fullscreen.png'), 'Toggle Full-Screen', self) fullscreen.setShortcut('Ctrl+F') fullscreen.setStatusTip('Toggle Full-Screen') fullscreen.triggered.connect(self.__toggleFullscreen) @@ -383,16 +416,20 @@ def __toggleFullscreen(self): else: self.showFullScreen() -################################################################################ + ################################################################################ + def keyPressEvent(self, event): if event.key() == QtCore.Qt.Key_Space: + # pylint: disable=no-member QtGui.qApp.request_update.emit() + # pylint: enable=no-member event.accept() def closeEvent(self, event): """Called when the window is closed @type event: QEvent @param event: The close event""" + del event self.__saveSettings() self.__windowClosed() @@ -411,7 +448,7 @@ def __restoreSettings(self): def __saveSettings(self): """Saves the windows settings""" - logger.info('Saving: %s' % self.settings.fileName()) + logger.info('Saving: %s', self.settings.fileName()) self.__plugins.saveState() @@ -427,7 +464,7 @@ def __saveSettings(self): self.size()) self.settings.setValue("%s/Position" % self.name, self.pos()) - + def __revertLayout(self): """Revert back to default window layout""" result = QtWidgets.QMessageBox.question( diff --git a/cuegui/cuegui/MenuActions.py b/cuegui/cuegui/MenuActions.py index 8f6fac07f..5c9b63139 100644 --- a/cuegui/cuegui/MenuActions.py +++ b/cuegui/cuegui/MenuActions.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -Provides actions and functions for right click menu items. -""" +"""Provides actions and functions for right click menu items.""" from __future__ import absolute_import @@ -38,6 +36,7 @@ import opencue import opencue.compiled_proto.job_pb2 +# pylint: disable=cyclic-import import cuegui.Action import cuegui.Comments import cuegui.Constants @@ -64,11 +63,15 @@ TOOLTIP = 1 ICON = 2 -# New icons are here: /usr/share/icons/crystalsvg/16x16 + +# pylint: disable=missing-function-docstring,no-self-use,unused-argument class AbstractActions(object): + """Parent class for all job-specific actions classes.""" + __iconCache = {} + def __init__(self, caller, updateCallable, selectedRpcObjectsCallable, sourceCallable): self._caller = caller self.__selectedRpcObjects = selectedRpcObjectsCallable @@ -109,13 +112,15 @@ def _getOnlyProcObjects(self, rpcObjects): def _getOnlyTaskObjects(self, rpcObjects): return list(filter(cuegui.Utils.isTask, self._getSelected(rpcObjects))) - def createAction(self, menu, title, tip = None, callback = None, icon = None): + def createAction(self, menu, title, tip=None, callback=None, icon=None): + """Creates a context menu action.""" if not tip: tip = title menu.addAction(cuegui.Action.create(menu, title, tip, callback, icon)) def addAction(self, menu, actionName, callback = None): - """Adds the requested menu item to the menu + """Adds the requested menu item to the menu. + @param menu: The menu that the action will be added to @type menu: QMenu @param actionName: The name of the action to add to the menu @@ -139,6 +144,7 @@ def addAction(self, menu, actionName, callback = None): else: icon_key = info[ICON] if icon_key not in self.__iconCache: + # pylint: disable=unidiomatic-typecheck if type(info[ICON]) is QtGui.QColor: pixmap = QtGui.QPixmap(100, 100) pixmap.fill(info[ICON]) @@ -158,30 +164,31 @@ def addAction(self, menu, actionName, callback = None): menu.addAction(self.__actionCache[key]) - def cuebotCall(self, callable, errorMessageTitle, *args): - """Makes the given call to the cuebot and if an exception occurs display - a critical message box. - @type callable: function - @param callable: The cuebot function to call. + def cuebotCall(self, functionToCall, errorMessageTitle, *args): + """Makes the given call to the Cuebot, displaying exception info if needed. + + @type functionToCall: function + @param functionToCall: The cuebot function to call. @type errorMessageTitle: string @param errorMessageTitle: The text to display in the title of the error message box. - @type callable: Variable arguments - @param callable: The arguments to pass to the callable + @type args: list + @param args: The arguments to pass to the callable @rtype: callable return type @return: Returns any results from the callable or None on exception""" try: - return callable(*args) - except Exception as e: + return functionToCall(*args) + except opencue.exception.CueException as e: logger.exception('Failed Cuebot call') QtWidgets.QMessageBox.critical(self._caller, errorMessageTitle, - e.details(), + str(e), QtWidgets.QMessageBox.Ok) return None def getText(self, title, body, default): """Prompts the user for text input. + @type title: string @param title: The title to display in the input dialog @type body: string @@ -190,40 +197,46 @@ def getText(self, title, body, default): @param default: The default text to provide in the input dialog @rtype: tuple(str, bool) @return: (input, choice)""" - (input, choice) = QtWidgets.QInputDialog.getText(self._caller, - title, - body, - QtWidgets.QLineEdit.Normal, - default) - return str(input), choice + (user_input, choice) = QtWidgets.QInputDialog.getText( + self._caller, title, body, QtWidgets.QLineEdit.Normal, default) + return str(user_input), choice class JobActions(AbstractActions): + """Actions for jobs.""" + def __init__(self, *args): AbstractActions.__init__(self, *args) unmonitor_info = ["Unmonitor", "Unmonitor selected jobs", "eject"] + def unmonitor(self, rpcObjects=None): self._caller.actionRemoveSelectedItems() view_info = ["View Job", None, "view"] + def view(self, rpcObjects=None): for job in self._getOnlyJobObjects(rpcObjects): + # pylint: disable=no-member QtGui.qApp.view_object.emit(job) + # pylint: enable=no-member viewDepends_info = ["&View Dependencies...", None, "log"] + def viewDepends(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) cuegui.DependDialog.DependDialog(jobs[0], self._caller).show() emailArtist_info = ["Email Artist...", None, "mail"] + def emailArtist(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) if jobs: - cuegui.EmailDialog.EmailDialog(jobs[0], [], self._caller).show() + cuegui.EmailDialog.EmailDialog(jobs[0], self._caller).show() setMinCores_info = ["Set Minimum Cores...", "Set Job(s) Minimum Cores", "configure"] + def setMinCores(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) if jobs: @@ -240,6 +253,7 @@ def setMinCores(self, rpcObjects=None): self._update() setMaxCores_info = ["Set Maximum Cores...", "Set Job(s) Maximum Cores", "configure"] + def setMaxCores(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) if jobs: @@ -256,6 +270,7 @@ def setMaxCores(self, rpcObjects=None): self._update() setPriority_info = ["Set Priority...", None, "configure"] + def setPriority(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) if jobs: @@ -272,11 +287,13 @@ def setPriority(self, rpcObjects=None): self._update() setMaxRetries_info = ["Set Max Retries...", None, "configure"] + def setMaxRetries(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) if jobs: title = "Set Max Retries" - body = "Please enter the number of retries that a frame should be allowed before it becomes dead:" + body = ('Please enter the number of retries that a frame should be ' + 'allowed before it becomes dead:') (value, choice) = QtWidgets.QInputDialog.getInt(self._caller, title, body, 0, 0, 10, 1) @@ -286,6 +303,7 @@ def setMaxRetries(self, rpcObjects=None): self._update() pause_info = ["&Pause", None, "pause"] + def pause(self, rpcObjects=None): """pause selected jobs""" jobs = self._getOnlyJobObjects(rpcObjects) @@ -295,6 +313,7 @@ def pause(self, rpcObjects=None): self._update() resume_info = ["&Unpause", None, "unpause"] + def resume(self, rpcObjects=None): """resume selected jobs""" jobs = self._getOnlyJobObjects(rpcObjects) @@ -304,6 +323,7 @@ def resume(self, rpcObjects=None): self._update() kill_info = ["&Kill", None, "kill"] + def kill(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) if jobs: @@ -315,6 +335,7 @@ def kill(self, rpcObjects=None): self._update() eatDead_info = ["Eat dead frames", None, "eat"] + def eatDead(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) if jobs: @@ -326,6 +347,7 @@ def eatDead(self, rpcObjects=None): self._update() autoEatOn_info = ["Enable auto eating", None, "eat"] + def autoEatOn(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) if jobs: @@ -335,6 +357,7 @@ def autoEatOn(self, rpcObjects=None): self._update() autoEatOff_info = ["Disable auto eating", None, "eat"] + def autoEatOff(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) if jobs: @@ -343,6 +366,7 @@ def autoEatOff(self, rpcObjects=None): self._update() retryDead_info = ["Retry dead frames", None, "retry"] + def retryDead(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) if jobs: @@ -355,6 +379,7 @@ def retryDead(self, rpcObjects=None): self._update() dropExternalDependencies_info = ["Drop External Dependencies", None, "kill"] + def dropExternalDependencies(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) if jobs: @@ -366,6 +391,7 @@ def dropExternalDependencies(self, rpcObjects=None): self._update() dropInternalDependencies_info = ["Drop Internal Dependencies", None, "kill"] + def dropInternalDependencies(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) if jobs: @@ -377,12 +403,14 @@ def dropInternalDependencies(self, rpcObjects=None): self._update() viewComments_info = ["Comments...", None, "comment"] + def viewComments(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) if jobs: cuegui.Comments.CommentListDialog(jobs[0], self._caller).show() dependWizard_info = ["Dependency &Wizard...", None, "configure"] + def dependWizard(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) if jobs: @@ -399,58 +427,60 @@ def __getJobRange(self, job): return (min(__minRange), max(__maxRange)) reorder_info = ["Reorder Frames...", None, "configure"] + def reorder(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) - if not jobs: return + if not jobs: + return __job = jobs[0] (__minRange, __maxRange) = self.__getJobRange(__job) title = "Reorder %s" % __job.data.name body = "What frame range should be reordered?" - (range, choice) = self.getText(title, body, "%s-%s" % (__minRange, __maxRange)) - if not choice: return + (frame_range, choice) = self.getText(title, body, "%s-%s" % (__minRange, __maxRange)) + if not choice: + return - body = "What order should the range %s take?" % range + body = "What order should the range %s take?" % frame_range items = list(opencue.compiled_proto.job_pb2.Order.keys()) - (order, choice) = QtWidgets.QInputDialog.getItem(self._caller, - title, - body, - sorted(items), - 0, - False) + (order, choice) = QtWidgets.QInputDialog.getItem( + self._caller, title, body, sorted(items), 0, False) if not choice: return self.cuebotCall( __job.reorderFrames, "Reorder Frames Failed", - range, getattr(opencue.compiled_proto.job_pb2, str(order))) + frame_range, getattr(opencue.compiled_proto.job_pb2, str(order))) stagger_info = ["Stagger Frames...", None, "configure"] + def stagger(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) - if not jobs: return + if not jobs: + return __job = jobs[0] (__minRange, __maxRange) = self.__getJobRange(__job) title = "Stagger %s" % __job.data.name body = "What frame range should be staggered?" - (range, choice) = self.getText(title, body, "%s-%s" % (__minRange, __maxRange)) - if not choice: return + (frameRange, choice) = self.getText(title, body, "%s-%s" % (__minRange, __maxRange)) + if not choice: + return - body = "What increment should the range %s be staggered?" % range - (increment, choice) = QtWidgets.QInputDialog.getInt(self._caller, - title, body, - 1, - 1, 100000, 1) + body = "What increment should the range %s be staggered?" % frameRange + (increment, choice) = QtWidgets.QInputDialog.getInt( + self._caller, title, body, 1, 1, 100000, 1) - if not choice: return + if not choice: + return self.cuebotCall(__job.staggerFrames, "Stagger Frames Failed", - range, int(increment)) + frameRange, int(increment)) unbook_info = ["Unbook Frames...", None, "kill"] + def unbook(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) if jobs: @@ -459,33 +489,28 @@ def unbook(self, rpcObjects=None): self._update() sendToGroup_info = ["Send To Group...", None, "configure"] + def sendToGroup(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) if not jobs: return title = "Send jobs to group" - groups = dict([(group.data.name, group) for group in opencue.api.findShow( - jobs[0].data.show).getGroups()]) + groups = { + group.data.name: group for group in opencue.api.findShow(jobs[0].data.show).getGroups()} body = "What group should these jobs move to?\n" + \ "\n".join([job.data.name for job in jobs]) - (group, choice) = QtWidgets.QInputDialog.getItem(self._caller, - title, - body, - sorted(groups.keys()), - 0, - False) + (group, choice) = QtWidgets.QInputDialog.getItem( + self._caller, title, body, sorted(groups.keys()), 0, False) if not choice: return groups[str(group)].reparentJobs(jobs) self._update() - - useLocalCores_info = ["Use local cores...", - "Set a single job to use the local desktop cores", - "configure"] + useLocalCores_info = [ + "Use local cores...", "Set a single job to use the local desktop cores", "configure"] def useLocalCores(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) @@ -495,56 +520,73 @@ def useLocalCores(self, rpcObjects=None): dialog.exec_() copyLogFileDir_info = ["Copy log file directory", None, "configure"] + def copyLogFileDir(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) if jobs: paths = [job.data.log_dir for job in jobs] - QtWidgets.QApplication.clipboard().setText(" ".join(paths), - QtGui.QClipboard.Clipboard) + QtWidgets.QApplication.clipboard().setText( + " ".join(paths), QtGui.QClipboard.Clipboard) + + setUserColor1_info = [ + "Set Color 1", "Set user defined background color", cuegui.Constants.COLOR_USER_1] - setUserColor1_info = ["Set Color 1", "Set user defined background color", cuegui.Constants.COLOR_USER_1] def setUserColor1(self, rpcObjects=None): self._caller.actionSetUserColor(cuegui.Constants.COLOR_USER_1) - setUserColor2_info = ["Set Color 2", "Set user defined background color", cuegui.Constants.COLOR_USER_2] + setUserColor2_info = [ + "Set Color 2", "Set user defined background color", cuegui.Constants.COLOR_USER_2] + def setUserColor2(self, rpcObjects=None): self._caller.actionSetUserColor(cuegui.Constants.COLOR_USER_2) - setUserColor3_info = ["Set Color 3", "Set user defined background color", cuegui.Constants.COLOR_USER_3] + setUserColor3_info = [ + "Set Color 3", "Set user defined background color", cuegui.Constants.COLOR_USER_3] + def setUserColor3(self, rpcObjects=None): self._caller.actionSetUserColor(cuegui.Constants.COLOR_USER_3) - setUserColor4_info = ["Set Color 4", "Set user defined background color", cuegui.Constants.COLOR_USER_4] + setUserColor4_info = [ + "Set Color 4", "Set user defined background color", cuegui.Constants.COLOR_USER_4] + def setUserColor4(self, rpcObjects=None): self._caller.actionSetUserColor(cuegui.Constants.COLOR_USER_4) clearUserColor_info = ["Clear", "Clear user defined background color", None] + def clearUserColor(self, rpcObjects=None): self._caller.actionSetUserColor(None) class LayerActions(AbstractActions): + """Actions for layers.""" + def __init__(self, *args): AbstractActions.__init__(self, *args) view_info = ["View Layer", None, "view"] + def view(self, rpcObjects=None): layers = self._getOnlyLayerObjects(rpcObjects) if layers: self._caller.handle_filter_layers_byLayer.emit([layer.data.name for layer in layers]) viewDepends_info = ["&View Dependencies...", None, "log"] + def viewDepends(self, rpcObjects=None): layers = self._getOnlyLayerObjects(rpcObjects) cuegui.DependDialog.DependDialog(layers[0], self._caller).show() - setMinCores_info = ["Set Minimum Cores", "Set the number of cores required for this layer", "configure"] + setMinCores_info = [ + "Set Minimum Cores", "Set the number of cores required for this layer", "configure"] + def setMinCores(self, rpcObjects=None): layers = self._getOnlyLayerObjects(rpcObjects) if layers: current = max([layer.data.min_cores for layer in layers]) title = "Set minimum number of cores required" - body = "Please enter the new minimum number of cores that frames in the selected layer(s) should require:" + body = ('Please enter the new minimum number of cores that frames in the ' + 'selected layer(s) should require:') (value, choice) = QtWidgets.QInputDialog.getDouble(self._caller, title, body, current, @@ -554,13 +596,16 @@ def setMinCores(self, rpcObjects=None): layer.setMinCores(float(value)) self._update() - setMinMemoryKb_info = ["Set Minimum Memory", "Set the amount of memory required for this layer", "configure"] + setMinMemoryKb_info = [ + "Set Minimum Memory", "Set the amount of memory required for this layer", "configure"] + def setMinMemoryKb(self, rpcObjects=None): layers = self._getOnlyLayerObjects(rpcObjects) if layers: current = max([layer.data.min_memory / 1048576 for layer in layers]) title = "Set minimum amount of memory required" - body = "Please enter the new minimum amount of memory in GB that frames in the selected layer(s) should require:" + body = ('Please enter the new minimum amount of memory in GB that frames ' + 'in the selected layer(s) should require:') (value, choice) = QtWidgets.QInputDialog.getDouble( self._caller, title, body, current, 0.01, 64.0, 1) if choice: @@ -568,9 +613,8 @@ def setMinMemoryKb(self, rpcObjects=None): layer.setMinMemory(int(value * 1048576)) self._update() - useLocalCores_info = ["Use local cores...", - "Set a single layer to use the local desktop cores.", - "configure"] + useLocalCores_info = [ + "Use local cores...", "Set a single layer to use the local desktop cores.", "configure"] def useLocalCores(self, rpcObjects=None): layers = self._getOnlyLayerObjects(rpcObjects) @@ -579,8 +623,8 @@ def useLocalCores(self, rpcObjects=None): dialog = cuegui.LocalBooking.LocalBookingDialog(layer, self._caller) dialog.exec_() - setProperties_info = ["Properties", None, "configure"] + def setProperties(self, rpcObjects=None): layers = self._getOnlyLayerObjects(rpcObjects) if layers: @@ -589,6 +633,7 @@ def setProperties(self, rpcObjects=None): self._update() setTags_info = ["Set Tags", None, "configure"] + def setTags(self, rpcObjects=None): layers = self._getOnlyLayerObjects(rpcObjects) if layers: @@ -597,6 +642,7 @@ def setTags(self, rpcObjects=None): self._update() kill_info = ["&Kill", None, "kill"] + def kill(self, rpcObjects=None): layers = self._getOnlyLayerObjects(rpcObjects) if layers: @@ -608,6 +654,7 @@ def kill(self, rpcObjects=None): self._update() eat_info = ["&Eat", None, "eat"] + def eat(self, rpcObjects=None): layers = self._getOnlyLayerObjects(rpcObjects) if layers: @@ -619,6 +666,7 @@ def eat(self, rpcObjects=None): self._update() retry_info = ["&Retry", None, "retry"] + def retry(self, rpcObjects=None): layers = self._getOnlyLayerObjects(rpcObjects) if layers: @@ -630,6 +678,7 @@ def retry(self, rpcObjects=None): self._update() retryDead_info = ["Retry dead frames", None, "retry"] + def retryDead(self, rpcObjects=None): layers = self._getOnlyLayerObjects(rpcObjects) if layers: @@ -641,6 +690,7 @@ def retryDead(self, rpcObjects=None): self._update() markdone_info = ["Mark done", None, "markdone"] + def markdone(self, rpcObjects=None): layers = self._getOnlyLayerObjects(rpcObjects) if layers: @@ -652,18 +702,22 @@ def markdone(self, rpcObjects=None): self._update() dependWizard_info = ["Dependency &Wizard...", None, "configure"] + def dependWizard(self, rpcObjects=None): layers = self._getOnlyLayerObjects(rpcObjects) if layers: - cuegui.DependWizard.DependWizard(self._caller, [self._getSource()], layers) + cuegui.DependWizard.DependWizard(self._caller, [self._getSource()], layers=layers) reorder_info = ["Reorder Frames...", None, "configure"] + def reorder(self, rpcObjects=None): layers = self._getOnlyLayerObjects(rpcObjects) - if not layers: return + if not layers: + return # Only allow multiple layers with the same range - if len(set([layer.data.range for layer in layers])) != 1: return + if len({layer.data.range for layer in layers}) != 1: + return __layer = layers[0] fs = FileSequence.FrameSet(__layer.data.range) @@ -679,29 +733,27 @@ def reorder(self, rpcObjects=None): else: title = "Reorder layer %s" % __layer.data.name - (range, choice) = self.getText(title, body, "%s-%s" % (__minRange, __maxRange)) + (frameRange, choice) = self.getText(title, body, "%s-%s" % (__minRange, __maxRange)) if not choice: return - body = "What order should the range %s take?" % range + body = "What order should the range %s take?" % frameRange items = list(opencue.compiled_proto.job_pb2.Order.keys()) - (order, choice) = QtWidgets.QInputDialog.getItem(self._caller, - title, - body, - sorted(items), - 0, - False) + (order, choice) = QtWidgets.QInputDialog.getItem( + self._caller, title, body, sorted(items), 0, False) if not choice: return for layer in layers: self.cuebotCall(layer.reorderFrames, "Reorder Frames Failed", - range, getattr(opencue.compiled_proto.job_pb2, str(order))) + frameRange, getattr(opencue.compiled_proto.job_pb2, str(order))) stagger_info = ["Stagger Frames...", None, "configure"] + def stagger(self, rpcObjects=None): layers = self._getOnlyLayerObjects(rpcObjects) - if not layers: return + if not layers: + return __layer = layers[0] fs = FileSequence.FrameSet(__layer.data.range) @@ -711,26 +763,28 @@ def stagger(self, rpcObjects=None): title = "Stagger %s" % __layer.data.name body = "What frame range should be staggered?" - (range, choice) = self.getText(title, body, "%s-%s" % (__minRange, __maxRange)) - if not choice: return - - body = "What increment should the range %s be staggered?" % range - (increment, choice) = QtWidgets.QInputDialog.getInt(self._caller, - title, body, - 1, - 1, 100000, 1) + (frameRange, choice) = self.getText(title, body, "%s-%s" % (__minRange, __maxRange)) + if not choice: + return - if not choice: return + body = "What increment should the range %s be staggered?" % frameRange + (increment, choice) = QtWidgets.QInputDialog.getInt( + self._caller, title, body, 1, 1, 100000, 1) + if not choice: + return self.cuebotCall(__layer.staggerFrames, "Stagger Frames Failed", - range, int(increment)) + frameRange, int(increment)) class FrameActions(AbstractActions): + """Actions for frames.""" + def __init__(self, *args): AbstractActions.__init__(self, *args) view_info = ["&View Log", None, "log"] + def view(self, rpcObjects=None): frames = self._getOnlyFrameObjects(rpcObjects) if frames: @@ -742,6 +796,7 @@ def view(self, rpcObjects=None): cuegui.Utils.popupFrameView(job, frame) tail_info = ["&Tail Log", None, "log"] + def tail(self, rpcObjects=None): frames = self._getOnlyFrameObjects(rpcObjects) if frames: @@ -753,12 +808,14 @@ def tail(self, rpcObjects=None): cuegui.Utils.popupFrameTail(job, frame) viewLastLog_info = ["View Last Log", None, "loglast"] + def viewLastLog(self, rpcObjects=None): frames = self._getOnlyFrameObjects(rpcObjects) if frames: job = self._getSource() path = cuegui.Utils.getFrameLogFile(job, frames[0]) - files = dict((int(j.split(".")[-1]), j) for j in glob.glob("%s.*" % path) if j[-1].isdigit()) + files = dict( + (int(j.split(".")[-1]), j) for j in glob.glob("%s.*" % path) if j[-1].isdigit()) if files: cuegui.Utils.popupView(files[sorted(files.keys())[-1]]) else: @@ -776,57 +833,71 @@ def useLocalCores(self, rpcObjects=None): dialog.exec_() xdiff2_info = ["View xdiff of 2 logs", None, "log"] + def xdiff2(self, rpcObjects=None): frames = self._getOnlyFrameObjects(rpcObjects) if len(frames) >= 2: cuegui.Utils.popupFrameXdiff(self._getSource(), frames[0], frames[1]) xdiff3_info = ["View xdiff of 3 logs", None, "log"] + def xdiff3(self, rpcObjects=None): frames = self._getOnlyFrameObjects(rpcObjects) if len(frames) >= 3: cuegui.Utils.popupFrameXdiff(self._getSource(), frames[0], frames[1], frames[2]) viewHost_info = ["View Host", None, "log"] + def viewHost(self, rpcObjects=None): frames = self._getOnlyFrameObjects(rpcObjects) - hosts = list(set([frame.data.last_resource.split("/")[0] for frame in frames if frame.data.last_resource])) + hosts = list({frame.data.last_resource.split("/")[0] + for frame in frames if frame.data.last_resource}) if hosts: + # pylint: disable=no-member QtGui.qApp.view_hosts.emit(hosts) QtGui.qApp.single_click.emit(opencue.api.findHost(hosts[0])) + # pylint: enable=no-member getWhatThisDependsOn_info = ["print getWhatThisDependsOn", None, "log"] + def getWhatThisDependsOn(self, rpcObjects=None): frame = self._getOnlyFrameObjects(rpcObjects)[0] - logger.info("type", "target", "anyFrame", "active", "dependErJob", "dependErLayer", "dependErFrame", "dependOnJob", "dependOnLayer", "dependOnFrame") for item in frame.getWhatThisDependsOn(): logger.info(item.data.type, item.data.target, item.data.any_frame, item.data.active) - logger.info("This:", item.data.depend_er_job, item.data.depend_er_layer, item.data.depend_er_frame, - "On:", item.data.depend_on_job, item.data.depend_on_layer, item.data.depend_on_frame) + logger.info( + "This: %s %s %s", item.data.depend_er_job, item.data.depend_er_layer, + item.data.depend_er_frame) + logger.info( + "On: %s %s %s", item.data.depend_on_job, item.data.depend_on_layer, + item.data.depend_on_frame) viewDepends_info = ["&View Dependencies...", None, "log"] + def viewDepends(self, rpcObjects=None): frames = self._getOnlyFrameObjects(rpcObjects) cuegui.DependDialog.DependDialog(frames[0], self._caller).show() getWhatDependsOnThis_info = ["print getWhatDependsOnThis", None, "log"] + def getWhatDependsOnThis(self, rpcObjects=None): frame = self._getOnlyFrameObjects(rpcObjects)[0] logger.info(frame.getWhatDependsOnThis()) retry_info = ["&Retry", None, "retry"] + def retry(self, rpcObjects=None): names = [frame.data.name for frame in self._getOnlyFrameObjects(rpcObjects)] if names: job = self._getSource() - if cuegui.Utils.questionBoxYesNo(self._caller, "Confirm", - "Retry selected frames?", - names): + if cuegui.Utils.questionBoxYesNo( + self._caller, "Confirm", "Retry selected frames?", names): job.retryFrames(name=names) self._update() previewMain_info = ["Preview Main", None, "previewMain"] + + # pylint: disable=broad-except def previewMain(self, rpcObjects=None): try: job = self._getSource() @@ -839,6 +910,8 @@ def previewMain(self, rpcObjects=None): "Error displaying preview frames, %s" % e) previewAovs_info = ["Preview All", None, "previewAovs"] + + # pylint: disable=broad-except def previewAovs(self, rpcObjects=None): try: job = self._getSource() @@ -850,6 +923,7 @@ def previewAovs(self, rpcObjects=None): QtWidgets.QMessageBox.critical(None, "Preview Error", "Error displaying preview frames, %s" % e) eat_info = ["&Eat", None, "eat"] + def eat(self, rpcObjects=None): names = [frame.data.name for frame in self._getOnlyFrameObjects(rpcObjects)] if names: @@ -860,6 +934,7 @@ def eat(self, rpcObjects=None): self._update() kill_info = ["&Kill", None, "kill"] + def kill(self, rpcObjects=None): names = [frame.data.name for frame in self._getOnlyFrameObjects(rpcObjects)] if names: @@ -870,6 +945,7 @@ def kill(self, rpcObjects=None): self._update() markAsWaiting_info = ["Mark as &waiting", None, "configure"] + def markAsWaiting(self, rpcObjects=None): names = [frame.data.name for frame in self._getOnlyFrameObjects(rpcObjects)] if names: @@ -881,6 +957,7 @@ def markAsWaiting(self, rpcObjects=None): self._update() dropDepends_info = ["D&rop depends", None, "configure"] + def dropDepends(self, rpcObjects=None): frames = self._getOnlyFrameObjects(rpcObjects) names = [frame.data.name for frame in frames] @@ -894,43 +971,44 @@ def dropDepends(self, rpcObjects=None): self._update() dependWizard_info = ["Dependency &Wizard...", None, "configure"] + def dependWizard(self, rpcObjects=None): frames = self._getOnlyFrameObjects(rpcObjects) if frames: - cuegui.DependWizard.DependWizard(self._caller, [self._getSource()], [], frames) + cuegui.DependWizard.DependWizard(self._caller, [self._getSource()], frames=frames) markdone_info = ["Mark done", None, "markdone"] + def markdone(self, rpcObjects=None): frames = self._getOnlyFrameObjects(rpcObjects) if frames: frameNames = [frame.data.name for frame in frames] - if cuegui.Utils.questionBoxYesNo(self._caller, "Confirm", - "Mark done all selected frames?\n" - "(Drops any dependencies that are waiting on these frames)", - frameNames): + if cuegui.Utils.questionBoxYesNo( + self._caller, "Confirm", + 'Mark done all selected frames?\n' + '(Drops any dependencies that are waiting on these frames)', frameNames): self._getSource().markdoneFrames(name=frameNames) self._update() reorder_info = ["Reorder...", None, "configure"] + def reorder(self, rpcObjects=None): frames = self._getOnlyFrameObjects(rpcObjects) - if not frames: return + if not frames: + return __job = self._getSource() title = "Reorder %s" % __job.data.name body = "How should these frames be reordered?" items = list(opencue.compiled_proto.job_pb2.Order.keys()) - (order, choice) = QtWidgets.QInputDialog.getItem(self._caller, - title, - body, - sorted(items), - 0, - False) - if not choice: return + (order, choice) = QtWidgets.QInputDialog.getItem( + self._caller, title, body, sorted(items), 0, False) + if not choice: + return # Store the proxy and a place for the frame numbers keyed to the layer name - __layersDict = dict([(layer.data.name, (layer, [])) for layer in __job.getLayers()]) + __layersDict = {layer.data.name: (layer, []) for layer in __job.getLayers()} # For each frame, store the number in the list for that layer for frame in frames: @@ -948,27 +1026,31 @@ def reorder(self, rpcObjects=None): getattr(opencue.compiled_proto.job_pb2, str(order))) copyLogFileName_info = ["Copy log file name", None, "configure"] + def copyLogFileName(self, rpcObjects=None): frames = self._getOnlyFrameObjects(rpcObjects) - if not frames: return + if not frames: + return job = self._getSource() paths = [cuegui.Utils.getFrameLogFile(job, frame) for frame in frames] QtWidgets.QApplication.clipboard().setText(paths, QtGui.QClipboard.Clipboard) eatandmarkdone_info = ["Eat and Mark done", None, "eatandmarkdone"] + def eatandmarkdone(self, rpcObjects=None): frames = self._getOnlyFrameObjects(rpcObjects) if frames: frameNames = [frame.data.name for frame in frames] - if cuegui.Utils.questionBoxYesNo(self._caller, "Confirm", - "Eat and Mark done all selected frames?\n" - "(Drops any dependencies that are waiting on these frames)\n\n" - "If a frame is part of a layer that will now only contain\n" - "eaten or succeeded frames, any dependencies on the\n" - "layer will be dropped as well.", - frameNames): + if cuegui.Utils.questionBoxYesNo( + self._caller, "Confirm", + "Eat and Mark done all selected frames?\n" + "(Drops any dependencies that are waiting on these frames)\n\n" + "If a frame is part of a layer that will now only contain\n" + "eaten or succeeded frames, any dependencies on the\n" + "layer will be dropped as well.", + frameNames): # Mark done the layers to drop their dependencies if the layer is done @@ -989,17 +1071,23 @@ def eatandmarkdone(self, rpcObjects=None): # Warning: The below assumes that eaten frames are desired to be markdone - # Wait for the markDoneFrames to be processed, then drop the dependencies on the layer if all frames are done + # Wait for the markDoneFrames to be processed, then drop the dependencies on + # the layer if all frames are done. layerNames = [frame.data.layer_name for frame in frames] time.sleep(1) for layer in self._getSource().getLayers(): if layer.data.name in layerNames: - if layer.data.layer_stats.eaten_frames + layer.data.layer_stats.succeeded_frames == layer.data.layer_stats.total_frames: + if ( + layer.data.layer_stats.eaten_frames + + layer.data.layer_stats.succeeded_frames == + layer.data.layer_stats.total_frames): layer.markdone() self._update() class ShowActions(AbstractActions): + """Actions for shows.""" + def __init__(self, *args): AbstractActions.__init__(self, *args) @@ -1011,7 +1099,8 @@ def properties(self, rpcObjects=None): createSubscription_info = ["Create Subscription...", None, "configure"] def createSubscription(self, rpcObjects=None): - d = cuegui.CreatorDialog.SubscriptionCreatorDialog(show=self._getOnlyShowObjects(rpcObjects)[0]) + d = cuegui.CreatorDialog.SubscriptionCreatorDialog( + show=self._getOnlyShowObjects(rpcObjects)[0]) d.exec_() viewTasks_info = ["View Tasks...", None, "view"] @@ -1022,6 +1111,8 @@ def viewTasks(self, rpcObjects=None): class RootGroupActions(AbstractActions): + """"Actions for root groups.""" + def __init__(self, *args): AbstractActions.__init__(self, *args) @@ -1029,7 +1120,8 @@ def __init__(self, *args): def properties(self, rpcObjects=None): rootgroups = self._getOnlyRootGroupObjects(rpcObjects) if rootgroups: - cuegui.ShowDialog.ShowDialog(opencue.api.findShow(rootgroups[0].data.name), self._caller).show() + cuegui.ShowDialog.ShowDialog( + opencue.api.findShow(rootgroups[0].data.name), self._caller).show() groupProperties_info = ["Group Properties...", None, "view"] def groupProperties(self, rpcObjects=None): @@ -1048,7 +1140,8 @@ def setCuewho(self, rpcObjects=None): (name, choice) = self.getText(title, body, cuegui.Utils.getUsername()) if choice: for rootgroup in rootgroups: - logger.info(subprocess.check_output("cuewho -s %s -who %s" % (rootgroup.data.name, name))) + logger.info(subprocess.check_output( + "cuewho -s %s -who %s" % (rootgroup.data.name, name))) showCuewho_info = ["Display Cuewho", None, "configure"] def showCuewho(self, rpcObjects=None): @@ -1058,11 +1151,9 @@ def showCuewho(self, rpcObjects=None): for rootgroup in rootgroups: cuewho = cuegui.Utils.getCuewho(rootgroup.data.name) extension = cuegui.Utils.getExtension(cuewho) - message.append("Cuewho for %s is: %s %s" % (rootgroup.data.name, cuewho, extension )) - QtWidgets.QMessageBox.information(self._caller, - "Show Cuewho", - '\n'.join(message), - QtWidgets.QMessageBox.Ok) + message.append("Cuewho for %s is: %s %s" % (rootgroup.data.name, cuewho, extension)) + QtWidgets.QMessageBox.information( + self._caller, "Show Cuewho", '\n'.join(message), QtWidgets.QMessageBox.Ok) createGroup_info = ["Create Group...", None, "configure"] def createGroup(self, rpcObjects=None): @@ -1092,6 +1183,8 @@ def serviceProperties(self, rpcObjects=None): class GroupActions(AbstractActions): + """Actions for groups.""" + def __init__(self, *args): AbstractActions.__init__(self, *args) @@ -1123,6 +1216,8 @@ def deleteGroup(self, rpcObjects=None): class SubscriptionActions(AbstractActions): + """Actions for subscriptions.""" + def __init__(self, *args): AbstractActions.__init__(self, *args) @@ -1137,9 +1232,9 @@ def editSize(self, rpcObjects=None): "contact the resource department." minSize = 0 decimalPlaces = 0 - (value, choice) = QtWidgets.QInputDialog.getDouble(self._caller, title, body, current/100.0, - minSize, cuegui.Constants.QT_MAX_INT, - decimalPlaces) + (value, choice) = QtWidgets.QInputDialog.getDouble( + self._caller, title, body, current/100.0, minSize, cuegui.Constants.QT_MAX_INT, + decimalPlaces) if choice: msg = QtWidgets.QMessageBox() msg.setText( @@ -1166,9 +1261,9 @@ def editBurst(self, rpcObjects=None): "subscription should be allowed to reach:" minSize = 0 decimalPlaces = 0 - (value, choice) = QtWidgets.QInputDialog.getDouble(self._caller, title, body, current/100.0, - minSize, cuegui.Constants.QT_MAX_INT, - decimalPlaces) + (value, choice) = QtWidgets.QInputDialog.getDouble( + self._caller, title, body, current/100.0, minSize, cuegui.Constants.QT_MAX_INT, + decimalPlaces) if choice: for sub in subs: self.cuebotCall(sub.setBurst, @@ -1190,28 +1285,37 @@ def delete(self, rpcObjects=None): class AllocationActions(AbstractActions): + """Actions for allocations.""" + def __init__(self, *args): AbstractActions.__init__(self, *args) class HostActions(AbstractActions): + """Actions for hosts.""" + def __init__(self, *args): AbstractActions.__init__(self, *args) viewComments_info = ["Comments...", None, "comment"] + def viewComments(self, rpcObjects=None): hosts = self._getOnlyHostObjects(rpcObjects) if hosts: cuegui.Comments.CommentListDialog(hosts[0], self._caller).show() viewProc_info = ["View Procs", None, "log"] + def viewProc(self, rpcObjects=None): hosts = self._getOnlyHostObjects(rpcObjects) - hosts = list(set([host.data.name for host in hosts])) + hosts = list({host.data.name for host in hosts}) if hosts: + # pylint: disable=no-member QtGui.qApp.view_procs.emit(hosts) + # pylint: enable=no-member lock_info = ["Lock Host", None, "lock"] + def lock(self, rpcObjects=None): hosts = self._getOnlyHostObjects(rpcObjects) for host in hosts: @@ -1219,6 +1323,7 @@ def lock(self, rpcObjects=None): self._update() unlock_info = ["Unlock Host", None, "lock"] + def unlock(self, rpcObjects=None): hosts = self._getOnlyHostObjects(rpcObjects) for host in hosts: @@ -1226,6 +1331,7 @@ def unlock(self, rpcObjects=None): self._update() delete_info = ["Delete Host", "Delete host from cuebot", "kill"] + def delete(self, rpcObjects=None): hosts = self._getOnlyHostObjects(rpcObjects) title = "Confirm" @@ -1244,6 +1350,7 @@ def delete(self, rpcObjects=None): self._update() rebootWhenIdle_info = ["Reboot when idle", None, "retry"] + def rebootWhenIdle(self, rpcObjects=None): hosts = self._getOnlyHostObjects(rpcObjects) title = "Confirm" @@ -1260,6 +1367,7 @@ def rebootWhenIdle(self, rpcObjects=None): self._update() addTags_info = ["Add Tags...", None, "configure"] + def addTags(self, rpcObjects=None): hosts = self._getOnlyHostObjects(rpcObjects) if hosts: @@ -1275,6 +1383,7 @@ def addTags(self, rpcObjects=None): self._update() removeTags_info = ["Remove Tags...", None, "configure"] + def removeTags(self, rpcObjects=None): hosts = self._getOnlyHostObjects(rpcObjects) if hosts: @@ -1290,40 +1399,39 @@ def removeTags(self, rpcObjects=None): self._update() renameTag_info = ["Rename Tag...", None, "configure"] + def renameTag(self, rpcObjects=None): hosts = self._getOnlyHostObjects(rpcObjects) if hosts: title = "Rename tag" body = "What tag should be renamed?" - (oldTag, choice) = QtWidgets.QInputDialog.getItem(self._caller, - title, body, - hosts[0].data.tags, - 0, False) - if not choice: return + (oldTag, choice) = QtWidgets.QInputDialog.getItem( + self._caller, title, body, hosts[0].data.tags, 0, False) + if not choice: + return oldTag = str(oldTag) title = "Rename tag" body = "What is the new name for the tag?" (newTag, choice) = self.getText(title, body, oldTag) - if not choice: return + if not choice: + return for host in hosts: - self.cuebotCall(host.renameTag, - "Rename Tags on %s Failed" % host.data.name, - oldTag, newTag) + self.cuebotCall( + host.renameTag, "Rename Tags on %s Failed" % host.data.name, oldTag, newTag) self._update() changeAllocation_info = ["Change Allocation...", None, "configure"] + def changeAllocation(self, rpcObjects=None): hosts = self._getOnlyHostObjects(rpcObjects) if hosts: - allocations = dict([(alloc.data.name, alloc) for alloc in opencue.api.getAllocations()]) + allocations = {alloc.data.name: alloc for alloc in opencue.api.getAllocations()} title = "Move host to allocation" body = "What allocation should the host(s) be moved to?" - (allocationName, choice) = QtWidgets.QInputDialog.getItem(self._caller, - title, body, - sorted(allocations.keys()), - 0, False) + (allocationName, choice) = QtWidgets.QInputDialog.getItem( + self._caller, title, body, sorted(allocations.keys()), 0, False) if choice: allocation = allocations[str(allocationName)] for host in hosts: @@ -1333,6 +1441,7 @@ def changeAllocation(self, rpcObjects=None): self._update() setRepair_info = ["Set Repair State", None, "configure"] + def setRepair(self, rpcObjects=None): hosts = self._getOnlyHostObjects(rpcObjects) repair = opencue.api.host_pb2.REPAIR @@ -1342,6 +1451,7 @@ def setRepair(self, rpcObjects=None): self._update() clearRepair_info = ["Clear Repair State", None, "configure"] + def clearRepair(self, rpcObjects=None): hosts = self._getOnlyHostObjects(rpcObjects) repair = opencue.api.host_pb2.REPAIR @@ -1353,36 +1463,45 @@ def clearRepair(self, rpcObjects=None): class ProcActions(AbstractActions): + """Actions for procs.""" + def __init__(self, *args): AbstractActions.__init__(self, *args) view_info = ["&View Job", None, "view"] + def view(self, rpcObjects=None): - for job in list(set([proc.data.job_name for proc in self._getOnlyProcObjects(rpcObjects)])): + for job in list({proc.data.job_name for proc in self._getOnlyProcObjects(rpcObjects)}): try: + # pylint: disable=no-member QtGui.qApp.view_object.emit(opencue.api.findJob(job)) - except Exception: - logger.warning("Unable to load: %s" % job) + # pylint: enable=no-member + except opencue.exception.CueException: + logger.warning("Unable to load: %s", job) kill_info = ["&Kill", None, "kill"] + def kill(self, rpcObjects=None): procs = self._getOnlyProcObjects(rpcObjects) if procs: - if cuegui.Utils.questionBoxYesNo(self._caller, "Confirm", - "Kill selected frames?", - ["%s -> %s @ %s" % (proc.data.job_name, proc.data.frame_name, proc.data.name) for proc in procs]): + if cuegui.Utils.questionBoxYesNo( + self._caller, "Confirm", "Kill selected frames?", + ["%s -> %s @ %s" % (proc.data.job_name, proc.data.frame_name, proc.data.name) + for proc in procs]): for proc in procs: self.cuebotCall(proc.kill, "Kill Proc %s Failed" % proc.data.name) self._update() unbook_info = ["Unbook", None, "eject"] + def unbook(self, rpcObjects=None): procs = self._getOnlyProcObjects(rpcObjects) if procs: - if cuegui.Utils.questionBoxYesNo(self._caller, "Confirm", - "Unbook selected frames?", - ["%s -> %s @ %s" % (proc.data.job_name, proc.data.frame_name, proc.data.name) for proc in procs]): + if cuegui.Utils.questionBoxYesNo( + self._caller, "Confirm", "Unbook selected frames?", + ["%s -> %s @ %s" % (proc.data.job_name, proc.data.frame_name, proc.data.name) + for proc in procs]): for proc in procs: self.cuebotCall(proc.unbook, "Unbook Proc %s Failed" % proc.data.name, @@ -1390,12 +1509,14 @@ def unbook(self, rpcObjects=None): self._update() unbookKill_info = ["Unbook and Kill", None, "unbookkill"] + def unbookKill(self, rpcObjects=None): procs = self._getOnlyProcObjects(rpcObjects) if procs: - if cuegui.Utils.questionBoxYesNo(self._caller, "Confirm", - "Unbook and Kill selected frames?", - ["%s -> %s @ %s" % (proc.data.job_name, proc.data.frame_name, proc.data.name) for proc in procs]): + if cuegui.Utils.questionBoxYesNo( + self._caller, "Confirm", "Unbook and Kill selected frames?", + ["%s -> %s @ %s" % (proc.data.job_name, proc.data.frame_name, proc.data.name) + for proc in procs]): for proc in procs: self.cuebotCall(proc.unbook, "Unbook and Kill Proc %s Failed" % proc.data.name, @@ -1404,10 +1525,13 @@ def unbookKill(self, rpcObjects=None): class DependenciesActions(AbstractActions): + """Actions for depends.""" + def __init__(self, *args): AbstractActions.__init__(self, *args) satisfy_info = ["Satisfy Dependency", None, "kill"] + def satisfy(self, rpcObjects=None): dependencies = self._getSelected(rpcObjects) for dependency in dependencies: @@ -1415,6 +1539,7 @@ def satisfy(self, rpcObjects=None): self._update() unsatisfy_info = ["Unsatisfy Dependency", None, "retry"] + def unsatisfy(self, rpcObjects=None): dependencies = self._getSelected(rpcObjects) for dependency in dependencies: @@ -1423,10 +1548,13 @@ def unsatisfy(self, rpcObjects=None): class FilterActions(AbstractActions): + """Actions for filters.""" + def __init__(self, *args): AbstractActions.__init__(self, *args) rename_info = ["Rename...", None, ""] + def rename(self, rpcObjects=None): filters = self._getSelected(rpcObjects) if filters: @@ -1439,45 +1567,51 @@ def rename(self, rpcObjects=None): self._update() delete_info = ["Delete", None, "kill"] + def delete(self, rpcObjects=None): filters = self._getSelected(rpcObjects) if filters: - if cuegui.Utils.questionBoxYesNo(self._caller, "Confirm", - "Delete selected filters?", - [filter.data.name for filter in filters]): - for filter in filters: - filter.delete() + if cuegui.Utils.questionBoxYesNo( + self._caller, "Confirm", "Delete selected filters?", + [selectedFilter.data.name for selectedFilter in filters]): + for filterToDelete in filters: + filterToDelete.delete() self._update() raiseOrder_info = ["Raise Order", None, ""] + def raiseOrder(self, rpcObjects=None): filters = self._getSelected(rpcObjects) - for filter in filters: - filter.raiseOrder() + for selectedFilter in filters: + selectedFilter.raiseOrder() self._update() lowerOrder_info = ["Lower Order", None, ""] + def lowerOrder(self, rpcObjects=None): filters = self._getSelected(rpcObjects) - for filter in filters: - filter.lowerOrder() + for selectedFilter in filters: + selectedFilter.lowerOrder() self._update() orderFirst_info = ["Order First", None, ""] + def orderFirst(self, rpcObjects=None): filters = self._getSelected(rpcObjects) - for filter in filters: - filter.orderFirst() + for selectedFilter in filters: + selectedFilter.orderFirst() self._update() orderLast_info = ["Order Last", None, ""] + def orderLast(self, rpcObjects=None): filters = self._getSelected(rpcObjects) - for filter in filters: - filter.orderLast() + for selectedFilter in filters: + selectedFilter.orderLast() self._update() setOrder_info = ["Set Order...", None, ""] + def setOrder(self, rpcObjects=None): filters = self._getSelected(rpcObjects) if filters: @@ -1493,6 +1627,8 @@ def setOrder(self, rpcObjects=None): class MatcherActions(AbstractActions): + """Actions for matchers.""" + def __init__(self, *args): AbstractActions.__init__(self, *args) @@ -1521,6 +1657,8 @@ def setValue(self, rpcObjects=None): class ActionActions(AbstractActions): + """Actions for filter actions.""" + def __init__(self, *args): AbstractActions.__init__(self, *args) @@ -1537,6 +1675,8 @@ def delete(self, rpcObjects=None): class TaskActions(AbstractActions): + """Actions for tasks.""" + def __init__(self, *args): AbstractActions.__init__(self, *args) @@ -1556,7 +1696,8 @@ def setMinCores(self, rpcObjects=None): task.setMinCores(float(value)) self._update() - clearAdjustment_info = ["Clear Minimum Core Adjustment", "Clear Task(s) Minimum Core Adjustment", "configure"] + clearAdjustment_info = [ + "Clear Minimum Core Adjustment", "Clear Task(s) Minimum Core Adjustment", "configure"] def clearAdjustment(self, rpcObjects=None): tasks = self._getSelected(rpcObjects) for task in tasks: @@ -1573,17 +1714,19 @@ def delete(self, rpcObjects=None): for task in tasks: task.delete() self._update() - + class LimitActions(AbstractActions): + """Actions for limits.""" + def __init__(self, *args): AbstractActions.__init__(self, *args) - create_info = ["Creaet Limit", None, "configure"] + create_info = ["Create Limit", None, "configure"] def create(self, rpcObjects=None): title = "Add Limit" body = "Enter a name for the new limit." - + (limit, choice) = self.getText(title, body, "") if choice: limit = limit.strip() @@ -1620,7 +1763,7 @@ def editMaxValue(self, rpcObjects=None): "Set Max Value on Limit %s Failed" % limit.data.name, int(value)) self._update() - + rename_info = ["Rename", None, "configure"] def rename(self, rpcObjects=None): limits = self._getSelected(rpcObjects) @@ -1633,9 +1776,12 @@ def rename(self, rpcObjects=None): self._update() +# pylint: disable=attribute-defined-outside-init class MenuActions(object): + """Provides access to common right click actions.""" + def __init__(self, caller, updateCallable, selectedRpcObjectsCallable, sourceCallable = None): - """This object provides access to common right click actions + """ @param caller: The Widget that is creating the menu @type caller: QWidget @param updateCallable: A callable that will update the display @@ -1731,7 +1877,7 @@ def tasks(self): if not hasattr(self, "_tasks"): self._tasks = TaskActions(*self.__getArgs()) return self._tasks - + def limits(self): if not hasattr(self, "_limits"): self._limits = LimitActions(*self.__getArgs()) diff --git a/cuegui/cuegui/MiscDialog.py b/cuegui/cuegui/MiscDialog.py index e10b5e554..c3b1185f8 100644 --- a/cuegui/cuegui/MiscDialog.py +++ b/cuegui/cuegui/MiscDialog.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Miscellaneous dialogs.""" + + from __future__ import absolute_import from __future__ import print_function from __future__ import division @@ -23,6 +26,8 @@ class RunLocalDialog(cuegui.AbstractDialog.AbstractDialog): + """Dialog for running a job on the user's local desktop cores.""" + def __init__(self, job, parent=None): cuegui.AbstractDialog.AbstractDialog.__init__(self, parent) layout = QtWidgets.QVBoxLayout(self) @@ -46,7 +51,8 @@ def __init__(self, job, parent=None): self.__localOnlyLabel = QtWidgets.QLabel("Only use local cores for this job?", self) self.__localOnlyCheckBox = QtWidgets.QCheckBox(self) - self.__buttons = self._newDialogButtonBox(QtWidgets.QDialogButtonBox.Ok | QtWidgets.QDialogButtonBox.Cancel) + self.__buttons = self._newDialogButtonBox( + QtWidgets.QDialogButtonBox.Ok | QtWidgets.QDialogButtonBox.Cancel) layout.addWidget(self.__descriptionLabel) self._addWidgetRow(self.__amountLabel, self.__amountSpinBox) @@ -55,4 +61,5 @@ def __init__(self, job, parent=None): layout.addWidget(self.__buttons) def results(self): + """Gets the user input results.""" return self.__amountSpinBox.value(), self.__localOnlyCheckBox.isChecked() diff --git a/cuegui/cuegui/Plugins.py b/cuegui/cuegui/Plugins.py index c57e64964..2b67ebd7e 100644 --- a/cuegui/cuegui/Plugins.py +++ b/cuegui/cuegui/Plugins.py @@ -83,9 +83,12 @@ class Plugins(object): + """Main class responsible for loading and managing plugins.""" + # Keyed to name. each is a dictionary with CLASS, DESCRIPTION and optionally CATEGORY __plugins = {} _loadedPaths = [] + def __init__(self, mainWindow, name): """Plugins class initialization. @param mainWindow: Application main window reference @@ -99,7 +102,9 @@ def __init__(self, mainWindow, name): self.__menu_separator = " \t-> " # Load plugin paths from the config file + # pylint: disable=no-member __pluginPaths = QtGui.qApp.settings.value("Plugin_Paths", []) + # pylint: enable=no-member for path in cuegui.Constants.DEFAULT_PLUGIN_PATHS + __pluginPaths: self.loadPluginPath(str(path)) @@ -118,74 +123,92 @@ def loadConfigFilePlugins(self, configGroup): The imported module must have an init function and a QMainWindow will be passed to it. """ + # pylint: disable=no-member __plugins = QtGui.qApp.settings.value("%s/Plugins" % configGroup, []) + # pylint: enable=no-member for plugin in __plugins: path = os.path.dirname(str(plugin)) if path: - logger.info("adding path " + path) + logger.info("adding path %s", path) sys.path.append(path) for plugin in __plugins: module = os.path.basename(str(plugin)) if module: - logger.info("loading module " + module) + logger.info("loading module %s", module) s_class = module.split(".")[-1] + # pylint: disable=broad-except try: m = __import__(module, globals(), locals(), [s_class]) m.init(self.mainWindow) - logger.info("plugin loaded %s" % module) + logger.info("plugin loaded %s", module) except Exception as e: - logger.warning("Failed to load plugin: %s" % s_class) + logger.warning("Failed to load plugin: %s", s_class) list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) - def __closePlugin(self, object): - """When a running plugin is closed, this is called and the running - plugin is deleted. If it is a dock widget then it is removed from the - main window. - @type object: Object - @param object: The object created by loadin""" + def __closePlugin(self, pluginBeingClosed): + """Event handler for when a plugin is closed. + + When a running plugin is closed, this is called and the running plugin is deleted. If it is + a dock widget then it is removed from the main window. + + @type pluginBeingClosed: Object + @param pluginBeingClosed: the plugin widget being closed""" for item in self.__running: - if item[1] == object: - if isinstance(object, QtWidgets.QDockWidget): - self.mainWindow.removeDockWidget(object) + if item[1] == pluginBeingClosed: + if isinstance(pluginBeingClosed, QtWidgets.QDockWidget): + self.mainWindow.removeDockWidget(pluginBeingClosed) self.__running.remove(item) return def runningList(self): - """Lists all running plugins + """Lists all running plugins. + @return: [("Class_Name_1", PluginClass1_Instance), ("Class_Name_2", PluginClass2_Instance)] @rtype: list""" return self.__running def saveState(self): """Saves the names of all open plugins. - Calls .saveSettings (if available) on all plugins.""" + + Calls .saveSettings (if available) on all plugins.""" opened = [] for plugin in self.__running: + # pylint: disable=broad-except try: if hasattr(plugin[1], "pluginSaveState"): opened.append("%s::%s" % (plugin[0], json.dumps(plugin[1].pluginSaveState()))) except Exception as e: - logger.warning("Error saving plugin state for: %s\n%s" % (plugin[0], e)) + logger.warning("Error saving plugin state for: %s\n%s", plugin[0], e) + # pylint: disable=no-member QtGui.qApp.settings.setValue("%s/Plugins_Opened" % self.name, opened) + # pylint: enable=no-member def restoreState(self): - """Loads any user defined pluggin directories. - Restores all open plugins. - Calls .restoreSettings (if available) on all plugins.""" - # Loads any user defined pluggin directories - for path in QtGui.qApp.settings.value("Plugins/Paths", []): + """Loads any user defined plugin directories and restores all open plugins. + + Calls .restoreSettings (if available) on all plugins.""" + # Loads any user defined plugin directories + # pylint: disable=no-member + pluginPaths = QtGui.qApp.settings.value("Plugins/Paths", []) + # pylint: enable=no-member + + for path in pluginPaths: self.loadPluginPath(str(path)) # Runs any plugins that were saved to the settings - for plugin in (QtGui.qApp.settings.value("%s/Plugins_Opened" % self.name) or []): + # pylint: disable=no-member + openPlugins = QtGui.qApp.settings.value("%s/Plugins_Opened" % self.name) or [] + # pylint: enable=no-member + for plugin in openPlugins: if '::' in plugin: plugin_name, plugin_state = str(plugin).split("::") self.launchPlugin(plugin_name, plugin_state) def launchPlugin(self, plugin_name, plugin_state): - """Launches the desired plugin + """Launches the desired plugin. + @param plugin_name: The name of the plugin as provided by PLUGIN_NAME @type plugin_name: string @param plugin_state: The state of the plugin's tab @@ -193,19 +216,23 @@ def launchPlugin(self, plugin_name, plugin_state): try: plugin_class = self.__plugins[plugin_name][CLASS] except KeyError: - logger.warning("Unable to launch previously open plugin, it no longer exists: %s" % plugin_name) + logger.warning( + "Unable to launch previously open plugin, it no longer exists: %s", plugin_name) return + # pylint: disable=broad-except try: plugin_instance = plugin_class(self.mainWindow) self.__running.append((plugin_name, plugin_instance)) plugin_instance.closed.connect(self.__closePlugin, QtCore.Qt.QueuedConnection) except Exception: - logger.warning("Failed to load plugin module: %s\n%s" % (plugin_name, - ''.join(traceback.format_exception(*sys.exc_info())) )) + logger.warning( + "Failed to load plugin module: %s\n%s", + plugin_name, ''.join(traceback.format_exception(*sys.exc_info()))) return if hasattr(plugin_instance, "pluginRestoreState"): + # pylint: disable=broad-except try: try: if plugin_state: @@ -223,11 +250,13 @@ def launchPlugin(self, plugin_name, plugin_state): else: state = None except Exception as e: - logger.warning("Failed to load state information stored as %s for %s, error was: %s" % (plugin_state, plugin_name, e)) + logger.warning( + "Failed to load state information stored as %s for %s, error was: %s", + plugin_state, plugin_name, e) state = None plugin_instance.pluginRestoreState(state) except Exception as e: - logger.warning("Error restoring plugin state for: %s" % plugin_name) + logger.warning("Error restoring plugin state for: %s", plugin_name) list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) def loadPluginPath(self, plugin_dir): @@ -245,22 +274,23 @@ def loadPluginPath(self, plugin_dir): for p in os.listdir(plugin_dir): name, ext = os.path.splitext(p) - if ext == ".py" and not name in ["__init__","Manifest","README"]: + if ext == ".py" and not name in ["__init__", "Manifest", "README"]: self.loadPlugin(name) sys.path = orig_sys_path else: - logger.warning("Unable to read the plugin path: %s" % plugin_dir) + logger.warning("Unable to read the plugin path: %s", plugin_dir) def loadPlugin(self, name): """Loads a single plugin that must be in the python path @param name: Name of the python module that contains a plugin @type name: string""" + # pylint: disable=broad-except try: - logger.info("Importing: %s" % name) + logger.info("Importing: %s", name) module = __import__(name, globals(), locals()) - logger.info("Has: %s" % dir(module)) - logger.info("Name: %s" % module.PLUGIN_NAME) - logger.info("Provides: %s" % module.PLUGIN_PROVIDES) + logger.info("Has: %s", dir(module)) + logger.info("Name: %s", module.PLUGIN_NAME) + logger.info("Provides: %s", module.PLUGIN_PROVIDES) # If a plugin requires a different app, do not use it # TODO: accept a list also, log it @@ -269,7 +299,7 @@ def loadPlugin(self, name): return newPlugin = {} - newPlugin[CLASS] = getattr(module, module.PLUGIN_PROVIDES) + newPlugin[CLASS] = getattr(module, module.PLUGIN_PROVIDES) newPlugin[DESCRIPTION] = str(module.PLUGIN_DESCRIPTION) if hasattr(module, "PLUGIN_CATEGORY"): @@ -277,9 +307,10 @@ def loadPlugin(self, name): self.__plugins[module.PLUGIN_NAME] = newPlugin - except Exception as e: - logger.warning("Failed to load plugin %s\n%s" % (name, - ''.join(traceback.format_exception(*sys.exc_info())) )) + except Exception: + logger.warning( + "Failed to load plugin %s\n%s", + name, ''.join(traceback.format_exception(*sys.exc_info()))) def setupPluginMenu(self, menu): """Adds a plugin menu option to the supplied menubar @@ -290,8 +321,9 @@ def setupPluginMenu(self, menu): # Create the submenus (ordered) submenus = {} menu_locations = {"root": []} - for category in set([plugin[CATEGORY] for plugin in list(self.__plugins.values()) - if CATEGORY in plugin]): + plugin_categories = { + plugin[CATEGORY] for plugin in list(self.__plugins.values()) if CATEGORY in plugin} + for category in plugin_categories: submenus[category] = QtWidgets.QMenu(category, menu) menu.addMenu(submenus[category]) menu_locations[category] = [] @@ -319,19 +351,23 @@ def _handlePluginMenu(self, action): plugin_name = str(action.text()).split("%s" % self.__menu_separator)[0] self.launchPlugin(plugin_name, "") + class Plugin(object): + """Represents a single plugin.""" + def __init__(self): self.__settings = [] - def pluginRestoreState(self, saved): + def pluginRestoreState(self, saved_settings): """Called on plugin start with any previously saved state. - @param settings: Last state of the plugin instance - @type settings: dict""" - if self.__settings and saved and isinstance(saved, dict): + + @param saved_settings: Last state of the plugin instance + @type saved_settings: dict""" + if self.__settings and saved_settings and isinstance(saved_settings, dict): for setting in self.__settings: item = setting[SETTINGS_KEY] - if item in saved: - setting[SETTINGS_SET](saved[item]) + if item in saved_settings: + setting[SETTINGS_SET](saved_settings[item]) def pluginSaveState(self): """Called on application exit and returns plugin state information. diff --git a/cuegui/cuegui/PreviewWidget.py b/cuegui/cuegui/PreviewWidget.py index e913cb56e..6e60a0a6a 100644 --- a/cuegui/cuegui/PreviewWidget.py +++ b/cuegui/cuegui/PreviewWidget.py @@ -13,16 +13,24 @@ # limitations under the License. +"""Widget for displaying a preview of a frame in an image viewer.""" + + from __future__ import absolute_import from __future__ import print_function from __future__ import division + +# pylint: disable=wrong-import-position from future import standard_library standard_library.install_aliases() +# pylint: enable=wrong-import-position import os -import time -import urllib.request, urllib.error, urllib.parse import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request import xml.etree.ElementTree as Et from PySide2 import QtCore @@ -59,7 +67,7 @@ def __init__(self, job, frame, aovs=False, parent=None): self.__itvFile = None layout = QtWidgets.QVBoxLayout(self) - + self.__msg = QtWidgets.QLabel("Waiting for preview images...", self) self.__progbar = QtWidgets.QProgressBar(self) @@ -67,14 +75,15 @@ def __init__(self, job, frame, aovs=False, parent=None): layout.addWidget(self.__progbar) def process(self): + """Opens the image viewer.""" items = [] http_host = self.__frame.resource().split("/")[0] http_port = self.__findHttpPort() - + aovs = "" if self.__aovs: aovs = "/aovs" - + playlist = urllib.request.urlopen("http://%s:%d%s" % (http_host, http_port, aovs)).read() for element in Et.fromstring(playlist).findall("page/edit/element"): items.append(element.text) @@ -84,25 +93,32 @@ def process(self): self.__itvFile = self.__writePlaylist(playlist) self.__previewThread = PreviewProcessorWatchThread(items, self) + # pylint: disable=no-member QtGui.qApp.threads.append(self.__previewThread) + # pylint: enable=no-member self.__previewThread.start() self.__progbar.setRange(0, len(items)) self.__previewThread.existCountChanged.connect(self.updateProgressDialog) self.__previewThread.timeout.connect(self.processTimedOut) - def updateProgressDialog(self, current, max): - if max != current: + def updateProgressDialog(self, current, max_progress): + """Updates the progress dialog.""" + if max_progress != current: self.__progbar.setValue(current) else: self.close() - + def processTimedOut(self): + """Event handler when the process has timed out.""" self.close() - QtWidgets.QMessageBox.critical(self, "Preview Timeout", "Unable to preview images, " + - "timed out while waiting for images to be copied.") + QtWidgets.QMessageBox.critical( + self, + "Preview Timeout", + "Unable to preview images, timed out while waiting for images to be copied.") - def __writePlaylist(self, data): + @staticmethod + def __writePlaylist(data): (fh, name) = tempfile.mkstemp(suffix=".itv", prefix="playlist") os.close(fh) fp = open(name, "w") @@ -140,7 +156,7 @@ class PreviewProcessorWatchThread(QtCore.QThread): def __init__(self, items, parent=None): QtCore.QThread.__init__(self, parent) self.__items = items - self.__timeout = 60 + (30 * len(items)) + self.__timeout = 60 + (30 * len(items)) def run(self): """ diff --git a/cuegui/cuegui/ProcMonitor.py b/cuegui/cuegui/ProcMonitor.py index 64b9fd2fa..63035d7da 100644 --- a/cuegui/cuegui/ProcMonitor.py +++ b/cuegui/cuegui/ProcMonitor.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Widget for displaying a list of procs.""" + + from __future__ import absolute_import from __future__ import print_function from __future__ import division @@ -35,10 +38,13 @@ class ProcMonitor(QtWidgets.QWidget): - """This contains the frame list table with controls at the top""" + """Widget for displaying a list of procs.""" + def __init__(self, parent): QtWidgets.QWidget.__init__(self, parent) + self.__filterByHostNameLastInput = None + self.procMonitorTree = cuegui.ProcMonitorTree.ProcMonitorTree(self) # Setup main vertical layout @@ -51,36 +57,43 @@ def __init__(self, parent): # This hlayout would contain any filter/control buttons hlayout = QtWidgets.QHBoxLayout() - self.__filterByHostNameSetup(hlayout) # Menu to filter by proc name + self.__filterByHostNameSetup(hlayout) hlayout.addStretch() - self.__refreshToggleCheckBoxSetup(hlayout) # Checkbox to enable/disable auto refresh - self.__refreshButtonSetup(hlayout) # Button to refresh - self.__clearButtonSetup(hlayout) # Button to clear all filters + self.__refreshToggleCheckBoxSetup(hlayout) + self.__refreshButtonSetup(hlayout) + self.__clearButtonSetup(hlayout) self.layout().addLayout(hlayout) self.layout().addWidget(self.procMonitorTree) - self.__viewProcsSetup() # For view_hosts signal - self.__hostDoubleClickedSetup() # Views procs when a host is double clicked + self.__viewProcsSetup() + self.__hostDoubleClickedSetup() + + self.__viewHostsSetup() - self.__viewHostsSetup() # For view_hosts signal - - if bool(int(QtGui.qApp.settings.value("AutoRefreshMonitorProc", 1))): # For refresh on launch + # pylint: disable=no-member + if bool(int(QtGui.qApp.settings.value("AutoRefreshMonitorProc", 1))): self.updateRequest() + # pylint: enable=no-member def updateRequest(self): + """Requests an update to the widget's contents.""" self.procMonitorTree.updateRequest() def getColumnVisibility(self): + """Gets a list of whether table columns are visible.""" return self.procMonitorTree.getColumnVisibility() def setColumnVisibility(self, settings): + """Sets whether table columns are visible.""" self.procMonitorTree.setColumnVisibility(settings) def getColumnOrder(self): + """Gets table column order.""" return self.procMonitorTree.getColumnOrder() def setColumnOrder(self, settings): + """Sets table column order.""" self.procMonitorTree.setColumnOrder(settings) # ============================================================================== @@ -130,7 +143,9 @@ def __refreshToggleCheckBoxSetup(self, layout): def __refreshToggleCheckBoxHandle(self, state): self.procMonitorTree.enableRefresh = bool(state) + # pylint: disable=no-member QtGui.qApp.settings.setValue("AutoRefreshMonitorProc", int(bool(state))) + # pylint: enable=no-member # ============================================================================== # Button to refresh @@ -179,7 +194,9 @@ def __clearButtonHandle(self): # Monitors and handles the view_procs signal # ============================================================================== def __viewProcsSetup(self): + # pylint: disable=no-member QtGui.qApp.view_procs.connect(self.__viewProcsHandle) + # pylint: enable=no-member def __viewProcsHandle(self, hosts): self.procMonitorTree.procSearch.options['host'] = hosts @@ -189,7 +206,9 @@ def __viewProcsHandle(self, hosts): # Views procs when a host is double clicked # ============================================================================== def __hostDoubleClickedSetup(self): + # pylint: disable=no-member QtGui.qApp.view_object.connect(self.__hostDoubleClickedHandle) + # pylint: enable=no-member def __hostDoubleClickedHandle(self, rpcObject): if cuegui.Utils.isHost(rpcObject): @@ -200,7 +219,9 @@ def __hostDoubleClickedHandle(self, rpcObject): # Monitors and handles the view_hosts signal # ============================================================================== def __viewHostsSetup(self): + # pylint: disable=no-member QtGui.qApp.view_hosts.connect(self.__viewHostsHandle) + # pylint: enable=no-member def __viewHostsHandle(self, hosts): if hosts: diff --git a/cuegui/cuegui/ProcMonitorTree.py b/cuegui/cuegui/ProcMonitorTree.py index a784fff37..efbf0c86f 100644 --- a/cuegui/cuegui/ProcMonitorTree.py +++ b/cuegui/cuegui/ProcMonitorTree.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -A frame list based on AbstractTreeWidget -""" +"""Tree widget for displaying a list of procs.""" from __future__ import absolute_import @@ -43,36 +41,41 @@ class ProcMonitorTree(cuegui.AbstractTreeWidget.AbstractTreeWidget): + """Tree widget for displaying a list of procs.""" + def __init__(self, parent): self.startColumnsForType(cuegui.Constants.TYPE_PROC) - self.addColumn("Name", 150, id=1, - data=lambda proc: proc.data.name, - tip="Name of the running proc.") - self.addColumn("Cores", 50, id=2, - data=lambda proc: ("%.2f" % proc.data.reserved_cores), - tip="The number of cores reserved.") - self.addColumn("Mem Reserved", 100, id=3, - data=lambda proc: cuegui.Utils.memoryToString(proc.data.reserved_memory), - tip="The amount of memory reserved.") - self.addColumn("Mem Used", 100, id=4, - data=lambda proc: cuegui.Utils.memoryToString(proc.data.used_memory), - tip="The amount of memory used.") - self.addColumn("GPU Used", 100, id=5, - data=lambda proc: cuegui.Utils.memoryToString(proc.data.reserved_gpu), - tip="The amount of gpu memory used.") - self.addColumn("Age", 60, id=6, - data=lambda proc: cuegui.Utils.secondsToHHHMM(time.time() - proc.data.dispatch_time), - tip="The age of the running frame.") - self.addColumn("Unbooked", 80, id=7, - data=lambda proc: proc.data.unbooked, - tip="If the proc has been unbooked.\n If it is unbooked then" - "when the frame finishes the job will stop using this proc") - self.addColumn("Name", 300, id=8, - data=lambda proc: proc.data.frame_name , - tip="The name of the proc, includes frame number and layer name.") - self.addColumn("Job", 50, id=9, - data=lambda proc: proc.data.job_name , - tip="The job that this proc is running on.") + self.addColumn( + "Name", 150, id=1, data=lambda proc: proc.data.name, tip="Name of the running proc.") + self.addColumn( + "Cores", 50, id=2, data=lambda proc: ("%.2f" % proc.data.reserved_cores), + tip="The number of cores reserved.") + self.addColumn( + "Mem Reserved", 100, id=3, + data=lambda proc: cuegui.Utils.memoryToString(proc.data.reserved_memory), + tip="The amount of memory reserved.") + self.addColumn( + "Mem Used", 100, id=4, + data=lambda proc: cuegui.Utils.memoryToString(proc.data.used_memory), + tip="The amount of memory used.") + self.addColumn( + "GPU Used", 100, id=5, + data=lambda proc: cuegui.Utils.memoryToString(proc.data.reserved_gpu), + tip="The amount of gpu memory used.") + self.addColumn( + "Age", 60, id=6, + data=lambda proc: cuegui.Utils.secondsToHHHMM(time.time() - proc.data.dispatch_time), + tip="The age of the running frame.") + self.addColumn( + "Unbooked", 80, id=7, data=lambda proc: proc.data.unbooked, + tip="If the proc has been unbooked.\n If it is unbooked then" + "when the frame finishes the job will stop using this proc") + self.addColumn( + "Name", 300, id=8, data=lambda proc: proc.data.frame_name, + tip="The name of the proc, includes frame number and layer name.") + self.addColumn( + "Job", 50, id=9, data=lambda proc: proc.data.job_name, + tip="The job that this proc is running on.") self.procSearch = opencue.search.ProcSearch() @@ -86,12 +89,17 @@ def __init__(self, parent): self.itemDoubleClicked.connect(self.__itemDoubleClickedViewLog) # Don't use the standard space bar to refresh + # pylint: disable=no-member QtGui.qApp.request_update.connect(self.updateRequest) + # pylint: enable=no-member self.startTicksUpdate(40) # Don't start refreshing until the user sets a filter or hits refresh self.ticksWithoutUpdate = -1 + + # pylint: disable=no-member self.enableRefresh = bool(int(QtGui.qApp.settings.value("AutoRefreshMonitorProc", 1))) + # pylint: enable=no-member def tick(self): if self.ticksWithoutUpdate >= self.updateInterval and \ @@ -117,20 +125,27 @@ def __itemSingleClickedCopy(self, item, col): @param item: The item clicked on @type col: int @param col: The column clicked on""" + del item + del col selected = [proc.data.name for proc in self.selectedObjects() if cuegui.Utils.isProc(proc)] if selected: QtWidgets.QApplication.clipboard().setText(",".join(selected)) + # pylint: disable=no-self-use def __itemDoubleClickedViewLog(self, item, col): """Called when a proc is double clicked @type item: QTreeWidgetItem @param item: The item double clicked on @type col: int @param col: Column number double clicked on""" + del col job_name = item.rpcObject.data.job_name + # pylint: disable=no-member QtGui.qApp.view_object.emit(opencue.api.findJob(job_name)) + # pylint: enable=no-member def clearFilters(self): + """Removes all sorting and filtering to restore default state.""" self.clearSelection() self.procSearch = opencue.search.ProcSearch() self.sortByColumn(0, QtCore.Qt.AscendingOrder) @@ -141,6 +156,7 @@ def updateRequest(self): since last updated""" self.ticksWithoutUpdate = 999 + # pylint: disable=too-many-boolean-expressions def _getUpdate(self): """Returns the proper data from the cuebot""" try: @@ -155,21 +171,21 @@ def _getUpdate(self): not self.procSearch.options.get('durationRange'): return [] return opencue.api.getProcs(**self.procSearch.options) - except Exception as e: + except opencue.exception.CueException as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) return [] - def _createItem(self, object, parent = None): + def _createItem(self, rpcObject, parent=None): """Creates and returns the proper item - @type object: Proc - @param object: The object for this item + @type rpcObject: Proc + @param rpcObject: The object for this item @type parent: QTreeWidgetItem @param parent: Optional parent for this item @rtype: QTreeWidgetItem @return: The created item""" if not parent: parent = self - return ProcWidgetItem(object, parent) + return ProcWidgetItem(rpcObject, parent) def contextMenuEvent(self, e): """When right clicking on an item, this raises a context menu""" @@ -182,6 +198,8 @@ def contextMenuEvent(self, e): class ProcWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): - def __init__(self, object, parent): + """Widget item representing a single proc.""" + + def __init__(self, rpcObject, parent): cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( - self, cuegui.Constants.TYPE_PROC, object, parent) + self, cuegui.Constants.TYPE_PROC, rpcObject, parent) diff --git a/cuegui/cuegui/ProgressDialog.py b/cuegui/cuegui/ProgressDialog.py index b2fcee2ef..399a1c64a 100644 --- a/cuegui/cuegui/ProgressDialog.py +++ b/cuegui/cuegui/ProgressDialog.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -A progress dialog that accepts a list of work units and displays the progress. -""" +"""A progress dialog that accepts a list of work units and displays the progress.""" from __future__ import absolute_import @@ -37,8 +35,10 @@ class ProgressDialog(QtWidgets.QDialog): + """A progress dialog that accepts a list of work units and displays the progress.""" + def __init__(self, title, function, work, concurrent, cancelTitle, - cancelText, parent = None): + cancelText, parent=None): """Creates, displays and starts the progress bar. @type title: str @param title: The title for the progress bar @@ -87,7 +87,8 @@ def __init__(self, title, function, work, concurrent, cancelTitle, self.show() - for thread in range(max(concurrent, 1)): + # Submit a new unit of work to the threadpool for each concurrent thread. + for _ in range(max(concurrent, 1)): self._submitWork() def closeEvent(self, event): @@ -124,6 +125,7 @@ def __doWork(self): self.__workLock.unlock() if work: + # pylint: disable=broad-except try: self.__function(*work) except Exception as e: @@ -136,6 +138,9 @@ def __doneWork(self, work, result): @param work: From threadpool (unused) @type result: @param result: From threadpool (unused)""" + del work + del result + self.__count -= 1 self.__bar.setValue(self.__bar.value() + 1) @@ -157,9 +162,11 @@ def _submitWork(self): self.__count += 1 if hasattr(QtGui.qApp, "threadpool"): + # pylint: disable=no-member QtGui.qApp.threadpool.queue(self.__doWork, self.__doneWork, "getting data for %s" % self.__class__) + # pylint: enable=no-member else: logger.warning("threadpool not found, doing work in gui thread") self.__doneWork(None, self.__doWork()) diff --git a/cuegui/cuegui/Redirect.py b/cuegui/cuegui/Redirect.py index 88c6f1738..cdbaa5b35 100644 --- a/cuegui/cuegui/Redirect.py +++ b/cuegui/cuegui/Redirect.py @@ -13,17 +13,18 @@ # limitations under the License. -""" -An interface for redirecting resources from one -job to another job. -""" +"""An interface for redirecting resources from one job to another job. + +The concept here is that there is a target job that needs procs. The user would choose the job. +The highest core/memory value would be detected and would populate 2 text boxes for cores and +memory. The user could then adjust these and hit search. The search will find all hosts that have +frames running that can be redirected to the target job.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division -from past.builtins import cmp from builtins import str from builtins import range import os @@ -39,14 +40,6 @@ import cuegui.Utils -# The concept here is that there is a target job that needs -# procs. The user would choose the job. The highest core/memory -# value would be dected and would populate 2 text boxes for cores -# and memory. The user could then adjust these and hit search. -# -# The search will find all hosts that have frames running that can be -# redirected to the target job. - class ShowCombo(QtWidgets.QComboBox): """ A combo box for show selection @@ -57,6 +50,7 @@ def __init__(self, selected="pipe", parent=None): self.setCurrentIndex(self.findText(selected)) def refresh(self): + """Refreshes the show list.""" self.clear() shows = opencue.api.getActiveShows() shows.sort(key=lambda x: x.data.name) @@ -65,6 +59,7 @@ def refresh(self): self.addItem(show.data.name, show) def getShow(self): + """Gets the selected show.""" return str(self.setCurrentText()) @@ -88,9 +83,7 @@ def __init__(self, parent=None): self.__menu.triggered.connect(self.__afterClicked) def refresh(self): - """ - Refresh the full list of allocations. - """ + """Refreshes the full list of allocations.""" allocs = opencue.api.getAllocations() allocs.sort(key=lambda x: x.data.name) @@ -133,6 +126,7 @@ def __afterClicked(self, action): """ Execute after an allocation has been selected for filtering. """ + del action self.__setSelected() self.setText("Allocations (%d)" % len(self.__selected)) @@ -148,6 +142,7 @@ def __init__(self, parent=None): self.refresh() def refresh(self): + """Refreshes the list of job names.""" slist = opencue.api.getJobNames() slist.sort() @@ -172,8 +167,10 @@ def __init__(self, show, name, parent=None): self.__menu.aboutToShow.connect(self.__populate_menu) + # pylint: disable=inconsistent-return-statements def __loadShow(self, show): - self.__actions = { } + self.__actions = {} + # pylint: disable=bare-except try: if show: return show @@ -181,6 +178,7 @@ def __loadShow(self, show): return opencue.api.findShow(show.name()) def showChanged(self, show): + """Loads a new show.""" self.__show = self.__loadShow(show) def __populate_menu(self): @@ -196,6 +194,7 @@ def __populate_menu(self): self.__menu.addAction(action) def getChecked(self): + """Gets a list of action text for all selected actions.""" return [str(action.text()) for action in list(self.__actions.values()) if action.isChecked()] @@ -298,14 +297,18 @@ def _cfg(self): return self.__config def showChanged(self, show_index): + """Load a new show.""" + del show_index show = self.__show_combo.currentText() self.__current_show = opencue.api.findShow(str(show)) self.__include_group_btn.showChanged(self.__current_show) def detect(self, name=None): + """Populates initial values when the job name is changed.""" + del name try: job = opencue.api.findJob(str(self.__job_box.text())) - except: + except opencue.exception.CueException: return layers = job.getLayers() @@ -323,48 +326,63 @@ def detect(self, name=None): self.__show_combo.setCurrentIndex(self.__show_combo.findText(job.data.show)) def getJob(self): + """Gets the current job name.""" return str(self.__job_box.text()) def getCores(self): + """Gets the core count.""" return int(self.__cores_spin.value()) def getMemory(self): + """Gets the memory amount.""" return int(self.__mem_spin.value() * 1048576.0) def getJobBox(self): + """Gets the job box widget.""" return self.__job_box def getUpdateButton(self): + """Gets the update button widget.""" return self.__update_btn def getRedirectButton(self): + """Gets the redirect button widget.""" return self.__redirect_btn def getSelectAllButton(self): + """Gets the select all button widget.""" return self.__select_all_btn def getClearButton(self): + """Gets the clear button widget.""" return self.__clear_btn def getShow(self): + """Gets the current show.""" return self.__current_show def getAllocFilter(self): + """Gets the allocation filter.""" return self.__alloc_filter def getLimit(self): + """Gets the limit.""" return self.__limit_spin.value() def getCutoffTime(self): + """Gets the cutoff time.""" return int(self.__prh_spin.value() * 3600.0) def getRequiredService(self): + """Gets the required service name.""" return str(self.__require_services.text()).strip() def getJobNameExcludeRegex(self): + """Gets the regex of job name to exclude.""" return str(self.__exclude_regex.text()).strip() def getIncludedGroups(self): + """Gets the value of the include groups checkbox.""" return self.__include_group_btn.getChecked() @@ -416,24 +434,24 @@ def __get_selected_procs_by_alloc(self, selected_items): entry = self.__hosts.get(str(item.text())) alloc = entry.get('alloc') alloc_procs = procs_by_alloc.get(alloc, []) - alloc_procs.extend([proc for proc in entry["procs"]]) + alloc_procs.extend(list(entry["procs"])) procs_by_alloc[alloc] = alloc_procs return procs_by_alloc def __warn(self, msg): - ''' + """ Displays the given message for the user to acknowledge @param msg: The message to display @type msg: str - ''' + """ message = QtWidgets.QMessageBox(self) message.setText(msg) message.exec_() def __is_cross_show_safe(self, procs, target_show): - ''' + """ Determines whether or not it's safe to redirect cores from a show to another, based on user response to the warning message @@ -446,7 +464,7 @@ def __is_cross_show_safe(self, procs, target_show): @return: Whether or not it's safe to redirect the given procs to the target show @rtype: bool - ''' + """ xshow_jobs = [proc.getJob() for proc in procs if not proc.getJob().show() == target_show] @@ -464,7 +482,7 @@ def __is_cross_show_safe(self, procs, target_show): in xshow_jobs]) def __is_burst_safe(self, alloc, procs, show): - ''' + """ Determines whether or not it's safe to redirect cores by checking the burst target show burst and the number of cores being redirected. If there's a number of cores that may not be possible to pick up by the @@ -483,10 +501,12 @@ def __is_burst_safe(self, alloc, procs, show): @return: Whether or not it's safe to kill these cores based on the subscription burst of the target show @rtype: bool - ''' + """ # Skip if this check is disabled in the config + # pylint: disable=protected-access cfg = self.__controls._cfg() + # pylint: enable=protected-access wc_ok = cfg.get('wasted_cores_threshold', 100) if wc_ok < 0: return True @@ -524,12 +544,12 @@ def __is_burst_safe(self, alloc, procs, show): return False def redirect(self): - ''' + """ Redirect the selected procs to the target job, after running a few checks to verify it's safe to do that. @postcondition: The selected procs are redirected to the target job - ''' + """ # Get selected items items = [self.__model.item(row) for row @@ -567,11 +587,11 @@ def redirect(self): for item in selected_items: entry = self.__hosts.get(str(item.text())) procs = entry["procs"] + # pylint: disable=broad-except try: host = entry["host"] host.redirectToJob(procs, job) except Exception as e: - print(e) errors.append(str(e)) item.setIcon(QtGui.QIcon(QtGui.QPixmap(":retry.png"))) item.setEnabled(False) @@ -612,7 +632,7 @@ def update(self): self.__controls.getLimit(), self) progress.setWindowModality(QtCore.Qt.WindowModal) - for num, proc in enumerate(procs): + for proc in procs: if progress.wasCanceled(): break @@ -647,7 +667,7 @@ def update(self): cue_host = opencue.api.findHost(name) hosts[name] = { "host": cue_host, - "procs":[], + "procs": [], "mem": cue_host.data.idle_memory, "cores": int(cue_host.data.idle_cores), "time": 0, @@ -661,16 +681,15 @@ def update(self): host["procs"].append(proc) host["mem"] = host["mem"] + proc.data.reserved_memory host["cores"] = host["cores"] + proc.data.reserved_cores - host["time"] = host["time"] + (int(time.time()) - proc.data.dispatch_time); + host["time"] = host["time"] + (int(time.time()) - proc.data.dispatch_time) if host["cores"] >= self.__controls.getCores() and \ - host["mem"] >= self.__controls.getMemory() and \ - host["time"] < self.__controls.getCutoffTime(): - self.__addHost(host) - host["ok"] = True - ok = ok + 1 - progress.setValue(ok) - + host["mem"] >= self.__controls.getMemory() and \ + host["time"] < self.__controls.getCutoffTime(): + self.__addHost(host) + host["ok"] = True + ok = ok + 1 + progress.setValue(ok) progress.setValue(self.__controls.getLimit()) # Save this for later on @@ -692,7 +711,8 @@ def __addHost(self, entry): for proc in procs: checkbox.appendRow([QtGui.QStandardItem(proc.data.job_name), QtGui.QStandardItem(str(proc.data.reserved_cores)), - QtGui.QStandardItem("%0.2fGB" % (proc.data.reserved_memory / 1048576.0)), + QtGui.QStandardItem( + "%0.2fGB" % (proc.data.reserved_memory / 1048576.0)), QtGui.QStandardItem(cuegui.Utils.secondsToHHMMSS(time.time() - proc.data.dispatch_time)), QtGui.QStandardItem(proc.data.group_name), diff --git a/cuegui/cuegui/ServiceDialog.py b/cuegui/cuegui/ServiceDialog.py index 2eccc6d70..04f1b1a89 100644 --- a/cuegui/cuegui/ServiceDialog.py +++ b/cuegui/cuegui/ServiceDialog.py @@ -13,7 +13,7 @@ # limitations under the License. -"""Service related widgets.""" +"""Dialog for displaying and editing services.""" from __future__ import absolute_import @@ -34,9 +34,8 @@ class ServiceForm(QtWidgets.QWidget): - """ - An Widget for displaying and editing a service. - """ + """Widget for displaying and editing a service.""" + saved = QtCore.Signal(object) def __init__(self, parent=None): @@ -100,7 +99,6 @@ def __init__(self, parent=None): self.__buttons.accepted.connect(self.save) - def _cfg(self): """ Loads (if necessary) and returns the config values. @@ -178,7 +176,7 @@ def save(self): class ServiceManager(QtWidgets.QWidget): """ Wraps the ServiceForm widget with the logic and controls needed - to add, update, and detete services. + to add, update, and delete services. """ def __init__(self, show, parent=None): QtWidgets.QWidget.__init__(self, parent) @@ -218,6 +216,8 @@ def selected(self, item, old_item): """ Executes if an item is selected """ + del old_item + self.__new_service = False if not item: diff --git a/cuegui/cuegui/ShowDialog.py b/cuegui/cuegui/ShowDialog.py index 569e723ab..865e75aa6 100644 --- a/cuegui/cuegui/ShowDialog.py +++ b/cuegui/cuegui/ShowDialog.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -Displays the show dialog with show configuration options -""" +"""A dialog displaying show configuration options.""" from __future__ import absolute_import @@ -31,6 +29,8 @@ class ShowDialog(QtWidgets.QDialog): + """A dialog displaying show configuration options.""" + def __init__(self, show, parent=None): QtWidgets.QDialog.__init__(self, parent) @@ -127,8 +127,6 @@ def __createStatisticsPage(self): text.setPlainText("%s" % self.__show.data.show_stats) page.layout().addWidget(text) - #page.layout().setRowStretch(10, 100) - return page def __createRawShowDataPage(self): @@ -147,6 +145,7 @@ def __createRawShowDataPage(self): def __valueChanged(self, value=None): """Called when something changes to enable the save button""" + del value self.__btnSave.setEnabled(True) def __closeDialog(self): diff --git a/cuegui/cuegui/ShowsWidget.py b/cuegui/cuegui/ShowsWidget.py index 8dbe22cc2..b0343834f 100644 --- a/cuegui/cuegui/ShowsWidget.py +++ b/cuegui/cuegui/ShowsWidget.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Tree widget for displaying a list of shows.""" + + from __future__ import absolute_import from __future__ import print_function from __future__ import division @@ -34,6 +37,8 @@ class ShowsWidget(cuegui.AbstractTreeWidget.AbstractTreeWidget): + """Tree widget for displaying a list of shows.""" + def __init__(self, parent): self.startColumnsForType(cuegui.Constants.TYPE_SHOW) self.addColumn("Show Name", 90, id=1, @@ -58,7 +63,9 @@ def __init__(self, parent): self, self.updateSoon, self.selectedObjects) self.itemClicked.connect(self.__itemSingleClickedToDouble) + # pylint: disable=no-member QtGui.qApp.facility_changed.connect(self.__facilityChanged) + # pylint: enable=no-member self.setUpdateInterval(60) @@ -76,16 +83,17 @@ def __itemSingleClickedToDouble(self, item, col): @param col: Column number single clicked on""" self.itemDoubleClicked.emit(item, col) - def _createItem(self, object): + def _createItem(self, rpcObject): """Creates and returns the proper item""" - item = ShowWidgetItem(object, self) + item = ShowWidgetItem(rpcObject, self) return item + # pylint: disable=no-self-use def _getUpdate(self): """Returns the proper data from the cuebot""" try: return opencue.api.getActiveShows() - except Exception as e: + except opencue.exception.CueException as e: logger.critical(e) return [] @@ -103,8 +111,13 @@ def contextMenuEvent(self, e): menu.exec_(QtCore.QPoint(e.globalX(), e.globalY())) + def tick(self): + pass + class ShowWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): - def __init__(self, object, parent): + """Widget item representing a single show.""" + + def __init__(self, rpcObject, parent): cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( - self, cuegui.Constants.TYPE_SHOW, object, parent) + self, cuegui.Constants.TYPE_SHOW, rpcObject, parent) diff --git a/cuegui/cuegui/SplashWindow.py b/cuegui/cuegui/SplashWindow.py index 511967375..29c268c7a 100644 --- a/cuegui/cuegui/SplashWindow.py +++ b/cuegui/cuegui/SplashWindow.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Splash screen displayed on initial application launch.""" + + from __future__ import division from __future__ import print_function from __future__ import absolute_import @@ -30,6 +33,8 @@ class SplashWindow(object): + """Splash screen displayed on initial application launch.""" + def __init__(self, app, app_name, version, resource_path): self.app = app @@ -64,19 +69,23 @@ def _findSplash(self, app_name, version, resource_path): break if image is None: + # pylint: disable=broad-except try: image = self._GenerateMissingSplash(app_name) except Exception: return None + # pylint: disable=broad-except try: self._StampVersion(image, version) except Exception: pass return image - def _generateSplashFromImage(self, imagePath): + @staticmethod + def _generateSplashFromImage(imagePath): if os.path.isfile(imagePath): + # pylint: disable=broad-except try: return imagePath and QtGui.QImage(imagePath) except Exception: diff --git a/cuegui/cuegui/Style.py b/cuegui/cuegui/Style.py index 236054ff5..3f25bbed0 100644 --- a/cuegui/cuegui/Style.py +++ b/cuegui/cuegui/Style.py @@ -13,7 +13,7 @@ # limitations under the License. -"""a module for handling global style setup""" +"""Module for handling global style setup.""" from __future__ import print_function @@ -30,35 +30,42 @@ DEFAULT_FONT = "Luxi Sans" DEFAULT_FONT_SIZE = 10.0 +# pylint: disable=global-statement + ColorTheme = None IconTheme = None Font = None def loadColorTheme(name): - """changes the running color scheme of the app""" + """Changes the running color scheme of the app.""" global ColorTheme ColorTheme = importlib.import_module('.%s' % name, package='cuegui') ColorTheme.init() def setIconTheme(name): - """stes the icon theme for the app, not sure if this - can be changed on the fly yet""" + """Sets the icon theme for the app. + + Not sure if this can be changed on the fly yet.""" global IconTheme IconTheme = importlib.import_module('.icons_rcc', package='cuegui.images.%s' % name) def setFont(font): - """sets the application font""" + """Sets the application font.""" global Font Font = font + # pylint: disable=no-member QtGui.qApp.setFont(font) + # pylint: enable=no-member def init(): - """initialize the global style settings""" + """Initializes the global style settings.""" + # pylint: disable=no-member settings = QtGui.qApp.settings + # pylint: enable=no-member loadColorTheme(settings.value("Style/colorTheme", DEFAULT_COLOR)) setIconTheme(settings.value("Style/iconTheme", DEFAULT_ICON)) diff --git a/cuegui/cuegui/SubscriptionGraphWidget.py b/cuegui/cuegui/SubscriptionGraphWidget.py index 25981bf31..2e122328c 100644 --- a/cuegui/cuegui/SubscriptionGraphWidget.py +++ b/cuegui/cuegui/SubscriptionGraphWidget.py @@ -1,9 +1,26 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Widget for displaying graph of subscription usage.""" + + from builtins import str import opencue from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import cuegui.AbstractTreeWidget @@ -17,6 +34,8 @@ class SubscriptionGraphWidget(QtWidgets.QWidget): + """Widget for displaying graph of subscription usage.""" + def __init__(self, parent): QtWidgets.QWidget.__init__(self, parent) @@ -53,6 +72,7 @@ def __init__(self, parent): layout.addWidget(scroll) def create_widgets(self): + """Creates all of the contained widgets.""" self.clearLayout(self.mainLayout) for show in self.__shows: widget = QtWidgets.QWidget() @@ -70,7 +90,9 @@ def create_widgets(self): self.__timer.start() - def clearLayout(self, layout): + @staticmethod + def clearLayout(layout): + """Clears the widget layout.""" while layout.count() > 0: item = layout.takeAt(0) if not item: @@ -82,8 +104,9 @@ def clearLayout(self, layout): def __showMenuHandle(self, action): if action.text() == 'All Shows': try: - self.__shows = sorted(set([job.show() for job in opencue.api.getJobs(include_finished=True)])) - except Exception as e: + self.__shows = sorted( + {job.show() for job in opencue.api.getJobs(include_finished=True)}) + except opencue.exception.CueException: self.__shows = [] self.__showMenuUpdate() elif action.text() == 'Clear': @@ -110,8 +133,8 @@ def __showMenuUpdate(self): self.__showMenu.addSeparator() try: - shows = sorted(set([job.show() for job in opencue.api.getJobs(include_finished=True)])) - except Exception as e: + shows = sorted({job.show() for job in opencue.api.getJobs(include_finished=True)}) + except opencue.exception.CueException: shows = [] for show in shows: @@ -130,11 +153,15 @@ def __showMenuCheck(self): self.__showMenuUpdate() def update_data(self): + """Refreshes the displayed data.""" + # pylint: disable=protected-access for sub_bar in self.__subBars: sub_bar._getUpdate() class SubGraphTreeWidget(cuegui.AbstractTreeWidget.AbstractTreeWidget): + """Tree for displaying a subscription graph.""" + def __init__(self, parent): self.startColumnsForType(cuegui.Constants.TYPE_SUB) self.addColumn("_Name", 110, id=0, @@ -156,6 +183,7 @@ def __init__(self, parent): # self.setUpdateInterval(30) def setShow(self, show=None): + """Sets the current show.""" self._itemsLock.lockForWrite() try: if not show: @@ -165,18 +193,19 @@ def setShow(self, show=None): elif isinstance(show, str): try: self.__show = opencue.api.findShow(show) - except: + except opencue.exception.CueException: pass self._update() finally: self._itemsLock.unlock() def getShow(self): + """Gets the current show.""" return self.__show - def _createItem(self, object): - """Creates and returns the proper item""" - return SubscriptionWidgetItem(object, self) + def _createItem(self, rpcObject): + """Creates a widget item for the current subscription.""" + return SubscriptionWidgetItem(rpcObject, self) def _getUpdate(self): """Returns the proper data from the cuebot""" @@ -189,7 +218,7 @@ def _getUpdate(self): self._itemsLock.unlock() def contextMenuEvent(self, e): - """When right clicking on an item, this raises a context menu""" + """Event handler for showing the context menu.""" menu = QtWidgets.QMenu() self.__menuActions.subscriptions().addAction(menu, "editSize") @@ -204,11 +233,17 @@ def contextMenuEvent(self, e): menu.exec_(QtCore.QPoint(e.globalX(),e.globalY())) def createSubscription(self): + """Shows a dialog for creating a new subscription.""" d = cuegui.CreatorDialog.SubscriptionCreatorDialog(show=self.__show) d.exec_() + def tick(self): + pass + class SubscriptionWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): - def __init__(self, object, parent): + """Widget item for displaying a single subscription.""" + + def __init__(self, rpcObject, parent): cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( - self, cuegui.Constants.TYPE_SUB, object, parent) \ No newline at end of file + self, cuegui.Constants.TYPE_SUB, rpcObject, parent) diff --git a/cuegui/cuegui/SubscriptionsWidget.py b/cuegui/cuegui/SubscriptionsWidget.py index ea01d1946..3acb97921 100644 --- a/cuegui/cuegui/SubscriptionsWidget.py +++ b/cuegui/cuegui/SubscriptionsWidget.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Widget for listing and managing subscriptions.""" + + from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -34,10 +37,13 @@ class SubscriptionsWidget(QtWidgets.QWidget): + """Widget for listing and managing subscriptions.""" + def __init__(self, parent): QtWidgets.QWidget.__init__(self, parent) self.__show = None + self.__shows = None self.__comboShows = QtWidgets.QComboBox(self) self.__comboShows.setFocusPolicy(QtCore.Qt.NoFocus) @@ -61,8 +67,10 @@ def __init__(self, parent): self.__btnShowProperties.clicked.connect(self.__showProperties) self.__btnAddSubscription.clicked.connect(self.__addSubscription) self.__comboShows.currentIndexChanged.connect(self.setShow) + # pylint: disable=no-member QtGui.qApp.view_object.connect(self.setShow) QtGui.qApp.facility_changed.connect(self.changeFacility) + # pylint: enable=no-member self.__menuActions = cuegui.MenuActions.MenuActions( self, self.updateSoon, self.selectedObjects) @@ -70,17 +78,18 @@ def __init__(self, parent): self.changeFacility() def changeFacility(self): + """Changes the active facility.""" try: - self.__shows = dict([(show.name(), show) for show in opencue.api.getActiveShows()]) - except Exception: + self.__shows = {show.name(): show for show in opencue.api.getActiveShows()} + except opencue.exception.CueException: self.__shows = {} self.__comboShows.clear() - self.__comboShows.addItems(["Select Show:"] + - sorted(self.__shows.keys())) + self.__comboShows.addItems(["Select Show:"] + sorted(self.__shows.keys())) self.setShow() def setShow(self, show=""): - """Sets the show for the subscription list and combo box + """Sets the active show. + @type show: str or Show @param show: The show to monitor""" if isinstance(show, int): @@ -110,17 +119,22 @@ def setShow(self, show=""): self.__monitorSubscriptions.setShow(self.__show) def getShow(self): + """Gets the active show.""" return self.__show def getShowName(self): + """Gets the active show name.""" if self.__show: return self.__show.data.name return None def updateSoon(self): + """Requests an update of the subscriptions list.""" + # pylint: disable=protected-access self.__monitorSubscriptions._update() def selectedObjects(self): + """Gets a list of the active show(s).""" return [opencue.api.findShow(self.__show.name())] def __showProperties(self): @@ -136,28 +150,34 @@ def __addSubscription(self): self.updateSoon() def getColumnVisibility(self): + """Gets the table column visibility.""" return self.__monitorSubscriptions.getColumnVisibility() def setColumnVisibility(self, settings): + """Sets the table column visibility.""" self.__monitorSubscriptions.setColumnVisibility(settings) def getColumnOrder(self): + """Gets the table column order.""" return self.__monitorSubscriptions.getColumnOrder() def setColumnOrder(self, settings): + """Sets the table column order.""" self.__monitorSubscriptions.setColumnOrder(settings) class SubscriptionsTreeWidget(cuegui.AbstractTreeWidget.AbstractTreeWidget): - def __init__(self, parent): + """Tree widget for displaying a list of subscriptions.""" + def __init__(self, parent): self.startColumnsForType(cuegui.Constants.TYPE_SUB) self.addColumn("Alloc", 160, id=1, data=lambda sub: sub.data.allocation_name) self.addColumn("Usage", 70, id=2, - data=lambda sub: (sub.data.size and - ("%.2f%%" % (sub.data.reserved_cores / sub.data.size * 100)) - or 0), + data=lambda sub: ( + sub.data.size and + ("%.2f%%" % (sub.data.reserved_cores / sub.data.size * 100)) + or 0), sort=lambda sub: (sub.data.size and sub.data.reserved_cores / sub.data.size or 0)) self.addColumn("Size", 70, id=3, @@ -181,6 +201,7 @@ def __init__(self, parent): self.setUpdateInterval(30) def setShow(self, show=None): + """Sets the active show.""" self._itemsLock.lockForWrite() try: if not show: @@ -190,18 +211,19 @@ def setShow(self, show=None): elif isinstance(show, str): try: self.__show = opencue.api.findShow(show) - except: + except opencue.exception.CueException: pass self._update() finally: self._itemsLock.unlock() def getShow(self): + """Gets the active show.""" return self.__show - def _createItem(self, object): + def _createItem(self, rpcObject): """Creates and returns the proper item""" - return SubscriptionWidgetItem(object, self) + return SubscriptionWidgetItem(rpcObject, self) def _getUpdate(self): """Returns the proper data from the cuebot""" @@ -223,8 +245,13 @@ def contextMenuEvent(self, e): self.__menuActions.subscriptions().addAction(menu, "delete") menu.exec_(QtCore.QPoint(e.globalX(),e.globalY())) + def tick(self): + pass + class SubscriptionWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): - def __init__(self, object, parent): + """Widget item representing a single subscription.""" + + def __init__(self, rpcObject, parent): cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( - self, cuegui.Constants.TYPE_SUB, object, parent) + self, cuegui.Constants.TYPE_SUB, rpcObject, parent) diff --git a/cuegui/cuegui/TagsWidget.py b/cuegui/cuegui/TagsWidget.py index 77d9867eb..cde10b910 100644 --- a/cuegui/cuegui/TagsWidget.py +++ b/cuegui/cuegui/TagsWidget.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -A Widget for displaying and editing Tags -""" +"""A Widget for displaying and editing tags.""" from __future__ import absolute_import @@ -32,11 +30,11 @@ class TagsWidget(QtWidgets.QWidget): - """ - A Widget for displaying and editing Tags.Includes checkboxes for the given - list of standard tag options, and a textfield for the user to enter one or - more custom tags - """ + """A Widget for displaying and editing tags. + + Includes checkboxes for the given list of standard tag options, and a textfield for the + user to enter one or more custom tags.""" + def __init__(self, allowed_tags=None, parent=None): """ A Widget for displaying and editing Tags @@ -127,7 +125,7 @@ def get_tags(self): if self.__enable_custom.isChecked(): tags = str(self.__custom.text()) - tags = re.split('[\s,|]+', tags) + tags = re.split(r'[\s,|]+', tags) else: tags = [str(t.text()) for t in self.standard_tags.checkedBoxes()] return [tag.strip() for tag in tags if tag.strip().isalnum()] diff --git a/cuegui/cuegui/TasksDialog.py b/cuegui/cuegui/TasksDialog.py index 9fabb6f44..f624fcf6e 100644 --- a/cuegui/cuegui/TasksDialog.py +++ b/cuegui/cuegui/TasksDialog.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -Handles the dialog to display/modify a show's tasks -""" +"""Dialog to display/modify a show's tasks.""" from __future__ import absolute_import @@ -28,6 +26,8 @@ from PySide2 import QtCore from PySide2 import QtWidgets +import opencue.exception + import cuegui.AbstractTreeWidget import cuegui.AbstractWidgetItem import cuegui.Constants @@ -42,7 +42,9 @@ class TasksDialog(QtWidgets.QDialog): - def __init__(self, show, parent = None): + """Dialog to display/modify a show's tasks.""" + + def __init__(self, show, parent=None): QtWidgets.QDialog.__init__(self, parent) self.__show = show @@ -84,6 +86,7 @@ def __init__(self, show, parent = None): self.getDepartments() def getDepartments(self): + """Gets a list of deparments.""" selected = self.__comboDepartments.currentText() self.__departments = self.__show.getDepartments() @@ -94,15 +97,18 @@ def getDepartments(self): self.__comboDepartments.setCurrentIndex(departmentNames.index(selected)) def setDepartment(self, departmentName): + """Sets the department.""" for department in self.__departments: if department.data.name == departmentName: self.__tasks.setDepartment(department) self.__btnAddTask.setEnabled(not department.data.tiManaged) self.__checkManaged.setChecked(department.data.tiManaged) - self.__btnMinCores.setText(MANAGED_CORES_PREFIX + "%.02f" % department.data.minCores) + self.__btnMinCores.setText( + MANAGED_CORES_PREFIX + "%.02f" % department.data.minCores) return def setMinCores(self): + """Sets the department's min cores.""" __department = self.__tasks.department() if __department: (managedCores, choice) = self.__askManagedCores(__department) @@ -112,10 +118,12 @@ def setMinCores(self): self.getDepartments() def setManaged(self, checked): + """Sets the departments management.""" __department = self.__tasks.department() if not __department.data.tiManaged and checked: title = "Manage Department" - body = "What tiTask should be used to manage the %s department?" % __department.data.name + body = ("What tiTask should be used to manage the %s department?" % + __department.data.name) (tiTask, choice) = QtWidgets.QInputDialog.getText(self, title, body, QtWidgets.QLineEdit.Normal, @@ -126,9 +134,9 @@ def setManaged(self, checked): __department.enableTiManaged(str(tiTask), managedCores) if __department.data.tiManaged and not checked: - if cuegui.Utils.questionBoxYesNo(self, - "Confirm", - "Disable management of the %s department?" % __department.data.name): + if cuegui.Utils.questionBoxYesNo( + self, "Confirm", + "Disable management of the %s department?" % __department.data.name): __department.disableTiManaged() self.getDepartments() @@ -143,9 +151,13 @@ def __askManagedCores(self, department): return (managedCores, choice) def refresh(self): + """Requests a refresh of the tasks list.""" self.__tasks.updateRequest() + class TaskMonitorTree(cuegui.AbstractTreeWidget.AbstractTreeWidget): + """Tree widget for displaying a list of tasks.""" + def __init__(self, department, parent): self.startColumnsForType(cuegui.Constants.TYPE_TASK) self.addColumn("Shot", 100, id=1, @@ -165,18 +177,22 @@ def __init__(self, department, parent): self.__menuActions = cuegui.MenuActions.MenuActions( self, self.updateSoon, self.selectedObjects) self._timer.stop() + + self.__department = None self.setDepartment(department) def department(self): + """Gets the department.""" return self.__department def setDepartment(self, department): + """Sets the department.""" self.__department = department self._update() - def _createItem(self, object): + def _createItem(self, rpcObject): """Creates and returns the proper item""" - return TaskWidgetItem(object, self) + return TaskWidgetItem(rpcObject, self) def _update(self): """Adds the feature of forcing the items to be sorted by the first @@ -189,9 +205,8 @@ def _getUpdate(self): try: if self.__department: return self.__department.getTasks() - else: - return [] - except Exception as e: + return [] + except opencue.exception.CueException as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) return [] @@ -211,6 +226,7 @@ def contextMenuEvent(self, e): menu.exec_(e.globalPos()) def createTask(self): + """Creates a task.""" if self.__department: title = "Create Task" body = "What shot is this task for? " @@ -227,7 +243,13 @@ def createTask(self): self.__department.addTask(str(shot), float(minCores)) self._update() + def tick(self): + pass + + class TaskWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): - def __init__(self, object, parent): + """Widget item for displaying a single task.""" + + def __init__(self, rpcObject, parent): cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( - self, cuegui.Constants.TYPE_TASK, object, parent) + self, cuegui.Constants.TYPE_TASK, rpcObject, parent) diff --git a/cuegui/cuegui/TextEditDialog.py b/cuegui/cuegui/TextEditDialog.py index a53351976..d6a14526d 100644 --- a/cuegui/cuegui/TextEditDialog.py +++ b/cuegui/cuegui/TextEditDialog.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -A TextEdit dialog -""" +"""A TextEdit dialog.""" from __future__ import print_function @@ -27,8 +25,10 @@ class TextEditDialog(QtWidgets.QDialog): - def __init__(self, title, text, default = "", parent = None): - """A confirmation dialog + """A TextEdit dialog.""" + + def __init__(self, title, text, default="", parent=None): + """ @type title: string @param title: The title for the confirmation dialog @type text: string @@ -64,4 +64,5 @@ def __init__(self, title, text, default = "", parent = None): self.__textEdit.setFocus(QtCore.Qt.OtherFocusReason) def results(self): - return self.__textEdit.toPlainText() \ No newline at end of file + """Gets the dialog results as plaintext.""" + return self.__textEdit.toPlainText() diff --git a/cuegui/cuegui/ThreadPool.py b/cuegui/cuegui/ThreadPool.py index 7989df27b..73d791d50 100644 --- a/cuegui/cuegui/ThreadPool.py +++ b/cuegui/cuegui/ThreadPool.py @@ -62,6 +62,7 @@ def systemCpuCount(): """systemCpuCount()->int returns the # of procs on the system, linux only """ + # pylint: disable=bare-except try: return len([p for p in os.listdir("/sys/devices/system/cpu") if p.startswith("cpu")]) except: @@ -69,10 +70,7 @@ def systemCpuCount(): class ThreadPool(QtCore.QObject): - """ThreadPool(QtCore.QObject) - - ThreadPool is a general purpose work queue class - """ + """A general purpose work queue class.""" def __init__(self, num_threads, max_queue=20, parent=None): QtCore.QObject.__init__(self, parent=parent) @@ -86,33 +84,35 @@ def __init__(self, num_threads, max_queue=20, parent=None): self._q_queue = [] def start(self): + """Initializes the thread pool and starts running work.""" if self.__started: return self.__started = True for i in range(0, self.__num_threads): thread = ThreadPool.WorkerThread(i, self) + # pylint: disable=no-member QtGui.qApp.threads.append(thread) + # pylint: enable=no-member self.__threads.append(thread) self.__threads[i].start() self.__threads[i].workComplete.connect(self.runCallback, QtCore.Qt.BlockingQueuedConnection) - def queue(self, callable, callback, comment, *args): - """queue(callable work, callable callback, str comment, * args) - queue up a callable to be run from within a separate thread of execution - """ + def queue(self, callable_to_queue, callback, comment, *args): + """Queues up a callable to be run from within a separate thread of execution.""" self._q_mutex.lock() if not self.__started: self.start() if len(self._q_queue) <= self.__max_queue: - self._q_queue.append((callable,callback,comment,args)) + self._q_queue.append((callable_to_queue, callback, comment, args)) else: - logger.warning("Queue length exceeds %s" % self.__max_queue) + logger.warning("Queue length exceeds %s", self.__max_queue) self._q_mutex.unlock() self._q_empty.wakeAll() - def local(self, callable, callback, comment, *args): - work = (callable, callback, comment, args) + def local(self, callable_to_queue, callback, comment, *args): + """Executes a callable then immediately executes a callback, if given.""" + work = (callable_to_queue, callback, comment, args) if work[3]: result = work[0](*work[3]) else: @@ -120,19 +120,17 @@ def local(self, callable, callback, comment, *args): if work[1]: self.runCallback(work, result) + # pylint: disable=no-self-use def runCallback(self, work, result): - """runCallback(tuple work, object result) - runs the callback function - """ + """Runs the callback function.""" if work[1]: work[1](work, result) class WorkerThread(QtCore.QThread): - """WorkerThread + """A thread for parsing job log files. - A thread for parsing job log files. The log file is parsed - using SpiCue.cueprofile and emits a "parsingComplete" signal - when complete. + The log file is parsed using SpiCue.cueprofile and emits a "parsingComplete" signal + when complete. """ workComplete = QtCore.Signal(object, object) @@ -141,23 +139,28 @@ def __init__(self, name, parent): QtCore.QThread.__init__(self, parent) self.__parent = parent self.__name = name + self.__running = False + # pylint: disable=protected-access def run(self): self.__running = True while self.__running: work = None self.__parent._q_mutex.lock() + # pylint: disable=bare-except try: work = self.__parent._q_queue.pop(0) except: self.__parent._q_empty.wait(self.__parent._q_mutex) + # pylint: enable=bare-except self.__parent._q_mutex.unlock() if not work: continue + # pylint: disable=broad-except try: if work[3]: result = work[0](*work[3]) @@ -167,10 +170,12 @@ def run(self): self.workComplete.emit(work, result) del result except Exception as e: - logger.info("Error processing work:' %s ', %s" % (work[2], e)) - logger.info("Done:' %s '" % work[2]) + logger.info("Error processing work:' %s ', %s" , work[2], e) + # pylint: enable=broad-except + logger.info("Done:' %s '", work[2]) logger.debug("Thread Stopping") def stop(self): + """Stops the worker thread.""" self.__running = False self.__parent._q_empty.wakeAll() diff --git a/cuegui/cuegui/UnbookDialog.py b/cuegui/cuegui/UnbookDialog.py index 722ff3ab0..e36d3e6ac 100644 --- a/cuegui/cuegui/UnbookDialog.py +++ b/cuegui/cuegui/UnbookDialog.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Dialog for unbooking frames.""" + + from __future__ import absolute_import from __future__ import print_function from __future__ import division @@ -31,17 +34,22 @@ class UnbookDialog(cuegui.AbstractDialog.AbstractDialog): + """Dialog for unbooking frames.""" + def __init__(self, jobs, parent=None): cuegui.AbstractDialog.AbstractDialog.__init__(self, parent) layout = QtWidgets.QVBoxLayout(self) self.setWindowTitle("Unbook matching frames") - __descriptionLabel = QtWidgets.QLabel("Unbook and optionally kill the matching frames from the following jobs:", self) + __descriptionLabel = QtWidgets.QLabel( + "Unbook and optionally kill the matching frames from the following jobs:", self) self.__show = opencue.api.findShow(jobs[0].data.name.split("-")[0]) - self.__jobs = [job.data.name for job in jobs if job.data.name.startswith(self.__show.data.name)] - self.__subscriptions = [sub.data.name.split(".")[1] for sub in self.__show.getSubscriptions()] + self.__jobs = [ + job.data.name for job in jobs if job.data.name.startswith(self.__show.data.name)] + self.__subscriptions = [ + sub.data.name.split(".")[1] for sub in self.__show.getSubscriptions()] # Show list of jobs selected self.__jobList = QtWidgets.QTextEdit(self) @@ -85,11 +93,12 @@ def __init__(self, jobs, parent=None): self.__memoryRangeBox = self.__createRangeBox(layout, "Memory requirement", "Mb", 32768) # checkbox and LineEdit for amount or range of runtime - self.__runtimeRangeBox = self.__createRangeBox(layout, "Runtime requirement", "Minutes", 10000) + self.__runtimeRangeBox = self.__createRangeBox( + layout, "Runtime requirement", "Minutes", 10000) layout.addWidget(self.__buttons) - def __createRangeBox(self, layout, name, units, max): + def __createRangeBox(self, layout, name, units, max_frame): __group = QtWidgets.QGroupBox(name) __group.setCheckable(True) __group.setChecked(False) @@ -106,7 +115,7 @@ def __createRangeBox(self, layout, name, units, max): __layout.addWidget(__range, 1, 2) __min = QtWidgets.QSpinBox(self) - __min.setRange(0, max) + __min.setRange(0, max_frame) __layout.addWidget(__min, 0, 0) __minLabel = QtWidgets.QLabel(units) @@ -116,7 +125,7 @@ def __createRangeBox(self, layout, name, units, max): __layout.addWidget(__toLabel, 0, 2, QtCore.Qt.AlignHCenter) __max = QtWidgets.QSpinBox(self) - __max.setRange(0, max) + __max.setRange(0, max_frame) __layout.addWidget(__max, 0, 3) __maxLabel = QtWidgets.QLabel(units) @@ -136,7 +145,8 @@ def __createRangeBox(self, layout, name, units, max): return RangeBox(__group, __moreThan, __lessThan, __range, __min, __max) - def handleIntCriterion(self, mixed, convert=None): + @staticmethod + def handleIntCriterion(mixed, convert=None): """handleIntCriterion returns the proper subclass of IntSearchCriterion based on input from the user. There are a few formats which are accepted. @@ -162,16 +172,16 @@ def _convert(val): result = opencue.api.criterion_pb2.LessThanIntegerSearchCriterion( _convert(mixed[2:])) elif mixed.find("-") > -1: - min,max = mixed.split("-", 1) + min_frame, max_frame = mixed.split("-", 1) result = opencue.api.criterion_pb2.InRangeIntegerSearchCriterion( - _convert(min), _convert(max)) + _convert(min_frame), _convert(max_frame)) else: try: result = opencue.api.criterion_pb2.GreaterThanIntegerSearchCriterion( _convert(mixed)) except ValueError: raise Exception("invalid int search input value: " + str(mixed)) - elif issubclass(mixed.__class__, opencue.api.criterion_pb2.IntegerSearchCriterion): + elif issubclass(mixed.__class__, opencue.api.criterion_pb2.EqualsIntegerSearchCriterion): result = mixed elif not mixed: return [] @@ -190,23 +200,20 @@ def accept(self): procSearch.allocs = [str(checkedBox.text()) for checkedBox in self.__matrix.checkedBoxes()] memoryRange = self.__memoryRangeBox.result() if memoryRange: - procSearch.memoryRange = self.handleIntCriterion(memoryRange, lambda mb:(mb*1024)) + procSearch.memoryRange = self.handleIntCriterion(memoryRange, lambda mb: (mb*1024)) runtimeRange = self.__runtimeRangeBox.result() if runtimeRange: - procSearch.durationRange = self.handleIntCriterion(runtimeRange, lambda min:(min*60)) + procSearch.durationRange = self.handleIntCriterion( + runtimeRange, lambda rangeMin: (rangeMin*60)) if self.__redirect.isChecked(): # Select the show to redirect to title = "Select show" body = "Redirect to what show?" - shows = dict([(show.data.name, show) for show in opencue.api.getActiveShows()]) + shows = {show.data.name: show for show in opencue.api.getActiveShows()} items = [self.__jobs[0].split("-")[0]] + sorted(shows.keys()) - (show, choice) = QtWidgets.QInputDialog.getItem(self, - title, - body, - items, - 0, - False) + (show, choice) = QtWidgets.QInputDialog.getItem( + self, title, body, items, 0, False) if not choice: return show = shows[str(show)] @@ -215,23 +222,19 @@ def accept(self): title = "Select Redirection Type" body = "Redirect to a job or a group?" items = ["Job", "Group"] - (redirectTo, choice) = QtWidgets.QInputDialog.getItem(self, - title, - body, - items, - 0, - False) + (redirectTo, choice) = QtWidgets.QInputDialog.getItem( + self, title, body, items, 0, False) if not choice: return job = group = None if redirectTo == "Job": - jobs = dict([(job.data.name, job) for job in opencue.api.getJobs( - show=[show.data.name])]) - dialog = SelectItemsWithSearchDialog(self, - "Redirect to which job?", - list(jobs.keys()), - QtWidgets.QAbstractItemView.SingleSelection) + jobs = {job.data.name: job for job in opencue.api.getJobs(show=[show.data.name])} + dialog = SelectItemsWithSearchDialog( + self, + "Redirect to which job?", + list(jobs.keys()), + QtWidgets.QAbstractItemView.SingleSelection) dialog.exec_() selected = dialog.selected() if selected: @@ -242,13 +245,9 @@ def accept(self): elif redirectTo == "Group": title = "Select Redirection Group" body = "Redirect to which group?" - groups = dict([(group.data.name, group) for group in show.getGroups()]) - (group, choice) = QtWidgets.QInputDialog.getItem(self, - title, - body, - sorted(groups.keys()), - 0, - False) + groups = {group.data.name: group for group in show.getGroups()} + (group, choice) = QtWidgets.QInputDialog.getItem( + self, title, body, sorted(groups.keys()), 0, False) if not choice: return group = groups[str(group)] @@ -265,7 +264,7 @@ def accept(self): elif group: proc.redirectToGroup(group, kill) amount += 1 - except Exception: + except opencue.exception.CueException: pass self.__informationBox("Redirected procs", "Number of redirected procs: %d" % amount) @@ -278,7 +277,14 @@ def accept(self): if dialog.result(): self.close() else: - amount = opencue.api.unbookProcs(procSearch, False) + procs = opencue.api.getProcs(procSearch) + amount = 0 + for proc in procs: + try: + proc.unbook() + amount += 1 + except opencue.exception.CueException: + pass self.__informationBox("Unbooked frames", "Number of frames unbooked: %d" % amount) self.close() @@ -291,6 +297,8 @@ def __informationBox(self, title, message): class SelectItemsWithSearchDialog(cuegui.AbstractDialog.AbstractDialog): + """Dialog for selecting items via search.""" + def __init__(self, parent, header, items, selectionMode=QtWidgets.QAbstractItemView.MultiSelection): cuegui.AbstractDialog.AbstractDialog.__init__(self, parent) @@ -310,14 +318,17 @@ def __init__(self, parent, header, items, self.layout().addWidget(self.__buttons) def selected(self): + """Gets whether the item is selected.""" if self.result(): return self.__widget.selected() return None class SelectItemsWithSearchWidget(QtWidgets.QWidget): - def __init__(self, parent, header, items, - selectionMode=QtWidgets.QAbstractItemView.MultiSelection): + """Widget for selecting items via search.""" + + def __init__( + self, parent, header, items, selectionMode=QtWidgets.QAbstractItemView.MultiSelection): QtWidgets.QWidget.__init__(self, parent) QtWidgets.QGridLayout(self) @@ -339,11 +350,14 @@ def __init__(self, parent, header, items, self.filterJobs(None) def filterJobs(self, text): + """Filter the list of jobs by text match.""" self.__list.clear() - items = [item for item in self.__items if not text or re.search(str(text), item, re.IGNORECASE)] + items = [ + item for item in self.__items if not text or re.search(str(text), item, re.IGNORECASE)] self.__list.addItems(items) def selected(self): + """Gets whether the item is selected.""" selected = [] for num in range(self.__list.count()): if self.__list.item(num).isSelected(): @@ -354,15 +368,16 @@ def selected(self): class RangeBox(object): """Stores the parts the make up the range box and provides a way to query for the result""" - def __init__(self, group, moreThan, lessThan, range, min, max): + def __init__(self, group, moreThan, lessThan, rangeButton, minBox, maxBox): self.__group = group self.__moreThan = moreThan self.__lessThan = lessThan - self.__range = range - self.__min = min - self.__max = max + self.__range = rangeButton + self.__min = minBox + self.__max = maxBox def result(self): + """Gets the formatted string result.""" if self.__group.isChecked(): if self.__moreThan.isChecked(): return "gt%d" % self.__min.value() @@ -374,6 +389,8 @@ def result(self): class KillConfirmationDialog(QtWidgets.QDialog): + """Dialog for confirming frames should be killed.""" + def __init__(self, procSearch, parent=None): QtWidgets.QDialog.__init__(self, parent) layout = QtWidgets.QVBoxLayout(self) @@ -394,7 +411,9 @@ def __init__(self, procSearch, parent=None): # Show list of jobs selected self.__jobList = QtWidgets.QTextEdit(self) - self.__jobList.setText("\n".join(["%s %s" % (proc.data.jobName, proc.data.frameName) for proc in self.__procs])) + self.__jobList.setText( + "\n".join( + ["%s %s" % (proc.data.jobName, proc.data.frameName) for proc in self.__procs])) self.__jobList.setReadOnly(True) self.__jobList.setSizePolicy( QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Maximum)) @@ -412,10 +431,11 @@ def __init__(self, procSearch, parent=None): self.__buttons.rejected.connect(self.reject) def accept(self): + """Kills the procs.""" for proc in self.__procs: try: proc.kill() - except Exception: + except opencue.exception.CueException: pass if self.__amount == 1: diff --git a/cuegui/cuegui/Utils.py b/cuegui/cuegui/Utils.py index f41076f09..98ef4a6fb 100644 --- a/cuegui/cuegui/Utils.py +++ b/cuegui/cuegui/Utils.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -Utility functions. -""" +"""Utility functions.""" from __future__ import print_function @@ -32,6 +30,7 @@ import sys import time import traceback +import webbrowser from PySide2 import QtCore from PySide2 import QtGui @@ -53,8 +52,10 @@ __USERNAME = None -def questionBoxYesNo(parent, title, text, items = []): - """A simple yes/no alert box +# pylint: disable=dangerous-default-value +def questionBoxYesNo(parent, title, text, items=[]): + """A simple yes/no alert box. + @type parent: QObject @param parent: The parent for this object @type title: string @@ -68,33 +69,35 @@ def questionBoxYesNo(parent, title, text, items = []): def countObjectTypes(objects): + """Given a list of objects, returns a count of how many of each type there are.""" results = {"rootgroup": 0, "group": 0, "job": 0} - for object in objects: - if isJob(object): + for obj in objects: + if isJob(obj): results["job"] += 1 - elif isGroup(object): + elif isGroup(obj): results["group"] += 1 - elif isRootGroup(object): + elif isRootGroup(obj): results["rootgroup"] += 1 results["total"] = len(objects) return results def countJobTypes(objects): + """Given a list of jobs, returns a count of how many jobs have each status.""" results = {"paused": False, "unpaused": False, "hasDead": False, "autoEating": False, "notEating": False} - for object in objects: - if isJob(object): - if isinstance(object, opencue.wrappers.job.NestedJob): - object = object.asJob() - if object.data.is_paused: + for obj in objects: + if isJob(obj): + if isinstance(obj, opencue.wrappers.job.NestedJob): + obj = obj.asJob() + if obj.data.is_paused: results["paused"] = True else: results["unpaused"] = True - if object.data.job_stats.dead_frames: + if obj.data.job_stats.dead_frames: results["hasDead"] = True - if object.data.auto_eat: + if obj.data.auto_eat: results["autoEating"] = True else: results["notEating"] = True @@ -102,81 +105,83 @@ def countJobTypes(objects): def qvarToString(qv): - """converts a QVariant to a python string""" + """converts a QVariant to a python string.""" return str(qv) def qvarToFloat(qv): - """converts a Qvariant to a python float""" + """converts a Qvariant to a python float.""" return float(qv) -def isJob(object): +def isJob(obj): """Returns true of the object is a job, false if not @return: If the object is a job @rtype: bool""" - return object.__class__.__name__ in ["Job", "NestedJob"] + return obj.__class__.__name__ in ["Job", "NestedJob"] -def isLayer(object): +def isLayer(obj): """Returns true if the object is a layer, false if not @return: If the object is a layer @rtype: bool""" - return object.__class__.__name__ == "Layer" + return obj.__class__.__name__ == "Layer" -def isFrame(object): +def isFrame(obj): """Returns true if the object is frame, false if not @return: If the object is a frame @rtype: bool""" - return object.__class__.__name__ == "Frame" + return obj.__class__.__name__ == "Frame" -def isShow(object): +def isShow(obj): """Returns true if the object is a show, false if not @return: If the object is a show @rtype: bool""" - return object.__class__.__name__ == "Show" + return obj.__class__.__name__ == "Show" -def isRootGroup(object): +def isRootGroup(obj): """Returns true if the object is a root, false if not @return: If the object is a root group @rtype: bool""" - return isinstance(object, opencue.wrappers.group.NestedGroup) and not object.hasParent() + return isinstance(obj, opencue.wrappers.group.NestedGroup) and not obj.hasParent() -def isGroup(object): +def isGroup(obj): """Returns true if the object is a group, false if not @return: If the object is a group @rtype: bool""" + # isinstance is needed here due to NestedGroup's inheritance + # pylint: disable=unidiomatic-typecheck return ( - type(object) == opencue.wrappers.group.Group or - (isinstance(object, opencue.wrappers.group.NestedGroup) and object.hasParent())) + type(obj) == opencue.wrappers.group.Group or + (isinstance(obj, opencue.wrappers.group.NestedGroup) and obj.hasParent())) -def isHost(object): +def isHost(obj): """Returns true of the object is a host, false if not @return: If the object is a host @rtype: bool""" - return object.__class__.__name__ in ["NestedHost", "Host"] + return obj.__class__.__name__ in ["NestedHost", "Host"] -def isProc(object): +def isProc(obj): """Returns true if the object is a proc, false if not @return: If the object is a proc @rtype: bool""" - return object.__class__.__name__ in ["Proc", "NestedProc"] + return obj.__class__.__name__ in ["Proc", "NestedProc"] -def isTask(object): +def isTask(obj): """Returns true if the object is a task, false if not @return: If the object is a task @rtype: bool""" - return object.__class__.__name__ == "Task" + return obj.__class__.__name__ == "Task" -__REGEX_ID = re.compile("[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}") +__REGEX_ID = re.compile(r"[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}") def isStringId(value): @@ -188,6 +193,7 @@ def isStringId(value): return __REGEX_ID.match(value) +# pylint: disable=broad-except def getCuewho(show): """Returns the username that is cuewho for the given show @param show: Show name @@ -198,10 +204,11 @@ def getCuewho(show): file = open("/shots/%s/home/cue/cuewho.who" % show, "r") return file.read() except Exception as e: - logger.warning("Failed to update cuewho: %s\n%s" % (show, e)) + logger.warning("Failed to update cuewho: %s\n%s", show, e) return "Unknown" +# pylint: disable=global-statement def getUsername(): """Returns the username that this process is running under""" global __USERNAME @@ -210,19 +217,19 @@ def getUsername(): return __USERNAME +# pylint: disable=broad-except def getExtension(username): + """Gets a user's phone extension.""" try: - # TODO: Replace this with a direct call to the phone util that the - # phone widget uses once code is stable results = subprocess.check_output(['phone', username]) for line in results.splitlines(): - if line.find('Extension') != -1 and len(line.split()) == 2: + if 'Extension' in line and len(line.split()) == 2: return line.split()[-1] return "Unknown" except Exception as e: - logger.warning("Failed to update extension: %s\n%s" % (username, e)) + logger.warning("Failed to update extension: %s\n%s", username, e) return "" @@ -252,17 +259,18 @@ def findJob(job): return None if isStringId(job): return opencue.api.getJob(job) - if not re.search("^([a-z0-9\_]+)\-([a-z0-9\.\_]+)\-", job, re.IGNORECASE): + if not re.search(r"^([a-z0-9\_]+)\-([a-z0-9\.\_]+)\-", job, re.IGNORECASE): return None try: return opencue.api.findJob(job) except Exception as e: - logger.warning("Error loading job: %s" % job) - logger.debug("Error loading job: %s\n%s" % (job, e)) + logger.warning("Error loading job: %s", job) + logger.debug("Error loading job: %s\n%s", job, e) return None def shellOut(cmd): + """Runs a command in an external shell.""" os.system("%s &" % cmd) @@ -308,6 +316,7 @@ def exceptionOutput(e): def handleExceptions(function): + """Custom exception handler.""" def new(*args): try: return function(*args) @@ -320,9 +329,9 @@ def __splitTime(sec): """Takes an amount of seconds and returns a tuple for hours, minutes and seconds. @rtype: tuple(int, int, int) @return: A tuple that contains hours, minutes and seconds""" - min, sec = divmod(sec, 60) - hour, min = divmod(min, 60) - return (hour, min, sec) + minutes, sec = divmod(sec, 60) + hour, minutes = divmod(minutes, 60) + return (hour, minutes, sec) def secondsToHHMMSS(sec): @@ -370,14 +379,14 @@ def dateToMMDDHHMM(sec): return time.strftime("%m/%d %H:%M", time.localtime(sec)) -def memoryToString(kmem, unit = None): +def memoryToString(kmem, unit=None): + """Returns an amount of memory in a human-friendly string.""" k = 1024 if unit == "K" or not unit and kmem < k: return "%dK" % kmem - if unit == "M" or not unit and kmem < pow(k,2): + if unit == "M" or not unit and kmem < pow(k, 2): return "%dM" % (kmem // k) - if unit == "G" or not unit and kmem < pow(k,3): - return "%.01fG" % (float(kmem) / pow(k,2)) + return "%.01fG" % (float(kmem) / pow(k, 2)) def getResourceConfig(path=None): @@ -391,7 +400,6 @@ def getResourceConfig(path=None): @return: The entries in the given yaml file @rtype: dict """ - config = {} if not path: path = '{}/cue_resources.yaml'.format(cuegui.Constants.DEFAULT_INI_PATH) @@ -399,8 +407,7 @@ def getResourceConfig(path=None): with open(path, 'r') as fileObject: config = yaml.load(fileObject, Loader=yaml.SafeLoader) except (IOError, ScannerError) as e: - print ('WARNING: Could not read config file %s: %s' - % (path, e)) + print('WARNING: Could not read config file %s: %s' % (path, e)) return config @@ -409,12 +416,15 @@ def getResourceConfig(path=None): ################################################################################ def getFrameLogFile(job, frame): + """Get the log file associated with a frame.""" return os.path.join(job.data.log_dir, "%s.%s.rqlog" % (job.data.name, frame.data.name)) def getFrameLLU(job, frame): + """Get a frame's last log update time.""" __now = time.time() if __now - getattr(frame, "getFrameLLUTime", 0) >= 5: + # pylint: disable=bare-except try: frame.getFrameLLU = __now - os.stat(getFrameLogFile(job, frame)).st_mtime except: @@ -424,6 +434,7 @@ def getFrameLLU(job, frame): def getFrameLastLine(job, frame): + """Get the last line of a frame log.""" __now = time.time() if __now - getattr(frame, "getFrameLastLineTime", 0) >= 5: frame.getFrameLastLine = getLastLine(getFrameLogFile(job, frame)) @@ -432,11 +443,12 @@ def getFrameLastLine(job, frame): def getLastLine(path): - """Reads the last line from the file""" - ansiEscape = '(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]' + """Reads the last line from the file.""" + + ansiEscape = r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]' try: - fp=open(path, 'r') + fp = open(path, 'r') fp.seek(0, 2) backseek = min(4096, fp.tell()) @@ -454,47 +466,55 @@ def getLastLine(path): def popupTail(file, facility=None): + """Opens an xterm window showing the tail of the given file.""" if file and not popupWeb(file, facility): - JOB_LOG_CMD = "/usr/bin/xterm -sb -sl 4096 -n RQLOG -geometry 200x50+0+0 -title %s -e '/usr/bin/tail -n+0 -f %s'" % (os.path.basename(file), file) - shellOut(JOB_LOG_CMD) + job_log_cmd = ( + "/usr/bin/xterm -sb -sl 4096 -n RQLOG -geometry 200x50+0+0 -title %s " + "-e '/usr/bin/tail -n+0 -f %s'" % (os.path.basename(file), file)) + shellOut(job_log_cmd) def popupView(file, facility=None): + """Opens the given file in your editor.""" if file and not popupWeb(file, facility): editor_from_env = os.getenv('EDITOR') + # pylint: disable=no-member if editor_from_env: job_log_cmd = editor_from_env.split() elif QtGui.qApp.settings.contains('LogEditor'): job_log_cmd = QtGui.qApp.settings.value("LogEditor") else: job_log_cmd = cuegui.Constants.DEFAULT_EDITOR.split() + # pylint: enable=no-member job_log_cmd.append(str(file)) checkShellOut(job_log_cmd) def openURL(url): - import webbrowser + """Opens a URL.""" webbrowser.open_new(url) return True def popupWeb(file, facility=None): + """Opens a web browser.""" client = os.getenv('FACILITY', 'unknown') if client in ['yvr'] or (facility == 'yvr' and client in ['lax']): - import webbrowser webbrowser.open_new('' + file) return True return False -def popupFrameTail(job, frame, logNumber = 0): +def popupFrameTail(job, frame, logNumber=0): + """Opens a tail of a frame log.""" path = getFrameLogFile(job, frame) if logNumber: path += ".%s" % logNumber popupTail(path, job.data.facility) -def popupFrameView(job, frame, logNumber = 0): +def popupFrameView(job, frame, logNumber=0): + """Opens a frame.""" path = getFrameLogFile(job, frame) if logNumber: path += ".%s" % logNumber @@ -502,6 +522,7 @@ def popupFrameView(job, frame, logNumber = 0): def popupFrameXdiff(job, frame1, frame2, frame3 = None): + """Opens a frame xdiff.""" for command in ['/usr/bin/xxdiff', '/usr/local/bin/xdiff']: if os.path.isfile(command): @@ -511,50 +532,14 @@ def popupFrameXdiff(job, frame1, frame2, frame3 = None): shellOut(command) -def getOutputFromLayers(job, layers): - """Returns the output paths from the frame logs - @type job: Job - @param job: A job object - @type layers: list - @param layers: A list of at least one later - @rtype: list - @return: The path to the proxy SVI or the primary output.""" - paths = [] - for layer in layers: - svi_found = False - outputs = layer.getOutputPaths() - if outputs: - for path in outputs: - if path.find("_svi") != -1: - paths.append(path) - svi_found = True - break - if not svi_found: - paths.append(outputs[0]) - return paths - - -def getOutputFromFrame(job, layer, frame): - """Returns the output paths from a single frame - @type job: Job - @param job: A job object - @type frame: Frame - @param frame: This frame's log is checked for known output paths - @rtype: list - @return: A list of output paths""" - try: - main_output = layer.getOutputPaths()[0] - main_output = main_output.replace("#", "%04d" % frame.data.number) - return [main_output] - except IndexError: - return [] - - ################################################################################ # Drag and drop functions ################################################################################ def startDrag(dragSource, dropActions, objects): + """Event handler for when a drag starts.""" + del dropActions + mimeData = QtCore.QMimeData() mimeData.setText("\n".join(["%s" % job.data.name for job in objects])) @@ -587,39 +572,45 @@ def startDrag(dragSource, dropActions, objects): drag.exec_(QtCore.Qt.MoveAction) -def dragEnterEvent(event, format = "application/x-job-names"): - if event.mimeData().hasFormat(format): +def dragEnterEvent(event, mime_format="application/x-job-names"): + """Event handler for when a drag enters an area.""" + if event.mimeData().hasFormat(mime_format): event.accept() else: event.ignore() -def dragMoveEvent(event, format = "application/x-job-names"): - if event.mimeData().hasFormat(format): +def dragMoveEvent(event, mime_format="application/x-job-names"): + """Event handler for when a drag is moved.""" + if event.mimeData().hasFormat(mime_format): event.setDropAction(QtCore.Qt.CopyAction) event.accept() else: event.ignore() -def dropEvent(event, format = "application/x-job-names"): - if event.mimeData().hasFormat(format): - item = event.mimeData().data(format) +# pylint: disable=inconsistent-return-statements +def dropEvent(event, mime_format="application/x-job-names"): + """Event handler for when a drop occurs.""" + if event.mimeData().hasFormat(mime_format): + item = event.mimeData().data(mime_format) stream = QtCore.QDataStream(item, QtCore.QIODevice.ReadOnly) names = stream.readQString() event.accept() return [name for name in str(names).split(":") if name] -def mimeDataAdd(mimeData, format, objects): +def mimeDataAdd(mimeData, mimeFormat, objects): + """Sets mime data.""" data = QtCore.QByteArray() stream = QtCore.QDataStream(data, QtCore.QIODevice.WriteOnly) text = ":".join(objects) stream.writeQString(text) - mimeData.setData(format, data) + mimeData.setData(mimeFormat, data) def showErrorMessageBox(text, title="ERROR!", detailedText=None): + """Displays an error dialog.""" messageBox = QtWidgets.QMessageBox() messageBox.setIcon(QtWidgets.QMessageBox.Critical) messageBox.setText(text) @@ -629,7 +620,8 @@ def showErrorMessageBox(text, title="ERROR!", detailedText=None): messageBox.setStandardButtons(QtWidgets.QMessageBox.Close) return messageBox.exec_() + def shutdownThread(thread): - """Shutdown a WorkerThread.""" + """Shuts down a WorkerThread.""" thread.stop() return thread.wait(1500) diff --git a/cuegui/cuegui/__init__.py b/cuegui/cuegui/__init__.py index 196c475ba..d9c80f13d 100644 --- a/cuegui/cuegui/__init__.py +++ b/cuegui/cuegui/__init__.py @@ -10,4 +10,4 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file +# limitations under the License. diff --git a/cuegui/cuegui/__main__.py b/cuegui/cuegui/__main__.py index fcba4cb3f..770f951e2 100755 --- a/cuegui/cuegui/__main__.py +++ b/cuegui/cuegui/__main__.py @@ -15,6 +15,9 @@ # limitations under the License. +"""Entrypoint for the CueGUI application.""" + + from __future__ import absolute_import from __future__ import print_function from __future__ import division @@ -25,9 +28,9 @@ def main(): - cuegui.Main.cuecommander(sys.argv) + """Entrypoint for the CueGUI application.""" + cuegui.Main.cuecommander(sys.argv) if __name__ == '__main__': - main() - + main() diff --git a/cuegui/cuegui/eta.py b/cuegui/cuegui/eta.py index 0975f8b15..3a91a77e8 100644 --- a/cuegui/cuegui/eta.py +++ b/cuegui/cuegui/eta.py @@ -15,6 +15,9 @@ # limitations under the License. +"""Functions for estimating time remaining on a frame.""" + + from __future__ import division from __future__ import print_function from __future__ import absolute_import @@ -31,29 +34,29 @@ import opencue -############################################### -# Parses log files and job data for ETC info -############################################### - class FrameEtaGenerator(object): + """Parses log files and job data for ETC info.""" + def __init__(self): self.buildTimeCache = {} self.startTimeCache = {} - self.frame_results={'time_left':0,'total_completion':0} - self.time_left=0 - self.total_completion=0 - self.scene_build_time=0 - self.scene_build_seconds=0 - self.percents=[] - self.simTimes=[] - self.log='' - self.log_lines=0 - + self.frame_results = {'time_left': 0, 'total_completion': 0} + self.time_left = 0 + self.total_completion = 0 + self.scene_build_time = 0 + self.scene_build_seconds = 0 + self.percents = [] + self.simTimes = [] + self.log = '' + self.log_lines = 0 + + # pylint: disable=bare-except def GetFrameEta(self, job, frame): + """Gets ETA for the given frame.""" self.log = opencue.util.logPath(job, frame) if os.path.isfile(self.log): - self.log_lines=len(open(self.log).readlines()) - buildTime = self.GetFrameBuildTime(frame) + self.log_lines = len(open(self.log).readlines()) + self.GetFrameBuildTime(frame) try: layer = opencue.api.findLayer(job.data.name, frame.data.layer_name) if 'tango' in layer.data.services: @@ -64,118 +67,128 @@ def GetFrameEta(self, job, frame): self.Arnold(frame) except: pass - self.frame_results['time_left']=self.time_left - self.frame_results['total_completion']=self.total_completion - self.frame_results['scene_build_time']=self.scene_build_time - self.frame_results['scene_build_seconds']=self.scene_build_seconds + self.frame_results['time_left'] = self.time_left + self.frame_results['total_completion'] = self.total_completion + self.frame_results['scene_build_time'] = self.scene_build_time + self.frame_results['scene_build_seconds'] = self.scene_build_seconds try: - self.frame_results['percent_complete']=self.percents[0][0] + self.frame_results['percent_complete'] = self.percents[0][0] except: - self.frame_results['percent_complete']=0 + self.frame_results['percent_complete'] = 0 linecache.clearcache() return self.frame_results - def Tango(self, frame): - simTimes=[] + """Calculates ETA for a Tango frame.""" + del frame + + simTimes = [] if os.path.isfile(self.log): - line='' - line_counter=0 + line = '' + line_counter = 0 while 'Done with Frame' not in line: - line=linecache.getline(self.log,self.log_lines-line_counter) - line_counter+=1 + line = linecache.getline(self.log, self.log_lines-line_counter) + line_counter += 1 if line_counter == self.log_lines: break frameTime = line.split("=")[1].split(",")[0].strip(" ") # Extract frame time if float(frameTime) > 0: - simTimes.append(float(frameTime)) + simTimes.append(float(frameTime)) lastFrame = line.split('#')[1].split(".")[0].strip(" ") # Extract the last frame - line_counter=100 - line='' + line_counter = 100 + line = '' while 'Loading XML file:' not in line: - line=linecache.getline(self.log,line_counter) - line_counter+=1 + line = linecache.getline(self.log, line_counter) + line_counter += 1 if line_counter == self.log_lines: break xml_loc = line.split(":")[1].strip(" ").rstrip("\n") # Extract the XML from the line start_frame, end_frame = self.GetSimFrameRange(xml_loc) # Get the start and end points - simTimes=sorted(simTimes,reverse=True) + simTimes = sorted(simTimes, reverse=True) framesLeft = int(end_frame) - int(lastFrame) self.time_left = float(simTimes[0]) * float(framesLeft) - self.percents.append(((float(start_frame)/int(end_frame))*100,self.time_left)) - self.percents.append((0,0)) - self.percents=sorted(self.percents, reverse=True) + self.percents.append(((float(start_frame) / int(end_frame)) * 100, self.time_left)) + self.percents.append((0, 0)) + self.percents = sorted(self.percents, reverse=True) if len(self.percents) > 1: - self.total_completion = (self.percents[0][1] - self.percents[-1][1]) * (100 / (self.percents[0][0] - self.percents[-1][0])) - + self.total_completion = ( + (self.percents[0][1] - self.percents[-1][1]) * + (100 / (self.percents[0][0] - self.percents[-1][0]))) + def Svea(self, frame): + """Calculates ETA for a Svea frame.""" + del frame if os.path.isfile(self.log): - line='' + line = '' for line in reversed(open(self.log).readlines()): - #checks log directory for a percentage complete in reverse to limit time in log + # Checks log directory for a percentage complete in reverse to limit time in log. if 'Running generator batch' in line: + # pylint: disable=bare-except try: - time_on_log=self.GetSeconds(line) - line=line.split(' ') - current=float(line[16]) - total=float(line[18].split('\n')[0]) - percent=float(current / total) * 100 - self.percents.append((percent,time_on_log)) + time_on_log = self.GetSeconds(line) + line = line.split(' ') + current = float(line[16]) + total = float(line[18].split('\n')[0]) + percent = float(current / total) * 100 + self.percents.append((percent, time_on_log)) if len(self.percents) > 1: break except: pass if len(self.percents) > 1: self.percents = sorted(self.percents, reverse=True) - self.total_completion = (self.percents[0][1] - self.percents[-1][1]) * (100 / (self.percents[0][0] - self.percents[-1][0])) - self.time_passed = self.percents[0][1] + self.total_completion = ( + (self.percents[0][1] - self.percents[-1][1]) * + (100 / (self.percents[0][0] - self.percents[-1][0]))) self.time_left = self.total_completion * ((100-self.percents[0][0]) / 100) else: - self.percents.append((0,0)) + self.percents.append((0, 0)) def Arnold(self, frame): + """Calculates ETA for an Arnold frame.""" if os.path.isfile(self.log): buildTime = self.GetFrameBuildTime(frame) - self.scene_build_seconds=buildTime['scene_build_seconds'] - self.scene_build_time=buildTime['scene_build_time'] - if self.scene_build_seconds!=0: - #doesn't look for percentages if it can't find a scenebuild + self.scene_build_seconds = buildTime['scene_build_seconds'] + self.scene_build_time = buildTime['scene_build_time'] + if self.scene_build_seconds != 0: + # Doesn't look for percentages if it can't find a scenebuild. line = self.GetFrameStartTime(frame) - if line !='': - #doesn't look for anything else if it can't find a first % + if line != '': + # Doesn't look for anything else if it can't find a first %. self.GetPercent(line) - line_counter=0 - line='' + line_counter = 0 + line = '' while '% done' not in line: - line=linecache.getline(self.log,self.log_lines-line_counter) - line_counter+=1 + line = linecache.getline(self.log,self.log_lines-line_counter) + line_counter += 1 if line_counter == self.log_lines: break if line_counter != self.log_lines: self.GetPercent(line) if len(self.percents) > 1: - self.percents=sorted(self.percents, reverse=True) - if len(self.percents)==1 and self.percents[0][0] % 5 == 0: - self.total_completion=(self.percents[0][1])*(20) + self.percents = sorted(self.percents, reverse=True) + if len(self.percents) == 1 and self.percents[0][0] % 5 == 0: + self.total_completion = self.percents[0][1] * 20 else: - if self.percents[0][0]==self.percents[-1][0]: + if self.percents[0][0] == self.percents[-1][0]: self.percents[-1]=(self.percents[0][0]-5,self.scene_build_seconds) - self.total_completion = (self.percents[0][1] - self.percents[-1][1]) * (100 / (self.percents[0][0] - self.percents[-1][0])) - time_passed=self.percents[0][1] + self.total_completion = ( + (self.percents[0][1] - self.percents[-1][1]) * + (100 / (self.percents[0][0] - self.percents[-1][0]))) self.time_left = self.total_completion * ((100 - self.percents[0][0]) / 100) else: - self.percents.append((0,0)) - + self.percents.append((0, 0)) def GetFrameStartTime(self, frame): + """Gets a frame start time.""" key = (frame, frame.data.start_time) if key in self.startTimeCache: return self.startTimeCache[key] - # read the logFile here for time - result='' + # Read the logFile here for time. + result = '' for line in open(self.log): - if '% done'in line: - result=line + if '% done' in line: + result = line break if not result: return result @@ -183,46 +196,53 @@ def GetFrameStartTime(self, frame): return result def GetFrameBuildTime(self, frame): + """Gets a frame build time.""" key = (frame, frame.data.start_time) if key in self.buildTimeCache: return self.buildTimeCache[key] - # read the logFile here for time - result='' + # Read the logFile here for time. + result_line = None for line in open(self.log): - if 'Building scene done'in line: - result=line + if 'Building scene done' in line: + result_line = line break - if result != '': - result={'scene_build_seconds':self.GetSeconds(line), 'scene_build_time':line.split(' ')[3]} + if result_line is not None: + result = { + 'scene_build_seconds': self.GetSeconds(result_line), + 'scene_build_time': result_line.split(' ')[3]} else: - result={'scene_build_seconds':0, 'scene_build_time':0} + result = {'scene_build_seconds': 0, 'scene_build_time': 0} return result if not result: return result self.buildTimeCache[key] = result return result - def GetPercent(self,line): + def GetPercent(self, line): + """Gets a percentage from a given log line.""" + # pylint: disable=bare-except try: - percent_location=line.find('%') - percent=float(line[percent_location-3]+line[percent_location-2]+line[percent_location-1]) - time_on_log=self.GetSeconds(line) - self.percents.append((percent,time_on_log)) + percent_location = line.find('%') + percent = float( + line[percent_location-3] + line[percent_location-2] + line[percent_location-1]) + time_on_log = self.GetSeconds(line) + self.percents.append((percent, time_on_log)) except: pass - - def GetSeconds(self,line): - time_str=re.search('([0-9]+):([0-9]{2}):([0-9]{2})', line) - hour=int(time_str.group(1)) - minute=int(time_str.group(2)) - second=int(time_str.group(3)) - seconds=(hour*3600) + (minute*60) + second + + @staticmethod + def GetSeconds(line): + """Gets a number of seconds from a timestamp found in a log line.""" + time_str = re.search('([0-9]+):([0-9]{2}):([0-9]{2})', line) + hour = int(time_str.group(1)) + minute = int(time_str.group(2)) + second = int(time_str.group(3)) + seconds = (hour * 3600) + (minute * 60) + second return seconds - ############################################################################### - # Reads the SimRender XML to get the frame range - ############################################################################### - def GetSimFrameRange(self, xml_loc): + @staticmethod + def GetSimFrameRange(xml_loc): + """Reads the SimRender XML to get the frame range.""" try: name = xml.dom.minidom.parse(xml_loc) except IOError: @@ -233,25 +253,31 @@ def GetSimFrameRange(self, xml_loc): end_frame = global_tag.getElementsByTagName('end')[0].childNodes[0].nodeValue return start_frame, end_frame + def ETAString(job, frame): - eta=FrameEtaGenerator() - time_left=eta.GetFrameEta(job, frame)['time_left'] + """Calculates ETA and returns it as a formatted string.""" + eta = FrameEtaGenerator() + time_left = eta.GetFrameEta(job, frame)['time_left'] t = datetime.datetime.now() - now_epoch=time.mktime(t.timetuple()) - time_left=datetime.datetime.fromtimestamp(time_left+now_epoch).strftime('%m/%d %H:%M:%S') + now_epoch = time.mktime(t.timetuple()) + time_left = datetime.datetime.fromtimestamp(time_left+now_epoch).strftime('%m/%d %H:%M:%S') return time_left + def ETADateTime(job, frame): - eta=FrameEtaGenerator() - time_left=eta.GetFrameEta(job, frame)['time_left'] + """Calculates ETA and returns it as a datetime.""" + eta = FrameEtaGenerator() + time_left = eta.GetFrameEta(job, frame)['time_left'] t = datetime.datetime.now() - now_epoch=time.mktime(t.timetuple()) - time_left=datetime.datetime.fromtimestamp(time_left+now_epoch) + now_epoch = time.mktime(t.timetuple()) + time_left = datetime.datetime.fromtimestamp(time_left + now_epoch) return time_left + def ETASeconds(job, frame): - eta=FrameEtaGenerator() - time_left=eta.GetFrameEta(job, frame)['time_left'] + """Calculates ETA and returns it as a number of seconds.""" + eta = FrameEtaGenerator() + time_left = eta.GetFrameEta(job, frame)['time_left'] return time_left @@ -261,95 +287,19 @@ def __init__(self, func): self.func = func self.memoized = {} self.method_cache = {} - + def __call__(self, *args): - return self.cache_get(self.memoized, args, - lambda: self.func(*args)) - + return self.__cache_get(self.memoized, args, lambda: self.func(*args)) + def __get__(self, obj, objtype): - return self.cache_get(self.method_cache, obj, - lambda: self.__class__(functools.partial(self.func, obj))) - - def cache_get(self, cache, key, func): + return self.__cache_get( + self.method_cache, obj, lambda: self.__class__(functools.partial(self.func, obj))) + + @staticmethod + def __cache_get(cache, key, func): try: return cache[key] except KeyError: result = func() cache[key] = result return result -""" -class BackwardsReader: - #Read a file line by line, backwards - BLKSIZE = 4096 - - def readline(self): - while 1: - newline_pos = string.rfind(self.buf, "\n") - pos = self.file.tell() - if newline_pos != -1: - # Found a newline - line = self.buf[newline_pos+1:] - self.buf = self.buf[:newline_pos] - if pos != 0 or newline_pos != 0 or self.trailing_newline: - line += "\n" - return line - else: - if pos == 0: - # Start-of-file - return "" - else: - # Need to fill buffer - toread = min(self.BLKSIZE, pos) - self.file.seek(-toread, 1) - self.buf = self.file.read(toread) + self.buf - self.file.seek(-toread, 1) - if pos - toread == 0: - self.buf = "\n" + self.buf - - def __init__(self, file): - self.file = file - self.buf = "" - self.file.seek(-1, 2) - self.trailing_newline = 0 - lastchar = self.file.read(1) - if lastchar == "\n": - self.trailing_newline = 1 - self.file.seek(-1, 2) - - br = BackwardsReader(open(self.log)) - while 1: - line = br.readline() - if '% done'in line: - self.GetPercent(line) - break - if not line: - break - - for line in reversed(open(self.log).readlines()): - if '% done'in line: - self.GetPercent(line) - break - - file=PipeIt('tac %s' % self.log)[0] - for line in file:#os.popen('tac %s' % self.log): PipeIt('tac %s' % self.log)[0]: - if '% done'in line: - self.GetPercent(line) - break - - while '% done' not in line: - line=linecache.getline(self.log,self.log_lines-line_counter) - line_counter+=1 - if line_counter == self.log_lines: - break - if line_counter != self.log_lines: - self.GetPercent(line) - line_counter=0 - line='' - while '% done' not in line: - line=linecache.getline(self.log,line_counter) - line_counter+=1 - if line_counter == self.log_lines: - break - if line_counter != self.log_lines: - self.GetPercent(line) - """ diff --git a/cuegui/cuegui/plugins/AllocationsPlugin.py b/cuegui/cuegui/plugins/AllocationsPlugin.py index 896557a4f..b181a350b 100644 --- a/cuegui/cuegui/plugins/AllocationsPlugin.py +++ b/cuegui/cuegui/plugins/AllocationsPlugin.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Plugin for managing allocations.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -40,8 +43,10 @@ PLUGIN_REQUIRES = "CueCommander" PLUGIN_PROVIDES = "AllocationsDockWidget" + class AllocationsDockWidget(cuegui.AbstractDockWidget.AbstractDockWidget): - """This builds what is displayed on the dock widget""" + """Widget that lists allocations and allows management of them.""" + def __init__(self, parent): cuegui.AbstractDockWidget.AbstractDockWidget.__init__(self, parent, PLUGIN_NAME) @@ -55,11 +60,15 @@ def __init__(self, parent): self.__monitorAllocations.getColumnOrder, self.__monitorAllocations.setColumnOrder)]) + ################################################################################ # Allocations ################################################################################ + class MonitorAllocations(cuegui.AbstractTreeWidget.AbstractTreeWidget): + """Inner widget that builds and displays the allocation list.""" + def __init__(self, parent): self.startColumnsForType(cuegui.Constants.TYPE_ALLOC) self.addColumn("Name", 150, id=1, @@ -95,25 +104,28 @@ def __init__(self, parent): self.setDragEnabled(True) self.setDragDropMode(QtWidgets.QAbstractItemView.DragDrop) + # Signals are defined in code, so pylint thinks they don't exist. + # pylint: disable=no-member QtGui.qApp.facility_changed.connect(self._update) + # pylint: enable=no-member self.setUpdateInterval(60) - def _createItem(self, object): + def _createItem(self, rpcObject): """Creates and returns the proper item""" - return AllocationWidgetItem(object, self) + return AllocationWidgetItem(rpcObject, self) + # pylint: disable=no-self-use def _getUpdate(self): """Returns the proper data from the cuebot""" try: return opencue.api.getAllocations() - except Exception as e: + except opencue.exception.CueException as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) return [] def contextMenuEvent(self, e): """When right clicking on an item, this raises a context menu""" - pass def dragEnterEvent(self, event): cuegui.Utils.dragEnterEvent(event, "application/x-host-ids") @@ -135,7 +147,14 @@ def dropEvent(self, event): item.rpcObject.reparentHostIds(hostIds) self.updateSoon() + def tick(self): + # tick is unused in this widget + pass + + class AllocationWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): - def __init__(self, object, parent): + """Widget element representing a single allocation.""" + + def __init__(self, rpcObject, parent): cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( - self, cuegui.Constants.TYPE_ALLOC, object, parent) + self, cuegui.Constants.TYPE_ALLOC, rpcObject, parent) diff --git a/cuegui/cuegui/plugins/AttributesPlugin.py b/cuegui/cuegui/plugins/AttributesPlugin.py index c12a4a374..da4693d06 100644 --- a/cuegui/cuegui/plugins/AttributesPlugin.py +++ b/cuegui/cuegui/plugins/AttributesPlugin.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Plugin that lists attributes of the selected object.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -26,7 +29,8 @@ from PySide2 import QtWidgets import opencue -import opencue.compiled_proto +import opencue.compiled_proto.depend_pb2 +import opencue.wrappers.job import cuegui.AbstractDockWidget import cuegui.Logger @@ -42,15 +46,18 @@ class AttributesPlugin(cuegui.AbstractDockWidget.AbstractDockWidget): + """Plugin that lists attributes of the selected object.""" + def __init__(self, parent): cuegui.AbstractDockWidget.AbstractDockWidget.__init__( self, parent, PLUGIN_NAME, QtCore.Qt.RightDockWidgetArea) self.__attributes = Attributes(self) self.layout().addWidget(self.__attributes) + def getDependsForm(depends): - """This is a temporary method unil a new widget factory can be made - that creates uber hawt dependency widgets""" + """This is a temporary method until a new widget factory can be made + that creates uber hawt dependency widgets.""" if not depends: return None @@ -72,6 +79,7 @@ def getDependsForm(depends): return result + class Attributes(QtWidgets.QWidget): """Attributes The Attributes widget contains a path or some text @@ -98,10 +106,13 @@ def __init__(self, parent=None): self.__scrollWidget.layout().addWidget(self.__stack) self.__scrollArea.setWidget(self.__scrollWidget) layout.addWidget(self.__scrollArea) + # pylint: disable=no-member QtGui.qApp.single_click.connect(self.setWidget) + # pylint: enable=no-member self.__load = None + # pylint: disable=inconsistent-return-statements def setWidget(self, item): """If the item is a known object, then it will be displayed. @type item: any known object, otherwise ignored @@ -129,15 +140,18 @@ def setWidget(self, item): if hasattr(function, "preload"): if hasattr(QtGui.qApp, "threadpool"): self.__load = {"item": item, "function": function} + # pylint: disable=no-member QtGui.qApp.threadpool.queue(self.__getUpdate, self.__processResults, "getting data for %s" % self.__class__) + # pylint: enable=no-member else: logger.warning("threadpool not found, doing work in gui thread") return self.__createItemAttribute(item, function, function.preload(item)) else: return self.__createItemAttribute(item, function, None) + # pylint: disable=broad-except,inconsistent-return-statements def __getUpdate(self): """Called from the worker thread, gets results from preload""" try: @@ -151,6 +165,7 @@ def __getUpdate(self): def __processResults(self, work, result): """Unpacks the worker thread results and calls function to create widget""" + del work if result: self.__createItemAttribute(result["item"], result["function"], @@ -171,6 +186,12 @@ def __createItemAttribute(self, item, function, preload): class AbstractAttributes(QtWidgets.QTreeWidget): + """Abstract class for listing object attributes. + + Inheriting classes build on this, adding additional logic depending on the type of the + selected object. + """ + def __init__(self, rpcObject, preload, parent=None): QtWidgets.QTreeWidget.__init__(self, parent) @@ -206,6 +227,7 @@ def addData(parent, value): self.itemSelectionChanged.connect(self.itemSingleClickedCopy) def itemSingleClickedCopy(self): + """Event handler that copies the text of the selected line to the clipboard on click.""" selected = self.selectedItems() if selected: @@ -213,13 +235,18 @@ def itemSingleClickedCopy(self): class LayerAttributes(AbstractAttributes): + """Class for listing attributes of a layer.""" + NAME = "LayerAttributes" @staticmethod def preload(layer): + """Prepopulates needed layer information.""" return {"depends": layer.getWhatThisDependsOn()} + # pylint: disable=no-self-use def dataSource(self, layer, preload): + """Returns layer information structured as needed for the attributes list.""" d = { "id": opencue.util.id(layer), "layer": layer.data.name, @@ -263,9 +290,9 @@ def dataSource(self, layer, preload): "depends": getDependsForm(preload["depends"]), } - for num, output in enumerate(layer.getOutputPaths()): # Try to formulate a unique name the output. + # pylint: disable=bare-except try: # Outline only puts outputs in as filespecs, # so we're just going to assume it is. @@ -274,19 +301,26 @@ def dataSource(self, layer, preload): rep = "%s #%d" % (rep, num) except: rep = "output #%d" % num + # pylint: enable=bare-except d["outputs"][rep] = output return d + class JobAttributes(AbstractAttributes): + """Class for listing attributes of a job.""" + NAME = "JobAttributes" @staticmethod def preload(jobObject): + """Prepopulates needed job information.""" return {"depends": jobObject.getWhatThisDependsOn()} + # pylint: disable=no-self-use def dataSource(self, job, preload): + """Returns job information structured as needed for the attributes list.""" if isinstance(job, opencue.wrappers.job.NestedJob): job = job.asJob() d = { @@ -348,6 +382,7 @@ def dataSource(self, job, preload): for num, output in enumerate(outputs): # Try to formulate a unique name the output. + # pylint: disable=bare-except try: # Outline only puts outputs in as filespecs, # so we're just going to assume it is. @@ -356,14 +391,21 @@ def dataSource(self, job, preload): rep = "%s #%d" % (rep, num) except: rep = "output #%d" % num + # pylint: enable=bare-except entry[rep] = output return d + class HostAttributes(AbstractAttributes): + """Class for listing attributes of a host.""" + NAME = "Host" + # pylint: disable=no-self-use def dataSource(self, host, preload): + """Returns host information structured as needed for the attributes list.""" + del preload return {"hostname": host.data.name, "id": opencue.util.id(host), "alloc": host.data.alloc_name, diff --git a/cuegui/cuegui/plugins/LimitsPlugin.py b/cuegui/cuegui/plugins/LimitsPlugin.py index 07565822b..4a5947f54 100644 --- a/cuegui/cuegui/plugins/LimitsPlugin.py +++ b/cuegui/cuegui/plugins/LimitsPlugin.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Plugin for managing limits.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -29,17 +32,18 @@ class LimitsDockWidget(cuegui.AbstractDockWidget.AbstractDockWidget): - """This builds what is displayed on the dock widget""" - def __init__(self, parent): - super(LimitsDockWidget, self).__init__(parent, PLUGIN_NAME) - - self.__limitsWidget = cuegui.LimitsWidget.LimitsWidget(self) - - self.layout().addWidget(self.__limitsWidget) - - self.pluginRegisterSettings([("columnVisibility", - self.__limitsWidget.getColumnVisibility, - self.__limitsWidget.setColumnVisibility), - ("columnOrder", - self.__limitsWidget.getColumnOrder, - self.__limitsWidget.setColumnOrder)]) + """Widget for managing limits.""" + + def __init__(self, parent): + super(LimitsDockWidget, self).__init__(parent, PLUGIN_NAME) + + self.__limitsWidget = cuegui.LimitsWidget.LimitsWidget(self) + + self.layout().addWidget(self.__limitsWidget) + + self.pluginRegisterSettings([("columnVisibility", + self.__limitsWidget.getColumnVisibility, + self.__limitsWidget.setColumnVisibility), + ("columnOrder", + self.__limitsWidget.getColumnOrder, + self.__limitsWidget.setColumnOrder)]) diff --git a/cuegui/cuegui/plugins/LogViewPlugin.py b/cuegui/cuegui/plugins/LogViewPlugin.py index f49957538..fb40734e4 100644 --- a/cuegui/cuegui/plugins/LogViewPlugin.py +++ b/cuegui/cuegui/plugins/LogViewPlugin.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Plugin for viewing logs.""" + + from __future__ import division from __future__ import print_function from __future__ import absolute_import @@ -86,9 +89,10 @@ def __init__(self, parent): @param parent: The parent widget @type parent: QtWidgets.QWidget """ - super(LogTextEdit, self).__init__(parent) + self._context_menu = None + # Use a Fixed-Width font for easier debugging self.font = self.document().defaultFont() self.font.setFamily('Courier New') @@ -117,7 +121,6 @@ def context_menu(self): A custom context menu to pop up when the user Right-Clicks in the Log View. Triggered by the customContextMenuRequested signal """ - self._context_menu = QtWidgets.QMenu(self) self._context_menu.addAction(self.copy_action) self._context_menu.exec_(QtGui.QCursor.pos()) @@ -155,8 +158,9 @@ def copy_selection(self, mode): Copy (Ctrl + C) action. Stores the currently selected text in the clipboard. - @param mode: The QClipboard mode value (QtGui.QClipboard.Clipboard = GLOBAL - QtGui.QClipboard.Selection = Selection (middle-mouse)) + @param mode: The QClipboard mode value + (QtGui.QClipboard.Clipboard = GLOBAL + QtGui.QClipboard.Selection = Selection (middle-mouse)) @type mode: int """ selection = self.textCursor().selection() @@ -286,9 +290,7 @@ def line_number_area_paint_event(self, event): class LogViewWidget(QtWidgets.QWidget): - """ - Displays the log file for the selected frame - """ + """Displays the log file for the selected frame.""" def __init__(self, parent=None): """ @@ -421,7 +423,10 @@ def __init__(self, parent=None): self._cursor = self._content_box.textCursor() pos = QtCore.QPoint(0, 0) self._highlight_cursor = self._content_box.cursorForPosition(pos) + # Signals are defined in code, so pylint thinks they don't exist. + # pylint: disable=no-member QtGui.qApp.display_log_file_content.connect(self._set_log_files) + # pylint: enable=no-member self._log_scrollbar = self._content_box.verticalScrollBar() self._log_scrollbar.valueChanged.connect(self._set_scrollbar_value) @@ -511,7 +516,7 @@ def _set_log_file(self): log_index_txt = ('Log %s of %s%s' % (str(self._current_log_index+1), len(self._log_files), - (' (current)' if self._current_log_index == 0 + (' (current)' if self._current_log_index == 0 else ''))) self._log_index_label.setText(log_index_txt) @@ -723,10 +728,11 @@ def _clear_search_data(self): if not self._log_file: return - format = QtGui.QTextCharFormat() + charFormat = QtGui.QTextCharFormat() self._highlight_cursor.setPosition(QtGui.QTextCursor.Start) - self._highlight_cursor.movePosition(QtGui.QTextCursor.End, mode=QtGui.QTextCursor.KeepAnchor) - self._highlight_cursor.setCharFormat(format) + self._highlight_cursor.movePosition( + QtGui.QTextCursor.End, mode=QtGui.QTextCursor.KeepAnchor) + self._highlight_cursor.setCharFormat(charFormat) self._highlight_cursor.clearSelection() def _set_scrollbar_value(self, val): @@ -833,14 +839,15 @@ def _update_log(self): # Update the content in the gui (if necessary) current_text = (self._content_box.toPlainText() or '') new_text = content.lstrip(str(current_text)) - [x for x in new_text if x in PRINTABLE] if new_text: if self._new_log: self._content_box.setPlainText(content) else: self._content_box.appendPlainText(new_text) self._content_timestamp = time.time() + # pylint: disable=no-member QtGui.qApp.processEvents() + # pylint: enable=no-member # Adjust scrollbar value (if necessary) self._scrollbar_max = self._log_scrollbar.maximum() @@ -852,7 +859,7 @@ def _update_log(self): class LogViewPlugin(cuegui.AbstractDockWidget.AbstractDockWidget): """ Plugin for displaying the log file content for the selected frame with - the ability to perform regex-based search + the ability to perform regex-based search. """ def __init__(self, parent=None): @@ -869,6 +876,8 @@ def __init__(self, parent=None): class Highlighter(QtGui.QSyntaxHighlighter): + """Color-codes log text according to log content.""" + def __init__(self, parent=None): super(Highlighter, self).__init__(parent) @@ -894,7 +903,6 @@ def __init__(self, parent=None): self.completeFormat.setFontWeight(QtGui.QFont.Bold) self.completeFormat.setForeground(cuegui.Style.ColorTheme.LOG_COMPLETE) - def highlightBlock(self, text): if not self.on: return diff --git a/cuegui/cuegui/plugins/MonitorCuePlugin.py b/cuegui/cuegui/plugins/MonitorCuePlugin.py index f242aa5b9..6f07b80e0 100644 --- a/cuegui/cuegui/plugins/MonitorCuePlugin.py +++ b/cuegui/cuegui/plugins/MonitorCuePlugin.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Plugin for general administration of the show/job hierarchy.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -47,10 +50,13 @@ class MonitorCueDockWidget(cuegui.AbstractDockWidget.AbstractDockWidget): - """This builds what is displayed on the dock widget""" + """Plugin for general administration of the show/job hierarchy.""" + def __init__(self, parent): cuegui.AbstractDockWidget.AbstractDockWidget.__init__(self, parent, PLUGIN_NAME) + self.__showMenuActions = None + self.__monitorCue = cuegui.CueJobMonitorTree.CueJobMonitorTree(self) self.__toolbar = QtWidgets.QToolBar(self) self.__showMenuSetup() @@ -72,7 +78,9 @@ def __init__(self, parent): self.layout().addLayout(self.__hlayout) + # pylint: disable=no-member self.__monitorCue.view_object.connect(QtGui.qApp.view_object.emit) + # pylint: enable=no-member self.pluginRegisterSettings([("shows", self.__monitorCue.getShowNames, @@ -90,7 +98,10 @@ def __init__(self, parent): self.addShows([os.getenv('SHOW')]) def __cueStateBarSetup(self, layout): - if QtGui.qApp.settings.value("CueStateBar", False): + # pylint: disable=no-member + cueStateBarEnabled = QtGui.qApp.settings.value("CueStateBar", False) + # pylint: enable=no-member + if cueStateBarEnabled: self.__cueStateBar = cuegui.CueStateBarWidget.CueStateBarWidget(self.__monitorCue, self) layout.addWidget(self.__cueStateBar) @@ -149,9 +160,10 @@ def __buttonSetup(self, layout): btn.clicked.connect(self.__monitorCue.actionResumeSelectedItems) -################################################################################ -# Show selection menu -################################################################################ + ################################################################################ + # Show selection menu + ################################################################################ + def __showMenuSetup(self): """Sets up the show selection menu""" self.__showMenuBtn = QtWidgets.QPushButton("Shows ",self) @@ -164,7 +176,9 @@ def __showMenuSetup(self): self.__showMenuBtn.setFocusPolicy(QtCore.Qt.NoFocus) self.__showMenu.setFont(cuegui.Constants.STANDARD_FONT) self.__showMenu.triggered.connect(self.__showMenuHandle) + # pylint: disable=no-member QtGui.qApp.facility_changed.connect(self.__showMenuUpdate) + # pylint: enable=no-member self.__showMenuUpdate() @@ -205,7 +219,7 @@ def __showMenuUpdate(self): try: shows = sorted([show.name() for show in opencue.api.getActiveShows()]) - except Exception as e: + except opencue.exception.CueException as e: logger.critical(e) shows = [] @@ -310,24 +324,29 @@ def __jobSelectedHandle(self, job): else: self.__jobSelectedLineEdit.setText("") ################################################################################ + def addShows(self, shows): + """Adds a list of shows to be monitored.""" for show in shows: if show in self.__showMenuActions: self.__monitorCue.addShow(show, False) self.__showMenuActions[show].setChecked(True) - def pluginRestoreState(self, settings): + def pluginRestoreState(self, saved_settings): """Called on plugin start with any previously saved state. - @param settings: Last state of the plugin instance - @type settings: any""" - cuegui.AbstractDockWidget.AbstractDockWidget.pluginRestoreState(self, settings) + @param saved_settings: Last state of the plugin instance + @type saved_settings: any""" + cuegui.AbstractDockWidget.AbstractDockWidget.pluginRestoreState(self, saved_settings) + # pylint: disable=protected-access self.__monitorCue._update() + # pylint: enable=protected-access QtCore.QTimer.singleShot(1000, self.__monitorCue.expandAll) class JobSelectEditBox(QtWidgets.QLineEdit): - """An edit box intended for selecting matching jobs""" + """An edit box for selecting matching jobs.""" + def __init__(self, parent): QtWidgets.QLineEdit.__init__(self) self.parent = weakref.proxy(parent) diff --git a/cuegui/cuegui/plugins/MonitorHostsPlugin.py b/cuegui/cuegui/plugins/MonitorHostsPlugin.py index 1bf67bfc3..cee8413fe 100644 --- a/cuegui/cuegui/plugins/MonitorHostsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorHostsPlugin.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Plugin for viewing the list of hosts and performing administrative tasks.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -33,7 +36,8 @@ class HostMonitorDockWidget(cuegui.AbstractDockWidget.AbstractDockWidget): - """This builds what is displayed on the dock widget""" + """Plugin for viewing the list of hosts and performing administrative tasks.""" + def __init__(self, parent): cuegui.AbstractDockWidget.AbstractDockWidget.__init__(self, parent, PLUGIN_NAME) diff --git a/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py b/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py index f163be449..b731736c4 100644 --- a/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py @@ -13,6 +13,11 @@ # limitations under the License. +"""Plugin for listing details of the selected job. + +Job selection is triggered by other plugins using the application's view_object signal.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -40,6 +45,7 @@ class MonitorLayerFramesDockWidget(cuegui.AbstractDockWidget.AbstractDockWidget): """This builds a display that can monitor the layers and frames of a job.""" + def __init__(self, parent): """Creates the dock widget and docks it to the parent. @param parent: The main window to dock to @@ -58,14 +64,14 @@ def __init__(self, parent): self.__splitter.addWidget(self.__monitorLayers) self.__splitter.addWidget(self.__monitorFrames) - #Signals in: + # pylint: disable=no-member QtGui.qApp.view_object.connect(self.__setJob) QtGui.qApp.unmonitor.connect(self.__unmonitor) QtGui.qApp.facility_changed.connect(self.__setJob) + # pylint: enable=no-member self.__monitorLayers.handle_filter_layers_byLayer.connect(self.handleLayerFilter) self.__splitter.splitterMoved.connect(self.__splitterMoved) - self.pluginRegisterSettings([("splitterSize", self.__splitter.sizes, self.__splitter.setSizes), @@ -89,9 +95,11 @@ def __init__(self, parent): self.__monitorLayers.setColumnOrder)]) def handleLayerFilter(self, names): + """Event handler for filtering layers.""" self.__monitorFrames.filterLayersFromDoubleClick(names) def __splitterMoved(self, pos, index): + del index self.__monitorLayers.disableUpdate = not bool(pos) def dragEnterEvent(self, event): @@ -121,6 +129,7 @@ def __setJob(self, job = None): def __unmonitor(self, proxy): """Unmonitors the current job if it matches the supplied proxy. + @param proxy: A job proxy @type proxy: proxy""" if self.__job and self.__job == proxy: diff --git a/cuegui/cuegui/plugins/MonitorJobsPlugin.py b/cuegui/cuegui/plugins/MonitorJobsPlugin.py index 154a9d73c..c2f970b59 100644 --- a/cuegui/cuegui/plugins/MonitorJobsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorJobsPlugin.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Plugin for listing active jobs and managing them.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -45,27 +48,31 @@ class MonitorJobsDockWidget(cuegui.AbstractDockWidget.AbstractDockWidget): - """This builds what is displayed on the dock widget""" + """Plugin for listing active jobs and managing them.""" view_object = QtCore.Signal(object) def __init__(self, parent): cuegui.AbstractDockWidget.AbstractDockWidget.__init__(self, parent, PLUGIN_NAME) + self.__loadFinishedJobsCheckBox = None + self.jobMonitor = cuegui.JobMonitorTree.JobMonitorTree(self) self.__toolbar = QtWidgets.QToolBar(self) self._regexLoadJobsSetup(self.__toolbar) - #self.__toolbar.addSeparator() self._buttonSetup(self.__toolbar) self.layout().addWidget(self.__toolbar) self.layout().addWidget(self.jobMonitor) - #Signals in: + # Signals in + # pylint: disable=no-member QtGui.qApp.view_object.connect(self.addJob) QtGui.qApp.facility_changed.connect(self.jobMonitor.removeAllItems) - #Signals out: + # pylint: enable=no-member + + # Signals out self.jobMonitor.view_object.connect(self.view_object.emit) self.pluginRegisterSettings([("regexText", @@ -84,45 +91,49 @@ def __init__(self, parent): self.jobMonitor.getColumnOrder, self.jobMonitor.setColumnOrder)]) - def addJob(self, object): - if cuegui.Utils.isProc(object): - object = cuegui.Utils.findJob(object.data.job_name) - elif not cuegui.Utils.isJob(object): + def addJob(self, rpcObject): + """Adds a job to be monitored.""" + if cuegui.Utils.isProc(rpcObject): + rpcObject = cuegui.Utils.findJob(rpcObject.data.job_name) + elif not cuegui.Utils.isJob(rpcObject): return - self.jobMonitor.addJob(object) + self.jobMonitor.addJob(rpcObject) self.raise_() def getJobIds(self): + """Returns a list of the IDs of all jobs being monitored.""" return list(map(opencue.id, self.jobMonitor.getJobProxies())) def restoreJobIds(self, jobIds): + """Monitors a list of jobs.""" for jobId in jobIds: try: self.jobMonitor.addJob(jobId) - except opencue.EntityNotFoundException as e: + except opencue.EntityNotFoundException: logger.warning("Unable to load previously loaded job since " "it was moved to the historical " - "database: %s" % jobId) + "database: %s", jobId) - def pluginRestoreState(self, settings): + def pluginRestoreState(self, saved_settings): """Called on plugin start with any previously saved state. - @param settings: Last state of the plugin instance - @type settings: any""" - if isinstance(settings, dict): - cuegui.AbstractDockWidget.AbstractDockWidget.pluginRestoreState(self, settings) - elif settings: + @param saved_settings: Last state of the plugin instance + @type saved_settings: any""" + if isinstance(saved_settings, dict): + cuegui.AbstractDockWidget.AbstractDockWidget.pluginRestoreState(self, saved_settings) + + elif saved_settings: # old method that needs to go away - if len(settings) >= 1: - self.__regexLoadJobsEditBox.setText(settings[0]) - if len(settings) >= 2 and settings[1]: - for jobId in settings[1]: + if len(saved_settings) >= 1: + self.__regexLoadJobsEditBox.setText(saved_settings[0]) + if len(saved_settings) >= 2 and saved_settings[1]: + for jobId in saved_settings[1]: try: self.jobMonitor.addJob(jobId) - except opencue.EntityNotFoundException as e: + except opencue.EntityNotFoundException: logger.warning("Unable to load previously loaded job since" "it was moved to the historical " - "database: %s" % jobId) + "database: %s", jobId) def _regexLoadJobsSetup(self, layout): """Selects jobs by name substring. @@ -156,7 +167,8 @@ def _regexLoadJobsHandle(self): if cuegui.Utils.isStringId(substring): # If a uuid is provided, load it self.jobMonitor.addJob(substring) - elif load_finished_jobs or re.search("^([a-z0-9_]+)\-([a-z0-9\.]+)\-", substring, re.IGNORECASE): + elif load_finished_jobs or re.search( + r"^([a-z0-9_]+)\-([a-z0-9\.]+)\-", substring, re.IGNORECASE): # If show and shot is provided, or if "load finished" checkbox is checked, load all jobs for job in opencue.api.getJobs(substr=[substring], include_finished=True): self.jobMonitor.addJob(job) @@ -240,7 +252,10 @@ def _buttonSetup(self, layout): layout.addWidget(unpauseButton) unpauseButton.clicked.connect(self.jobMonitor.actionResumeSelectedItems) + class JobLoadFinishedCheckBox(QtWidgets.QCheckBox): + """Checkbox for controlling whether finished jobs appear in the list.""" + def __init__(self, parent): QtWidgets.QCheckBox.__init__(self, 'Load Finished') self.parent = weakref.proxy(parent) @@ -250,7 +265,10 @@ def __init__(self, parent): self.setToolTip(toolTip) + class JobRegexLoadEditBox(QtWidgets.QLineEdit): + """Textbox for searching for jobs to add to the list of monitored jobs.""" + def __init__(self, parent): QtWidgets.QLineEdit.__init__(self) self.parent = weakref.proxy(parent) @@ -290,12 +308,14 @@ def contextMenuEvent(self, e): menu.exec_(QtCore.QPoint(e.globalX(), e.globalY())) def actionClear(self): + """Clears the textbox.""" self.setText("") def _actionLoad(self): self.returnPressed.emit() def toggleReadOnly(self): + """Toggles the textbox readonly setting.""" self.setReadOnly(not self.isReadOnly()) def keyPressEvent(self, event): diff --git a/cuegui/cuegui/plugins/RedirectPlugin.py b/cuegui/cuegui/plugins/RedirectPlugin.py index 5af78a52c..1122e6baf 100644 --- a/cuegui/cuegui/plugins/RedirectPlugin.py +++ b/cuegui/cuegui/plugins/RedirectPlugin.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Plugin for redirecting procs from one job to another.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -29,6 +32,8 @@ class RedirectWidget(cuegui.AbstractDockWidget.AbstractDockWidget): + """Plugin for redirecting procs from one job to another.""" + def __init__(self, parent): cuegui.AbstractDockWidget.AbstractDockWidget.__init__(self, parent, PLUGIN_NAME) self.layout().addWidget(cuegui.Redirect.RedirectWidget(self)) diff --git a/cuegui/cuegui/plugins/ServicePlugin.py b/cuegui/cuegui/plugins/ServicePlugin.py index b69d89f36..cc2a99bef 100644 --- a/cuegui/cuegui/plugins/ServicePlugin.py +++ b/cuegui/cuegui/plugins/ServicePlugin.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Plugin for listing services and performing administrative tasks.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -31,10 +34,13 @@ class ServicesDockWidget(cuegui.AbstractDockWidget.AbstractDockWidget): - """This builds what is displayed on the dock widget""" + """Plugin for listing services and performing administrative tasks.""" + def __init__(self, parent): cuegui.AbstractDockWidget.AbstractDockWidget.__init__(self, parent, PLUGIN_NAME) self.setWindowTitle("Facility Service Defaults") self.__serviceManager = cuegui.ServiceDialog.ServiceManager(None, self) self.layout().addWidget(self.__serviceManager) + # pylint: disable=no-member QtGui.qApp.facility_changed.connect(self.__serviceManager.refresh) + # pylint: enable=no-member diff --git a/cuegui/cuegui/plugins/ShowsPlugin.py b/cuegui/cuegui/plugins/ShowsPlugin.py index e1abd60eb..89ef37754 100644 --- a/cuegui/cuegui/plugins/ShowsPlugin.py +++ b/cuegui/cuegui/plugins/ShowsPlugin.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Plugin for listing shows and performing administrative tasks.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -29,7 +32,8 @@ class ShowsDockWidget(cuegui.AbstractDockWidget.AbstractDockWidget): - """This builds what is displayed on the dock widget""" + """Plugin for listing shows and performing administrative tasks.""" + def __init__(self, parent): super(ShowsDockWidget, self).__init__(parent, PLUGIN_NAME) diff --git a/cuegui/cuegui/plugins/SubscriptionsGraphPlugin.py b/cuegui/cuegui/plugins/SubscriptionsGraphPlugin.py index 9e895aaf5..80dde33ca 100644 --- a/cuegui/cuegui/plugins/SubscriptionsGraphPlugin.py +++ b/cuegui/cuegui/plugins/SubscriptionsGraphPlugin.py @@ -1,3 +1,21 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Plugin for listing a show's subscriptions and a visualization of what's being consumed.""" + + from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -8,6 +26,7 @@ import cuegui.AbstractDockWidget import cuegui.SubscriptionGraphWidget + PLUGIN_NAME = 'Subscription Graphs' PLUGIN_CATEGORY = "Cuecommander" PLUGIN_DESCRIPTION = "An administrator interface to subscriptions" @@ -16,7 +35,8 @@ class SubscriptionGraphDockWidget(cuegui.AbstractDockWidget.AbstractDockWidget): - """This builds what is displayed on the dock widget""" + """Containing widget for this plugin.""" + def __init__(self, parent): cuegui.AbstractDockWidget.AbstractDockWidget.__init__(self, parent, PLUGIN_NAME) diff --git a/cuegui/cuegui/plugins/SubscriptionsPlugin.py b/cuegui/cuegui/plugins/SubscriptionsPlugin.py index f0cd946a7..7c0d11731 100644 --- a/cuegui/cuegui/plugins/SubscriptionsPlugin.py +++ b/cuegui/cuegui/plugins/SubscriptionsPlugin.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Plugin for managing show subscriptions.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -32,7 +35,8 @@ class SubscriptionDockWidget(cuegui.AbstractDockWidget.AbstractDockWidget): - """This builds what is displayed on the dock widget""" + """Widget that lists shows and the subscriptions they have.""" + def __init__(self, parent): cuegui.AbstractDockWidget.AbstractDockWidget.__init__(self, parent, PLUGIN_NAME) diff --git a/cuegui/tests/CueJobMonitorTree_tests.py b/cuegui/tests/CueJobMonitorTree_tests.py index 28859001f..87dd0a635 100644 --- a/cuegui/tests/CueJobMonitorTree_tests.py +++ b/cuegui/tests/CueJobMonitorTree_tests.py @@ -13,20 +13,24 @@ # limitations under the License. -import mock +"""Tests for cuegui.CueJobMonitorTree.""" + + import unittest +import mock import PySide2.QtCore import PySide2.QtGui import PySide2.QtWidgets -import cuegui.CueJobMonitorTree -import cuegui.plugins.MonitorCuePlugin -import cuegui.Style import opencue.compiled_proto.job_pb2 import opencue.compiled_proto.show_pb2 import opencue.wrappers.show +import cuegui.CueJobMonitorTree +import cuegui.plugins.MonitorCuePlugin +import cuegui.Style + from . import test_utils diff --git a/cuegui/tests/FilterDialog_tests.py b/cuegui/tests/FilterDialog_tests.py index 52d6b926a..071a2dab9 100644 --- a/cuegui/tests/FilterDialog_tests.py +++ b/cuegui/tests/FilterDialog_tests.py @@ -13,21 +13,26 @@ # limitations under the License. -import mock +"""Tests for cuegui.FilterDialog.""" + + import unittest +import mock + import PySide2.QtCore import PySide2.QtGui import PySide2.QtWidgets import PySide2.QtTest -import cuegui.FilterDialog -import cuegui.Style import opencue.compiled_proto.show_pb2 import opencue.compiled_proto.filter_pb2 import opencue.wrappers.filter import opencue.wrappers.show +import cuegui.FilterDialog +import cuegui.Style + from . import test_utils @@ -45,8 +50,9 @@ def setUp(self, getStubMock): id='filter-one-id', name='filterOne', order=1, enabled=True) self.filter = opencue.wrappers.filter.Filter(filterProto) - getStubMock.return_value.GetFilters.return_value = opencue.compiled_proto.show_pb2.ShowGetFiltersResponse( - filters=opencue.compiled_proto.filter_pb2.FilterSeq(filters=[filterProto])) + getStubMock.return_value.GetFilters.return_value = \ + opencue.compiled_proto.show_pb2.ShowGetFiltersResponse( + filters=opencue.compiled_proto.filter_pb2.FilterSeq(filters=[filterProto])) self.parentWidget = PySide2.QtWidgets.QWidget() self.filterDialog = cuegui.FilterDialog.FilterDialog(self.show, parent=self.parentWidget) @@ -159,8 +165,9 @@ def setUp(self, getStubMock): opencue.compiled_proto.filter_pb2.Filter( id='filter-two-id', name='filterTwo', order=2, enabled=False), ] - getStubMock.return_value.GetFilters.return_value = opencue.compiled_proto.show_pb2.ShowGetFiltersResponse( - filters=opencue.compiled_proto.filter_pb2.FilterSeq(filters=filters)) + getStubMock.return_value.GetFilters.return_value = \ + opencue.compiled_proto.show_pb2.ShowGetFiltersResponse( + filters=opencue.compiled_proto.filter_pb2.FilterSeq(filters=filters)) self.parentWidget = PySide2.QtWidgets.QWidget() self.filterDialog = cuegui.FilterDialog.FilterDialog(show, parent=self.parentWidget) @@ -257,8 +264,10 @@ def test_shouldAddMatcher(self, getItemMock, getTextMock): self.filter.createMatcher.assert_called_with(matcherSubject, matcherType, matcherText) self.assertEqual(3, self.matcherMonitorTree.topLevelItemCount()) matcherWidget = self.matcherMonitorTree.topLevelItem(0) - self.assertEqual('FACILITY', self.matcherMonitorTree.itemWidget(matcherWidget, 0).currentText()) - self.assertEqual('CONTAINS', self.matcherMonitorTree.itemWidget(matcherWidget, 1).currentText()) + self.assertEqual( + 'FACILITY', self.matcherMonitorTree.itemWidget(matcherWidget, 0).currentText()) + self.assertEqual( + 'CONTAINS', self.matcherMonitorTree.itemWidget(matcherWidget, 1).currentText()) self.assertEqual(matcherText, self.matcherMonitorTree.itemWidget(matcherWidget, 2).text()) @mock.patch('PySide2.QtWidgets.QInputDialog.getText') diff --git a/cuegui/tests/FrameMonitorTree_tests.py b/cuegui/tests/FrameMonitorTree_tests.py index ff7dee5ab..e28e8229f 100644 --- a/cuegui/tests/FrameMonitorTree_tests.py +++ b/cuegui/tests/FrameMonitorTree_tests.py @@ -13,23 +13,27 @@ # limitations under the License. -import mock +"""Tests for cuegui.FrameMonitorTree.""" + + import unittest +import mock import PySide2.QtCore import PySide2.QtGui import PySide2.QtTest import PySide2.QtWidgets +import opencue.compiled_proto.job_pb2 +import opencue.wrappers.frame +import opencue.wrappers.job + import cuegui.Constants import cuegui.FrameMonitor import cuegui.FrameMonitorTree import cuegui.Main import cuegui.plugins.MonitorJobDetailsPlugin import cuegui.Style -import opencue.compiled_proto.job_pb2 -import opencue.wrappers.frame -import opencue.wrappers.job from . import test_utils @@ -115,10 +119,11 @@ def test_tickFullUpdate(self, getFramesMock, getUpdatedFramesMock): getUpdatedFramesMock.assert_not_called() def test_getCores(self): - frame = opencue.wrappers.frame.Frame(opencue.compiled_proto.job_pb2.Frame(last_resource='foo/125.82723')) + frame = opencue.wrappers.frame.Frame( + opencue.compiled_proto.job_pb2.Frame(last_resource='foo/125.82723')) self.assertEqual(125.82723, self.frameMonitorTree.getCores(frame)) - self.assertEqual('125.83', self.frameMonitorTree.getCores(frame, format=True)) + self.assertEqual('125.83', self.frameMonitorTree.getCores(frame, format_as_string=True)) @mock.patch.object(cuegui.FrameMonitorTree.FrameContextMenu, 'exec_') def test_rightClickItem(self, execMock): diff --git a/cuegui/tests/MenuActions_tests.py b/cuegui/tests/MenuActions_tests.py index a1490da44..87282eefe 100644 --- a/cuegui/tests/MenuActions_tests.py +++ b/cuegui/tests/MenuActions_tests.py @@ -13,21 +13,19 @@ # limitations under the License. +"""Tests for cuegui.MenuActions.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import -from builtins import * -import mock import unittest +import mock import PySide2.QtGui import PySide2.QtWidgets -import cuegui.Constants -import cuegui.CueJobMonitorTree -import cuegui.Main -import cuegui.MenuActions import opencue.compiled_proto.depend_pb2 import opencue.compiled_proto.facility_pb2 import opencue.compiled_proto.filter_pb2 @@ -50,6 +48,11 @@ import opencue.wrappers.subscription import opencue.wrappers.task +import cuegui.Constants +import cuegui.CueJobMonitorTree +import cuegui.Main +import cuegui.MenuActions + _GB_TO_KB = 1024 * 1024 @@ -94,7 +97,7 @@ def test_emailArtist(self, emailDialogMock): self.job_actions.emailArtist(rpcObjects=[job]) - emailDialogMock.assert_called_with(job, [], self.widgetMock) + emailDialogMock.assert_called_with(job, self.widgetMock) emailDialogMock.return_value.show.assert_called() @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') @@ -534,7 +537,8 @@ def test_view(self): @mock.patch('cuegui.DependDialog.DependDialog') def test_viewDepends(self, dependDialogMock): - layer = opencue.wrappers.layer.Layer(opencue.compiled_proto.job_pb2.Layer(name='arbitrary-name')) + layer = opencue.wrappers.layer.Layer( + opencue.compiled_proto.job_pb2.Layer(name='arbitrary-name')) self.layer_actions.viewDepends(rpcObjects=[layer]) @@ -641,7 +645,8 @@ def test_setTags(self, layerTagsDialogMock): @mock.patch.object(opencue.wrappers.layer.Layer, 'kill') @mock.patch('cuegui.Utils.questionBoxYesNo', return_value=True) def test_kill(self, yesNoMock, killMock): - layer = opencue.wrappers.layer.Layer(opencue.compiled_proto.job_pb2.Layer(name='arbitrary-name')) + layer = opencue.wrappers.layer.Layer( + opencue.compiled_proto.job_pb2.Layer(name='arbitrary-name')) self.layer_actions.kill(rpcObjects=[layer]) @@ -725,7 +730,7 @@ def test_dependWizard(self, dependWizardMock): self.layer_actions.dependWizard(rpcObjects=layers) - dependWizardMock.assert_called_with(self.widgetMock, [self.job], layers) + dependWizardMock.assert_called_with(self.widgetMock, [self.job], layers=layers) @mock.patch.object(opencue.wrappers.layer.Layer, 'reorderFrames', autospec=True) @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') @@ -954,7 +959,7 @@ def test_dependWizard(self, dependWizardMock): self.frame_actions.dependWizard(rpcObjects=frames) - dependWizardMock.assert_called_with(self.widgetMock, [self.job], [], frames) + dependWizardMock.assert_called_with(self.widgetMock, [self.job], frames=frames) @mock.patch('cuegui.Utils.questionBoxYesNo', return_value=True) def test_markdone(self, yesNoMock): @@ -980,7 +985,8 @@ def test_reorder(self, getItemMock, reorderFramesMock): self.frame_actions.reorder(rpcObjects=[frame]) - reorderFramesMock.assert_called_with(layer, str(frame_num), opencue.compiled_proto.job_pb2.REVERSE) + reorderFramesMock.assert_called_with( + layer, str(frame_num), opencue.compiled_proto.job_pb2.REVERSE) @mock.patch('PySide2.QtWidgets.QApplication.clipboard') @mock.patch('cuegui.Utils.getFrameLogFile') @@ -998,8 +1004,10 @@ def test_copyLogFileName(self, getFrameLogFileMock, clipboardMock): def test_eatandmarkdone(self, yesNoMock, markdoneMock): layer_name = 'layer-name' frames = [ - opencue.wrappers.frame.Frame(opencue.compiled_proto.job_pb2.Frame(name='frame1', layer_name=layer_name)), - opencue.wrappers.frame.Frame(opencue.compiled_proto.job_pb2.Frame(name='frame2', layer_name=layer_name))] + opencue.wrappers.frame.Frame( + opencue.compiled_proto.job_pb2.Frame(name='frame1', layer_name=layer_name)), + opencue.wrappers.frame.Frame( + opencue.compiled_proto.job_pb2.Frame(name='frame2', layer_name=layer_name))] layer = opencue.wrappers.layer.Layer( opencue.compiled_proto.job_pb2.Layer( name=layer_name, layer_stats=opencue.compiled_proto.job_pb2.LayerStats( @@ -1133,6 +1141,7 @@ def test_delete(self): class AllocationActionsTests(unittest.TestCase): + # pylint: disable=attribute-defined-outside-init def test_init(self): self.widgetMock = mock.Mock() cuegui.MenuActions.AllocationActions(self.widgetMock, mock.Mock(), None, None) @@ -1386,66 +1395,66 @@ def setUp(self): @mock.patch('PySide2.QtWidgets.QInputDialog.getText') def test_rename(self, getTextMock): - filter = opencue.wrappers.filter.Filter(opencue.compiled_proto.filter_pb2.Filter()) - filter.setName = mock.MagicMock() + filter_wrapper = opencue.wrappers.filter.Filter(opencue.compiled_proto.filter_pb2.Filter()) + filter_wrapper.setName = mock.MagicMock() newName = 'newFilterName' getTextMock.return_value = (newName, True) - self.filter_actions.rename(rpcObjects=[filter]) + self.filter_actions.rename(rpcObjects=[filter_wrapper]) - filter.setName.assert_called_with(newName) + filter_wrapper.setName.assert_called_with(newName) @mock.patch('cuegui.Utils.questionBoxYesNo', new=mock.Mock(return_value=True)) def test_delete(self): - filter = opencue.wrappers.filter.Filter(opencue.compiled_proto.filter_pb2.Filter()) - filter.delete = mock.MagicMock() + filter_wrapper = opencue.wrappers.filter.Filter(opencue.compiled_proto.filter_pb2.Filter()) + filter_wrapper.delete = mock.MagicMock() - self.filter_actions.delete(rpcObjects=[filter]) + self.filter_actions.delete(rpcObjects=[filter_wrapper]) - filter.delete.assert_called() + filter_wrapper.delete.assert_called() def test_raiseOrder(self): - filter = opencue.wrappers.filter.Filter(opencue.compiled_proto.filter_pb2.Filter()) - filter.raiseOrder = mock.MagicMock() + filter_wrapper = opencue.wrappers.filter.Filter(opencue.compiled_proto.filter_pb2.Filter()) + filter_wrapper.raiseOrder = mock.MagicMock() - self.filter_actions.raiseOrder(rpcObjects=[filter]) + self.filter_actions.raiseOrder(rpcObjects=[filter_wrapper]) - filter.raiseOrder.assert_called() + filter_wrapper.raiseOrder.assert_called() def test_lowerOrder(self): - filter = opencue.wrappers.filter.Filter(opencue.compiled_proto.filter_pb2.Filter()) - filter.lowerOrder = mock.MagicMock() + filter_wrapper = opencue.wrappers.filter.Filter(opencue.compiled_proto.filter_pb2.Filter()) + filter_wrapper.lowerOrder = mock.MagicMock() - self.filter_actions.lowerOrder(rpcObjects=[filter]) + self.filter_actions.lowerOrder(rpcObjects=[filter_wrapper]) - filter.lowerOrder.assert_called() + filter_wrapper.lowerOrder.assert_called() def test_orderFirst(self): - filter = opencue.wrappers.filter.Filter(opencue.compiled_proto.filter_pb2.Filter()) - filter.orderFirst = mock.MagicMock() + filter_wrapper = opencue.wrappers.filter.Filter(opencue.compiled_proto.filter_pb2.Filter()) + filter_wrapper.orderFirst = mock.MagicMock() - self.filter_actions.orderFirst(rpcObjects=[filter]) + self.filter_actions.orderFirst(rpcObjects=[filter_wrapper]) - filter.orderFirst.assert_called() + filter_wrapper.orderFirst.assert_called() def test_orderLast(self): - filter = opencue.wrappers.filter.Filter(opencue.compiled_proto.filter_pb2.Filter()) - filter.orderLast = mock.MagicMock() + filter_wrapper = opencue.wrappers.filter.Filter(opencue.compiled_proto.filter_pb2.Filter()) + filter_wrapper.orderLast = mock.MagicMock() - self.filter_actions.orderLast(rpcObjects=[filter]) + self.filter_actions.orderLast(rpcObjects=[filter_wrapper]) - filter.orderLast.assert_called() + filter_wrapper.orderLast.assert_called() @mock.patch('PySide2.QtWidgets.QInputDialog.getInt') def test_setOrder(self, getTextMock): - filter = opencue.wrappers.filter.Filter(opencue.compiled_proto.filter_pb2.Filter()) - filter.setOrder = mock.MagicMock() - newOrder = 47 - getTextMock.return_value = (newOrder, True) + filter_wrapper = opencue.wrappers.filter.Filter(opencue.compiled_proto.filter_pb2.Filter()) + filter_wrapper.setOrder = mock.MagicMock() + new_order = 47 + getTextMock.return_value = (new_order, True) - self.filter_actions.setOrder(rpcObjects=[filter]) + self.filter_actions.setOrder(rpcObjects=[filter_wrapper]) - filter.setOrder.assert_called_with(newOrder) + filter_wrapper.setOrder.assert_called_with(new_order) @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) diff --git a/cuegui/tests/Redirect_tests.py b/cuegui/tests/Redirect_tests.py index e4432d843..d1e7ce5dd 100644 --- a/cuegui/tests/Redirect_tests.py +++ b/cuegui/tests/Redirect_tests.py @@ -13,17 +13,21 @@ # limitations under the License. -import mock +"""Tests for cuegui.Redirect.""" + + import unittest +import mock import PySide2.QtCore import PySide2.QtGui -import cuegui.Redirect -import cuegui.Style import opencue.compiled_proto.show_pb2 import opencue.wrappers.show +import cuegui.Redirect +import cuegui.Style + from . import test_utils diff --git a/cuegui/tests/Utils_tests.py b/cuegui/tests/Utils_tests.py index 0d9bd39b3..dddf46423 100644 --- a/cuegui/tests/Utils_tests.py +++ b/cuegui/tests/Utils_tests.py @@ -13,9 +13,13 @@ # limitations under the License. -import mock +"""Tests for cuegui.Utils.""" + + import unittest +import mock + import opencue.compiled_proto.job_pb2 import opencue.wrappers.job import cuegui.Utils diff --git a/cuegui/tests/images/images_tests.py b/cuegui/tests/images/images_tests.py index 6e76baad4..60f893bef 100644 --- a/cuegui/tests/images/images_tests.py +++ b/cuegui/tests/images/images_tests.py @@ -13,8 +13,12 @@ # limitations under the License. +"""Tests for cuegui.images.""" + + import unittest +# pylint: disable=unused-import import cuegui.images.icons_rcc import cuegui.images.bluecurve.icons_rcc import cuegui.images.crystal.icons_rcc diff --git a/cuegui/tests/plugins/LogViewPlugin_tests.py b/cuegui/tests/plugins/LogViewPlugin_tests.py index 74ffa3847..9ceb4a5b6 100644 --- a/cuegui/tests/plugins/LogViewPlugin_tests.py +++ b/cuegui/tests/plugins/LogViewPlugin_tests.py @@ -13,10 +13,14 @@ # limitations under the License. +"""Tests for cuegui.plugins.LogViewPlugin.""" + + import os -import mock import unittest +import mock + import pyfakefs.fake_filesystem_unittest import PySide2.QtCore import PySide2.QtGui @@ -47,6 +51,7 @@ Donec porta gravida eros id vulputate. Phasellus vel nisl arcu.''' +# pylint: disable=no-member class LogViewPluginTests(pyfakefs.fake_filesystem_unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.MagicMock()) def setUp(self): @@ -79,19 +84,25 @@ def test_shouldUpdateLogFile(self): def test_shouldHighlightAllSearchResults(self): PySide2.QtGui.qApp.display_log_file_content.emit([self.logPath1, self.logPath2]) - self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState(PySide2.QtCore.Qt.CheckState.Unchecked) + self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( + PySide2.QtCore.Qt.CheckState.Unchecked) self.logViewPlugin.logview_widget._search_box.setText('lorem') self.logViewPlugin.logview_widget._search_button.click() matches = self.logViewPlugin.logview_widget._matches self.assertEqual([(0, 5), (127, 5)], matches) - self.assertTrue(self.__isHighlighted(self.logViewPlugin.logview_widget._content_box, matches[0][0], matches[0][1])) - self.assertTrue(self.__isHighlighted(self.logViewPlugin.logview_widget._content_box, matches[1][0], matches[1][1])) + self.assertTrue( + self.__isHighlighted( + self.logViewPlugin.logview_widget._content_box, matches[0][0], matches[0][1])) + self.assertTrue( + self.__isHighlighted( + self.logViewPlugin.logview_widget._content_box, matches[1][0], matches[1][1])) def test_shouldMoveCursorToSecondSearchResult(self): PySide2.QtGui.qApp.display_log_file_content.emit([self.logPath1, self.logPath2]) - self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState(PySide2.QtCore.Qt.CheckState.Unchecked) + self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( + PySide2.QtCore.Qt.CheckState.Unchecked) self.logViewPlugin.logview_widget._search_box.setText('lorem') self.logViewPlugin.logview_widget._search_button.click() @@ -104,7 +115,8 @@ def test_shouldMoveCursorToSecondSearchResult(self): def test_shouldMoveCursorLastSearchResult(self): PySide2.QtGui.qApp.display_log_file_content.emit([self.logPath1, self.logPath2]) - self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState(PySide2.QtCore.Qt.CheckState.Unchecked) + self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( + PySide2.QtCore.Qt.CheckState.Unchecked) self.logViewPlugin.logview_widget._search_box.setText('lorem') self.logViewPlugin.logview_widget._search_button.click() @@ -117,14 +129,17 @@ def test_shouldMoveCursorLastSearchResult(self): def test_shouldPerformCaseInsensitiveSearch(self): PySide2.QtGui.qApp.display_log_file_content.emit([self.logPath1, self.logPath2]) - self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState(PySide2.QtCore.Qt.CheckState.Checked) + self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( + PySide2.QtCore.Qt.CheckState.Checked) self.logViewPlugin.logview_widget._search_box.setText('lorem') self.logViewPlugin.logview_widget._search_button.click() matches = self.logViewPlugin.logview_widget._matches self.assertEqual([(127, 5)], matches) - self.assertTrue(self.__isHighlighted(self.logViewPlugin.logview_widget._content_box, matches[0][0], matches[0][1])) + self.assertTrue( + self.__isHighlighted( + self.logViewPlugin.logview_widget._content_box, matches[0][0], matches[0][1])) @staticmethod def __isHighlighted(textBox, startPosition, selectionLength): @@ -135,5 +150,6 @@ def __isHighlighted(textBox, startPosition, selectionLength): selectionLength) return cursor.charFormat().background() == PySide2.QtCore.Qt.red + if __name__ == '__main__': unittest.main() diff --git a/cuegui/tests/test_utils.py b/cuegui/tests/test_utils.py index 5e2685049..81f22f6cc 100644 --- a/cuegui/tests/test_utils.py +++ b/cuegui/tests/test_utils.py @@ -13,12 +13,16 @@ # limitations under the License. +"""Common utility functions for CueGUI test code.""" + + import cuegui.Main __QAPPLICATION_SINGLETON = None +# pylint: disable=global-statement def createApplication(): global __QAPPLICATION_SINGLETON if __QAPPLICATION_SINGLETON is None: diff --git a/pycue/opencue/wrappers/service.py b/pycue/opencue/wrappers/service.py index acacb233a..1a941772b 100644 --- a/pycue/opencue/wrappers/service.py +++ b/pycue/opencue/wrappers/service.py @@ -192,3 +192,19 @@ def setTags(self, tags): :param: new list of service tags """ self.data.tags[:] = tags + + def timeout(self): + """Gets the default service timeout.""" + return self.data.timeout + + def setTimeout(self, timeout): + """Sets the default service timeout.""" + self.data.timeout = timeout + + def timeoutLLU(self): + """Gets the default service LLU timeout.""" + return self.data.timeout + + def setTimeoutLLU(self, timeout_llu): + """Sets the default service LLU timeout.""" + self.data.timeout_llu = timeout_llu From f757be0641a06b248d93b0d2bc53c53c00fcacd2 Mon Sep 17 00:00:00 2001 From: Lars van der Bijl <285658+larsbijl@users.noreply.github.com> Date: Tue, 9 Feb 2021 08:36:18 +0000 Subject: [PATCH 052/277] Remove ptree feature (#914) This is unused Closes #913 Co-authored-by: Lars van der Bijl --- rqd/rqd/rqconstants.py | 3 --- rqd/rqd/rqmachine.py | 8 -------- rqd/tests/rqmachine_tests.py | 4 ---- 3 files changed, 15 deletions(-) diff --git a/rqd/rqd/rqconstants.py b/rqd/rqd/rqconstants.py index 0a1508e45..7b5ce3886 100644 --- a/rqd/rqd/rqconstants.py +++ b/rqd/rqd/rqconstants.py @@ -79,9 +79,6 @@ RQD_UID = 0 RQD_GID = 0 -# ptree reporting is not actually used, and could be slow -ENABLE_PTREE = False - # Nimby behavior: CHECK_INTERVAL_LOCKED = 60 # = seconds to wait before checking if the user has become idle MINIMUM_IDLE = 900 # seconds of idle time required before nimby unlocks diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index e2cc365a0..92823cf3e 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -219,8 +219,6 @@ def rssUpdate(self, frames): rss = 0 vsize = 0 pcpu = 0 - if rqd.rqconstants.ENABLE_PTREE: - ptree = [] for pid, data in pids.items(): if data["session"] == session: try: @@ -250,8 +248,6 @@ def rssUpdate(self, frames): pcpu += pidPcpu pidData[pid] = totalTime, seconds, pidPcpu - if rqd.rqconstants.ENABLE_PTREE: - ptree.append({"pid": pid, "seconds": seconds, "total_time": totalTime}) except Exception as e: log.warning('Failure with pid rss update due to: %s at %s' % \ (e, traceback.extract_tb(sys.exc_info()[2]))) @@ -271,10 +267,6 @@ def rssUpdate(self, frames): frame.runFrame.attributes["pcpu"] = str(pcpu) - if rqd.rqconstants.ENABLE_PTREE: - frame.runFrame.attributes["ptree"] = str(yaml.load("list: %s" % ptree, - Loader=yaml.SafeLoader)) - # Store the current data for the next check self.__pidHistory = pidData diff --git a/rqd/tests/rqmachine_tests.py b/rqd/tests/rqmachine_tests.py index eadf63f54..963a10a0d 100644 --- a/rqd/tests/rqmachine_tests.py +++ b/rqd/tests/rqmachine_tests.py @@ -278,7 +278,6 @@ def test_isUserLoggedInWithNoDisplayOrProcess(self, processIterMock): @mock.patch('time.time', new=mock.MagicMock(return_value=1570057887.61)) def test_rssUpdate(self): rqd.rqconstants.SYS_HERTZ = 100 - rqd.rqconstants.ENABLE_PTREE = True pid = 105 frameId = 'unused-frame-id' self.fs.create_file('/proc/%d/stat' % pid, contents=PROC_PID_STAT) @@ -295,9 +294,6 @@ def test_rssUpdate(self): self.assertEqual(4356, updatedFrameInfo.max_vsize) self.assertEqual(4356, updatedFrameInfo.vsize) self.assertAlmostEqual(0.034444696691, float(updatedFrameInfo.attributes['pcpu'])) - self.assertEqual( - {'list': [{'seconds': 1277.4100000000035, 'total_time': 44, 'pid': '105'}]}, - eval(updatedFrameInfo.attributes['ptree'])) @mock.patch.object( rqd.rqmachine.Machine, '_Machine__enabledHT', new=mock.MagicMock(return_value=False)) From a61cc48c42afcb9a9c3a802357047783902b4a5e Mon Sep 17 00:00:00 2001 From: Lars van der Bijl <285658+larsbijl@users.noreply.github.com> Date: Tue, 9 Feb 2021 15:57:12 +0000 Subject: [PATCH 053/277] Add RQD overrides for gRPC ports and default facility. (#911) --- rqd/rqd/__main__.py | 3 --- rqd/rqd/cuerqd.py | 2 -- rqd/rqd/rqconstants.py | 46 ++++++++------------------------------- rqd/rqd/rqcore.py | 2 +- rqd/rqd/rqmachine.py | 2 +- rqd/tests/rqcore_tests.py | 2 +- 6 files changed, 12 insertions(+), 45 deletions(-) diff --git a/rqd/rqd/__main__.py b/rqd/rqd/__main__.py index 81130fdc1..e69209828 100755 --- a/rqd/rqd/__main__.py +++ b/rqd/rqd/__main__.py @@ -130,9 +130,6 @@ def main(): logging.warning('RQD Starting Up') - if rqd.rqconstants.FACILITY == 'abq': - os.environ['TZ'] = 'PST8PDT' - rqCore = rqd.rqcore.RqCore(optNimbyOff) rqCore.start() diff --git a/rqd/rqd/cuerqd.py b/rqd/rqd/cuerqd.py index 538218d7d..2a14bdfef 100755 --- a/rqd/rqd/cuerqd.py +++ b/rqd/rqd/cuerqd.py @@ -28,8 +28,6 @@ import logging as log import os import random -import re -import sys import grpc diff --git a/rqd/rqd/rqconstants.py b/rqd/rqd/rqconstants.py index 7b5ce3886..56170aa64 100644 --- a/rqd/rqd/rqconstants.py +++ b/rqd/rqd/rqconstants.py @@ -28,7 +28,6 @@ import logging import os import platform -import re import subprocess import sys import traceback @@ -119,43 +118,11 @@ LOAD_MODIFIER = 0 # amount to add/subtract from load if subprocess.getoutput('/bin/su --help').find('session-command') != -1: - SU_ARGUEMENT = '--session-command' + SU_ARGUMENT = '--session-command' else: - SU_ARGUEMENT = '-c' - -SP_OS = FACILITY = '' -proc = None -# Try to read facility and os from studio environment -if os.path.isfile('/usr/local/stdenv/.cshrc'): - proc = subprocess.Popen( - "csh -c 'unsetenv SP_PATH ; setenv CONSOLE 1 ; setenv HOME / ;" - " source /usr/local/stdenv/.cshrc ; echo $SP_OS $FACILITY'", - shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) -elif os.path.isfile('/etc/csh.cshrc'): - # For maa on centos - proc = subprocess.Popen("csh -c 'source /etc/csh.cshrc ; echo $SP_OS $FACILITY'", - shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - -# If we have a popen process and it has successfully been run, -# get os and facility from result. -if proc: - out, err = proc.communicate() - if proc.returncode == 0: - SP_OS, FACILITY = out.split()[-2:] - -if not 3 <= len(SP_OS) <= 10 or not re.match('^[A-Za-z0-9]*$', SP_OS): - if SP_OS: - logging.warning('SP_OS value of %s is out of allowed range' % SP_OS) - SP_OS = platform.system() - -if len(FACILITY) != 3 or not re.match('^[A-Za-z0-9]*$', FACILITY): - if FACILITY: - logging.warning('FACILITY value of %s is out of allowed range' % FACILITY) - FACILITY = DEFAULT_FACILITY - -# maa is small so decrease the ping in interval -if FACILITY == 'maa': - RQD_MAX_PING_INTERVAL_SEC = 30 + SU_ARGUMENT = '-c' + +SP_OS = platform.system() try: if os.path.isfile(CONFIG_FILE): @@ -165,6 +132,11 @@ config = configparser.RawConfigParser() logging.info('Loading config {}'.format(CONFIG_FILE)) config.read(CONFIG_FILE) + + if config.has_option(__section, "RQD_GRPC_PORT"): + RQD_GRPC_PORT = config.getint(__section, "RQD_GRPC_PORT") + if config.has_option(__section, "CUEBOT_GRPC_PORT"): + CUEBOT_GRPC_PORT = config.getint(__section, "CUEBOT_GRPC_PORT") if config.has_option(__section, "OVERRIDE_CORES"): OVERRIDE_CORES = config.getint(__section, "OVERRIDE_CORES") if config.has_option(__section, "OVERRIDE_PROCS"): diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index f0f34a63f..a8041727d 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -264,7 +264,7 @@ def runLinux(self): rqd.rqutil.permissionsHigh() try: if rqd.rqconstants.RQD_BECOME_JOB_USER: - tempCommand += ["/bin/su", runFrame.user_name, rqd.rqconstants.SU_ARGUEMENT, + tempCommand += ["/bin/su", runFrame.user_name, rqd.rqconstants.SU_ARGUMENT, '"' + self._createCommandFile(runFrame.command) + '"'] else: tempCommand += [self._createCommandFile(runFrame.command)] diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 92823cf3e..68a450ba8 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -407,7 +407,7 @@ def __initMachineStats(self, pathCpuInfo=None): """Updates static machine information during initialization""" self.__renderHost.name = self.getHostname() self.__renderHost.boot_time = self.getBootTime() - self.__renderHost.facility = rqd.rqconstants.FACILITY + self.__renderHost.facility = rqd.rqconstants.DEFAULT_FACILITY self.__renderHost.attributes['SP_OS'] = rqd.rqconstants.SP_OS self.updateMachineStats() diff --git a/rqd/tests/rqcore_tests.py b/rqd/tests/rqcore_tests.py index 6c9de1c8e..89a845ed6 100644 --- a/rqd/tests/rqcore_tests.py +++ b/rqd/tests/rqcore_tests.py @@ -550,7 +550,7 @@ def test_unlockAllWhenNimbyLocked(self): class FrameAttendantThreadTests(pyfakefs.fake_filesystem_unittest.TestCase): def setUp(self): self.setUpPyfakefs() - rqd.rqconstants.SU_ARGUEMENT = '-c' + rqd.rqconstants.SU_ARGUMENT = '-c' @mock.patch('platform.system', new=mock.Mock(return_value='Linux')) @mock.patch('tempfile.gettempdir') From 2ff7e237ff42d025157a71846ab0f5d76843547f Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Wed, 10 Feb 2021 08:47:43 -0800 Subject: [PATCH 054/277] Fix reading RQD_TAGS value from config file. (#916) --- rqd/rqd/rqconstants.py | 11 ++- rqd/tests/rqconstants_tests.py | 147 +++++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 3 deletions(-) create mode 100644 rqd/tests/rqconstants_tests.py diff --git a/rqd/rqd/rqconstants.py b/rqd/rqd/rqconstants.py index 56170aa64..010fbb964 100644 --- a/rqd/rqd/rqconstants.py +++ b/rqd/rqd/rqconstants.py @@ -128,8 +128,13 @@ if os.path.isfile(CONFIG_FILE): # Hostname can come from here: rqutil.getHostname() __section = "Override" - import configparser - config = configparser.RawConfigParser() + import six + from six.moves import configparser + if six.PY2: + ConfigParser = configparser.SafeConfigParser + else: + ConfigParser = configparser.RawConfigParser + config = ConfigParser() logging.info('Loading config {}'.format(CONFIG_FILE)) config.read(CONFIG_FILE) @@ -162,7 +167,7 @@ if config.has_option(__section, "RQD_BECOME_JOB_USER"): RQD_BECOME_JOB_USER = config.getboolean(__section, "RQD_BECOME_JOB_USER") if config.has_option(__section, "RQD_TAGS"): - RQD_TAGS = config.getint(__section, "RQD_TAGS") + RQD_TAGS = config.get(__section, "RQD_TAGS") if config.has_option(__section, "DEFAULT_FACILITY"): DEFAULT_FACILITY = config.get(__section, "DEFAULT_FACILITY") if config.has_option(__section, "LAUNCH_FRAME_USER_GID"): diff --git a/rqd/tests/rqconstants_tests.py b/rqd/tests/rqconstants_tests.py new file mode 100644 index 000000000..7eed236c5 --- /dev/null +++ b/rqd/tests/rqconstants_tests.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python + +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import + +import os.path +import shutil +import tempfile +import unittest +import uuid + +import mock +import pyfakefs.fake_filesystem_unittest + +import six + +import rqd.rqconstants +import rqd.rqcore +import rqd.rqmachine +import rqd.rqnimby +import rqd.rqutil +import rqd.compiled_proto.report_pb2 + +from .rqmachine_tests import ( + CPUINFO, + CUDAINFO, + LOADAVG_LOW_USAGE, + MEMINFO_MODERATE_USAGE, + PROC_STAT, +) + + +if not six.PY2: + import importlib + + reload = importlib.reload + + +class MockConfig(object): + def __init__(self, tempdir, content): + config = os.path.join(tempdir, str(uuid.uuid4())) + self.patcher = mock.patch("sys.argv", ["rqd", "-c", config]) + + with open(config, "w") as f: + print(content, file=f) + + def __enter__(self): + self.patcher.start() + reload(rqd.rqconstants) + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.patcher.stop() + + def __call__(self, func): + def decorator(*args, **kwargs): + with self: + return func(*args, **kwargs) + + return decorator + + +@mock.patch("subprocess.getoutput", new=mock.MagicMock(return_value=CUDAINFO)) +@mock.patch.object( + rqd.rqutil.Memoize, "isCached", new=mock.MagicMock(return_value=False) +) +@mock.patch("platform.system", new=mock.MagicMock(return_value="Linux")) +@mock.patch("os.statvfs", new=mock.MagicMock()) +@mock.patch( + "rqd.rqutil.getHostname", new=mock.MagicMock(return_value="arbitrary-hostname") +) +class RqConstantTests(pyfakefs.fake_filesystem_unittest.TestCase): + + tempdir = tempfile.mkdtemp() + + def setUp(self): + self.setUpPyfakefs() + self.fs.add_real_directory(self.tempdir) + self.fs.create_file("/proc/cpuinfo", contents=CPUINFO) + self.loadavg = self.fs.create_file("/proc/loadavg", contents=LOADAVG_LOW_USAGE) + self.procStat = self.fs.create_file("/proc/stat", contents=PROC_STAT) + self.meminfo = self.fs.create_file( + "/proc/meminfo", contents=MEMINFO_MODERATE_USAGE + ) + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def makeRqMachine(self): + rqCore = mock.MagicMock(spec=rqd.rqcore.RqCore) + nimby = mock.MagicMock(spec=rqd.rqnimby.Nimby) + rqCore.nimby = nimby + nimby.active = False + nimby.locked = False + coreDetail = rqd.compiled_proto.report_pb2.CoreDetail(total_cores=2) + machine = rqd.rqmachine.Machine(rqCore, coreDetail) + + machine.renderHost = machine.__dict__["_Machine__renderHost"] + + return machine + + @MockConfig( + tempdir, + """ +[Override] +DEFAULT_FACILITY = test_facility +""", + ) + def test_facility(self): + self.assertEqual(rqd.rqconstants.DEFAULT_FACILITY, "test_facility") + + machine = self.makeRqMachine() + self.assertEqual(machine.renderHost.facility, "test_facility") + + @MockConfig( + tempdir, + """ +[Override] +RQD_TAGS = test_tag1 test_tag2 test_tag3 +""", + ) + def test_tags(self): + self.assertEqual(rqd.rqconstants.RQD_TAGS, "test_tag1 test_tag2 test_tag3") + + machine = self.makeRqMachine() + self.assertEqual(machine.renderHost.facility, "cloud") + self.assertTrue( + set(["test_tag1", "test_tag2", "test_tag3"]).issubset( + machine.renderHost.tags + ) + ) From 97a72a08fb3389ca7744583f1538f4460bb9b8a8 Mon Sep 17 00:00:00 2001 From: Lars van der Bijl <285658+larsbijl@users.noreply.github.com> Date: Sun, 14 Feb 2021 10:39:39 +0000 Subject: [PATCH 055/277] Remove unused config and task locks (#910) Co-authored-by: Lars van der Bijl --- .../main/resources/conf/ddl/oracle/demo_data.sql | 16 ---------------- .../resources/conf/ddl/postgres/demo_data.sql | 16 ---------------- .../test/resources/conf/ddl/oracle/test_data.sql | 15 --------------- .../resources/conf/ddl/postgres/test_data.sql | 16 ---------------- 4 files changed, 63 deletions(-) diff --git a/cuebot/src/main/resources/conf/ddl/oracle/demo_data.sql b/cuebot/src/main/resources/conf/ddl/oracle/demo_data.sql index a62190f09..fc97a477f 100644 --- a/cuebot/src/main/resources/conf/ddl/oracle/demo_data.sql +++ b/cuebot/src/main/resources/conf/ddl/oracle/demo_data.sql @@ -70,25 +70,9 @@ Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN, -- SPLIT HERE! Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAA12','postprocess',0,10,524288,'util') --- SPLIT HERE! -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000000','MAX_PING_TIME',300,0,null,0) --- SPLIT HERE! -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000001','MIN_CORES_REQUIRED',10,0,null,0) --- SPLIT HERE! -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000002','MIN_MEM_REQUIRED',0,500000,null,0) --- SPLIT HERE! -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000003','MAX_PPS',0,20,null,0) -- SPLIT HERE! Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000005','MAX_FRAME_RETRIES',16,0,null,0) --- SPLIT HERE! -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000006','MAX_LAYER_COUNT_PER_JOB',250,0,null,0) --- SPLIT HERE! -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000007','MAX_FRAME_COUNT_PER_JOB',0,250000,null,0) --- SPLIT HERE! -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000008','DEFAULT_FRAME_RETRIES',2,0,null,0) --- SPLIT HERE! -Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000004','LOCK_SHOW_ALERTS',0,300) -- SPLIT HERE! Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000002','LOCK_HARDWARE_STATE_CHECK',0,30) -- SPLIT HERE! diff --git a/cuebot/src/main/resources/conf/ddl/postgres/demo_data.sql b/cuebot/src/main/resources/conf/ddl/postgres/demo_data.sql index 2f8b89f4e..ab2787dca 100644 --- a/cuebot/src/main/resources/conf/ddl/postgres/demo_data.sql +++ b/cuebot/src/main/resources/conf/ddl/postgres/demo_data.sql @@ -72,24 +72,8 @@ Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN, Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAA12','postprocess',false,10,524288,'util'); -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000000','MAX_PING_TIME',300,0,null,false); - -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000001','MIN_CORES_REQUIRED',10,0,null,false); - -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000002','MIN_MEM_REQUIRED',0,500000,null,false); - -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000003','MAX_PPS',0,20,null,false); - Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000005','MAX_FRAME_RETRIES',16,0,null,false); -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000006','MAX_LAYER_COUNT_PER_JOB',250,0,null,false); - -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000007','MAX_FRAME_COUNT_PER_JOB',0,250000,null,false); - -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000008','DEFAULT_FRAME_RETRIES',2,0,null,false); - - -Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000004','LOCK_SHOW_ALERTS',0,300); Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000002','LOCK_HARDWARE_STATE_CHECK',0,30); diff --git a/cuebot/src/test/resources/conf/ddl/oracle/test_data.sql b/cuebot/src/test/resources/conf/ddl/oracle/test_data.sql index 37713d006..bdc0e4c8f 100644 --- a/cuebot/src/test/resources/conf/ddl/oracle/test_data.sql +++ b/cuebot/src/test/resources/conf/ddl/oracle/test_data.sql @@ -105,24 +105,9 @@ Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN, Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS,INT_CORES_MAX,INT_GPU_MIN) values ('488c75f0-eae4-4dd0-83e0-29b982adbbff','cuda',1,100,3354624,'cuda',0,262144) -- SPLIT HERE! -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000000','MAX_PING_TIME',300,0,null,0) --- SPLIT HERE! -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000001','MIN_CORES_REQUIRED',10,0,null,0) --- SPLIT HERE! -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000002','MIN_MEM_REQUIRED',0,500000,null,0) --- SPLIT HERE! -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000003','MAX_PPS',0,20,null,0) -- SPLIT HERE! Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000005','MAX_FRAME_RETRIES',16,0,null,0) --- SPLIT HERE! -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000006','MAX_LAYER_COUNT_PER_JOB',250,0,null,0) --- SPLIT HERE! -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000007','MAX_FRAME_COUNT_PER_JOB',0,250000,null,0) --- SPLIT HERE! -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000008','DEFAULT_FRAME_RETRIES',2,0,null,0) --- SPLIT HERE! -Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000004','LOCK_SHOW_ALERTS',0,300) -- SPLIT HERE! Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000002','LOCK_HARDWARE_STATE_CHECK',0,30) -- SPLIT HERE! diff --git a/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql b/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql index 9907b22ff..b5596b91d 100644 --- a/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql +++ b/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql @@ -105,24 +105,8 @@ Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN, Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS,INT_CORES_MAX,INT_GPU_MIN) values ('488c75f0-eae4-4dd0-83e0-29b982adbbff','cuda',true,100,3354624,'cuda',0,262144) -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000000','MAX_PING_TIME',300,0,null,false) - -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000001','MIN_CORES_REQUIRED',10,0,null,false) - -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000002','MIN_MEM_REQUIRED',0,500000,null,false) - -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000003','MAX_PPS',0,20,null,false) - Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000005','MAX_FRAME_RETRIES',16,0,null,false) -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000006','MAX_LAYER_COUNT_PER_JOB',250,0,null,false) - -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000007','MAX_FRAME_COUNT_PER_JOB',0,250000,null,false) - -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000008','DEFAULT_FRAME_RETRIES',2,0,null,false) - - -Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000004','LOCK_SHOW_ALERTS',0,300) Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000002','LOCK_HARDWARE_STATE_CHECK',0,30) From 15ff97ea396606b3b644f52896d5509c356d7897 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Tue, 16 Feb 2021 13:21:06 -0800 Subject: [PATCH 056/277] [PyOutline] Add spec_version (#919) * Add spec_version to outline.cfg * Add spec_version unittest --- pyoutline/etc/outline.cfg | 1 + pyoutline/outline/backend/cue.py | 24 ++++++++++-- pyoutline/tests/specver_test.py | 63 ++++++++++++++++++++++++++++++++ requirements.txt | 1 + 4 files changed, 85 insertions(+), 4 deletions(-) create mode 100644 pyoutline/tests/specver_test.py diff --git a/pyoutline/etc/outline.cfg b/pyoutline/etc/outline.cfg index b80006f0d..6d0f129a4 100644 --- a/pyoutline/etc/outline.cfg +++ b/pyoutline/etc/outline.cfg @@ -5,6 +5,7 @@ wrapper_dir = %(home)s/wrappers user_dir = bin_dir = %(home)s/bin backend = cue +spec_version = 1.11 facility = local domain = example.com maxretries = 2 diff --git a/pyoutline/outline/backend/cue.py b/pyoutline/outline/backend/cue.py index 6808ff7e2..1d91eb70a 100644 --- a/pyoutline/outline/backend/cue.py +++ b/pyoutline/outline/backend/cue.py @@ -32,6 +32,7 @@ from xml.dom.minidom import parseString from xml.etree import ElementTree as Et +from packaging.version import Version import six import FileSequence @@ -216,6 +217,10 @@ def serialize_simple(launcher): return _serialize(launcher, use_pycuerun=False) +def _warning_spec_version(spec_version, feature): + logger.warning("spec_version=%s doesn't support %s", spec_version, feature) + + def _serialize(launcher, use_pycuerun): """ Serialize the outline part of the given L{OutlineLauncher} into a @@ -229,6 +234,8 @@ def _serialize(launcher, use_pycuerun): """ ol = launcher.get_outline() + spec_version = Version(outline.config.get("outline", "spec_version")) + root = Et.Element("spec") depends = Et.Element("depends") @@ -246,7 +253,10 @@ def _serialize(launcher, use_pycuerun): j = Et.SubElement(root, "job", {"name": ol.get_name()}) sub_element(j, "paused", str(launcher.get("pause"))) - sub_element(j, "priority", str(launcher.get("priority"))) + if spec_version >= Version("1.11"): + sub_element(j, "priority", str(launcher.get("priority"))) + elif launcher.get("priority"): + _warning_spec_version(spec_version, "priority") sub_element(j, "maxretries", str(launcher.get("maxretries"))) sub_element(j, "autoeat", str(launcher.get("autoeat"))) @@ -309,10 +319,16 @@ def _serialize(launcher, use_pycuerun): sub_element(spec_layer, "memory", "%s" % (layer.get_arg("memory"))) if layer.get_arg("timeout"): - sub_element(spec_layer, "timeout", "%s" % (layer.get_arg("timeout"))) + if spec_version >= Version("1.10"): + sub_element(spec_layer, "timeout", "%s" % (layer.get_arg("timeout"))) + else: + _warning_spec_version(spec_version, "timeout") if layer.get_arg("timeout_llu"): - sub_element(spec_layer, "timeout_llu", "%s" % (layer.get_arg("timeout_llu"))) + if spec_version >= Version("1.10"): + sub_element(spec_layer, "timeout_llu", "%s" % (layer.get_arg("timeout_llu"))) + else: + _warning_spec_version(spec_version, "timeout_llu") if os.environ.get("OL_TAG_OVERRIDE", False): sub_element(spec_layer, "tags", @@ -352,7 +368,7 @@ def _serialize(launcher, use_pycuerun): xml = [ '', '', + '"http://localhost:8080/spcue/dtd/cjsl-%s.dtd">' % spec_version, Et.tostring(root).decode() ] diff --git a/pyoutline/tests/specver_test.py b/pyoutline/tests/specver_test.py new file mode 100644 index 000000000..24d92df07 --- /dev/null +++ b/pyoutline/tests/specver_test.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Tests for the outline.cfg spec_version +""" + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import unittest + +from xml.etree import ElementTree as Et + +import outline +import outline.modules.shell + + +class SpecVersiondTest(unittest.TestCase): + def _makeSpec(self): + ol = outline.Outline(name="spec_version_test") + layer = outline.modules.shell.Shell("test_layer", command=["/bin/ls"]) + layer.set_arg("timeout", 420) + layer.set_arg("timeout_llu", 4200) + ol.add_layer(layer) + l = outline.cuerun.OutlineLauncher(ol) + l.set_flag("priority", 42) + return Et.fromstring(l.serialize()) + + def test_1_9(self): + outline.config.set("outline", "spec_version", "1.9") + root = self._makeSpec() + self.assertIsNone(root.find("job/layers/layer/timeout")) + self.assertIsNone(root.find("job/layers/layer/timeout_llu")) + self.assertIsNone(root.find("job/priority")) + + def test_1_10(self): + outline.config.set("outline", "spec_version", "1.10") + root = self._makeSpec() + self.assertEqual(root.find("job/layers/layer/timeout").text, "420") + self.assertEqual(root.find("job/layers/layer/timeout_llu").text, "4200") + self.assertIsNone(root.find("job/priority")) + + def test_1_11(self): + outline.config.set("outline", "spec_version", "1.11") + root = self._makeSpec() + self.assertEqual(root.find("job/layers/layer/timeout").text, "420") + self.assertEqual(root.find("job/layers/layer/timeout_llu").text, "4200") + self.assertEqual(root.find("job/priority").text, "42") diff --git a/requirements.txt b/requirements.txt index fc1123fad..f5c9a537f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ futures==3.2.0;python_version<"3.0" grpcio==1.16.0 grpcio-tools==1.16.0 mock==2.0.0 +packaging==20.9 pathlib==1.0.1;python_version<"3.4" psutil==5.6.6 pyfakefs==3.6 From 00963b10de8e23f6d1e37563bf7b8a45c37605d0 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 16 Feb 2021 21:45:15 +0000 Subject: [PATCH 057/277] Notes from Feb 3 TSC meeting. (#923) --- tsc/meetings/2021-02-03.md | 76 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 tsc/meetings/2021-02-03.md diff --git a/tsc/meetings/2021-02-03.md b/tsc/meetings/2021-02-03.md new file mode 100644 index 000000000..a2a5e4b2b --- /dev/null +++ b/tsc/meetings/2021-02-03.md @@ -0,0 +1,76 @@ +# OpenCue TSC Meeting Notes 3 Feb 2021 + +Secretary: Brian Cipriano + +TSC Attendees: + +* [ ] Matt Chambers +* [x] Brian Cipriano +* [x] Greg Denton +* [x] Ben Dines +* [ ] Dave Fellows +* [x] Daniel Neilson +* [ ] Alex Schworer +* [ ] Erik Strauss +* [x] Lars van der Bijl + +Committer Attendees: + +* [ ] Christian Smith +* [x] Diego Tavares +* [x] Idris Miles + +Agenda/Notes: + +* Goals for 2021 + * User survey + * Still waiting on results. + * New user UX + * Split demo_data.sql + * Issue filed: https://github.com/AcademySoftwareFoundation/OpenCue/issues/891 + * Keep using Docker compose? + * Let's look at survey responses to see if this is a useful tool still. + * Review Daniel's writeup for new action items + * Fix cuebot:latest tag for sandbox setup + * podman issue + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/898 + * This appears to be related to some Dockerfile syntax we're using which podman doesn't + support. + * Docs refresh + * API reference + * Existing issue: https://github.com/AcademySoftwareFoundation/OpenCue/issues/559 + * We are able to generate HTML, we just need to publish it somewhere. + * This tool also supports versioning in theory, we could have a dropdown where users can + select a specific OpenCue version to view reference of. Let's start by publishing the + latest version, then go from there. + * pyoutline examples + * Existing issue: https://github.com/AcademySoftwareFoundation/OpenCue/issues/177 + * Needs more work. + * These are all listed in the Roadmap now. + * CII badge updates + * Brian: done a lot of work to finish linting all Python code. pycue, pyoutline, cuegui all + done, a few components left. + * Drop Oracle support + * Existing issue: https://github.com/AcademySoftwareFoundation/OpenCue/issues/840 + * Needs more work. + * GPU support + * Existing issue: https://github.com/AcademySoftwareFoundation/OpenCue/issues/459 + * Linux-specific PR: https://github.com/AcademySoftwareFoundation/OpenCue/pull/760 + * PR is close but was filed a while ago, so needs a merge from master and resolve conflicts. + * GUI to add new shows + * Filed issue: https://github.com/AcademySoftwareFoundation/OpenCue/issues/892 + * Discussion ongoing. + * Expand DCC plugins + * Houdini Tops scheduler node could be a good target. + * Logging solution + * PR out with Loki solution: https://github.com/AcademySoftwareFoundation/OpenCue/pull/880 + * Needs review. + * CSP terraform docs + * Brian has some initial stuff from the Google side, need to take some time to see what's + there and plan next steps. Will start email thread once that's done. +* Other current work updates + * Lars + * Adding priority to pyoutline. + * PR: https://github.com/AcademySoftwareFoundation/OpenCue/pull/625 + * Idris + * Node graph PR: including vendor icons? Otherwise, PR ready for review From 58c81fb5267df2bb65985915e0b0c07c6ab0c2da Mon Sep 17 00:00:00 2001 From: Lars van der Bijl <285658+larsbijl@users.noreply.github.com> Date: Fri, 19 Feb 2021 17:55:14 +0000 Subject: [PATCH 058/277] Remove special case for playblast. (#912) --- rqd/rqd/rqconstants.py | 3 --- rqd/rqd/rqmachine.py | 2 -- 2 files changed, 5 deletions(-) diff --git a/rqd/rqd/rqconstants.py b/rqd/rqd/rqconstants.py index 010fbb964..651e61337 100644 --- a/rqd/rqd/rqconstants.py +++ b/rqd/rqd/rqconstants.py @@ -114,7 +114,6 @@ OVERRIDE_NIMBY = None # True to turn on, False to turn off OVERRIDE_HOSTNAME = None # Force to use this hostname ALLOW_GPU = False -ALLOW_PLAYBLAST = False LOAD_MODIFIER = 0 # amount to add/subtract from load if subprocess.getoutput('/bin/su --help').find('session-command') != -1: @@ -156,8 +155,6 @@ OVERRIDE_HOSTNAME = config.get(__section, "OVERRIDE_HOSTNAME") if config.has_option(__section, "GPU"): ALLOW_GPU = config.getboolean(__section, "GPU") - if config.has_option(__section, "PLAYBLAST"): - ALLOW_PLAYBLAST = config.getboolean(__section, "PLAYBLAST") if config.has_option(__section, "LOAD_MODIFIER"): LOAD_MODIFIER = config.getint(__section, "LOAD_MODIFIER") if config.has_option(__section, "RQD_USE_IP_AS_HOSTNAME"): diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 68a450ba8..5391e40d0 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -309,8 +309,6 @@ def __getGpuValues(self): if not hasattr(self, 'gpuNotSupported'): if not hasattr(self, 'gpuResults'): self.gpuResults = {'total': 0, 'free': 0, 'updated': 0} - if rqd.rqconstants.ALLOW_PLAYBLAST and not rqd.rqconstants.ALLOW_GPU: - return {'total': 262144, 'free': 262144, 'updated': 0} if not rqd.rqconstants.ALLOW_GPU: self.gpuNotSupported = True return self.gpuResults From 5729fb2fdbfc43add031eff5175fd43ec9128674 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 22 Feb 2021 16:57:37 +0000 Subject: [PATCH 059/277] Lint CueSubmit code. (#920) --- ci/run_python_tests.sh | 3 ++ cuesubmit/cuesubmit/Config.py | 15 +++--- cuesubmit/cuesubmit/Constants.py | 5 ++ cuesubmit/cuesubmit/JobTypes.py | 11 +++-- cuesubmit/cuesubmit/Layer.py | 6 ++- cuesubmit/cuesubmit/Submission.py | 30 +++++++----- cuesubmit/cuesubmit/Util.py | 15 ++++-- cuesubmit/cuesubmit/Validators.py | 27 ++++++----- cuesubmit/cuesubmit/__main__.py | 9 +++- cuesubmit/cuesubmit/ui/Command.py | 24 ++++++++-- cuesubmit/cuesubmit/ui/Frame.py | 18 +++++++ cuesubmit/cuesubmit/ui/Job.py | 29 ++++++++++-- cuesubmit/cuesubmit/ui/SettingsWidgets.py | 44 +++++++++++++++-- cuesubmit/cuesubmit/ui/Style.py | 21 ++++++++ cuesubmit/cuesubmit/ui/Submit.py | 58 ++++++++++++++++++----- cuesubmit/cuesubmit/ui/Widgets.py | 53 ++++++++++++++++----- cuesubmit/cuesubmit/ui/__init__.py | 13 +++++ cuesubmit/tests/Config_tests.py | 3 ++ cuesubmit/tests/Layer_tests.py | 3 ++ cuesubmit/tests/Submission_tests.py | 29 +++++++----- cuesubmit/tests/Validators_tests.py | 7 ++- 21 files changed, 333 insertions(+), 90 deletions(-) diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index 7f0e9c0a4..c65436072 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -34,5 +34,8 @@ if [[ "$1" == "--lint" ]]; then cd cuegui && PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main cuegui --ignore=cuegui/images,cuegui/images/crystal && cd .. cd cuegui && PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests && cd .. + + cd cuesubmit && PYTHONPATH=../pycue:../pyoutline python -m pylint --rcfile=../ci/pylintrc_main cuesubmit && cd .. + cd cuesubmit && PYTHONPATH=../pycue:../pyoutline python -m pylint --rcfile=../ci/pylintrc_test tests && cd .. fi diff --git a/cuesubmit/cuesubmit/Config.py b/cuesubmit/cuesubmit/Config.py index 97efb9a44..99f76242c 100644 --- a/cuesubmit/cuesubmit/Config.py +++ b/cuesubmit/cuesubmit/Config.py @@ -13,22 +13,26 @@ # limitations under the License. +"""Provides custom configuration to override default CueSubmit functionality. + +Uses a YAML config file to override Constant.py values. Path is specified using the +"CUESUBMIT_CONFIG_FILE" environment variable. An example config file is contained in the +top level cuesubmit folder.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import + import os import yaml -""" -Overwrite Constant.py values with a yaml config file, with it's path specified with the -"CUESUBMIT_CONFIG_FILE" environment variable. An example config file is contained in the -top level cuesubmit folder. -""" CONFIG_FILE_ENV_VAR = 'CUESUBMIT_CONFIG_FILE' def getConfigValues(): + """Reads the config file from disk and returns the values it defines.""" configData = {} configFile = os.environ.get(CONFIG_FILE_ENV_VAR) if configFile and os.path.exists(configFile): @@ -43,4 +47,3 @@ def getConfigValues(): class CuesubmitConfigError(Exception): """Thrown when an error occurs reading the config file.""" - pass diff --git a/cuesubmit/cuesubmit/Constants.py b/cuesubmit/cuesubmit/Constants.py index ef876a104..123f32723 100644 --- a/cuesubmit/cuesubmit/Constants.py +++ b/cuesubmit/cuesubmit/Constants.py @@ -13,6 +13,11 @@ # limitations under the License. +"""Constants used throughout the code. + +Some values can be overridden by custom config, see Config.py.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import diff --git a/cuesubmit/cuesubmit/JobTypes.py b/cuesubmit/cuesubmit/JobTypes.py index a0918f8bb..3c867b921 100644 --- a/cuesubmit/cuesubmit/JobTypes.py +++ b/cuesubmit/cuesubmit/JobTypes.py @@ -13,6 +13,11 @@ # limitations under the License. +"""Base Job Types available in the UI. + +Plugin apps can subclass this to change out the mapping to enable customized settings widgets.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -23,9 +28,9 @@ class JobTypes(object): """Base Job Types available in the UI. - Plugin apps can subclass this to change out the mapping - to enable customized settings widgets. - """ + + Plugin apps can subclass this to change out the mapping to enable customized + settings widgets.""" SHELL = 'Shell' MAYA = 'Maya' diff --git a/cuesubmit/cuesubmit/Layer.py b/cuesubmit/cuesubmit/Layer.py index ad88384c6..ccb0e034f 100644 --- a/cuesubmit/cuesubmit/Layer.py +++ b/cuesubmit/cuesubmit/Layer.py @@ -13,13 +13,17 @@ # limitations under the License. +"""Data object for storing settings about a Layer.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import - from builtins import str from builtins import object + + class DependType(object): """Types of Dependencies available in the UI.""" diff --git a/cuesubmit/cuesubmit/Submission.py b/cuesubmit/cuesubmit/Submission.py index e6d66402c..94a1f97b9 100644 --- a/cuesubmit/cuesubmit/Submission.py +++ b/cuesubmit/cuesubmit/Submission.py @@ -13,21 +13,25 @@ # limitations under the License. +"""Code for constructing a job submission and sending it to the Cuebot.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import from builtins import str -from cuesubmit import Constants -from cuesubmit import JobTypes import outline import outline.cuerun import outline.modules.shell +from cuesubmit import Constants +from cuesubmit import JobTypes + def buildMayaCmd(layerData): - """From a layer, build a Maya Render command.""" + """From a layer, builds a Maya Render command.""" camera = layerData.cmd.get('camera') mayaFile = layerData.cmd.get('mayaFile') if not mayaFile: @@ -41,7 +45,7 @@ def buildMayaCmd(layerData): def buildNukeCmd(layerData): - """From a layer, build a Nuke Render command.""" + """From a layer, builds a Nuke Render command.""" writeNodes = layerData.cmd.get('writeNodes') nukeFile = layerData.cmd.get('nukeFile') if not nukeFile: @@ -55,13 +59,13 @@ def buildNukeCmd(layerData): def buildBlenderCmd(layerData): - """From a layer, build a Blender render command.""" + """From a layer, builds a Blender render command.""" blenderFile = layerData.cmd.get('blenderFile') outputPath = layerData.cmd.get('outputPath') outputFormat = layerData.cmd.get('outputFormat') if not blenderFile: raise ValueError('No Blender file provided. Cannot submit job.') - + renderCommand = '{renderCmd} -b -noaudio {blenderFile}'.format( renderCmd=Constants.BLENDER_RENDER_CMD, blenderFile=blenderFile) if outputPath: @@ -74,7 +78,8 @@ def buildBlenderCmd(layerData): def buildLayer(layerData, command, lastLayer=None): - """Create a PyOutline Layer for the given layerData. + """Creates a PyOutline Layer for the given layerData. + @type layerData: ui.Layer.LayerData @param layerData: layer data from the ui @type command: str @@ -82,10 +87,7 @@ def buildLayer(layerData, command, lastLayer=None): @type lastLayer: outline.layer.Layer @param lastLayer: layer that this new layer should be dependent on if dependType is set. """ - if float(layerData.cores) >= 2: - threadable = True - else: - threadable = False + threadable = float(layerData.cores) >= 2 layer = outline.modules.shell.Shell( layerData.name, command=command.split(), chunk=layerData.chunk, threads=float(layerData.cores), range=str(layerData.layerRange), threadable=threadable) @@ -102,26 +104,30 @@ def buildLayer(layerData, command, lastLayer=None): def buildMayaLayer(layerData, lastLayer): + """Builds a PyOutline layer running a Maya command.""" mayaCmd = buildMayaCmd(layerData) return buildLayer(layerData, mayaCmd, lastLayer) def buildNukeLayer(layerData, lastLayer): + """Builds a PyOutline layer running a Nuke command.""" nukeCmd = buildNukeCmd(layerData) return buildLayer(layerData, nukeCmd, lastLayer) def buildBlenderLayer(layerData, lastLayer): + """Builds a PyOutline layer running a Blender command.""" blenderCmd = buildBlenderCmd(layerData) return buildLayer(layerData, blenderCmd, lastLayer) def buildShellLayer(layerData, lastLayer): + """Builds a PyOutline layer running a shell command.""" return buildLayer(layerData, layerData.cmd['commandTextBox'], lastLayer) def submitJob(jobData): - """Submit the job using the PyOutline API.""" + """Submits the job using the PyOutline API.""" ol = outline.Outline( jobData['name'], shot=jobData['shot'], show=jobData['show'], user=jobData['username']) lastLayer = None diff --git a/cuesubmit/cuesubmit/Util.py b/cuesubmit/cuesubmit/Util.py index 58762ddd1..1b25da8ef 100644 --- a/cuesubmit/cuesubmit/Util.py +++ b/cuesubmit/cuesubmit/Util.py @@ -12,6 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. + +"""Utility functions used throughout the application.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -23,26 +27,27 @@ def getLimits(): - """Return a list of limit names from cuebot.""" + """Returns a list of limit names from cuebot.""" return [limit.name() for limit in opencue.api.getLimits()] def getServices(): - """Return a list of service names from cuebot.""" + """Returns a list of service names from cuebot.""" return [service.name() for service in opencue.api.getDefaultServices()] def getShows(): - """Return a list of show names from cuebot.""" + """Returns a list of show names from cuebot.""" return [show.name() for show in opencue.api.getShows()] def getAllocations(): - """Return a list of Allocations from cuebot.""" + """Returns a list of Allocations from cuebot.""" return opencue.api.getAllocations() def getPresetFacility(): + """Returns the default facility defined via environment variable, if set.""" if os.getenv('RENDER_TO', None): return os.environ['RENDER_TO'] if os.getenv('FACILITY', None): @@ -51,7 +56,7 @@ def getPresetFacility(): def getFacilities(allocations): - """Return a list of facility names from the allocations.""" + """Returns a list of facility names from the allocations.""" default_facilities = [Constants.DEFAULT_FACILITY_TEXT] facilities = set(alloc.data.facility for alloc in allocations) return default_facilities + list(facilities) diff --git a/cuesubmit/cuesubmit/Validators.py b/cuesubmit/cuesubmit/Validators.py index 35b89753e..540f92e21 100644 --- a/cuesubmit/cuesubmit/Validators.py +++ b/cuesubmit/cuesubmit/Validators.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Validator functions.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -21,43 +24,43 @@ def matchLettersAndNumbersOnly(value): - """Match strings of letters and numbers.""" - if re.match('^[a-zA-Z0-9]+$', value): + """Matches strings of letters and numbers.""" + if re.match(r'^[a-zA-Z0-9]+$', value): return True return False def matchNoSpecialCharactersOnly(value): - """Match strings containing letters, numbers, '.', '-','_', \t and \n""" - if re.match('^[a-zA-Z0-9.\-_\s]+$', value): + """Matches strings containing letters, numbers, '.', '-','_', \t and \n""" + if re.match(r'^[a-zA-Z0-9.\-_\s]+$', value): return True return False def matchLettersOnly(value): - """Match strings container letters only.""" - if re.match('^[a-zA-Z]+$', value): + """Matches strings container letters only.""" + if re.match(r'^[a-zA-Z]+$', value): return True return False def matchNoSpaces(value): - """Match strings with no spaces.""" - if re.search('\s', value): + """Matches strings with no spaces.""" + if re.search(r'\s', value): return False return True def matchNumbersOnly(value): - """Match strings with numbers and '.' only.""" - if re.match('^[0-9.]+$', value): + """Matches strings with numbers and '.' only.""" + if re.match(r'^[0-9.]+$', value): return True return False def matchPositiveIntegers(value): - """Match integers greater than 0.""" - if re.match('^[0-9]+$', value) and int(value) >= 1: + """Matches integers greater than 0.""" + if re.match(r'^[0-9]+$', value) and int(value) >= 1: return True return False diff --git a/cuesubmit/cuesubmit/__main__.py b/cuesubmit/cuesubmit/__main__.py index 75415fc9e..50c111ab0 100644 --- a/cuesubmit/cuesubmit/__main__.py +++ b/cuesubmit/cuesubmit/__main__.py @@ -15,6 +15,9 @@ # limitations under the License. +"""Entrypoint for the CueSubmit application.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -29,13 +32,14 @@ class CueSubmitApp(QtWidgets.QApplication): - """Standalone submission application""" + """Standalone submission application.""" def __init__(self, args): super(CueSubmitApp, self).__init__(args) self.mainWindow = CueSubmitMainWindow(Constants.SUBMIT_APP_WINDOW_TITLE) def startup(self): + """Initializes the application and makes it appear on screen.""" self.setApplicationName(Constants.SUBMIT_APP_WINDOW_TITLE) Style.init() self.mainWindow.show() @@ -43,7 +47,7 @@ def startup(self): class CueSubmitMainWindow(QtWidgets.QMainWindow): - """Main Window object for the standalone submission""" + """Main Window object for the standalone submission.""" def __init__(self, name, *args, **kwargs): super(CueSubmitMainWindow, self).__init__(*args, **kwargs) @@ -59,6 +63,7 @@ def __init__(self, name, *args, **kwargs): def main(): + """Entrypoint for the CueSubmit application.""" app = CueSubmitApp(sys.argv) app.startup() app.exec_() diff --git a/cuesubmit/cuesubmit/ui/Command.py b/cuesubmit/cuesubmit/ui/Command.py index fc987b1b7..6377f2ce3 100644 --- a/cuesubmit/cuesubmit/ui/Command.py +++ b/cuesubmit/cuesubmit/ui/Command.py @@ -1,3 +1,21 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Help Widget that contains a text box for entering a shell command.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -8,9 +26,7 @@ class CueCommandWidget(Widgets.CueHelpWidget): - """Help Widget that contains a text box for entering a - shell command. - """ + """Help Widget that contains a text box for entering a shell command.""" helpText = 'Enter a shell command to run' textChanged = QtCore.Signal() @@ -22,6 +38,7 @@ def __init__(self, parent=None): self.setupConnections() def setupConnections(self): + """Sets up widget signals.""" self.commandTextBox.commandBox.textChanged.connect(self.textChanged.emit) def setText(self, text): @@ -60,6 +77,7 @@ def __init__(self, *args, **kwargs): self.setupUi() def setupUi(self): + """Creates the widget layout.""" self.setLayout(self.mainLayout) self.mainLayout.addWidget(self.label, 0, 0, 1, 1) self.mainLayout.addWidget(self.commandBox, 1, 0, 1, 4) diff --git a/cuesubmit/cuesubmit/ui/Frame.py b/cuesubmit/cuesubmit/ui/Frame.py index 8c2d209b0..4dec785ed 100644 --- a/cuesubmit/cuesubmit/ui/Frame.py +++ b/cuesubmit/cuesubmit/ui/Frame.py @@ -1,3 +1,21 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Widget for entering a frame spec.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import diff --git a/cuesubmit/cuesubmit/ui/Job.py b/cuesubmit/cuesubmit/ui/Job.py index 25dedc869..c9ac15bca 100644 --- a/cuesubmit/cuesubmit/ui/Job.py +++ b/cuesubmit/cuesubmit/ui/Job.py @@ -1,3 +1,21 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Widget showing a tree view of a job and its layers.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -37,6 +55,7 @@ def __init__(self, parent=None): self.selected = self.jobRow.child(0) def setupUi(self): + """Creates the widget layout.""" self.setLayout(self.mainLayout) self.table.setModel(self.model) header = self.table.header() @@ -52,6 +71,7 @@ def setupUi(self): self.initLayers() def setupConnections(self): + """Sets up widget signals.""" self.table.selectionModel().selectionChanged.connect(self.updateSelection) self.addLayerButton.clicked.connect(self.newLayer) self.deleteLayerButton.clicked.connect(self.removeRow) @@ -59,6 +79,7 @@ def setupConnections(self): self.downButton.clicked.connect(self.moveDown) def setupButtons(self): + """Creates buttons working with job layers.""" self.addLayerButton.setAccessibleName('editLayer') self.addLayerButton.setText('+') self.deleteLayerButton.setAccessibleName('editLayer') @@ -170,8 +191,7 @@ def getCurrentRow(self): item = self.getSelectedItem() if item: return item.row() - else: - return None + return None def getDependOnItem(self): """Return the layer that the current layer depends on. @@ -181,8 +201,7 @@ def getDependOnItem(self): currentRow = self.getCurrentRow() if currentRow == 0: return None - else: - return self.jobRow.child(currentRow - 1, 0) + return self.jobRow.child(currentRow - 1, 0) def getSelectedItem(self): """Return the selected item from the tree. @@ -281,6 +300,7 @@ def updateSelection(self, selectionItem): class CueJobTree(QtWidgets.QTreeView): + """Inner table for displaying job data.""" def __init__(self, parent=None): super(CueJobTree, self).__init__(parent=parent) @@ -290,6 +310,7 @@ def __init__(self, parent=None): class CueJobModel(QtGui.QStandardItemModel): + """Data model for a job, in Qt format.""" def __init__(self): super(CueJobModel, self).__init__() diff --git a/cuesubmit/cuesubmit/ui/SettingsWidgets.py b/cuesubmit/cuesubmit/ui/SettingsWidgets.py index c616844c4..43b7ed7dd 100644 --- a/cuesubmit/cuesubmit/ui/SettingsWidgets.py +++ b/cuesubmit/cuesubmit/ui/SettingsWidgets.py @@ -1,3 +1,21 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Widgets to provide application specific settings.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -10,7 +28,7 @@ class BaseSettingsWidget(QtWidgets.QWidget): - """Swappable widget to provide application specific settings. """ + """Swappable widget to provide application specific settings.""" dataChanged = QtCore.Signal(object) @@ -32,6 +50,7 @@ def setCommandData(self, commandData): class InMayaSettings(BaseSettingsWidget): """Settings widget to be used when launching from within Maya.""" + # pylint: disable=keyword-arg-before-vararg,unused-argument def __init__(self, cameras=None, filename=None, parent=None, *args, **kwargs): super(InMayaSettings, self).__init__(parent=parent) self.mayaFileInput = Widgets.CueLabelLineEdit('Maya File:', filename) @@ -40,6 +59,7 @@ def __init__(self, cameras=None, filename=None, parent=None, *args, **kwargs): self.setupUi() def setupUi(self): + """Creates the Maya-specific widget layout.""" self.mainLayout.addWidget(self.mayaFileInput) self.selectorLayout.addWidget(self.cameraSelector) self.selectorLayout.addSpacerItem(Widgets.CueSpacerItem(Widgets.SpacerTypes.HORIZONTAL)) @@ -59,6 +79,7 @@ def getCommandData(self): class BaseMayaSettings(BaseSettingsWidget): """Standard Maya settings widget to be used from outside Maya.""" + # pylint: disable=keyword-arg-before-vararg,unused-argument def __init__(self, parent=None, *args, **kwargs): super(BaseMayaSettings, self).__init__(parent=parent) self.mayaFileInput = Widgets.CueLabelLineEdit('Maya File:') @@ -66,9 +87,11 @@ def __init__(self, parent=None, *args, **kwargs): self.setupConnections() def setupUi(self): + """Creates the widget layout with a single input for the path to the Maya scene.""" self.mainLayout.addWidget(self.mayaFileInput) def setupConnections(self): + """Sets up widget signals.""" self.mayaFileInput.lineEdit.textChanged.connect(self.dataChanged.emit) def setCommandData(self, commandData): @@ -83,6 +106,7 @@ def getCommandData(self): class InNukeSettings(BaseSettingsWidget): """Settings widget to be used when launching from within Nuke.""" + # pylint: disable=keyword-arg-before-vararg,unused-argument def __init__(self, writeNodes=None, filename=None, parent=None, *args, **kwargs): super(InNukeSettings, self).__init__(parent=parent) self.fileInput = Widgets.CueLabelLineEdit('Nuke File:', filename) @@ -92,6 +116,7 @@ def __init__(self, writeNodes=None, filename=None, parent=None, *args, **kwargs) self.setupUi() def setupUi(self): + """Creates the Nuke-specific widget layout.""" self.mainLayout.addWidget(self.fileInput) self.selectorLayout.addWidget(self.writeNodeSelector) self.selectorLayout.addSpacerItem(Widgets.CueSpacerItem(Widgets.SpacerTypes.HORIZONTAL)) @@ -111,6 +136,7 @@ def getCommandData(self): class BaseNukeSettings(BaseSettingsWidget): """Standard Nuke settings widget to be used from outside Nuke.""" + # pylint: disable=keyword-arg-before-vararg,unused-argument def __init__(self, parent=None, *args, **kwargs): super(BaseNukeSettings, self).__init__(parent=parent) self.fileInput = Widgets.CueLabelLineEdit('Nuke File:') @@ -118,9 +144,11 @@ def __init__(self, parent=None, *args, **kwargs): self.setupConnections() def setupUi(self): + """Creates the widget layout with a single input for the path to the Nuke script.""" self.mainLayout.addWidget(self.fileInput) def setupConnections(self): + """Sets up widget signals.""" self.fileInput.lineEdit.textChanged.connect(self.dataChanged.emit) def setCommandData(self, commandData): @@ -135,6 +163,7 @@ def getCommandData(self): class ShellSettings(BaseSettingsWidget): """Basic settings widget for submitting simple shell commands.""" + # pylint: disable=keyword-arg-before-vararg,unused-argument def __init__(self, parent=None, *args, **kwargs): super(ShellSettings, self).__init__(parent=parent) @@ -144,9 +173,11 @@ def __init__(self, parent=None, *args, **kwargs): self.setupConnections() def setupUi(self): + """Creates the widget layout with a single input for the shell command.""" self.mainLayout.addWidget(self.commandTextBox) def setupConnections(self): + """Sets up widget signals.""" self.commandTextBox.textChanged.connect(lambda: self.dataChanged.emit(None)) def getCommandData(self): @@ -159,6 +190,7 @@ def setCommandData(self, commandData): class BaseBlenderSettings(BaseSettingsWidget): """Standard Blender settings widget to be used from outside Blender.""" + # pylint: disable=keyword-arg-before-vararg,unused-argument def __init__(self, parent=None, *args, **kwargs): super(BaseBlenderSettings, self).__init__(parent=parent) self.fileInput = Widgets.CueLabelLineEdit('Blender File:') @@ -172,22 +204,24 @@ def __init__(self, parent=None, *args, **kwargs): self.outputLayout = QtWidgets.QHBoxLayout() self.setupUi() self.setupConnections() - + def setupUi(self): + """Creates the Blender-specific widget layout.""" self.mainLayout.addWidget(self.fileInput) self.mainLayout.addLayout(self.outputLayout) self.outputLayout.addWidget(self.outputPath) self.outputLayout.addWidget(self.outputSelector) - + def setupConnections(self): + """Sets up widget signals.""" self.fileInput.lineEdit.textChanged.connect(self.dataChanged.emit) self.outputPath.lineEdit.textChanged.connect(self.dataChanged.emit) - + def setCommandData(self, commandData): self.fileInput.setText(commandData.get('nukeFile', '')) self.outputPath.setText(commandData.get('outputPath', '')) self.outputSelector.setChecked(commandData.get('outputFormat', '')) - + def getCommandData(self): return { 'blenderFile': self.fileInput.text(), diff --git a/cuesubmit/cuesubmit/ui/Style.py b/cuesubmit/cuesubmit/ui/Style.py index 2effc446e..6e3211b94 100644 --- a/cuesubmit/cuesubmit/ui/Style.py +++ b/cuesubmit/cuesubmit/ui/Style.py @@ -1,3 +1,21 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Widget styling information.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -166,11 +184,14 @@ UNDERLINE_HIGHLIGHT = 'background-color: rgb(10, 90, 240);' +# pylint: disable=global-variable-undefined def setFont(font): """sets the application font""" global Font Font = font + # pylint: disable=no-member QtGui.qApp.setFont(font) + # pylint: enable=no-member def init(): diff --git a/cuesubmit/cuesubmit/ui/Submit.py b/cuesubmit/cuesubmit/ui/Submit.py index 809323119..fb0c6de9e 100644 --- a/cuesubmit/cuesubmit/ui/Submit.py +++ b/cuesubmit/cuesubmit/ui/Submit.py @@ -1,3 +1,21 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Widgets for cancel and submit buttons.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -39,14 +57,17 @@ def __init__(self, parent=None): self.setupConnections() def setupConnections(self): + """Sets up widget signals.""" self.submitButton.pressed.connect(self.submitPressed) self.cancelButton.pressed.connect(self.cancelPressed) def submitPressed(self): + """Handler for when submit button has been pressed.""" self.state = "submitted" self.submitted.emit() def cancelPressed(self): + """Handler for when cancel button has been pressed.""" self.state = "cancelled" self.cancelled.emit() @@ -54,7 +75,9 @@ def cancelPressed(self): class CueSubmitWidget(QtWidgets.QWidget): """Central widget for submission.""" - def __init__(self, settingsWidgetType, jobTypes=JobTypes.JobTypes, parent=None, *args, **kwargs): + # pylint: disable=keyword-arg-before-vararg + def __init__( + self, settingsWidgetType, jobTypes=JobTypes.JobTypes, parent=None, *args, **kwargs): super(CueSubmitWidget, self).__init__(parent) self.startupErrors = list() self.skipDataChangedEvent = False @@ -107,8 +130,9 @@ def __init__(self, settingsWidgetType, jobTypes=JobTypes.JobTypes, parent=None, ) shows = Util.getShows() if not shows: - self.startupErrors.append("No shows exist yet. Please create some or contact your OpenCue administrator to create one!\n" +\ - "You won't be able to submit job for non-existent show!\n") + self.startupErrors.append( + "No shows exist yet. Please create some or contact your OpenCue administrator " + "to create one!\nYou won't be able to submit a job for a non-existent show!\n") shows = [''] # to allow building UI self.showSelector = Widgets.CueSelectPulldown( 'Show:', shows[0], @@ -183,17 +207,21 @@ def __init__(self, settingsWidgetType, jobTypes=JobTypes.JobTypes, parent=None, self.jobDataChanged() def showEvent(self, event): + del event + if self.startupErrors: for startupError in self.startupErrors: msgBox = Widgets.CueMessageBox(startupError, title="No Shows Exist", parent=self) msgBox.show() - msgBox.centerOnScreen() # explicitly center on desktop center as parent is not shown yet + # Explicitly center on desktop center as parent is not shown yet. + msgBox.centerOnScreen() - # Raise at least one of the errors so the user gets feedback in the event the GUI wasn't built - # or shown properly. + # Raise at least one of the errors so the user gets feedback in the event the GUI + # wasn't built or shown properly. raise opencue.exception.CueException(self.startupErrors[0]) def setupConnections(self): + """Sets up widget signals.""" self.submitButtons.cancelled.connect(self.cancel) self.submitButtons.submitted.connect(self.submit) self.jobTreeWidget.selectionChanged.connect(self.jobLayerSelectionChanged) @@ -209,6 +237,7 @@ def setupConnections(self): self.dependSelector.optionsMenu.triggered.connect(self.dependencyChanged) def setupUi(self): + """Creates the widget layout.""" self.setLayout(self.mainLayout) self.scrollingLayout.addWidget(self.titleLogo) @@ -275,7 +304,8 @@ def getJobData(self): 'layers': self.jobTreeWidget.getAllLayers() } facility = self.facilitySelector.text() - jobData['facility'] = facility if facility and facility != Constants.DEFAULT_FACILITY_TEXT else None + jobData['facility'] = ( + facility if facility and facility != Constants.DEFAULT_FACILITY_TEXT else None) return jobData def jobLayerSelectionChanged(self, layerObject): @@ -345,10 +375,12 @@ def updateSettingsWidget(self, layerType): self.settingsLayout.addWidget(self.settingsWidget) def errorInJobData(self, message): + """Displays an error box about invalid job data.""" Widgets.CueMessageBox(message, title="Error in Job Data", parent=self).show() return False def validate(self, jobData): + """Validates job data.""" errMessage = 'ERROR: Job not submitted!\n' if not self.jobNameInput.validateText(): return self.errorInJobData(errMessage + 'Invalid job name.') @@ -379,10 +411,12 @@ def validate(self, jobData): def updateCompleters(self): """Update the line edit completers after submission.""" - self.jobNameInput.lineEdit.completerStrings = self.getFilteredHistorySetting('submit/jobName') - self.shotInput.lineEdit.completerStrings = self.getFilteredHistorySetting('submit/shotName') - self.layerNameInput.lineEdit.completerStrings = self.getFilteredHistorySetting('submit/layerName') - + self.jobNameInput.lineEdit.completerStrings = self.getFilteredHistorySetting( + 'submit/jobName') + self.shotInput.lineEdit.completerStrings = self.getFilteredHistorySetting( + 'submit/shotName') + self.layerNameInput.lineEdit.completerStrings = self.getFilteredHistorySetting( + 'submit/layerName') def getFilteredHistorySetting(self, setting): """Return a list of strings for the provided setting. @@ -392,6 +426,7 @@ def getFilteredHistorySetting(self, setting): @rtype: list @return: A list of strings of setting values. """ + # pylint: disable=broad-except try: return [str(value) for value in self.getHistorySetting(setting) if value is not None] except Exception: @@ -447,6 +482,7 @@ def updateSettingItem(self, setting, value, maxSettings=10): @type maxSettings: int @param maxSettings: maximum number of items to keep a history of """ + # pylint: disable=broad-except try: if not value: return diff --git a/cuesubmit/cuesubmit/ui/Widgets.py b/cuesubmit/cuesubmit/ui/Widgets.py index 7b21f5473..b1b6f0b54 100644 --- a/cuesubmit/cuesubmit/ui/Widgets.py +++ b/cuesubmit/cuesubmit/ui/Widgets.py @@ -1,3 +1,21 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Collection of utility widgets used throughout the main widget code.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -42,6 +60,7 @@ def __init__(self, labelText=None, defaultText='', tooltip=None, validators=None self.setAutoFillBackground(True) def setupUi(self): + """Creates the widget layout.""" self.setLayout(self.mainLayout) self.mainLayout.addWidget(self.label, 0, 0, 1, 1) self.mainLayout.addWidget(self.lineEdit, 1, 0, 1, 4) @@ -49,6 +68,7 @@ def setupUi(self): self.label.setStyleSheet(Style.LABEL_TEXT) def setupConnections(self): + """Sets up widget signals.""" self.lineEdit.textChanged.connect(self.validateText) self.lineEdit.focusChange.connect(self.textFocusChange) @@ -72,9 +92,8 @@ def validateText(self): if all(results): self.label.setStyleSheet(Style.LABEL_TEXT) return True - else: - self.label.setStyleSheet(Style.INVALID_TEXT) - return False + self.label.setStyleSheet(Style.INVALID_TEXT) + return False def text(self): """Return the current text. @@ -95,6 +114,7 @@ def __init__(self, defaultText=None, completerStrings=None, parent=None): self.index = -1 self.setSizePolicy(QtWidgets.QSizePolicy.MinimumExpanding, QtWidgets.QSizePolicy.Fixed) self.completer = QtWidgets.QCompleter() + # pylint: disable=c-extension-no-member try: self.completerModel = QtCore.QStringListModel() except AttributeError: @@ -150,7 +170,8 @@ def tabPressed(self): class CueSelectPulldown(QtWidgets.QWidget): """A button that acts like a dropdown and supports multiselect.""" - def __init__(self, labelText=None, emptyText='[None]', options=None, multiselect=True, parent=None): + def __init__( + self, labelText=None, emptyText='[None]', options=None, multiselect=True, parent=None): super(CueSelectPulldown, self).__init__(parent=parent) self.multiselect = multiselect self.emptyText = emptyText @@ -168,6 +189,7 @@ def __init__(self, labelText=None, emptyText='[None]', options=None, multiselect self.setupConnections() def setupUi(self): + """Creates the widget layout.""" self.mainLayout.setVerticalSpacing(1) self.mainLayout.addWidget(self.label, 0, 0, 1, 1) self.mainLayout.addWidget(self.toolButton, 1, 0, 1, 1) @@ -175,6 +197,7 @@ def setupUi(self): self.toolButton.setPopupMode(QtWidgets.QToolButton.InstantPopup) def setupConnections(self): + """Sets up widget signals.""" self.optionsMenu.triggered.connect(self.updateLabel) def setOptions(self, options): @@ -272,12 +295,14 @@ def __init__(self, label=None, parent=None): self.setupConnections() def setupUi(self): + """Creates the widget layout.""" self.setLayout(self.mainLayout) self.mainLayout.addWidget(self.label) self.mainLayout.addWidget(self.toggle) self.mainLayout.addSpacerItem(CueSpacerItem(SpacerTypes.HORIZONTAL)) def setupConnections(self): + """Sets up widget signals.""" self.toggle.valueChanged.connect(self.valueChanged.emit) self.toggle.sliderPressed.connect(self.sliderPressed.emit) self.toggle.sliderMoved.connect(self.sliderMoved.emit) @@ -300,6 +325,7 @@ def __init__(self, *args, **kwargs): self.setStyleSheet(Style.TOGGLE_DEFAULT) def setupConnections(self): + """Sets up widget signals.""" self.valueChanged.connect(self.change) self.sliderPressed.connect(self.toggle) @@ -350,6 +376,7 @@ def __init__(self, parent=None): self.setupHelpConnections() def setupHelpConnections(self): + """Sets up widget signal for the help button.""" self.helpButton.clicked.connect(self.toggleHelp) def setHelpText(self): @@ -417,18 +444,16 @@ def separatorLine(): class CueMessageBox(QtWidgets.QMessageBox): - ''' Display QMessageBox with message and OK button. + """A QMessageBox with message and OK button.""" + + def __init__(self, message, title=None, parent=None): + """ @type message: str @param message: error message @type title: str @param title: box title @type parent: QWidget - @param parent: parent object, used for centering, deleting - @type centerOnScreen: bool - @param centerOnScreen: useful mainly for rare cases that parent is not shown yet for centering on desktop - If parent is shown, QMessageBox gets centered into it properly. - ''' - def __init__(self, message, title=None, parent=None): + @param parent: parent object""" super(CueMessageBox, self).__init__(parent) self.setIcon(QtWidgets.QMessageBox.Information) @@ -437,8 +462,10 @@ def __init__(self, message, title=None, parent=None): self.setStandardButtons(QtWidgets.QMessageBox.Ok) def centerOnScreen(self): - ''' Useful mainly for rare cases that parent is not shown yet for centering on desktop - If parent is shown, QMessageBox gets centered into it properly.''' + """Centers the message box on screen. + + Useful mainly for rare cases that parent is not shown yet for centering on desktop. + If parent is shown, QMessageBox gets centered into it properly.""" size = self.size() desktopSize = QtWidgets.QDesktopWidget().screenGeometry() top = (desktopSize.height() / 2) - (size.height() / 2) diff --git a/cuesubmit/cuesubmit/ui/__init__.py b/cuesubmit/cuesubmit/ui/__init__.py index e69de29bb..d9c80f13d 100644 --- a/cuesubmit/cuesubmit/ui/__init__.py +++ b/cuesubmit/cuesubmit/ui/__init__.py @@ -0,0 +1,13 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/cuesubmit/tests/Config_tests.py b/cuesubmit/tests/Config_tests.py index 702573d75..12ce31fba 100644 --- a/cuesubmit/tests/Config_tests.py +++ b/cuesubmit/tests/Config_tests.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Tests for cuesubmit.Config""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import diff --git a/cuesubmit/tests/Layer_tests.py b/cuesubmit/tests/Layer_tests.py index b2efa99a7..7ba737530 100644 --- a/cuesubmit/tests/Layer_tests.py +++ b/cuesubmit/tests/Layer_tests.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Tests for cuesubmit.Layer""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import diff --git a/cuesubmit/tests/Submission_tests.py b/cuesubmit/tests/Submission_tests.py index a3b0d51b8..751df915a 100644 --- a/cuesubmit/tests/Submission_tests.py +++ b/cuesubmit/tests/Submission_tests.py @@ -13,17 +13,22 @@ # limitations under the License. +"""Tests for cuesubmit.Submission""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import -import mock import unittest +import mock + +import outline.depend + import cuesubmit.JobTypes import cuesubmit.Layer import cuesubmit.Submission -import outline.depend MAYA_LAYER_DATA = { @@ -67,9 +72,9 @@ def testSubmitMayaJob(self, launchMock): 'layers': [cuesubmit.Layer.LayerData.buildFactory(**MAYA_LAYER_DATA)], }) - outline = launchMock.call_args[0][0] - self.assertEqual(1, len(outline.get_layers())) - layer = outline.get_layer(MAYA_LAYER_DATA['name']) + ol = launchMock.call_args[0][0] + self.assertEqual(1, len(ol.get_layers())) + layer = ol.get_layer(MAYA_LAYER_DATA['name']) self.assertEqual(MAYA_LAYER_DATA['name'], layer.get_name()) self.assertEqual( [ @@ -90,9 +95,9 @@ def testSubmitNukeJob(self, launchMock): 'layers': [cuesubmit.Layer.LayerData.buildFactory(**NUKE_LAYER_DATA)], }) - outline = launchMock.call_args[0][0] - self.assertEqual(1, len(outline.get_layers())) - layer = outline.get_layer(NUKE_LAYER_DATA['name']) + ol = launchMock.call_args[0][0] + self.assertEqual(1, len(ol.get_layers())) + layer = ol.get_layer(NUKE_LAYER_DATA['name']) self.assertEqual(NUKE_LAYER_DATA['name'], layer.get_name()) self.assertEqual( [ @@ -153,8 +158,8 @@ def testSubmitJobWithFacility(self, launchMock): 'facility': 'my-facility' }) - outline = launchMock.call_args[0][0] - facility = outline.get_facility() + ol = launchMock.call_args[0][0] + facility = ol.get_facility() self.assertEqual('my-facility', facility) def testSubmitJobWithoutFacility(self, launchMock): @@ -166,8 +171,8 @@ def testSubmitJobWithoutFacility(self, launchMock): 'layers': [cuesubmit.Layer.LayerData.buildFactory(**NUKE_LAYER_DATA)] }) - outline = launchMock.call_args[0][0] - self.assertIsNone(outline.get_facility()) + ol = launchMock.call_args[0][0] + self.assertIsNone(ol.get_facility()) if __name__ == '__main__': diff --git a/cuesubmit/tests/Validators_tests.py b/cuesubmit/tests/Validators_tests.py index 00b99615f..0a5ef78eb 100644 --- a/cuesubmit/tests/Validators_tests.py +++ b/cuesubmit/tests/Validators_tests.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Tests for cuesubmit.Validators""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -29,6 +32,7 @@ moreThan3Chars, ) + class ValidatorsTests(unittest.TestCase): def testMatchLettersAndNumbersOnly(self): @@ -73,7 +77,8 @@ def testMatchNoSpaces(self): def testMatchNumbersOnly(self): self.assertTrue(matchNumbersOnly('0123')) self.assertTrue(matchNumbersOnly('3.14')) - self.assertTrue(matchNumbersOnly('800.555.555')) # bit weird, but thats how the function is written + # bit weird, but that's how the function is written + self.assertTrue(matchNumbersOnly('800.555.555')) self.assertFalse(matchNumbersOnly('')) From c918cec17d0d241f19c1513d5fc3f7f52673a8ed Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 22 Feb 2021 18:01:29 +0000 Subject: [PATCH 060/277] Lint RQD code. (#921) --- ci/run_python_tests.sh | 3 + rqd/rqd/__main__.py | 29 +++-- rqd/rqd/cuerqd.py | 177 ++++++++++++++++++++---------- rqd/rqd/rqconstants.py | 34 +++--- rqd/rqd/rqcore.py | 155 +++++++++++++++++--------- rqd/rqd/rqdservicers.py | 21 ++-- rqd/rqd/rqexceptions.py | 14 ++- rqd/rqd/rqmachine.py | 102 ++++++++++------- rqd/rqd/rqnetwork.py | 47 +++++--- rqd/rqd/rqnimby.py | 18 ++- rqd/rqd/rqswap.py | 34 +++--- rqd/rqd/rqutil.py | 29 ++--- rqd/tests/cuerqd_tests.py | 7 +- rqd/tests/rqconstants_tests.py | 4 +- rqd/tests/rqcore_tests.py | 25 +++-- rqd/tests/rqmachine_tests.py | 16 ++- rqd/tests/rqnimby_tests.py | 7 +- rqd/tests/test_cuebot_listener.py | 92 ++++++++++++---- 18 files changed, 537 insertions(+), 277 deletions(-) diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index c65436072..db3f68fd0 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -37,5 +37,8 @@ if [[ "$1" == "--lint" ]]; then cd cuesubmit && PYTHONPATH=../pycue:../pyoutline python -m pylint --rcfile=../ci/pylintrc_main cuesubmit && cd .. cd cuesubmit && PYTHONPATH=../pycue:../pyoutline python -m pylint --rcfile=../ci/pylintrc_test tests && cd .. + + cd rqd && python -m pylint --rcfile=../ci/pylintrc_main rqd --ignore=rqd/compiled_proto && cd .. + cd rqd && python -m pylint --rcfile=../ci/pylintrc_test tests && cd .. fi diff --git a/rqd/rqd/__main__.py b/rqd/rqd/__main__.py index e69209828..4c0a46bfe 100755 --- a/rqd/rqd/__main__.py +++ b/rqd/rqd/__main__.py @@ -16,7 +16,7 @@ """ -Initializes and starts rqd. +Initializes and starts RQD. - RQD allows the cuebot to launch frames on a remote host. - RQD monitors the resources on a machine. @@ -61,12 +61,13 @@ def setupLogging(): """Sets up the logging for RQD. - Logs to /var/log/messages""" + + Logs to /var/log/messages""" # TODO(bcipriano) These should be config based. (Issue #72) consoleFormat = '%(asctime)s %(levelname)-9s rqd3-%(module)-10s %(message)s' - consoleLevel = logging.DEBUG - fileFormat = '%(asctime)s %(levelname)-9s rqd3-%(module)-10s %(message)s' - fileLevel = logging.WARNING # Equal to or greater than the consoleLevel + consoleLevel = logging.DEBUG + fileFormat = '%(asctime)s %(levelname)-9s rqd3-%(module)-10s %(message)s' + fileLevel = logging.WARNING # Equal to or greater than the consoleLevel logging.basicConfig(level=consoleLevel, format=consoleFormat) if platform.system() in ('Linux', 'Darwin'): @@ -98,7 +99,9 @@ def usage(): print(" Defaults to /etc/rqd3/rqd3.conf", file=s) print(" Config file is optional", file=s) + def main(): + """Entrypoint for RQD.""" setupLogging() if platform.system() == 'Linux' and os.getuid() != 0 and \ @@ -107,23 +110,20 @@ def main(): sys.exit(1) try: - opts, argv = getopt.getopt(sys.argv[1:], 'hdc:', ['help', - 'daemon', - 'nimbyoff', - 'update']) + opts, _ = getopt.getopt(sys.argv[1:], 'hdc:', ['help', 'daemon', 'nimbyoff', 'update']) except getopt.GetoptError: usage() sys.exit(1) optNimbyOff = False - for o, a in opts: - if o in ["-h", "--help"]: + for option, _ in opts: + if option in ["-h", "--help"]: usage() sys.exit(0) - if o in ["-d", "--daemon"]: + if option in ["-d", "--daemon"]: # TODO(bcipriano) Background the process. (Issue #153) pass - if o in ["--nimbyoff"]: + if option in ["--nimbyoff"]: optNimbyOff = True rqd.rqutil.permissionsLow() @@ -135,5 +135,4 @@ def main(): if __name__ == "__main__": - main() - + main() diff --git a/rqd/rqd/cuerqd.py b/rqd/rqd/cuerqd.py index 2a14bdfef..005c92d4f 100755 --- a/rqd/rqd/cuerqd.py +++ b/rqd/rqd/cuerqd.py @@ -14,9 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" -Displays information from or sends a command to an RQD host -""" + +"""Displays information from or sends a command to an RQD host.""" + from __future__ import print_function from __future__ import absolute_import @@ -37,6 +37,10 @@ class RqdHost(object): + """An RQD host. + + This class supplies a wrapper for querying or sending commands to an RQD host.""" + def __init__(self, rqdHost, rqdPort=rqd.rqconstants.RQD_GRPC_PORT): self.rqdHost = rqdHost self.rqdPort = rqdPort @@ -46,156 +50,208 @@ def __init__(self, rqdHost, rqdPort=rqd.rqconstants.RQD_GRPC_PORT): self.frameStub = rqd.compiled_proto.rqd_pb2_grpc.RunningFrameStub(channel) def status(self): + """Fetches and returns the host status report.""" return self.stub.ReportStatus(rqd.compiled_proto.rqd_pb2.RqdStaticReportStatusRequest()) def getRunningFrame(self, frameId): + """Returns the host's currently running frame.""" return self.stub.GetRunFrame( rqd.compiled_proto.rqd_pb2.RqdStaticGetRunFrameRequest(frame_id=frameId)) def nimbyOff(self): + """Disables Nimby on the host.""" print(self.rqdHost, "Turning off Nimby") - log.info("rqd nimbyoff by {0}".format(os.environ.get("USER"))) + log.info("rqd nimbyoff by %s", os.environ.get("USER")) self.stub.NimbyOff(rqd.compiled_proto.rqd_pb2.RqdStaticNimbyOffRequest()) def nimbyOn(self): + """Enables Nimby on the host.""" print(self.rqdHost, "Turning on Nimby") - log.info("rqd nimbyon by {0}".format(os.environ.get("USER"))) + log.info("rqd nimbyon by %s", os.environ.get("USER")) self.stub.NimbyOn(rqd.compiled_proto.rqd_pb2.RqdStaticNimbyOnRequest()) def lockAll(self): - print(self.rqdHost,"Locking all cores") + """Locks all of the host's cores.""" + print(self.rqdHost, "Locking all cores") self.stub.LockAll(rqd.compiled_proto.rqd_pb2.RqdStaticLockAllRequest()) def unlockAll(self): - print(self.rqdHost,"Unlocking all cores") + """Unlocks all of the host's cores.""" + print(self.rqdHost, "Unlocking all cores") self.stub.UnlockAll(rqd.compiled_proto.rqd_pb2.RqdStaticUnlockAllRequest()) def lock(self, cores): + """Locks the given number of cores.""" cores = int(cores) - print(self.rqdHost,"Locking %d cores" % cores) + print(self.rqdHost, "Locking %d cores" % cores) self.stub.Lock(rqd.compiled_proto.rqd_pb2.RqdStaticLockRequest(cores=cores)) def unlock(self, cores): + """Unlocks the given number of cores.""" cores = int(cores) - print(self.rqdHost,"Unlocking %d cores" % cores) + print(self.rqdHost, "Unlocking %d cores" % cores) self.stub.Unlock(rqd.compiled_proto.rqd_pb2.RqdStaticUnlockRequest(cores=cores)) def shutdownRqdIdle(self): - print(self.rqdHost,"Sending shutdownRqdIdle command") + """Shuts down the host when idle.""" + print(self.rqdHost, "Sending shutdownRqdIdle command") self.stub.ShutdownRqdIdle(rqd.compiled_proto.rqd_pb2.RqdStaticShutdownIdleRequest()) def shutdownRqdNow(self): - print(self.rqdHost,"Sending shutdownRqdNow command") + """Shuts down the host now.""" + print(self.rqdHost, "Sending shutdownRqdNow command") self.stub.ShutdownRqdNow(rqd.compiled_proto.rqd_pb2.RqdStaticShutdownNowRequest()) def restartRqdIdle(self): - print(self.rqdHost,"Sending restartRqdIdle command") + """Restarts RQD on the host when idle.""" + print(self.rqdHost, "Sending restartRqdIdle command") self.stub.RestartRqdIdle(rqd.compiled_proto.rqd_pb2.RqdStaticRestartIdleRequest()) def restartRqdNow(self): - print(self.rqdHost,"Sending restartRqdNow command") + """Restarts RQD on the host now.""" + print(self.rqdHost, "Sending restartRqdNow command") self.stub.RestartRqdNow(rqd.compiled_proto.rqd_pb2.RqdStaticRestartNowRequest()) def rebootIdle(self): - print(self.rqdHost,"Sending rebootIdle command") + """Reboots the host when idle.""" + print(self.rqdHost, "Sending rebootIdle command") self.stub.RebootIdle(rqd.compiled_proto.rqd_pb2.RqdStaticRebootIdleRequest()) def rebootNow(self): - print(self.rqdHost,"Sending rebootNow command") + """Reboots the host now.""" + print(self.rqdHost, "Sending rebootNow command") self.stub.RebootNow(rqd.compiled_proto.rqd_pb2.RqdStaticRebootNowRequest()) def launchFrame(self, frame): - self.stub.LaunchFrame(rqd.compiled_proto.rqd_pb2.RqdStaticLaunchFrameRequest(run_frame=frame)) + """Launches a frame on the host.""" + self.stub.LaunchFrame( + rqd.compiled_proto.rqd_pb2.RqdStaticLaunchFrameRequest(run_frame=frame)) def killFrame(self, frameId, message): + """Kills a frame on the host.""" runFrame = self.getRunningFrame(frameId) self.frameStub.Kill(run_frame=runFrame, message=message) def main(): + """Entrypoint for the commandline interface.""" + parser = argparse.ArgumentParser() - parser.add_argument('host', nargs='?', default='localhost', help='RQD hostname (defaults to localhost)') - parser.add_argument('-s', action='store_true', help='Print RQD status') - parser.add_argument('-v', action='store_true', help='Print RQD version') - parser.add_argument('--lp', metavar='coreID', nargs='+', help='Lock the specified cores') - parser.add_argument('--ulp', metavar='coreID', nargs='+', help='Unlock the specified cores') - parser.add_argument('--lh', action='store_true', help='Lock all cores for the specified host') - parser.add_argument('--ulh', action='store_true', help='Unlock all cores for the specified host') - parser.add_argument('--nimbyon', action='store_true', help="Turn on 'Not in my back yard' (NIMBY) to stop processing on the specified host") - parser.add_argument('--nimbyoff', action='store_true', help="Turn off 'Not in my back yard' (NIMBY) to start processing on the specified host") - parser.add_argument('--exit', action='store_true', help='Lock host, wait until machine is idle, and then shutdown RQD. Any unlock command cancels this request.') - parser.add_argument('--exit_now', action='store_true', help='KILL ALL running frames and shutdown RQD') - parser.add_argument('--restart', action='store_true', help='Lock host, wait until machine is idle, and then restart RQD. Any unlock command cancels this request') - parser.add_argument('--restart_now', action='store_true', help='KILL ALL running frames and restart RQD') - parser.add_argument('--reboot', action='store_true', help='Lock host, wait until machine is idle, and then REBOOT machine. Any unlock command cancels this request.') - parser.add_argument('--reboot_now', action='store_true', help='KILL ALL running frames and REBOOT machine') - parser.add_argument('--kill', metavar='frameID', nargs='+', help='Attempts to kill the given frame via its ICE proxy') - parser.add_argument('--getproxy', metavar='frameID', nargs='+', help='Returns the proxy for the given frameid') - parser.add_argument('--test_edu_frame',action='store_true', help='Launch an edu frame test on an idle core (or first core if none are available)') - parser.add_argument('--test_script_frame', action='store_true', help='Launch a script frame test on an idle core (or first core if none are available)') - parser.add_argument('--test_script_frame_mac', action='store_true', help='Launch a script frame test for macOS on an idle core (or first core if none are available)') - + parser.add_argument( + 'host', nargs='?', default='localhost', help='RQD hostname (defaults to localhost)') + parser.add_argument( + '-s', action='store_true', help='Print RQD status') + parser.add_argument( + '-v', action='store_true', help='Print RQD version') + parser.add_argument( + '--lp', metavar='coreID', nargs='+', help='Lock the specified cores') + parser.add_argument( + '--ulp', metavar='coreID', nargs='+', help='Unlock the specified cores') + parser.add_argument( + '--lh', action='store_true', help='Lock all cores for the specified host') + parser.add_argument( + '--ulh', action='store_true', help='Unlock all cores for the specified host') + parser.add_argument( + '--nimbyon', action='store_true', + help="Turn on 'Not in my back yard' (NIMBY) to stop processing on the specified host") + parser.add_argument( + '--nimbyoff', action='store_true', + help="Turn off 'Not in my back yard' (NIMBY) to start processing on the specified host") + parser.add_argument( + '--exit', action='store_true', + help='Lock host, wait until machine is idle, and then shutdown RQD. Any unlock ' + 'command cancels this request.') + parser.add_argument( + '--exit_now', action='store_true', help='KILL ALL running frames and shutdown RQD') + parser.add_argument( + '--restart', action='store_true', + help='Lock host, wait until machine is idle, and then restart RQD. Any unlock ' + 'command cancels this request') + parser.add_argument( + '--restart_now', action='store_true', help='KILL ALL running frames and restart RQD') + parser.add_argument( + '--reboot', action='store_true', + help='Lock host, wait until machine is idle, and then REBOOT machine. Any unlock ' + 'command cancels this request.') + parser.add_argument( + '--reboot_now', action='store_true', help='KILL ALL running frames and REBOOT machine') + parser.add_argument( + '--kill', metavar='frameID', nargs='+', + help='Attempts to kill the given frame via its ICE proxy') + parser.add_argument( + '--getproxy', metavar='frameID', nargs='+', help='Returns the proxy for the given frameid') + parser.add_argument( + '--test_edu_frame', action='store_true', + help='Launch an edu frame test on an idle core (or first core if none are available)') + parser.add_argument( + '--test_script_frame', action='store_true', + help='Launch a script frame test on an idle core (or first core if none are available)') + parser.add_argument( + '--test_script_frame_mac', action='store_true', + help='Launch a script frame test for macOS on an idle core (or first core if ' + 'none are available)') + args = parser.parse_args() rqdHost = RqdHost(args.host) - + if args.s: print(rqdHost.status()) - + if args.v: tagPrefix = 'rqdv-' for tag in rqdHost.status().host.tags: if tag.startswith(tagPrefix): print("version =", tag[len(tagPrefix):]) - + if args.nimbyoff: rqdHost.nimbyOff() - + if args.nimbyon: rqdHost.nimbyOn() - + if args.lp is not None: for arg in args.lp: rqdHost.lock(arg) - + if args.ulp is not None: - for arg in args.ulp: + for arg in args.ulp: rqdHost.unlock(arg) - + if args.lh is not None: rqdHost.lockAll() - + if args.ulh is not None: rqdHost.unlockAll() - + if args.exit_now: rqdHost.shutdownRqdNow() - + elif args.exit: rqdHost.shutdownRqdIdle() - + if args.restart_now: rqdHost.restartRqdNow() - + elif args.restart: rqdHost.restartRqdIdle() - + if args.reboot_now: rqdHost.rebootNow() - + elif args.reboot: rqdHost.rebootIdle() - + if args.kill is not None: for arg in args.kill: rqdHost.killFrame(arg, "Killed by %s using cuerqd.py" % os.environ.get("USER")) - + if args.getproxy is not None: for arg in args.getproxy: frameProxy = rqdHost.getRunningFrame(arg) print(frameProxy) - + if args.test_edu_frame: print("Launching edu test frame (logs to /mcp)") frameNum = "0001" @@ -204,7 +260,12 @@ def main(): runFrame.job_name = "edu-trn_jwelborn-jwelborn_teapot_bty" runFrame.frame_id = "FD1S3I154O646UGSNN%s" % frameNum runFrame.frame_name = "%s-teapot_bty_3D" % frameNum - runFrame.command = "/usr/bin/env VNP_APPLICATION_TIME=1197683283873 /usr/bin/env VNP_VCR_SESSION=3411896 /usr/bin/env PROFILE=default /shots/edu/home/perl/etc/qwrap.cuerun /shots/edu/trn_jwelborn/cue/jwelborn olrun /shots/edu/trn_jwelborn/cue/cue_archive/edu-trn_jwelborn-jwelborn_teapot_bty/v4/teapot_bty.outline %d -batch -event teapot_bty_3D" % int(frameNum) + runFrame.command = ( + "/usr/bin/env VNP_APPLICATION_TIME=1197683283873 /usr/bin/env VNP_VCR_" + "SESSION=3411896 /usr/bin/env PROFILE=default " + "/shots/edu/home/perl/etc/qwrap.cuerun /shots/edu/trn_jwelborn/cue/jwelborn " + "olrun /shots/edu/trn_jwelborn/cue/cue_archive/edu-trn_jwelborn-jwelborn_teapot_bty" + "/v4/teapot_bty.outline %d -batch -event teapot_bty_3D" % int(frameNum)) runFrame.user_name = "jwelborn" runFrame.log_dir = "/mcp" # This would be on the shottree runFrame.show = "edu" @@ -248,7 +309,7 @@ def main(): runFrame.uid = 10164 runFrame.num_cores = 1 rqdHost.launchFrame(runFrame) - - + + if __name__ == "__main__": main() diff --git a/rqd/rqd/rqconstants.py b/rqd/rqd/rqconstants.py index 651e61337..07a621456 100644 --- a/rqd/rqd/rqconstants.py +++ b/rqd/rqd/rqconstants.py @@ -13,18 +13,18 @@ # limitations under the License. -""" -Constants. -""" +"""Constants used throughout RQD.""" from __future__ import print_function from __future__ import division from __future__ import absolute_import +# pylint: disable=wrong-import-position from future import standard_library standard_library.install_aliases() -import subprocess +# pylint: enable=wrong-import-position + import logging import os import platform @@ -40,9 +40,9 @@ VERSION = 'dev' if 'CUEBOT_HOSTNAME' in os.environ: - CUEBOT_HOSTNAME = os.environ['CUEBOT_HOSTNAME'] + CUEBOT_HOSTNAME = os.environ['CUEBOT_HOSTNAME'] else: - CUEBOT_HOSTNAME = 'localhost' + CUEBOT_HOSTNAME = 'localhost' RQD_TIMEOUT = 10000 DEFAULT_FACILITY = 'cloud' @@ -79,12 +79,16 @@ RQD_GID = 0 # Nimby behavior: -CHECK_INTERVAL_LOCKED = 60 # = seconds to wait before checking if the user has become idle -MINIMUM_IDLE = 900 # seconds of idle time required before nimby unlocks -MINIMUM_MEM = 524288 # If available memory drops below this amount, lock nimby (need to take into account cache) +# Number of seconds to wait before checking if the user has become idle. +CHECK_INTERVAL_LOCKED = 60 +# Seconds of idle time required before nimby unlocks. +MINIMUM_IDLE = 900 +# If available memory drops below this amount, lock nimby (need to take into account cache). +MINIMUM_MEM = 524288 MINIMUM_SWAP = 1048576 -MAXIMUM_LOAD = 75 # If (machine load * 100 / cores) goes over this amount, don't unlock nimby - # 1.5 would mean a max load of 1.5 per core +# If (machine load * 100 / cores) goes over this amount, don't unlock nimby. +# 1.5 would mean a max load of 1.5 per core +MAXIMUM_LOAD = 75 EXITSTATUS_FOR_FAILED_LAUNCH = 256 EXITSTATUS_FOR_NIMBY_KILL = 286 @@ -134,7 +138,7 @@ else: ConfigParser = configparser.RawConfigParser config = ConfigParser() - logging.info('Loading config {}'.format(CONFIG_FILE)) + logging.info('Loading config %s', CONFIG_FILE) config.read(CONFIG_FILE) if config.has_option(__section, "RQD_GRPC_PORT"): @@ -169,6 +173,8 @@ DEFAULT_FACILITY = config.get(__section, "DEFAULT_FACILITY") if config.has_option(__section, "LAUNCH_FRAME_USER_GID"): LAUNCH_FRAME_USER_GID = config.getint(__section, "LAUNCH_FRAME_USER_GID") +# pylint: disable=broad-except except Exception as e: - logging.warning("Failed to read values from config file %s due to %s at %s" % (CONFIG_FILE, e, traceback.extract_tb(sys.exc_info()[2]))) - + logging.warning( + "Failed to read values from config file %s due to %s at %s", + CONFIG_FILE, e, traceback.extract_tb(sys.exc_info()[2])) diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index a8041727d..dd58e35a6 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -Main RQD module, handles gRPC function implementation and job launching. -""" +"""Main RQD module, handles gRPC function implementation and job launching.""" from __future__ import print_function @@ -72,6 +70,7 @@ def __init__(self, rqCore, runFrame, frameInfo): def __createEnvVariables(self): """Define the environmental variables for the frame""" # If linux specific, they need to move into self.runLinux() + # pylint: disable=attribute-defined-outside-init self.frameEnv = {} self.frameEnv["PATH"] = self.rqCore.machine.getPathEnv() self.frameEnv["TERM"] = "unknown" @@ -131,9 +130,11 @@ def _createCommandFile(self, command): rqexe.close() os.chmod(commandFile, 0o777) return commandFile + # pylint: disable=broad-except except Exception as e: - log.critical("Unable to make command file: %s due to %s at %s" % ( - commandFile, e, traceback.extract_tb(sys.exc_info()[2]))) + log.critical( + "Unable to make command file: %s due to %s at %s", + commandFile, e, traceback.extract_tb(sys.exc_info()[2])) def __writeHeader(self): """Writes the frame's log header""" @@ -163,11 +164,11 @@ def __writeHeader(self): if 'CPU_LIST' in self.runFrame.attributes: print('Hyper-threading enabled', file=self.rqlog) + # pylint: disable=broad-except except Exception as e: - log.critical("Unable to write header to rqlog: " - "%s due to %s at %s" % - (self.runFrame.log_dir_file, e, - traceback.extract_tb(sys.exc_info()[2]))) + log.critical( + "Unable to write header to rqlog: %s due to %s at %s", + self.runFrame.log_dir_file, e, traceback.extract_tb(sys.exc_info()[2])) def __writeFooter(self): """Writes frame's log footer""" @@ -190,16 +191,20 @@ def __writeFooter(self): print("%-20s%s" % ("stime", self.frameInfo.stime), file=self.rqlog) print("%-20s%s" % ("renderhost", self.rqCore.machine.getHostname()), file=self.rqlog) print("="*59, file=self.rqlog) + + # pylint: disable=broad-except except Exception as e: - log.critical("Unable to write footer: %s due to %s at %s" % - (self.runFrame.log_dir_file, e, - traceback.extract_tb(sys.exc_info()[2]))) + log.critical( + "Unable to write footer: %s due to %s at %s", + self.runFrame.log_dir_file, e, traceback.extract_tb(sys.exc_info()[2])) def __sendFrameCompleteReport(self): """Send report to cuebot that frame has finished""" report = rqd.compiled_proto.report_pb2.FrameCompleteReport() + # pylint: disable=no-member report.host.CopyFrom(self.rqCore.machine.getHostInfo()) report.frame.CopyFrom(self.frameInfo.runningFrameInfo()) + # pylint: enable=no-member if self.frameInfo.exitStatus is None: report.exit_status = 1 @@ -224,18 +229,22 @@ def __cleanup(self): if os.path.isfile(location): try: os.remove(location) + # pylint: disable=broad-except except Exception as e: - log.warning("Unable to delete file: %s due to %s at %s" % ( - location, e, traceback.extract_tb(sys.exc_info()[2]))) + log.warning( + "Unable to delete file: %s due to %s at %s", + location, e, traceback.extract_tb(sys.exc_info()[2])) finally: rqd.rqutil.permissionsLow() # Close log file try: self.rqlog.close() + # pylint: disable=broad-except except Exception as e: - log.warning("Unable to close file: %s due to %s at %s" % ( - self.runFrame.log_file, e, traceback.extract_tb(sys.exc_info()[2]))) + log.warning( + "Unable to close file: %s due to %s at %s", + self.runFrame.log_file, e, traceback.extract_tb(sys.exc_info()[2])) def runLinux(self): """The steps required to handle a frame under linux""" @@ -270,6 +279,7 @@ def runLinux(self): tempCommand += [self._createCommandFile(runFrame.command)] # Actual cwd is set by /shots/SHOW/home/perl/etc/qwrap.cuerun + # pylint: disable=subprocess-popen-preexec-fn frameInfo.forkedCommand = subprocess.Popen(tempCommand, env=self.frameEnv, cwd=self.rqCore.machine.getTempPath(), @@ -305,8 +315,9 @@ def runLinux(self): frameInfo.utime = statFile.readline().split()[1] frameInfo.stime = statFile.readline().split()[1] statFile.close() + # pylint: disable=broad-except except Exception: - pass # This happens when frames are killed + pass # This happens when frames are killed self.__writeFooter() self.__cleanup() @@ -327,9 +338,11 @@ def runWindows(self): stdin=subprocess.PIPE, stdout=self.rqlog, stderr=self.rqlog) - except: - log.critical("Failed subprocess.Popen: Due to: \n%s" % ''.join( - traceback.format_exception(*sys.exc_info()))) + # pylint: disable=broad-except + except Exception: + log.critical( + "Failed subprocess.Popen: Due to: \n%s", + ''.join(traceback.format_exception(*sys.exc_info()))) frameInfo.pid = frameInfo.forkedCommand.pid @@ -364,6 +377,7 @@ def runDarwin(self): tempCommand = ["/usr/bin/su", frameInfo.runFrame.user_name, "-c", '"' + self._createCommandFile(frameInfo.runFrame.command) + '"'] + # pylint: disable=subprocess-popen-preexec-fn frameInfo.forkedCommand = subprocess.Popen(tempCommand, env=self.frameEnv, cwd=self.rqCore.machine.getTempPath(), @@ -395,19 +409,19 @@ def runDarwin(self): self.__writeFooter() self.__cleanup() - def waitForFile(self, filepath, maxTries=5): + @staticmethod + def waitForFile(filepath, maxTries=5): + """Waits for a file to exist.""" tries = 0 while tries < maxTries: if os.path.exists(filepath): return - else: - tries += 1 - time.sleep(0.5 * tries) + tries += 1 + time.sleep(0.5 * tries) raise IOError("Failed to create %s" % filepath) def runUnknown(self): - """The steps required to handle a frame under an unknown OS""" - pass + """The steps required to handle a frame under an unknown OS.""" def run(self): """Thread initialization""" @@ -415,6 +429,7 @@ def run(self): runFrame = self.runFrame + # pylint: disable=too-many-nested-blocks try: runFrame.job_temp_dir = os.path.join(self.rqCore.machine.getTempPath(), runFrame.job_name) @@ -443,6 +458,7 @@ def run(self): try: os.makedirs(runFrame.log_dir) os.chmod(runFrame.log_dir, 0o777) + # pylint: disable=broad-except except Exception as e: # This is expected to fail when called in abq # But the directory should now be visible @@ -466,6 +482,7 @@ def run(self): rotateCount += 1 os.rename(runFrame.log_dir_file, "%s.%s" % (runFrame.log_dir_file, rotateCount)) + # pylint: disable=broad-except except Exception as e: err = "Unable to rotate previous log file due to %s" % e # Windows might fail while trying to rotate logs for checking if file is @@ -478,11 +495,13 @@ def run(self): try: self.rqlog = open(runFrame.log_dir_file, "w", 1) self.waitForFile(runFrame.log_dir_file) + # pylint: disable=broad-except except Exception as e: err = "Unable to write to %s due to %s" % (runFrame.log_dir_file, e) raise RuntimeError(err) try: os.chmod(runFrame.log_dir_file, 0o666) + # pylint: disable=broad-except except Exception as e: err = "Failed to chmod log file! %s due to %s" % (runFrame.log_dir_file, e) log.warning(err) @@ -502,10 +521,11 @@ def run(self): else: self.runUnknown() - except Exception as e: - log.critical("Failed launchFrame: For %s due to: \n%s" % ( - runFrame.frame_id, - ''.join(traceback.format_exception(*sys.exc_info())))) + # pylint: disable=broad-except + except Exception: + log.critical( + "Failed launchFrame: For %s due to: \n%s", + runFrame.frame_id, ''.join(traceback.format_exception(*sys.exc_info()))) # Notifies the cuebot that there was an error launching self.frameInfo.exitStatus = rqd.rqconstants.EXITSTATUS_FOR_FAILED_LAUNCH # Delay keeps the cuebot from spamming failing booking requests @@ -516,7 +536,8 @@ def run(self): self.rqCore.deleteFrame(self.runFrame.frame_id) self.__sendFrameCompleteReport() - time_till_next = (self.rqCore.intervalStartTime + self.rqCore.intervalSleepTime) - time.time() + time_till_next = ( + (self.rqCore.intervalStartTime + self.rqCore.intervalSleepTime) - time.time()) if time_till_next > (2 * rqd.rqconstants.RQD_MIN_PING_INTERVAL_SEC): self.rqCore.onIntervalThread.cancel() self.rqCore.onInterval(rqd.rqconstants.RQD_MIN_PING_INTERVAL_SEC) @@ -609,8 +630,11 @@ def onInterval(self, sleepTime=None): self.onIntervalThread = threading.Timer(self.intervalSleepTime, self.onInterval) self.intervalStartTime = time.time() self.onIntervalThread.start() + # pylint: disable=broad-except except Exception as e: - log.critical('Unable to schedule a ping due to {0} at {1}'.format(e, traceback.extract_tb(sys.exc_info()[2]))) + log.critical( + 'Unable to schedule a ping due to %s at %s', + e, traceback.extract_tb(sys.exc_info()[2])) try: if self.__whenIdle and not self.__cache: @@ -618,14 +642,18 @@ def onInterval(self, sleepTime=None): self.shutdownRqdNow() else: log.warning('Shutdown requested but a user is logged in.') - + # pylint: disable=broad-except except Exception as e: - log.warning('Unable to shutdown due to {0} at {1}'.format(e, traceback.extract_tb(sys.exc_info()[2]))) + log.warning( + 'Unable to shutdown due to %s at %s', e, traceback.extract_tb(sys.exc_info()[2])) try: self.sendStatusReport() + # pylint: disable=broad-except except Exception as e: - log.critical('Unable to send status report due to {0} at {1}'.format(e, traceback.extract_tb(sys.exc_info()[2]))) + log.critical( + 'Unable to send status report due to %s at %s', + e, traceback.extract_tb(sys.exc_info()[2])) def updateRss(self): """Triggers and schedules the updating of rss information""" @@ -633,7 +661,8 @@ def updateRss(self): try: self.machine.rssUpdate(self.__cache) finally: - self.updateRssThread = threading.Timer(rqd.rqconstants.RSS_UPDATE_INTERVAL, self.updateRss) + self.updateRssThread = threading.Timer( + rqd.rqconstants.RSS_UPDATE_INTERVAL, self.updateRss) self.updateRssThread.start() def getFrame(self, frameId): @@ -659,7 +688,8 @@ def storeFrame(self, frameId, runningFrame): self.__threadLock.acquire() try: if frameId in self.__cache: - raise rqd.rqexceptions.RqdException("frameId " + frameId + " is already running on this machine") + raise rqd.rqexceptions.RqdException( + "frameId " + frameId + " is already running on this machine") self.__cache[frameId] = runningFrame finally: self.__threadLock.release() @@ -681,12 +711,14 @@ def killAllFrame(self, reason): @param reason: Reason for requesting all frames to be killed""" if self.__cache: - log.warning("killAllFrame called due to: %s\n%s" % (reason, ",".join(self.getFrameKeys()))) + log.warning( + "killAllFrame called due to: %s\n%s", reason, ",".join(self.getFrameKeys())) while self.__cache: if reason.startswith("NIMBY"): # Since this is a nimby kill, ignore any frames that are ignoreNimby - frameKeys = [frame.frameId for frame in list(self.__cache.values()) if not frame.ignoreNimby] + frameKeys = [ + frame.frameId for frame in list(self.__cache.values()) if not frame.ignoreNimby] else: frameKeys = list(self.__cache.keys()) @@ -707,6 +739,7 @@ def releaseCores(self, reqRelease, releaseHT=None): @param reqRelease: Number of cores to release, 100 = 1 physical core""" self.__threadLock.acquire() try: + # pylint: disable=no-member self.cores.booked_cores -= reqRelease maxRelease = (self.cores.total_cores - self.cores.locked_cores - @@ -715,6 +748,7 @@ def releaseCores(self, reqRelease, releaseHT=None): if maxRelease > 0: self.cores.idle_cores += min(maxRelease, reqRelease) + # pylint: enable=no-member if releaseHT: self.machine.releaseHT(releaseHT) @@ -722,13 +756,16 @@ def releaseCores(self, reqRelease, releaseHT=None): finally: self.__threadLock.release() + # pylint: disable=no-member if self.cores.idle_cores > self.cores.total_cores: log.critical( - "idle_cores (%d) have become greater than total_cores (%d): %s at %s" % ( - self.cores.idle_cores, self.cores.total_cores, sys.exc_info()[0], - traceback.extract_tb(sys.exc_info()[2]))) + "idle_cores (%d) have become greater than total_cores (%d): %s at %s", + self.cores.idle_cores, self.cores.total_cores, sys.exc_info()[0], + traceback.extract_tb(sys.exc_info()[2])) + # pylint: enable=no-member - def respawn_rqd(self): + @staticmethod + def respawn_rqd(): """Restarts RQD""" os.system("/etc/init.d/rqd3 restart") @@ -751,6 +788,8 @@ def shutdown(self): def handleExit(self, signalnum, flag): """Shutdown threads and exit RQD.""" + del signalnum + del flag self.shutdown() self.network.stopGrpc() sys.exit() @@ -760,8 +799,7 @@ def launchFrame(self, runFrame): If a problem is encountered, a CueException will be thrown. @type runFrame: RunFrame @param runFrame: rqd_pb2.RunFrame""" - log.info("Running command %s for %s" % (runFrame.command, - runFrame.frame_id)) + log.info("Running command %s for %s", runFrame.command, runFrame.frame_id) log.debug(runFrame) # @@ -801,10 +839,12 @@ def launchFrame(self, runFrame): # See if all requested cores are available self.__threadLock.acquire() try: + # pylint: disable=no-member if self.cores.idle_cores < runFrame.num_cores: err = "Not launching, insufficient idle cores" log.critical(err) raise rqd.rqexceptions.CoreReservationFailureException(err) + # pylint: enable=no-member if runFrame.environment.get('CUE_THREADABLE') == '1': reserveHT = self.machine.reserveHT(runFrame.num_cores) @@ -812,8 +852,10 @@ def launchFrame(self, runFrame): runFrame.attributes['CPU_LIST'] = reserveHT # They must be available at this point, reserve them + # pylint: disable=no-member self.cores.idle_cores -= runFrame.num_cores self.cores.booked_cores += runFrame.num_cores + # pylint: enable=no-member finally: self.__threadLock.release() @@ -822,16 +864,18 @@ def launchFrame(self, runFrame): runningFrame.frameAttendantThread.start() def getRunningFrame(self, frameId): + """Gets the currently running frame.""" try: return self.__cache[frameId] except KeyError: - log.info("frameId {} is not running on this machine".format(frameId)) + log.info("frameId %s is not running on this machine", frameId) return None def getCoreInfo(self): + """Gets the core info report.""" return self.cores - def reportStatus(self, current=None): + def reportStatus(self): """Replies with hostReport""" return self.machine.getHostReport() @@ -920,6 +964,7 @@ def onNimbyUnlock(self, asOf=None): """This is called by nimby when it unlocks the machine due to sufficent idle. A new report is sent to the cuebot. @param asOf: Time when idle state began, if known.""" + del asOf self.sendStatusReport() def lock(self, reqLock): @@ -930,12 +975,14 @@ def lock(self, reqLock): sendUpdate = False self.__threadLock.acquire() try: + # pylint: disable=no-member numLock = min(self.cores.total_cores - self.cores.locked_cores, reqLock) if numLock > 0: self.cores.locked_cores += numLock self.cores.idle_cores -= min(numLock, self.cores.idle_cores) sendUpdate = True + # pylint: enable=no-member finally: self.__threadLock.release() @@ -950,10 +997,12 @@ def lockAll(self): sendUpdate = False self.__threadLock.acquire() try: + # pylint: disable=no-member if self.cores.locked_cores < self.cores.total_cores: self.cores.locked_cores = self.cores.total_cores self.cores.idle_cores = 0 sendUpdate = True + # pylint: enable=no-member finally: self.__threadLock.release() @@ -971,7 +1020,8 @@ def unlock(self, reqUnlock): sendUpdate = False - if self.__whenIdle or self.__reboot or self.__respawn or self.machine.state != rqd.compiled_proto.host_pb2.UP: + if (self.__whenIdle or self.__reboot or self.__respawn + or self.machine.state != rqd.compiled_proto.host_pb2.UP): sendUpdate = True self.__whenIdle = False @@ -981,11 +1031,13 @@ def unlock(self, reqUnlock): self.__threadLock.acquire() try: + # pylint: disable=no-member numUnlock = min(self.cores.locked_cores, reqUnlock) if numUnlock > 0: self.cores.locked_cores -= numUnlock self.cores.idle_cores += numUnlock sendUpdate = True + # pylint: enable=no-member finally: self.__threadLock.release() @@ -1001,7 +1053,8 @@ def unlockAll(self): sendUpdate = False - if self.__whenIdle or self.__reboot or self.__respawn or self.machine.state != rqd.compiled_proto.host_pb2.UP: + if (self.__whenIdle or self.__reboot or self.__respawn + or self.machine.state != rqd.compiled_proto.host_pb2.UP): sendUpdate = True self.__whenIdle = False @@ -1011,11 +1064,13 @@ def unlockAll(self): self.__threadLock.acquire() try: + # pylint: disable=no-member if self.cores.locked_cores > 0: if not self.nimby.locked: self.cores.idle_cores += self.cores.locked_cores self.cores.locked_cores = 0 sendUpdate = True + # pylint: enable=no-member finally: self.__threadLock.release() @@ -1025,7 +1080,9 @@ def unlockAll(self): self.sendStatusReport() def sendStatusReport(self): + """Sends the current host report to Cuebot.""" self.network.reportStatus(self.machine.getHostReport()) def isWaitingForIdle(self): + """Returns whether the host is waiting until idle to take some action.""" return self.__whenIdle diff --git a/rqd/rqd/rqdservicers.py b/rqd/rqd/rqdservicers.py index b9fa27b4a..a56fdf442 100644 --- a/rqd/rqd/rqdservicers.py +++ b/rqd/rqd/rqdservicers.py @@ -13,6 +13,9 @@ # limitations under the License. +"""Implements the server side of the RQD gRPC service.""" + + from __future__ import absolute_import from __future__ import print_function from __future__ import division @@ -26,7 +29,7 @@ class RqdInterfaceServicer(rqd.compiled_proto.rqd_pb2_grpc.RqdInterfaceServicer): - """Service interface for RqdStatic gRPC definition""" + """Service interface for RqdStatic gRPC definition.""" def __init__(self, rqCore): self.rqCore = rqCore @@ -40,7 +43,8 @@ def LaunchFrame(self, request, context): def ReportStatus(self, request, context): """RPC call that returns reportStatus""" log.info("Request received: reportStatus") - return rqd.compiled_proto.rqd_pb2.RqdStaticReportStatusResponse(host_report=self.rqCore.reportStatus()) + return rqd.compiled_proto.rqd_pb2.RqdStaticReportStatusResponse( + host_report=self.rqCore.reportStatus()) def GetRunningFrameStatus(self, request, context): """RPC call to return the frame info for the given frame id""" @@ -49,11 +53,10 @@ def GetRunningFrameStatus(self, request, context): if frame: return rqd.compiled_proto.rqd_pb2.RqdStaticGetRunningFrameStatusResponse( running_frame_info=frame.runningFrameInfo()) - else: - context.set_details( - "The requested frame was not found. frameId: {}".format(request.frameId)) - context.set_code(grpc.StatusCode.NOT_FOUND) - return rqd.compiled_proto.rqd_pb2.RqdStaticGetRunningFrameStatusResponse() + context.set_details( + "The requested frame was not found. frameId: {}".format(request.frameId)) + context.set_code(grpc.StatusCode.NOT_FOUND) + return rqd.compiled_proto.rqd_pb2.RqdStaticGetRunningFrameStatusResponse() def KillRunningFrame(self, request, context): """RPC call that kills the running frame with the given id""" @@ -116,7 +119,7 @@ def NimbyOff(self, request, context): def Lock(self, request, context): """RPC call that locks a specific number of cores""" - log.info("Request received: lock %d" % request.cores) + log.info("Request received: lock %d", request.cores) self.rqCore.lock(request.cores) return rqd.compiled_proto.rqd_pb2.RqdStaticLockResponse() @@ -128,7 +131,7 @@ def LockAll(self, request, context): def Unlock(self, request, context): """RPC call that unlocks a specific number of cores""" - log.info("Request received: unlock %d" % request.cores) + log.info("Request received: unlock %d", request.cores) self.rqCore.unlock(request.cores) return rqd.compiled_proto.rqd_pb2.RqdStaticUnlockResponse() diff --git a/rqd/rqd/rqexceptions.py b/rqd/rqd/rqexceptions.py index f8443ee35..c5ac586cc 100644 --- a/rqd/rqd/rqexceptions.py +++ b/rqd/rqd/rqexceptions.py @@ -13,19 +13,25 @@ # limitations under the License. +"""Custom exception classes used throughout RQD.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import class RqdException(Exception): - pass + """Generic exception from the RQD code.""" + class CoreReservationFailureException(Exception): - pass + """RQD failed to reserve the required number of cores.""" + class DuplicateFrameViolationException(Exception): - pass + """RQD attempted to book a frame that was already running.""" + class InvalidUserException(Exception): - pass + """RQD attempted to assume the role of an invalid user.""" diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 5391e40d0..915c13afc 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -13,27 +13,28 @@ # limitations under the License. -""" -Machine information access module. -""" +"""Machine information access module.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division +# pylint: disable=wrong-import-position from future import standard_library standard_library.install_aliases() +# pylint: enable=wrong-import-position + from builtins import str from builtins import range from builtins import object +import ctypes import errno import logging as log import math import os import platform -import psutil import re import subprocess import sys @@ -41,12 +42,14 @@ import time import traceback +# pylint: disable=import-error,wrong-import-position if platform.system() in ('Linux', 'Darwin'): import resource - import yaml elif platform.system() == "win32": - import win32process import win32api +# pylint: enable=import-error,wrong-import-position + +import psutil import rqd.compiled_proto.host_pb2 import rqd.compiled_proto.report_pb2 @@ -82,10 +85,14 @@ def __init__(self, rqCore, coreInfo): self.__initMachineStats() self.__bootReport = rqd.compiled_proto.report_pb2.BootReport() + # pylint: disable=no-member self.__bootReport.core_info.CopyFrom(self.__coreInfo) + # pylint: enable=no-member self.__hostReport = rqd.compiled_proto.report_pb2.HostReport() + # pylint: disable=no-member self.__hostReport.core_info.CopyFrom(self.__coreInfo) + # pylint: enable=no-member self.__pidHistory = {} @@ -95,10 +102,12 @@ def isNimbySafeToRunJobs(self): """Returns False if nimby should be triggered due to resource limits""" if platform.system() == "Linux": self.updateMachineStats() + # pylint: disable=no-member if self.__renderHost.free_mem < rqd.rqconstants.MINIMUM_MEM: return False if self.__renderHost.free_swap < rqd.rqconstants.MINIMUM_SWAP: return False + # pylint: enable=no-member return True def isNimbySafeToUnlock(self): @@ -109,6 +118,7 @@ def isNimbySafeToUnlock(self): return False return True + # pylint: disable=no-self-use @rqd.rqutil.Memoize def isDesktop(self): """Returns True if machine starts in run level 5 (X11) @@ -126,6 +136,8 @@ def isDesktop(self): return False def isUserLoggedIn(self): + """Returns whether a user is logged into the machine RQD is running on.""" + # For non-headless systems, first check to see if there # is a user logged into the display. displayNums = [] @@ -155,9 +167,7 @@ def isUserLoggedIn(self): # is what shows up when gdm is running and # showing a login screen. if cols[0] != '(unknown)': - log.warning( - 'User {} logged into display :{}'.format( - cols[0], displayNum)) + log.warning('User %s logged into display :%s', cols[0], displayNum) return True # When there is a display, the above code is considered @@ -203,9 +213,11 @@ def rssUpdate(self, frames): "start_time": statFields[21], } - except Exception as e: - log.exception('failed to read stat file for pid %s' % pid) + # pylint: disable=broad-except + except Exception: + log.exception('failed to read stat file for pid %s', pid) + # pylint: disable=too-many-nested-blocks try: now = int(time.time()) pidData = {"time": now} @@ -225,7 +237,8 @@ def rssUpdate(self, frames): rss += int(data["rss"]) vsize += int(data["vsize"]) - # jiffies used by this process, last two means that dead children are counted + # jiffies used by this process, last two means that dead + # children are counted totalTime = int(data["utime"]) + \ int(data["stime"]) + \ int(data["cutime"]) + \ @@ -236,21 +249,26 @@ def rssUpdate(self, frames): float(data["start_time"]) / rqd.rqconstants.SYS_HERTZ if seconds: if pid in self.__pidHistory: - # Percent cpu using decaying average, 50% from 10 seconds ago, 50% from last 10 seconds: - oldTotalTime, oldSeconds, oldPidPcpu = self.__pidHistory[pid] - #checking if already updated data + # Percent cpu using decaying average, 50% from 10 seconds + # ago, 50% from last 10 seconds: + oldTotalTime, oldSeconds, oldPidPcpu = \ + self.__pidHistory[pid] + # checking if already updated data if seconds != oldSeconds: - pidPcpu = (totalTime - oldTotalTime) / float(seconds - oldSeconds) - pcpu += (oldPidPcpu + pidPcpu) / 2 # %cpu + pidPcpu = ((totalTime - oldTotalTime) / + float(seconds - oldSeconds)) + pcpu += (oldPidPcpu + pidPcpu) / 2 # %cpu pidData[pid] = totalTime, seconds, pidPcpu else: pidPcpu = totalTime / seconds pcpu += pidPcpu pidData[pid] = totalTime, seconds, pidPcpu + # pylint: disable=broad-except except Exception as e: - log.warning('Failure with pid rss update due to: %s at %s' % \ - (e, traceback.extract_tb(sys.exc_info()[2]))) + log.warning( + 'Failure with pid rss update due to: %s at %s', + e, traceback.extract_tb(sys.exc_info()[2])) rss = (rss * resource.getpagesize()) // 1024 vsize = int(vsize/1024) @@ -270,8 +288,9 @@ def rssUpdate(self, frames): # Store the current data for the next check self.__pidHistory = pidData + # pylint: disable=broad-except except Exception as e: - log.exception('Failure with rss update due to: {0}'.format(e)) + log.exception('Failure with rss update due to: %s', e) def getLoadAvg(self): """Returns average number of processes waiting to be served @@ -305,6 +324,7 @@ def getGpuMemory(self): """Returns the available gpu memory in kb for CUE_GPU_MEMORY""" return self.__getGpuValues()['free'] + # pylint: disable=attribute-defined-outside-init def __getGpuValues(self): if not hasattr(self, 'gpuNotSupported'): if not hasattr(self, 'gpuResults'): @@ -324,19 +344,23 @@ def __getGpuValues(self): results = cudaInfo.splitlines()[-1].split() # TotalMem 1023 Mb FreeMem 968 Mb # The int(math.ceil(int(x) / 32.0) * 32) rounds up to the next multiple of 32 - self.gpuResults['total'] = int(math.ceil(int(results[1]) / 32.0) * 32) * KILOBYTE + self.gpuResults['total'] = ( + int(math.ceil(int(results[1]) / 32.0) * 32) * KILOBYTE) self.gpuResults['free'] = int(results[4]) * KILOBYTE self.gpuResults['updated'] = time.time() + # pylint: disable=broad-except except Exception as e: - log.warning('Failed to get FreeMem from cudaInfo due to: %s at %s' % \ - (e, traceback.extract_tb(sys.exc_info()[2]))) + log.warning( + 'Failed to get FreeMem from cudaInfo due to: %s at %s', + e, traceback.extract_tb(sys.exc_info()[2])) return self.gpuResults def __getSwapout(self): if platform.system() == "Linux": try: return str(int(self.__vmstat.getRecentPgoutRate())) - except: + # pylint: disable=broad-except + except Exception: return str(0) return str(0) @@ -345,8 +369,7 @@ def getTimezone(self): """Returns the desired timezone""" if time.tzname[0] == 'IST': return 'IST' - else: - return 'PST8PDT' + return 'PST8PDT' @rqd.rqutil.Memoize def getHostname(self): @@ -365,7 +388,7 @@ def getTempPath(self): """Returns the correct mcp path for the given machine""" if platform.system() == "win32": return win32api.GetTempPath() - elif os.path.isdir("/mcp/"): + if os.path.isdir("/mcp/"): return "/mcp/" return '%s/' % tempfile.gettempdir() @@ -375,6 +398,7 @@ def reboot(self): log.warning("Rebooting machine") subprocess.Popen(['/usr/bin/sudo','/sbin/reboot', '-f']) + # pylint: disable=no-member def __initMachineTags(self): """Sets the hosts tags""" self.__renderHost.tags.append("rqdv-%s" % rqd.rqconstants.VERSION) @@ -398,6 +422,7 @@ def __initMachineTags(self): self.__renderHost.tags.append(os.uname()[2].replace(".EL.spi", "").replace("smp", "")) def testInitMachineStats(self, pathCpuInfo): + """Initializes machine stats outside of normal startup process. Used for testing.""" self.__initMachineStats(pathCpuInfo=pathCpuInfo) return self.__renderHost, self.__coreInfo @@ -455,7 +480,6 @@ def __initMachineStats(self, pathCpuInfo=None): self.__renderHost.total_swap = int(stat.ullTotalPageFile / 1024) # Windows CPU information - import psutil logical_core_count = psutil.cpu_count(logical=True) actual_core_count = psutil.cpu_count(logical=False) hyperthreadingMultiplier = logical_core_count // actual_core_count @@ -463,7 +487,6 @@ def __initMachineStats(self, pathCpuInfo=None): __totalCores = logical_core_count * rqd.rqconstants.CORE_VALUE __numProcs = 1 # TODO: figure out how to count sockets in Python - # All other systems will just have one proc/core if not __numProcs or not __totalCores: __numProcs = 1 @@ -490,13 +513,15 @@ def __initMachineStats(self, pathCpuInfo=None): self.__renderHost.cores_per_proc = __totalCores // __numProcs if hyperthreadingMultiplier > 1: - self.__renderHost.attributes['hyperthreadingMultiplier'] = str(hyperthreadingMultiplier) + self.__renderHost.attributes['hyperthreadingMultiplier'] = str(hyperthreadingMultiplier) def getWindowsMemory(self): - # From http://stackoverflow.com/questions/2017545/get-memory-usage-of-computer-in-windows-with-python - import ctypes + """Gets information on system memory, Windows compatible version.""" + # From + # http://stackoverflow.com/questions/2017545/get-memory-usage-of-computer-in-windows-with-python if not hasattr(self, '__windowsStat'): class MEMORYSTATUSEX(ctypes.Structure): + """Represents Windows memory information.""" _fields_ = [("dwLength", ctypes.c_uint), ("dwMemoryLoad", ctypes.c_uint), ("ullTotalPhys", ctypes.c_ulonglong), @@ -517,6 +542,7 @@ def __init__(self): return self.__windowsStat def updateMacMemory(self): + """Updates the internal store of memory available, macOS compatible version.""" memsizeOutput = subprocess.getoutput('sysctl hw.memsize').strip() memsizeRegex = re.compile(r'^hw.memsize: (?P[\d]+)$') memsizeMatch = memsizeRegex.match(memsizeOutput) @@ -638,23 +664,25 @@ def reserveHT(self, reservedCores): log.debug('Taskset: Can not reserveHT with fractional cores') return None - log.debug('Taskset: Requesting reserve of %d' % (reservedCores // 100)) + log.debug('Taskset: Requesting reserve of %d', (reservedCores // 100)) if len(self.__tasksets) < reservedCores // 100: - err = 'Not launching, insufficient hyperthreading cores to reserve based on reservedCores' + err = ('Not launching, insufficient hyperthreading cores to reserve ' + 'based on reservedCores') log.critical(err) raise rqd.rqexceptions.CoreReservationFailureException(err) tasksets = [] - for x in range(reservedCores // 100): + for _ in range(reservedCores // 100): core = self.__tasksets.pop() tasksets.append(str(core)) tasksets.append(str(core + self.__coreInfo.total_cores // 100)) - log.debug('Taskset: Reserving cores - %s' % ','.join(tasksets)) + log.debug('Taskset: Reserving cores - %s', ','.join(tasksets)) return ','.join(tasksets) + # pylint: disable=inconsistent-return-statements def releaseHT(self, reservedHT): """ Release cores used by taskset Format: 0,1,8,9 @@ -666,7 +694,7 @@ def releaseHT(self, reservedHT): if not self.__enabledHT(): return None - log.debug('Taskset: Releasing cores - %s' % reservedHT) + log.debug('Taskset: Releasing cores - %s', reservedHT) for core in reservedHT.split(','): if int(core) < self.__coreInfo.total_cores // 100: self.__tasksets.add(int(core)) diff --git a/rqd/rqd/rqnetwork.py b/rqd/rqd/rqnetwork.py index 0639a5943..da5bee5cc 100644 --- a/rqd/rqd/rqnetwork.py +++ b/rqd/rqd/rqnetwork.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -Network specific module, implements the interface with gRPC. -""" +"""Network specific module, implements the interface with gRPC.""" from __future__ import absolute_import @@ -43,6 +41,7 @@ class RunningFrame(object): + """Represents a running frame.""" def __init__(self, rqCore, runFrame): self.rqCore = rqCore @@ -97,12 +96,12 @@ def kill(self, message=""): """Kills the frame""" log.info("Request recieved: kill") if self.frameAttendantThread is None: - log.warning("Kill requested before frameAttendantThread is created " - "for: %s" % self.frameId) + log.warning( + "Kill requested before frameAttendantThread is created for: %s", self.frameId) elif self.frameAttendantThread.isAlive() and self.pid is None: - log.warning("Kill requested before pid is available for: %s" - % self.frameId) + log.warning("Kill requested before pid is available for: %s", self.frameId) elif self.frameAttendantThread.isAlive(): + # pylint: disable=broad-except try: if not self.killMessage and message: self.killMessage = message @@ -115,21 +114,21 @@ def kill(self, message=""): finally: rqd.rqutil.permissionsLow() except OSError as e: - log.warning("kill() tried to kill a non-existant pid for: %s " - "Error: %s" % (self.frameId, e)) + log.warning( + "kill() tried to kill a non-existant pid for: %s Error: %s", self.frameId, e) except Exception as e: - log.warning("kill() encountered an unknown error: %s" % e) + log.warning("kill() encountered an unknown error: %s", e) else: - log.warning("Kill requested after frameAttendantThread has exited " - "for: %s" % self.frameId) + log.warning( + "Kill requested after frameAttendantThread has exited for: %s", self.frameId) self.rqCore.deleteFrame(self.frameId) class GrpcServer(object): - """ - gRPC server class for managing messages from cuebot back to rqd. - This is used for controlling the render host and task actions initiated by cuebot and cuegui. - """ + """gRPC server class for managing messages from Cuebot back to RQD. + + This is used for controlling the render host and task actions initiated by + Cuebot and CueGUI.""" def __init__(self, rqCore): self.rqCore = rqCore @@ -139,25 +138,31 @@ def __init__(self, rqCore): self.server.add_insecure_port('[::]:{0}'.format(rqd.rqconstants.RQD_GRPC_PORT)) def addServicers(self): + """Registers the gRPC servicers defined in rqdservicers.py.""" for servicer in self.servicers: addFunc = getattr(rqd.compiled_proto.rqd_pb2_grpc, 'add_{0}_to_server'.format(servicer)) servicerClass = getattr(rqd.rqdservicers, servicer) addFunc(servicerClass(self.rqCore), self.server) def connectGrpcWithRetries(self): + """Connects to the Cuebot gRPC, with built-in retries.""" while True: try: self.rqCore.grpcConnected() break except grpc.RpcError as exc: + # pylint: disable=no-member if exc.code() == grpc.StatusCode.UNAVAILABLE: - log.warning('GRPC connection failed. Retrying in {} seconds'.format( - rqd.rqconstants.RQD_GRPC_CONNECTION_ATTEMPT_SLEEP_SEC)) + log.warning( + 'GRPC connection failed. Retrying in %s seconds', + rqd.rqconstants.RQD_GRPC_CONNECTION_ATTEMPT_SLEEP_SEC) time.sleep(rqd.rqconstants.RQD_GRPC_CONNECTION_ATTEMPT_SLEEP_SEC) else: raise exc + # pylint: enable=no-member def serve(self): + """Starts serving gRPC.""" self.addServicers() self.server.start() if rqd.rqconstants.RQD_GRPC_RETRY_CONNECTION: @@ -166,14 +171,17 @@ def serve(self): self.rqCore.grpcConnected() def serveForever(self): + """Starts serving gRPC, then enters a loop to keep running until killed.""" self.serve() self.stayAlive() def shutdown(self): + """Stops the gRPC server.""" log.info('Stopping grpc server.') self.server.stop(0) def stayAlive(self): + """Runs forever until killed.""" try: while True: time.sleep(rqd.rqconstants.RQD_GRPC_SLEEP_SEC) @@ -190,14 +198,17 @@ def __init__(self, rqCore): self.channel = None def start_grpc(self): + """Starts the gRPC server.""" self.grpcServer = GrpcServer(self.rqCore) self.grpcServer.serveForever() def stopGrpc(self): + """Stops the gRPC server.""" self.grpcServer.shutdown() del self.grpcServer def closeChannel(self): + """Closes the gRPC channel.""" self.channel.close() del self.channel self.channel = None diff --git a/rqd/rqd/rqnimby.py b/rqd/rqd/rqnimby.py index a8bf98457..ad006fca5 100644 --- a/rqd/rqd/rqnimby.py +++ b/rqd/rqd/rqnimby.py @@ -58,6 +58,8 @@ def __init__(self, rqCore): def signalHandler(self, sig, frame): """If a signal is detected, call .stop()""" + del sig + del frame self.stop() def lockNimby(self): @@ -83,12 +85,12 @@ def _openEvents(self): try: for device in os.listdir("/dev/input/"): if device.startswith("event") or device.startswith("mice"): - log.debug("Found device: %s" % device) + log.debug("Found device: %s", device) try: self.fileObjList.append(open("/dev/input/%s" % device, "rb")) except IOError as e: # Bad device found - log.debug("IOError: Failed to open %s, %s" % ("/dev/input/%s" % device, e)) + log.debug("IOError: Failed to open %s, %s", "/dev/input/%s" % device, e) finally: rqd.rqutil.permissionsLow() @@ -99,7 +101,8 @@ def _closeEvents(self): for fileObj in self.fileObjList: try: fileObj.close() - except: + # pylint: disable=broad-except + except Exception: pass self.fileObjList = [] @@ -110,7 +113,8 @@ def lockedInUse(self): self._openEvents() try: self.results = select.select(self.fileObjList, [], [], 5) - except: + # pylint: disable=broad-except + except Exception: pass if self.active and self.results[0] == []: self.lockedIdle() @@ -128,7 +132,8 @@ def lockedIdle(self): try: self.results = select.select(self.fileObjList, [], [], rqd.rqconstants.MINIMUM_IDLE) - except: + # pylint: disable=broad-except + except Exception: pass if self.active and self.results[0] == [] and \ self.rqCore.machine.isNimbySafeToUnlock(): @@ -150,7 +155,8 @@ def unlockedIdle(self): try: self._openEvents() self.results = select.select(self.fileObjList, [], [], 5) - except: + # pylint: disable=broad-except + except Exception: pass if not self.rqCore.machine.isNimbySafeToRunJobs(): log.warning("memory threshold has been exceeded, locking nimby") diff --git a/rqd/rqd/rqswap.py b/rqd/rqd/rqswap.py index 26e4bb37b..eae7259dd 100644 --- a/rqd/rqd/rqswap.py +++ b/rqd/rqd/rqswap.py @@ -13,9 +13,7 @@ # limitations under the License. -""" -Utility classes and functions to get virtual memory page out number. -""" +"""Utility classes and functions to get virtual memory page out number.""" from __future__ import with_statement @@ -36,23 +34,16 @@ class VmStatException(Exception): - """ - Something for clients to catch if something - goes wrong in here. - """ - pass + """Something for clients to catch if something goes wrong in here.""" class SampleData(object): - """ - Sample data container. - """ + """Sample data container.""" def __init__(self, epochTime, pgoutNum): """ Constructor. """ - self.__epochTimeInSecond = epochTime self.__pgpgout = pgoutNum @@ -60,9 +51,7 @@ def __repr__(self): """ Return string representation. """ - - return "(" + str(self.__epochTimeInSecond) + ", " + \ - str(self.__pgpgout) + ")" + return "(" + str(self.__epochTimeInSecond) + ", " + str(self.__pgpgout) + ")" def get_epoch_time(self): """ @@ -100,6 +89,7 @@ def run(self): self.__kwargs) timer.start() timer.join() + # pylint: disable=broad-except except Exception: # Catch all exceptions here. pass @@ -147,7 +137,7 @@ def __getPgoutNum(self): pgpgoutNum = int(matchObj.group(1)) break except IOError: - log.warn("Failed to open /proc/vmstat file.") + log.warning("Failed to open /proc/vmstat file.") if foundPgpgout: with self.__lock: @@ -155,9 +145,10 @@ def __getPgoutNum(self): pgpgoutNum)) del self.__sampleData[:-self.__sampleSize] else: - log.warn("Could not get pgpgout number.") + log.warning("Could not get pgpgout number.") def getPgoutRate(self): + """Gets the pgout rate.""" currentSampleData = self.__getSampleDataCopy() currentTime = time.time() sampleDataLen = len(currentSampleData) @@ -171,15 +162,18 @@ def getPgoutRate(self): if (currentSampleData[i].get_epoch_time() < currentTime - self.__sampleSize * self.__interval - 2): continue - weightedSum += \ - weight * (currentSampleData[i].getPgoutNumber() - currentSampleData[i - 1].getPgoutNumber()) / \ - (currentSampleData[i].getEpochTime() - currentSampleData[i - 1].getEpochTime()) + weightedSum += (weight * + (currentSampleData[i].getPgoutNumber() - + currentSampleData[i - 1].getPgoutNumber()) / + (currentSampleData[i].getEpochTime() - + currentSampleData[i - 1].getEpochTime())) totalWeight += weight weight += 1 return weightedSum / (totalWeight * self.__interval) def getRecentPgoutRate(self): + """Gets the recent pgout rate.""" currentSampleData = self.__getSampleDataCopy() sampleDataLen = len(currentSampleData) if sampleDataLen < 2: diff --git a/rqd/rqd/rqutil.py b/rqd/rqd/rqutil.py index 67a50abe0..0253dcff0 100644 --- a/rqd/rqd/rqutil.py +++ b/rqd/rqd/rqutil.py @@ -14,9 +14,7 @@ # limitations under the License. -""" -Utility functions. -""" +"""Utility functions.""" from __future__ import absolute_import @@ -29,7 +27,6 @@ import logging as log import os import platform -import random import socket import subprocess import threading @@ -46,6 +43,8 @@ class Memoize(object): + """Decorator used to cache the results of functions that only need to be run once.""" + def __init__(self, func): self.func = func self.memoized = {} @@ -59,13 +58,17 @@ def __get__(self, obj, objtype): return self.cacheGet( self.methodCache, obj, lambda: self.__class__(functools.partial(self.func, obj))) - def isCached(self, cache, key): - """Mocked in tests to disable caching as needed.""" + @staticmethod + def isCached(cache, key): + """Returns whether a given function has been cached already. + + Mocked in tests to disable caching as needed.""" if key in cache: return True return False def cacheGet(self, cache, key, func): + """Gets the cached result of a function.""" if not self.isCached(cache, key): cache[key] = func() return cache[key] @@ -80,6 +83,7 @@ def permissionsHigh(): os.seteuid(os.getuid()) try: os.setgroups(HIGH_PERMISSION_GROUPS) + # pylint: disable=broad-except except Exception: pass @@ -108,6 +112,7 @@ def permissionsUser(uid, gid): username = pwd.getpwuid(uid).pw_name groups = [20] + [g.gr_gid for g in grp.getgrall() if username in g.gr_mem] os.setgroups(groups) + # pylint: disable=broad-except except Exception: pass os.setegid(gid) @@ -121,6 +126,7 @@ def __becomeRoot(): os.seteuid(os.getuid()) try: os.setgroups(HIGH_PERMISSION_GROUPS) + # pylint: disable=broad-except except Exception: pass @@ -136,7 +142,7 @@ def checkAndCreateUser(username): except KeyError: subprocess.check_call([ 'useradd', - '-p', str(uuid.uuid4()), # generate a random password + '-p', str(uuid.uuid4()), # generate a random password username ]) @@ -145,8 +151,7 @@ def getHostIp(): """Returns the machine's local ip address""" if rqd.rqconstants.RQD_USE_IPV6_AS_HOSTNAME: return socket.getaddrinfo(socket.gethostname(), None, socket.AF_INET6)[0][4][0] - else: - return socket.gethostbyname(socket.gethostname()) + return socket.gethostbyname(socket.gethostname()) def getHostname(): @@ -154,11 +159,9 @@ def getHostname(): try: if rqd.rqconstants.OVERRIDE_HOSTNAME: return rqd.rqconstants.OVERRIDE_HOSTNAME - elif rqd.rqconstants.RQD_USE_IP_AS_HOSTNAME or \ - rqd.rqconstants.RQD_USE_IPV6_AS_HOSTNAME: + if rqd.rqconstants.RQD_USE_IP_AS_HOSTNAME or rqd.rqconstants.RQD_USE_IPV6_AS_HOSTNAME: return getHostIp() - else: - return socket.gethostbyaddr(socket.gethostname())[0].split('.')[0] + return socket.gethostbyaddr(socket.gethostname())[0].split('.')[0] except (socket.herror, socket.gaierror): log.warning("Failed to resolve hostname to IP, falling back to local hostname") return socket.gethostname() diff --git a/rqd/tests/cuerqd_tests.py b/rqd/tests/cuerqd_tests.py index b323a884d..d1ef540a1 100644 --- a/rqd/tests/cuerqd_tests.py +++ b/rqd/tests/cuerqd_tests.py @@ -15,14 +15,19 @@ # limitations under the License. +"""Tests for rqd.cuerqd.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import + from builtins import str -import mock import sys import unittest +import mock + import rqd.cuerqd import rqd.compiled_proto.rqd_pb2 diff --git a/rqd/tests/rqconstants_tests.py b/rqd/tests/rqconstants_tests.py index 7eed236c5..0df71790c 100644 --- a/rqd/tests/rqconstants_tests.py +++ b/rqd/tests/rqconstants_tests.py @@ -15,6 +15,9 @@ # limitations under the License. +"""Tests for rqd.rqconstants.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import @@ -22,7 +25,6 @@ import os.path import shutil import tempfile -import unittest import uuid import mock diff --git a/rqd/tests/rqcore_tests.py b/rqd/tests/rqcore_tests.py index 89a845ed6..90ef4618b 100644 --- a/rqd/tests/rqcore_tests.py +++ b/rqd/tests/rqcore_tests.py @@ -1,5 +1,4 @@ #!/usr/bin/env python - # Copyright Contributors to the OpenCue Project # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,15 +14,18 @@ # limitations under the License. +"""Tests for rqd.rqcore.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import from builtins import str -import mock import os.path import unittest +import mock import pyfakefs.fake_filesystem_unittest import rqd.compiled_proto.host_pb2 @@ -249,10 +251,12 @@ def test_releaseCores(self): num_booked_cores = 7 num_cores_to_release = 5 self.rqcore.cores = rqd.compiled_proto.report_pb2.CoreDetail( - total_cores=50, idle_cores=num_idle_cores, locked_cores=2, booked_cores=num_booked_cores) + total_cores=50, idle_cores=num_idle_cores, locked_cores=2, + booked_cores=num_booked_cores) self.rqcore.releaseCores(num_cores_to_release) + # pylint: disable=no-member self.assertEqual(num_booked_cores-num_cores_to_release, self.rqcore.cores.booked_cores) self.assertEqual(num_idle_cores+num_cores_to_release, self.rqcore.cores.idle_cores) @@ -364,13 +368,13 @@ def test_getRunningFrame(self): self.assertEqual(frame, self.rqcore.getRunningFrame(frameId)) self.assertIsNone(self.rqcore.getRunningFrame('some-unknown-frame-id')) - @mock.patch.object(rqd.rqcore.RqCore, 'respawn_rqd', autospec=True) + @mock.patch.object(rqd.rqcore.RqCore, 'respawn_rqd') def test_restartRqdNowNoFrames(self, respawnMock): self.nimbyMock.return_value.active = False self.rqcore.restartRqdNow() - respawnMock.assert_called_with(self.rqcore) + respawnMock.assert_called_with() @mock.patch.object(rqd.rqcore.RqCore, 'killAllFrame', autospec=True) def test_restartRqdNowWithFrames(self, killAllFrameMock): @@ -383,13 +387,13 @@ def test_restartRqdNowWithFrames(self, killAllFrameMock): killAllFrameMock.assert_called_with(self.rqcore, mock.ANY) - @mock.patch.object(rqd.rqcore.RqCore, 'respawn_rqd', autospec=True) + @mock.patch.object(rqd.rqcore.RqCore, 'respawn_rqd') def test_restartRqdIdleNoFrames(self, respawnMock): self.nimbyMock.return_value.active = False self.rqcore.restartRqdIdle() - respawnMock.assert_called_with(self.rqcore) + respawnMock.assert_called_with() @mock.patch.object(rqd.rqcore.RqCore, 'respawn_rqd') def test_restartRqdIdleWithFrames(self, respawnMock): @@ -471,6 +475,7 @@ def test_lock(self): self.rqcore.lock(20) + # pylint: disable=no-member self.assertEqual(20, self.rqcore.cores.idle_cores) self.assertEqual(30, self.rqcore.cores.locked_cores) @@ -481,6 +486,7 @@ def test_lockMoreCoresThanThereAre(self): self.rqcore.lock(100) + # pylint: disable=no-member self.assertEqual(0, self.rqcore.cores.idle_cores) self.assertEqual(50, self.rqcore.cores.locked_cores) @@ -491,6 +497,7 @@ def test_lockAll(self): self.rqcore.lockAll() + # pylint: disable=no-member self.assertEqual(0, self.rqcore.cores.idle_cores) self.assertEqual(50, self.rqcore.cores.locked_cores) @@ -502,6 +509,7 @@ def test_unlock(self): self.rqcore.unlock(20) + # pylint: disable=no-member self.assertEqual(30, self.rqcore.cores.idle_cores) self.assertEqual(20, self.rqcore.cores.locked_cores) @@ -513,6 +521,7 @@ def test_unlockMoreCoresThanThereAre(self): self.rqcore.unlock(100) + # pylint: disable=no-member self.assertEqual(50, self.rqcore.cores.idle_cores) self.assertEqual(0, self.rqcore.cores.locked_cores) @@ -525,6 +534,7 @@ def test_unlockAll(self): self.rqcore.unlockAll() + # pylint: disable=no-member self.assertEqual(50, self.rqcore.cores.idle_cores) self.assertEqual(0, self.rqcore.cores.locked_cores) @@ -537,6 +547,7 @@ def test_unlockAllWhenNimbyLocked(self): self.rqcore.unlockAll() + # pylint: disable=no-member self.assertEqual(40, self.rqcore.cores.idle_cores) self.assertEqual(0, self.rqcore.cores.locked_cores) diff --git a/rqd/tests/rqmachine_tests.py b/rqd/tests/rqmachine_tests.py index 963a10a0d..7c7a650e1 100644 --- a/rqd/tests/rqmachine_tests.py +++ b/rqd/tests/rqmachine_tests.py @@ -1,5 +1,4 @@ #!/usr/bin/env python - # Copyright Contributors to the OpenCue Project # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,14 +14,17 @@ # limitations under the License. +"""Tests for rqd.rqmachine.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import -import mock import os import unittest +import mock import pyfakefs.fake_filesystem_unittest import rqd.rqconstants @@ -35,6 +37,7 @@ import rqd.compiled_proto.report_pb2 import rqd.compiled_proto.rqd_pb2 + CPUINFO = """processor : 0 vendor_id : GenuineIntel cpu family : 6 @@ -289,6 +292,7 @@ def test_rssUpdate(self): self.machine.rssUpdate(frameCache) updatedFrameInfo = frameCache[frameId].runningFrameInfo() + # pylint: disable=no-member self.assertEqual(616, updatedFrameInfo.max_rss) self.assertEqual(616, updatedFrameInfo.rss) self.assertEqual(4356, updatedFrameInfo.max_vsize) @@ -365,6 +369,7 @@ def test_reboot(self, popenMock): 'subprocess.getoutput', new=mock.MagicMock(return_value=' TotalMem 1023 Mb FreeMem 968 Mb')) def test_getHostInfo(self): + # pylint: disable=no-member hostInfo = self.machine.getHostInfo() self.assertEqual(4105212, hostInfo.free_swap) @@ -396,6 +401,8 @@ def test_getHostReport(self): hostReport = self.machine.getHostReport() + # pylint: disable=no-member + # Verify host info was copied into the report. self.assertEqual(4105212, hostReport.host.free_swap) self.assertEqual(25699176, hostReport.host.free_mem) @@ -408,6 +415,8 @@ def test_getHostReport(self): def test_getBootReport(self): bootReport = self.machine.getBootReport() + # pylint: disable=no-member + # Verify host info was copied into the report. self.assertEqual(4105212, bootReport.host.free_swap) self.assertEqual(25699176, bootReport.host.free_mem) @@ -434,6 +443,7 @@ def test_tags(self): self.assertTrue(all(tag in machine.__dict__['_Machine__renderHost'].tags for tag in tags)) + class CpuinfoTests(unittest.TestCase): def setUp(self): @@ -474,6 +484,8 @@ def __cpuinfoTestHelper(self, pathCpuInfo): pathCpuInfo = os.path.join(os.path.dirname(__file__), 'cpuinfo', pathCpuInfo) renderHost, coreInfo = self.rqd.machine.testInitMachineStats(pathCpuInfo) totalCores, coresPerProc, numProcs = pathCpuInfo.split('_')[-1].split('-')[:3] + + # pylint: disable=no-member self.assertEqual(renderHost.num_procs, int(numProcs)) self.assertEqual(renderHost.cores_per_proc, int(coresPerProc) * 100) self.assertEqual(coreInfo.total_cores, int(totalCores) * 100) diff --git a/rqd/tests/rqnimby_tests.py b/rqd/tests/rqnimby_tests.py index a50777ef8..aa6846f44 100644 --- a/rqd/tests/rqnimby_tests.py +++ b/rqd/tests/rqnimby_tests.py @@ -1,5 +1,4 @@ #!/usr/bin/env python - # Copyright Contributors to the OpenCue Project # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,13 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. + +"""Tests for rqd.rqnimby.""" + + from __future__ import print_function from __future__ import division from __future__ import absolute_import -import mock import unittest +import mock import pyfakefs.fake_filesystem_unittest import rqd.rqcore diff --git a/rqd/tests/test_cuebot_listener.py b/rqd/tests/test_cuebot_listener.py index 1098f4241..160951f05 100644 --- a/rqd/tests/test_cuebot_listener.py +++ b/rqd/tests/test_cuebot_listener.py @@ -65,7 +65,7 @@ def shutdown(self): def stayAlive(self): try: while True: - time.sleep(rqd.rqconstants.RQD_GRPC_SLEEP) + time.sleep(rqd.rqconstants.RQD_GRPC_SLEEP_SEC) except KeyboardInterrupt: self.server.stop(0) @@ -92,14 +92,14 @@ def _trackUpdateTime(self, report): for host in sorted(self.statusCheckin.keys()): secondsSinceLast = now - self.statusCheckin[host]["last"] if host == report.host.name: - print(" >", end=' ') + print(" >", end=' ') else: - print(" ", end=' ') - print(host.ljust(15) \ - , str(int(secondsSinceLast)).ljust(10) \ - , str(self.statusCheckin[host]["report"].host.load).ljust(5) \ - , str(self.statusCheckin[host]["report"].host.freeMem).ljust(10) \ - , ",".join(self.statusCheckin[host]["report"].host.tags)) + print(" ", end=' ') + print( + host.ljust(15), str(int(secondsSinceLast)).ljust(10), + str(self.statusCheckin[host]["report"].host.load).ljust(5), + str(self.statusCheckin[host]["report"].host.freeMem).ljust(10), + ",".join(self.statusCheckin[host]["report"].host.tags)) def ReportRqdStartup(self, request, context): report = request.bootReport @@ -111,10 +111,25 @@ def ReportRqdStartup(self, request, context): sys.stdout.write("1") sys.stdout.flush() elif self.verbose == 2: - print("%s : startup.host - nimbyEnabled = %s, numProcs = %s, coresPerProc = %d, load = %s, bootTime = %s" % (report.host.name, report.host.nimbyEnabled, report.host.numProcs, report.host.coresPerProc, report.host.load,report.host.bootTime)) - print("%s : startup.host - totalSwap = %s, totalMem = %s, totalMcp = %s, freeSwap = %s, freeMem = %s, freeMcp = %s" % (report.host.name, report.host.totalSwap, report.host.totalMem, report.host.totalMcp, report.host.freeSwap, report.host.freeMem, report.host.freeMcp)) - print("%s : startup.host - tags = %s, state = %s" % (report.host.name, report.host.tags, report.host.state)) - print("%s : startup.coreInfo - totalCores = %s, idleCores = %s, lockedCores = %s, bookedCores = %s" % (report.host.name, report.coreInfo.totalCores, report.coreInfo.idleCores, report.coreInfo.lockedCores, report.coreInfo.bookedCores)) + print( + "%s : startup.host - nimbyEnabled = %s, numProcs = %s, coresPerProc = %d, " + "load = %s, bootTime = %s" % ( + report.host.name, report.host.nimbyEnabled, report.host.numProcs, + report.host.coresPerProc, report.host.load, report.host.bootTime)) + print( + "%s : startup.host - totalSwap = %s, totalMem = %s, totalMcp = %s, " + "freeSwap = %s, freeMem = %s, freeMcp = %s" % ( + report.host.name, report.host.totalSwap, report.host.totalMem, + report.host.totalMcp, report.host.freeSwap, report.host.freeMem, + report.host.freeMcp)) + print( + "%s : startup.host - tags = %s, state = %s" % ( + report.host.name, report.host.tags, report.host.state)) + print( + "%s : startup.coreInfo - totalCores = %s, idleCores = %s, lockedCores = %s, " + "bookedCores = %s" % ( + report.host.name, report.coreInfo.totalCores, report.coreInfo.idleCores, + report.coreInfo.lockedCores, report.coreInfo.bookedCores)) elif self.verbose == 3: print("Receiving reportRqdStartup") print(report) @@ -131,12 +146,30 @@ def ReportStatus(self, request, context): sys.stdout.write(".") sys.stdout.flush() elif self.verbose == 2: - print("%s : status.host - nimbyEnabled = %s, numProcs = %s, coresPerProc = %d, load = %s, bootTime = %s" % (report.host.name, report.host.nimbyEnabled, report.host.numProcs, report.host.coresPerProc, report.host.load,report.host.bootTime)) - print("%s : status.host - totalSwap = %s, totalMem = %s, totalMcp = %s, freeSwap = %s, freeMem = %s, freeMcp = %s" % (report.host.name, report.host.totalSwap, report.host.totalMem, report.host.totalMcp, report.host.freeSwap, report.host.freeMem, report.host.freeMcp)) - print("%s : status.host - tags = %s, state = %s" % (report.host.name, report.host.tags, report.host.state)) + print( + "%s : status.host - nimbyEnabled = %s, numProcs = %s, coresPerProc = %d, " + "load = %s, bootTime = %s" % ( + report.host.name, report.host.nimbyEnabled, report.host.numProcs, + report.host.coresPerProc, report.host.load,report.host.bootTime)) + print( + "%s : status.host - totalSwap = %s, totalMem = %s, totalMcp = %s, " + "freeSwap = %s, freeMem = %s, freeMcp = %s" % ( + report.host.name, report.host.totalSwap, report.host.totalMem, + report.host.totalMcp, report.host.freeSwap, report.host.freeMem, + report.host.freeMcp)) + print( + "%s : status.host - tags = %s, state = %s" % ( + report.host.name, report.host.tags, report.host.state)) for job in report.frames: - print("%s : status.frames[x] - frameId = %s, jobId = %s, numCores = %d, usedMem = %s" % (report.host.name, job.frameId, job.jobId, job.numCores, job.usedMem)) - print("%s : status.coreInfo - totalCores = %s, idleCores = %s, lockedCores = %s, bookedCores = %s" % (report.host.name, report.coreInfo.totalCores, report.coreInfo.idleCores, report.coreInfo.lockedCores, report.coreInfo.bookedCores)) + print( + "%s : status.frames[x] - frameId = %s, jobId = %s, numCores = %d, " + "usedMem = %s" % ( + report.host.name, job.frameId, job.jobId, job.numCores, job.usedMem)) + print( + "%s : status.coreInfo - totalCores = %s, idleCores = %s, lockedCores = %s, " + "bookedCores = %s" % ( + report.host.name, report.coreInfo.totalCores, report.coreInfo.idleCores, + report.coreInfo.lockedCores, report.coreInfo.bookedCores)) elif self.verbose == 3: print("Receiving reportStatus") print(report) @@ -153,11 +186,28 @@ def ReportRunningFrameCompletion(self, request, context): sys.stdout.write("X") sys.stdout.flush() elif self.verbose == 2: - print("%s : FrameCompletion.host - nimbyEnabled = %s, numProcs = %s, coresPerProc = %d, load = %s, bootTime = %s" % (report.host.name, report.host.nimbyEnabled, report.host.numProcs, report.host.coresPerProc, report.host.load,report.host.bootTime)) - print("%s : FrameCompletion.host - totalSwap = %s, totalMem = %s, totalMcp = %s, freeSwap = %s, freeMem = %s, freeMcp = %s" % (report.host.name, report.host.totalSwap, report.host.totalMem, report.host.totalMcp, report.host.freeSwap, report.host.freeMem, report.host.freeMcp)) + print( + "%s : FrameCompletion.host - nimbyEnabled = %s, numProcs = %s, " + "coresPerProc = %d, load = %s, bootTime = %s" % ( + report.host.name, report.host.nimbyEnabled, report.host.numProcs, + report.host.coresPerProc, report.host.load,report.host.bootTime)) + print( + "%s : FrameCompletion.host - totalSwap = %s, totalMem = %s, totalMcp = %s, " + "freeSwap = %s, freeMem = %s, freeMcp = %s" % ( + report.host.name, report.host.totalSwap, report.host.totalMem, + report.host.totalMcp, report.host.freeSwap, report.host.freeMem, + report.host.freeMcp)) print("%s : FrameCompletion.host - tags = %s" % (report.host.name, report.host.tags)) - print("%s : FrameCompletion.frame - jobId = %s, frameId = %s, numCores = %d, usedMem = %d" % (report.host.name, report.frame.jobId, report.frame.frameId, report.frame.numCores, report.frame.usedMem)) - print("%s : FrameCompletion - exitStatus = %s, exitSignal = %s, runTime = %s, maxRss = %s" % (report.host.name, report.exitStatus, report.exitSignal, report.runTime, report.maxRss)) + print( + "%s : FrameCompletion.frame - jobId = %s, frameId = %s, numCores = %d, " + "usedMem = %d" % ( + report.host.name, report.frame.jobId, report.frame.frameId, + report.frame.numCores, report.frame.usedMem)) + print( + "%s : FrameCompletion - exitStatus = %s, exitSignal = %s, " + "runTime = %s, maxRss = %s" % ( + report.host.name, report.exitStatus, report.exitSignal, report.runTime, + report.maxRss)) elif self.verbose == 3: print("Receiving reportRunningFrameCompletion") print(report) From ca6f4ba917a2a95a01de8a83ce029b05ffa3cd16 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 22 Feb 2021 18:08:40 +0000 Subject: [PATCH 061/277] Add unit tests for DependWizard. (#922) --- cuegui/tests/DependWizard_tests.py | 271 +++++++++++++++++++++++++++++ 1 file changed, 271 insertions(+) create mode 100644 cuegui/tests/DependWizard_tests.py diff --git a/cuegui/tests/DependWizard_tests.py b/cuegui/tests/DependWizard_tests.py new file mode 100644 index 000000000..0e8667460 --- /dev/null +++ b/cuegui/tests/DependWizard_tests.py @@ -0,0 +1,271 @@ +# Copyright (c) OpenCue Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Tests for cuegui.DependWizard.""" + + +import unittest + +import mock +import PySide2.QtCore +import PySide2.QtGui +import PySide2.QtWidgets +import PySide2.QtTest + +import opencue.compiled_proto.job_pb2 +import opencue.wrappers.frame +import opencue.wrappers.layer +import opencue.wrappers.job + +import cuegui.DependWizard +import cuegui.Style + +from . import test_utils + + +@mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) +class DependWizardTests(unittest.TestCase): + + @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) + def setUp(self): + test_utils.createApplication() + PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + cuegui.Style.init() + + self.parentWidget = PySide2.QtWidgets.QWidget() + + @mock.patch('cuegui.Cuedepend.createJobOnLayerDepend') + @mock.patch('opencue.api.findJob') + @mock.patch('opencue.api.getJobNames') + def test_job_on_layer(self, get_job_names_mock, find_job_mock, create_depend_mock): + show_name = 'arbitraryshow' + shot_name = 'sh01' + user_name = 'arbitraryuser' + job_prefix = '%s-%s-%s' % (show_name, shot_name, user_name) + job_dependon_name = '%s_dependon' % job_prefix + job_depender_name = '%s_depender' % job_prefix + layer_dependon_name = 'arbitraryLayerName' + + # Create depend-on job with one layer + job_dependon = opencue.wrappers.job.Job( + opencue.compiled_proto.job_pb2.Job(id=job_dependon_name, name=job_dependon_name, + show='arbitraryshow')) + layer_dependon = opencue.wrappers.layer.Layer( + opencue.compiled_proto.job_pb2.Layer(name=layer_dependon_name)) + job_dependon.getLayers = lambda: [layer_dependon] + + # Create depend-er job, no layers needed + job_depender = opencue.wrappers.job.Job( + opencue.compiled_proto.job_pb2.Job(id=job_depender_name, name=job_depender_name, + show='arbitraryshow')) + + # API mocks to allow wizard to load job information + get_job_names_mock.return_value = [job_dependon_name, job_depender_name] + find_job_mock.side_effect = lambda name: job_dependon if name == job_dependon_name else None + + # Create the Depend Wizard + depend_wizard = cuegui.DependWizard.DependWizard(self.parentWidget, [job_depender]) + + # Select job-on-layer then go to next page + depend_type_page = depend_wizard.page(cuegui.DependWizard.PAGE_SELECT_DEPEND_TYPE) + jol_option = depend_type_page._PageDependType__options[cuegui.DependWizard.JOL] + jol_option.setChecked(True) + depend_wizard.next() + self.assertEqual(cuegui.DependWizard.PAGE_SELECT_ONJOB, depend_wizard.currentId()) + + # Ensure the depend-on job is the only one selected, then go to next page + select_on_job_page = depend_wizard.page(cuegui.DependWizard.PAGE_SELECT_ONJOB) + job_list = select_on_job_page._PageSelectOnJob__jobList + for job_list_index in range(job_list.count()): + if job_list.item(job_list_index).text() == job_dependon_name: + job_list.item(job_list_index).setSelected(True) + else: + job_list.item(job_list_index).setSelected(False) + depend_wizard.next() + self.assertEqual(cuegui.DependWizard.PAGE_SELECT_ONLAYER, depend_wizard.currentId()) + + # Ensure the depend-on layer is the only one selected, then go to the next page + select_on_layer_page = depend_wizard.page(cuegui.DependWizard.PAGE_SELECT_ONLAYER) + layer_list = select_on_layer_page._PageSelectOnLayer__onLayerList + for layer_list_index in range(layer_list.count()): + if layer_list.item(layer_list_index).text() == layer_dependon_name: + layer_list.item(layer_list_index).setSelected(True) + else: + layer_list.item(layer_list_index).setSelected(False) + depend_wizard.next() + self.assertEqual(cuegui.DependWizard.PAGE_CONFIRMATION, depend_wizard.currentId()) + + # Proceed past confirmation screen to create the depend + depend_wizard.next() + + create_depend_mock.assert_called_with( + job_depender_name, job_dependon_name, layer_dependon_name) + + @mock.patch('cuegui.Cuedepend.createLayerOnJobDepend') + @mock.patch('opencue.api.findJob') + @mock.patch('opencue.api.getJobNames') + def test_layer_on_job(self, get_job_names_mock, find_job_mock, create_depend_mock): + show_name = 'arbitraryshow' + shot_name = 'sh01' + user_name = 'arbitraryuser' + job_prefix = '%s-%s-%s' % (show_name, shot_name, user_name) + job_dependon_name = '%s_dependon' % job_prefix + job_depender_name = '%s_depender' % job_prefix + layer_depender_name = 'arbitraryLayerName' + + # Create depend-on job, no layers needed + job_dependon = opencue.wrappers.job.Job( + opencue.compiled_proto.job_pb2.Job(id=job_dependon_name, name=job_dependon_name, + show='arbitraryshow')) + + # Create depend-er job with one layer + job_depender = opencue.wrappers.job.Job( + opencue.compiled_proto.job_pb2.Job(id=job_depender_name, name=job_depender_name, + show='arbitraryshow')) + layer_depender = opencue.wrappers.layer.Layer( + opencue.compiled_proto.job_pb2.Layer(name=layer_depender_name)) + job_depender.getLayers = lambda: [layer_depender] + + # API mocks to allow wizard to load job information + get_job_names_mock.return_value = [job_dependon_name, job_depender_name] + find_job_mock.side_effect = lambda name: job_dependon if name == job_dependon_name else None + + # Create the Depend Wizard + depend_wizard = cuegui.DependWizard.DependWizard(self.parentWidget, [job_depender]) + + # Select layer-on-job then go to next page + depend_type_page = depend_wizard.page(cuegui.DependWizard.PAGE_SELECT_DEPEND_TYPE) + loj_option = depend_type_page._PageDependType__options[cuegui.DependWizard.LOJ] + loj_option.setChecked(True) + depend_wizard.next() + self.assertEqual(cuegui.DependWizard.PAGE_SELECT_JOB_LAYER, depend_wizard.currentId()) + + # Ensure the depend-er layer is the only one selected, then go to next page + select_job_layer_page = depend_wizard.page(cuegui.DependWizard.PAGE_SELECT_JOB_LAYER) + layer_list = select_job_layer_page._PageSelectLayer__layerList + for layer_list_index in range(layer_list.count()): + if layer_list.item(layer_list_index).text() == layer_depender_name: + layer_list.item(layer_list_index).setSelected(True) + else: + layer_list.item(layer_list_index).setSelected(False) + depend_wizard.next() + self.assertEqual(cuegui.DependWizard.PAGE_SELECT_ONJOB, depend_wizard.currentId()) + + # Ensure the depend-on job is the only one selected, then go to next page + select_on_job_page = depend_wizard.page(cuegui.DependWizard.PAGE_SELECT_ONJOB) + job_list = select_on_job_page._PageSelectOnJob__jobList + for job_list_index in range(job_list.count()): + if job_list.item(job_list_index).text() == job_dependon_name: + job_list.item(job_list_index).setSelected(True) + else: + job_list.item(job_list_index).setSelected(False) + depend_wizard.next() + self.assertEqual(cuegui.DependWizard.PAGE_CONFIRMATION, depend_wizard.currentId()) + + # Proceed past confirmation screen to create the depend + depend_wizard.next() + + create_depend_mock.assert_called_with( + job_depender_name, layer_depender_name, job_dependon_name) + + @mock.patch('cuegui.Cuedepend.createFrameOnFrameDepend') + @mock.patch('opencue.api.findJob') + @mock.patch('opencue.api.getJobNames') + def test_frame_on_frame(self, get_job_names_mock, find_job_mock, create_depend_mock): + show_name = 'arbitraryshow' + shot_name = 'sh01' + user_name = 'arbitraryuser' + job_prefix = '%s-%s-%s' % (show_name, shot_name, user_name) + job_dependon_name = '%s_dependon' % job_prefix + layer_dependon_name = 'layerDependonName' + frame_dependon_num = 1 + job_depender_name = '%s_depender' % job_prefix + layer_depender_name = 'layerDependerName' + frame_depender_name = '0040-%s' % layer_depender_name + frame_depender_num = 40 + + # Create depend-on job with one layer, no frames needed (frame number only is used) + job_dependon = opencue.wrappers.job.Job( + opencue.compiled_proto.job_pb2.Job(id=job_dependon_name, name=job_dependon_name, + show='arbitraryshow')) + layer_dependon = opencue.wrappers.layer.Layer( + opencue.compiled_proto.job_pb2.Layer(name=layer_dependon_name)) + job_dependon.getLayers = lambda: [layer_dependon] + + # Create depend-er job with one layer and one frame + job_depender = opencue.wrappers.job.Job( + opencue.compiled_proto.job_pb2.Job(id=job_depender_name, name=job_depender_name, + show='arbitraryshow')) + layer_depender = opencue.wrappers.layer.Layer( + opencue.compiled_proto.job_pb2.Layer(name=layer_depender_name)) + frame_depender = opencue.wrappers.frame.Frame( + opencue.compiled_proto.job_pb2.Frame( + id='arbitrary-frame-id', name=frame_depender_name, layer_name=layer_depender_name, + number=frame_depender_num)) + + # API mocks to allow wizard to load job information + get_job_names_mock.return_value = [job_dependon_name, job_depender_name] + find_job_mock.side_effect = lambda name: job_dependon if name == job_dependon_name else None + + # Create the Depend Wizard + depend_wizard = cuegui.DependWizard.DependWizard( + self.parentWidget, [job_depender], layers=[layer_depender], frames=[frame_depender]) + + # Select frame-on-frame then go to next page + depend_type_page = depend_wizard.page(cuegui.DependWizard.PAGE_SELECT_DEPEND_TYPE) + loj_option = depend_type_page._PageDependType__options[cuegui.DependWizard.FOF] + loj_option.setChecked(True) + depend_wizard.next() + self.assertEqual(cuegui.DependWizard.PAGE_SELECT_ONJOB, depend_wizard.currentId()) + + # Ensure the depend-on job is the only one selected, then go to next page + select_on_job_page = depend_wizard.page(cuegui.DependWizard.PAGE_SELECT_ONJOB) + job_list = select_on_job_page._PageSelectOnJob__jobList + for job_list_index in range(job_list.count()): + if job_list.item(job_list_index).text() == job_dependon_name: + job_list.item(job_list_index).setSelected(True) + else: + job_list.item(job_list_index).setSelected(False) + depend_wizard.next() + self.assertEqual(cuegui.DependWizard.PAGE_SELECT_ONLAYER, depend_wizard.currentId()) + + # Ensure the depend-on layer is the only one selected, then go to the next page + select_on_layer_page = depend_wizard.page(cuegui.DependWizard.PAGE_SELECT_ONLAYER) + layer_list = select_on_layer_page._PageSelectOnLayer__onLayerList + for layer_list_index in range(layer_list.count()): + if layer_list.item(layer_list_index).text() == layer_dependon_name: + layer_list.item(layer_list_index).setSelected(True) + else: + layer_list.item(layer_list_index).setSelected(False) + depend_wizard.next() + self.assertEqual(cuegui.DependWizard.PAGE_SELECT_ONFRAME, depend_wizard.currentId()) + + # Input the depend-on frame number as the depend-on frame, then go to the next page + select_on_frame_page = depend_wizard.page(cuegui.DependWizard.PAGE_SELECT_ONFRAME) + frame_field = select_on_frame_page._PageSelectOnFrame__frame + frame_field.setText(str(frame_dependon_num)) + depend_wizard.next() + self.assertEqual(cuegui.DependWizard.PAGE_CONFIRMATION, depend_wizard.currentId()) + + # Proceed past confirmation screen to create the depend + depend_wizard.next() + + create_depend_mock.assert_called_with( + job_depender_name, layer_depender_name, frame_depender_num, job_dependon_name, + layer_dependon_name, frame_dependon_num) + + +if __name__ == '__main__': + unittest.main() From 0d5c0c770c7411e6126626c1b7b69b5bfde83c5d Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 22 Feb 2021 18:24:11 +0000 Subject: [PATCH 062/277] Add unit tests for LayerDialog. (#926) --- ci/pylintrc_test | 1 + cuegui/cuegui/LayerDialog.py | 4 +- cuegui/tests/LayerDialog_tests.py | 257 ++++++++++++++++++++++++++++++ 3 files changed, 260 insertions(+), 2 deletions(-) create mode 100644 cuegui/tests/LayerDialog_tests.py diff --git a/ci/pylintrc_test b/ci/pylintrc_test index a436236f2..e4f09c50d 100644 --- a/ci/pylintrc_test +++ b/ci/pylintrc_test @@ -73,6 +73,7 @@ disable=arguments-differ, too-many-lines, too-many-locals, too-many-public-methods, + too-many-statements, unused-argument, unused-variable, useless-object-inheritance diff --git a/cuegui/cuegui/LayerDialog.py b/cuegui/cuegui/LayerDialog.py index 989df17cd..c337dda80 100644 --- a/cuegui/cuegui/LayerDialog.py +++ b/cuegui/cuegui/LayerDialog.py @@ -180,7 +180,7 @@ def __init__(self, layers, parent=None): # Limits self.__limits = LayerLimitsWidget(self.__layers, self) - ## GPU Memory + # GPU Memory self.__gpu = SlideSpinner(self) self.__gpu.slider.setMinimumWidth(200) self.__gpu.slider.setRange(self.gpu_min_kb, self.gpu_max_kb // self.gpu_tick_kb) @@ -470,7 +470,7 @@ def apply(self): class LayerTagsDialog(QtWidgets.QDialog): - """Dialog for displayer a layer's tags.""" + """Dialog for displaying a layer's tags.""" def __init__(self, layers, parent=None): QtWidgets.QDialog.__init__(self, parent) diff --git a/cuegui/tests/LayerDialog_tests.py b/cuegui/tests/LayerDialog_tests.py new file mode 100644 index 000000000..1f9624d59 --- /dev/null +++ b/cuegui/tests/LayerDialog_tests.py @@ -0,0 +1,257 @@ +# Copyright (c) OpenCue Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Tests for cuegui.LayerDialog.""" + + +import unittest + +import mock + +import PySide2.QtCore +import PySide2.QtGui +import PySide2.QtWidgets + +import opencue.compiled_proto.show_pb2 +import opencue.compiled_proto.filter_pb2 +import opencue.compiled_proto.job_pb2 +import opencue.compiled_proto.limit_pb2 +import opencue.wrappers.filter +import opencue.wrappers.layer +import opencue.wrappers.limit +import opencue.wrappers.show + +import cuegui.LayerDialog +import cuegui.Style +import cuegui.Utils + +from . import test_utils + + +@mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) +class LayerPropertiesDialogTests(unittest.TestCase): + + @mock.patch('opencue.api.getLimits') + @mock.patch('opencue.api.getLayer') + @mock.patch('opencue.cuebot.Cuebot.getStub') + def setUp(self, get_stub_mock, get_layer_mock, get_limits_mock): + test_utils.createApplication() + PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + cuegui.Style.init() + + self.layers = { + 'layer1Id': opencue.wrappers.layer.Layer( + opencue.compiled_proto.job_pb2.Layer( + id='layer1Id', name='layer1Name', range='1-5', tags=['tag1', 'tag2'], + min_cores=1, max_cores=3, is_threadable=False, min_memory=2097152, min_gpu=1, + chunk_size=1, timeout=30, timeout_llu=1, memory_optimizer_enabled=True, + limits=['limit1Name', 'limit2Name'])), + 'layer2Id': opencue.wrappers.layer.Layer( + opencue.compiled_proto.job_pb2.Layer( + id='layer2Id', name='layer2Name', range='2-22', tags=['tag2', 'tag3'], + min_cores=2, max_cores=2, is_threadable=True, min_memory=6291456, min_gpu=2, + chunk_size=5, timeout=60, timeout_llu=5, memory_optimizer_enabled=False, + limits=['limit2Name', 'limit3Name'])), + } + + get_layer_mock.side_effect = lambda layer_id: self.layers[layer_id] + get_limits_mock.return_value = [ + opencue.wrappers.limit.Limit( + opencue.compiled_proto.limit_pb2.Limit(id='limit1Id', name='limit1Name')), + opencue.wrappers.limit.Limit( + opencue.compiled_proto.limit_pb2.Limit(id='limit2Id', name='limit2Name')), + opencue.wrappers.limit.Limit( + opencue.compiled_proto.limit_pb2.Limit(id='limit3Id', name='limit3Name')), + opencue.wrappers.limit.Limit( + opencue.compiled_proto.limit_pb2.Limit(id='limit4Id', name='limit4Name')), + ] + + self.parent_widget = PySide2.QtWidgets.QWidget() + self.layer_properties_dialog = cuegui.LayerDialog.LayerPropertiesDialog( + ['layer1Id', 'layer2Id'], parent=self.parent_widget) + + def test__should_display_current_values(self): + default_config = cuegui.Utils.getResourceConfig() + + self.assertEqual( + int(self.layer_properties_dialog.mem_min_gb * 1024 * 1024), + self.layer_properties_dialog._LayerPropertiesDialog__mem.slider.minimum()) + self.assertEqual( + default_config['max_memory'] * 1024 * 1024, + self.layer_properties_dialog._LayerPropertiesDialog__mem.slider.maximum()) + # Layer with the higher min_memory determines the initial value. + self.assertEqual( + 6291456, self.layer_properties_dialog._LayerPropertiesDialog__mem.slider.value()) + + # Is memory optimizer is on for any layer, it shows as checked in the dialog. + self.assertTrue(self.layer_properties_dialog._LayerPropertiesDialog__mem_opt.isChecked()) + + self.assertEqual( + 0, + self.layer_properties_dialog._LayerPropertiesDialog__core.minimum()) + self.assertEqual( + default_config['max_cores'], + self.layer_properties_dialog._LayerPropertiesDialog__core.maximum()) + # Layer with the higher min_cores determines the initial value. + self.assertEqual( + 2, + self.layer_properties_dialog._LayerPropertiesDialog__core.value()) + + self.assertEqual( + 0, + self.layer_properties_dialog._LayerPropertiesDialog__max_cores.minimum()) + self.assertEqual( + default_config['max_cores'], + self.layer_properties_dialog._LayerPropertiesDialog__max_cores.maximum()) + # Layer with the higher max_cores determines the initial value. + self.assertEqual( + 3, + self.layer_properties_dialog._LayerPropertiesDialog__max_cores.value()) + + # Is any layer is threadable, it shows as checked in the dialog. + self.assertTrue(self.layer_properties_dialog._LayerPropertiesDialog__thread.isChecked()) + + self.assertEqual( + int(self.layer_properties_dialog.gpu_min_gb * 1024 * 1024), + self.layer_properties_dialog._LayerPropertiesDialog__gpu.slider.minimum()) + self.assertEqual( + int(self.layer_properties_dialog.gpu_max_gb * 1024 * 1024) // + int(self.layer_properties_dialog.gpu_tick_gb * 1024 * 1024), + self.layer_properties_dialog._LayerPropertiesDialog__gpu.slider.maximum()) + + # Layer with the highest timeout determines the initial value. + self.assertEqual(60, self.layer_properties_dialog._LayerPropertiesDialog__timeout.value()) + + # Layer with the highest LLU timeout determines the initial value. + self.assertEqual( + 5, self.layer_properties_dialog._LayerPropertiesDialog__timeout_llu.value()) + + # Tags list should contain a union of tags in all layers, without duplicating any tags + # which appear in multiple layers. + self.assertListEqual( + ['tag1', 'tag2', 'tag3'], + sorted( + self.layer_properties_dialog._LayerPropertiesDialog__tags._tags_widget.get_tags())) + + # Limits list will contain a list of all limits defined by opencue.api.getLimits. The only + # checked ones should be the union of limits in all layers, without duplicating any limits + # which appear in multiple layers. + limits_widget = self.layer_properties_dialog._LayerPropertiesDialog__limits._limits_widget + self.assertListEqual( + ['limit1Name', 'limit2Name', 'limit3Name'], sorted(limits_widget.get_selected_limits())) + + def test__should_fail_on_memory_too_high(self): + self.layer_properties_dialog._LayerPropertiesDialog__mem.slider.setValue( + self.layer_properties_dialog.mem_max_kb * 2) + self.assertFalse(self.layer_properties_dialog.verify()) + + def test__should_fail_on_memory_too_low(self): + self.layer_properties_dialog._LayerPropertiesDialog__mem.slider.setValue( + self.layer_properties_dialog.mem_min_kb / 3) + self.assertFalse(self.layer_properties_dialog.verify()) + + def test__should_fail_on_gpu_too_high(self): + self.layer_properties_dialog._LayerPropertiesDialog__gpu.slider.setValue( + self.layer_properties_dialog.gpu_max_kb * 2) + self.assertFalse(self.layer_properties_dialog.verify()) + + def test__should_fail_on_gpu_too_low(self): + self.layer_properties_dialog._LayerPropertiesDialog__gpu.slider.setValue( + self.layer_properties_dialog.gpu_min_kb / 3) + self.assertFalse(self.layer_properties_dialog.verify()) + + def test__should_apply_new_settings(self): + layer1_mock = mock.Mock() + layer1_mock.limits.return_value = ['limit1Name', 'limit2Name'] + layer2_mock = mock.Mock() + layer2_mock.limits.return_value = ['limit2Name', 'limit3Name'] + self.layer_properties_dialog._LayerPropertiesDialog__layers = [layer1_mock, layer2_mock] + self.layer_properties_dialog._LayerPropertiesDialog__tags._LayerTagsWidget__layers = [ + layer1_mock, layer2_mock] + self.layer_properties_dialog._LayerPropertiesDialog__limits._LayerLimitsWidget__layers = [ + layer1_mock, layer2_mock] + + new_mem_value = self.layer_properties_dialog.mem_max_kb + self.layer_properties_dialog._LayerPropertiesDialog__mem.parent().parent().enable(True) + self.layer_properties_dialog._LayerPropertiesDialog__mem.slider.setValue(new_mem_value) + + new_mem_opt_is_enabled = False + self.layer_properties_dialog._LayerPropertiesDialog__mem_opt.parent().parent().enable(True) + self.layer_properties_dialog._LayerPropertiesDialog__mem_opt.setChecked( + new_mem_opt_is_enabled) + + new_min_cores = 10 + self.layer_properties_dialog._LayerPropertiesDialog__core.parent().parent().enable(True) + self.layer_properties_dialog._LayerPropertiesDialog__core.setDisabled(False) + self.layer_properties_dialog._LayerPropertiesDialog__core.setValue(new_min_cores) + + new_max_cores = 12 + self.layer_properties_dialog._LayerPropertiesDialog__max_cores.parent().parent().enable( + True) + self.layer_properties_dialog._LayerPropertiesDialog__max_cores.setValue(new_max_cores) + + new_is_threadable = False + self.layer_properties_dialog._LayerPropertiesDialog__thread.parent().parent().enable(True) + self.layer_properties_dialog._LayerPropertiesDialog__thread.setChecked(new_is_threadable) + + new_min_gpu = 6 + self.layer_properties_dialog._LayerPropertiesDialog__gpu.parent().parent().enable(True) + self.layer_properties_dialog._LayerPropertiesDialog__gpu.slider.setValue(new_min_gpu) + + new_timeout = 20 + self.layer_properties_dialog._LayerPropertiesDialog__timeout.parent().parent().enable(True) + self.layer_properties_dialog._LayerPropertiesDialog__timeout.setValue(new_timeout) + + new_timeout_llu = 15 + self.layer_properties_dialog._LayerPropertiesDialog__timeout_llu.parent().parent().enable( + True) + self.layer_properties_dialog._LayerPropertiesDialog__timeout_llu.setValue(new_timeout_llu) + + new_tags = ['newTag1', 'newTag2'] + self.layer_properties_dialog._LayerPropertiesDialog__tags.parent().enable(True) + self.layer_properties_dialog._LayerPropertiesDialog__tags._tags_widget.set_tags(new_tags) + + new_limits = ['limit3Name', 'limit4Name'] + self.layer_properties_dialog._LayerPropertiesDialog__limits.parent().enable(True) + self.layer_properties_dialog._LayerPropertiesDialog__limits._limits_widget.enable_limits( + limits_to_enable=new_limits) + + self.layer_properties_dialog.apply() + layer1_mock.setMinMemory.assert_called_with(new_mem_value) + layer2_mock.setMinMemory.assert_called_with(new_mem_value) + layer1_mock.enableMemoryOptimizer.assert_called_with(new_mem_opt_is_enabled) + layer2_mock.enableMemoryOptimizer.assert_called_with(new_mem_opt_is_enabled) + layer1_mock.setMinCores.assert_called_with(100 * new_min_cores) + layer2_mock.setMinCores.assert_called_with(100 * new_min_cores) + layer1_mock.setMaxCores.assert_called_with(100 * new_max_cores) + layer2_mock.setMaxCores.assert_called_with(100 * new_max_cores) + layer1_mock.setThreadable.assert_called_with(new_is_threadable) + layer2_mock.setThreadable.assert_called_with(new_is_threadable) + layer1_mock.setMinGpu.assert_called_with( + new_min_gpu * self.layer_properties_dialog.gpu_tick_kb) + layer2_mock.setMinGpu.assert_called_with( + new_min_gpu * self.layer_properties_dialog.gpu_tick_kb) + layer1_mock.setTimeout.assert_called_with(new_timeout) + layer2_mock.setTimeout.assert_called_with(new_timeout) + layer1_mock.setTimeoutLLU.assert_called_with(new_timeout_llu) + layer2_mock.setTimeoutLLU.assert_called_with(new_timeout_llu) + layer1_mock.setTags.assert_called_with(new_tags) + layer2_mock.setTags.assert_called_with(new_tags) + layer1_mock.addLimit.assert_has_calls( + [mock.call('limit3Id'), mock.call('limit4Id')], any_order=True) + layer1_mock.dropLimit.assert_has_calls( + [mock.call('limit1Id'), mock.call('limit2Id')], any_order=True) + layer2_mock.addLimit.assert_has_calls([mock.call('limit4Id')]) + layer2_mock.dropLimit.assert_has_calls([mock.call('limit2Id')]) From 4175096c25306aeffb6ef2be2785f796422c3c67 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 23 Feb 2021 21:22:36 +0000 Subject: [PATCH 063/277] Add unit tests for outline.layer. (#928) --- pyoutline/outline/depend.py | 2 + pyoutline/outline/event.py | 4 +- pyoutline/outline/io.py | 2 +- pyoutline/outline/layer.py | 13 +- pyoutline/tests/layer_test.py | 369 +++++++++++++++++++++++++++++++++- 5 files changed, 370 insertions(+), 20 deletions(-) diff --git a/pyoutline/outline/depend.py b/pyoutline/outline/depend.py index c2b83d59a..9caffab85 100644 --- a/pyoutline/outline/depend.py +++ b/pyoutline/outline/depend.py @@ -53,6 +53,7 @@ class DependType(object): sim = LayerOnSimFrame prev = PreviousFrame + def parse_require_str(require): """ Parse a require string and returns its components. @@ -65,6 +66,7 @@ def parse_require_str(require): return (parts[0], DependType.FrameByFrame) return (parts[0], getattr(DependType, parts[1])) + class Depend(object): """A dependency""" diff --git a/pyoutline/outline/event.py b/pyoutline/outline/event.py index 39301bf1a..4ba5edf84 100644 --- a/pyoutline/outline/event.py +++ b/pyoutline/outline/event.py @@ -54,7 +54,7 @@ class EventHandler(object): """ def __init__(self, component): self.__component = component - self.__listeners = { } + self.__listeners = {} def add_event_listener(self, event_type, callback): """ @@ -63,7 +63,7 @@ def add_event_listener(self, event_type, callback): """ logger.debug("adding event listener %s", event_type) if event_type not in self.__listeners: - self.__listeners[event_type] = [ ] + self.__listeners[event_type] = [] self.__listeners[event_type].append(callback) # pylint: disable=broad-except diff --git a/pyoutline/outline/io.py b/pyoutline/outline/io.py index e5d7a5551..e127a65fc 100644 --- a/pyoutline/outline/io.py +++ b/pyoutline/outline/io.py @@ -122,7 +122,7 @@ def __init__(self, path, **args): object.__init__(self) self.__path = resolve(str(path)) - self.__attributes = { } + self.__attributes = {} self.__attributes["checked"] = False self.__attributes["mkdir"] = False self.__attributes.update(args) diff --git a/pyoutline/outline/layer.py b/pyoutline/outline/layer.py index b4e74a833..9fcdea1be 100644 --- a/pyoutline/outline/layer.py +++ b/pyoutline/outline/layer.py @@ -105,14 +105,14 @@ def __init__(self, name, **args): self.__parent = None # Contains IO objects that are considered input. - self.__input = { } + self.__input = {} # Contains IO objects that are considered output. - self.__output = { } + self.__output = {} # A dictionary of environment variables to apply before execute. - self.__env = { } - self.__env.update(args.get("env", { })) + self.__env = {} + self.__env.update(args.get("env", {})) # Children are unregistered layers that are executed # after the parent layer. @@ -224,7 +224,7 @@ def get_default_args(self, merge=None): # would be defined within the relevant backend module or in # the outline configuration file. - defaults = { } + defaults = {} # By default all layers are registerd. Registered layers show up # as discrete layers. Unregisterd layers are generally embedded @@ -606,6 +606,7 @@ def sym_file(self, src, rename=None): return self.__outline.get_session().sym_file(src, layer=self, rename=rename) + def put_file(self, src, rename=None): """ Copy the given file into the layer's session path. If @@ -1168,7 +1169,7 @@ def execute(self, frame): Perform pre-propcess execute methods and call the super class's exceute method. """ - super().execute(frame) + super(LayerPreProcess, self).execute(frame) self.__save_outputs() def get_frame_range(self): diff --git a/pyoutline/tests/layer_test.py b/pyoutline/tests/layer_test.py index 567f4aa32..f1fd61b9f 100644 --- a/pyoutline/tests/layer_test.py +++ b/pyoutline/tests/layer_test.py @@ -33,6 +33,7 @@ import mock import outline +import outline.layer import outline.io import outline.modules.shell @@ -321,17 +322,6 @@ def test_after_init_current(self): # Ensure that the layer has the right ol reference self.assertEqual(ol, ol.get_layer('test').get_outline()) - def test_dependency_creation(self): - with test_utils.TemporarySessionDirectory(): - outline.Outline.current = None - ol = outline.Outline('after_init') - ol.add_layer(TestAfterInit('test')) - ol.add_layer(TestB('testb', require='test')) - ol.setup() - - # check the depend was setup properly - self.assertEqual(1, len(ol.get_layer('testb').get_depends())) - def test_type_arg(self): """Test to ensure the type argument is handled properly.""" outline.Outline.current = None @@ -364,6 +354,238 @@ def test_set_name(self): self.assertRaises( outline.LayerException, self.layer.set_name, 'this-should-fail') + def test_should_fail_on_invalid_parent(self): + invalid_object = 'some-string-object' + + self.assertRaises(outline.LayerException, self.layer.set_parent, invalid_object) + + def test_should_fail_on_invalid_child(self): + invalid_object = 'some-string-object' + + self.assertRaises(outline.LayerException, self.layer.add_child, invalid_object) + + def test_should_add_event_listener(self): + event_type = 'arbitrary-event-type' + callback = lambda x: x + + self.layer.add_event_listener(event_type, callback) + + self.assertEqual([callback], self.layer.get_event_handler().get_event_listeners(event_type)) + + def test_should_fail_on_invalid_layer_type(self): + invalid_layer_type = 'invalid-layer-type' + + self.assertRaises(outline.LayerException, self.layer.set_type, invalid_layer_type) + + def test_should_create_dependency_from_outline(self): + with test_utils.TemporarySessionDirectory(): + outline.Outline.current = None + ol = outline.Outline('after_init') + ol.add_layer(TestAfterInit('test')) + ol.add_layer(TestB('testb', require='test')) + ol.setup() + + new_depend = ol.get_layer('testb').get_depends()[0] + self.assertEqual('testb', new_depend.get_dependant_layer().get_name()) + self.assertEqual('test', new_depend.get_depend_on_layer().get_name()) + + def test_should_create_dependency_from_layer(self): + depend_on_layer = TestAfterInit('test') + depend_er_layer = TestB('testb') + + depend_er_layer.depend_on(depend_on_layer) + + new_depend = depend_er_layer.get_depends()[0] + self.assertEqual('testb', new_depend.get_dependant_layer().get_name()) + self.assertEqual('test', new_depend.get_depend_on_layer().get_name()) + + def test_should_skip_duplicate_depend(self): + depend_on_layer = TestAfterInit('test') + depend_er_layer = TestB('testb') + depend_er_layer.depend_on(depend_on_layer) + num_depends = len(depend_er_layer.get_depends()) + + depend_er_layer.depend_on(depend_on_layer) + + self.assertEqual(num_depends, len(depend_er_layer.get_depends())) + + def test_should_skip_depend_on_self(self): + depend_layer = TestAfterInit('test') + + depend_layer.depend_on(depend_layer) + + self.assertEqual(0, len(depend_layer.get_depends())) + + def test_should_remove_depend(self): + depend_on_layer = TestAfterInit('test') + depend_er_layer = TestB('testb') + depend_er_layer.depend_on(depend_on_layer) + depend_to_remove = depend_er_layer.get_depends()[0] + + depend_er_layer.undepend(depend_to_remove) + + self.assertEqual(0, len(depend_er_layer.get_depends())) + + def test_should_get_dependents(self): + with test_utils.TemporarySessionDirectory(): + depend_on_layer = TestAfterInit('test') + depend_er_layer = TestB('testb', require='test') + outline.Outline.current = None + ol = outline.Outline('after_init') + ol.add_layer(depend_on_layer) + ol.add_layer(depend_er_layer) + ol.setup() + + depend = depend_on_layer.get_dependents()[0] + self.assertEqual('testb', depend.get_dependant_layer().get_name()) + self.assertEqual('test', depend.get_depend_on_layer().get_name()) + + def test_should_add_inputs(self): + layer = TestAfterInit('test') + unnamed_input = outline.io.Path('/path/to/first/input') + named_input = outline.io.Path('/path/to/second/input') + bare_path = '/path/to/third/input' + + layer.add_input(None, unnamed_input) + layer.add_input('named-input-name', named_input) + layer.add_input('bare-path-name', bare_path) + + inputs = layer.get_inputs() + self.assertEqual(unnamed_input, inputs['input0']) + self.assertEqual(named_input, inputs['named-input-name']) + self.assertEqual(bare_path, inputs['bare-path-name'].get_path()) + self.assertEqual(unnamed_input, layer.get_input('input0')) + self.assertEqual(named_input, layer.get_input('named-input-name')) + self.assertEqual(bare_path, layer.get_input('bare-path-name').get_path()) + + def test_should_fail_on_duplicate_input(self): + layer = TestAfterInit('test') + input_to_add = outline.io.Path('/path/to/input') + layer.add_input('input-name', input_to_add) + + self.assertRaises(outline.LayerException, layer.add_input, 'input-name', input_to_add) + + def test_should_check_input(self): + layer = TestAfterInit('test') + input_to_check = outline.io.Path('/path/to/input') + input_to_check.set_attribute('checked', True) + input_to_check.exists = mock.Mock() + input_to_check.exists.return_value = True + layer.add_input('input-name', input_to_check) + + layer.check_input() + + input_to_check.exists.assert_called() + + def test_should_skip_checking_input(self): + layer = TestAfterInit('test') + input_to_check = outline.io.Path('/path/to/input') + input_to_check.set_attribute('checked', False) + input_to_check.exists = mock.Mock() + input_to_check.exists.return_value = True + layer.add_input('input-name', input_to_check) + + layer.check_input() + + input_to_check.exists.assert_not_called() + + def test_should_fail_on_nonexistent_input(self): + layer = TestAfterInit('test') + input_to_check = outline.io.Path('/path/to/input') + input_to_check.set_attribute('checked', True) + input_to_check.exists = mock.Mock() + input_to_check.exists.return_value = False + layer.add_input('input-name', input_to_check) + + self.assertRaises(outline.LayerException, layer.check_input) + + def test_should_set_attribute_on_all_inputs(self): + layer = TestAfterInit('test') + input1 = outline.io.Path('/path/to/input') + input2 = outline.io.Path('/path/to/another/input') + layer.add_input('input-one', input1) + layer.add_input('input-two', input2) + attr_name = 'arbitrary-attribute-name' + attr_val = 'arbitrary-attribute-value' + + layer.set_input_attribute(attr_name, attr_val) + + self.assertEqual(attr_val, input1.get_attribute(attr_name)) + self.assertEqual(attr_val, input2.get_attribute(attr_name)) + + def test_should_add_outputs(self): + layer = TestAfterInit('test') + unnamed_output = outline.io.Path('/path/to/first/output') + named_output = outline.io.Path('/path/to/second/output') + bare_path = '/path/to/third/output' + + layer.add_output(None, unnamed_output) + layer.add_output('named-output-name', named_output) + layer.add_output('bare-path-name', bare_path) + + outputs = layer.get_outputs() + self.assertEqual(unnamed_output, outputs['output0']) + self.assertEqual(named_output, outputs['named-output-name']) + self.assertEqual(bare_path, outputs['bare-path-name'].get_path()) + self.assertEqual(unnamed_output, layer.get_output('output0')) + self.assertEqual(named_output, layer.get_output('named-output-name')) + self.assertEqual(bare_path, layer.get_output('bare-path-name').get_path()) + + def test_should_fail_on_duplicate_output(self): + layer = TestAfterInit('test') + output_to_add = outline.io.Path('/path/to/output') + layer.add_output('output-name', output_to_add) + + self.assertRaises(outline.LayerException, layer.add_output, 'output-name', output_to_add) + + def test_should_check_output(self): + layer = TestAfterInit('test') + output_to_check = outline.io.Path('/path/to/output') + output_to_check.set_attribute('checked', True) + output_to_check.exists = mock.Mock() + output_to_check.exists.return_value = True + layer.add_output('output-name', output_to_check) + + layer.check_output() + + output_to_check.exists.assert_called() + + def test_should_skip_checking_output(self): + layer = TestAfterInit('test') + output_to_check = outline.io.Path('/path/to/output') + output_to_check.set_attribute('checked', False) + output_to_check.exists = mock.Mock() + output_to_check.exists.return_value = True + layer.add_output('output-name', output_to_check) + + layer.check_output() + + output_to_check.exists.assert_not_called() + + def test_should_fail_on_nonexistent_output(self): + layer = TestAfterInit('test') + output_to_check = outline.io.Path('/path/to/output') + output_to_check.set_attribute('checked', True) + output_to_check.exists = mock.Mock() + output_to_check.exists.return_value = False + layer.add_output('output-name', output_to_check) + + self.assertRaises(outline.LayerException, layer.check_output) + + def test_should_set_attribute_on_all_outputs(self): + layer = TestAfterInit('test') + output1 = outline.io.Path('/path/to/output') + output2 = outline.io.Path('/path/to/another/output') + layer.add_output('output-one', output1) + layer.add_output('output-two', output2) + attr_name = 'arbitrary-attribute-name' + attr_val = 'arbitrary-attribute-value' + + layer.set_output_attribute(attr_name, attr_val) + + self.assertEqual(attr_val, output1.get_attribute(attr_name)) + self.assertEqual(attr_val, output1.get_attribute(attr_name)) + class OutputRegistrationTest(unittest.TestCase): @@ -407,6 +629,131 @@ def disabled__test_output_passing(self): self.assertEqual(1, len(layer1.get_outputs())) +class FrameTests(unittest.TestCase): + + def test_should_return_first_frame_of_outline_range(self): + """Test getting/setting the frame range. If the frame + range is not set on a layer, then it should default to + the outline frame range. + """ + frame_range = '55-65' + outline.Outline.current = None + ol = outline.Outline('after_init') + ol.set_frame_range(frame_range) + layer = outline.layer.Frame('layer-name') + ol.add_layer(layer) + self.assertEqual('55', layer.get_frame_range()) + + def test_should_return_default_frame_range(self): + """Test getting/setting the frame range. If the frame + range is not set on a layer, then it should default to + the outline frame range. + """ + outline.Outline.current = None + ol = outline.Outline('after_init') + layer = outline.layer.Frame('layer-name') + ol.add_layer(layer) + self.assertEqual(outline.layer.DEFAULT_FRAME_RANGE, layer.get_frame_range()) + + +class LayerPreProcessTests(unittest.TestCase): + + def test_should_be_util_type_and_preprocess_service(self): + parent_layer_name = 'parent-layer' + parent_layer = outline.layer.Layer(parent_layer_name) + preprocess_layer = outline.layer.LayerPreProcess(parent_layer) + + self.assertEqual('Util', preprocess_layer.get_type()) + self.assertEqual('preprocess', preprocess_layer.get_service()) + + def test_should_setup_depend_and_preprocess_layer(self): + parent_layer_name = 'parent-layer' + preprocess_layer_name = '%s_preprocess' % parent_layer_name + parent_layer = outline.layer.Layer(parent_layer_name) + preprocess_layer = outline.layer.LayerPreProcess(parent_layer) + + self.assertEqual(preprocess_layer_name, preprocess_layer.get_name()) + self.assertEqual( + preprocess_layer_name, parent_layer.get_depends()[0].get_depend_on_layer().get_name()) + self.assertEqual(preprocess_layer, parent_layer.get_preprocess_layers()[0]) + + def test_should_get_parent_layer(self): + parent_layer_name = 'parent-layer' + parent_layer = outline.layer.Layer(parent_layer_name) + preprocess_layer = outline.layer.LayerPreProcess(parent_layer) + + self.assertEqual(parent_layer, preprocess_layer.get_creator()) + + def test_should_get_first_frame_of_layer_range(self): + parent_layer_name = 'parent-layer' + parent_layer = outline.layer.Layer(parent_layer_name) + parent_layer.set_frame_range('540-600') + preprocess_layer = outline.layer.LayerPreProcess(parent_layer) + + self.assertEqual('540', preprocess_layer.get_frame_range()) + + @mock.patch('outline.Layer.system', new=mock.Mock()) + def test_should_call_put_data_on_parent(self): + os.environ = {} + + outline.Outline.current = None + ol = outline.Outline('outline-name') + parent_layer_name = 'parent-layer' + parent_layer = outline.layer.Layer(parent_layer_name) + parent_layer.put_data = mock.Mock() + ol.add_layer(parent_layer) + + preprocess_layer = outline.layer.LayerPreProcess(parent_layer) + output = outline.io.Path('/path/to/output') + preprocess_layer.add_output('output-name', output) + ol.add_layer(preprocess_layer) + + with test_utils.TemporarySessionDirectory(): + ol.setup() + preprocess_layer.execute(1) + + parent_layer.put_data.assert_called_with( + 'ol:outputs', {'output-name': output}, force=True) + + +class LayerPostProcessTests(unittest.TestCase): + + def test_should_be_util_type(self): + parent_layer_name = 'parent-layer' + parent_layer = outline.layer.Layer(parent_layer_name) + postprocess_layer = outline.layer.LayerPostProcess(parent_layer) + + self.assertEqual('Util', postprocess_layer.get_type()) + + def test_should_setup_depend(self): + parent_layer_name = 'parent-layer' + postprocess_layer_name = '%s_postprocess' % parent_layer_name + parent_layer = outline.layer.Layer(parent_layer_name) + postprocess_layer = outline.layer.LayerPostProcess(parent_layer) + + self.assertEqual(postprocess_layer_name, postprocess_layer.get_name()) + self.assertEqual( + parent_layer_name, postprocess_layer.get_depends()[0].get_depend_on_layer().get_name()) + + def test_should_get_parent_layer(self): + parent_layer_name = 'parent-layer' + parent_layer = outline.layer.Layer(parent_layer_name) + postprocess_layer = outline.layer.LayerPostProcess(parent_layer) + + self.assertEqual(parent_layer, postprocess_layer.get_creator()) + + +class OutlinePostCommandTests(unittest.TestCase): + + def test_should_be_post_type_and_postprocess_service(self): + parent_layer_name = 'parent-layer' + parent_layer = outline.layer.Layer(parent_layer_name) + post_layer = outline.layer.OutlinePostCommand(parent_layer) + + self.assertEqual('Post', post_layer.get_type()) + self.assertEqual('postprocess', post_layer.get_service()) + + class TestAfterInit(outline.Layer): def __init__(self, name, **args): outline.Layer.__init__(self, name, **args) From d84a10ff08848fade4f4ec07091eba2ab9fc32e3 Mon Sep 17 00:00:00 2001 From: Lars van der Bijl <285658+larsbijl@users.noreply.github.com> Date: Fri, 26 Feb 2021 13:59:44 +0000 Subject: [PATCH 064/277] Show example of using loki for logging (#880) * Show example of using loki for logging Store logs for the RQD, Cuebot and DB pod's in loki and make the datasource available in grafana. * make loki a sub section to monitoring * Add prometheus, postgres exporter, and grafana dashboards Hookup the prometheus to the cuebot and postgres metrics with a default dashboard to give some insight. * Remove logging from base docker-compose Fix spell in readme --- sandbox/README.md | 22 + .../config/grafana/dashboards/PostgreSQL.json | 2266 +++++++++++++++++ sandbox/config/grafana/dashboards/cuebot.json | 948 +++++++ .../provisioning/dashboards/opencue.yml | 24 + .../provisioning/datasources/datasource.yml | 25 + sandbox/config/loki/loki.yaml | 54 + sandbox/config/prometheus/prometheus.yml | 14 + sandbox/docker-compose.monitoring.yml | 79 + sandbox/docker-compose.yml | 13 - 9 files changed, 3432 insertions(+), 13 deletions(-) create mode 100644 sandbox/config/grafana/dashboards/PostgreSQL.json create mode 100644 sandbox/config/grafana/dashboards/cuebot.json create mode 100644 sandbox/config/grafana/provisioning/dashboards/opencue.yml create mode 100644 sandbox/config/grafana/provisioning/datasources/datasource.yml create mode 100644 sandbox/config/loki/loki.yaml create mode 100644 sandbox/config/prometheus/prometheus.yml create mode 100644 sandbox/docker-compose.monitoring.yml diff --git a/sandbox/README.md b/sandbox/README.md index 8aad50ec8..8fcc71fa8 100644 --- a/sandbox/README.md +++ b/sandbox/README.md @@ -7,3 +7,25 @@ machine. To learn how to run the sandbox environment, see https://www.opencue.io/docs/quick-starts/. + +## Monitoring + +To get started with monitoring there is also an additional docker-compose which sets up +monitoring for key services. + +This can be started from the OpenCue root directory with: +```bash +docker-compose --project-directory . -f sandbox/docker-compose.yml -f sandbox/docker-compose.monitoring.yml up +``` + +Spins up a monitoring stack + +http://localhost:3000/ + +login: admin +pass: admin + +### Loki logging + +Too use loki to store logs requires installing the docker drivers. see: +https://grafana.com/docs/loki/latest/clients/docker-driver/ \ No newline at end of file diff --git a/sandbox/config/grafana/dashboards/PostgreSQL.json b/sandbox/config/grafana/dashboards/PostgreSQL.json new file mode 100644 index 000000000..dcaaf940c --- /dev/null +++ b/sandbox/config/grafana/dashboards/PostgreSQL.json @@ -0,0 +1,2266 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Dashboard for PostgreSQL Statistics.", + "editable": true, + "gnetId": 6742, + "graphTooltip": 0, + "id": 145, + "iteration": 1612456597382, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 34, + "panels": [], + "title": "Settings", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#299c46", + "#7eb26d", + "#d44a3a" + ], + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pg_static", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{short_version}}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 54, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pg_settings_max_connections{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Max Connections", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 6, + "y": 1 + }, + "id": 56, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pg_settings_shared_buffers_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Shared Buffers", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 10, + "y": 1 + }, + "id": 58, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pg_settings_effective_cache_size_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Effective Cache", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 14, + "y": 1 + }, + "id": 60, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pg_settings_maintenance_work_mem_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Maintenance Work Mem", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 66, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pg_settings_work_mem_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Work Mem", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "prometheus", + "decimals": 1, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 32, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pg_settings_max_wal_size_bytes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Max WAL Size", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 4 + }, + "id": 62, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pg_settings_random_page_cost{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Random Page Cost", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 4 + }, + "id": 70, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pg_settings_seq_page_cost", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Seq Page Cost", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 6, + "y": 4 + }, + "id": 64, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pg_settings_max_worker_processes{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Max Worker Processes", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 10, + "y": 4 + }, + "id": 68, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "pg_settings_max_parallel_workers{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Max Parallel Workers", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 36, + "panels": [], + "title": "Connection / Transaction Statistics", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "pg_stat_activity_count{instance=\"$instance\", datname=\"$db\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{state}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Connections", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(pg_stat_database_xact_commit{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "commits", + "refId": "A" + }, + { + "expr": "irate(pg_stat_database_xact_rollback{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "rollbacks", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Transactions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(pg_stat_database_tup_fetched{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SELECT (index scan)", + "refId": "A" + }, + { + "expr": "irate(pg_stat_database_tup_returned{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SELECT (table scan)", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Read Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "Rows", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(pg_stat_database_tup_inserted{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "INSERT", + "refId": "A" + }, + { + "expr": "irate(pg_stat_database_tup_updated{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "UPDATE", + "refId": "B" + }, + { + "expr": "irate(pg_stat_database_tup_deleted{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "DELETE", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Change Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Rows", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "hiddenSeries": false, + "id": 42, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pg_stat_activity_max_tx_duration{instance=\"$instance\", datname=\"$db\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max_tx_duration [{{state}}]", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Longest Transaction", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "hiddenSeries": false, + "id": 44, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pg_stat_database_blks_hit{instance=\"$instance\", datname=\"$db\"} / (pg_stat_database_blks_read{instance=\"$instance\", datname=\"$db\"} + pg_stat_database_blks_hit{instance=\"$instance\", datname=\"$db\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cache Hit Rate", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache Hit Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 4, + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 50, + "panels": [], + "title": "misc", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 17, + "x": 0, + "y": 35 + }, + "hiddenSeries": false, + "id": 46, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(pg_stat_bgwriter_buffers_backend{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "buffers_backend", + "refId": "A" + }, + { + "expr": "irate(pg_stat_bgwriter_buffers_alloc{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "buffers_alloc", + "refId": "B" + }, + { + "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "backend_fsync", + "refId": "C" + }, + { + "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "buffers_checkpoint", + "refId": "D" + }, + { + "expr": "irate(pg_stat_bgwriter_buffers_clean{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "buffers_clean", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Buffers (bgwriter)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 7, + "x": 17, + "y": 35 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(pg_stat_database_conflicts{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "conflicts", + "refId": "B" + }, + { + "expr": "irate(pg_stat_database_deadlocks{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "deadlocks", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Conflicts/Deadlocks", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "decimals": 0, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 17, + "x": 0, + "y": 41 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pg_locks_count{instance=\"$instance\", datname=\"$db\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{mode}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Lock Tables", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Total amount of data written to temporary files by queries in this database. All temporary files are counted, regardless of why the temporary file was created, and regardless of the log_temp_files setting.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 7, + "x": 17, + "y": 41 + }, + "hiddenSeries": false, + "id": 40, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(pg_stat_database_temp_bytes{instance=\"$instance\", datname=\"$db\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Temp Bytes", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Temp File", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 48 + }, + "hiddenSeries": false, + "id": 38, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.7", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(pg_stat_bgwriter_checkpoint_write_time{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "write_time - Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk.", + "refId": "B" + }, + { + "expr": "irate(pg_stat_bgwriter_checkpoint_sync_time{instance=\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "sync_time - Total amount of time that has been spent in the portion of checkpoint processing where files are synchronized to disk.", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Checkpoint Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": false, + "text": "cuebot:9187", + "value": "cuebot:9187" + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(pg_up, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "cuebot", + "value": "cuebot" + }, + "datasource": "prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "Database", + "multi": false, + "name": "db", + "options": [], + "query": "label_values(pg_stat_database_tup_fetched{datname!~\"template.*|postgres\"},datname)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "PostgreSQL", + "uid": "w8OVoJYGz", + "version": 1 + } \ No newline at end of file diff --git a/sandbox/config/grafana/dashboards/cuebot.json b/sandbox/config/grafana/dashboards/cuebot.json new file mode 100644 index 000000000..38e5edd3e --- /dev/null +++ b/sandbox/config/grafana/dashboards/cuebot.json @@ -0,0 +1,948 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "OpenCue Metrics for the Cue server", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "id": 2, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(cue_host_balance_success_total[1m])", + "legendFormat": "balance success", + "refId": "A" + }, + { + "expr": "rate(cue_host_balance_failed_total[1m])", + "legendFormat": "balance failed", + "refId": "B" + }, + { + "expr": "rate(cue_killed_offender_procs_total[1m])", + "legendFormat": "killed offender procs", + "refId": "C" + }, + { + "expr": "rate(cue_killed_oom_procs_total[1m])", + "legendFormat": "killed oom procs", + "refId": "D" + }, + { + "expr": "rate(cue_cleared_procs_total[1m])", + "legendFormat": "cleared procs", + "refId": "E" + }, + { + "expr": "rate(cue_req_errors_total[1m])", + "legendFormat": "req errors", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "System calls", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 6, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "booking errors", + "color": "#C4162A" + }, + { + "alias": "booking retries", + "color": "#FADE2A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": true, + "targets": [ + { + "expr": "rate(cue_booking_retries_total[1m])", + "legendFormat": "booking retries", + "refId": "B" + }, + { + "expr": "rate(cue_booking_errors_total[1m])", + "legendFormat": "booking errors", + "refId": "A" + }, + { + "expr": "rate(cue_booked_procs_total[1m])", + "legendFormat": "booked procs", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "More System calls", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "It has a total of 6, if it stays at 6 it's likely a host has deadlocked and will need to be found and killed.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 6 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cue_booking_threads_total", + "interval": "", + "legendFormat": "booking threads", + "refId": "C" + }, + { + "expr": "cue_dispatch_threads_total", + "interval": "", + "legendFormat": "dispatch threads", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Booking and Dispatch threads", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "It has a total of 6, if it stays at 6 it's likely a host has deadlocked and will need to be found and killed.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 6 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cue_manage_threads_total", + "interval": "", + "legendFormat": "manage threads", + "refId": "C" + }, + { + "expr": "cue_report_threads_total", + "interval": "", + "legendFormat": "report threads", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Manage and Report threads", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(cue_booking_executed_total[1m])", + "instant": false, + "interval": "", + "legendFormat": "executed", + "refId": "A" + }, + { + "expr": "rate(cue_dispatch_waiting_total[1m])", + "legendFormat": "waiting", + "refId": "E" + }, + { + "expr": "rate(cue_dispatch_remaining_capacity_total[1m])", + "interval": "", + "legendFormat": "remaining capacity", + "refId": "D" + }, + { + "expr": "rate(cue_booking_rejected_total[1m])", + "interval": "", + "legendFormat": "rejected", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Booking over 1 min", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 12 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(cue_manage_executed_total[1m])", + "interval": "", + "legendFormat": "executed", + "refId": "A" + }, + { + "expr": "rate(cue_manage_rejected_total[1m])", + "interval": "", + "legendFormat": "rejected", + "refId": "B" + }, + { + "expr": "rate(cue_manage_remaining_capacity_total[1m])", + "legendFormat": "remaining capacity", + "refId": "D" + }, + { + "expr": "rate(cue_manage_waiting_total[1m])", + "legendFormat": "waiting", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Manage over 1 min", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(cue_dispatch_executed_total[1m])", + "format": "time_series", + "interval": "", + "legendFormat": "executed", + "refId": "A" + }, + { + "expr": "rate(cue_dispatch_remaining_capacity_total[1m])", + "interval": "", + "legendFormat": "remaining capacity", + "refId": "D" + }, + { + "expr": "rate(cue_dispatch_waiting_total[1m])", + "interval": "", + "legendFormat": "waiting", + "refId": "E" + }, + { + "expr": "rate(cue_dispatch_rejected_total[1m])", + "interval": "", + "legendFormat": "rejected", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Dispatch over 1 min", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 18 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(cue_report_executed_total[1m])", + "legendFormat": "executed", + "refId": "A" + }, + { + "expr": "rate(cue_report_rejected_total[1m])", + "interval": "", + "legendFormat": "rejected", + "refId": "B" + }, + { + "expr": "rate(cue_report_remaining_capacity_total[1m])", + "interval": "", + "legendFormat": "remaining capacity", + "refId": "D" + }, + { + "expr": "rate(cue_report_waiting_total[1m])", + "interval": "", + "legendFormat": "waiting", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Report over 1 min", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Cuebot", + "uid": "0000000087", + "version": 1 +} \ No newline at end of file diff --git a/sandbox/config/grafana/provisioning/dashboards/opencue.yml b/sandbox/config/grafana/provisioning/dashboards/opencue.yml new file mode 100644 index 000000000..dbb442cd0 --- /dev/null +++ b/sandbox/config/grafana/provisioning/dashboards/opencue.yml @@ -0,0 +1,24 @@ +# config file version +apiVersion: 1 + +providers: +- name: opencue + # org id. will default to orgId 1 if not specified + org_id: 1 + # name of the dashboard folder. Required + folder: '' + # provider type. Required + type: 'file' + # disable dashboard deletion + disableDeletion: false + # enable dashboard editing + editable: true + # how often Grafana will scan for changed dashboards + updateIntervalSeconds: 100 + # allow updating provisioned dashboards from the UI + allowUiUpdates: true + options: + # path to dashboard files on disk. Required + path: /etc/grafana/dashboards + # use folder names from filesystem to create folders in Grafana + foldersFromFilesStructure: true \ No newline at end of file diff --git a/sandbox/config/grafana/provisioning/datasources/datasource.yml b/sandbox/config/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 000000000..593b8276f --- /dev/null +++ b/sandbox/config/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,25 @@ +# config file version +apiVersion: 1 + +deleteDatasources: + - name: loki + +datasources: + - name: loki + type: loki + access: proxy + orgId: 1 + url: http://loki:3100 + basicAuth: false + isDefault: true + version: 1 + editable: false + - name: prometheus + type: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 + basicAuth: false + isDefault: false + version: 1 + editable: false \ No newline at end of file diff --git a/sandbox/config/loki/loki.yaml b/sandbox/config/loki/loki.yaml new file mode 100644 index 000000000..149c10501 --- /dev/null +++ b/sandbox/config/loki/loki.yaml @@ -0,0 +1,54 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +ingester: + lifecycler: + address: 127.0.0.1 + ring: + kvstore: + store: inmemory + replication_factor: 1 + final_sleep: 0s + chunk_idle_period: 5m + chunk_retain_period: 30s + +schema_config: + configs: + - from: 2020-01-20 + store: boltdb + object_store: filesystem + schema: v9 + index: + prefix: index_ + period: 168h + +storage_config: + boltdb: + directory: /tmp/loki/index + + filesystem: + directory: /tmp/loki/chunks + +limits_config: + enforce_metric_name: false + reject_old_samples: true + reject_old_samples_max_age: 168h + +chunk_store_config: + max_look_back_period: 0 + +table_manager: + chunk_tables_provisioning: + inactive_read_throughput: 0 + inactive_write_throughput: 0 + provisioned_read_throughput: 0 + provisioned_write_throughput: 0 + index_tables_provisioning: + inactive_read_throughput: 0 + inactive_write_throughput: 0 + provisioned_read_throughput: 0 + provisioned_write_throughput: 0 + retention_deletes_enabled: false + retention_period: 0 \ No newline at end of file diff --git a/sandbox/config/prometheus/prometheus.yml b/sandbox/config/prometheus/prometheus.yml new file mode 100644 index 000000000..632b12ca6 --- /dev/null +++ b/sandbox/config/prometheus/prometheus.yml @@ -0,0 +1,14 @@ +global: + scrape_interval: 30s + scrape_timeout: 10s + +scrape_configs: + - job_name: metrics + static_configs: + - targets: + - 'metrics:8302' + + - job_name: db + static_configs: + - targets: + - 'db-exporter:9187' diff --git a/sandbox/docker-compose.monitoring.yml b/sandbox/docker-compose.monitoring.yml new file mode 100644 index 000000000..3f6e451f3 --- /dev/null +++ b/sandbox/docker-compose.monitoring.yml @@ -0,0 +1,79 @@ +version: '3' + +services: + db: + logging: + driver: loki + options: + loki-url: http://localhost:3100/loki/api/v1/push + loki-external-labels: job=db + + db-exporter: + image: wrouesnel/postgres_exporter + environment: + - DATA_SOURCE_URI=db:5432/postgres?sslmode=disable + - DATA_SOURCE_USER=cuebot + - DATA_SOURCE_PASS=$POSTGRES_PASSWORD + - PG_EXPORTER_AUTO_DISCOVER_DATABASES=true + ports: + - 9187:9187 + depends_on: + - db + + cuebot: + logging: + driver: loki + options: + loki-url: http://localhost:3100/loki/api/v1/push + loki-external-labels: job=cuebot + + rqd: + logging: + driver: loki + options: + loki-url: http://localhost:3100/loki/api/v1/push + loki-external-labels: job=rqd + + metrics: + restart: always + build: + context: ./ + dockerfile: ./connectors/prometheus_metrics/Dockerfile + environment: + - CUEBOT_HOSTS=cuebot + depends_on: + - cuebot + links: + - cuebot + ports: + - "8302:8302" + + prometheus: + image: prom/prometheus:v2.21.0 + ports: + - 9000:9090 + volumes: + - ./sandbox/config/prometheus:/etc/prometheus/ + - prometheus-data:/prometheus + command: --web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml + + grafana: + image: grafana/grafana:7.1.1 + volumes: + - ./sandbox/config/grafana/provisioning:/etc/grafana/provisioning + - ./sandbox/config/grafana/dashboards:/etc/grafana/dashboards + ports: + - "3000:3000" + + loki: + image: grafana/loki:v1.3.0 + volumes: + - ./sandbox/config/loki:/etc/config + entrypoint: + - /usr/bin/loki + - -config.file=/etc/config/loki.yaml + ports: + - "3100:3100" + +volumes: + prometheus-data: \ No newline at end of file diff --git a/sandbox/docker-compose.yml b/sandbox/docker-compose.yml index 33938c06f..09bdf7eb7 100644 --- a/sandbox/docker-compose.yml +++ b/sandbox/docker-compose.yml @@ -55,16 +55,3 @@ services: volumes: - /tmp/rqd/logs:/tmp/rqd/logs - /tmp/rqd/shots:/tmp/rqd/shots - - metrics: - build: - context: ./ - dockerfile: ./connectors/prometheus_metrics/Dockerfile - environment: - - CUEBOT_HOSTS=cuebot - depends_on: - - cuebot - links: - - cuebot - ports: - - "8302:8302" \ No newline at end of file From 8371757aa44e13bce302fe1f8478887654b913c8 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sun, 21 Mar 2021 10:29:25 -0700 Subject: [PATCH 065/277] Change mem type from INT to BIGINT (#936) --- VERSION.in | 2 +- .../ddl/postgres/migrations/V10__Change_mem_type.sql | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 cuebot/build/resources/main/conf/ddl/postgres/migrations/V10__Change_mem_type.sql diff --git a/VERSION.in b/VERSION.in index ce609caf8..9a7d84f2a 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.8 \ No newline at end of file +0.9 \ No newline at end of file diff --git a/cuebot/build/resources/main/conf/ddl/postgres/migrations/V10__Change_mem_type.sql b/cuebot/build/resources/main/conf/ddl/postgres/migrations/V10__Change_mem_type.sql new file mode 100644 index 000000000..31f0c2afa --- /dev/null +++ b/cuebot/build/resources/main/conf/ddl/postgres/migrations/V10__Change_mem_type.sql @@ -0,0 +1,10 @@ +-- Change memory type from INT to BIGINT. + +ALTER TABLE "job_mem" ALTER COLUMN "int_max_rss" TYPE BIGINT; +ALTER TABLE "job_mem" ALTER COLUMN "int_max_vss" TYPE BIGINT; +ALTER TABLE "job_resource" ALTER COLUMN "int_max_rss" TYPE BIGINT; +ALTER TABLE "job_resource" ALTER COLUMN "int_max_vss" TYPE BIGINT; +ALTER TABLE "layer_mem" ALTER COLUMN "int_max_rss" TYPE BIGINT; +ALTER TABLE "layer_mem" ALTER COLUMN "int_max_vss" TYPE BIGINT; +ALTER TABLE "layer_resource" ALTER COLUMN "int_max_rss" TYPE BIGINT; +ALTER TABLE "layer_resource" ALTER COLUMN "int_max_vss" TYPE BIGINT; From 103f3354e6a9d17b98f1e4939169574ad4629169 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Mon, 22 Mar 2021 10:30:40 -0700 Subject: [PATCH 066/277] Fix serivce update (#937) --- cuegui/cuegui/ServiceDialog.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cuegui/cuegui/ServiceDialog.py b/cuegui/cuegui/ServiceDialog.py index 04f1b1a89..7949f8f6b 100644 --- a/cuegui/cuegui/ServiceDialog.py +++ b/cuegui/cuegui/ServiceDialog.py @@ -115,6 +115,7 @@ def setService(self, service): """ Update the form with data from the given service. """ + self.__service = service self.__buttons.setDisabled(False) self.name.setText(service.data.name) self.threadable.setChecked(service.data.threadable) @@ -159,7 +160,7 @@ def save(self): service = opencue.wrappers.service.Service() if self.__service: - service.data.id = self.__service.id + service.data.id = self.__service.data.id service.setName(str(self.name.text())) service.setThreadable(self.threadable.isChecked()) service.setMinCores(self.min_cores.value()) From d21ddcfc0f61d44b4c7e6c034c4888ce5e1e78ab Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Tue, 23 Mar 2021 12:55:55 -0700 Subject: [PATCH 067/277] Fix #936 (#938) --- VERSION.in | 2 +- .../conf/ddl/postgres/migrations/V10__Change_mem_type.sql | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename cuebot/{build/resources/main => src/main/resources}/conf/ddl/postgres/migrations/V10__Change_mem_type.sql (100%) diff --git a/VERSION.in b/VERSION.in index 9a7d84f2a..68c123cf1 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.9 \ No newline at end of file +0.10 diff --git a/cuebot/build/resources/main/conf/ddl/postgres/migrations/V10__Change_mem_type.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V10__Change_mem_type.sql similarity index 100% rename from cuebot/build/resources/main/conf/ddl/postgres/migrations/V10__Change_mem_type.sql rename to cuebot/src/main/resources/conf/ddl/postgres/migrations/V10__Change_mem_type.sql From c93e8629f641a95f47617c50865f7e316e94f5c8 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Wed, 24 Mar 2021 11:48:22 -0700 Subject: [PATCH 068/277] Reset outline.Outline.current in PyOutline tests. (#942) --- pyoutline/tests/specver_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyoutline/tests/specver_test.py b/pyoutline/tests/specver_test.py index 24d92df07..3244e6807 100644 --- a/pyoutline/tests/specver_test.py +++ b/pyoutline/tests/specver_test.py @@ -32,6 +32,8 @@ class SpecVersiondTest(unittest.TestCase): def _makeSpec(self): + # Ensure to reset current + outline.Outline.current = None ol = outline.Outline(name="spec_version_test") layer = outline.modules.shell.Shell("test_layer", command=["/bin/ls"]) layer.set_arg("timeout", 420) From 4195dddc5b5d16bb7b1ae31ec2de2707985d232f Mon Sep 17 00:00:00 2001 From: Lars van der Bijl <285658+larsbijl@users.noreply.github.com> Date: Fri, 2 Apr 2021 14:22:12 +0100 Subject: [PATCH 069/277] Fix typo's and spelling mistakes. (#934) Co-authored-by: Lars van der Bijl --- .../com/imageworks/spcue/BuildableJob.java | 4 ++-- .../com/imageworks/spcue/dao/LayerDao.java | 2 +- cuegui/cuegui/Cuedepend.py | 6 ++--- pycue/opencue/util.py | 2 +- pyoutline/bin/cuerunbase.py | 2 +- pyoutline/outline/cuerun.py | 4 ++-- pyoutline/outline/depend.py | 10 ++++----- pyoutline/outline/io.py | 4 ++-- pyoutline/outline/layer.py | 14 ++++++------ pyoutline/outline/loader.py | 22 +++++++++---------- pyoutline/outline/util.py | 8 +++---- 11 files changed, 39 insertions(+), 39 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/BuildableJob.java b/cuebot/src/main/java/com/imageworks/spcue/BuildableJob.java index cfa60ed60..8719f51d6 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/BuildableJob.java +++ b/cuebot/src/main/java/com/imageworks/spcue/BuildableJob.java @@ -70,12 +70,12 @@ public void addBuildableLayer(BuildableLayer layer) { } /** - * Add a key/value pair environement var to job + * Add a key/value pair environment var to job * * @param key * @param value */ - public void addEnvironementVariable(String key, String value) { + public void addEnvironmentVariable(String key, String value) { env.put(key, value); } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/LayerDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/LayerDao.java index 7843d8e8d..243cbbce9 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/LayerDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/LayerDao.java @@ -167,7 +167,7 @@ public interface LayerDao { void insertLayerEnvironment(LayerInterface layer, String key, String value); /** - * Insert a map key/value pairs into the layer environement + * Insert a map key/value pairs into the layer environment * * @param layer * @param env diff --git a/cuegui/cuegui/Cuedepend.py b/cuegui/cuegui/Cuedepend.py index 2997513e5..7335f6d59 100644 --- a/cuegui/cuegui/Cuedepend.py +++ b/cuegui/cuegui/Cuedepend.py @@ -22,7 +22,7 @@ JobOnJob / joj JobOnLayer / jol JobOnFrame / jof LayerOnJob / loj LayerOnLayer / lol LayerOnFrame / lof FrameOnJob / foj FrameOnLayer / fol FrameOnFrame / fof - FrameByFRame / fbf HardDepend / hd + FrameByFrame / fbf HardDepend / hd """ @@ -44,13 +44,13 @@ ERR_INVALID_ON_LAYER = ( "Error, a dependency of this type requires a valid layer name to depend on. See -on-layer.") ERR_INVALID_ON_FRAME = ( - "Error, a dependency of this type requries a valid frame name to depend on. See -on-frame.") + "Error, a dependency of this type requires a valid frame name to depend on. See -on-frame.") ERR_INVALID_ER_JOB = ( "Error, a dependency of this type requires a valid job name to depend on. See -job.") ERR_INVALID_ER_LAYER = ( "Error, a dependency of this type requires a valid layer name to depend on. See -layer.") ERR_INVALID_ER_FRAME = ( - "Error, a dependency of this type requries a valid frame name to depend on. See -frame.") + "Error, a dependency of this type requires a valid frame name to depend on. See -frame.") def __is_valid(value, error): diff --git a/pycue/opencue/util.py b/pycue/opencue/util.py index 1eb704696..7183036c3 100644 --- a/pycue/opencue/util.py +++ b/pycue/opencue/util.py @@ -118,7 +118,7 @@ def _proxies(entities): def rep(entity): """rep(entity) - Extracts a string repesentation of a opencue entity""" + Extracts a string representation of a opencue entity""" try: return entity.name # pylint: disable=bare-except diff --git a/pyoutline/bin/cuerunbase.py b/pyoutline/bin/cuerunbase.py index 89da42b6e..7839305b2 100644 --- a/pyoutline/bin/cuerunbase.py +++ b/pyoutline/bin/cuerunbase.py @@ -68,7 +68,7 @@ def handle_core_arguments(): --repos = specify an alternate outline repository All of these options need to be handled to setup the right - environement to imported the versioned cuerun/outline code. + environment to imported the versioned cuerun/outline code. """ repos = None diff --git a/pyoutline/outline/cuerun.py b/pyoutline/outline/cuerun.py index 7f40364f8..2abf7352d 100644 --- a/pyoutline/outline/cuerun.py +++ b/pyoutline/outline/cuerun.py @@ -74,7 +74,7 @@ def get_launch_facility(): def launch(ol, use_pycuerun=True, **args): """ - A simple convinience method for launching an outline script with + A simple convenience method for launching an outline script with the most common options. If you need additional options, use the OutlineLauncher class. @@ -181,7 +181,7 @@ def setup(self): if self.get("range_default"): fully_baked = True for layer in self.__outline.get_layers(): - # Frames dont' have a range by default. + # Frames don't have a range by default. if isinstance(layer, Frame): continue if not layer.get_arg("range"): diff --git a/pyoutline/outline/depend.py b/pyoutline/outline/depend.py index 9caffab85..cb83d576b 100644 --- a/pyoutline/outline/depend.py +++ b/pyoutline/outline/depend.py @@ -108,18 +108,18 @@ def get_type(self): """ return self.__type - def is_propigated(self): + def is_propagated(self): """ - A propigated dependency is propigated to to others layers + A propagated dependency is propagated to to others layers automatically. For example when a L{Layer} A depends on L{Layer} B through - a propigated dependency, then setting up a dependency from L{Layer} C to + a propagated dependency, then setting up a dependency from L{Layer} C to L{Layer} A would automatically create a depend from L{Layer} C to L{Layer} B. Depends that are automatically setup between L{LayerPreProcess} and - a L{Layer} are propigated dependencies. + a L{Layer} are propagated dependencies. :rtype: boolean - :return: True if he depend is propigated, false if it is not. + :return: True if he depend is propagated, false if it is not. """ return self.__propigate diff --git a/pyoutline/outline/io.py b/pyoutline/outline/io.py index e127a65fc..67d4b1138 100644 --- a/pyoutline/outline/io.py +++ b/pyoutline/outline/io.py @@ -361,11 +361,11 @@ def get_res(self): def get_rep(self): """ - Return the repesentation. The repesentation is + Return the representation. The representation is the oav_resolution_colorspace. :rtype: string - :return: the repesentation. + :return: the representation. """ return self.__fs.getDirname().rsplit("/", 2)[1] diff --git a/pyoutline/outline/layer.py b/pyoutline/outline/layer.py index 9fcdea1be..b0e43bde0 100644 --- a/pyoutline/outline/layer.py +++ b/pyoutline/outline/layer.py @@ -416,17 +416,17 @@ def execute(self, frame): # Check for the existance of required inputs. self.check_input(frames) - # Set all post set shot environement variables. + # Set all post set shot environment variables. for env_k, env_v in self.__outline.get_env().items(): if not env_v[1]: - logger.info("Setting post-set shot environement var: %s %s", + logger.info("Setting post-set shot environment var: %s %s", env_k, env_v[0]) os.environ[env_k] = env_v[0] # Set all layer specific post set shot env variables try: for env_k, env_v in self.__env.items(): - logger.info("Setting post-set shot environement var: %s %s", + logger.info("Setting post-set shot environment var: %s %s", env_k, env_v) os.environ[str(env_k)] = str(env_v) except AttributeError: @@ -628,7 +628,7 @@ def put_file(self, src, rename=None): def get_file(self, name, check=True, new=False): """ - Retrieve the sesion path path to the given file. The + Retrieve the session path path to the given file. The file does not have to exist. :type name: str @@ -741,7 +741,7 @@ def get_local_frame_set(self, start_frame): # Remove the duplicates out of our frame range. # frame_range = FileSequence.FrameSet(self.get_frame_range()) - frame_set = outline.util.deaggregate_frame_set(frame_range) + frame_set = outline.util.disaggregate_frame_set(frame_range) # # Now find the index for the current frame and start @@ -871,10 +871,10 @@ def depend_on(self, on_layer, depend_type=outline.depend.DependType.FrameByFrame # C, which means layer D must now also depend on layer B. # # Currently this creates a depend-all (LayerOnLayer) between - # the propigated depends. + # the propagated depends. # for depend in on_layer.get_depends(): - if depend.is_propigated(): + if depend.is_propagated(): for my_depend in self.get_depends(): dependant = my_depend.get_depend_on_layer() logger.info( diff --git a/pyoutline/outline/loader.py b/pyoutline/outline/loader.py index f58ad46ea..c5a54f126 100644 --- a/pyoutline/outline/loader.py +++ b/pyoutline/outline/loader.py @@ -97,7 +97,7 @@ def load_outline(path): def load_json(json_str): """ - Parse a json repesentation of an outline file. + Parse a json representation of an outline file. :type json_str: str :param json_str: A json string. @@ -296,7 +296,7 @@ def __init__(self, name=None, frame_range=None, path=None, self.__args = {} # - # See contsants for the description of outline modes + # See constants for the description of outline modes # self.__mode = outline.constants.OUTLINE_MODE_INIT @@ -307,7 +307,7 @@ def __init__(self, name=None, frame_range=None, path=None, self.__layers = [] # - # A hash of environement variables that are passed up + # A hash of environment variables that are passed up # to opencue and then set before each frame is run. # These are set "pre" setshot, so they can be used # to modify setshot behavior. @@ -680,17 +680,17 @@ def get_frame_range(self): def set_env(self, key, value, pre=False): """ - Set an environment variable that is propigated to + Set an environment variable that is propagated to every frame. :type key: str - :param key: Name of environement variable. + :param key: Name of environment variable. :type value: str :param value: Value to associate with the name. :type pre: boolean - :param pre: If this value is set to true, the environement + :param pre: If this value is set to true, the environment variable is applied pre-setshot. The default is for the environment variable to be set post set shot. @@ -712,11 +712,11 @@ def set_env(self, key, value, pre=False): def get_env(self, key=None): """ - Return the environement hash setup using set_env. + Return the environment hash setup using set_env. :rtype: dict - :return: the dictionary of values that will be propigated into - every frame's environement on the cue. + :return: the dictionary of values that will be propagated into + every frame's environment on the cue. """ if key: return self.__env[key][0] @@ -749,7 +749,7 @@ def set_arg(self, key, value): def get_arg(self, key, default=None): """ - Return the value assoiciated with the given key. Throw an + Return the value associated with the given key. Throw an OutlineException if the key does not exist. If a default value is provided then that value is returned instead of throwing an OutlineException. @@ -789,7 +789,7 @@ def put_file(self, src, rename=None): def get_file(self, name, check=True, new=False): """ - Retrieve the sesion path path to the given file. The + Retrieve the session path path to the given file. The file does not have to exist. :type name: str diff --git a/pyoutline/outline/util.py b/pyoutline/outline/util.py index 82f7c1153..c5edd0f2d 100644 --- a/pyoutline/outline/util.py +++ b/pyoutline/outline/util.py @@ -31,16 +31,16 @@ from .config import config -def deaggregate_frame_set(frameset): - """Deaggregates a FileSequence.FrameSet into its individual frames +def disaggregate_frame_set(frameset): + """Disaggregates a FileSequence.FrameSet into its individual frames and removes duplicates. FrameSet objects can have duplicates if the user specifies duplicates, which they tend to do even though they don't want duplicates. :type frameset: FileSequence.FrameSet - :param frameset: The frameset to deaggregate + :param frameset: The frameset to disaggregate :rtype: List - :return: The list of deaggregated frames. + :return: The list of disaggregated frames. """ # This is not a Set because sets are unordered. From 64ce8a4c8321faa86ba6e07fe9c84483fd18b6cb Mon Sep 17 00:00:00 2001 From: Lars van der Bijl <285658+larsbijl@users.noreply.github.com> Date: Fri, 2 Apr 2021 14:24:06 +0100 Subject: [PATCH 070/277] Rename the demo data to seed data (#932) Without it the cuebot will not run jobs. Renaming it to seed will make it cleaner it's needed but will likely need to be changed after starting. Co-authored-by: Lars van der Bijl --- .github/workflows/packaging-pipeline.yml | 2 +- .github/workflows/release-pipeline.yml | 4 ++-- ci/extract_schema.sh | 2 +- cuebot/oracle/xe/run_db_container.sh | 4 ++-- .../conf/ddl/postgres/{demo_data.sql => seed_data.sql} | 2 -- sandbox/flyway.Dockerfile | 2 +- sandbox/migrate.sh | 2 +- sandbox/setup-database.sh | 4 ++-- 8 files changed, 10 insertions(+), 12 deletions(-) rename cuebot/src/main/resources/conf/ddl/postgres/{demo_data.sql => seed_data.sql} (97%) diff --git a/.github/workflows/packaging-pipeline.yml b/.github/workflows/packaging-pipeline.yml index 7ff1568fe..920d92ca7 100644 --- a/.github/workflows/packaging-pipeline.yml +++ b/.github/workflows/packaging-pipeline.yml @@ -140,7 +140,7 @@ jobs: aws s3 cp LICENSE s3://${S3_BUCKET}/opencue/${BUILD_ID}/ aws s3 cp VERSION s3://${S3_BUCKET}/opencue/${BUILD_ID}/ aws s3 cp "${GITHUB_WORKSPACE}/artifacts/schema-${BUILD_ID}.sql" s3://${S3_BUCKET}/opencue/${BUILD_ID}/ - aws s3 cp "${GITHUB_WORKSPACE}/artifacts/demo_data-${BUILD_ID}.sql" s3://${S3_BUCKET}/opencue/${BUILD_ID}/ + aws s3 cp "${GITHUB_WORKSPACE}/artifacts/seed_data-${BUILD_ID}.sql" s3://${S3_BUCKET}/opencue/${BUILD_ID}/ aws s3 cp "${GITHUB_WORKSPACE}/artifacts/build_metadata.json" s3://${S3_BUCKET}/opencue/${BUILD_ID}/ - name: Display artifacts diff --git a/.github/workflows/release-pipeline.yml b/.github/workflows/release-pipeline.yml index 59f5da59d..d87637d61 100644 --- a/.github/workflows/release-pipeline.yml +++ b/.github/workflows/release-pipeline.yml @@ -161,8 +161,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ${{ github.workspace }}/artifacts/demo_data-${{ env.BUILD_ID }}.sql - asset_name: demo_data-${{ env.BUILD_ID }}.sql + asset_path: ${{ github.workspace }}/artifacts/seed_data-${{ env.BUILD_ID }}.sql + asset_name: seed_data-${{ env.BUILD_ID }}.sql asset_content_type: application/octet-stream - name: Upload Cuebot JAR diff --git a/ci/extract_schema.sh b/ci/extract_schema.sh index c4365962e..ec38b6344 100755 --- a/ci/extract_schema.sh +++ b/ci/extract_schema.sh @@ -34,7 +34,7 @@ docker exec -t --user=${DB_USER} ${PG_CONTAINER} pg_dump --no-privileges --no-ow | tee "${ARTIFACT_DIRECTORY}/schema-${BUILD_ID}.sql" # The demo data gets its own build artifact too. -cp "${SCHEMA_DIRECTORY}/demo_data.sql" "${ARTIFACT_DIRECTORY}/demo_data-${BUILD_ID}.sql" +cp "${SCHEMA_DIRECTORY}/seed_data.sql" "${ARTIFACT_DIRECTORY}/seed_data-${BUILD_ID}.sql" docker kill ${FLYWAY_CONTAINER} docker kill ${PG_CONTAINER} diff --git a/cuebot/oracle/xe/run_db_container.sh b/cuebot/oracle/xe/run_db_container.sh index fd307bd02..4d82a2e83 100755 --- a/cuebot/oracle/xe/run_db_container.sh +++ b/cuebot/oracle/xe/run_db_container.sh @@ -12,7 +12,7 @@ ORACLE_RPM="oracle-xe-11.2.0-1.0.x86_64.rpm.zip" ORACLE_DOCKER_REPO="https://github.com/oracle/docker-images.git" DOCKER_NAME="oracle-xe" ORACLE_SQL_FILE='/tmp/oracle_ddl/schema.sql' -ORACLE_SQL_DATA_FILE='/tmp/oracle_ddl/demo_data.sql' +ORACLE_SQL_DATA_FILE='/tmp/oracle_ddl/seed_data.sql' CUE_DB_USER='CUE' @@ -54,7 +54,7 @@ if [ "$3" = "--build-prod" ]; then echo "Applying Schema..." docker exec oracle-xe /bin/bash -c "mkdir $(dirname $ORACLE_SQL_FILE)" docker cp ${CUEBOT_ROOT_DIRECTORY}/src/main/resources/conf/ddl/oracle/schema.sql oracle-xe:$ORACLE_SQL_FILE - docker cp ${CUEBOT_ROOT_DIRECTORY}/src/test/resources/conf/ddl/oracle/demo_data.sql oracle-xe:$ORACLE_SQL_DATA_FILE + docker cp ${CUEBOT_ROOT_DIRECTORY}/src/test/resources/conf/ddl/oracle/seed_data.sql oracle-xe:$ORACLE_SQL_DATA_FILE docker cp ${CUEBOT_ROOT_DIRECTORY}/oracle/xe/apply_schema.sh oracle-xe:/tmp/ docker cp ${CUEBOT_ROOT_DIRECTORY}/oracle/xe/apply_schema.py oracle-xe:/tmp/ docker exec oracle-xe /bin/bash -c "/tmp/apply_schema.sh $2 $CUE_DB_USER $ORACLE_SQL_FILE $ORACLE_SQL_DATA_FILE" diff --git a/cuebot/src/main/resources/conf/ddl/postgres/demo_data.sql b/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql similarity index 97% rename from cuebot/src/main/resources/conf/ddl/postgres/demo_data.sql rename to cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql index ab2787dca..ebebaa75d 100644 --- a/cuebot/src/main/resources/conf/ddl/postgres/demo_data.sql +++ b/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql @@ -63,8 +63,6 @@ Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN, Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA7','katana',true,100,2097152,'general | desktop | util'); -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA8','shake',false,100,2097152,'general | desktop'); - Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA9','nuke',false,100,2097152,'general | desktop'); Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAA11','preprocess',false,10,393216,'util'); diff --git a/sandbox/flyway.Dockerfile b/sandbox/flyway.Dockerfile index a7c6ebf45..900c4449c 100644 --- a/sandbox/flyway.Dockerfile +++ b/sandbox/flyway.Dockerfile @@ -12,7 +12,7 @@ RUN ["cp", "/usr/share/java/postgresql-jdbc.jar", "jars/"] RUN ["mkdir", "/opt/migrations"] RUN ["mkdir", "/opt/scripts"] COPY ./cuebot/src/main/resources/conf/ddl/postgres/migrations /opt/migrations -COPY ./cuebot/src/main/resources/conf/ddl/postgres/demo_data.sql /opt/scripts +COPY ./cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql /opt/scripts COPY ./sandbox/migrate.sh /opt/scripts/ CMD ["/bin/bash"] diff --git a/sandbox/migrate.sh b/sandbox/migrate.sh index 57892b606..529dca834 100755 --- a/sandbox/migrate.sh +++ b/sandbox/migrate.sh @@ -18,5 +18,5 @@ done # Check if a show exists, if not apply demo data if psql -c "select 1 from show"|grep "(0 rows)"; then - psql -a -f /opt/scripts/demo_data.sql + psql -a -f /opt/scripts/seed_data.sql fi diff --git a/sandbox/setup-database.sh b/sandbox/setup-database.sh index 56761a61f..50383ba95 100755 --- a/sandbox/setup-database.sh +++ b/sandbox/setup-database.sh @@ -149,7 +149,7 @@ read -n 1 -p "Select mode of population: " POPULATING_OPT if [[ $POPULATING_OPT -eq 1 ]] then wget ${BASE_URL}"${VERSION}"/schema-"${VERSION}".sql -P ./db-data/ - wget ${BASE_URL}"${VERSION}"/demo_data-"${VERSION}".sql -P ./db-data/ + wget ${BASE_URL}"${VERSION}"/seed_data-"${VERSION}".sql -P ./db-data/ echo "" echo "Populating the database schema and some initial data" @@ -164,7 +164,7 @@ elif [[ $POPULATING_OPT -eq 2 ]] then brew install flyway || flyway -url=jdbc:postgresql://$DB_HOST/$DB_NAME -user="$USER" -n -locations=filesystem:/cuebot/src/main/resources/conf/ddl/postgres/migrations migrate - psql -h $DB_HOST -f /cuebot/src/main/resources/conf/ddl/postgres/demo_data.sql $DB_NAME + psql -h $DB_HOST -f /cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql $DB_NAME else echo "" echo "Invalid option!" From 070e23a4e6f8a3b6225390ac4a2befdff3976634 Mon Sep 17 00:00:00 2001 From: Idris Miles Date: Sat, 3 Apr 2021 11:55:26 +0100 Subject: [PATCH 071/277] Add create show dialog (#892) (#930) * Add create show dialog (#892) - Implement a dialog for creating shows and setting up their subscriptions. - Add right click action to show widget for creating new shows. * Replace create show menu action with button --- cuegui/cuegui/CreateShowDialog.py | 206 +++++++++++++++++++++++++++ cuegui/cuegui/plugins/ShowsPlugin.py | 12 ++ 2 files changed, 218 insertions(+) create mode 100644 cuegui/cuegui/CreateShowDialog.py diff --git a/cuegui/cuegui/CreateShowDialog.py b/cuegui/cuegui/CreateShowDialog.py new file mode 100644 index 000000000..8bb733151 --- /dev/null +++ b/cuegui/cuegui/CreateShowDialog.py @@ -0,0 +1,206 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""A dialog for creating new shows.""" + + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + + +from PySide2 import QtCore +from PySide2 import QtWidgets + +import opencue + + +class CreateShowDialog(QtWidgets.QDialog): + """A dialog for creating new shows. + _________________________________________________ + | Show name |__Enter show name here__| | + | | + | subscriptions_______________________________ | + | |_| local.general size |_____| burst |____| | + | |_| local.desktop size |_____| burst |____| | + | |_| cloud.general size |_____| burst |____| | + | | + | |_create_| |_cancel_| | + |_______________________________________________| + """ + + def __init__(self, parent=None): + QtWidgets.QDialog.__init__(self, parent) + self.subscription_fields = [] + + self.setWindowTitle("Create New Show") + self.setAttribute(QtCore.Qt.WA_DeleteOnClose) + self.setSizeGripEnabled(True) + + self.__create_btn = QtWidgets.QPushButton("Create", self) + self.__cancel_btn = QtWidgets.QPushButton("Close", self) + + self.__name_label = QtWidgets.QLabel("Show name") + self.__name_field = QtWidgets.QLineEdit() + self.__name_field.setPlaceholderText("Enter show name here") + + self.__subscription_grpbox = self.__createSubscriptionWidget() + + QtWidgets.QGridLayout(self) + self.layout().addWidget(self.__name_label, 0, 0, 1, 1) + self.layout().addWidget(self.__name_field, 0, 1, 1, 2) + self.layout().addWidget(self.__subscription_grpbox, 1, 0, 4, 3) + self.layout().addWidget(self.__create_btn, 5, 1) + self.layout().addWidget(self.__cancel_btn, 5, 2) + + self.__create_btn.clicked.connect(self.__createShow) + self.__cancel_btn.clicked.connect(self.__cancelDialog) + self.adjustSize() + + def __createSubscriptionWidget(self): + """Create the groupbox widget containing subscription fields""" + widget = QtWidgets.QGroupBox("Subscriptions") + layout = QtWidgets.QGridLayout() + + layout.addWidget(QtWidgets.QLabel("Allocation"), 0, 0 , 1, 1) + layout.addWidget(QtWidgets.QLabel("Size"), 0, 1 , 1, 1) + layout.addWidget(QtWidgets.QLabel("Burst"), 0, 2 , 1, 1) + + row = 1 + for allocation in opencue.api.getAllocations(): + alloc_checkbox = QtWidgets.QCheckBox(allocation.name()) + layout.addWidget(alloc_checkbox, row, 0 , 1, 1) + + size_spinbox = QtWidgets.QDoubleSpinBox(self) + size_spinbox.setMaximum(1000000) + size_spinbox.setValue(100) + layout.addWidget(size_spinbox, row, 1 , 1, 1) + + burst_spinbox = QtWidgets.QDoubleSpinBox(self) + burst_spinbox.setMaximum(1000000) + burst_spinbox.setValue(100) + layout.addWidget(burst_spinbox, row, 2 , 1, 1) + + self.subscription_fields.append({ + "allocation": allocation, + "enabled": alloc_checkbox, + "size": size_spinbox, + "burst": burst_spinbox + }) + + row += 1 + + widget.setLayout(layout) + return widget + + def __createShow(self): + """Create the show and specified subscriptions""" + if not self.__validate(): + return + + show = self.tryCreateShow() + if not show: + return + + for subscription in self.subscription_fields: + self.tryCreateSubscription(show, subscription) + + self.accept() + + def __cancelDialog(self): + """Abort creating a new show""" + self.reject() + + def __validate(self): + """Validate fields before creating a show""" + if not self.__validateName(): + return False + + if not self.__validateNoDuplicateShow(): + return False + + return True + + def __validateName(self): + """Validate the name field""" + show_name = self.__name_field.text() + if not show_name: + QtWidgets.QMessageBox.critical( + self, + "Invalid Show Name", + "Please enter a valid show name.", + QtWidgets.QMessageBox.Ok + ) + return False + return True + + def __validateNoDuplicateShow(self): + """Validate an existing show with the same name doesn't exist""" + show_name = self.__name_field.text() + try: + opencue.api.findShow(show_name) + except opencue.EntityNotFoundException: + return True + + QtWidgets.QMessageBox.critical( + self, + "Show Already Exists", + "A show with that name already exists, please enter a unique show name.", + QtWidgets.QMessageBox.Ok + ) + + return False + + def tryCreateShow(self): + """Try to create the show in OpenCue + + @return: An opencue.wrappers.show.Show if successful + """ + try: + show = opencue.api.createShow(self.__name_field.text()) + return show + except opencue.exception.CueException as e: + QtWidgets.QMessageBox.critical( + self, + "Failed To Create Show", + str(e), + QtWidgets.QMessageBox.Ok + ) + + def tryCreateSubscription(self, show, subscription): + """Try to create a subscription for the show in OpenCue + + @type show: opencue.wrappers.show.Show + @param show: The show to create a subscription for. + @type subscription: dict + @param subscription: A dictionary containing the Allocation instance + along with the other subscription field widgets. + """ + if not subscription["enabled"].isChecked(): + return + + try: + show.createSubscription( + subscription["allocation"], + float(subscription["size"].value()), + float(subscription["burst"].value()) + ) + except opencue.exception.CueException as e: + QtWidgets.QMessageBox.critical( + self, + "Failed To Create Subscription", + str(e), + QtWidgets.QMessageBox.Ok + ) diff --git a/cuegui/cuegui/plugins/ShowsPlugin.py b/cuegui/cuegui/plugins/ShowsPlugin.py index 89ef37754..5b0b94aad 100644 --- a/cuegui/cuegui/plugins/ShowsPlugin.py +++ b/cuegui/cuegui/plugins/ShowsPlugin.py @@ -20,7 +20,10 @@ from __future__ import division from __future__ import absolute_import +from PySide2 import QtWidgets + import cuegui.AbstractDockWidget +import cuegui.CreateShowDialog import cuegui.ShowsWidget @@ -38,7 +41,11 @@ def __init__(self, parent): super(ShowsDockWidget, self).__init__(parent, PLUGIN_NAME) self.__showsWidget = cuegui.ShowsWidget.ShowsWidget(self) + self.__createShowButton = QtWidgets.QPushButton("Create Show") + self.__createShowButton.setFixedWidth(150) + self.__createShowButton.clicked.connect(self.onCreateShowClicked) + self.layout().addWidget(self.__createShowButton) self.layout().addWidget(self.__showsWidget) self.pluginRegisterSettings([("columnVisibility", @@ -47,3 +54,8 @@ def __init__(self, parent): ("columnOrder", self.__showsWidget.getColumnOrder, self.__showsWidget.setColumnOrder)]) + + def onCreateShowClicked(self): + """Show the dialog for creating new shows""" + d = cuegui.CreateShowDialog.CreateShowDialog(self) + d.exec_() From a77c1ffa438f82b7a63d470814ce5d3020a1331b Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sun, 4 Apr 2021 14:37:58 -0700 Subject: [PATCH 072/277] Remove blackout (#946) * Remove blackout * Bump up version --- VERSION.in | 2 +- .../java/com/imageworks/spcue/DeedEntity.java | 14 ----- .../com/imageworks/spcue/dao/BookingDao.java | 8 --- .../com/imageworks/spcue/dao/DeedDao.java | 16 ------ .../spcue/dao/oracle/BookingDaoJdbc.java | 55 ------------------- .../spcue/dao/oracle/DeedDaoJdbc.java | 21 ------- .../spcue/dao/oracle/WhiteboardDaoJdbc.java | 6 -- .../spcue/dao/postgres/BookingDaoJdbc.java | 55 ------------------- .../spcue/dao/postgres/DeedDaoJdbc.java | 21 ------- .../spcue/dao/postgres/WhiteboardDaoJdbc.java | 6 -- .../spcue/dispatcher/HostReportHandler.java | 10 ---- .../imageworks/spcue/servant/ManageDeed.java | 24 -------- .../spcue/service/BookingManager.java | 7 --- .../spcue/service/BookingManagerService.java | 6 -- .../spcue/service/OwnerManager.java | 15 ----- .../spcue/service/OwnerManagerService.java | 10 ---- .../spcue/test/dao/oracle/DeedDaoTests.java | 43 --------------- .../spcue/test/dao/postgres/DeedDaoTests.java | 43 --------------- .../spcue/test/service/OwnerManagerTests.java | 35 ------------ proto/host.proto | 26 --------- pycue/opencue/wrappers/deed.py | 47 ---------------- pycue/tests/wrappers/deed_test.py | 30 ---------- 22 files changed, 1 insertion(+), 499 deletions(-) diff --git a/VERSION.in b/VERSION.in index 68c123cf1..51176c7c8 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.10 +0.11 diff --git a/cuebot/src/main/java/com/imageworks/spcue/DeedEntity.java b/cuebot/src/main/java/com/imageworks/spcue/DeedEntity.java index 51d252352..fe518ef20 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/DeedEntity.java +++ b/cuebot/src/main/java/com/imageworks/spcue/DeedEntity.java @@ -25,20 +25,6 @@ public class DeedEntity extends Entity { public String host; public String show; - /** - * The owner can set a black out time for booking where Cue will not - * automatically book the cores, even if NIMBY locked. - * - * This is measured in seconds past midnight. - */ - public int blackoutStart = 0; - public int blackoutStop = 0; - - /** - * Quickly disable and enable the current black out time settings. - */ - public boolean isBlackoutEnabled = false; - public String getName() { return String.format("%s.%s", owner, host); } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/BookingDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/BookingDao.java index 4b362f000..9fe9703cb 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/BookingDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/BookingDao.java @@ -126,14 +126,6 @@ void insertLocalHostAssignment(HostInterface host, FrameInterface frame, */ boolean hasActiveLocalJob(HostInterface host); - /** - * Return true if the host is in blackout time. - * - * @param h - * @return - */ - boolean isBlackoutTime(HostInterface h); - /** * Delete the given LocalHostAssignment. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/DeedDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/DeedDao.java index 555e6beeb..b02b58222 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/DeedDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/DeedDao.java @@ -66,22 +66,6 @@ public interface DeedDao { */ List getDeeds(OwnerEntity owner); - /** - * Enable/Disable the blackout time. - * - * @param value - */ - void updateBlackoutTimeEnabled(DeedEntity deed, boolean value); - - /** - * Set blackout times. During blackout times, machines - * cannot be booked. - * - * @param start - * @param stop - */ - void setBlackoutTime(DeedEntity deed, int startSeconds, int stopSeconds); - /** * * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/BookingDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/BookingDaoJdbc.java index d44423045..220983026 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/BookingDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/BookingDaoJdbc.java @@ -250,61 +250,6 @@ public boolean hasActiveLocalJob(HostInterface host) { Integer.class, host.getHostId()) > 0; } - private static final String IS_BLACKOUT_TIME = - "SELECT " + - "int_blackout_start,"+ - "int_blackout_duration " + - "FROM " + - "host,"+ - "deed "+ - "WHERE " + - "host.pk_host = deed.pk_host " + - "AND " + - "deed.b_blackout = 1 " + - "AND " + - "host.pk_host = ? "; - - public static final RowMapper BLACKOUT_MAPPER = - new RowMapper() { - public Boolean mapRow(final ResultSet rs, int rowNum) throws SQLException { - - int startTimeSeconds = rs.getInt("int_backout_start"); - int stopTimeSeconds = rs.getInt("int_blackout_stop"); - if (stopTimeSeconds <= startTimeSeconds) { - stopTimeSeconds = stopTimeSeconds + 86400; - } - - Calendar startTime = Calendar.getInstance(); - startTime.set(Calendar.HOUR_OF_DAY, 0); - startTime.set(Calendar.MINUTE, 0); - startTime.set(Calendar.SECOND, 0); - startTime.add(Calendar.SECOND, startTimeSeconds); - - Calendar stopTime = Calendar.getInstance(); - stopTime.set(Calendar.HOUR_OF_DAY, 0); - stopTime.set(Calendar.MINUTE, 0); - stopTime.set(Calendar.SECOND, 0); - stopTime.add(Calendar.SECOND, stopTimeSeconds); - - Calendar now = Calendar.getInstance(); - if (now.compareTo(startTime) >= 0 && now.compareTo(stopTime) <= 0) { - return true; - } - - return false; - } - }; - - @Override - public boolean isBlackoutTime(HostInterface h) { - try { - return getJdbcTemplate().queryForObject(IS_BLACKOUT_TIME, - BLACKOUT_MAPPER, h.getHostId()); - } catch (Exception e) { - return false; - } - } - @Override public int getCoreUsageDifference(LocalHostAssignment l, int coreUnits) { return getJdbcTemplate().queryForObject( diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DeedDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DeedDaoJdbc.java index 80adf6db5..68ab67521 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DeedDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DeedDaoJdbc.java @@ -41,9 +41,6 @@ public DeedEntity mapRow(ResultSet rs, int rowNum) throws SQLException { o.id = rs.getString("pk_deed"); o.owner = rs.getString("str_username"); o.host = rs.getString("str_hostname"); - o.isBlackoutEnabled = rs.getBoolean("b_blackout"); - o.blackoutStart = rs.getInt("int_blackout_start"); - o.blackoutStop = rs.getInt("int_blackout_stop"); return o; } }; @@ -94,9 +91,6 @@ public DeedEntity insertDeed(OwnerEntity owner, HostInterface host) { private static final String QUERY_FOR_DEED = "SELECT " + "deed.pk_deed, "+ - "deed.b_blackout,"+ - "deed.int_blackout_start,"+ - "deed.int_blackout_stop, " + "host.str_name as str_hostname, " + "owner.str_username " + "FROM " + @@ -121,20 +115,5 @@ public List getDeeds(OwnerEntity owner) { QUERY_FOR_DEED + " AND owner.pk_owner = ?", DEED_MAPPER, owner.getId()); } - - @Override - public void setBlackoutTime(DeedEntity deed, int startSeconds, int stopSeconds) { - getJdbcTemplate().update( - "UPDATE deed SET int_blackout_start = ?, " + - "int_blackout_stop = ? WHERE deed.pk_deed = ?", - startSeconds, stopSeconds, deed.getId()); - } - - @Override - public void updateBlackoutTimeEnabled(DeedEntity deed, boolean bool) { - getJdbcTemplate().update( - "UPDATE deed SET b_blackout = ? WHERE deed.pk_deed = ?", - bool, deed.getId()); - } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/WhiteboardDaoJdbc.java index 97ddcb927..75d631d4a 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/WhiteboardDaoJdbc.java @@ -882,9 +882,6 @@ public Deed mapRow(ResultSet rs, int rowNum) throws SQLException { .setId(SqlUtil.getString(rs, "pk_deed")) .setHost(SqlUtil.getString(rs, "str_host")) .setOwner(SqlUtil.getString(rs, "str_username")) - .setBlackout(rs.getBoolean("b_blackout")) - .setBlackoutStartTime(rs.getInt("int_blackout_start")) - .setBlackoutStopTime(rs.getInt("int_blackout_stop")) .build(); } }; @@ -2166,9 +2163,6 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "host.str_name AS str_host,"+ "show.str_name AS str_show,"+ "owner.str_username," + - "deed.b_blackout,"+ - "deed.int_blackout_start,"+ - "deed.int_blackout_stop,"+ "deed.pk_deed " + "FROM " + "deed,"+ diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/BookingDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/BookingDaoJdbc.java index b6ec245f1..550cddc17 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/BookingDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/BookingDaoJdbc.java @@ -250,61 +250,6 @@ public boolean hasActiveLocalJob(HostInterface host) { Integer.class, host.getHostId()) > 0; } - private static final String IS_BLACKOUT_TIME = - "SELECT " + - "int_blackout_start,"+ - "int_blackout_duration " + - "FROM " + - "host,"+ - "deed "+ - "WHERE " + - "host.pk_host = deed.pk_host " + - "AND " + - "deed.b_blackout = true " + - "AND " + - "host.pk_host = ? "; - - public static final RowMapper BLACKOUT_MAPPER = - new RowMapper() { - public Boolean mapRow(final ResultSet rs, int rowNum) throws SQLException { - - int startTimeSeconds = rs.getInt("int_backout_start"); - int stopTimeSeconds = rs.getInt("int_blackout_stop"); - if (stopTimeSeconds <= startTimeSeconds) { - stopTimeSeconds = stopTimeSeconds + 86400; - } - - Calendar startTime = Calendar.getInstance(); - startTime.set(Calendar.HOUR_OF_DAY, 0); - startTime.set(Calendar.MINUTE, 0); - startTime.set(Calendar.SECOND, 0); - startTime.add(Calendar.SECOND, startTimeSeconds); - - Calendar stopTime = Calendar.getInstance(); - stopTime.set(Calendar.HOUR_OF_DAY, 0); - stopTime.set(Calendar.MINUTE, 0); - stopTime.set(Calendar.SECOND, 0); - stopTime.add(Calendar.SECOND, stopTimeSeconds); - - Calendar now = Calendar.getInstance(); - if (now.compareTo(startTime) >= 0 && now.compareTo(stopTime) <= 0) { - return true; - } - - return false; - } - }; - - @Override - public boolean isBlackoutTime(HostInterface h) { - try { - return getJdbcTemplate().queryForObject(IS_BLACKOUT_TIME, - BLACKOUT_MAPPER, h.getHostId()); - } catch (Exception e) { - return false; - } - } - @Override public int getCoreUsageDifference(LocalHostAssignment l, int coreUnits) { return getJdbcTemplate().queryForObject( diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DeedDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DeedDaoJdbc.java index b002d3033..6d461423f 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DeedDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DeedDaoJdbc.java @@ -41,9 +41,6 @@ public DeedEntity mapRow(ResultSet rs, int rowNum) throws SQLException { o.id = rs.getString("pk_deed"); o.owner = rs.getString("str_username"); o.host = rs.getString("str_hostname"); - o.isBlackoutEnabled = rs.getBoolean("b_blackout"); - o.blackoutStart = rs.getInt("int_blackout_start"); - o.blackoutStop = rs.getInt("int_blackout_stop"); return o; } }; @@ -94,9 +91,6 @@ public DeedEntity insertDeed(OwnerEntity owner, HostInterface host) { private static final String QUERY_FOR_DEED = "SELECT " + "deed.pk_deed, "+ - "deed.b_blackout,"+ - "deed.int_blackout_start,"+ - "deed.int_blackout_stop, " + "host.str_name as str_hostname, " + "owner.str_username " + "FROM " + @@ -121,20 +115,5 @@ public List getDeeds(OwnerEntity owner) { QUERY_FOR_DEED + " AND owner.pk_owner = ?", DEED_MAPPER, owner.getId()); } - - @Override - public void setBlackoutTime(DeedEntity deed, int startSeconds, int stopSeconds) { - getJdbcTemplate().update( - "UPDATE deed SET int_blackout_start = ?, " + - "int_blackout_stop = ? WHERE deed.pk_deed = ?", - startSeconds, stopSeconds, deed.getId()); - } - - @Override - public void updateBlackoutTimeEnabled(DeedEntity deed, boolean bool) { - getJdbcTemplate().update( - "UPDATE deed SET b_blackout = ? WHERE deed.pk_deed = ?", - bool, deed.getId()); - } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index 687e91ecd..95712d605 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -880,9 +880,6 @@ public Deed mapRow(ResultSet rs, int rowNum) throws SQLException { .setId(SqlUtil.getString(rs,"pk_deed")) .setHost(SqlUtil.getString(rs,"str_host")) .setOwner(SqlUtil.getString(rs,"str_username")) - .setBlackout(rs.getBoolean("b_blackout")) - .setBlackoutStartTime(rs.getInt("int_blackout_start")) - .setBlackoutStopTime(rs.getInt("int_blackout_stop")) .build(); } }; @@ -2160,9 +2157,6 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "host.str_name AS str_host,"+ "show.str_name AS str_show,"+ "owner.str_username," + - "deed.b_blackout,"+ - "deed.int_blackout_start,"+ - "deed.int_blackout_stop,"+ "deed.pk_deed " + "FROM " + "deed,"+ diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java index 815689bb4..69b602430 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java @@ -289,16 +289,6 @@ else if (!dispatchSupport.isCueBookable(host)) { return; } - /* - * Check for NIMBY blackout time. - */ - /* - if (bookingManager.isBlackOutTime(host)) { - logger.trace(host + " is blacked out."); - return ; - } - */ - /* * Check if the host prefers a show. If it does , dispatch * to that show first. diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageDeed.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageDeed.java index f92d03dea..057be1843 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageDeed.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageDeed.java @@ -30,10 +30,6 @@ import com.imageworks.spcue.grpc.host.DeedGetOwnerRequest; import com.imageworks.spcue.grpc.host.DeedGetOwnerResponse; import com.imageworks.spcue.grpc.host.DeedInterfaceGrpc; -import com.imageworks.spcue.grpc.host.DeedSetBlackoutTimeEnabledRequest; -import com.imageworks.spcue.grpc.host.DeedSetBlackoutTimeEnabledResponse; -import com.imageworks.spcue.grpc.host.DeedSetBlackoutTimeRequest; -import com.imageworks.spcue.grpc.host.DeedSetBlackoutTimeResponse; import com.imageworks.spcue.grpc.host.Host; import com.imageworks.spcue.grpc.host.Owner; import com.imageworks.spcue.service.OwnerManager; @@ -65,23 +61,6 @@ public void getOwner(DeedGetOwnerRequest request, StreamObserver responseObserver) { - ownerManager.setBlackoutTime(toEntity(request.getDeed()), request.getStartTime(), request.getStopTime()); - responseObserver.onNext(DeedSetBlackoutTimeResponse.newBuilder().build()); - responseObserver.onCompleted(); - } - - @Override - public void setBlackoutTimeEnabled(DeedSetBlackoutTimeEnabledRequest request, - StreamObserver responseObserver) { - ownerManager.setBlackoutTimeEnabled(toEntity(request.getDeed()), request.getEnabled()); - responseObserver.onNext(DeedSetBlackoutTimeEnabledResponse.newBuilder().build()); - responseObserver.onCompleted(); - } - - public OwnerManager getOwnerManager() { return ownerManager; } @@ -104,9 +83,6 @@ private DeedEntity toEntity(Deed deed) { entity.host = deed.getHost(); entity.owner = deed.getOwner(); entity.show = deed.getShow(); - entity.isBlackoutEnabled = deed.getBlackout(); - entity.blackoutStart = deed.getBlackoutStartTime(); - entity.blackoutStop = deed.getBlackoutStopTime(); return entity; } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/BookingManager.java b/cuebot/src/main/java/com/imageworks/spcue/service/BookingManager.java index 51a04bfa3..6c348eeb4 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/BookingManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/BookingManager.java @@ -133,13 +133,6 @@ public void createLocalHostAssignment(DispatchHost host, */ void removeInactiveLocalHostAssignment(LocalHostAssignment lha); - /** - * - * @param host - * @return - */ - boolean isBlackOutTime(HostInterface host); - /** * Return true if the host is running more cores than the maximum allowed. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/BookingManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/BookingManagerService.java index 91abdab4b..02b2fe948 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/BookingManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/BookingManagerService.java @@ -172,12 +172,6 @@ public void createLocalHostAssignment(DispatchHost host, FrameInterface frame, bookingDao.insertLocalHostAssignment(host, frame, lja); } - @Override - @Transactional(propagation = Propagation.REQUIRED, readOnly=true) - public boolean isBlackOutTime(HostInterface host) { - return bookingDao.isBlackoutTime(host); - } - @Override @Transactional(propagation = Propagation.REQUIRED, readOnly=true) public boolean hasResourceDeficit(HostInterface host) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/OwnerManager.java b/cuebot/src/main/java/com/imageworks/spcue/service/OwnerManager.java index ca57a52e7..d639405ee 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/OwnerManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/OwnerManager.java @@ -81,13 +81,6 @@ public interface OwnerManager { */ DeedEntity takeOwnership(OwnerEntity owner, HostInterface host); - /** - * - * @param deed - * @param value - */ - void setBlackoutTimeEnabled(DeedEntity deed, boolean value); - /** * * @param id @@ -95,14 +88,6 @@ public interface OwnerManager { */ DeedEntity getDeed(String id); - /** - * - * @param deed - * @param startSeconds - * @param stopSeconds - */ - void setBlackoutTime(DeedEntity deed, int startSeconds, int stopSeconds); - /** * Deletes a deed for the specified host. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/OwnerManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/OwnerManagerService.java index 30273968f..d3fb63ee6 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/OwnerManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/OwnerManagerService.java @@ -70,16 +70,6 @@ public DeedEntity getDeed(String id) { return deedDao.getDeed(id); } - @Override - public void setBlackoutTime(DeedEntity deed, int startSeconds, int stopSeconds) { - deedDao.setBlackoutTime(deed, startSeconds, stopSeconds); - } - - @Override - public void setBlackoutTimeEnabled(DeedEntity deed, boolean value) { - deedDao.updateBlackoutTimeEnabled(deed, value); - } - @Override public DeedEntity takeOwnership(OwnerEntity owner, HostInterface host) { if (!hostDao.isNimbyHost(host)) { diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/DeedDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/DeedDaoTests.java index d174c089e..788fe040c 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/DeedDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/DeedDaoTests.java @@ -165,48 +165,5 @@ public void tesGetDeeds() { assertEquals(1, deedDao.getDeeds(o).size()); assertEquals(d, deedDao.getDeeds(o).get(0)); } - - - @Test - @Transactional - @Rollback(true) - public void testEnableDisableBlackoutTime() { - - DispatchHost host = createHost(); - ShowInterface s = adminManager.findShowEntity("pipe"); - OwnerEntity o = ownerManager.createOwner("squarepants", s); - DeedEntity d = deedDao.insertDeed(o, host); - - deedDao.updateBlackoutTimeEnabled(d, true); - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT b_blackout FROM deed WHERE pk_deed=?", - Integer.class, d.getId())); - - deedDao.updateBlackoutTimeEnabled(d, false); - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT b_blackout FROM deed WHERE pk_deed=?", - Integer.class, d.getId())); - } - - @Test - @Transactional - @Rollback(true) - public void testSetBlackOutTimes() { - - DispatchHost host = createHost(); - ShowInterface s = adminManager.findShowEntity("pipe"); - OwnerEntity o = ownerManager.createOwner("squarepants", s); - DeedEntity d = deedDao.insertDeed(o, host); - - deedDao.setBlackoutTime(d, 3600, 7200); - - assertEquals(Integer.valueOf(3600), jdbcTemplate.queryForObject( - "SELECT int_blackout_start FROM deed WHERE pk_deed=?", - Integer.class, d.getId())); - - assertEquals(Integer.valueOf(7200), jdbcTemplate.queryForObject( - "SELECT int_blackout_stop FROM deed WHERE pk_deed=?", - Integer.class, d.getId())); - } } diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DeedDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DeedDaoTests.java index 3b06eb59c..23a93a2c1 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DeedDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DeedDaoTests.java @@ -165,48 +165,5 @@ public void tesGetDeeds() { assertEquals(1, deedDao.getDeeds(o).size()); assertEquals(d, deedDao.getDeeds(o).get(0)); } - - - @Test - @Transactional - @Rollback(true) - public void testEnableDisableBlackoutTime() { - - DispatchHost host = createHost(); - ShowInterface s = adminManager.findShowEntity("pipe"); - OwnerEntity o = ownerManager.createOwner("squarepants", s); - DeedEntity d = deedDao.insertDeed(o, host); - - deedDao.updateBlackoutTimeEnabled(d, true); - assertTrue(jdbcTemplate.queryForObject( - "SELECT b_blackout FROM deed WHERE pk_deed=?", - Boolean.class, d.getId())); - - deedDao.updateBlackoutTimeEnabled(d, false); - assertFalse(jdbcTemplate.queryForObject( - "SELECT b_blackout FROM deed WHERE pk_deed=?", - Boolean.class, d.getId())); - } - - @Test - @Transactional - @Rollback(true) - public void testSetBlackOutTimes() { - - DispatchHost host = createHost(); - ShowInterface s = adminManager.findShowEntity("pipe"); - OwnerEntity o = ownerManager.createOwner("squarepants", s); - DeedEntity d = deedDao.insertDeed(o, host); - - deedDao.setBlackoutTime(d, 3600, 7200); - - assertEquals(Integer.valueOf(3600), jdbcTemplate.queryForObject( - "SELECT int_blackout_start FROM deed WHERE pk_deed=?", - Integer.class, d.getId())); - - assertEquals(Integer.valueOf(7200), jdbcTemplate.queryForObject( - "SELECT int_blackout_stop FROM deed WHERE pk_deed=?", - Integer.class, d.getId())); - } } diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/OwnerManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/OwnerManagerTests.java index 38d178444..e11bab099 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/OwnerManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/OwnerManagerTests.java @@ -175,41 +175,6 @@ public void testGetDeed() { assertEquals(d, ownerManager.getDeed(d.id)); } - @Test - @Transactional - @Rollback(true) - public void testSetBlackoutTimes() { - OwnerEntity o = ownerManager.createOwner("spongebob", - adminManager.findShowEntity("pipe")); - - DispatchHost host = createHost(); - DeedEntity d = ownerManager.takeOwnership(o, host); - - ownerManager.setBlackoutTime(d, 0, 3600); - - assertEquals(0, deedDao.getDeed(d.id).blackoutStart); - assertEquals(3600, deedDao.getDeed(d.id).blackoutStop); - } - - @Test - @Transactional - @Rollback(true) - public void testEnableDisableBlackout() { - OwnerEntity o = ownerManager.createOwner("spongebob", - adminManager.findShowEntity("pipe")); - - DispatchHost host = createHost(); - DeedEntity d = ownerManager.takeOwnership(o, host); - - ownerManager.setBlackoutTimeEnabled(d, true); - - assertTrue(deedDao.getDeed(d.id).isBlackoutEnabled); - - ownerManager.setBlackoutTimeEnabled(d, false); - - assertFalse(deedDao.getDeed(d.id).isBlackoutEnabled); - } - @Test @Transactional @Rollback(true) diff --git a/proto/host.proto b/proto/host.proto index 7d715a4de..90c957816 100644 --- a/proto/host.proto +++ b/proto/host.proto @@ -22,12 +22,6 @@ service DeedInterface { // Returns the owner for these settings. rpc GetOwner(DeedGetOwnerRequest) returns (DeedGetOwnerResponse); - - // Sets a blackout time for the host. - rpc SetBlackoutTime(DeedSetBlackoutTimeRequest) returns (DeedSetBlackoutTimeResponse); - - // Enable/Disable blackout time without changing the times. - rpc SetBlackoutTimeEnabled(DeedSetBlackoutTimeEnabledRequest) returns (DeedSetBlackoutTimeEnabledResponse); } service HostInterface { @@ -235,9 +229,6 @@ message Deed { string host = 2; string owner = 3; string show = 4; - bool blackout = 5; - int32 blackout_start_time = 6; - int32 blackout_stop_time = 7; } message DeedSeq { @@ -440,23 +431,6 @@ message DeedGetOwnerResponse { Owner owner = 1; } -// SetBlackoutTimeEnabled -message DeedSetBlackoutTimeEnabledRequest { - Deed deed = 1; - bool enabled = 2; -} - -message DeedSetBlackoutTimeEnabledResponse {} // Empty - -// SetBlackoutTime -message DeedSetBlackoutTimeRequest { - Deed deed = 1; - int32 start_time = 2; - int32 stop_time = 3; -} - -message DeedSetBlackoutTimeResponse {} // Empty - // HOST ---- // AddComment message HostAddCommentRequest { diff --git a/pycue/opencue/wrappers/deed.py b/pycue/opencue/wrappers/deed.py index d29ec8854..bdd0933cd 100644 --- a/pycue/opencue/wrappers/deed.py +++ b/pycue/opencue/wrappers/deed.py @@ -52,29 +52,6 @@ def getOwner(self): self.stub.GetOwner(host_pb2.DeedGetOwnerRequest(deed=self.data), timeout=Cuebot.Timeout).owner) - def setBlackoutTime(self, startTime, stopTime): - """Sets a blackout time for the host. - - :type startTime: int - :param startTime: blackout start time as an epoch - :type stopTime: int - :param stopTime: blackout stop time as an epoch - """ - self.stub.SetBlackoutTime( - host_pb2.DeedSetBlackoutTimeRequest( - deed=self.data, start_time=startTime, stop_time=stopTime), - timeout=Cuebot.Timeout) - - def setBlackoutTimeEnabled(self, enabled): - """Enable/disable the host blackout time without changing the times. - - :type enabled: bool - :param enabled: enable/disable blackout time - """ - self.stub.SetBlackoutTimeEnabled( - host_pb2.DeedSetBlackoutTimeEnabledRequest(deed=self.data, enabled=enabled), - timeout=Cuebot.Timeout) - def id(self): """Returns the id of the deed. @@ -106,27 +83,3 @@ def show(self): :return: name of the deed show """ return self.data.show - - def blackout(self): - """Returns whether the blackout time is enabled. - - :rtype: bool - :return: whether the blackout is enabled - """ - return self.data.blackout - - def blackoutStartTime(self): - """Returns the blackout start time as an epoch. - - :rtype: int - :return: blackout start time as an epoch - """ - return self.data.blackout_start_time - - def blackoutStopTime(self): - """Returns the blackout end time as an epoch. - - :rtype: int - :return: blackout end time as an epoch - """ - return self.data.blackout_stop_time diff --git a/pycue/tests/wrappers/deed_test.py b/pycue/tests/wrappers/deed_test.py index 8fd0833f6..6d762fe4b 100644 --- a/pycue/tests/wrappers/deed_test.py +++ b/pycue/tests/wrappers/deed_test.py @@ -72,36 +72,6 @@ def testGetOwner(self, getStubMock): host_pb2.DeedGetOwnerRequest(deed=deed.data), timeout=mock.ANY) self.assertEqual(owner.name(), TEST_DEED_OWNER) - def testSetBlackoutTime(self, getStubMock): - stubMock = mock.Mock() - stubMock.SetBlackoutTime.return_value = host_pb2.DeedSetBlackoutTimeResponse() - getStubMock.return_value = stubMock - - testStartTime = 100 - testStopTime = 200 - deed = opencue.wrappers.deed.Deed(host_pb2.Deed(id=TEST_DEED_ID)) - deed.setBlackoutTime(testStartTime, testStopTime) - - stubMock.SetBlackoutTime.assert_called_with( - host_pb2.DeedSetBlackoutTimeRequest(deed=deed.data, - start_time=testStartTime, - stop_time=testStopTime), - timeout=mock.ANY) - - def testSetBlackoutTimeEnabled(self, getStubMock): - stubMock = mock.Mock() - stubMock.SetBlackoutTimeEnabled.return_value = host_pb2.DeedSetBlackoutTimeEnabledResponse() - getStubMock.return_value = stubMock - - testBlackoutEnabled = True - deed = opencue.wrappers.deed.Deed(host_pb2.Deed(id=TEST_DEED_ID)) - deed.setBlackoutTimeEnabled(testBlackoutEnabled) - - stubMock.SetBlackoutTimeEnabled.assert_called_with( - host_pb2.DeedSetBlackoutTimeEnabledRequest(deed=deed.data, - enabled=testBlackoutEnabled), - timeout=mock.ANY) - if __name__ == '__main__': unittest.main() From 401a33ba54b936246a64a38ada671f62d18b2257 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Tue, 6 Apr 2021 10:56:01 -0700 Subject: [PATCH 073/277] Auto-delete the down state hosts (#945) * Add testDeleteDownHosts * Add testDeleteDownHosts --- .../com/imageworks/spcue/dao/HostDao.java | 5 ++++ .../spcue/dao/oracle/HostDaoJdbc.java | 5 ++++ .../spcue/dao/postgres/HostDaoJdbc.java | 28 +++++++++++++++++++ .../service/MaintenanceManagerSupport.java | 18 ++++++++++++ .../spring/applicationContext-service.xml | 1 + cuebot/src/main/resources/opencue.properties | 3 ++ .../spcue/test/dao/postgres/HostDaoTests.java | 24 ++++++++++++++++ 7 files changed, 84 insertions(+) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java index 31b7d475d..04cd49f46 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java @@ -65,6 +65,11 @@ public interface HostDao { */ void deleteHost(HostInterface host); + /** + * deletes the down state hosts + */ + void deleteDownHosts(); + /** * updates a host with the passed hardware state * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/HostDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/HostDaoJdbc.java index a6ef7a079..15c74278c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/HostDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/HostDaoJdbc.java @@ -488,6 +488,11 @@ public void deleteHost(HostInterface host) { "DELETE FROM host WHERE pk_host=?",host.getHostId()); } + @Override + public void deleteDownHosts() { + // Not implemented. + } + @Override public void updateHostState(HostInterface host, HardwareState state) { getJdbcTemplate().update( diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java index bf42f0ecb..1efd6b597 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java @@ -489,6 +489,34 @@ public void deleteHost(HostInterface host) { "DELETE FROM host WHERE pk_host=?",host.getHostId()); } + private static final String DELETE_DOWN_HOST_COMMENTS = + "DELETE " + + "FROM " + + "comments " + + "USING " + + "host_stat " + + "WHERE " + + "comments.pk_host = host_stat.pk_host " + + "AND " + + "host_stat.str_state = ?"; + + private static final String DELETE_DOWN_HOSTS = + "DELETE " + + "FROM " + + "host " + + "USING " + + "host_stat " + + "WHERE " + + "host.pk_host = host_stat.pk_host " + + "AND " + + "host_stat.str_state=?"; + + @Override + public void deleteDownHosts() { + getJdbcTemplate().update(DELETE_DOWN_HOST_COMMENTS, HardwareState.DOWN.toString()); + getJdbcTemplate().update(DELETE_DOWN_HOSTS, HardwareState.DOWN.toString()); + } + @Override public void updateHostState(HostInterface host, HardwareState state) { getJdbcTemplate().update( diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/MaintenanceManagerSupport.java b/cuebot/src/main/java/com/imageworks/spcue/service/MaintenanceManagerSupport.java index a1b1ecadf..fdc44eddd 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/MaintenanceManagerSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/MaintenanceManagerSupport.java @@ -22,6 +22,8 @@ import java.util.List; import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; import org.springframework.jdbc.CannotGetJdbcConnectionException; import com.imageworks.spcue.FrameInterface; @@ -29,6 +31,7 @@ import com.imageworks.spcue.PointDetail; import com.imageworks.spcue.VirtualProc; import com.imageworks.spcue.dao.FrameDao; +import com.imageworks.spcue.dao.HostDao; import com.imageworks.spcue.dao.MaintenanceDao; import com.imageworks.spcue.dao.ProcDao; import com.imageworks.spcue.dispatcher.DispatchSupport; @@ -41,12 +44,17 @@ public class MaintenanceManagerSupport { private static final Logger logger = Logger.getLogger(MaintenanceManagerSupport.class); + @Autowired + private Environment env; + private MaintenanceDao maintenanceDao; private ProcDao procDao; private FrameDao frameDao; + private HostDao hostDao; + private JobManager jobManager; private DispatchSupport dispatchSupport; @@ -90,6 +98,12 @@ public void checkHardwareState() { int hosts = maintenanceDao.setUpHostsToDown(); if (hosts > 0) { clearDownProcs(); + + boolean autoDeleteDownHosts = env.getProperty( + "maintenance.auto_delete_down_hosts", Boolean.class, false); + if (autoDeleteDownHosts) { + hostDao.deleteDownHosts(); + } } clearOrphanedProcs(); } finally { @@ -193,6 +207,10 @@ public void setFrameDao(FrameDao frameDao) { this.frameDao = frameDao; } + public void setHostDao(HostDao hostDao) { + this.hostDao = hostDao; + } + public DispatchSupport getDispatchSupport() { return dispatchSupport; } diff --git a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml index c30812aa5..b7f6e3d14 100644 --- a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml +++ b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml @@ -342,6 +342,7 @@ + diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index 20d85970b..f77e10273 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -56,3 +56,6 @@ dispatcher.host_frame_dispatch_max=12 # Jobs will be archived to the history tables after being completed for this long. history.archive_jobs_cutoff_hours=72 + +# Delete down hosts automatically. +maintenance.auto_delete_down_hosts=false diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java index 9327fc8dd..b53e1078a 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java @@ -389,6 +389,30 @@ public void testDeleteHost() { assertEquals(hostDao.hostExists(TEST_HOST),false); } + @Test + @Transactional + @Rollback(true) + public void testDeleteDownHosts() { + for (int i = 0; i < 3; i++) { + String name = TEST_HOST + i; + hostDao.insertRenderHost(buildRenderHost(name), + hostManager.getDefaultAllocationDetail(), + false); + if (i != 1) { + HostEntity host = hostDao.findHostDetail(name); + assertEquals(name, host.name); + hostDao.updateHostState(host, HardwareState.DOWN); + } + } + + hostDao.deleteDownHosts(); + + for (int i = 0; i < 3; i++) { + String name = TEST_HOST + i; + assertEquals(hostDao.hostExists(name), i == 1); + } + } + @Test @Transactional @Rollback(true) From f833ed6dcbefe52237dbbb779951c9b34a4e5e9b Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Tue, 6 Apr 2021 11:37:09 -0700 Subject: [PATCH 074/277] Optimize bookingqueue (#949) * Update dispatchQuery to use min_cores Sorting jobs only by priority causes a situation where low priority jobs can get starved by a constant flow of high priority jobs. The new formula adds a modifier to the sorting rank to take into account the number of cores the job is requesting and also the number of days the job is waiting on the queue. Priorities numbers over 200 will mostly override the formula and work as a priority only based scheduling. sort = priority + (100 * (1 - (job.cores/job.int_min_cores))) + (age in days) Besides that, also take layer_int_cores_min into account when filtering folder_resourse limitations to avoid allocating more cores than the folder limits. (cherry picked from commit 566411aeeddc60983a30eabe121fd03263d05525) * Revert "Update dispatchQuery to use min_cores" This reverts commit 2eb4936c * Avoid repeated bookingQueue tasks It was observed that the booking performance is degrating over time. Around 4 tasks are submitted for each booking executed. This solution avoid creating more tasks if there's already a task waiting to be executed. (cherry picked from commit 57ef9ab989c4d82585b31c95d93b969f6e95e4cd) --- .../spcue/dispatcher/BookingQueue.java | 23 +++++++++++++++++-- .../dispatcher/commands/DispatchBookHost.java | 10 ++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java index ace595ea2..0c7563dbf 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java @@ -24,6 +24,9 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.imageworks.spcue.dispatcher.commands.DispatchBookHost; import org.apache.log4j.Logger; public class BookingQueue extends ThreadPoolExecutor { @@ -41,6 +44,12 @@ public class BookingQueue extends ThreadPoolExecutor { private QueueRejectCounter rejectCounter = new QueueRejectCounter(); + private Cache bookingCache = CacheBuilder.newBuilder() + .expireAfterWrite(3, TimeUnit.MINUTES) + // Invalidate entries that got executed by the threadpool and lost their reference + .weakValues() + .build(); + public BookingQueue(int sleepTimeMs) { super(THREADS_MINIMUM, THREADS_MAXIMUM, THREADS_KEEP_ALIVE_SECONDS, TimeUnit.SECONDS, new LinkedBlockingQueue(INITIAL_QUEUE_SIZE)); @@ -48,8 +57,12 @@ public BookingQueue(int sleepTimeMs) { this.setRejectedExecutionHandler(rejectCounter); } - public void execute(Runnable r) { - if (!isShutdown.get()) { + public void execute(DispatchBookHost r) { + if (isShutdown.get()) { + return; + } + if (bookingCache.getIfPresent(r.getKey()) == null){ + bookingCache.put(r.getKey(), r); super.execute(r); } } @@ -100,7 +113,13 @@ protected void beforeExecute(Thread t, Runnable r) { protected void afterExecute(Runnable r, Throwable t) { super.afterExecute(r, t); + + // Invalidate cache to avoid having to wait for GC to mark processed entries collectible + DispatchBookHost h = (DispatchBookHost)r; + bookingCache.invalidate(h.getKey()); + if (sleepTime() < 100) { + logger.info("BookingQueue cleanup executed."); getQueue().clear(); } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java index 6ddaa9a3b..c8971a0f4 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java @@ -37,34 +37,44 @@ public class DispatchBookHost implements Runnable { private JobInterface job = null; private DispatchHost host; private Dispatcher dispatcher; + private String key; public DispatchHost getDispatchHost() { + this.key = host.getId(); return host; } public DispatchBookHost(DispatchHost host, Dispatcher d) { this.host = host; + this.key = host.getId(); this.dispatcher = d; } public DispatchBookHost(DispatchHost host, JobInterface job, Dispatcher d) { this.host = host; this.job = job; + this.key = host.getId() + "_job_" + job.getJobId(); this.dispatcher = d; } public DispatchBookHost(DispatchHost host, GroupInterface group, Dispatcher d) { this.host = host; this.group = group; + this.key = host.getId() + "_group_" + group.getGroupId(); this.dispatcher = d; } public DispatchBookHost(DispatchHost host, ShowInterface show, Dispatcher d) { this.host = host; this.show = show; + this.key = host.getId() + "_name_" + show.getName(); this.dispatcher = d; } + public String getKey() { + return this.key; + } + public void run() { new DispatchCommandTemplate() { public void wrapDispatchCommand() { From 746f4e62ab6e5f174eb6efdfde2bea0761a5fa91 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Thu, 8 Apr 2021 08:23:03 -0700 Subject: [PATCH 075/277] Update MonitorJobsPlugin to check for empty string on search. (#952) --- cuegui/cuegui/plugins/MonitorJobsPlugin.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cuegui/cuegui/plugins/MonitorJobsPlugin.py b/cuegui/cuegui/plugins/MonitorJobsPlugin.py index c2f970b59..5f9c5b20e 100644 --- a/cuegui/cuegui/plugins/MonitorJobsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorJobsPlugin.py @@ -45,6 +45,7 @@ PLUGIN_CATEGORY = "Cuetopia" PLUGIN_DESCRIPTION = "Monitors a list of jobs" PLUGIN_PROVIDES = "MonitorJobsDockWidget" +REGEX_EMPTY_STRING = re.compile("^$") class MonitorJobsDockWidget(cuegui.AbstractDockWidget.AbstractDockWidget): @@ -173,9 +174,10 @@ def _regexLoadJobsHandle(self): for job in opencue.api.getJobs(substr=[substring], include_finished=True): self.jobMonitor.addJob(job) else: - # Otherwise, just load current matching jobs - for job in opencue.api.getJobs(regex=[substring]): - self.jobMonitor.addJob(job) + # Otherwise, just load current matching jobs (except for the empty string) + if not re.search(REGEX_EMPTY_STRING, substring): + for job in opencue.api.getJobs(regex=[substring]): + self.jobMonitor.addJob(job) def _buttonSetup(self, layout): clearButton = QtWidgets.QPushButton("Clr") From c437bfd3c89c78a0e8adca2208fdebb48e46606d Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Thu, 8 Apr 2021 08:25:37 -0700 Subject: [PATCH 076/277] New RQD hosts set allocation based on their facility and tags. (#941) --- .../spcue/service/HostManagerService.java | 24 ++++- .../dispatcher/HostReportHandlerTests.java | 92 +++++++++++++++++++ 2 files changed, 115 insertions(+), 1 deletion(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java index ee081ecc7..9bbaaa6e4 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java @@ -23,6 +23,7 @@ import java.util.List; import org.apache.log4j.Logger; +import org.springframework.dao.EmptyResultDataAccessException; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -152,7 +153,28 @@ public DispatchHost createHost(HostReport report) { @Transactional(propagation = Propagation.REQUIRED) public DispatchHost createHost(RenderHost rhost) { - return createHost(rhost, getDefaultAllocationDetail()); + // Find suitable allocation with facility and tags. + AllocationEntity alloc = null; + if (rhost.getTagsCount() > 0) { + String facility = rhost.getFacility(); + for (String tag : rhost.getTagsList()) { + try { + alloc = allocationDao.findAllocationEntity(facility, tag); + logger.info("set " + rhost.getName() + + " to the given allocation " + alloc.getName()); + break; + } + catch (EmptyResultDataAccessException e) { + // Allocation doesn't exist. ignore. + } + } + } + if (alloc == null) { + alloc = getDefaultAllocationDetail(); + logger.info("set " + rhost.getName() + + " to the default allocation " + alloc.getName()); + } + return createHost(rhost, alloc); } @Transactional(propagation = Propagation.REQUIRED) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java index 81fe8fd68..f24375cce 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java @@ -27,9 +27,11 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.transaction.annotation.Transactional; +import com.imageworks.spcue.AllocationEntity; import com.imageworks.spcue.DispatchHost; import com.imageworks.spcue.dispatcher.Dispatcher; import com.imageworks.spcue.dispatcher.HostReportHandler; +import com.imageworks.spcue.FacilityInterface; import com.imageworks.spcue.grpc.host.HardwareState; import com.imageworks.spcue.grpc.host.LockState; import com.imageworks.spcue.grpc.report.CoreDetail; @@ -58,6 +60,7 @@ public class HostReportHandlerTests extends TransactionalTest { Dispatcher dispatcher; private static final String HOSTNAME = "beta"; + private static final String NEW_HOSTNAME = "gamma"; @Before public void setTestMode() { @@ -106,6 +109,29 @@ private static RenderHost getRenderHost() { .build(); } + private static RenderHost getNewRenderHost(String tags) { + return RenderHost.newBuilder() + .setName(NEW_HOSTNAME) + .setBootTime(1192369572) + .setFreeMcp(76020) + .setFreeMem(53500) + .setFreeSwap(20760) + .setLoad(0) + .setTotalMcp(195430) + .setTotalMem(8173264) + .setTotalSwap(20960) + .setNimbyEnabled(false) + .setNumProcs(2) + .setCoresPerProc(100) + .addTags(tags) + .setState(HardwareState.UP) + .setFacility("spi") + .putAttributes("SP_OS", "Linux") + .putAttributes("freeGpu", String.format("%d", CueUtil.MB512)) + .putAttributes("totalGpu", String.format("%d", CueUtil.MB512)) + .build(); + } + @Test @Transactional @Rollback(true) @@ -121,5 +147,71 @@ public void testHandleHostReport() { DispatchHost host = getHost(); assertEquals(host.lockState, LockState.OPEN); } + + @Test + @Transactional + @Rollback(true) + public void testHandleHostReportWithNewAllocation() { + FacilityInterface facility = adminManager.getFacility( + "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA0"); + assertEquals(facility.getName(), "spi"); + + AllocationEntity detail = new AllocationEntity(); + detail.name = "test"; + detail.tag = "test"; + adminManager.createAllocation(facility, detail); + detail = adminManager.findAllocationDetail("spi", "test"); + + boolean isBoot = true; + CoreDetail cores = getCoreDetail(200, 200, 0, 0); + HostReport report = HostReport.newBuilder() + .setHost(getNewRenderHost("test")) + .setCoreInfo(cores) + .build(); + + hostReportHandler.handleHostReport(report, isBoot); + DispatchHost host = hostManager.findDispatchHost(NEW_HOSTNAME); + assertEquals(host.getAllocationId(), detail.id); + } + + @Test + @Transactional + @Rollback(true) + public void testHandleHostReportWithExistentAllocation() { + AllocationEntity alloc = adminManager.getAllocationDetail( + "00000000-0000-0000-0000-000000000006"); + assertEquals(alloc.getName(), "spi.general"); + + boolean isBoot = true; + CoreDetail cores = getCoreDetail(200, 200, 0, 0); + HostReport report = HostReport.newBuilder() + .setHost(getNewRenderHost("general")) + .setCoreInfo(cores) + .build(); + + hostReportHandler.handleHostReport(report, isBoot); + DispatchHost host = hostManager.findDispatchHost(NEW_HOSTNAME); + assertEquals(host.getAllocationId(), alloc.id); + } + + @Test + @Transactional + @Rollback(true) + public void testHandleHostReportWithNonExistentTags() { + AllocationEntity alloc = adminManager.getAllocationDetail( + "00000000-0000-0000-0000-000000000002"); + assertEquals(alloc.getName(), "lax.unassigned"); + + boolean isBoot = true; + CoreDetail cores = getCoreDetail(200, 200, 0, 0); + HostReport report = HostReport.newBuilder() + .setHost(getNewRenderHost("nonexistent")) + .setCoreInfo(cores) + .build(); + + hostReportHandler.handleHostReport(report, isBoot); + DispatchHost host = hostManager.findDispatchHost(NEW_HOSTNAME); + assertEquals(host.getAllocationId(), alloc.id); + } } From ab3f211bc0a0aa07b11c1c24a49bfd6b286b8531 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Sat, 10 Apr 2021 08:27:19 -0400 Subject: [PATCH 077/277] Notes from last three TSC meetings. (#956) --- tsc/meetings/2021-02-17.md | 109 +++++++++++++++++++++++++++++++++++++ tsc/meetings/2021-03-03.md | 91 +++++++++++++++++++++++++++++++ tsc/meetings/2021-03-31.md | 41 ++++++++++++++ 3 files changed, 241 insertions(+) create mode 100644 tsc/meetings/2021-02-17.md create mode 100644 tsc/meetings/2021-03-03.md create mode 100644 tsc/meetings/2021-03-31.md diff --git a/tsc/meetings/2021-02-17.md b/tsc/meetings/2021-02-17.md new file mode 100644 index 000000000..ee95b60a1 --- /dev/null +++ b/tsc/meetings/2021-02-17.md @@ -0,0 +1,109 @@ +# OpenCue TSC Meeting Notes 17 Feb 2021 + +Secretary: Brian Cipriano + +TSC Attendees: + +* [ ] Matt Chambers +* [x] Brian Cipriano +* [ ] Greg Denton +* [x] Ben Dines +* [ ] Dave Fellows +* [x] Daniel Neilson +* [ ] Alex Schworer +* [ ] Erik Strauss +* [x] Lars van der Bijl + +Committer Attendees: + +* [ ] Christian Smith +* [x] Diego Tavares +* [x] Idris Miles + +Agenda/Notes: + +* Goals for 2021 + * User survey + * Responses: https://www.surveymonkey.com/results/SM-8DS6VQ9K9/ + * New user UX + * Split demo_data.sql + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/891 + * Progress: https://github.com/AcademySoftwareFoundation/OpenCue/pull/910 + * Maybe just a rename for now? seed_data.sql + * Could add explanation of demo data and what each piece means + * Keep using Docker compose? + * Look at survey responses to see if this is a useful tool still. + * improvements + * move to toplevel? + * Review Daniel's writeup for new action items + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/905 + * Document config options for RQD and Cuebot + * Make RQD gRPC port configurable (done) + * Rename/split demo_data.sql (in progress) + * Document how facilities/tags/allocations work + * This is in glossary, but deserves a dedicated doc page, perhaps a reference on how + an OpenCue deployment might be structured (also addresses point 6 in Daniel's + writeup) + * Could add explanation of demo data and what each piece means + * Clean hardcoded facilities from RQD source (done) + * Clean SPI-specific environment variables from RQD source, or convert them into fully + supported, documented ones + * Why does RQD limit facility names to three chars? + * Fix cuebot:latest tag for sandbox setup + * Published new release which will fix the problem for now. + * Need to address in more detail soon. + * Proposal: publish Docker images for every new patch version (i.e. every new commit to + master) + * Pro: master branch in repo will always match :latest tag in Docker Hub + * Pro: following deployment guide using combo of source checkout + docker images + will work fine + * Con: following deployment guide using combo of release artifacts + docker images + means images will be ahead of release artifacts (because guide assumes :latest tag + for images) + * This should be ok most of the time, as changes tend to be + backwards-compatible (e.g. old schema versions still exist in the system) + * We could change layout of deployment guide, split into two options: + * Deploy from a release (uses release artifacts and docker images + with : tag) + * Deploy from master (uses source code and docker images with :latest tag) + * This would also help simplify the docs, there are too many options for + deploying each component which is confusing for new users. + * podman issue + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/898 + * This appears to be related to some Dockerfile syntax we're using which podman doesn't + support. + * Auto close github issues + * Docs refresh + * API reference + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/559 + * We are able to generate HTML, we just need to publish it somewhere + * No other progress yet. + * pyoutline examples + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/177 + * No progress yet. + * User guide for CueGUI + * Configuration guide for Cuebot + * Configuration guide for RQD + * dedicated doc page, a reference on how an OpenCue deployment might/should be structured, + with examples and pointers to seed data + * UX expert we can contact? + * CII badge updates + * Linting done. Brian working on resolving some additional SonarCloud issues. + * Brian to go through CII checklist and start to fill in more details. + * Drop Oracle support + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/840 + * PR in review: https://github.com/AcademySoftwareFoundation/OpenCue/pull/895 + * GPU support + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/459 + * Starting PR in review: https://github.com/AcademySoftwareFoundation/OpenCue/issues/459 + * GUI to add new shows + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/892 + * Some discussion but coding has not started + * Expand DCC plugins + * Houdini? + * Katana, needs followup + * Logging solution + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/880 + * PR approved, needs to resolve some failing checks. + * CSP Terraform docs + * No progress yet. diff --git a/tsc/meetings/2021-03-03.md b/tsc/meetings/2021-03-03.md new file mode 100644 index 000000000..ad6b4e44e --- /dev/null +++ b/tsc/meetings/2021-03-03.md @@ -0,0 +1,91 @@ +# OpenCue TSC Meeting Notes 3 Mar 2021 + +Secretary: Brian Cipriano + +TSC Attendees: + +* [ ] Matt Chambers +* [x] Brian Cipriano +* [ ] Greg Denton +* [x] Ben Dines +* [ ] Dave Fellows +* [x] Daniel Neilson +* [ ] Alex Schworer +* [ ] Erik Strauss +* [x] Lars van der Bijl + +Committer Attendees: + +* [ ] Christian Smith +* [x] Diego Tavares +* [x] Idris Miles + +Agenda/Notes: + +* Goals for 2021 + * User survey + * ASWF Graduation + * Brian working through CII checklist, should be completed soon. + * Brian also preparing proposal and presentation for next TAC meeting, will be circulated + soon. + * Quality gate failing due to test coverage, this should not block graduation but requires + some ongoing work to resolve ASAP. + * New user UX + * Rename demo_data.sql + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/891 + * Progress: https://github.com/AcademySoftwareFoundation/OpenCue/pull/910 + * No new updates. + * Review Daniel's writeup for new action items + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/905 + * See takeaways from last meeting. + * Brian to file bugs for each of these. + * Fix cuebot:latest tag for sandbox setup + * Published new release which will fix the problem for now. + * Need to address in more detail soon. + * Proposal: publish Docker images for every new patch version (i.e. every new commit to + master) + * See pros/cons from last meeting. + * No progress implementing this yet. + * Docs refresh + * User guide refresh + * Split into "install from latest release" and "install from master" + * Needed to assist transition to publish Docker images on all commits + * No progress. + * API reference + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/559 + * We are able to generate HTML, we just need to publish it somewhere + * No other progress yet. + * pyoutline examples + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/177 + * No progress yet. + * User guide for CueGUI + * No progress. + * Configuration guide for Cuebot + * No progress. + * Configuration guide for RQD + * No progress. + * Drop Oracle support + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/840 + * PR in review: https://github.com/AcademySoftwareFoundation/OpenCue/pull/895 + * Ready for review? Yes + * GPU support + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/459 + * Starting PR in review: https://github.com/AcademySoftwareFoundation/OpenCue/issues/459 + * New PR: https://github.com/AcademySoftwareFoundation/OpenCue/pull/924 + * Ready for review + * GUI to add new shows + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/892 + * Coding done, PR approved and ready to merge. + * Expand DCC plugins + * No progress. + * Logging solution + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/880 + * PR approved and merged. + * Blog post on this topic? + * CSP Terraform docs + * No progress yet. + * Auto-close Github issues + * No progress. +* Other current work updates + * SPI several bug fixes, PRs coming soon, mostly GUI issues + * A few FRs to be filed as issues and discussed soon diff --git a/tsc/meetings/2021-03-31.md b/tsc/meetings/2021-03-31.md new file mode 100644 index 000000000..37829c441 --- /dev/null +++ b/tsc/meetings/2021-03-31.md @@ -0,0 +1,41 @@ +# OpenCue TSC Meeting Notes 31 Mar 2021 + +Secretary: Brian Cipriano + +TSC Attendees: + +* [ ] Matt Chambers +* [x] Brian Cipriano +* [x] Greg Denton +* [x] Ben Dines +* [ ] Dave Fellows +* [x] Daniel Neilson +* [ ] Alex Schworer +* [ ] Erik Strauss +* [ ] Lars van der Bijl + +Committer Attendees: + +* [ ] Christian Smith +* [x] Diego Tavares +* [ ] Idris Miles + +Agenda/Notes: + +* Goals for 2021 + * User survey + * ASWF Graduation + * CII badge complete + * Proposal submitted + * Waiting on TAC for consideration + * New user UX + * Brian planning to start on Python distribution updates, publish to PyPI + * Docs refresh + * API reference + * Let's publish instructions for doing local build for now +* Other current work updates? + * SPI merged with OSS version, CueGUI and other bug fixes incoming +* User survey impressions + * PR/outreach, highlight new features/integrations + * Could make videos e.g. demonstrating CueGUI + * More windows support/docs From bb5e56dfea691ef296684f2968ae1aff530066af Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sun, 11 Apr 2021 11:02:32 -0700 Subject: [PATCH 078/277] Fix AllocationDao type mismatch. (#953) --- .../spcue/dao/postgres/AllocationDaoJdbc.java | 6 +++--- .../test/dao/postgres/AllocationDaoTests.java | 20 +++++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/AllocationDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/AllocationDaoJdbc.java index c57e1d8e8..1ceda9b5c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/AllocationDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/AllocationDaoJdbc.java @@ -109,7 +109,7 @@ public void insertAllocation(FacilityInterface facility, AllocationEntity detail Integer.class, new_alloc_name) > 0) { getJdbcTemplate().update( - "UPDATE alloc SET b_enabled=1 WHERE str_name=?", + "UPDATE alloc SET b_enabled = true WHERE str_name=?", new_alloc_name); } else { @@ -196,8 +196,8 @@ public CallableStatement createCallableStatement(Connection con) throws SQLExcep } public void setDefaultAllocation(AllocationInterface a) { - getJdbcTemplate().update("UPDATE alloc SET b_default = 0 WHERE b_default = 1"); - getJdbcTemplate().update("UPDATE alloc SET b_default = 1 WHERe pk_alloc=?", + getJdbcTemplate().update("UPDATE alloc SET b_default = false WHERE b_default = true"); + getJdbcTemplate().update("UPDATE alloc SET b_default = true WHERE pk_alloc=?", a.getAllocationId()); } diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/AllocationDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/AllocationDaoTests.java index 669ecb981..bbc6914a4 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/AllocationDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/AllocationDaoTests.java @@ -182,6 +182,26 @@ public void testUpdateAllocationBillable() { "SELECT b_billable FROM alloc WHERE pk_alloc=?", Boolean.class, alloc.getId())); } + + @Test + @Transactional + @Rollback(true) + public void testSetDefaultAllocation() { + AllocationEntity newAlloc = new AllocationEntity(); + newAlloc.name = "spi.new_alloc"; + newAlloc.tag = "new_alloc"; + allocDao.insertAllocation( + facilityDao.getFacility("spi"), newAlloc); + + allocDao.setDefaultAllocation(newAlloc); + AllocationEntity defaultAlloc = allocDao.getDefaultAllocationEntity(); + assertEquals(newAlloc.getAllocationId(), defaultAlloc.getAllocationId()); + assertEquals(newAlloc.name, defaultAlloc.name); + assertEquals(newAlloc.tag, defaultAlloc.tag); + assertEquals( + facilityDao.getFacility("spi").getFacilityId(), + defaultAlloc.getFacilityId()); + } } From af75c67a3e00a20098dd63bd5d9a8004ff9db2ad Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sun, 11 Apr 2021 11:03:23 -0700 Subject: [PATCH 079/277] Handle facility.name format in ManageAllocation. (#955) --- .../spcue/servant/ManageAllocation.java | 16 ++- .../test/servant/FakeStreamObserver.java | 37 +++++ .../test/servant/ManageAllocationTests.java | 127 ++++++++++++++++++ 3 files changed, 176 insertions(+), 4 deletions(-) create mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/servant/FakeStreamObserver.java create mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/servant/ManageAllocationTests.java diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageAllocation.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageAllocation.java index 02781bbcd..1857cc653 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageAllocation.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageAllocation.java @@ -122,10 +122,18 @@ public void get(AllocGetRequest request, StreamObserver respon } } + private AllocationEntity findAllocationDetail(String facility, String name) { + // If they pass name in the format ., just remove the facility. + if (CueUtil.verifyAllocationNameFormat(name)) { + name = CueUtil.splitAllocationName(name)[1]; + } + return adminManager.findAllocationDetail(facility, name); + } + @Override public void delete( AllocDeleteRequest request, StreamObserver responseObserver) { - AllocationEntity alloc = adminManager.findAllocationDetail( + AllocationEntity alloc = findAllocationDetail( request.getAllocation().getFacility(), request.getAllocation().getName()); adminManager.deleteAllocation(alloc); responseObserver.onNext(AllocDeleteResponse.newBuilder().build()); @@ -186,7 +194,7 @@ public void reparentHosts( public void setBillable( AllocSetBillableRequest request, StreamObserver responseObserver) { - AllocationEntity alloc = adminManager.findAllocationDetail( + AllocationEntity alloc = findAllocationDetail( request.getAllocation().getFacility(), request.getAllocation().getName()); adminManager.setAllocationBillable(alloc, request.getValue()); responseObserver.onNext(AllocSetBillableResponse.newBuilder().build()); @@ -196,7 +204,7 @@ public void setBillable( @Override public void setName( AllocSetNameRequest request, StreamObserver responseObserver) { - AllocationEntity alloc = adminManager.findAllocationDetail( + AllocationEntity alloc = findAllocationDetail( request.getAllocation().getFacility(), request.getAllocation().getName()); adminManager.setAllocationName(alloc, request.getName()); responseObserver.onNext(AllocSetNameResponse.newBuilder().build()); @@ -206,7 +214,7 @@ public void setName( @Override public void setTag( AllocSetTagRequest request, StreamObserver responseObserver) { - AllocationEntity alloc = adminManager.findAllocationDetail( + AllocationEntity alloc = findAllocationDetail( request.getAllocation().getFacility(), request.getAllocation().getName()); adminManager.setAllocationTag(alloc, request.getTag()); responseObserver.onNext(AllocSetTagResponse.newBuilder().build()); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/servant/FakeStreamObserver.java b/cuebot/src/test/java/com/imageworks/spcue/test/servant/FakeStreamObserver.java new file mode 100644 index 000000000..3ac9bcc93 --- /dev/null +++ b/cuebot/src/test/java/com/imageworks/spcue/test/servant/FakeStreamObserver.java @@ -0,0 +1,37 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.imageworks.spcue.test.servant; + +import io.grpc.stub.StreamObserver; + +public class FakeStreamObserver implements StreamObserver { + + @Override + public void onNext(T value) { + } + + @Override + public void onError(Throwable t) { + } + + @Override + public void onCompleted() { + } +} + diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/servant/ManageAllocationTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/servant/ManageAllocationTests.java new file mode 100644 index 000000000..c63f292c6 --- /dev/null +++ b/cuebot/src/test/java/com/imageworks/spcue/test/servant/ManageAllocationTests.java @@ -0,0 +1,127 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.imageworks.spcue.test.servant; + +import javax.annotation.Resource; + +import io.grpc.stub.StreamObserver; + +import org.junit.Test; +import org.springframework.dao.EmptyResultDataAccessException; +import org.springframework.test.annotation.Rollback; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; +import org.springframework.test.context.support.AnnotationConfigContextLoader; +import org.springframework.transaction.annotation.Transactional; + +import com.imageworks.spcue.config.TestAppConfig; +import com.imageworks.spcue.dao.AllocationDao; +import com.imageworks.spcue.dao.FacilityDao; +import com.imageworks.spcue.grpc.facility.AllocCreateRequest; +import com.imageworks.spcue.grpc.facility.AllocCreateResponse; +import com.imageworks.spcue.grpc.facility.AllocDeleteRequest; +import com.imageworks.spcue.grpc.facility.AllocDeleteResponse; +import com.imageworks.spcue.grpc.facility.Allocation; +import com.imageworks.spcue.grpc.facility.Facility; +import com.imageworks.spcue.servant.ManageAllocation; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + + +@Transactional +@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) +public class ManageAllocationTests extends AbstractTransactionalJUnit4SpringContextTests { + + @Resource + AllocationDao allocationDao; + + @Resource + FacilityDao facilityDao; + + @Resource + ManageAllocation manageAllocation; + + @Test + @Transactional + @Rollback(true) + public void testCreate() { + Facility facility = Facility.newBuilder() + .setName(facilityDao.getFacility("spi").getName()) + .build(); + + // Use . name + AllocCreateRequest request = AllocCreateRequest.newBuilder() + .setName("spi.test_tag") + .setTag("test_tag") + .setFacility(facility) + .build(); + + FakeStreamObserver responseObserver = + new FakeStreamObserver(); + manageAllocation.create(request, responseObserver); + + allocationDao.findAllocationEntity("spi", "test_tag"); + } + + @Test + @Transactional + @Rollback(true) + public void testDelete() { + Facility facility = Facility.newBuilder() + .setName(facilityDao.getFacility("spi").getName()) + .build(); + + // Non . name should work too. + AllocCreateRequest createRequest = AllocCreateRequest.newBuilder() + .setName("test_tag") + .setTag("test_tag") + .setFacility(facility) + .build(); + + FakeStreamObserver createResponseObserver = + new FakeStreamObserver(); + manageAllocation.create(createRequest, createResponseObserver); + + Allocation allocation = Allocation.newBuilder() + .setName("spi.test_tag") + .setTag("test_tag") + .setFacility("spi") + .build(); + + AllocDeleteRequest deleteRequest = AllocDeleteRequest.newBuilder() + .setAllocation(allocation) + .build(); + + FakeStreamObserver deleteResponseObserver = + new FakeStreamObserver(); + + manageAllocation.delete(deleteRequest, deleteResponseObserver); + + try { + allocationDao.findAllocationEntity("spi", "test_tag"); + fail("Expected exception"); + } catch (EmptyResultDataAccessException e) { + assertEquals(e.getMessage(), + "Incorrect result size: expected 1, actual 0"); + } + } +} + + From e26413257031b159af3e54ce1d8125f156b67588 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sun, 11 Apr 2021 11:03:44 -0700 Subject: [PATCH 080/277] PyOutline: Add facility support. (#954) --- pyoutline/outline/loader.py | 2 ++ pyoutline/tests/json/facility.json | 12 ++++++++++++ pyoutline/tests/json_test.py | 6 ++++++ 3 files changed, 20 insertions(+) create mode 100644 pyoutline/tests/json/facility.json diff --git a/pyoutline/outline/loader.py b/pyoutline/outline/loader.py index c5a54f126..265272be6 100644 --- a/pyoutline/outline/loader.py +++ b/pyoutline/outline/loader.py @@ -124,6 +124,8 @@ def decode_layer(layer): if "name" in data: ol.set_name(data["name"]) + if "facility" in data: + ol.set_facility(data["facility"]) if "range" in data: ol.set_frame_range(data["range"]) diff --git a/pyoutline/tests/json/facility.json b/pyoutline/tests/json/facility.json new file mode 100644 index 000000000..0876b6872 --- /dev/null +++ b/pyoutline/tests/json/facility.json @@ -0,0 +1,12 @@ +{ + "name": "shell_command", + "facility": "test_facility", + "range": "1", + "layers": [ + { + "name": "shell_layer", + "module": "outline.modules.shell.Shell", + "command": ["/bin/ls"] + } + ] +} diff --git a/pyoutline/tests/json_test.py b/pyoutline/tests/json_test.py index dc0fe48fa..551beec8a 100644 --- a/pyoutline/tests/json_test.py +++ b/pyoutline/tests/json_test.py @@ -77,6 +77,12 @@ def testJsonFile(self, systemMock): systemMock.assert_has_calls([mock.call(['/bin/ls'], frame=1000)]) self.assertEqual('LAYER_VALUE', os.environ['LAYER_KEY']) + def testFacility(self): + """Test facility from JSON""" + with open(os.path.join(JSON_DIR, 'facility.json')) as fp: + ol = outline.load_json(fp.read()) + self.assertEqual('test_facility', ol.get_facility()) + if __name__ == '__main__': unittest.main() From 9e0b6bf208de0ff1f9fd66ca3451071fa7162e28 Mon Sep 17 00:00:00 2001 From: "Marcelo F. Bortolini" Date: Wed, 14 Apr 2021 19:16:18 +0200 Subject: [PATCH 081/277] Remove compiled python from tar.gz releases. (#957) --- sandbox/install-client-archives.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sandbox/install-client-archives.sh b/sandbox/install-client-archives.sh index 66dfe31f1..be82dd76b 100755 --- a/sandbox/install-client-archives.sh +++ b/sandbox/install-client-archives.sh @@ -38,6 +38,16 @@ for PACKAGE in "${CLIENT_PACKAGES[@]}"; do pip install -r ${REQUIREMENTS_GUI} fi cd ${PACKAGE}-${VERSION}-all + + # remove *.pyc files and __pycache__ folders contained on + # -all.tar.gz. As these files might be generated from + # a different operating system and/or python version than current host has + # `python setup.py install` may raise a `ValueError: bad marshal data` error. + # Removing these files before invoking `setup.py` prevent this error. + # NOTE: Temporary solution until pip distribution is ready. + find . -path '*/__pycache__*' -delete + find . -name '*.pyc' -type f -delete + python setup.py install cd .. done From 19f0b1de4d19223a5e9d655fe39095be6b34a4e5 Mon Sep 17 00:00:00 2001 From: Lars van der Bijl <285658+larsbijl@users.noreply.github.com> Date: Sat, 17 Apr 2021 13:49:18 +0100 Subject: [PATCH 082/277] Simplify docker compose setup. (#933) Place docker-compose at the root of the project. Remove some of the variable user needed to setup with defaults. As we can asume a person is running this as a test and not in production with docker-compose. process to start is now. `docker-compose up` in the root of the project. Co-authored-by: Lars van der Bijl --- sandbox/docker-compose.yml => docker-compose.yml | 8 ++++---- sandbox/docker-compose.monitoring.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) rename sandbox/docker-compose.yml => docker-compose.yml (85%) diff --git a/sandbox/docker-compose.yml b/docker-compose.yml similarity index 85% rename from sandbox/docker-compose.yml rename to docker-compose.yml index 09bdf7eb7..554735581 100644 --- a/sandbox/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: image: postgres environment: - POSTGRES_USER=cuebot - - POSTGRES_PASSWORD=$POSTGRES_PASSWORD + - POSTGRES_PASSWORD=cuebot_password - POSTGRES_DB=cuebot ports: - "5432:5432" @@ -22,7 +22,7 @@ services: - db environment: - PGUSER=cuebot - - PGPASSWORD=$POSTGRES_PASSWORD + - PGPASSWORD=cuebot_password - PGDATABASE=cuebot - PGHOST=db - PGPORT=5432 @@ -39,8 +39,8 @@ services: - flyway restart: always environment: - - CUE_FRAME_LOG_DIR=$CUE_FRAME_LOG_DIR - command: --datasource.cue-data-source.jdbc-url=jdbc:postgresql://db/cuebot --datasource.cue-data-source.username=cuebot --datasource.cue-data-source.password=$POSTGRES_PASSWORD + - CUE_FRAME_LOG_DIR=/tmp/rqd/logs + command: --datasource.cue-data-source.jdbc-url=jdbc:postgresql://db/cuebot --datasource.cue-data-source.username=cuebot --datasource.cue-data-source.password=cuebot_password rqd: image: opencue/rqd diff --git a/sandbox/docker-compose.monitoring.yml b/sandbox/docker-compose.monitoring.yml index 3f6e451f3..2b5ab78ee 100644 --- a/sandbox/docker-compose.monitoring.yml +++ b/sandbox/docker-compose.monitoring.yml @@ -13,7 +13,7 @@ services: environment: - DATA_SOURCE_URI=db:5432/postgres?sslmode=disable - DATA_SOURCE_USER=cuebot - - DATA_SOURCE_PASS=$POSTGRES_PASSWORD + - DATA_SOURCE_PASS=cuebot_password - PG_EXPORTER_AUTO_DISCOVER_DATABASES=true ports: - 9187:9187 From 7cce111306ccfa6c94f12806c8668866f2ee70ad Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Sat, 17 Apr 2021 08:53:46 -0400 Subject: [PATCH 083/277] Minor cleanup to API reference docs. (#959) --- docs/.gitignore | 1 + docs/_static/images/aswf_aqua.png | Bin 0 -> 33393 bytes docs/_templates/footer.html | 4 ++-- docs/conf.py | 4 ++-- 4 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 docs/.gitignore create mode 100644 docs/_static/images/aswf_aqua.png diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 000000000..a485625d4 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +/_build diff --git a/docs/_static/images/aswf_aqua.png b/docs/_static/images/aswf_aqua.png new file mode 100644 index 0000000000000000000000000000000000000000..52bf2be87edec6d4d7ffe14137b98d63cc3d4771 GIT binary patch literal 33393 zcmX_o1yq#Z^ZwG(A|l--QqtWBQj!WQsdSg*(jcI8NH>zwNXLRGjSJEZ(y)XeyTB6v zcm4dn|2@ZJoPBxkotgX0GtbQRgU$<8Vgech5C}x9_FP#H1j1SvMu43T~(nmbv_f>Ud zsT0GrWU>CvbUJ?vGEYt8hMWUbWZj*e*b{)G7Qx8-Afdgx44ih($GG|oQ&Of$uZ+s2x*y~VpAq+d%7FBDKa z^9X%X-X-sgacG&@>-yR6GV;l!J+Z}DfzKC;klJ|577I}^vbgBfCl$1FSQ+E;QiiI`E7Xy zOEsT?N2xaB6S%Vnyc2GCu!|m3WiDvxe;s%m`6R?03y$4oSS|@#jdFbL`;q4|Z+nY% zKU|i8$b)>MRrnl5<;%LC@mFCif@OK@?C#pA>d#6`;MD<#>ATD8r15f_HJJhFkUCa-qMtA$<{`qJty{ zecIl7`5d~aPvux@0y1K_42Bke#B#dPvl?n|?9WxeWx0FmE%ONMVMpiTM&zNYEUWB; zCuJ={nPLj^U2Nt1uvnxj}jZjmNm)j={Y|Xm0A1r*p-9kk=pGW zJF9Cl_gnTZ+QMy#S(aEDdL=7nYc{{cpgu7QNdwP5`&3nkMS9D1fSM!Mg!f$hl!;u5 zktuGcjM=}TsUtMlnX$txBOw*Pkv<7o#)!c)C^1s1``TQ56)QAIPwDmD(iVuzQ)Nx& zl=zzDTJT1A$Dc30*L%&q^kQdW8q9s9a2GRFO>i z)GsW)g}NPs^JzF=t&G*I3*D;rdq0LYjk8nPNZD%amUMCHb@)Td@~i6+VSYxCH1<`8 zZa)%96%0k`bsw>zZfPq+qt6rLXhLZ(uT?K^lagh2a!48Kq>Q{Xw@YZ+u@l)CvasZ| z>$_&DH{(a@#V84q;^HSS&J!`-`1hfT;#Zj!4oF^%;@t#a&X3j}e#=O=?Y^C1uUtEt z73Sw$?0q}*fF&zBQT%D8K;@4TN`1H1Q^K0&tn=OMp_0${Q<>s?vMy(c_v4Glh(388&#J1&J2y1l#zg@Gn_^v?f%4E zy#aobIl`7NW9|3zx$t}PJ()co$3?n~f0uBZ8RPNCsS`|%YdvFbs>Z1+nbvqK-ybUO z?#!|;bG@Vd3m=sr#88J@LtxGFfppuLFMm%w2%&jAE*BZW$&1C#+zB-v>Pc6@YitFu z>hyNL8_E|s>XHtB=Xw9YIQEzyw{4@44z*fm4jZ@)9y!WZntH*9N$@gicd6=-(ty5P zq%s7tJ-TzIF4-iNyLPyLD(>D(^X7%~RJr70HhDfb$G?!FFsgV^H1TCfP}cwH-p;^p ztdK>hi{A1{-EsW!L7p~-(T!J%3fZ+pNaef_GvPSSry;H_#r=ksRi^ngQ=F~OoxAE` zrbv-m2(blgXis6VwpVy3c8?wnRZogQ!e@APUB2Z zV=r|gneeW7qwaL>Axn{Lgh6G59RZxjGVq%x4@SVvWxWvqLZkEve%=9heXxWd*E<7m zNMN8U_BqLW zL#I_EX1UC_IM}ol;8PxCp{D_|)C=h#2Fi=NrhNu~v0|=}0;bR^iHYafav6o8h_G_V zw~(&=PTfCfJl`ZrkyNhFi8@YFz}pf@S#^)A))1U@a&bEpFB6gJEE=*8!9^c=GBv45 z@cmt-bnY-uXe)^jmdkHwdR+_)c04xa0B~p2_;ml$I(R#tMl-npA$s9XymkrrbXWfU zb-ryUUWA+|_Ne9CTfP;Yb7YGPdxfLUuSgL@_#b}IE$;`euGN(1b&$|~*|jlo;dceb z)T?rD#GX=)%0FA(TzMCNWr4)0geILX*O2+%^9zwb;`P$jhJI~64$LFpY8nNPFH&>s zz6JJC6Vo3fU|mtcTH5YRGEpNGzPK2B7)%KkOlLZ7KT}ark!&S>mS$2&;2YfkhkC`Q zT#z7*c}eE za>EhpxG-!{%q4H?Fjo*R7deZUALTu!anqC!u6M-zN+dc61wpr3oi0^PA%mv&PsgIq z1~sR0nBlTN2|9LcAk9C9*br}QJ^4HQZ?Tpvk+w1I*IeOCp^jiTJy&Q_3OswmU2N3h zGXNfb36o)q+Mpm2!RDwM0ZJZxUw(GSHA=WoX_k|r-D+%gxk1*4FZE!fR-UB=@HZ$I4(~SyZpB``g3aYlyK|kNgtH63SA}!mkQR6$zlCQJU0fZErgNfTWAE>c*WNv$kD*3@YS#7hIr z6av9Wm7Fc#_u_{*=9vG(wqA6?R&Gqc``SCNo=ZGufm%w1FnT=_>tXWQ&B7F3SkV>Aa`=sz`cAZ1 z_`?D265?Ye+faZy;q$AR2GGZmGGYE=bBr8zjc7Y7qsoZE+QvsYPL?Vkv>e zgf6r>B^GR+6Y!SG+w8L!m_`c=HxJcuTeO1A^Vgv-S{s_1HXs(i@i|V#bZ1;v!N`du zWHY##E%j>2oBjyc$ln>t6yLnuv1QwFKVXds%w~N|K656;^R>i;X9C&TlAQC4j&x>O zR-CQNwu`&D6mZ$ku|w>YKO7@#!hQMDc$kaoEh-+cK)oH35Oc1}GfzcPwyx!v5|#`- zg27NPWZibxmt)N}b_C@mk>qpkM)9s`{Mt;@uR_@--D2S$DdvQlVFmfs%Zw#(9Epcu zkA&rju>)pXh7reX>sYfj+vcGM9kU+}WLKn4B}BfcbjGv004pqD>-c4M{Z>Ql+OXry zs(synS~6`EG-m;QWtuAnf(o?C@LVp#+>OSLs6+Ve&1zU=Ve|7zEl&fvs@YxI@MOd_Ub8V3`Rr;>90r9X4r2`0cf*g$(fN3+}R{ zgzC+BFWXw366!2`UW$|H3)+LSPw38d`P>M9Ukfpzg$pv6{Ul8ux8SVdgZOeW)(YIi zZR?vBR{v~@w;4V=rJ&E*9L4ec(c4RYl982L`j=yey9eTNz0EdaA>6Gaei<$xWpH-{ z+TUzwfAd>A`yeD5!xr;(&P!H7*EZ5{v3aqSfmq(T>MCbvxq?#C9d2?Uxa7{kkxXyB zIc`^upCgE0d2?8JOP8g`E(s)g@IweOP@3O8rq;WWMhy4LF~_|{eQdEpPGyBto|yHg z77Q}^glvXy`IE~rJT-0N8l-}NF;AJsM*OnUVRJCI@-ytU)rJDLMn+85FZrvO7o-b~ z?fg2Q_&*4sGJ%yo_WlQImziIP*XR#`+<1FtSy)fIgj;!gO5u&-=AK8gEf(W@kK9yA zP;~DxyfR*_jN-V_5z>MpXJaQqGcUXbS|3pbO=UJ8a0@+}R1B-$?wKVD!n+djUPUH! zWA!}HuebC!<#@nXeW=^TPrXSp$-9)(fi7?7(uIVB0#f<>ai9a$#~2 z@q$gv#q+DJ7YuYQ_Dymbeby)5qNCIS)7KV9H+Btir-P%@g7k0&1)+e-;@T_4Md$oS z^LFs^=WM9UFPxRe_E4k+^6uu^Xg8mc@a*pU?=>75yg}?{RO_mEqLBu8v3b{>4`v0 zp&N#}WLA#&ojk-ZnGy3xqD`dJbzt>&$J}$r>bvOSn1`dSsUE^=yYrq$Q z1yrE_xSR}n~qOP!_IT@tE!xrHyoc;*w|d^Ps)eT39WYckky_~W@fDt z_F#m(@-Vl(%j^CNex;uWu|MplcPMS|Jo>fJ1z+*OPV)7}eYvWNU>Pnb_f_+EOREyajeF85)2Vihe=l@p;yzu$+8F1nORzpiI-RdH&DQV}m&x9x}_0P=+8IrVp-LKul zt@@-Qc|O6W=Qv|E$<({F<@7;)B8t4jYUpi9>G%Cd9pRi}rclo=}V4*|Ybp83e?b*@0xXWDpojNuQGEIrjISK{}IlBI-S@!CJ7`BWzIpL&3HuktPh-Df#JkAhrCUu1ZD>BP*JirW zX^K#Vp|I(Y{}XOm*lVscqc_2zz%ipR{J=&_cl- zr1-BE!AqnVX_k9{CDwJ~Z))i^GslSsID#iKlZE+@Cm7bu+(L)!rJjygW0`42q7C88 zTcVy}sy@|qzoAtm1a?}2=X)c|wSn?HdNbCe1+2jv>NORbXX%yt0ZP1^)ho`CY2f`R ze>3)KV@Rfw9{NjlrYx`6CTa0GCRD=_RcfCra|xEC?oi49K@1eq5n1*MrH!i*dVPZw zRr~ET)Hv!YJWyfj2n?62^$>Xefh=inG=KpzzxO-_Uolbu2hj{FW%7I1Q1ZNtxBnV`X4?20C$+|{vtbC#AsBc+i zH+v+V>1PG7dD`2UKqqRor*}MZ#LFPe@qqOi`mV_Sqg0~FKn>c*9)gL5#thygKN&JV z^7y5}?`q?toE!_eT;n8{2+paK)9uZEf^kElF~@;Ian`bkWV$N&onG1#39se?+a&f? zK;8Kckvqx?6ypaX_dp^P%A)bg-mLp~;F?ls=#lJo(soJh>=k>#Gir)T52Z!qtaVm8 zsb0$>*VqVqh@<7K#^N3RxJ>2Qp{P!9?)KKV+r>clNsSsJqQiawGP6foWLVV5vo5?g^QhhFCku9ZFy_CEdi8(S?hk8wBKxYcm6m&L5bnOcNeLD|D!DI~MM z9esT|Ni==Q`t|1l4I0qipi{l}XQ%x6;5Q*1#={r4)qNHp`yty~JYyrqBro38a**Fu z#K?Liu>(?Y{s}{H>zlg{&sC68;y;CJ@vlwENhX?T zx?)`K`3`D@h%s`vQW~3vU_q3HLFn}P0=(!SA{v`a#FQbV;uWP`TTlC4cTJ*!ba-Kg zUbdy*B{}z7#4v|%0odbVOF8C=P27ITsB&Z#zcr#wg$yeR1pS5r9lc-r5CqONQo%E%uSl-&ycX>mklSW_6>uyTGFdQYF|8`n>{#>IptOUn1J8PFNu`f z$qPO?BA_#8>g*LfoD1$dqbDPd0NA*6lB|Hnr@!)7y4#oVk|@uN=H_W?n01Tu=}k|T z=C!OW77iwXl$9a@VoFvfG3Reb>OEaa2#c)82Cg+Szzc zY3}rq`-yqnJ^b*cKKI-usa%%L@xRxF)^C3OInsrSGwI}Bq~B&KTC#;(BkV3`h;H5` zYJWom8`b}jC&4V9ZqQ@Lj@nSkGe3T-!Z!V?YUecWJvnkH-_Q|XT?7>)9JycN*H*JkM7saAzfUJ3zl0lA!Ll^Aq!z8;oxgU|KY zZ~uzl8+tA|Hu4Pb@WXqWb_`sRcUb<-Uz-`*L_xYxqnht2X}YeAF1# znWIj2TjC?n{M29L^o$;sRFT=@0Tzu;5|ht$UIQ2<`e?sU*YE zO@!({Kh>2DCR2nM4yu@>M#1GLnwhpl_yYWOK)5P0rJGb>7V$6|bYTq<4Ip|M)SH2V}gV zRq38rW|rh*lMq3t%JmGy6mAi(p{m4ZkcYml>}LyUvN#a%Ha}al5j_0gaY;ENqpOFL zA;`Lg1z8c_*6%K1{Y0Vp>DJ7BcbIc@#;PW_wbUu&%S~{*?PI@3!;0soiPRX3)yq&% zo$NtU+_oD#Nqf>D=4xuP7WXLu6T*R8W~n@3)An_v`?-dI{a9e=3OFT;X=M!wX7z|! z776()e+a6Ce#yiS(7V$bo_p+w1nAS%SDSMUT$-5%39OTWq)-=BUeF*IYWgV`|6@)# zJpT~KgfnK47Pk$;L0R1XV=ko5f#V?3lIR+{oLx#2;zU3Ok{5hv!mja%A^Vq#6e23s z{e|*I55SskBOZRI&CV61q;dl(giUTAnCE2S^K2nxjXRDN3E$CTxOf(++SsVM#-fo1VyIs3#cq!SwOoq%_PGbXk|LLj zz$lmi>WmM`a&;&v+)9ubOi7tRU~m~)s>jG|VxkGSkgp~lXp_-vXVo-W-aUIv{f`~x z{TGsnmbe+>iBmwtkOq-1Dsj|(3tBzl8ylHUY`nG#S*J!}9Q{bjoa?pvO$!$-XQpwF z2ioMofxtXz+CHf@ArBdp1MJDB|H_2$1&QRd!NQ8Z2R9Fk$X<&h&fcnr+wYB<`Z+$& z5E|tA-z$e%u(51epv=7jm++ZPZQ@W0f?L;;RB>aZj*cEh3t^ zi_AUybn2l4vx4|6Xn4U=KYG9QQG@A)lQJRtK(3&7FIPBC%Rr95`To`RTj1SVLgmKP zp+~MWg@mBsQoqYJRx$uA0r|@jWL{^!dYMihR)#p0sK1T|=)GfC~6XOKa zN@dcic!36(wSTnA)E+B&>tOQnv$V%>eieKWBZkwjP7?$fJZ-HR_e2gfJG1V5@-Fb3 zKKGfQV!L5#b4JX}>U;5GK>!=G(gB1#E)QGu8nS_6wG!BsVok6CLBKc-Ko9`tTD?Ae z^3)`Tb>p8DG&Uy}>VuE!iKE8jZ_r@AgXYP>aF}uVkg{%)Amw&!>Hz5YO;b~KyRRNN z_LLHiYZ2=3bc;OXE~Q`xjO^E&1gmWW(pf$mGMYTROr&XFY54mpfdhv6Wj2r1Q%m;H zLphxrXZWadfqK?u$zU=HB*&6kmVxVh+DzDf0d2CXxIW&a@|cu|Ne3a<((D8Ui+J*ohQ4{?!R*n|llk ze81m)t3Rf6w0XXw!|p3R0$nx5upbh>3)wew3B8GRp5(5)THG-aOKqb6Ei!nN!2Rw% ze^O|;Wk8S1;i)J;hyk~zs~We;vYW*I6=>p7_>znr*iA5!h=JHA5yb(FlSn0mTP9|Q zy|fzg%u4r)TKUy^eP2CI>0jYsuCrQbotPc{0$pC+VjI7(#0D~qagSG|h zSbM4yp#r`Ys;hK1I*z(pm*8d9deCPlUqNMuM;F0RUm9sW)~s^5WCY z@G26vo`a9`7Yz&-=N}MtCTem3>9S~SZE$$9V>W8%A!>k`{|_*7ZCcNXsLZGA}cd}Y& z>ml}6AP?Yu7tX}G0O>ofm(sa$91NAF5o-(1D<<&m2>S{Hbz}K%hY*qJA~S}TyGkoW z77ST`zSO53@g}rkBep)8WBm1;MORaJzp>3#v6A_PvM#zY5uv|A=-;P27V9PtWA z@y`)Isa##MBx9MY{WXE}mdKYAba=@NYp34`LQDz0! z|5%#HBfl09N7Ko)@+XN0T0te|*@d+7!f$~>Nx_B1!#$UAcL?4Sb=aYs_#(qjlOHRO`j8vPlO2B9qXqK3i zdT~rke>@M^v+iGXl8^stCob9?#02b?Ot%jBqP9NoKG2B^K+Zgndu3-9l8jOCMj-or zIW4f79dVWhV9Wo$3j)5I;2>Tq)6`3gzaX|;=Z>K+Hxqv|`kC`zyI`WwWCt6!74Uv^ zwDgtzF7Edyz<3riETtAG0&bf6x#N~`%ryb|$}G;VUW+BGfJH+T(4{b2%OwH`aceoA zsk=L`b#J@gct!}dmWLAgx2;f0Yu`t|+wZ&P(d2(TzB6U*#<5Q+V@!rFjhlHto?HHh z=Fy%#fz?7nu{~Nz20lgdPX`0TupZ@11-%x`D8+f~+6JH|EV#*>++ zbhlCoQ_)HMFQ}p)-v_)>%4K|6KbRa>8%d-rMwcBr?-Z_fmOQ_tOrO}GQ3K|d9?*&) zT3DgGIB}%LF2hG3F!cNCfcMe-E8hAnK5y>=b%s&H=U{%oHbtEKBpPYs^}&waVx{4L zpU@x7Z%oLf*N<{?e#OPPhP?_Au-~FP8HEo_+s8;RtI;YwZ2T32iCFvY0l$ zUeq3HMu(!5rBx8d0NyAwTz~fxSGF0etLCik;7)t+&OU#U&Xhf|H%ZuN;{X2#KpeMi86YoZ0Mg_wkkKxC z3PM zT#($mJ^>p~a<3S6js5`QNU>C}vUq#;STdk+#K+ucQ zb=Yv>6#cXtxCI7sY6CuLN{QuhRP`C<^ex|b_;r$hX39l-C+;@-xgVbK1oQAFfYM*HNf9jUJGJAOWOqo zL)U&r=vnkMc>=jh`^*D>EoOrm$S%Mf+P7!p(LakZ0;=YNleV`gMvUKUsYkWWtcm+< z&kz98*$qIkT&daOFaZFHiypNy0cA)rGypSmhZPdJDPASpZg&Bo6Z#&Z5Zjemy?emf zmZuP-YG%JJ@$)0K=#ypEZK2wH1SI{6o3B>yg!jYcLwMx5^gAk)o~21YFTHNsU}glA zvb?uZVddPwKmAkl9HSJ^dbo&T=SV*ZWIJ1UTms0&fn+r-^j8581QX)VxrQMIvzw(T zY0yRLHoMLzW0UtQs%d9airprT8p1bsh9Jkw*TnRH)39^3m6{c zBrf2NXSFtvFj?X4$Y^V9v`nD;$W5>|9tb!`k?g%CO}4tSXjK^vV#QPCy^>R&rl zyyCAg*I>f@(2>iS9qN6G@TCiZk8^+DY=ot7I)Fn6EfPD#PCE>FNhbx9HUDKN%4?_h zBl@LB1m{?BmYs#vXmu=FQ+*P59SoyCv}%%AgKux+XS~%h-Jm1U2t0@eh5&NpFCj0^_gUN9SIY zwMxrCDk<_ET05md<*!p3AXyLEJv*`MDa(J3AtX>FJ)i!5>Pq{YDEXVToIkMX{Nw)8 z-`L9TnB~uaZ_1*9%RyE z$xO&T5}iblXH!&?yq?5Nbu^F5%mVrmCHe=IurA_v z+%Nh#B!>XItF>4pN{RC}w!9nQ-y!f=C2|*1#RG_0(Y>Lz!1v3jI5AL*bLeXc%34qr zTPrGL*%QgWlA_p|(^5hY)T(mM;o4UjC703*#e_XjG74_Tios;9BzDpUrhTZ+*srGoBnr7%0g1F@VDHg|JrW?jCK3trjGLt& zJ7%S~BXNN{6ILo+k|f*-T7`YBE=3HGVFS-|Rs+)7XSM)|C=yqDRRz9UBklo%XFW(; zLnwCBaCjliYI2V7bWT64lWgo3FUDF^tD)CnI#3I!wkQ5tCKj^zjXA6nH=P!vC?p~fnzLq;}=gHBQ&z=$Q`|FArhX>Z%U%B_H?`P@){(Wdv0mv zwumgr2L`z>%ox>&GnTL}C_0ctLI(3`@Y(OKgHA5i0e2DWm|epQM;!)3yxtm5s;g#m z4DbzEF<{py*nKl+kB_5M;Y(WcecC2&4W9p{-l3F`5xTL=^BzbbPki}k zZTA|i)YgC``1rMV&r6>h%m#~j!bqTd_Kg7r%$c@nKg8g|uo|L_ztE)LCARAs5O^7Vn@{##RBGo+%z z_`eBvnX2W2`e5g)rrP;u10JK8xuMoLMI1s8N1=O7Wze#~YEw(O@kI!FuZ15Bie z@o+0`e*VXxh_p)7{kyhXsx$=JXtU%m%@R|i{h2UQ9zVzY(Dg$3DaH{m>$r0}DCOH* zTV&uP1~Yx?Rp`);S9I0s`=(_dxT;8Jo@jUezD&3&hY|6_J(?&r+%{4pH;*5*SwVyE zM@32)*T%Yi8ot$=dcw9adl-S~O=>vD!jEM3USJHM8@H*+QcmawuOFbYy2eDyA^Oib z_I@qg@O}j$)Bd^#HF>YP-Q<;5S09?)rHRZ>E;?-!dy``Xr1DgZJU&jL-tcWo=6=^_ z$yvC~FpPX#F3gfbN*ji$A8QCI6XQ15gxMG-`CJnS!6c64SxyxZ}3g zJmU0M;1H~)M^O*brFp=S-ec*sk7p(C4Xe@)egN3=?5e7)`&R|_8~0OKUs-ps*fW71%nhsjb8;FcXG zVuam|@0u*3r*pW104h)Hjz+okV5H3;j_cXLY8F!@3j!1DW@cd{HG1sEc^YXvRfj7z z=OWe~`E=k7Z(0L$XM(x@g*?)6T$<1!oVE$sUsZv^VKl)oeflnKrTc zU`VgikqbSLR95ChKPy$z!w$+c|Am5a==v6x`WB`z>0SzVoqJ!eshr~I zDQ8jRW4M`*Eihr=kT^5dsPRcTuqLVZZR4>Vp_@f0iDfsX)CiO8SA(HfFm{bjV!`RZ z0)LtBv}N_TVZlO$IFBi3L{o@&+4KoKzC>DTCuvi}Py@sxltrVPc77Rsxu>1b*VVMB zUE%_J4eCo8>H-e_j=i*K?N!YNf_D1a9J8bh7|+ zXjVpr=|w>2Y1opsl;ozkZiV}0*=dKITtaxB( zY(VO~SM8K*WtQD2+D$)MTdo$Nv;39EL?q_4oBqO#c>I#C9|H04==oj7T^x_;}D zB_=0QAo*u_TCMY=5yY1Ym7w&R>fqq#XCA1-X3=Yv?hxb)RHTs1?!*6vBb08KzF^u` z{w*zSbRuBaFf69YX*30HobC9P$??4ONCp~#KuM+$L;-G~YbPE6bY3uSvSGOmhRYZz zAW0tS5rTrYHOZub;%_`hqk!WM%me^kF4^^)puOKq-GlWd>Q&9szq=TH=5>un4}$V% zulR^hNT{D9A%AO5XxIQ}q4S)39lJ2-~J*~ghXK?(O()@lJdWfnFsuQz{puDBR0 zxOSYa>zguNPa`}T;qr$~P%v%S|8e5gH^6g&R1M*?Frl;; zOj8nDd-#j7BRP~B6$lBfj2L<_9k>==1|gE_^V?V#fpB7|dCIREJ|OgReZLG{Z3`p= zG<(*?w;bH>meXWw&u#8*UkWxRoQEtc-+TXO^mdN<0~bPu4At|lVFIshA`hcV=4{Zd;6P0!=?kK!bLF zV&-asbx&1``rP9Em6HpO_X+Ad5Ub=mRzpBlNKtuKK_9S8va|#g@1Wq`Bj?5bcQ?Vf z;ft1a-;QoB+CsPV)jwr8UphE_5n5~zFXqKd!LBK8oOK4;TUBg59Y7ODbwPDvu4^@z zX8tlsI$Kf81<;4yGZt9vrfm!>(b`$)!5{#>Hw-2}OJZjPU6c3lmp^K}rhh5wWq#h( z@lH znFKBp10_76LEunXR@A7@`LyUS&I8BL;UVnM`Lei_(GXt|Z`1rMzjCgZMF~zLJ_e4G z&bdqcm3x#m+J1RhT)+%#Pv=eW;ZD3+Ryqx83+K~a+IU8+WQwti)qW|4tANIn`-0Pl zv`bBG;No{hpHmuDw4#(%M&t^HFMwOHaXK9!)nf2U5YOj+$`9Q7uP2BOmjescGe;#Ir`951(}*M%{2nO{ccSO>nU zZ;pjCB?pO|$1F3Y{nS*g?G!@|dMREV@vCUtPl^)@(jR0#@c`KwC&BVp0pt(`p+EK7gqHHw)nMH39RF4F?a+A)(G`WPZ@oFeoJ7 znz~60+T82L1b3p50=eKq^IOb`0#tucY|28I=_^adSmifL#rA}Y!SD|x5fB5;Ut4yw|{aY2<%`}kVKrx+1ePzP*qxu z`3}>grL&yPKl1hh9iW4;_~#M@&iz~~<62NfOJt*4!o3l{=T0u!ow`23H-O-|)u<(G zR!od$vHklP#DqG)o<8zSFf1yu+IS`B(37#ISn-+nK%QO`^zp{G5eHy_tP<^b&ACZ7 zaejOns|KXg^i<0`$8zv%y@pNvb0N)#;;oD9z)16Q%9IEP4aFWIFhG=E-avThc+MWu z#B)_DuHWy^}I4Q9?HQ8C%NP4|G;Q@BCc6{KBB2Jm$4ZB>Kc2c#uYopRvunI3zlWwoy6! zpPMssiMwxR4VkVnU#gyv^cYs0cPLIzyv6-hXVmbW)bPI0Ur;G-_>wFT3}n;E2X)Es z$J6FyIAJ>HUur(`NSx<5*$hMq>(>!_6CY{e!P-oV`=jE@WKK@=@uY!1Fma(eAoDfMB>do!2 zlt0O_&8n>$W57iog5-L6jy0-Yk>O$N1ao7vo@EI`JjT?oq*KsPfaBXs*W8D7>cO1lCzin`Y~eK}UR$Q#MYiNFP!lwY`( zNHrIC3to7ZWA#zFOj#Pw9?#l}B2ZI#2h>_~Vmyght@WBubN@rJREmEUtRz;zS-B;cpre2E6+2av)27Qbg z_Hmiw6`OW{%?Rrebvw_?-~J?CgTa+*K}J5~Y3;MRbHs3;tX7s@i^c?B`aPK?O8{$u zt1(1w`uQr9XhJch8dUy`gW9o{m{d0}OK~#}yT|asbvGdG4Brd7BUtl+dyB)|G1h=7 zloG$Tw4c=Hu;~f8?d{Md69vt~MlxG~QxJLhpnH3}k3C?m@*>qMfA=C@Hvs}-kb1VF z{$%i8WvzN&1F9+p>&|>9bMgVLuV<%9%KN5jm~%|{>4_x5^XmB(1mk3c+8p{C*0Od( za{091gS~RA6)}!0xk~VI#-5%}UM+>0x?Boe?qTWmzVfWO$M3wmR@WMvN7tIc4maQm z7<;|3=XzEsBf4cpd`xO~_7k1X-aXkP=K@21Xny{sC%Qf^JK4WrWHQ&gMsi%{(-VRJ*VJ2wMfHAPpmcY4cS?5;4U$SD64Kq$f^-f@H;4#GcXzkaARW>r z1CGEwVPGbZDWKWADRnq9I*#hM6$wg~^F>>6} z9z6xSjM1rLK4{X43qBtwR%>0Q9a_f@GoC=}=pxI(i_S1BpwqAER7Kz?w z(FUm^L&G(>4AhbVmCdnSqb|{{p5HnZgTGBHe)W0KTA>sKaf^Npcux&&v*j5x~yx35E<=-yC>=?n-J4?tIqiy{-4zyHz>0G)Ed;R+=(fT%auRv2dVY0Vfj?Ln24GQ<$390k(1L?XtKwq3$foniNXVW799g7 z3#-LLY2h3XXJ_Rg^NhX+bt?m2W_r69s7w3RZHssAK|%#L#Wq)quu{35?^+-SJ-zFW z-Fmh+SJQ(~^+UWAncq}^`tf%=2agGrd$R$Be1@EQN&o-#OuEHhZK=@dFt)9obUTYH9WALL|(J_JmNpMauLX2xOMyG~!M95uX~}o&QaE*Q(X3@C#ll~MK*!)5-!Iujve4n= z?!p6TjDR6&Kce_PaoShJ+(r7i`g}QHh>5erjnyv-F5M$(S8BN1ilApHkQIpv9tO+Bd(C|G_~R3s{8 zP`^d*J6DHuT-1|@s~q?iC~S#jinv~9`^#Y@c)t$Y>P5R89s@?d*ypvzrXbQ^U6b>x ziS#*XiMDSzqcAvJS1uZw?>cV<^>(%tZKbq<@*EH7Pgy|BJyG&S%2BP@4T^!z+yG$< z>gmg~26j+3vK&z?gSsFbci0K@>O`iKqc<-QJ-dWmat-VEG=wnCvl@HSZ{@98e6~q8 zX-8lLjN^gn2S%XL5S6c2I(0h%kV z<~0C8*zQDv8<@p*NxThD*7XYVx zD_iTw=OY1_nXW(4W*;>rpm5SA&}Ac_`@Dg_58Mg?=E-1D8l#@F&Qp%6w-#?bnkmBcAm^4uK_S9|S zfAvhL2t-HRfSvw^ZRDvA=>-5B*7!#KGttD$ki+E&4)j||;|_18$8usHRJ=EBe(Og< zpSZ{wM!$V?HVF9H{?aFHraZ}m&AG9OS`Q(jzPW~cqKGGfestq5Y*Cs*uIu}M0Y+0T zg?R=T3gBk=8q)mI7fZyh$ZZ47RLQvDJ7pHfF~=+ZbhxsU!`7|-ntz>`6hJ?dfAIO` z&GkiCb%t4PIKVe6IY9{9Y*EVs16UqKA^TnmlKYhLIA&q9{+I~u(o!~ZyIIzH&@Jkj zk!3iO{`lq~6`}S0I&J_1 zaja{V7o8p~&z{WA=j{GYIv)De21Rq7w=cm87!%L4ixMfAw(850Xj*f#)%d|r%01wb zi)owKSr_BLV(=tomb?AQEj!FD7{;<&vyvIYBSm4j&o4f%dDfrWDiKiD7-)5$cAAslOfKoHZa3 zM$UHJLnJsl?~>c?1v4uNbdlHM8}W#7I-~xEfqRvW)8$0&d?8w-Fwn zffB>`+OlzHbG4I@_M~|_=gFnc;d5i|6B_fGQ{OHW-HH($3>Y*S2_$V&b!T@BgGo6T z9#JvPe@zgS;xU+-wYP>FwISpvxqoV13EDdRU-WmdOpRH5PFa)AExraE%=~_&WcqwVF2j}+P!I*a6 zcYJ-T_Cb45UK-LZlBbx;dnR?(V`0YiUdk)%E~9rz4#pMA)mzV)&`8iOeb{2xN6I+` zrIuT(#pSN2P8FGQLZ!kroO@G*d!WGmoD@LSR_&zy$fH?f5gKr8`nVtm#&S?gclzz# zWw7X@5<|SYn#kQ}XYc%`fr&Pcahy&hb4bejKl?EBC$SGW3$>%>xloD&k7!&UF>~@j zx_9uniBK~iDeKJ0CIe2`3|ZA$X_W%XETK<2NF#Aw!zn00=XD*2jy@jW4+*8D}ym(FEB=kd5ilf#b$lh(nPQUhFg6(a-C z4}9gq0FPvMX7m*hXgH~y^40vcum0Jb9Q za}G$p#O?(siCnFtOz2Q!as97g?x3x1&*g%K!as!9D}rlY_-|Qm2|5ltFw;9r^&SPi z5MTBpv$9tIt1z^d&F+W1fK<+qJVX~=wnNe@ux^wFFfx8jB6t`VRdm^RXxwp^C2;HRbgZ25`dOLHOh^ zI_14->P{|P{3}&~o~atYN(P*?ag=TOA1ok6br=0v$=~0+NGx#0u}D0&?Ua8-F|=Td zSOIKxJz0%TjnBgGdH`K75POj?K09r`zYomrThu1xpsfv{EyB~;G~AKEEl0U(PWQN2 z9)eo~ga9_EGoG=Ug%??Kv3#Rby39!WA+`p|l;tl^9ci)Yi#g(C>rtMZ{FV{3Nk8Ur zwPgh<1mQtQa+^_G5i|9SqO3kX%A#tUT{JGGBf)sLWm9Mvj;Bkh-JwVy;ejz5dglEB zvj!g|AOwGnii2MSMjjt_g&{@1{QJs_j7DKLOye0f6iWrbAB&yE-5a+NiA7J!fO4A;3VPvDECv7Pw&e z%jLQ6-3a%u)O?)8-%pnSw8E5mw`cjLE#D2xfI~lpSA>wwYtL$l;9!c3Gojx?$&m5! zs~}RC&Nujp$KFw*0?fQ7C*=rHzH|R&SZCwWlCbmQyR$Jsm0oQoy3eP(t{4TQjve_V zr1RqYpt`F7+BOfP`qH)%=@_^&>tBMOxU&BIP6UjH)|>Qka*=>)C7o7`CGLWFp~;3= z1rJx2_*7{j$yn`r31MNz=i`A#zIVR%CsFi$+jR@ypp3dDxE50Pb*vwWdR00b;iz-Z z;L~eolAHG3FVCTn9zEx;C82hwcOR=?`r+J2{d%I0|SXaiM0i~iz z3Uey9B>;WI|H(2<0yl}E*Q3p3WGnD0rssMH2w-p@=BkGHBr@wYJeP~z0(nb#e^u!45gA{Ra4((x{0$H? zHOt)RA_S47n}gArB*N@;OM^=t%aBnEspdPqY&^AuquE8ypFOGae}t+|T~0+(&;)7%%|)5Rxh;oA763AD4fuKZ1W~k-tGmYaztJ!pBxB`y-4k zuPH{iRLX8AkyTGcvcKFD>#pXH+GzF6v`MQQTv+{5tQQFlMvf~Ui)I+b<`GrJA=t9R zz?gb5M;THKC)q27=FQ&6&Oa)4Hs|i=ameqV)T*Yd$q~0!NDbv>`IDuKJyQ0E<{!T< zJJ;~TIt&J}EYobEvK?ldg9A3a{7XV-?>-3V0ZGRZKLcAJmLJ3bS|Nz{HSg3o6kIUzp3A6Tm)Y60CrY*R ze-!@cXpJJ&wuM>jLY?hG^f@zgUg=dRX;xw%9ullHx6 zso1~o5hea)I^X#N?wl&B1ykGM%^7_$?V=l&MVWsafP|ej3Aq~g*$zfRHlCZGd( z`%hM5jnW`UExl6|C+SM6m?H6i&vFigOuDCZGd1Bjv10==aSo?94F%hgRa;Pnue(&A z{3IhK^~%ekRW{E4%(+_k#@{KN_y0^6@Bzbvwsx!Z?mGbmJoeVN?bhq>SPb>8YU!r$ zofS=4bm~EW;Hn|;sI56|!7>?E*=;bS3ijC}&`+j33sXKT!WKjTUo8<4WqtPlDk&5d z{nObLl_^v)9sLt<%|-y(n2vn1+K2!+wv;0AO6niqUGOa*avQA}_NT-juj@e%c1ZYV zh9XpyCchGRG>fcbJWihh2prYj?56Am9^jDw&PYhAkqm{gp1+jMsx)#9BLjG9F})x% zCn?}mMLHQ=DO|w-5S?@a@#2d#BKW^s)ck-BgS&NQV4ECS>&uu#!7wm0z*0|GdLC%( zR0assw~GItQ|x>|^YH9bJ)Xk=jL1e3S44@|%9p|Aj(kksDVcZz`5Y)*%=&-7{ZbAI z=xlu~1Lz3D%KU3yFwrt|N{E(%2%T;T0OsmO0&O=&K#Z*;Q!9~L@22xVPCA1Fp;~gs zS8F8F+)~e`Ph2r~!v)c0R)fsyra{2G2Z@9eUy}d7t42rSdH^LEK&mdK0pLBogClHU zP7g-^?^*2VUNZIcT7A`)VY0n<{|hn>f_|{RH2ClAf16I7eJJq7<#M%j74(Fx-P*h5 zw7KNq)Jp4^>-F4*|BDFJma}v%)_EI3lBM4!k>=zmj?-M&Ww_}X;AVjPdCkaWwBv9C zWUs)2+?Nt}Blk2q*dv)}|IJp0i=5ee%{|f-B|PhO6lq1u$2+bFO689QGta+X@;3-@ z8Yi+!>MvDpaX~A7I8-n(~ zfIC-&f(d&p*P8s@rr-a<13?RfPo^_Q@b2htLWrz5P#$zKyH!4%Q0?z=|rV zrtmVw7AgZ8KwacjT@FMnN>{3=rjfGvEXTWfZxCWT723Z{P#lg$07L5q(}2#5o7Og< z{%5&<%jbotM0W>f)^xUxbU|f8`wL9N_!|J5M)5CLktx`CRHpB)0zJZZfOHPKtUgM) zF@mLRs-(%;voxPm1DZ8Ajs>@PMEZYD+-Ow}V{?_#|N7TijJDeZ+S1V)Y0T=S`oo`h zUxa6Dcsxq;GP1$q@V_IxY6}8;EGBOCLXFmcH2f3d1c!H6PXqYe&8$nYP2HmZ)Dj>w zVH&D>$o)8ruq6k0E;+)#d=-oNSX3E|vb@ z&6h_m(Y0t4xU_$@6z1@!4`e@ zsol5|ROB#DWd?N^Gz0;?pam8bUg-XxM~;F7(d2)^hOmFP!?4Rey1}nV4pg((8Xpuk zDQG-cnSJE9sgN+L@<5sg*du^2;f1hH8rR#%b2ldSr*8+^(d5tn7YdYbvc@*jh93JV zNhws;wqmd6q7j9Hl;povs#$w>8^m#~_&JSaR#8J0FfH2uzyn(*KpNMxg{mNm_n!^E zT>o=FA`Ws>e|tX_6-;dx3P5eAS?Bg?$(_4OjoaF-*P8~DB^2G)q;W<`!|+nh1qP+g zjRDkvQPK-cl=OrB+z?d$D=9wey1cmWot*%7B<&^9ex3Sd$=<{bydox8iVF9JdV~i! z0;tUImh+8)7B~RVDLmS;#}cw>noL+$=T3RjU~j2UJ<%a#f_2W1QGv!}R@Hxli7aLG z-lrdY#<1Sjf`RbBtRkK$7sSvhyhBa&Y%cBJkB4qdxPbcO@ecbHts}Gv=j}l*l73p9 zqtxDG`tSVX9oI^@K((9eO?va5)6Ecoaf_3$w|$FzAmXQWv>I_qa~6i;1`INiff~oX zJZ~ZilJr$h{NuDDqeXi0y*hs1l#2$=^s~Phu6y2~)h259hjk#z$v4aY?(;#mAnG^5 zyCDC~I{Wd^2QSZZ&durUE4EdWd{A-#U>Rq)eOaDQC+^ut=&iK}Pq*?y zb@lWR#VZZ3@2ZSiA`0GShFI%~;+Rxd(nnpU=s;AMRsB-(3q`yXnQbZYk<=>NQof|g zPPF;%N2Cr$o3*O?7(WW{<9y?+_v0H}{yJm%4%}QRaK+da3_c#-JXK5eN~y7^$mvUfjj(S=&8kh36GV zDCEwYPT=$O4r6K#kT0=lwDW3BiTaNU1%u2qjP1`c+pJ}^y{~NFIe_Q~R zSMy_pp-}iu+Je2nCYqz)GY54T_(^2{L(AqB*%1XSxOzp-&@b3aMNN+XaNA7kEEZ?2#zB)k&6QwYu?RcP zHWj(=#P)ek1N+iC6o#KY+$bM+D*|%W-xj zc_z{Dx9>x0B_aNM7gterj)I3P3P^4|Po7(Vg@626k2-2HjfoF*>r;*W2D4G^JB0R{h*a-VSV0ckwD#7&Y0wzcBcChV$sRy2M&1e zdc*xYxXF}`6wf(qpTM7~==eGt$spRw=F-khV_Gt!^EbhRT12_@8*;=?M#bir53xxC z-d8jH5A`R$0Dz1_2xrL4^)jYH0Vf?PPmoyc>#t2I?H8xNPAnHny-U-Brr5HJ#n z58$scWjcB!zh)jNriQ-i<%cVLg-&nX&18S42YzRg8*AYTCmq|`b<%*vKnGA^*~R|8 z0@2P>`x1%dFn-vg=;dW2?T}%%zatpSq!v9hkn&GV2}^FjN8b0os?jBdP(d{|YUs4{ zubf@$#&gWSi)xlCd!khBc_lEbB}#5JhTOKk;jW3oKuHz%4bw~6b*Y+IWZVwjJ<|B& z=^KHTjG3FbiAvh8{<^~1ef2p((IivAr1fM+?(~yQV5h9VS=IaQzc?dS5$f?8J!gu0 z(iv|2x08bOLRX4p=8F6`)lwtNhTu%?d@FHn@&hHVbXd_JBPLed>s^4X;L3;-B~{zgks=bu_+q0tYEt7L=}nnZexm zYH;QASbQ?GpUv27jODhAFfuN6P^XvTrCw(|;u6G-+>PG=I8@e#E+W|?TJ&mcmHK13 z?iRj*UBg_wy!9(llEkB#4f<=WPv{$cBa{qPIYBN0y$Y02Uexg{fq=c0IFI_eUOmzi z>2EH3?RP;BP{ZTrX~lC4bXInUWWGYF5zDKtJ1KPzeR{O3+AFF6YeClv_g(d&PnSk`Gg46I6Fgq%w=v}L)0HQ>+0 zNa8MjlYBu($2iX%{eY$Vk?mYF6Fiw-#I0_+wCv%*$kAxka}4c=t@BB|1gq^6Z1;F} zHFwaB*<3+~nB|&7Z0+sgIT=2ObJF>2K^=dvO#KP>Slo5k?%kFh)zZ)=-j#LuI>6Ts z{ND0$6bMTr?nDo4f7_sV{~UGueT3I^R;j5hRXX$LcGAI5Bn}LE8z;#T(d`uh1> zPAXi>nODL_6qfuT*n_OmM5C9(=j_8>JYQR84QBpxAJ+uUFLvT|+Kzr&-_Rp@px6`O zaTjJgfjT%!td3QGrT4BolPprwCcSVoCUr&lF?82J)FtElg+YYyB{+sJ5-AP$QrWWx zG5sSRoQybb{0h%sRY7q_DWx&Z&ghCOk@LmiZuQeXsK$|Q^Yh%2MhC7 zwxH4>v~m7UQHKoJn-;yc(OF#>p66Tq<{K$O$m;zgxX=EPkx=#B^9?rRygF5?NLNi& zifY!}y{9jWhdQGzZy*1*6E8VqH&i&?MzP!OoKgunIeGN@Gv|zna9AvVKx8c!{?&R@ zu{y+Ea~@|V;#@K5Eb5}O{fi@k{HS(rbe2_vb82%x3D&E7um%ocPg ztFc`3&bk(61C!BD5}5nge5o2B$}t9iD2Jg{08;&9yin)fGm4;XVXM&)`V|Ie&`iR%ejEcoy z%m_c5ZL*GgYWN0n%cGG~*f=KznO1!R=J(lT648X555Wy(F|4kh2O_+wg{r^G_hu=? zih6vEm%>}>?$M8qOYAG?sDzofOrjlyKKupX&7lHqk?2xCRd`2F&xUW(ZD0D}#QL0G zDD3fslMwCpF02?(zlnkX0iOhTmCmHzwQ$%|a&t2|=sP?PFkfhsQ-@AY*r#`-JxV<* zTlTZ87JJS^cja=i<(E(mvMjkay(3Qt#{lZ%{yL~<3rQO%Q|=Gv8aWjtV9n|m^6Sg> zLQ;BKYYS$7ssRLC4PBm$0vep4VKFeM& zKWWa=Lsll3x;t1Pp(=1!)ZTY)<;B6oYTrEU{;afg?+NYFL7LT!Z))nIEX2xVVCeb^ zHATj;i(f40h=bk#o#5*kK6$Rw&ma~Zrb47(FG-R7mUSlk1BsU^W`c<&X--6IVQ}ReOmep1cazvizz`wES)F^y<4kdsNZCZ8CEn^@Ki)ckR13oQ@_2 zj|1RWyMMboqP*Ql22W=;T1LM~h|181>(!@?Dj{=>8vzv;;@@WG0xSIevDOrH!cDdO z5p=GJJdD-4p31~V$AhO&sZ=HM5_gq}(^{}Q&;8$>qxu9%wU+Y{mvRG>0)$HTO}?*5 znSlKs+vGNv;+KkKmDwe_D5Dghgr>e;4^;VCZu7 z=XF`ohkU_3>Fr~LULO9p`SD0sdC+732}+L~@Wop-(6sw-tmVe_ZFjp;i=+s8v}A); z;(+s-DBzxg#S0*N%AQLc6%l(!+|mPHlhs5Ac@ z%N^=+nwg5%O?+XsYvIg`!iK1QKJyKQL)|k4ssUyXMI_i z=Rv^FQ>x&{lMT{jGqzU)(rNR5MFt^t$>8t6mM*P-MTS6B5#}soToL9AAu4!vAfTo% zT3L{jW>u1}fz}-zC9>h#`2_YCPU=9V5`WuJJ#p*^$G%C9WCA&pAS)|D8s4TaxM>~E z@3wOv!yhE_(Zz$rzTD@M_X?Wu`qkXdA9J1@A9*!#$g67Qow6Dzu-GG#SLiB#mc@(F zcr09$BJ;!l&1uScTAvcTb$e;!ERxuqaYaa|?UVG)lPZ!8MCTIO?iM2hLB9UO(K>xw zKoWA2V>Hh_t*&xv2UkEf;)1{V7?QLeHO>qomWO2PWDF`QhCy`#kyIm24JH)8?`qg; zFp07i5^#(cVx|w`~yO0-QL>$E})miQ1 zb(7KCqQTBH(#fqWob{eo+>!r4JiZzi*B)F5t7!1xZojc$7s>cGE*JE;1oRW+Z6k)y zj*=J8yD<3;ng1sLQP9ym%8hHZLa>Oi(JB}bk9sCnhVZF~M0dMNt|&(zgmhkWiMd`= zXuEEgJl){=&* z3!tc0OIA^wWWdZeadF)1#{Z0V3%3ZL%_;0Km3jmCLUW z%5XhMfv+r4e&?`PwGM+y6__rjQtK)&4_OwDbS?B2eKIz~yRxfQd5Es3`&RQktOGU9 zrlk@QD}h?^r#;vYjLwQaHU}pyk;o2O{k2?^&bw3D6D}bSLFip#nj(QNco3G`F(Ft- zddJ<2MI7}>nK#1)^R9#d{^I>9tW9ID?Kp|;L|PZup0j)Vhr=F&ssM9pmpRk?*r(e} zAQzQl;&6lIBCYwn;Mu*-@F39!-Xz@O3FjjqjO$n|l4{@(0yp(d&FQ-5F5BZ9~s?^?4@)8?3oVazBhAg_YB&imX_azfo{Bzrd8!d@wR@p`T+W`a^bG3r(0^!uM9P~fR~Pc|*PR3Rf{0=fo`RIAa1iZ-Od`%`UqU5tp%nSe@Z~Q8vhAw3F z{M(nFpTfo&+Z=`em;0Gqqo&-Ch!MfLK$&DVa07_lL?pF{Pd&f;c8N(iO1w`fV{r6j zTzmyCCg!AC8(xpG^*!`r_BinwqO<;CH^WCwLO4w9b=wM)*(MBo-%_j$n%|jBY?S=xJuAlvn&UFsd%k5#}LNz zZu9C>pUc43<>8Ygy-6+)pvdcpJk%M!a1BC%8-qA1oxI_jt>Oc$C?!iSuK0mH>3FAT zT=!%0)A}cnOIxku7(3u7ZM!B8_}&O=l=kII<*!wPQ7-!vf3zl}C2`EH$)5Lg9cupIZ?2E(9^h_qP!-g`#0PyfmyklVed>xgvLJhM#@nB~u_7#5OZ znUZhL#9P^^3dfcWya9ZX#1Pw~4a#mu`3KMEFFVjcbw(hmO=bfClay+Ma;wydD_`*; zkS|F3egUFXOu7pVzQ0fCDz7Z#AB2Mo)oS&ie>E^Zd&Wp*Xy<)Wgzd% z`4uDUn`icmJ>EjXF2I5kBCpRWUg_fKVkZO=dxrI^KuZ}g)`HIJ{_;Xby9VQ~Y`15= zwYT@KsJM)v8eXCZJ4L`f9a?x{r`&r1a}IVogV2+GE1q7?pN6F%x~G(`7+0x2#W^5A z2t5M8W(E5UbG4qQGWL)e(BZL5tOCQ(f_t||)oa~E$amO&L*!KaB6mTrF*>B8O)|et zbVqJ`xd0mZv@WB)=yMi4O0LPpL>lx!HUkF@pk0U!h$E~7*CA1UygaP7G_VW_WijEg zV%-}7v%WzX_iml{ii*a^)CXQVg0v;ubH0tgdBz#}$Q5jnt{?Oqlc4S@%KlP@Az1gI7W|trMCuO9SMP zWP1|p+WxrY?E9d6NhD7%FoJLSe8Qzaqm)p1@y^Z^G4$!NtulHY_{CC+?nAkc9H1o? zeF%4jviBQ0$Vu6<6dZ$#BV5Qp9^LE#zW=Q-a(Nbg9@Y|4o_K@VrDXaM{5*!iqlH$m z8-E0zd}7@Qny-~QodMufLmbs$v`fYBp5CEBXF5t>)Mpn+9U<$FZ6b?4dXH-4Jw5RE z%$F0lsWfK}$!Khqaij`FrR=KN8znh0Idbj~p1$rbHLQ@Vh}0U>e9$O9g};Ka@I*jH z>B?g!qcx;*|20Jz_I31`Cy^zjE4mt&sQ-@wgOG4JNt4-S?7Dgo&|Dk!TQ2^a)%{V! ztKb*zQE;pxgH>Fhn?;LzhHADMWsv;-eSq;NJo$GqdmaA6?zCs{j8_^YDL+V}XvE2u z0tY$_^}al>3OPDLUjXY8D0^J2HM9fs8-5>>bdUsCmH}CR_sz}IdL)emv;Dh6B^@$X zqH7Bv(U_^4Rq=rj6^?fjngS-nMp&QLRkNjl|9I`wbI8y}KBTFlSXy#O*;^^A_zOGc zxEu9SAoMUc#3#&D77A=_q#!n)W~rrFYV)Va3(l&covDyD02L|g)zqG<2l+qc_wQqg zejjEJ1xg~k9bh{xLXaYYA}pG}<-|mi?nd-N1kfsVpD!amxVU%z@$q?DpAUI-I&%@{ zvy%DJu2ZyHH+1D&fx%w-d=UG8#tU; z{^gbT{qTH>)I@Mo;7Uz8Fywr@S)6Sbr?&uD*3qI8hx?uh`0C&&rm1emf(73gHJd;= z7tQ_{HH5-?o0v+OOThzTj#6Rs6v}^Yt7|Q^OS^ckdD_g(AiP}JrLXM#JiUs;BRFO_ zh(+K}33c#(a5VMy0$ zCjNEwn>k9xxAJ0ubZjajHR1<|W*(T=J;ze$Rrepp40n&c@V6baIytW4S;b3)@|OSr z{-)wLJ&IhzE9W7xCBZ$p0#Da4B$XgHc!D3U$BXKVn;7GuOKDYgPx5oj0&I^-|Z9*>*Ln$;TRW^rD5pl_xtlyYm z>qN)Y_Su_Jo!tN^)8nJB5h-r$y=RW99A?gErF)PI;DYn%^W}jX1IIZP1d3J{F2!HJ(pCG`<7#v*KzTHRK z>H@Cwe(q;*qmMGP*989+^-BY+>HtG*yB7AG#F-0_FLKHdLJ>c z2MZZsbCH)EMXZo=)p@JNkHz3aD$GKwBPI8F47ZQ6?eiFqYJ+B~8jnw@o)0JXF?<{G z*o9rX%@gHMsTOvZ)b=054D6=U$pVB7HZX)q9?P2eM@f?eMv)2V6KU>Cw~$lDSxsro z2F*6C6eb-c4XAk68xcXAZM$$uHhTgx`9dk5_lh0vEst%#NDhxNoSn5?`Wcw&DK@I( zF7UL>ItPAP?>_2>C+pb55JaxnWAH@V@Ut~u%HR3a?|3JpNZQ26`HHlEX(*hL5VuSQ!t8M1c%Rgz zKp_GE4yDLIs!A?h5g0FVuK^2o@UIU*fx$4N`)Mq_PvVcYHcc{X&o_h%VtF)nAof-; zO@&&;eZ<_8qYi;XBWR`5nZjjj896c22V&A1P<>RLoG?u-nn4b+Yur7lt|McV2hR;A zEES#jNWP|aN{fRURtr04{;>7iAw=bZ_@e z+q+Jw?cEFiur*owC9hc9+ShbjF{9Yei1lARJ>=gdMwq;hV%u$K`^oMq zGaFUh4CyFx*(OtcjS&Cctf+GbOUK{M9g+u8yEKFrv`{s38>8vS#IjA! zRQY$>CI5Al+QuA}X(H|RZ?fcZ9Ugl67oK0T@IrN<61PfH3I zCnq1D8mM!Vznu^#XmE`$E$&8j+pODi){Wl${5usbAJ9?tgKpTj@RcXjCNr&rUC*_mO7lQAmC;ZYM1y8Ug zE}8=4iw}EG-11zjq%BhE3P_ovf383w<1+_=PM4Cj(yeBdug}c*qva}Nr&4*i{b=mF zJ;*KhPOLC3GvIM@$Ce3zbB-}*Reh*>@q~RiLczM6vfgbY-hMuYPVd(nfN0%6JX#34 zG%xA;tV$S3)Szje;(IML1R?;(rZW@qpeJvl614%FOT^O0E~Rdd)qJOJkoo2lPcw}K zT!(njV65fajFj4@lo^YJ?{+7Ya3K@V)s-kgWO9NS`v~(+Naujpwh=-<)?e9w5xh+A z8HQr4AXg+H_Ry2I_zKnR?~;r_)CG0cQ7A?>x24KCy*@}XHdM!1$+1u`vbD^6SjMf7 zUv;;%3Bg5pMMQQG_6?UK*)y3Zn`Hlu=)`9;FH*eF?qql_Xc?s(Uzu72my6mO6(fEM z2Tf+J6T-o&=TegS=MJrT4Mx(%@#7`jYmV1UB$ar+i%^WZOIH0885xF_GqOfp2lS2R z+AH{8)ZJ(4t&PZ8>x91{zAGNL^zn(21R>|$M*^0A(c84HtD`*E!V2bo+o=7qFi_3r zW{4N1Wv+@Vk%v{eTr!|@0d!XHIC*@2lT!G4!mmV=Jn%MVr5>B(n2|?82{+WshW-lX zIv248PJ=4ecCCu%28(F=m_2ZC@HeU;Ienu+*#Rk)L>FxpGZKA^Y}_;ok~2-cSfY7!fa6}Au>kUKz>bGAHV4~iW(!FC%}df#Px^A==YLzh0&V7 zUK(}`^JD0qVoRkCMWo~*kg(-a_uAS?Ze7_CG|Kjm2jE@>kcAFViZYTiyXttZ_@I}J z{btZpN&dPO=2Qx{IJd6mTq^ud*_W=RKiW*k7bRAR?FLWx3A^>p+NJi&6+#8^Q!IhivqX%tG- zA?MEzkj)S%FFt()DA6l-0wksMh7ic=K#BzZ0AVS*{J(#J{Y%OIGrYU;+(`mzye1s* OQB`~is+PC-@c#g5PoS&- literal 0 HcmV?d00001 diff --git a/docs/_templates/footer.html b/docs/_templates/footer.html index 6ce7a2807..674845d7e 100644 --- a/docs/_templates/footer.html +++ b/docs/_templates/footer.html @@ -3,7 +3,7 @@ {% block extrafooter %}
-
+
Documentation Distributed under CC BY 4.0
LF Projects, LLC uses various trademarks. For our trademark, privacy and antitrust policies, code of conduct, @@ -21,4 +21,4 @@
{{ super() }} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/docs/conf.py b/docs/conf.py index ba0826c30..d273b2c57 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -25,9 +25,9 @@ author = u'Academy Software Foundation' # The short X.Y version -version = u'0.3.6' +version = u'built from source' # The full version, including alpha/beta/rc tags -release = u'0.3.6' +release = u'built from source' # -- General configuration --------------------------------------------------- From 481e12c162e88bad70e69897675850c1f97e0291 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sat, 24 Apr 2021 01:54:43 -0700 Subject: [PATCH 084/277] Support layer 'cores' (#960) --- pyoutline/outline/backend/cue.py | 7 +++++- pyoutline/tests/backend/cue_test.py | 36 +++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/pyoutline/outline/backend/cue.py b/pyoutline/outline/backend/cue.py index 1d91eb70a..6cd03105d 100644 --- a/pyoutline/outline/backend/cue.py +++ b/pyoutline/outline/backend/cue.py @@ -308,7 +308,12 @@ def _serialize(launcher, use_pycuerun): sub_element(spec_layer, "chunk", str(layer.get_chunk_size())) # opencue specific options - if layer.get_arg("threads"): + # Keeping 'threads' for backward compatibility + if layer.get_arg("cores"): + if layer.get_arg("threads"): + logger.warning("%s has both cores and threads. Use cores.", layer.get_name()) + sub_element(spec_layer, "cores", "%0.1f" % (layer.get_arg("cores"))) + elif layer.get_arg("threads"): sub_element(spec_layer, "cores", "%0.1f" % (layer.get_arg("threads"))) if layer.is_arg_set("threadable"): diff --git a/pyoutline/tests/backend/cue_test.py b/pyoutline/tests/backend/cue_test.py index 029a1ec3f..89c7bcac7 100644 --- a/pyoutline/tests/backend/cue_test.py +++ b/pyoutline/tests/backend/cue_test.py @@ -92,6 +92,42 @@ def testSerializeShellOutline(self): self.assertEqual(0, len(list(outlineXml.find('depends')))) +class CoresTest(unittest.TestCase): + def setUp(self): + # Ensure to reset current + outline.Outline.current = None + + def create(self): + ol = outline.Outline() + layer = outline.Layer("test") + ol.add_layer(layer) + return ol, layer + + def assertCores(self, ol, v): + launcher = outline.cuerun.OutlineLauncher(ol, user=TEST_USER) + outlineXml = ET.fromstring(outline.backend.cue.serialize(launcher)) + job = outlineXml.find('job') + layer = job.find('layers').find('layer') + self.assertEqual(v, layer.find('cores').text) + + def testCores(self): + ol, layer = self.create() + layer.set_arg("cores", 42) + self.assertCores(ol, "42.0") + + def testThreads(self): + ol, layer = self.create() + layer.set_arg("threads", 4) + self.assertCores(ol, "4.0") + + def testCoresAndThreads(self): + ol, layer = self.create() + layer.set_arg("cores", 8) + layer.set_arg("threads", 4) + # cores overrides threads + self.assertCores(ol, "8.0") + + class BuildCommandTest(unittest.TestCase): def setUp(self): path = os.path.join(SCRIPTS_DIR, 'shell.outline') From 49bdf56289660a592bcce74405b3629fa8c7db13 Mon Sep 17 00:00:00 2001 From: Lars van der Bijl <285658+larsbijl@users.noreply.github.com> Date: Sat, 24 Apr 2021 10:05:38 +0100 Subject: [PATCH 085/277] Remove Oracle DB support. (#895) * Remove Oracle DB support. * remove trackit. Co-authored-by: Lars van der Bijl --- cuebot/build.gradle | 12 +- cuebot/oracle/xe/.gitignore | 1 - cuebot/oracle/xe/README.md | 43 - cuebot/oracle/xe/apply_schema.py | 40 - cuebot/oracle/xe/apply_schema.sh | 14 - cuebot/oracle/xe/build_oracle_base_image.sh | 42 - cuebot/oracle/xe/run_db_container.sh | 61 - cuebot/oracle/xe/setup_db.sh | 16 - .../imageworks/spcue/config/AppConfig.java | 12 - .../spcue/config/DatabaseEngine.java | 7 +- .../spcue/config/OracleDatabaseCondition.java | 19 - .../dao/criteria/FrameSearchFactory.java | 9 +- .../spcue/dao/criteria/HostSearchFactory.java | 10 +- .../spcue/dao/criteria/JobSearchFactory.java | 9 +- .../spcue/dao/criteria/ProcSearchFactory.java | 9 +- .../spcue/dao/criteria/oracle/Criteria.java | 304 -- .../dao/criteria/oracle/FrameSearch.java | 217 -- .../spcue/dao/criteria/oracle/HostSearch.java | 56 - .../spcue/dao/criteria/oracle/JobSearch.java | 63 - .../spcue/dao/criteria/oracle/ProcSearch.java | 127 - .../spcue/dao/oracle/ActionDaoJdbc.java | 164 - .../spcue/dao/oracle/AllocationDaoJdbc.java | 205 -- .../spcue/dao/oracle/BookingDaoJdbc.java | 363 --- .../spcue/dao/oracle/CommentDaoJdbc.java | 134 - .../spcue/dao/oracle/DeedDaoJdbc.java | 119 - .../spcue/dao/oracle/DepartmentDaoJdbc.java | 84 - .../spcue/dao/oracle/DependDaoJdbc.java | 751 ----- .../spcue/dao/oracle/DispatchQuery.java | 1293 -------- .../spcue/dao/oracle/DispatcherDaoJdbc.java | 412 --- .../spcue/dao/oracle/FacilityDaoJdbc.java | 88 - .../spcue/dao/oracle/FilterDaoJdbc.java | 199 -- .../spcue/dao/oracle/FrameDaoJdbc.java | 1104 ------- .../spcue/dao/oracle/GroupDaoJdbc.java | 430 --- .../spcue/dao/oracle/HistoricalDaoJdbc.java | 52 - .../spcue/dao/oracle/HostDaoJdbc.java | 711 ----- .../spcue/dao/oracle/JobDaoJdbc.java | 966 ------ .../spcue/dao/oracle/LayerDaoJdbc.java | 847 ----- .../spcue/dao/oracle/LimitDaoJdbc.java | 110 - .../spcue/dao/oracle/MaintenanceDaoJdbc.java | 90 - .../spcue/dao/oracle/MatcherDaoJdbc.java | 109 - .../dao/oracle/NestedWhiteboardDaoJdbc.java | 485 --- .../spcue/dao/oracle/OwnerDaoJdbc.java | 127 - .../spcue/dao/oracle/PointDaoJdbc.java | 222 -- .../spcue/dao/oracle/ProcDaoJdbc.java | 904 ------ .../spcue/dao/oracle/RedirectDaoJdbc.java | 119 - .../spcue/dao/oracle/ServiceDaoJdbc.java | 256 -- .../spcue/dao/oracle/ShowDaoJdbc.java | 221 -- .../spcue/dao/oracle/SubscriptionDaoJdbc.java | 229 -- .../spcue/dao/oracle/TaskDaoJdbc.java | 259 -- .../spcue/dao/oracle/TrackitDaoJdbc.java | 108 - .../spcue/dao/oracle/WhiteboardDaoJdbc.java | 2195 ------------- .../service/DepartmentManagerService.java | 66 - .../resources/conf/ddl/oracle/demo_data.sql | 83 - .../conf/ddl/oracle/migrations/.keep | 0 .../oracle/migrations/V2__Add_limit_table.sql | 19 - .../main/resources/conf/ddl/oracle/schema.sql | 2737 ----------------- .../spring/applicationContext-dao-oracle.xml | 147 - .../spring/applicationContext-service.xml | 1 - .../spring/applicationContext-trackit.xml | 4 - cuebot/src/main/resources/log4j.properties | 1 - cuebot/src/main/resources/opencue.properties | 14 - .../spcue/config/TestAppConfig.java | 15 - .../spcue/test/AssumingOracleEngine.java | 58 - .../spcue/test/AssumingTrackitEnabled.java | 50 - .../spcue/test/TestDatabaseSetup.java | 217 -- .../spcue/test/dao/oracle/ActionDaoTests.java | 208 -- .../test/dao/oracle/AllocationDaoTests.java | 185 -- .../test/dao/oracle/BookingDaoTests.java | 456 --- .../test/dao/oracle/CommentDaoTests.java | 238 -- .../spcue/test/dao/oracle/DeedDaoTests.java | 169 - .../test/dao/oracle/DepartmentDaoTests.java | 107 - .../spcue/test/dao/oracle/DependDaoTests.java | 455 --- .../test/dao/oracle/DispatcherDaoTests.java | 522 ---- .../dao/oracle/FacilityInterfaceDaoTests.java | 78 - .../spcue/test/dao/oracle/FilterDaoTests.java | 302 -- .../spcue/test/dao/oracle/FrameDaoTests.java | 659 ---- .../spcue/test/dao/oracle/GroupDaoTests.java | 393 --- .../test/dao/oracle/HistoricalDaoTests.java | 82 - .../spcue/test/dao/oracle/HostDaoTests.java | 571 ---- .../spcue/test/dao/oracle/JobDaoTests.java | 681 ---- .../spcue/test/dao/oracle/LayerDaoTests.java | 742 ----- .../spcue/test/dao/oracle/LimitDaoTests.java | 137 - .../test/dao/oracle/MaintenanceDaoTests.java | 76 - .../test/dao/oracle/MatcherDaoTests.java | 140 - .../dao/oracle/NestedWhiteboardDaoTests.java | 68 - .../spcue/test/dao/oracle/OwnerDaoTests.java | 131 - .../spcue/test/dao/oracle/PointDaoTests.java | 171 - .../spcue/test/dao/oracle/ProcDaoTests.java | 846 ----- .../test/dao/oracle/ServiceDaoTests.java | 210 -- .../spcue/test/dao/oracle/ShowDaoTests.java | 241 -- .../test/dao/oracle/SubscriptionDaoTests.java | 262 -- .../spcue/test/dao/oracle/TaskDaoTests.java | 288 -- .../test/dao/oracle/TrackitDaoTests.java | 60 - .../test/dao/oracle/WhiteboardDaoTests.java | 1279 -------- .../test/service/DepartmentManagerTests.java | 25 - .../resources/conf/ddl/oracle/test_data.sql | 118 - .../spring/applicationContext-assumptions.xml | 6 - .../applicationContext-oracle-datasource.xml | 59 - cuebot/src/test/resources/log4j.properties | 1 - cuebot/src/test/resources/opencue.properties | 10 - cuegui/cuegui/MenuActions.py | 1 - 101 files changed, 6 insertions(+), 27810 deletions(-) delete mode 100644 cuebot/oracle/xe/.gitignore delete mode 100644 cuebot/oracle/xe/README.md delete mode 100644 cuebot/oracle/xe/apply_schema.py delete mode 100755 cuebot/oracle/xe/apply_schema.sh delete mode 100644 cuebot/oracle/xe/build_oracle_base_image.sh delete mode 100755 cuebot/oracle/xe/run_db_container.sh delete mode 100755 cuebot/oracle/xe/setup_db.sh delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/config/OracleDatabaseCondition.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/criteria/oracle/Criteria.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/criteria/oracle/FrameSearch.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/criteria/oracle/HostSearch.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/criteria/oracle/JobSearch.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/criteria/oracle/ProcSearch.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/ActionDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/AllocationDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/BookingDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/CommentDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DeedDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DepartmentDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DependDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DispatchQuery.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DispatcherDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/FacilityDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/FilterDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/FrameDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/GroupDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/HistoricalDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/HostDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/JobDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/LayerDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/LimitDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/MaintenanceDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/MatcherDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/NestedWhiteboardDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/OwnerDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/PointDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/ProcDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/RedirectDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/ServiceDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/ShowDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/SubscriptionDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/TaskDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/TrackitDaoJdbc.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/oracle/WhiteboardDaoJdbc.java delete mode 100644 cuebot/src/main/resources/conf/ddl/oracle/demo_data.sql delete mode 100644 cuebot/src/main/resources/conf/ddl/oracle/migrations/.keep delete mode 100644 cuebot/src/main/resources/conf/ddl/oracle/migrations/V2__Add_limit_table.sql delete mode 100644 cuebot/src/main/resources/conf/ddl/oracle/schema.sql delete mode 100644 cuebot/src/main/resources/conf/spring/applicationContext-dao-oracle.xml delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/AssumingOracleEngine.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/AssumingTrackitEnabled.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/TestDatabaseSetup.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/ActionDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/AllocationDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/BookingDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/CommentDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/DeedDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/DepartmentDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/DependDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/DispatcherDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/FacilityInterfaceDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/FilterDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/FrameDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/GroupDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/HistoricalDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/HostDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/JobDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/LayerDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/LimitDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/MaintenanceDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/MatcherDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/NestedWhiteboardDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/OwnerDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/PointDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/ProcDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/ServiceDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/ShowDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/SubscriptionDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/TaskDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/TrackitDaoTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/WhiteboardDaoTests.java delete mode 100644 cuebot/src/test/resources/conf/ddl/oracle/test_data.sql delete mode 100644 cuebot/src/test/resources/conf/spring/applicationContext-oracle-datasource.xml diff --git a/cuebot/build.gradle b/cuebot/build.gradle index 2af5f04d4..d4f00e193 100644 --- a/cuebot/build.gradle +++ b/cuebot/build.gradle @@ -130,12 +130,7 @@ jacocoTestReport { fileTree(dir: it, exclude: [ // Exclude proto files' generated Java code. - 'com/imageworks/spcue/grpc/**', - // Exclude Oracle classes; our standard test environment isn't currently - // able to run Oracle unit tests, so coverage of those files will always - // read as 0% even though tests do exist. - 'com/imageworks/spcue/dao/oracle/**', - 'com/imageworks/spcue/dao/criteria/oracle/**',]) + 'com/imageworks/spcue/grpc/**']) })) } } @@ -146,11 +141,6 @@ sonarqube { property "sonar.organization", "academysoftwarefoundation" property "sonar.projectKey", "AcademySoftwareFoundation_OpenCue_Cuebot" property "sonar.projectName", "OpenCue Cuebot" - // SonarCloud will pick up the JaCoCo report automatically, but has its own options - // for excluding files. We don't need to exclude generated code here as it isn't - // checked into the repository, so SonarCloud doesn't even know it exists. - property "sonar.coverage.exclusions", "src/main/java/com/imageworks/spcue/dao/oracle/**," + - "src/main/java/com/imageworks/spcue/dao/criteria/oracle/**" // NOTE: sonar.login must be provided manually, like: // ./gradlew sonarqube -Dsonar.login= diff --git a/cuebot/oracle/xe/.gitignore b/cuebot/oracle/xe/.gitignore deleted file mode 100644 index 110868150..000000000 --- a/cuebot/oracle/xe/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/docker_setup \ No newline at end of file diff --git a/cuebot/oracle/xe/README.md b/cuebot/oracle/xe/README.md deleted file mode 100644 index 80cae882b..000000000 --- a/cuebot/oracle/xe/README.md +++ /dev/null @@ -1,43 +0,0 @@ - -NOTE: This is a stop gap only intended to be used until the postgres swap is ready. - - -### Initial Setup -To install and run Oracle XE, you'll need to clone the `docker-images` repo from Oracle. You'll also -need to download the Oracle XE rpm from Oracle and copy it into the `docker_setup` folder. -The following commands detail the steps including cleanup. Please note that the `docker_setup` folder is in the -`.gitignore`. - -``` -mkdir docker_setup -cd docker_setup -git clone https://github.com/oracle/docker-images.git -cp docker-images/OracleDatabase/SingleInstance/dockerfiles/11.2.0.2/* ./ -cp ../oracle-xe-11.2.0-1.0.x86_64.rpm.zip ./ -rm -rf docker-images -``` - -### To Build -Oracle requires more shared mem than what Docker is provided by default, make sure to include the "--shm-size" flag. -`docker build --shm-size=1G -t oracle-xe Dockerfile.xe .` - -### To Run -Set the password to whatever is the password for the application. -`docker run --shm-size=1G -p 1521:1521 -e ORACLE_PWD= oracle-xe` - -### To Connect -Oracle SID: `XE` -Port: `1521` -Use `sys as sysdba` user with provided password -Failing to set a password in the docker run command will generate a random password - -### Script to build -The above steps can all be accomplished by running the `./run_db_container.sh` script. -It requires two arguments, the system password for the database and a password to create for the cue user. - -### To Run: -`export PROJECT_ID='YOUR GCP PROJECT ID'` -`./run_db_container.sh [--build-prod]` - -### Populating the Schema -Using the `--build-prod` will apply the db schema from `src/main/resources/conf/ddl/oracle/schema.sql`. diff --git a/cuebot/oracle/xe/apply_schema.py b/cuebot/oracle/xe/apply_schema.py deleted file mode 100644 index a6e2cbe20..000000000 --- a/cuebot/oracle/xe/apply_schema.py +++ /dev/null @@ -1,40 +0,0 @@ - -import argparse -import cx_Oracle - - -SPLITTER_KEY = '-- SPLIT HERE!' - - -def get_statements(sql_file): - with open(sql_file) as file_handle: - statement = '' - for line in file_handle: - if line.startswith(SPLITTER_KEY): - yield statement - statement = '' - else: - statement += line - - -def main(user, pwd, sql_file, sql_data_file=None): - print "CONNECTING: {} {} {}".format(user, pwd, sql_file) - connection = cx_Oracle.connect(user, pwd) - cursor = connection.cursor() - for statement in get_statements(sql_file): - cursor.execute(statement) - if sql_data_file: - print 'APPLYING DATA FILE: {}'.format(sql_data_file) - for statement in get_statements(sql_data_file): - cursor.execute(statement) - cursor.close() - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('-u', '--user', help='db user', default='cue') - parser.add_argument('-p', '--pwd', help='db password to connect to cue user') - parser.add_argument('-s', '--sql', help='path to SQL schema file') - parser.add_argument('-d', '--sql-data', help='path to SQL file with inital data to populate') - args = parser.parse_args() - main(args.user, args.pwd, args.sql, args.sql_data) diff --git a/cuebot/oracle/xe/apply_schema.sh b/cuebot/oracle/xe/apply_schema.sh deleted file mode 100755 index 1a088480a..000000000 --- a/cuebot/oracle/xe/apply_schema.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -echo "Applying Database schema..." - -# Install pip -curl "https://bootstrap.pypa.io/get-pip.py" -o "/tmp/get-pip.py" -python /tmp/get-pip.py -rm /tmp/get-pip.py - -# install cx_Oracle -pip install cx_Oracle - -su oracle -c "LD_LIBRARY_PATH=/u01/app/oracle/product/11.2.0/xe/lib/ bash -c \"python /tmp/apply_schema.py -p $1 -u $2 -s $3 -d $4\"" -echo "Finished applying schema." diff --git a/cuebot/oracle/xe/build_oracle_base_image.sh b/cuebot/oracle/xe/build_oracle_base_image.sh deleted file mode 100644 index 507c0659d..000000000 --- a/cuebot/oracle/xe/build_oracle_base_image.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/sh - -set -e - -timestamp=$(date +%Y%m%d%H%M%S) - - -INSTANCE_NAME="oracle-build-$timestamp" -ZONE="us-central1-c" -SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -IMAGE_NAME="oracle-base-$timestamp" - -echo "Starting instance $INSTANCE_NAME..." - -gcloud --project $PROJECT_ID compute instances create $INSTANCE_NAME \ - --machine-type=n1-standard-2 --network=default --zone=$ZONE \ - --image-project=eip-images --image-family=centos-7-drawfork - -INSTANCE_IP=$(gcloud --project $PROJECT_ID compute instances describe $INSTANCE_NAME \ - --zone $ZONE --format="value(networkInterfaces[0].accessConfigs[0].natIP)") - -echo "Waiting for instance to become available..." - -sleep 60 - -ssh $SSH_OPTS $INSTANCE_IP sudo yum install -y yum-utils device-mapper-persistent-data lvm2 -ssh $SSH_OPTS $INSTANCE_IP sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo -ssh $SSH_OPTS $INSTANCE_IP sudo yum install -y docker-ce - -script_path=$(dirname $0)/setupDB.sh -script_dest="/etc/opencue/" -ssh $SSH_OPTS $INSTANCE_IP sudo mkdir $script_dest -ssh $SSH_OPTS $INSTANCE_IP sudo chmod 777 $script_dest -scp $script_path $INSTANCE_IP:$script_dest - -gcloud --project $PROJECT_ID --quiet compute instances delete $INSTANCE_NAME \ - --zone=$ZONE --keep-disks=boot -gcloud --project $PROJECT_ID compute images create $IMAGE_NAME \ - --family=cue-oracle-base --source-disk=$INSTANCE_NAME --source-disk-zone=$ZONE - -echo "Image $IMAGE_NAME created." - diff --git a/cuebot/oracle/xe/run_db_container.sh b/cuebot/oracle/xe/run_db_container.sh deleted file mode 100755 index 4d82a2e83..000000000 --- a/cuebot/oracle/xe/run_db_container.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/sh - -set -e - -cd $(dirname $0) - -DEP_BUCKET="GCS BUCKET NAME" # Where the oracle rpm was copied. - -XE_DIRECTORY=`pwd` -CUEBOT_ROOT_DIRECTORY=$(dirname $(dirname $XE_DIRECTORY)) -ORACLE_RPM="oracle-xe-11.2.0-1.0.x86_64.rpm.zip" -ORACLE_DOCKER_REPO="https://github.com/oracle/docker-images.git" -DOCKER_NAME="oracle-xe" -ORACLE_SQL_FILE='/tmp/oracle_ddl/schema.sql' -ORACLE_SQL_DATA_FILE='/tmp/oracle_ddl/seed_data.sql' -CUE_DB_USER='CUE' - - -if (( $# < 2 )); then - echo "Please pass a password as the first argument!" - echo "Usage:" - echo " ./run_db_container.sh sys_db_password cue_db_password [--build-prod]" - exit 1 -fi - -if [ ! -d "./docker_setup" ]; then - mkdir docker_setup - cd docker_setup - gsutil -m cp "${DEP_BUCKET}/${ORACLE_RPM}" ./ - git clone "${ORACLE_DOCKER_REPO}" - cp docker-images/OracleDatabase/SingleInstance/dockerfiles/11.2.0.2/* ./ - rm -rf docker-images -else - cd docker_setup -fi - -echo "Attempting to stop any running docker images" -docker stop "${DOCKER_NAME}" || : -docker rm "${DOCKER_NAME}" || : -echo "Building new docker container" -docker build --shm-size=4g -t "${DOCKER_NAME}" -f Dockerfile.xe . -echo "Running new docker container" -docker run -itd --shm-size=1g --name "${DOCKER_NAME}" -p 1521:1521 -p 8080:8080 -e ORACLE_PWD=$1 "${DOCKER_NAME}" - -echo "Waiting for DB to come up..." -sleep 90 - -echo "Configuring DB..." -docker cp ../setup_db.sh oracle-xe:/tmp/setup_db.sh -docker exec oracle-xe /bin/bash -c "/tmp/setup_db.sh $CUE_DB_USER $2" - - -if [ "$3" = "--build-prod" ]; then - echo "Applying Schema..." - docker exec oracle-xe /bin/bash -c "mkdir $(dirname $ORACLE_SQL_FILE)" - docker cp ${CUEBOT_ROOT_DIRECTORY}/src/main/resources/conf/ddl/oracle/schema.sql oracle-xe:$ORACLE_SQL_FILE - docker cp ${CUEBOT_ROOT_DIRECTORY}/src/test/resources/conf/ddl/oracle/seed_data.sql oracle-xe:$ORACLE_SQL_DATA_FILE - docker cp ${CUEBOT_ROOT_DIRECTORY}/oracle/xe/apply_schema.sh oracle-xe:/tmp/ - docker cp ${CUEBOT_ROOT_DIRECTORY}/oracle/xe/apply_schema.py oracle-xe:/tmp/ - docker exec oracle-xe /bin/bash -c "/tmp/apply_schema.sh $2 $CUE_DB_USER $ORACLE_SQL_FILE $ORACLE_SQL_DATA_FILE" -fi diff --git a/cuebot/oracle/xe/setup_db.sh b/cuebot/oracle/xe/setup_db.sh deleted file mode 100755 index 7b6803c9d..000000000 --- a/cuebot/oracle/xe/setup_db.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -echo "Setting up Database settings..." -echo "CREATING USER: $1 - $2" -results=`su -p oracle -c "sqlplus system/$1 as sysdba" << EOF - alter system set processes=300 scope=spfile; - alter system reset sessions scope=spfile sid='*'; - alter system set local_listener='(DESCRIPTION=(ADDRESS=(PROTOCOL=IPC)(KEY=EXTPROC_FOR_XE)))' SCOPE=BOTH; - CREATE USER $1 IDENTIFIED BY $2; - GRANT CONNECT, RESOURCE, DBA TO $1; - shutdown immediate; - startup; - exit; -EOF` -echo $results -echo "Finished configuring database." diff --git a/cuebot/src/main/java/com/imageworks/spcue/config/AppConfig.java b/cuebot/src/main/java/com/imageworks/spcue/config/AppConfig.java index 6aea9b48f..4edac2826 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/config/AppConfig.java +++ b/cuebot/src/main/java/com/imageworks/spcue/config/AppConfig.java @@ -40,17 +40,11 @@ "classpath:conf/spring/applicationContext-grpcServer.xml", "classpath:conf/spring/applicationContext-service.xml", "classpath:conf/spring/applicationContext-jms.xml", - "classpath:conf/spring/applicationContext-trackit.xml", "classpath:conf/spring/applicationContext-criteria.xml"}) @EnableConfigurationProperties @PropertySource({"classpath:opencue.properties"}) public class AppConfig { - @Configuration - @Conditional(OracleDatabaseCondition.class) - @ImportResource({"classpath:conf/spring/applicationContext-dao-oracle.xml"}) - static class OracleEngineConfig {} - @Configuration @Conditional(PostgresDatabaseCondition.class) @ImportResource({"classpath:conf/spring/applicationContext-dao-postgres.xml"}) @@ -63,12 +57,6 @@ public DataSource cueDataSource() { return DataSourceBuilder.create().build(); } - @Bean - @ConfigurationProperties(prefix="datasource.trackit-data-source") - public DataSource trackitDataSource() { - return DataSourceBuilder.create().build(); - } - @Bean public ServletRegistrationBean jobLaunchServlet() { ServletRegistrationBean b = new ServletRegistrationBean<>(); diff --git a/cuebot/src/main/java/com/imageworks/spcue/config/DatabaseEngine.java b/cuebot/src/main/java/com/imageworks/spcue/config/DatabaseEngine.java index 71a059305..0ae36a296 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/config/DatabaseEngine.java +++ b/cuebot/src/main/java/com/imageworks/spcue/config/DatabaseEngine.java @@ -1,14 +1,9 @@ package com.imageworks.spcue.config; public enum DatabaseEngine { - ORACLE, POSTGRES; public static DatabaseEngine fromEnv() { - String envValue = System.getenv("CUEBOT_DB_ENGINE"); - if (envValue == null) { - return POSTGRES; - } - return DatabaseEngine.valueOf(envValue.toUpperCase()); + return POSTGRES; } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/config/OracleDatabaseCondition.java b/cuebot/src/main/java/com/imageworks/spcue/config/OracleDatabaseCondition.java deleted file mode 100644 index 7739ec885..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/config/OracleDatabaseCondition.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.imageworks.spcue.config; - -import org.springframework.context.annotation.Condition; -import org.springframework.context.annotation.ConditionContext; -import org.springframework.core.type.AnnotatedTypeMetadata; - -public class OracleDatabaseCondition implements Condition { - - @Override - public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { - String dbEngine = System.getenv("CUEBOT_DB_ENGINE"); - if (dbEngine == null) { - return false; - } - DatabaseEngine selectedDatabaseEngine = DatabaseEngine.valueOf(dbEngine.toUpperCase()); - return selectedDatabaseEngine.equals(DatabaseEngine.ORACLE); - } - -} diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/FrameSearchFactory.java b/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/FrameSearchFactory.java index b2a30b7b0..7a957bdd9 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/FrameSearchFactory.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/FrameSearchFactory.java @@ -30,14 +30,7 @@ public class FrameSearchFactory { private DatabaseEngine dbEngine; public FrameSearchInterface create() { - if (dbEngine.equals(DatabaseEngine.POSTGRES)) { - return new FrameSearch(); - } else if (dbEngine.equals(DatabaseEngine.ORACLE)) { - return new com.imageworks.spcue.dao.criteria.oracle.FrameSearch(); - } else { - throw new RuntimeException( - "current database engine is not supported by FrameSearchFactory"); - } + return new FrameSearch(); } public FrameSearchInterface create(List frameIds) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/HostSearchFactory.java b/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/HostSearchFactory.java index a1f7fe6f5..d11093810 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/HostSearchFactory.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/HostSearchFactory.java @@ -27,15 +27,7 @@ public class HostSearchFactory { private DatabaseEngine dbEngine; public HostSearchInterface create(HostSearchCriteria criteria) { - if (dbEngine.equals(DatabaseEngine.POSTGRES)) { - return new HostSearch(criteria); - } else if (dbEngine.equals(DatabaseEngine.ORACLE)) { - return new com.imageworks.spcue.dao.criteria.oracle.HostSearch(criteria); - } else { - throw new RuntimeException( - "current database engine is not supported by HostSearchFactory"); - } - + return new HostSearch(criteria); } public HostSearchInterface create(AllocationEntity allocEntity) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/JobSearchFactory.java b/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/JobSearchFactory.java index df3b7635c..a9468a098 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/JobSearchFactory.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/JobSearchFactory.java @@ -26,14 +26,7 @@ public class JobSearchFactory { private DatabaseEngine dbEngine; public JobSearchInterface create() { - if (dbEngine.equals(DatabaseEngine.POSTGRES)) { - return new JobSearch(); - } else if (dbEngine.equals(DatabaseEngine.ORACLE)) { - return new com.imageworks.spcue.dao.criteria.oracle.JobSearch(); - } else { - throw new RuntimeException( - "current database engine is not supported by JobSearchFactory"); - } + return new JobSearch(); } public JobSearchInterface create(JobSearchCriteria criteria) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/ProcSearchFactory.java b/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/ProcSearchFactory.java index 58ace1be5..d754dce13 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/ProcSearchFactory.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/ProcSearchFactory.java @@ -25,14 +25,7 @@ public class ProcSearchFactory { private DatabaseEngine dbEngine; public ProcSearchInterface create() { - if (dbEngine.equals(DatabaseEngine.POSTGRES)) { - return new ProcSearch(); - } else if (dbEngine.equals(DatabaseEngine.ORACLE)) { - return new com.imageworks.spcue.dao.criteria.oracle.ProcSearch(); - } else { - throw new RuntimeException( - "current database engine is not supported by ProcSearchFactory"); - } + return new ProcSearch(); } public ProcSearchInterface create(ProcSearchCriteria criteria) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/oracle/Criteria.java b/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/oracle/Criteria.java deleted file mode 100644 index 393848454..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/oracle/Criteria.java +++ /dev/null @@ -1,304 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.imageworks.spcue.dao.criteria.oracle; - -import java.security.Timestamp; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import com.google.common.collect.ImmutableList; - -import com.imageworks.spcue.dao.criteria.CriteriaInterface; -import com.imageworks.spcue.dao.criteria.Phrase; -import com.imageworks.spcue.dao.criteria.Sort; -import com.imageworks.spcue.grpc.criterion.EqualsFloatSearchCriterion; -import com.imageworks.spcue.grpc.criterion.EqualsIntegerSearchCriterion; -import com.imageworks.spcue.grpc.criterion.GreaterThanFloatSearchCriterion; -import com.imageworks.spcue.grpc.criterion.GreaterThanIntegerSearchCriterion; -import com.imageworks.spcue.grpc.criterion.InRangeFloatSearchCriterion; -import com.imageworks.spcue.grpc.criterion.InRangeIntegerSearchCriterion; -import com.imageworks.spcue.grpc.criterion.LessThanFloatSearchCriterion; -import com.imageworks.spcue.grpc.criterion.LessThanIntegerSearchCriterion; - -public abstract class Criteria implements CriteriaInterface { - - List chunks = new ArrayList(12); - List values = new ArrayList(32); - - boolean built = false; - private int firstResult = 1; - private int maxResults = 0; - private ArrayList order = new ArrayList(); - - abstract void buildWhereClause(); - - public String toString() { return this.getWhereClause(); } - - public void setFirstResult(int firstResult) { - this.firstResult = Math.max(firstResult, 1); - } - - public void setMaxResults(int maxResults) { - this.maxResults = maxResults; - } - - public void addSort(Sort sort) { - this.order.add(sort); - } - - public List getValues() { - return values; - } - - public Object[] getValuesArray() { - return values.toArray(); - } - - public String getWhereClause() { - build(); - return generateWhereClause(); - } - - public String getFilteredQuery(String query) { - build(); - return queryWithPaging(query); - } - - private void build() { - if (!built) { - buildWhereClause(); - } - built = true; - } - - private String generateWhereClause() { - return chunks.stream() - .map(StringBuilder::toString) - .collect(Collectors.joining(" AND ")); - } - - private String queryWithPaging(String query) { - if (firstResult > 1 || maxResults > 0) { - if (order.size() == 0) { - query = query.replaceFirst("SELECT ", "SELECT ROWNUM AS RN,"); - } else { - query = query.replaceFirst("SELECT ", "SELECT row_number() OVER (" + getOrder() + ") AS RN, "); - } - } - - StringBuilder sb = new StringBuilder(4096); - if (maxResults > 0 || firstResult > 1) { - sb.append("SELECT * FROM ( "); - } - - sb.append(query); - sb.append(" "); - if (chunks.size() > 0) { - sb.append("AND "); - sb.append( - chunks.stream() - .map(StringBuilder::toString) - .collect(Collectors.joining(" AND "))); - } - - if (firstResult > 1 || maxResults > 0) { - sb.append(") WHERE "); - } - - if (firstResult > 1) { - sb.append (" RN >= ? "); - values.add(firstResult); - } - - if (maxResults > 0) { - if (firstResult > 1) { - sb.append(" AND "); - } - sb.append(" RN < ? "); - values.add(firstResult + maxResults); - } - - return sb.toString(); - } - - private String getOrder() { - if (order.size() < 1) { - return ""; - } - return " ORDER BY " + order.stream() - .map(sort -> sort.getColumn() + " " + sort.getDirection().toString()) - .collect(Collectors.joining(", ")); - } - - void addPhrase(String col, Collection s) { - if (s == null || s.size() == 0) { return; } - - StringBuilder sb = new StringBuilder(1024); - sb.append("("); - for (String w: s) { - sb.append(col); - sb.append("=?"); - sb.append(" OR "); - values.add(w); - } - sb.delete(sb.length()-4, sb.length()); - sb.append(")"); - chunks.add(sb); - } - - void addPhrases(Collection phrases, String inclusion) { - if (phrases.size() == 0) { return; } - StringBuilder sb = new StringBuilder(1024); - sb.append("("); - for (Phrase p: phrases) { - sb.append(p.getColumn()); - sb.append(p.getComparison()); - sb.append("?"); - sb.append(" "); - sb.append(inclusion); - sb.append(" "); - values.add(p.getValue()); - } - sb.delete(sb.length()-4, sb.length()); - sb.append(")"); - chunks.add(sb); - } - - void addPhrase(String col, String v) { - if (v == null) { return; } - addPhrase(col, ImmutableList.of(v)); - } - - void addRegexPhrase(String col, Set s) { - if (s == null) { return; } - if (s.size() == 0) { return; } - StringBuilder sb = new StringBuilder(1024); - sb.append("("); - for (String w: s) { - sb.append(String.format("REGEXP_LIKE(%s,?)", col)); - sb.append(" OR "); - values.add(w); - } - sb.delete(sb.length()-4, sb.length()); - sb.append(")"); - chunks.add(sb); - } - - void addLikePhrase(String col, Set s) { - if (s == null) { return; } - if (s.size() == 0) { return; } - StringBuilder sb = new StringBuilder(1024); - sb.append("("); - for (String w: s) { - sb.append(col); - sb.append(" LIKE ?"); - sb.append(" OR "); - values.add("%" + w + "%"); - } - sb.delete(sb.length()-4, sb.length()); - sb.append(")"); - chunks.add(sb); - } - - void addGreaterThanTimestamp(String col, Timestamp timestamp) { - if (timestamp == null) { return; } - StringBuilder sb = new StringBuilder(128); - sb.append("("); - sb.append(col); - sb.append(" > ?"); - sb.append(") "); - values.add(timestamp); - chunks.add(sb); - } - - void addLessThanTimestamp(String col, Timestamp timestamp) { - if (timestamp == null) { return; } - StringBuilder sb = new StringBuilder(128); - sb.append("("); - sb.append(col); - sb.append(" < ?"); - sb.append(") "); - values.add(timestamp); - chunks.add(sb); - } - - void addRangePhrase(String col, EqualsIntegerSearchCriterion criterion) { - StringBuilder sb = new StringBuilder(128); - sb.append(" " + col + " = ?"); - chunks.add(sb); - values.add(criterion.getValue()); - } - - void addRangePhrase(String col, LessThanIntegerSearchCriterion criterion) { - StringBuilder sb = new StringBuilder(128); - sb.append(" " + col + "<=? "); - chunks.add(sb); - values.add(criterion.getValue()); - } - - void addRangePhrase(String col, GreaterThanIntegerSearchCriterion criterion) { - StringBuilder sb = new StringBuilder(128); - sb.append(" " + col + " >= ? "); - chunks.add(sb); - values.add(criterion.getValue()); - } - - void addRangePhrase(String col, InRangeIntegerSearchCriterion criterion) { - StringBuilder sb = new StringBuilder(128); - sb.append(" " + col + " >= ? "); - chunks.add(sb); - values.add(criterion.getMin()); - values.add(criterion.getMax()); - } - - void addRangePhrase(String col, EqualsFloatSearchCriterion criterion) { - StringBuilder sb = new StringBuilder(128); - sb.append(" " + col + " = ?"); - chunks.add(sb); - values.add(criterion.getValue()); - } - - void addRangePhrase(String col, LessThanFloatSearchCriterion criterion) { - StringBuilder sb = new StringBuilder(128); - sb.append(" " + col + " <= ? "); - chunks.add(sb); - values.add(criterion.getValue()); - } - - void addRangePhrase(String col, GreaterThanFloatSearchCriterion criterion) { - StringBuilder sb = new StringBuilder(128); - sb.append(" " + col + " >= ? "); - chunks.add(sb); - values.add(criterion.getValue()); - } - - void addRangePhrase(String col, InRangeFloatSearchCriterion criterion) { - StringBuilder sb = new StringBuilder(128); - sb.append(" " + col + " >= ? "); - chunks.add(sb); - values.add(criterion.getMin()); - values.add(criterion.getMax()); - } - - boolean isValid(String v) { - return v != null && !v.isEmpty(); - } -} diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/oracle/FrameSearch.java b/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/oracle/FrameSearch.java deleted file mode 100644 index f04e1de71..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/oracle/FrameSearch.java +++ /dev/null @@ -1,217 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.imageworks.spcue.dao.criteria.oracle; - -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import com.google.common.collect.ImmutableList; -import org.apache.log4j.Logger; - -import com.imageworks.spcue.FrameInterface; -import com.imageworks.spcue.JobInterface; -import com.imageworks.spcue.LayerInterface; -import com.imageworks.spcue.dao.criteria.FrameSearchInterface; -import com.imageworks.spcue.grpc.job.FrameSearchCriteria; -import com.imageworks.spcue.grpc.job.FrameState; -import com.imageworks.spcue.util.CueUtil; -import com.imageworks.spcue.util.FrameSet; - -public class FrameSearch extends Criteria implements FrameSearchInterface { - private static final int MAX_RESULTS = 1000; - private static final Logger logger = Logger.getLogger(FrameSearch.class); - private static final Pattern PATTERN_SINGLE_FRAME = Pattern.compile("^([0-9]+)$"); - private static final Pattern PATTERN_RANGE = Pattern.compile("^([0-9]+)\\-([0-9]+)$"); - private static final Pattern PATTERN_FLOAT_RANGE = Pattern.compile("^([0-9\\.]+)\\-([0-9\\.]+)$"); - private static final int RANGE_MAX_SIZE = 1000; - - private FrameSearchCriteria criteria; - private String sortedQuery; - - public FrameSearch() { - criteria = FrameSearchInterface.criteriaFactory(); - } - - @Override - public FrameSearchCriteria getCriteria() { - return criteria; - } - - @Override - public void setCriteria(FrameSearchCriteria criteria) { - this.criteria = criteria; - } - - @Override - public String getSortedQuery(String query) { - if (built) { - return sortedQuery; - } - - int limit = criteria.getLimit(); - int page = criteria.getPage(); - - if (limit <= 0 || limit >= MAX_RESULTS) { - criteria = criteria.toBuilder().setLimit(MAX_RESULTS).build(); - } - if (page <= 0) { - page = 1; - } - - StringBuilder sb = new StringBuilder(query.length() + 256); - sb.append("SELECT * FROM ("); - sb.append(getFilteredQuery(query)); - sb.append(" ) WHERE row_number > ?"); - sb.append(" AND row_number <= ?"); - values.add((page - 1) * limit); - values.add(page * limit); - sortedQuery = sb.toString(); - return sortedQuery; - } - - @Override - public void filterByFrameIds(List frameIds) { - criteria = criteria.toBuilder().addAllIds(frameIds).build(); - } - - @Override - public void filterByJob(JobInterface job) { - addPhrase("job.pk_job", job.getJobId()); - } - - @Override - public void filterByFrame(FrameInterface frame) { - filterByFrameIds(ImmutableList.of(frame.getFrameId())); - } - - @Override - public void filterByLayer(LayerInterface layer) { - addPhrase("layer.pk_layer", layer.getLayerId()); - } - - @Override - public void filterByLayers(List layers) { - addPhrase( - "layer.pk_layer", - layers.stream().map(LayerInterface::getLayerId).collect(Collectors.toList())); - } - - @Override - public void filterByFrameStates(List frameStates) { - addPhrase( - "frame.str_state", - frameStates.stream().map(FrameState::toString).collect(Collectors.toSet())); - } - - @Override - public void filterByFrameSet(String frameSet) { - StringBuilder sb = new StringBuilder(8096); - Matcher matchRange = PATTERN_RANGE.matcher(frameSet); - Matcher matchSingle = PATTERN_SINGLE_FRAME.matcher(frameSet); - - if (matchSingle.matches()) { - sb.append("frame.int_number=?"); - values.add(Integer.valueOf(matchSingle.group(1))); - } else if (matchRange.matches()) { - sb.append(" ( frame.int_number >= ? AND "); - sb.append(" frame.int_number <= ? )"); - values.add(Integer.valueOf(matchRange.group(1))); - values.add(Integer.valueOf(matchRange.group(2))); - } else { - FrameSet set = new FrameSet(frameSet); - int num_frames = set.size(); - if (num_frames <= RANGE_MAX_SIZE) { - sb.append("("); - for (int i=0; i= ? AND frame.int_mem_max_used <= ?) "); - } - else { - values.add(CueUtil.GB * Float.valueOf(range)); - sb.append(" frame.int_mem_max_used >= ? "); - } - } catch (RuntimeException e) { - logger.warn("Failed to convert float range: " + range + "," + e); - } - chunks.add(sb); - } - - @Override - public void filterByDurationRange(String range) { - StringBuilder sb = new StringBuilder(128); - Matcher matchRange = PATTERN_FLOAT_RANGE.matcher(range); - try { - if (matchRange.matches()) { - values.add((int) (3600 * Float.valueOf(matchRange.group(1)))); - values.add((int) (3600 * Float.valueOf(matchRange.group(2)))); - sb.append(" (frame.str_state != 'Waiting' "); - sb.append(" AND find_duration(frame.ts_started, frame.ts_stopped) "); - sb.append(" BETWEEN ? AND ? )"); - } - else { - values.add((int) (3600 * Float.valueOf(range))); - sb.append(" (frame.str_state != 'Waiting' AND "); - sb.append("find_duration(frame.ts_started, frame.ts_stopped) >= ?) "); - } - } catch (RuntimeException e) { - logger.warn("Failed to convert float range: " + range + "," + e); - // a cast failed, ignore for now. - } - chunks.add(sb); - } - - @Override - public void filterByChangeDate(int changeDate) { - StringBuilder sb = new StringBuilder(); - sb.append("frame.ts_updated > ?"); - chunks.add(sb); - values.add(new java.sql.Timestamp( changeDate * 1000L)); - } - - @Override - void buildWhereClause() { - addPhrase("frame.pk_frame", criteria.getIdsList()); - - addPhrase("frame.str_name", criteria.getFramesList()); - addPhrase("layer.str_name", criteria.getLayersList()); - filterByFrameStates(criteria.getStates().getFrameStatesList()); - if (isValid(criteria.getFrameRange())) { filterByFrameSet(criteria.getFrameRange()); } - if (isValid(criteria.getMemoryRange())) { filterByMemoryRange(criteria.getMemoryRange()); } - if (isValid(criteria.getDurationRange())) { filterByDurationRange(criteria.getDurationRange()); } - if (criteria.getChangeDate() > 0) { filterByChangeDate(criteria.getChangeDate()); } - } -} diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/oracle/HostSearch.java b/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/oracle/HostSearch.java deleted file mode 100644 index b7720537d..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/oracle/HostSearch.java +++ /dev/null @@ -1,56 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.imageworks.spcue.dao.criteria.oracle; - -import java.util.HashSet; -import java.util.Set; - -import com.imageworks.spcue.AllocationInterface; -import com.imageworks.spcue.dao.criteria.HostSearchInterface; -import com.imageworks.spcue.grpc.host.HardwareState; -import com.imageworks.spcue.grpc.host.HostSearchCriteria; - -public class HostSearch extends Criteria implements HostSearchInterface { - private HostSearchCriteria criteria; - - public HostSearch(HostSearchCriteria criteria) { - this.criteria = criteria; - } - - public HostSearchCriteria getCriteria() { - return this.criteria; - } - - public void filterByAlloc(AllocationInterface alloc) { - addPhrase("host.pk_alloc", alloc.getAllocationId()); - } - - @Override - public void buildWhereClause() { - addPhrase("host.pk_host", criteria.getIdsList()); - addPhrase("host.str_name", criteria.getHostsList()); - addPhrase("host.str_name", new HashSet<>(criteria.getSubstrList())); - addRegexPhrase("host.str_name", new HashSet<>(criteria.getRegexList())); - addPhrase("alloc.str_name", criteria.getAllocsList()); - Set items = new HashSet<>(criteria.getStates().getStateCount()); - for (HardwareState w: criteria.getStates().getStateList()) { - items.add(w.toString()); - } - addPhrase("host_stat.str_state", items); - } -} diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/oracle/JobSearch.java b/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/oracle/JobSearch.java deleted file mode 100644 index c22631274..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/oracle/JobSearch.java +++ /dev/null @@ -1,63 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.imageworks.spcue.dao.criteria.oracle; - -import java.util.HashSet; - -import com.imageworks.spcue.ShowInterface; -import com.imageworks.spcue.dao.criteria.JobSearchInterface; -import com.imageworks.spcue.grpc.job.JobSearchCriteria; - -public final class JobSearch extends Criteria implements JobSearchInterface { - private JobSearchCriteria criteria; - - public JobSearch() { - criteria = JobSearchInterface.criteriaFactory(); - } - - @Override - public JobSearchCriteria getCriteria() { - return criteria; - } - - @Override - public void setCriteria(JobSearchCriteria criteria) { - this.criteria = criteria; - } - - @Override - public void filterByShow(ShowInterface show) { - addPhrase("job.pk_show", show.getShowId()); - } - - @Override - void buildWhereClause() { - addPhrase("job.pk_job", criteria.getIdsList()); - addPhrase("job.str_name", criteria.getJobsList()); - addLikePhrase("job.str_name", new HashSet<>(criteria.getSubstrList())); - addRegexPhrase("job.str_name", new HashSet<>(criteria.getRegexList())); - addPhrase("job.str_shot", criteria.getShotsList()); - addPhrase("show.str_name", criteria.getShowsList()); - addPhrase("job.str_user", criteria.getUsersList()); - if (criteria.getIncludeFinished()) { - chunks.add(new StringBuilder(" ROWNUM < 200 ")); - } else { - addPhrase("job.str_state", "Pending"); - } - } -} diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/oracle/ProcSearch.java b/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/oracle/ProcSearch.java deleted file mode 100644 index 078b90cb6..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/oracle/ProcSearch.java +++ /dev/null @@ -1,127 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.imageworks.spcue.dao.criteria.oracle; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import com.imageworks.spcue.GroupInterface; -import com.imageworks.spcue.HostInterface; -import com.imageworks.spcue.JobInterface; -import com.imageworks.spcue.dao.criteria.Phrase; -import com.imageworks.spcue.dao.criteria.ProcSearchInterface; -import com.imageworks.spcue.dao.criteria.Sort; -import com.imageworks.spcue.grpc.criterion.GreaterThanIntegerSearchCriterion; -import com.imageworks.spcue.grpc.criterion.InRangeIntegerSearchCriterion; -import com.imageworks.spcue.grpc.criterion.LessThanIntegerSearchCriterion; -import com.imageworks.spcue.grpc.host.ProcSearchCriteria; - -public class ProcSearch extends Criteria implements ProcSearchInterface { - - private ProcSearchCriteria criteria; - private Set notJobs = new HashSet<>(); - private Set notGroups = new HashSet<>(); - - public ProcSearch() { - criteria = ProcSearchInterface.criteriaFactory(); - } - - public ProcSearchCriteria getCriteria() { - return criteria; - } - - public void setCriteria(ProcSearchCriteria criteria) { - this.criteria = criteria; - } - - public void notJobs(List jobs) { - for (JobInterface job: jobs) { - notJobs.add(new Phrase("proc.pk_job","!=", job.getJobId())); - } - } - - public void notGroups(List groups) { - for (GroupInterface group: groups) { - notGroups.add(new Phrase("folder.pk_folder","!=", group.getGroupId())); - } - } - - public void filterByDurationRange(LessThanIntegerSearchCriterion criterion) { - StringBuilder sb = new StringBuilder(128); - sb.append(" (find_duration(proc.ts_dispatched, null) <= ?) "); - chunks.add(sb); - values.add(criterion.getValue()); - } - - public void filterByDurationRange(GreaterThanIntegerSearchCriterion criterion) { - StringBuilder sb = new StringBuilder(128); - sb.append(" (find_duration(proc.ts_dispatched, null) >= ?) "); - chunks.add(sb); - values.add(criterion.getValue()); - } - - public void filterByDurationRange(InRangeIntegerSearchCriterion criterion) { - StringBuilder sb = new StringBuilder(128); - sb.append(" (find_duration(proc.ts_dispatched, null) BETWEEN ? AND ? )"); - chunks.add(sb); - values.add(criterion.getMin()); - values.add(criterion.getMax()); - } - - public void filterByHost(HostInterface host) { - addPhrase("host.pk_host", host.getHostId()); - } - - public void sortByHostName() { - addSort(Sort.asc("host.str_name")); - } - - public void sortByDispatchedTime() { - addSort(Sort.asc("proc.ts_dispatched")); - } - - public void sortByBookedTime() { - addSort(Sort.asc("proc.ts_booked")); - } - - @Override - void buildWhereClause() { - addPhrases(notJobs, "AND"); - addPhrases(notGroups, "AND"); - - addPhrase("host.str_name", criteria.getHostsList()); - addPhrase("job.str_name", criteria.getJobsList()); - addPhrase("layer.str_name", criteria.getLayersList()); - addPhrase("show.str_name", criteria.getShowsList()); - addPhrase("alloc.str_name", criteria.getAllocsList()); - - if (criteria.getMemoryRangeCount() > 0) { - addRangePhrase("proc.int_mem_reserved", criteria.getMemoryRange(0)); - } - - if (criteria.getDurationRangeCount() > 0) { - filterByDurationRange(criteria.getDurationRange(0)); - } - - setFirstResult(criteria.getFirstResult()); - if (criteria.getMaxResultsCount() > 0) { - setMaxResults(criteria.getMaxResults(0)); - } - } -} diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/ActionDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/ActionDaoJdbc.java deleted file mode 100644 index a393caa82..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/ActionDaoJdbc.java +++ /dev/null @@ -1,164 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.ActionEntity; -import com.imageworks.spcue.ActionInterface; -import com.imageworks.spcue.FilterInterface; -import com.imageworks.spcue.SpcueRuntimeException; -import com.imageworks.spcue.dao.ActionDao; -import com.imageworks.spcue.grpc.filter.ActionType; -import com.imageworks.spcue.grpc.filter.ActionValueType; -import com.imageworks.spcue.util.SqlUtil; - -public class ActionDaoJdbc extends JdbcDaoSupport implements ActionDao { - - public static final String INSERT_ACTION = - "INSERT INTO " + - "action " + - "(" + - "pk_action,pk_filter,str_action,str_value_type,b_stop" + - ") VALUES (?,?,?,?,?)"; - - public void createAction(ActionEntity action) { - action.id = SqlUtil.genKeyRandom(); - boolean stopAction = ActionType.STOP_PROCESSING.equals(action.type); - getJdbcTemplate().update(INSERT_ACTION, - action.id, action.filterId,action.type.toString(), - action.valueType.toString(), stopAction); - updateAction(action); - } - - private static final String GET_ACTION = - "SELECT "+ - "action.*," + - "filter.pk_show "+ - "FROM " + - "action,"+ - "filter " + - "WHERE " + - "action.pk_filter = filter.pk_filter"; - - public ActionEntity getAction(String id) { - return getJdbcTemplate().queryForObject( - GET_ACTION + " AND pk_action=?", - ACTION_DETAIL_MAPPER, id); - } - - public ActionEntity getAction(ActionInterface action) { - return getJdbcTemplate().queryForObject( - GET_ACTION + " AND pk_action=?", - ACTION_DETAIL_MAPPER, action.getActionId()); - } - - public List getActions(FilterInterface filter) { - return getJdbcTemplate().query( - GET_ACTION + " AND filter.pk_filter=? ORDER BY b_stop ASC, ts_created ASC", - ACTION_DETAIL_MAPPER, filter.getFilterId()); - } - - public void updateAction(ActionEntity action) { - if (action.isNew()) { - throw new SpcueRuntimeException("unable to update action that is not already commited"); - } - - // first we clear out all values - - getJdbcTemplate().update( - "UPDATE action SET str_value=NULL,int_value=NULL,b_value=NULL,float_value=NULL WHERE pk_action=?", - action.getActionId()); - - StringBuilder query = new StringBuilder(1024); - query.append("UPDATE action SET str_action=?,str_value_type=?"); - - List args = new ArrayList(4); - args.add(action.type.toString()); - args.add(action.valueType.toString()); - - switch(action.valueType) { - case GROUP_TYPE: - query.append(",pk_folder=? WHERE pk_action=?"); - args.add(action.groupValue); - break; - - case STRING_TYPE: - query.append(",str_value=? WHERE pk_action=?"); - args.add(action.stringValue); - break; - - case INTEGER_TYPE: - query.append(",int_value=? WHERE pk_action=?"); - args.add(action.intValue); - break; - - case FLOAT_TYPE: - query.append(",float_value=? WHERE pk_action=?"); - args.add(action.floatValue); - break; - - case BOOLEAN_TYPE: - query.append(",b_value=? WHERE pk_action=?"); - args.add(action.booleanValue); - break; - - case NONE_TYPE: - query.append(" WHERE pk_action=?"); - break; - - default: - throw new SpcueRuntimeException("invalid action value type: " + action.valueType); - } - - args.add(action.id); - getJdbcTemplate().update(query.toString(), - args.toArray()); - - } - - public void deleteAction(ActionInterface action) { - getJdbcTemplate().update("DELETE FROM action WHERE pk_action=?",action.getActionId()); - } - - public static final RowMapper ACTION_DETAIL_MAPPER = new RowMapper() { - public ActionEntity mapRow(ResultSet rs, int rowNum) throws SQLException { - ActionEntity action = new ActionEntity(); - action.id = rs.getString("pk_action"); - action.showId = rs.getString("pk_show"); - action.filterId = rs.getString("pk_filter"); - action.booleanValue = rs.getBoolean("b_value"); - action.groupValue = rs.getString("pk_folder"); - action.intValue = rs.getLong("int_value"); - action.floatValue = rs.getFloat("float_value"); - action.type = ActionType.valueOf(rs.getString("str_action")); - action.valueType = ActionValueType.valueOf(rs.getString("str_value_type")); - action.stringValue = rs.getString("str_value"); - return action; - } - }; -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/AllocationDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/AllocationDaoJdbc.java deleted file mode 100644 index 2f4a44687..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/AllocationDaoJdbc.java +++ /dev/null @@ -1,205 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.CallableStatement; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Map; -import java.util.regex.Pattern; - -import org.springframework.dao.DataIntegrityViolationException; -import org.springframework.jdbc.core.CallableStatementCreator; -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.SqlParameter; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.AllocationEntity; -import com.imageworks.spcue.AllocationInterface; -import com.imageworks.spcue.EntityRemovalError; -import com.imageworks.spcue.FacilityInterface; -import com.imageworks.spcue.dao.AllocationDao; -import com.imageworks.spcue.util.SqlUtil; - -public class AllocationDaoJdbc extends JdbcDaoSupport implements AllocationDao { - - public static RowMapper ALLOC_MAPPER = new RowMapper() { - public AllocationEntity mapRow(ResultSet rs, int rowNum) throws SQLException { - AllocationEntity alloc = new AllocationEntity(); - alloc.id = rs.getString("pk_alloc"); - alloc.facilityId = rs.getString("pk_facility"); - alloc.name = rs.getString("str_name"); - alloc.tag = rs.getString("str_tag"); - return alloc; - } - }; - - private static final String GET_ALLOCATION = - "SELECT " + - "alloc.pk_facility,"+ - "alloc.pk_alloc, " + - "alloc.str_name, "+ - "alloc.str_tag, " + - "facility.str_name AS facility_name " + - "FROM " + - "alloc, " + - "facility " + - "WHERE " + - "alloc.pk_facility = facility.pk_facility "; - - public AllocationEntity getAllocationEntity(String id) { - return getJdbcTemplate().queryForObject( - GET_ALLOCATION + " AND pk_alloc=?", - ALLOC_MAPPER, id); - } - - public AllocationEntity findAllocationEntity(String facility, String name) { - return getJdbcTemplate().queryForObject( - GET_ALLOCATION + " AND alloc.str_name=?", - ALLOC_MAPPER, String.format("%s.%s", facility, name)); - } - - @Override - public AllocationEntity findAllocationEntity(String name) { - return getJdbcTemplate().queryForObject( - GET_ALLOCATION + " AND alloc.str_name=?", - ALLOC_MAPPER, name); - } - - private static final String INSERT_ALLOCATION = - "INSERT INTO " + - "alloc " + - "(" + - "pk_alloc,"+ - "pk_facility,"+ - "str_name, "+ - "str_tag "+ - ") VALUES (?,?,?,?)"; - - public void insertAllocation(FacilityInterface facility, AllocationEntity detail) { - - String new_alloc_name = String.format("%s.%s", - facility.getName(), detail.getName()); - /* - * Checks if the allocation already exits. - */ - if (getJdbcTemplate().queryForObject( - "SELECT COUNT(1) FROM alloc WHERE str_name=?", - Integer.class, new_alloc_name) > 0) { - - getJdbcTemplate().update( - "UPDATE alloc SET b_enabled=1 WHERE str_name=?", - new_alloc_name); - } - else { - detail.id = SqlUtil.genKeyRandom(); - detail.name = new_alloc_name; - getJdbcTemplate().update(INSERT_ALLOCATION, - detail.id, facility.getFacilityId(), - detail.name, detail.tag); - } - } - - public void deleteAllocation(AllocationInterface a) { - if (getJdbcTemplate().queryForObject( - "SELECT COUNT(1) FROM host WHERE pk_alloc=?", Integer.class, - a.getAllocationId()) > 0) { - throw new EntityRemovalError("allocation still contains hosts", a); - } - - if (getJdbcTemplate().queryForObject( - "SELECT b_default FROM alloc WHERE pk_alloc=?", Integer.class, - a.getAllocationId()) > 0) { - throw new EntityRemovalError("you cannot delete the default allocation", a); - } - - /* - * Allocations are logged in historical data so once they are used you - * can't specifically delete them. They are disabled instead. - */ - try { - getJdbcTemplate().update("DELETE FROM alloc WHERE pk_alloc=?", - a.getAllocationId()); - } catch (DataIntegrityViolationException e) { - getJdbcTemplate().update("UPDATE alloc SET b_enabled = 0 WHERe pk_alloc = ?", - a.getAllocationId()); - } - } - - public void updateAllocationName(AllocationInterface a, String name) { - - if (!Pattern.matches("^\\w+$", name)) { - throw new IllegalArgumentException("The new allocation name" + - "must be alpha numeric and not contain the facility prefix."); - } - - String[] parts = a.getName().split("\\.", 2); - String new_name = String.format("%s.%s", parts[0], name); - - getJdbcTemplate().update( - "UPDATE alloc SET str_name=? WHERE pk_alloc=?", - new_name, a.getAllocationId()); - } - - public void updateAllocationTag(AllocationInterface a, String tag) { - getJdbcTemplate().update("UPDATE alloc SET str_tag=? WHERE pk_alloc=?", - tag, a.getAllocationId()); - - getJdbcTemplate().update("UPDATE host_tag SET str_tag=? WHERE " + - "host_tag.str_tag_type='Alloc' AND pk_host IN " + - "(SELECT pk_host FROM host WHERE host.pk_alloc=?)", tag, - a.getAllocationId()); - - for (Map e: getJdbcTemplate().queryForList( - "SELECT pk_host FROM host WHERE pk_alloc=?",a.getAllocationId())) { - final String pk_host = (String) e.get("pk_host"); - getJdbcTemplate().call(new CallableStatementCreator() { - public CallableStatement createCallableStatement(Connection con) throws SQLException { - CallableStatement c = con.prepareCall("{ call recalculate_tags(?) }"); - c.setString(1, pk_host); - return c; - } - }, new ArrayList()); - } - } - - public void setDefaultAllocation(AllocationInterface a) { - getJdbcTemplate().update("UPDATE alloc SET b_default = 0 WHERE b_default = 1"); - getJdbcTemplate().update("UPDATE alloc SET b_default = 1 WHERe pk_alloc=?", - a.getAllocationId()); - } - - public AllocationEntity getDefaultAllocationEntity() { - return getJdbcTemplate().queryForObject( - GET_ALLOCATION + " AND alloc.b_default = 1 AND ROWNUM = 1", - ALLOC_MAPPER); - } - - @Override - public void updateAllocationBillable(AllocationInterface alloc, boolean value) { - getJdbcTemplate().update( - "UPDATE alloc SET b_billable = ? WHERE pk_alloc = ?", - value, alloc.getAllocationId()); - - } -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/BookingDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/BookingDaoJdbc.java deleted file mode 100644 index 220983026..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/BookingDaoJdbc.java +++ /dev/null @@ -1,363 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Calendar; -import java.util.List; - -import org.springframework.dao.DataAccessException; -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.FrameInterface; -import com.imageworks.spcue.HostInterface; -import com.imageworks.spcue.JobInterface; -import com.imageworks.spcue.LayerInterface; -import com.imageworks.spcue.LocalHostAssignment; -import com.imageworks.spcue.dao.BookingDao; -import com.imageworks.spcue.dispatcher.ResourceReservationFailureException; -import com.imageworks.spcue.grpc.renderpartition.RenderPartitionType; -import com.imageworks.spcue.util.SqlUtil; - -public class BookingDaoJdbc extends - JdbcDaoSupport implements BookingDao { - - private static final String INSERT_LOCAL_JOB_ASSIGNMENT = - "INSERT INTO " + - "host_local " + - "(" + - "pk_host_local,"+ - "pk_job,"+ - "pk_layer,"+ - "pk_frame,"+ - "str_type,"+ - "pk_host,"+ - "int_mem_max,"+ - "int_mem_idle,"+ - "int_cores_max,"+ - "int_cores_idle,"+ - "int_gpu_idle,"+ - "int_gpu_max,"+ - "int_threads "+ - ") " + - "VALUES " + - "(?,?,?,?,?,?,?,?,?,?,?,?,?)"; - - @Override - public void insertLocalHostAssignment(HostInterface h, JobInterface job, LocalHostAssignment l) { - l.id = SqlUtil.genKeyRandom(); - l.name = String.format("%s->%s", h.getName(), job.getName()); - l.setHostId(h.getHostId()); - l.setJobId(job.getJobId()); - l.setType(RenderPartitionType.JOB_PARTITION); - l.setIdleCoreUnits(l.getMaxCoreUnits()); - l.setIdleMemory(l.getMaxMemory()); - l.setIdleGpu(l.getMaxGpu()); - - getJdbcTemplate().update( - INSERT_LOCAL_JOB_ASSIGNMENT, - l.id, - job.getJobId(), - l.getLayerId(), - l.getFrameId(), - l.getType().toString(), - h.getHostId(), - l.getMaxMemory(), - l.getMaxMemory(), - l.getMaxCoreUnits(), - l.getMaxCoreUnits(), - l.getMaxGpu(), - l.getMaxGpu(), - l.getThreads()); - } - - @Override - public void insertLocalHostAssignment(HostInterface h, LayerInterface layer, LocalHostAssignment l) { - l.id = SqlUtil.genKeyRandom(); - l.name = String.format("%s->%s", h.getName(), layer.getName()); - l.setHostId(h.getHostId()); - l.setJobId(layer.getJobId()); - l.setLayerId(layer.getLayerId()); - l.setType(RenderPartitionType.LAYER_PARTITION); - l.setIdleCoreUnits(l.getMaxCoreUnits()); - l.setIdleMemory(l.getMaxMemory()); - l.setIdleGpu(l.getMaxGpu()); - - getJdbcTemplate().update( - INSERT_LOCAL_JOB_ASSIGNMENT, - l.id, - l.getJobId(), - l.getLayerId(), - l.getFrameId(), - l.getType().toString(), - h.getHostId(), - l.getMaxMemory(), - l.getMaxMemory(), - l.getMaxCoreUnits(), - l.getMaxCoreUnits(), - l.getMaxGpu(), - l.getMaxGpu(), - l.getThreads()); - } - - @Override - public void insertLocalHostAssignment(HostInterface h, FrameInterface frame, LocalHostAssignment l) { - l.id = SqlUtil.genKeyRandom(); - l.name = String.format("%s->%s", h.getName(), frame.getName()); - l.setHostId(h.getHostId()); - l.setJobId(frame.getJobId()); - l.setLayerId(frame.getLayerId()); - l.setFrameId(frame.getFrameId()); - l.setType(RenderPartitionType.FRAME_PARTITION); - l.setIdleCoreUnits(l.getMaxCoreUnits()); - l.setIdleMemory(l.getMaxMemory()); - l.setIdleGpu(l.getMaxGpu()); - - getJdbcTemplate().update( - INSERT_LOCAL_JOB_ASSIGNMENT, - l.id, - l.getJobId(), - l.getLayerId(), - l.getFrameId(), - l.getType().toString(), - h.getHostId(), - l.getMaxMemory(), - l.getMaxMemory(), - l.getMaxCoreUnits(), - l.getMaxCoreUnits(), - l.getMaxGpu(), - l.getMaxGpu(), - l.getThreads()); - } - public static final RowMapper LJA_MAPPER = - new RowMapper() { - public LocalHostAssignment mapRow(final ResultSet rs, int rowNum) throws SQLException { - LocalHostAssignment l = new LocalHostAssignment(); - l.id = rs.getString("pk_host_local"); - l.setMaxCoreUnits(rs.getInt("int_cores_max")); - l.setMaxMemory(rs.getLong("int_mem_max")); - l.setMaxGpu(rs.getLong("int_gpu_max")); - l.setThreads(rs.getInt("int_threads")); - l.setIdleCoreUnits(rs.getInt("int_cores_idle")); - l.setIdleMemory(rs.getLong("int_mem_idle")); - l.setIdleGpu(rs.getLong("int_gpu_idle")); - l.setJobId(rs.getString("pk_job")); - l.setLayerId(rs.getString("pk_layer")); - l.setFrameId(rs.getString("pk_frame")); - l.setHostId(rs.getString("pk_host")); - l.setType(RenderPartitionType.valueOf(rs.getString("str_type"))); - return l; - } - }; - - private static final String QUERY_FOR_LJA = - "SELECT " + - "pk_host_local,"+ - "pk_job,"+ - "pk_layer," + - "pk_frame,"+ - "pk_host,"+ - "int_mem_idle,"+ - "int_mem_max,"+ - "int_cores_idle,"+ - "int_cores_max,"+ - "int_gpu_idle,"+ - "int_gpu_max,"+ - "int_threads, "+ - "str_type " + - "FROM " + - "host_local "; - - @Override - public List getLocalJobAssignment(HostInterface host) { - return getJdbcTemplate().query( - QUERY_FOR_LJA + - "WHERE " + - "host_local.pk_host = ? ", - LJA_MAPPER, host.getHostId()); - } - - @Override - public LocalHostAssignment getLocalJobAssignment(String id) { - return getJdbcTemplate().queryForObject( - QUERY_FOR_LJA + " WHERE pk_host_local = ?", - LJA_MAPPER, id); - } - - @Override - public LocalHostAssignment getLocalJobAssignment(String hostId, String jobId) { - return getJdbcTemplate().queryForObject( - QUERY_FOR_LJA + " WHERE pk_host = ? and pk_job = ?", - LJA_MAPPER, hostId, jobId); - } - - @Override - public boolean deleteLocalJobAssignment(LocalHostAssignment l) { - return getJdbcTemplate().update( - "DELETE FROM host_local WHERE pk_host_local = ?", - l.getId()) > 0; - } - - private static final String HAS_LOCAL_JOB = - "SELECT " + - "COUNT(1) " + - "FROM " + - "host_local " + - "WHERE " + - "host_local.pk_host = ? "; - - @Override - public boolean hasLocalJob(HostInterface host) { - return getJdbcTemplate().queryForObject(HAS_LOCAL_JOB, - Integer.class, host.getHostId()) > 0; - } - - private static final String HAS_ACTIVE_LOCAL_JOB = - "SELECT " + - "COUNT(1) " + - "FROM " + - "host_local, " + - "proc " + - "WHERE " + - "host_local.pk_host = proc.pk_host " + - "AND " + - "proc.b_local = 1 " + - "AND " + - "host_local.pk_host = ? "; - - @Override - public boolean hasActiveLocalJob(HostInterface host) { - return getJdbcTemplate().queryForObject(HAS_ACTIVE_LOCAL_JOB, - Integer.class, host.getHostId()) > 0; - } - - @Override - public int getCoreUsageDifference(LocalHostAssignment l, int coreUnits) { - return getJdbcTemplate().queryForObject( - "SELECT ? - int_cores_max FROM host_local WHERE pk_host_local=?", - Integer.class, coreUnits, l.getId()); - } - - private static final String UPDATE_MAX_CORES = - "UPDATE " + - "host_local " + - "SET " + - "int_cores_idle = int_cores_idle + (? - int_cores_max), " + - "int_cores_max = ? "+ - "WHERE " + - "pk_host_local = ? "; - - @Override - public boolean updateMaxCores(LocalHostAssignment l, int coreUnits) { - return getJdbcTemplate().update(UPDATE_MAX_CORES, - coreUnits, coreUnits, l.getId()) > 0; - } - - private static final String UPDATE_MAX_MEMORY = - "UPDATE " + - "host_local " + - "SET " + - "int_mem_idle = int_mem_idle + (? - int_mem_max), " + - "int_mem_max = ? "+ - "WHERE " + - "pk_host_local = ? "; - - @Override - public boolean updateMaxMemory(LocalHostAssignment l, long maxMemory) { - return getJdbcTemplate().update( - UPDATE_MAX_MEMORY, maxMemory, maxMemory, l.getId()) > 0; - } - - private static final String UPDATE_MAX_GPU = - "UPDATE " + - "host_local " + - "SET " + - "int_gpu_idle = int_gpu_idle + (? - int_gpu_max), " + - "int_gpu_max = ? "+ - "WHERE " + - "pk_host_local = ? "; - - @Override - public boolean updateMaxGpu(LocalHostAssignment l, long maxGpu) { - return getJdbcTemplate().update( - UPDATE_MAX_GPU, maxGpu, maxGpu, l.getId()) > 0; - } - - @Override - public boolean deactivate(LocalHostAssignment l) { - return getJdbcTemplate().update( - "UPDATE host_local SET b_active = 0 WHERE " + - "pk_host_local = ? AND b_active = 1", - l.getId()) > 0; - } - - /** - * - * @param id - * @param cores - * @return - */ - @Override - public boolean allocateCoresFromHost(HostInterface h, int cores) { - - try { - return getJdbcTemplate().update( - "UPDATE host SET int_cores_idle = int_cores_idle - ? " + - "WHERE pk_host = ?", - cores, h.getHostId()) > 0; - } catch (DataAccessException e) { - throw new ResourceReservationFailureException("Failed to allocate " + - cores + " from host, " + e); - } - - } - - /** - * - * @param id - * @param cores - * @return - */ - @Override - public boolean deallocateCoresFromHost(HostInterface h, int cores) { - try { - return getJdbcTemplate().update( - "UPDATE host SET int_cores_idle = int_cores_idle + ? WHERE pk_host = ?", - cores, h.getHostId()) > 0; - } catch (DataAccessException e) { - throw new ResourceReservationFailureException("Failed to de-allocate " + - cores + " from host, " + e); - } - } - - @Override - public boolean hasResourceDeficit(HostInterface host) { - return getJdbcTemplate().queryForObject( - "SELECT COUNT(1) FROM host_local WHERE " + - "(int_cores_max < int_cores_max - int_cores_idle OR " + - "int_gpu_max < int_gpu_max - int_gpu_idle OR " + - "int_mem_max < int_mem_max - int_mem_idle) AND " + - "host_local.pk_host= ?", - Integer.class, host.getHostId()) > 0; - } -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/CommentDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/CommentDaoJdbc.java deleted file mode 100644 index 3da40621b..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/CommentDaoJdbc.java +++ /dev/null @@ -1,134 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Map; - -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.CommentDetail; -import com.imageworks.spcue.HostInterface; -import com.imageworks.spcue.JobInterface; -import com.imageworks.spcue.dao.CommentDao; -import com.imageworks.spcue.util.SqlUtil; - -public class CommentDaoJdbc extends JdbcDaoSupport implements CommentDao { - - public void deleteComment(String id) { - /* - * Checks what type of comment we have. - */ - Map type = getJdbcTemplate().queryForMap( - "SELECT pk_job, pk_host FROM comments WHERE pk_comment=?",id); - - /* - * If the comment is deleted successfully, check if we need to unset - * the b_comment boolean flag. - */ - if (getJdbcTemplate().update( - "DELETE FROM comments WHERE pk_comment=?",id) > 0) { - if (type.get("pk_job") != null) { - getJdbcTemplate().update("UPDATE job SET b_comment=0 WHERE job.pk_job = ? AND " + - "(SELECT COUNT(1) FROM comments c WHERE c.pk_job = job.pk_job) = 0",type.get("pk_job")); - } - else if (type.get("pk_host") != null) { - getJdbcTemplate().update("UPDATE host SET b_comment=0 WHERE host.pk_host = ? AND " + - "(SELECT COUNT(1) FROM comments c WHERE c.pk_host = host.pk_host) = 0",type.get("pk_host")); - } - } - } - - private static final RowMapper COMMENT_DETAIL_MAPPER = - new RowMapper() { - public CommentDetail mapRow(ResultSet rs, int row) throws SQLException { - CommentDetail d = new CommentDetail(); - d.id = rs.getString("pk_comment"); - d.message = rs.getString("str_message"); - d.subject = rs.getString("str_subject"); - d.timestamp = rs.getTimestamp("ts_created"); - d.user = rs.getString("str_user"); - return d; - } - }; - - public CommentDetail getCommentDetail(String id) { - return getJdbcTemplate().queryForObject( - "SELECT * FROM comments WHERE pk_comment=?", - COMMENT_DETAIL_MAPPER, id); - } - - public void updateComment(CommentDetail comment) { - getJdbcTemplate().update( - "UPDATE comments SET str_message=?,str_subject=? WHERE pk_comment=?", - comment.message, comment.subject, comment.id); - } - - public void updateCommentMessage(String id, String message) { - getJdbcTemplate().update( - "UPDATE comments SET str_message=? WHERE pk_comment=?", - message,id); - } - - public void updateCommentSubject(String id, String subject) { - getJdbcTemplate().update( - "UPDATE comments SET str_subject=? WHERE pk_comment=?", - subject,id); - } - - private static final String INSERT_JOB_COMMENT = - "INSERT INTO " + - "comments " + - "(" + - "pk_comment,pk_job,str_user,str_subject,str_message"+ - ") VALUES (?,?,?,?,?)"; - - public void insertComment(JobInterface job, CommentDetail comment) { - comment.id = SqlUtil.genKeyRandom(); - getJdbcTemplate().update(INSERT_JOB_COMMENT, - comment.id, job.getJobId(), comment.user, - comment.subject, comment.message); - getJdbcTemplate().update( - "UPDATE job SET b_comment=1 WHERE pk_job=?", - job.getJobId()); - } - - private static final String INSERT_HOST_COMMENT = - "INSERT INTO " + - "comments " + - "(" + - "pk_comment,pk_host,str_user,str_subject,str_message"+ - ") VALUES (?,?,?,?,?)"; - - - public void insertComment(HostInterface host, CommentDetail comment) { - comment.id = SqlUtil.genKeyRandom(); - getJdbcTemplate().update(INSERT_HOST_COMMENT, - comment.id, host.getHostId(), comment.user, - comment.subject, comment.message); - getJdbcTemplate().update( - "UPDATE host SET b_comment=1 WHERE pk_host=?", - host.getHostId()); - } - -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DeedDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DeedDaoJdbc.java deleted file mode 100644 index 68ab67521..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DeedDaoJdbc.java +++ /dev/null @@ -1,119 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; - -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.DeedEntity; -import com.imageworks.spcue.HostInterface; -import com.imageworks.spcue.OwnerEntity; -import com.imageworks.spcue.dao.DeedDao; -import com.imageworks.spcue.util.SqlUtil; - -public class DeedDaoJdbc extends JdbcDaoSupport implements DeedDao { - - public static final RowMapper - DEED_MAPPER = new RowMapper() { - public DeedEntity mapRow(ResultSet rs, int rowNum) throws SQLException { - DeedEntity o = new DeedEntity(); - o.id = rs.getString("pk_deed"); - o.owner = rs.getString("str_username"); - o.host = rs.getString("str_hostname"); - return o; - } - }; - - @Override - public boolean deleteDeed(DeedEntity deed) { - return getJdbcTemplate().update( - "DELETE FROM deed WHERE pk_deed = ?", - deed.getId()) > 0; - } - - @Override - public boolean deleteDeed(HostInterface host) { - return getJdbcTemplate().update( - "DELETE FROM deed WHERE pk_host = ?", - host.getHostId()) > 0; - } - - @Override - public void deleteDeeds(OwnerEntity owner) { - getJdbcTemplate().update( - "DELETE FROM deed WHERE pk_owner = ?", - owner.getId()); - } - - private static final String INSERT_DEED = - "INSERT INTO " + - "deed " + - "("+ - "pk_deed,"+ - "pk_owner,"+ - "pk_host " + - ") "+ - "VALUES (?,?,?)"; - - public DeedEntity insertDeed(OwnerEntity owner, HostInterface host) { - DeedEntity deed = new DeedEntity(); - deed.id = SqlUtil.genKeyRandom(); - deed.host = host.getName(); - deed.owner = owner.name; - - getJdbcTemplate().update(INSERT_DEED, - deed.getId(), owner.getId(), host.getId()); - - return deed; - } - - private static final String QUERY_FOR_DEED = - "SELECT " + - "deed.pk_deed, "+ - "host.str_name as str_hostname, " + - "owner.str_username " + - "FROM " + - "deed,"+ - "host,"+ - "owner " + - "WHERE " + - "deed.pk_owner = owner.pk_owner " + - "AND " + - "deed.pk_host = host.pk_host "; - - @Override - public DeedEntity getDeed(String id) { - return getJdbcTemplate().queryForObject( - QUERY_FOR_DEED + " AND pk_deed = ?", - DEED_MAPPER, id); - } - - @Override - public List getDeeds(OwnerEntity owner) { - return getJdbcTemplate().query( - QUERY_FOR_DEED + " AND owner.pk_owner = ?", - DEED_MAPPER, owner.getId()); - } -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DepartmentDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DepartmentDaoJdbc.java deleted file mode 100644 index 32ddaaa28..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DepartmentDaoJdbc.java +++ /dev/null @@ -1,84 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; - -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.DepartmentEntity; -import com.imageworks.spcue.DepartmentInterface; -import com.imageworks.spcue.dao.DepartmentDao; -import com.imageworks.spcue.util.SqlUtil; - -public class DepartmentDaoJdbc extends JdbcDaoSupport implements DepartmentDao { - - public static final RowMapper DEPARTMENT_MAPPER = new RowMapper() { - public DepartmentInterface mapRow(ResultSet rs, int rowNum) throws SQLException { - DepartmentEntity d = new DepartmentEntity(); - d.id = rs.getString("pk_dept"); - d.name = rs.getString("str_name"); - return d; - } - }; - - @Override - public boolean departmentExists(String name) { - return getJdbcTemplate().queryForObject( - "SELECT COUNT(1) FROM dept WHERE str_name=?", - Integer.class, name) > 0; - } - - @Override - public DepartmentInterface findDepartment(String name) { - return getJdbcTemplate().queryForObject( - "SELECT pk_dept, str_name FROM dept WHERE str_name=?", - DEPARTMENT_MAPPER, name); - } - - @Override - public DepartmentInterface getDefaultDepartment() { - return getJdbcTemplate().queryForObject( - "SELECT pk_dept, str_name FROM dept WHERE b_default=1", - DEPARTMENT_MAPPER); - } - - @Override - public DepartmentInterface getDepartment(String id) { - return getJdbcTemplate().queryForObject( - "SELECT pk_dept, str_name FROM dept WHERE pk_dept=?", - DEPARTMENT_MAPPER, id); - } - - @Override - public void deleteDepartment(DepartmentInterface d) { - getJdbcTemplate().update("DELETE FROM dept WHERE pk_dept=?", - d.getDepartmentId()); - } - - @Override - public void insertDepartment(String name) { - getJdbcTemplate().update("INSERT INTO dept (pk_dept,str_name) VALUES (?,?)", - SqlUtil.genKeyRandom(), name); - } -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DependDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DependDaoJdbc.java deleted file mode 100644 index 72c03da34..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DependDaoJdbc.java +++ /dev/null @@ -1,751 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; - -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.FrameInterface; -import com.imageworks.spcue.JobInterface; -import com.imageworks.spcue.LayerInterface; -import com.imageworks.spcue.LightweightDependency; -import com.imageworks.spcue.dao.DependDao; -import com.imageworks.spcue.depend.DependException; -import com.imageworks.spcue.depend.FrameByFrame; -import com.imageworks.spcue.depend.FrameOnFrame; -import com.imageworks.spcue.depend.FrameOnJob; -import com.imageworks.spcue.depend.FrameOnLayer; -import com.imageworks.spcue.depend.JobOnFrame; -import com.imageworks.spcue.depend.JobOnJob; -import com.imageworks.spcue.depend.JobOnLayer; -import com.imageworks.spcue.depend.LayerOnFrame; -import com.imageworks.spcue.depend.LayerOnJob; -import com.imageworks.spcue.depend.LayerOnLayer; -import com.imageworks.spcue.depend.PreviousFrame; -import com.imageworks.spcue.grpc.depend.DependTarget; -import com.imageworks.spcue.grpc.depend.DependType; -import com.imageworks.spcue.util.SqlUtil; - -public class DependDaoJdbc extends JdbcDaoSupport implements DependDao { - - public static final RowMapper DEPEND_MAPPER = new RowMapper() { - public LightweightDependency mapRow(ResultSet rs, int row) throws SQLException { - LightweightDependency d = new LightweightDependency(); - d.id = rs.getString("pk_depend"); - d.type = DependType.valueOf(rs.getString("str_type")); - d.target = DependTarget.valueOf(rs.getString("str_target")); - d.anyFrame = rs.getBoolean("b_any"); - d.parent = rs.getString("pk_parent"); - d.active = rs.getBoolean("b_active"); - d.dependErFrameId = rs.getString("pk_frame_depend_er"); - d.dependOnFrameId = rs.getString("pk_frame_depend_on"); - d.dependErLayerId = rs.getString("pk_layer_depend_er"); - d.dependOnLayerId =rs.getString("pk_layer_depend_on"); - d.dependOnJobId = rs.getString("pk_job_depend_on"); - d.dependErJobId = rs.getString("pk_job_depend_er"); - return d; - } - }; - - private static final String INSERT_DEPEND = - "INSERT INTO " + - "depend " + - "(" + - "pk_depend,"+ - "pk_parent,"+ - "pk_job_depend_er," + - "pk_layer_depend_er," + - "pk_frame_depend_er," + - "pk_job_depend_on," + - "pk_layer_depend_on," + - "pk_frame_depend_on," + - "str_type," + - "b_any, " + - "str_target, " + - "b_active, " + - "str_signature, "+ - "b_composite " + - ") " + - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; - - @Override - public void insertDepend(JobOnJob d) { - d.setId(SqlUtil.genKeyRandom()); - getJdbcTemplate().update(INSERT_DEPEND, - d.getId(), - null, - d.getDependErJob().getJobId(), - null, - null, - d.getDependOnJob().getJobId(), - null, - null, - DependType.JOB_ON_JOB.toString(), - d.isAnyFrame(), - d.getTarget().toString(), - d.isActive(), - d.getSignature(), - d.isComposite()); - } - - @Override - public void insertDepend(JobOnLayer d) { - d.setId(SqlUtil.genKeyRandom()); - getJdbcTemplate().update(INSERT_DEPEND, - d.getId(), - null, - d.getDependErJob().getJobId(), - null, - null, - d.getDependOnLayer().getJobId(), - d.getDependOnLayer().getLayerId(), - null, - DependType.JOB_ON_LAYER.toString(), - d.isAnyFrame(), - d.getTarget().toString(), - d.isActive(), - d.getSignature(), - d.isComposite()); - } - - @Override - public void insertDepend(JobOnFrame d) { - d.setId(SqlUtil.genKeyRandom()); - getJdbcTemplate().update(INSERT_DEPEND, - d.getId(), - null, - d.getDependErJob().getJobId(), - null, - null, - d.getDependOnFrame().getJobId(), - d.getDependOnFrame().getLayerId(), - d.getDependOnFrame().getFrameId(), - DependType.JOB_ON_FRAME.toString(), - d.isAnyFrame(), - d.getTarget().toString(), - d.isActive(), - d.getSignature(), - d.isComposite()); - } - - @Override - public void insertDepend(LayerOnJob d) { - d.setId(SqlUtil.genKeyRandom()); - getJdbcTemplate().update(INSERT_DEPEND, - d.getId(), - null, - d.getDependErLayer().getJobId(), - d.getDependErLayer().getLayerId(), - null, - d.getDependOnJob().getJobId(), - null, - null, - DependType.LAYER_ON_JOB.toString(), - d.isAnyFrame(), - d.getTarget().toString(), - d.isActive(), - d.getSignature(), - d.isComposite()); - } - - @Override - public void insertDepend(LayerOnLayer d) { - d.setId(SqlUtil.genKeyRandom()); - getJdbcTemplate().update(INSERT_DEPEND, - d.getId(), - null, - d.getDependErLayer().getJobId(), - d.getDependErLayer().getLayerId(), - null, - d.getDependOnLayer().getJobId(), - d.getDependOnLayer().getLayerId(), - null, - DependType.LAYER_ON_LAYER.toString(), - d.isAnyFrame(), - d.getTarget().toString(), - d.isActive(), - d.getSignature(), - d.isComposite()); - } - - @Override - public void insertDepend(LayerOnFrame d) { - d.setId(SqlUtil.genKeyRandom()); - getJdbcTemplate().update(INSERT_DEPEND, - d.getId(), - null, - d.getDependErLayer().getJobId(), - d.getDependErLayer().getLayerId(), - null, - d.getDependOnFrame().getJobId(), - d.getDependOnFrame().getLayerId(), - d.getDependOnFrame().getFrameId(), - DependType.LAYER_ON_FRAME.toString(), - d.isAnyFrame(), - d.getTarget().toString(), - d.isActive(), - d.getSignature(), - d.isComposite()); - } - - @Override - public void insertDepend(FrameOnJob d) { - d.setId(SqlUtil.genKeyRandom()); - getJdbcTemplate().update(INSERT_DEPEND, - d.getId(), - null, - d.getDependErFrame().getJobId(), - d.getDependErFrame().getLayerId(), - d.getDependErFrame().getFrameId(), - d.getDependOnJob().getJobId(), - null, - null, - DependType.FRAME_ON_JOB.toString(), - d.isAnyFrame(), - d.getTarget().toString(), - d.isActive(), - d.getSignature(), - d.isComposite()); - } - - @Override - public void insertDepend(FrameOnLayer d) { - d.setId(SqlUtil.genKeyRandom()); - getJdbcTemplate().update(INSERT_DEPEND, - d.getId(), - null, - d.getDependErFrame().getJobId(), - d.getDependErFrame().getLayerId(), - d.getDependErFrame().getFrameId(), - d.getDependOnLayer().getJobId(), - d.getDependOnLayer().getLayerId(), - null, - DependType.FRAME_ON_LAYER.toString(), - d.isAnyFrame(), - d.getTarget().toString(), - d.isActive(), - d.getSignature(), - d.isComposite()); - } - - @Override - public void insertDepend(PreviousFrame d) { - d.setId(SqlUtil.genKeyRandom()); - getJdbcTemplate().update(INSERT_DEPEND, - d.getId(), - null, - d.getDependErLayer().getJobId(), - d.getDependErLayer().getLayerId(), - null, - d.getDependOnLayer().getJobId(), - d.getDependOnLayer().getLayerId(), - null, - DependType.PREVIOUS_FRAME.toString(), - d.isAnyFrame(), - d.getTarget().toString(), - d.isActive(), - d.getSignature(), - d.isComposite()); - } - - @Override - public void insertDepend(FrameOnFrame d) { - d.setId(SqlUtil.genKeyRandom()); - String parentId = null; - if (d.getParent() != null) { - parentId =d.getParent().getId(); - } - - getJdbcTemplate().update(INSERT_DEPEND, - d.getId(), - parentId, - d.getDependErFrame().getJobId(), - d.getDependErFrame().getLayerId(), - d.getDependErFrame().getFrameId(), - d.getDependOnFrame().getJobId(), - d.getDependOnFrame().getLayerId(), - d.getDependOnFrame().getFrameId(), - DependType.FRAME_ON_FRAME.toString(), - d.isAnyFrame(), - d.getTarget().toString(), - d.isActive(), - d.getSignature(), - d.isComposite()); - } - - @Override - public void insertDepend(FrameByFrame d) { - d.setId(SqlUtil.genKeyRandom()); - getJdbcTemplate().update(INSERT_DEPEND, - d.getId(), - null, - d.getDependErLayer().getJobId(), - d.getDependErLayer().getLayerId(), - null, - d.getDependOnLayer().getJobId(), - d.getDependOnLayer().getLayerId(), - null, - DependType.FRAME_BY_FRAME.toString(), - d.isAnyFrame(), - d.getTarget().toString(), - d.isActive(), - d.getSignature(), - d.isComposite()); - } - - private static final String UPDATE_FRAME_STATE = - "UPDATE " + - "frame " + - "SET " + - "str_state='DEPEND' " + - "WHERE " + - "int_depend_count != 0 " + - "AND " + - "frame.str_state NOT IN ('SUCCEEDED','EATEN','RUNNING','DEPEND') " + - "AND " + - "frame.pk_frame = ?"; - - @Override - public void updateFrameState(FrameInterface f) { - getJdbcTemplate().update(UPDATE_FRAME_STATE, - f.getFrameId()); - } - - private static final String UPDATE_DEPEND_COUNT = - "UPDATE " + - "frame " + - "SET " + - "int_depend_count = int_depend_count + 1 " + - "WHERE " + - "pk_frame = ?"; - - @Override - public void incrementDependCount(FrameInterface f) { - int result = getJdbcTemplate().update(UPDATE_DEPEND_COUNT, - f.getFrameId()); - if (result == 0) { - throw new DependException("updating the depend count for " + - " the frame " + f.getName() + " in job " + f.getJobId() + - "failed."); - } - } - - private static final String DECREMENT_DEPEND_COUNT = - "UPDATE " + - "frame " + - "SET " + - "int_depend_count = int_depend_count -1 " + - "WHERE " + - "pk_frame = ? " + - "AND " + - "int_depend_count > 0"; - - @Override - public boolean decrementDependCount(FrameInterface f) { - return getJdbcTemplate().update(DECREMENT_DEPEND_COUNT, - f.getFrameId()) == 1; - } - - private static final String[] DELETE_DEPEND = { - "DELETE FROM depend WHERE pk_parent=?", - "DELETE FROM depend WHERE pk_depend=?" - }; - - @Override - public void deleteDepend(LightweightDependency depend) { - if (depend.type.equals(DependType.FRAME_BY_FRAME)) { - getJdbcTemplate().update(DELETE_DEPEND[0], depend.getId()); - } - getJdbcTemplate().update(DELETE_DEPEND[1], depend.getId()); - } - - private static final String GET_LIGHTWEIGHT_DEPEND = - "SELECT * FROM depend WHERE pk_depend=?"; - - @Override - public LightweightDependency getDepend(String id) { - return getJdbcTemplate().queryForObject( - GET_LIGHTWEIGHT_DEPEND, - DEPEND_MAPPER, id); - } - - private static final String GET_LIGHTWEIGHT_DEPEND_BY_SIGNATURE = - "SELECT * FROM depend WHERE str_signature=?"; - - @Override - public LightweightDependency getDependBySignature(String s) { - return getJdbcTemplate().queryForObject( - GET_LIGHTWEIGHT_DEPEND_BY_SIGNATURE, - DEPEND_MAPPER, s); - } - - private static final String GET_WHAT_DEPENDS_ON_JOB = - "SELECT " + - "depend.pk_depend," + - "depend.str_type," + - "depend.str_target,"+ - "depend.b_any,"+ - "depend.pk_parent,"+ - "depend.b_active," + - "depend.pk_frame_depend_er,"+ - "depend.pk_frame_depend_on,"+ - "depend.pk_layer_depend_er,"+ - "depend.pk_layer_depend_on,"+ - "depend.pk_job_depend_er,"+ - "depend.pk_job_depend_on "+ - "FROM " + - "depend " + - "WHERE " + - "pk_job_depend_on=? " + - "AND " + - "b_active = 1 " + - "AND " + - "str_type IN (?,?,?)"; - - @Override - public List getWhatDependsOn(JobInterface job) { - return getJdbcTemplate().query(GET_WHAT_DEPENDS_ON_JOB, - DEPEND_MAPPER, job.getJobId(), - DependType.JOB_ON_JOB.toString(), - DependType.LAYER_ON_JOB.toString(), - DependType.FRAME_ON_JOB.toString()); - } - - private static final String GET_WHAT_DEPENDS_ON_JOB_WITH_TARGET = - "SELECT " + - "depend.pk_depend," + - "depend.str_type," + - "depend.str_target,"+ - "depend.b_any,"+ - "depend.pk_parent,"+ - "depend.b_active," + - "depend.pk_frame_depend_er,"+ - "depend.pk_frame_depend_on,"+ - "depend.pk_layer_depend_er,"+ - "depend.pk_layer_depend_on,"+ - "depend.pk_job_depend_er,"+ - "depend.pk_job_depend_on "+ - "FROM " + - "depend " + - "WHERE " + - "pk_job_depend_on=? " + - "AND " + - "b_active = 1 " + - "AND " + - "str_target = ? " + - "AND " + - "str_type IN (?,?,?)"; - - @Override - public List getWhatDependsOn(JobInterface job, DependTarget target) { - if (target.equals(DependTarget.ANY_TARGET)) { - return getWhatDependsOn(job); - } - else { - return getJdbcTemplate().query(GET_WHAT_DEPENDS_ON_JOB_WITH_TARGET, - DEPEND_MAPPER, job.getJobId(), target.toString(), - DependType.JOB_ON_JOB.toString(), - DependType.LAYER_ON_JOB.toString(), - DependType.FRAME_ON_JOB.toString()); - } - } - - private static final String GET_WHAT_DEPENDS_ON_LAYER = - "SELECT " + - "depend.pk_depend," + - "depend.str_type," + - "depend.str_target,"+ - "depend.b_any,"+ - "depend.pk_parent,"+ - "depend.b_active," + - "depend.pk_frame_depend_er,"+ - "depend.pk_frame_depend_on,"+ - "depend.pk_layer_depend_er,"+ - "depend.pk_layer_depend_on,"+ - "depend.pk_job_depend_er,"+ - "depend.pk_job_depend_on "+ - "FROM " + - "depend " + - "WHERE " + - "pk_job_depend_on=? " + - "AND " + - "pk_layer_depend_on=? " + - "AND " + - "str_type IN (?,?,?) " + - "AND " + - "b_active = ?"; - - @Override - public List getWhatDependsOn(LayerInterface layer) { - return getJdbcTemplate().query(GET_WHAT_DEPENDS_ON_LAYER, - DEPEND_MAPPER, layer.getJobId(), layer.getLayerId(), - DependType.JOB_ON_LAYER.toString(), - DependType.LAYER_ON_LAYER.toString(), - DependType.FRAME_ON_LAYER.toString(), - true); - } - - @Override - public List getWhatDependsOn(LayerInterface layer, boolean active) { - return getJdbcTemplate().query(GET_WHAT_DEPENDS_ON_LAYER, - DEPEND_MAPPER, layer.getJobId(), layer.getLayerId(), - DependType.JOB_ON_LAYER.toString(), - DependType.LAYER_ON_LAYER.toString(), - DependType.FRAME_ON_LAYER.toString(), - active); - } - - - private static final String GET_WHAT_DEPENDS_ON_FRAME = - "SELECT " + - "depend.pk_depend," + - "depend.str_type," + - "depend.str_target,"+ - "depend.b_any,"+ - "depend.pk_parent,"+ - "depend.b_active," + - "depend.pk_frame_depend_er,"+ - "depend.pk_frame_depend_on,"+ - "depend.pk_layer_depend_er,"+ - "depend.pk_layer_depend_on,"+ - "depend.pk_job_depend_er,"+ - "depend.pk_job_depend_on "+ - "FROM " + - "depend " + - "WHERE " + - "b_active = ? " + - "AND " + - "pk_job_depend_on = ? " + - "AND " + - "(pk_frame_depend_on = ? " + - "AND " + - "str_type IN (?,?,?)) " + - "OR " + - "(pk_layer_depend_on = ? AND str_type = ? AND b_any = 1)"; - - @Override - public List getWhatDependsOn(FrameInterface frame) { - return getWhatDependsOn(frame, true); - } - - @Override - public List getWhatDependsOn(FrameInterface frame, boolean active) { - return getJdbcTemplate().query(GET_WHAT_DEPENDS_ON_FRAME, - DEPEND_MAPPER, active, frame.getJobId(), frame.getFrameId(), - DependType.FRAME_ON_FRAME.toString(), - DependType.LAYER_ON_FRAME.toString(), - DependType.JOB_ON_FRAME.toString(), - frame.getLayerId(), - DependType.LAYER_ON_LAYER.toString()); - } - - private static final String SET_INACTIVE = - "UPDATE " + - "depend " + - "SET " + - "b_active=0,"+ - "ts_satisfied=systimestamp,"+ - "str_signature=pk_depend "+ - "WHERE " + - "pk_depend = ? " + - "AND " + - "b_active = 1 " + - "AND " + - "b_composite = 0"; - - @Override - public boolean setInactive(LightweightDependency depend) { - depend.active = getJdbcTemplate().update(SET_INACTIVE, depend.getId()) == 1; - return depend.active; - } - - private static final String SET_ACTIVE = - "UPDATE " + - "depend " + - "SET " + - "b_active=1 "+ - "WHERE " + - "pk_depend=? " + - "AND "+ - "b_active=0"; - - @Override - public boolean setActive(LightweightDependency depend) { - if (!depend.type.equals(DependType.FRAME_ON_FRAME) - && !depend.type.equals(DependType.LAYER_ON_LAYER)) { - return false; - } - depend.active = getJdbcTemplate().update( - SET_ACTIVE, depend.getId()) == 1; - return depend.active; - } - - private static final String GET_CHILD_DEPENDS = - "SELECT " + - "depend.pk_depend," + - "depend.str_type," + - "depend.str_target,"+ - "depend.b_any,"+ - "depend.pk_parent,"+ - "depend.b_active," + - "depend.pk_frame_depend_er,"+ - "depend.pk_frame_depend_on,"+ - "depend.pk_layer_depend_er,"+ - "depend.pk_layer_depend_on,"+ - "depend.pk_job_depend_er,"+ - "depend.pk_job_depend_on "+ - "FROM " + - "depend " + - "WHERE " + - "depend.pk_job_depend_er = ? " + - "AND " + - "depend.pk_job_depend_on = ? " + - "AND " + - "depend.pk_parent = ? " + - "AND " + - "depend.b_active = 1 "; - - @Override - public List getChildDepends(LightweightDependency depend) { - return getJdbcTemplate().query(GET_CHILD_DEPENDS, DEPEND_MAPPER, - depend.dependErJobId, depend.dependOnJobId, depend.id); - } - - private static final String GET_WHAT_THIS_JOB_DEPENDS_ON = - "SELECT " + - "depend.pk_depend," + - "depend.str_type," + - "depend.str_target,"+ - "depend.b_any,"+ - "depend.pk_parent,"+ - "depend.b_active," + - "depend.pk_frame_depend_er,"+ - "depend.pk_frame_depend_on,"+ - "depend.pk_layer_depend_er,"+ - "depend.pk_layer_depend_on,"+ - "depend.pk_job_depend_er,"+ - "depend.pk_job_depend_on "+ - "FROM " + - "depend " + - "WHERE " + - "depend.pk_job_depend_er=? " + - "AND " + - "depend.b_active=1 " + - "AND " + - "depend.pk_parent IS NULL "; - - @Override - public List getWhatThisDependsOn(JobInterface job, DependTarget target) { - String query = GET_WHAT_THIS_JOB_DEPENDS_ON; - Object[] values = new Object[] { job.getJobId() }; - if (!target.equals(DependTarget.ANY_TARGET)) { - query = query + " AND depend.str_target = ?"; - values = new Object[] { job.getJobId(), target.toString() }; - } - return getJdbcTemplate().query(query,DEPEND_MAPPER, values); - - } - - private static final String GET_WHAT_THIS_LAYER_DEPENDS_ON = - "SELECT " + - "depend.pk_depend," + - "depend.str_type," + - "depend.str_target,"+ - "depend.b_any,"+ - "depend.pk_parent,"+ - "depend.b_active," + - "depend.pk_frame_depend_er,"+ - "depend.pk_frame_depend_on,"+ - "depend.pk_layer_depend_er,"+ - "depend.pk_layer_depend_on,"+ - "depend.pk_job_depend_er,"+ - "depend.pk_job_depend_on "+ - "FROM " + - "depend " + - "WHERE " + - "depend.pk_layer_depend_er=? " + - "AND " + - "depend.b_active=1 " + - "AND " + - "depend.pk_parent IS NULL " + - "AND " + - "depend.str_type IN (?,?,?,?) "; - - @Override - public List getWhatThisDependsOn(LayerInterface layer, DependTarget target) { - if (!target.equals(DependTarget.ANY_TARGET)) { - String query = GET_WHAT_THIS_LAYER_DEPENDS_ON + " AND str_target = ?"; - return getJdbcTemplate().query(query, DEPEND_MAPPER, - layer.getLayerId(), DependType.LAYER_ON_JOB.toString(), - DependType.LAYER_ON_LAYER.toString(), DependType.LAYER_ON_FRAME.toString(), - DependType.FRAME_BY_FRAME.toString(), target.toString()); - } - else { - return getJdbcTemplate().query(GET_WHAT_THIS_LAYER_DEPENDS_ON, DEPEND_MAPPER, - layer.getLayerId(), DependType.LAYER_ON_JOB.toString(), - DependType.LAYER_ON_LAYER.toString(), DependType.LAYER_ON_FRAME.toString(), - DependType.FRAME_BY_FRAME.toString()); - } - } - - private static final String GET_WHAT_THIS_FRAME_DEPENDS_ON = - "SELECT " + - "depend.pk_depend," + - "depend.str_type," + - "depend.str_target,"+ - "depend.b_any,"+ - "depend.pk_parent,"+ - "depend.b_active," + - "depend.pk_frame_depend_er,"+ - "depend.pk_frame_depend_on,"+ - "depend.pk_layer_depend_er,"+ - "depend.pk_layer_depend_on,"+ - "depend.pk_job_depend_er,"+ - "depend.pk_job_depend_on "+ - "FROM " + - "depend " + - "WHERE " + - "depend.pk_frame_depend_er=? " + - "AND " + - "depend.b_active=1 " + - "AND " + - "depend.str_type IN (?,?,?) "; - - @Override - public List getWhatThisDependsOn(FrameInterface frame, DependTarget target) { - if (!target.equals(DependTarget.ANY_TARGET)) { - String query = GET_WHAT_THIS_FRAME_DEPENDS_ON + " AND depend.str_target = ?"; - return getJdbcTemplate().query(query, DEPEND_MAPPER, - frame.getFrameId(), DependType.FRAME_ON_JOB.toString(), - DependType.FRAME_ON_LAYER.toString(), DependType.FRAME_ON_FRAME.toString(), - target.toString()); - } - else { - return getJdbcTemplate().query(GET_WHAT_THIS_FRAME_DEPENDS_ON, DEPEND_MAPPER, - frame.getFrameId(), DependType.FRAME_ON_JOB.toString(), - DependType.FRAME_ON_LAYER.toString(), DependType.FRAME_ON_FRAME.toString()); - } - } -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DispatchQuery.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DispatchQuery.java deleted file mode 100644 index 1577921d7..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DispatchQuery.java +++ /dev/null @@ -1,1293 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -public class DispatchQuery { - - public static final String FIND_JOBS_BY_SHOW = - "/* FIND_JOBS_BY_SHOW */ " + - "SELECT pk_job, int_priority, rank FROM ( " + - "SELECT " + - "ROW_NUMBER() OVER (ORDER BY job_resource.int_priority DESC) AS rank, " + - "job.pk_job, " + - "job_resource.int_priority " + - "FROM " + - "job , " + - "job_resource , " + - "folder , " + - "folder_resource, " + - "point , " + - "layer , " + - "layer_stat , " + - "host " + - "WHERE " + - "job.pk_job = job_resource.pk_job " + - "AND job.pk_folder = folder.pk_folder " + - "AND folder.pk_folder = folder_resource.pk_folder " + - "AND folder.pk_dept = point.pk_dept " + - "AND folder.pk_show = point.pk_show " + - "AND job.pk_job = layer.pk_job " + - "AND job_resource.pk_job = job.pk_job " + - "AND (CASE WHEN layer_stat.int_waiting_count > 0 THEN layer_stat.pk_layer ELSE NULL END) = layer.pk_layer " + - "AND " + - "(" + - "folder_resource.int_max_cores = -1 " + - "OR " + - "folder_resource.int_cores < folder_resource.int_max_cores " + - ") " + - "AND job.str_state = 'PENDING' " + - "AND job.b_paused = 0 " + - "AND job.pk_show = ? " + - "AND job.pk_facility = ? " + - "AND (job.str_os = ? OR job.str_os IS NULL)" + - "AND (CASE WHEN layer_stat.int_waiting_count > 0 THEN 1 ELSE NULL END) = 1 " + - "AND layer.int_cores_min <= ? " + - "AND layer.int_mem_min <= ? " + - "AND layer.b_threadable >= ? " + - "AND layer.int_gpu_min BETWEEN ? AND ? " + - "AND job_resource.int_cores + layer.int_cores_min < job_resource.int_max_cores " + - "AND CATSEARCH(host.str_tags, layer.str_tags, ?) > 0 " + - "AND layer.pk_layer IN (" + - "SELECT " + - "l.pk_layer " + - "FROM " + - "layer l " + - "LEFT JOIN layer_limit ON layer_limit.pk_layer = l.pk_layer " + - "LEFT JOIN limit_record ON limit_record.pk_limit_record = layer_limit.pk_limit_record " + - "LEFT JOIN (" + - "SELECT " + - "limit_record.pk_limit_record, " + - "SUM(layer_stat.int_running_count) AS int_sum_running " + - "FROM " + - "layer_limit " + - "LEFT JOIN limit_record ON layer_limit.pk_limit_record = limit_record.pk_limit_record " + - "LEFT JOIN layer_stat ON layer_stat.pk_layer = layer_limit.pk_layer " + - "GROUP BY limit_record.pk_limit_record) AS sum_running " + - "ON limit_record.pk_limit_record = sum_running.pk_limit_record " + - "WHERE " + - "sum_running.int_sum_running < limit_record.int_max_value " + - "OR sum_running.int_sum_running IS NULL " + - ") " + - ") WHERE rank < ?"; - - - public static final String FIND_JOBS_BY_GROUP = - FIND_JOBS_BY_SHOW - .replace( - "FIND_JOBS_BY_SHOW", - "FIND_JOBS_BY_GROUP") - .replace( - "AND job.pk_show = ? ", - "AND job.pk_folder = ? "); - - - /** - * Dispatch a host in local booking mode. - */ - public static final String FIND_JOBS_BY_LOCAL = - "/* FIND_JOBS_BY_LOCAL */ SELECT pk_job,float_tier, rank FROM ( " + - "SELECT " + - "ROW_NUMBER() OVER (ORDER BY " + - "host_local.float_tier ASC " + - ") AS rank, " + - "job.pk_job, " + - "host_local.float_tier " + - "FROM " + - "job, " + - "host_local " + - "WHERE " + - "job.pk_job = host_local.pk_job " + - "AND " + - "host_local.pk_host = ? " + - "AND " + - "job.str_state = 'PENDING' " + - "AND " + - "job.b_paused = 0 " + - "AND " + - "job.pk_facility = ? " + - "AND " + - "(job.str_os = ? OR job.str_os IS NULL)" + - "AND " + - "job.pk_job IN ( " + - "SELECT " + - "l.pk_job " + - "FROM " + - "job j, " + - "layer l, " + - "layer_stat lst, " + - "host h, " + - "host_local " + - "WHERE " + - "j.pk_job = l.pk_job " + - "AND " + - "j.pk_job = host_local.pk_job " + - "AND " + - "h.pk_host = host_local.pk_host " + - "AND " + - "h.pk_host = ? " + - "AND " + - "j.str_state = 'PENDING' " + - "AND " + - "j.b_paused = 0 " + - "AND " + - "j.pk_facility = ? " + - "AND " + - "(j.str_os = ? OR j.str_os IS NULL)" + - "AND " + - "(CASE WHEN lst.int_waiting_count > 0 THEN lst.pk_layer ELSE NULL END) = l.pk_layer " + - "AND " + - "(CASE WHEN lst.int_waiting_count > 0 THEN 1 ELSE NULL END) = 1 " + - "AND " + - "l.int_mem_min <= host_local.int_mem_idle " + - "AND " + - "l.int_gpu_min <= host_local.int_gpu_idle " + - "AND " + - "l.pk_layer IN (" + - "SELECT " + - "la.pk_layer " + - "FROM " + - "layer la " + - "LEFT JOIN layer_limit ON layer_limit.pk_layer = la.pk_layer " + - "LEFT JOIN limit_record ON limit_record.pk_limit_record = layer_limit.pk_limit_record " + - "LEFT JOIN (" + - "SELECT " + - "limit_record.pk_limit_record, " + - "SUM(layer_stat.int_running_count) AS int_sum_running " + - "FROM " + - "layer_limit " + - "LEFT JOIN limit_record ON layer_limit.pk_limit_record = limit_record.pk_limit_record " + - "LEFT JOIN layer_stat ON layer_stat.pk_layer = layer_limit.pk_layer " + - "GROUP BY limit_record.pk_limit_record) AS sum_running " + - "ON limit_record.pk_limit_record = sum_running.pk_limit_record " + - "WHERE " + - "sum_running.int_sum_running < limit_record.int_max_value " + - "OR sum_running.int_sum_running IS NULL " + - ") " + - ") " + - ") WHERE rank < 5"; - - /** - * This query is run before a proc is dispatched to the next frame. - * It checks to see if there is another job someplace that is - * under its minimum and can take the proc. - * - * The current job the proc is on is excluded. This should only be run - * if the excluded job is actually over its min proc. - * - * Does not unbook for Utility frames - * - */ - public static final String FIND_UNDER_PROCED_JOB_BY_FACILITY = - "SELECT " + - "1 " + - "FROM " + - "job, " + - "job_resource, " + - "folder, " + - "folder_resource " + - "WHERE " + - "job.pk_job = job_resource.pk_job " + - "AND " + - "job.pk_folder = folder.pk_folder " + - "AND " + - "folder.pk_folder = folder_resource.pk_folder " + - "AND " + - "(folder_resource.int_max_cores = -1 OR folder_resource.int_cores < folder_resource.int_max_cores) " + - "AND " + - "job_resource.float_tier < 1.00 " + - "AND " + - "job_resource.int_cores < job_resource.int_min_cores " + - "AND " + - "job.str_state = 'PENDING' " + - "AND " + - "job.b_paused = 0 " + - "AND " + - "job.pk_show = ? " + - "AND " + - "job.pk_facility = ? " + - "AND " + - "(job.str_os = ? OR job.str_os IS NULL)" + - "AND " + - "job.pk_job IN ( " + - "SELECT /* index (h i_str_host_tag) */ " + - "l.pk_job " + - "FROM " + - "job j, " + - "layer l, " + - "layer_stat lst, " + - "host h " + - "WHERE " + - "j.pk_job = l.pk_job " + - "AND " + - "j.str_state = 'PENDING' " + - "AND " + - "j.b_paused = 0 " + - "AND " + - "j.pk_show = ? " + - "AND " + - "j.pk_facility = ? " + - "AND " + - "(j.str_os = ? OR j.str_os IS NULL)" + - "AND " + - "(CASE WHEN lst.int_waiting_count > 0 THEN lst.pk_layer ELSE NULL END) = l.pk_layer " + - "AND " + - "(CASE WHEN lst.int_waiting_count > 0 THEN 1 ELSE NULL END) = 1 " + - "AND " + - "l.int_cores_min <= ? " + - "AND " + - "l.int_mem_min <= ? " + - "AND " + - "l.int_gpu_min = ? " + - "AND " + - "CATSEARCH(h.str_tags, l.str_tags, ?) > 0" + - "AND " + - "l.pk_layer IN (" + - "SELECT " + - "la.pk_layer " + - "FROM " + - "layer la " + - "LEFT JOIN layer_limit ON layer_limit.pk_layer = la.pk_layer " + - "LEFT JOIN limit_record ON limit_record.pk_limit_record = layer_limit.pk_limit_record " + - "LEFT JOIN (" + - "SELECT " + - "limit_record.pk_limit_record, " + - "SUM(layer_stat.int_running_count) AS int_sum_running " + - "FROM " + - "layer_limit " + - "LEFT JOIN limit_record ON layer_limit.pk_limit_record = limit_record.pk_limit_record " + - "LEFT JOIN layer_stat ON layer_stat.pk_layer = layer_limit.pk_layer " + - "GROUP BY limit_record.pk_limit_record) AS sum_running " + - "ON limit_record.pk_limit_record = sum_running.pk_limit_record " + - "WHERE " + - "sum_running.int_sum_running < limit_record.int_max_value " + - "OR sum_running.int_sum_running IS NULL " + - ") " + - ") " + - "AND ROWNUM < 2 "; - - /** - * This query is run before a proc is dispatched to the next frame. - * It checks to see if there is another job someplace that is - * at a higher priority and can take the proc. - * - * The current job the proc is on is excluded. This should only be run - * if the excluded job is actually over its min proc. - * - * Does not unbook for Utility frames - * - */ - public static final String HIGHER_PRIORITY_JOB_BY_FACILITY_EXISTS = - "SELECT " + - "1 " + - "FROM " + - "job, " + - "job_resource, " + - "folder, " + - "folder_resource " + - "WHERE " + - "job.pk_job = job_resource.pk_job " + - "AND " + - "job.pk_folder = folder.pk_folder " + - "AND " + - "folder.pk_folder = folder_resource.pk_folder " + - "AND " + - "(folder_resource.int_max_cores = -1 OR folder_resource.int_cores < folder_resource.int_max_cores) " + - "AND " + - "job_resource.int_priority > ?" + - "AND " + - "job_resource.int_cores < job_resource.int_max_cores " + - "AND " + - "job.str_state = 'PENDING' " + - "AND " + - "job.b_paused = 0 " + - "AND " + - "job.pk_facility = ? " + - "AND " + - "(job.str_os = ? OR job.str_os IS NULL)" + - "AND " + - "job.pk_job IN ( " + - "SELECT /* index (h i_str_host_tag) */ " + - "l.pk_job " + - "FROM " + - "job j, " + - "layer l, " + - "layer_stat lst, " + - "host h " + - "WHERE " + - "j.pk_job = l.pk_job " + - "AND " + - "j.str_state = 'PENDING' " + - "AND " + - "j.b_paused = 0 " + - "AND " + - "j.pk_facility = ? " + - "AND " + - "(j.str_os = ? OR j.str_os IS NULL)" + - "AND " + - "(CASE WHEN lst.int_waiting_count > 0 THEN lst.pk_layer ELSE NULL END) = l.pk_layer " + - "AND " + - "(CASE WHEN lst.int_waiting_count > 0 THEN 1 ELSE NULL END) = 1 " + - "AND " + - "l.int_cores_min <= ? " + - "AND " + - "l.int_mem_min <= ? " + - "AND " + - "l.int_gpu_min = ? " + - "AND " + - "CATSEARCH(h.str_tags, l.str_tags, ?) > 0" + - "AND " + - "l.pk_layer IN (" + - "SELECT " + - "la.pk_layer " + - "FROM " + - "layer la " + - "LEFT JOIN layer_limit ON layer_limit.pk_layer = la.pk_layer " + - "LEFT JOIN limit_record ON limit_record.pk_limit_record = layer_limit.pk_limit_record " + - "LEFT JOIN (" + - "SELECT " + - "limit_record.pk_limit_record, " + - "SUM(layer_stat.int_running_count) AS int_sum_running " + - "FROM " + - "layer_limit " + - "LEFT JOIN limit_record ON layer_limit.pk_limit_record = limit_record.pk_limit_record " + - "LEFT JOIN layer_stat ON layer_stat.pk_layer = layer_limit.pk_layer " + - "GROUP BY limit_record.pk_limit_record) AS sum_running " + - "ON limit_record.pk_limit_record = sum_running.pk_limit_record " + - "WHERE " + - "sum_running.int_sum_running < limit_record.int_max_value " + - "OR sum_running.int_sum_running IS NULL " + - ") " + - ") " + - "AND ROWNUM < 2 "; - - /** - * Finds the next frame in a job for a proc. - */ - public static final String FIND_DISPATCH_FRAME_BY_JOB_AND_PROC = - "SELECT "+ - "show_name, "+ - "job_name, " + - "pk_job,"+ - "pk_show,"+ - "pk_facility,"+ - "str_name,"+ - "str_shot,"+ - "str_user,"+ - "int_uid,"+ - "str_log_dir,"+ - "frame_name, "+ - "frame_state, "+ - "pk_frame, "+ - "pk_layer, "+ - "int_retries, "+ - "int_version, " + - "layer_name, " + - "layer_type, "+ - "b_threadable,"+ - "int_cores_min,"+ - "int_cores_max,"+ - "int_mem_min,"+ - "int_gpu_min,"+ - "str_cmd, "+ - "str_range,"+ - "int_chunk_size, "+ - "str_services " + - "FROM (SELECT " + - "ROW_NUMBER() OVER ( ORDER BY " + - "frame.int_dispatch_order ASC, " + - "frame.int_layer_order ASC " + - ") LINENUM, " + - "job.str_show AS show_name, "+ - "job.str_name AS job_name, " + - "job.pk_job,"+ - "job.pk_show,"+ - "job.pk_facility,"+ - "job.str_name,"+ - "job.str_shot,"+ - "job.str_user,"+ - "job.int_uid,"+ - "job.str_log_dir,"+ - "frame.str_name AS frame_name, "+ - "frame.str_state AS frame_state, "+ - "frame.pk_frame, "+ - "frame.pk_layer, "+ - "frame.int_retries, "+ - "frame.int_version, " + - "layer.str_name AS layer_name, " + - "layer.str_type AS layer_type, "+ - "layer.b_threadable,"+ - "layer.int_cores_min,"+ - "layer.int_cores_max,"+ - "layer.int_mem_min,"+ - "layer.int_gpu_min,"+ - "layer.str_cmd, "+ - "layer.str_range, "+ - "layer.int_chunk_size, "+ - "layer.str_services " + - "FROM " + - "job,"+ - "frame," + - "layer " + - "WHERE " + - "frame.pk_layer = layer.pk_layer " + - "AND " + - "layer.pk_job = job.pk_job " + - "AND " + - "layer.int_cores_min <= ? " + - "AND " + - "layer.int_mem_min <= ? " + - "AND " + - "layer.int_gpu_min BETWEEN ? AND ? " + - "AND " + - "frame.str_state='WAITING' " + - "AND " + - "job.pk_job=? "+ - "AND layer.pk_layer IN ( " + - "SELECT /*+ index (h i_str_host_tag) */ " + - "pk_layer " + - "FROM " + - "layer l,"+ - "host h " + - "WHERE " + - "l.pk_job= ? " + - "AND " + - "CATSEARCH(h.str_tags, l.str_tags, ?) > 0 "+ - ") " + - "AND " + - "layer.pk_layer IN (" + - "SELECT " + - "la.pk_layer " + - "FROM " + - "layer la " + - "LEFT JOIN layer_limit ON layer_limit.pk_layer = la.pk_layer " + - "LEFT JOIN limit_record ON limit_record.pk_limit_record = layer_limit.pk_limit_record " + - "LEFT JOIN (" + - "SELECT " + - "limit_record.pk_limit_record, " + - "SUM(layer_stat.int_running_count) AS int_sum_running " + - "FROM " + - "layer_limit " + - "LEFT JOIN limit_record ON layer_limit.pk_limit_record = limit_record.pk_limit_record " + - "LEFT JOIN layer_stat ON layer_stat.pk_layer = layer_limit.pk_layer " + - "GROUP BY limit_record.pk_limit_record) AS sum_running " + - "ON limit_record.pk_limit_record = sum_running.pk_limit_record " + - "WHERE " + - "sum_running.int_sum_running < limit_record.int_max_value " + - "OR sum_running.int_sum_running IS NULL " + - ") " + - ") WHERE LINENUM <= ?"; - - /** - * Find the next frame in a job for a host. - */ - public static final String FIND_DISPATCH_FRAME_BY_JOB_AND_HOST = - "SELECT " + - "show_name, "+ - "job_name, " + - "pk_job,"+ - "pk_show,"+ - "pk_facility,"+ - "str_name,"+ - "str_shot,"+ - "str_user,"+ - "int_uid,"+ - "str_log_dir,"+ - "frame_name, "+ - "frame_state, "+ - "pk_frame, "+ - "pk_layer, "+ - "int_retries, "+ - "int_version, " + - "layer_name, " + - "layer_type, "+ - "int_cores_min,"+ - "int_cores_max,"+ - "b_threadable,"+ - "int_mem_min,"+ - "int_gpu_min,"+ - "str_cmd, "+ - "str_range,"+ - "int_chunk_size, "+ - "str_services " + - "FROM (SELECT " + - "ROW_NUMBER() OVER ( ORDER BY " + - "frame.int_dispatch_order ASC, " + - "frame.int_layer_order ASC " + - ") LINENUM, " + - "job.str_show AS show_name, "+ - "job.str_name AS job_name, " + - "job.pk_job,"+ - "job.pk_show,"+ - "job.pk_facility,"+ - "job.str_name,"+ - "job.str_shot,"+ - "job.str_user,"+ - "job.int_uid,"+ - "job.str_log_dir,"+ - "frame.str_name AS frame_name, "+ - "frame.str_state AS frame_state, "+ - "frame.pk_frame, "+ - "frame.pk_layer, "+ - "frame.int_retries, "+ - "frame.int_version, "+ - "layer.str_name AS layer_name, " + - "layer.str_type AS layer_type, "+ - "layer.int_cores_min,"+ - "layer.int_cores_max,"+ - "layer.b_threadable,"+ - "layer.int_mem_min,"+ - "layer.int_gpu_min,"+ - "layer.str_cmd, "+ - "layer.str_range, "+ - "layer.int_chunk_size, "+ - "layer.str_services " + - "FROM " + - "job,"+ - "frame," + - "layer " + - "WHERE " + - "frame.pk_layer = layer.pk_layer " + - "AND " + - "layer.pk_job = job.pk_job " + - "AND " + - "layer.int_cores_min <= ? " + - "AND " + - "layer.int_mem_min <= ? " + - "AND " + - "layer.b_threadable >= ? " + - "AND " + - "layer.int_gpu_min BETWEEN ? AND ? " + - "AND " + - "frame.str_state='WAITING' " + - "AND " + - "job.pk_job=? "+ - "AND " + - "layer.pk_layer IN ( " + - "SELECT /*+ index (h i_str_host_tag) */ " + - "pk_layer " + - "FROM " + - "layer l,"+ - "host h " + - "WHERE " + - "l.pk_job=? " + - "AND " + - "CATSEARCH(h.str_tags, l.str_tags,?) > 0 "+ - ") " + - "AND " + - "layer.pk_layer IN (" + - "SELECT " + - "la.pk_layer " + - "FROM " + - "layer la " + - "LEFT JOIN layer_limit ON layer_limit.pk_layer = la.pk_layer " + - "LEFT JOIN limit_record ON limit_record.pk_limit_record = layer_limit.pk_limit_record " + - "LEFT JOIN (" + - "SELECT " + - "limit_record.pk_limit_record, " + - "SUM(layer_stat.int_running_count) AS int_sum_running " + - "FROM " + - "layer_limit " + - "LEFT JOIN limit_record ON layer_limit.pk_limit_record = limit_record.pk_limit_record " + - "LEFT JOIN layer_stat ON layer_stat.pk_layer = layer_limit.pk_layer " + - "GROUP BY limit_record.pk_limit_record) AS sum_running " + - "ON limit_record.pk_limit_record = sum_running.pk_limit_record " + - "WHERE " + - "sum_running.int_sum_running < limit_record.int_max_value " + - "OR sum_running.int_sum_running IS NULL " + - ") " + - ") WHERE LINENUM <= ?"; - - - public static final String FIND_LOCAL_DISPATCH_FRAME_BY_JOB_AND_PROC = - "SELECT "+ - "show_name, "+ - "job_name, " + - "pk_job,"+ - "pk_show,"+ - "pk_facility,"+ - "str_name,"+ - "str_shot,"+ - "str_user,"+ - "int_uid,"+ - "str_log_dir,"+ - "frame_name, "+ - "frame_state, "+ - "pk_frame, "+ - "pk_layer, "+ - "int_retries, "+ - "int_version, " + - "layer_name, " + - "layer_type, "+ - "b_threadable,"+ - "int_cores_min,"+ - "int_cores_max,"+ - "int_mem_min,"+ - "int_gpu_min,"+ - "str_cmd, "+ - "str_range,"+ - "int_chunk_size, "+ - "str_services " + - "FROM (SELECT " + - "ROW_NUMBER() OVER ( ORDER BY " + - "frame.int_dispatch_order ASC, " + - "frame.int_layer_order ASC " + - ") LINENUM, " + - "job.str_show AS show_name, "+ - "job.str_name AS job_name, " + - "job.pk_job,"+ - "job.pk_show,"+ - "job.pk_facility,"+ - "job.str_name,"+ - "job.str_shot,"+ - "job.str_user,"+ - "job.int_uid,"+ - "job.str_log_dir,"+ - "frame.str_name AS frame_name, "+ - "frame.str_state AS frame_state, "+ - "frame.pk_frame, "+ - "frame.pk_layer, "+ - "frame.int_retries, "+ - "frame.int_version, " + - "layer.str_name AS layer_name, " + - "layer.str_type AS layer_type, "+ - "layer.b_threadable,"+ - "layer.int_cores_min,"+ - "layer.int_cores_max,"+ - "layer.int_mem_min,"+ - "layer.int_gpu_min,"+ - "layer.str_cmd, "+ - "layer.str_range, "+ - "layer.int_chunk_size, "+ - "layer.str_services " + - "FROM " + - "job,"+ - "frame," + - "layer " + - "WHERE " + - "frame.pk_layer = layer.pk_layer " + - "AND " + - "layer.pk_job = job.pk_job " + - "AND " + - "layer.int_mem_min <= ? " + - "AND " + - "layer.int_gpu_min <= ? " + - "AND " + - "frame.str_state='WAITING' " + - "AND " + - "job.pk_job=? "+ - "AND " + - "layer.pk_layer IN (" + - "SELECT " + - "la.pk_layer " + - "FROM " + - "layer la " + - "LEFT JOIN layer_limit ON layer_limit.pk_layer = la.pk_layer " + - "LEFT JOIN limit_record ON limit_record.pk_limit_record = layer_limit.pk_limit_record " + - "LEFT JOIN (" + - "SELECT " + - "limit_record.pk_limit_record, " + - "SUM(layer_stat.int_running_count) AS int_sum_running " + - "FROM " + - "layer_limit " + - "LEFT JOIN limit_record ON layer_limit.pk_limit_record = limit_record.pk_limit_record " + - "LEFT JOIN layer_stat ON layer_stat.pk_layer = layer_limit.pk_layer " + - "GROUP BY limit_record.pk_limit_record) AS sum_running " + - "ON limit_record.pk_limit_record = sum_running.pk_limit_record " + - "WHERE " + - "sum_running.int_sum_running < limit_record.int_max_value " + - "OR sum_running.int_sum_running IS NULL " + - ") " + - ") WHERE LINENUM <= ?"; - - /** - * Find the next frame in a job for a host. - */ - public static final String FIND_LOCAL_DISPATCH_FRAME_BY_JOB_AND_HOST = - "SELECT " + - "show_name, "+ - "job_name, " + - "pk_job,"+ - "pk_show,"+ - "pk_facility,"+ - "str_name,"+ - "str_shot,"+ - "str_user,"+ - "int_uid,"+ - "str_log_dir,"+ - "frame_name, "+ - "frame_state, "+ - "pk_frame, "+ - "pk_layer, "+ - "int_retries, "+ - "int_version, " + - "layer_name, " + - "layer_type, "+ - "int_cores_min,"+ - "int_cores_max,"+ - "b_threadable,"+ - "int_mem_min,"+ - "int_gpu_min,"+ - "str_cmd, "+ - "str_range,"+ - "int_chunk_size, "+ - "str_services " + - "FROM (SELECT " + - "ROW_NUMBER() OVER ( ORDER BY " + - "frame.int_dispatch_order ASC, " + - "frame.int_layer_order ASC " + - ") LINENUM, " + - "job.str_show AS show_name, "+ - "job.str_name AS job_name, " + - "job.pk_job,"+ - "job.pk_show,"+ - "job.pk_facility,"+ - "job.str_name,"+ - "job.str_shot,"+ - "job.str_user,"+ - "job.int_uid,"+ - "job.str_log_dir,"+ - "frame.str_name AS frame_name, "+ - "frame.str_state AS frame_state, "+ - "frame.pk_frame, "+ - "frame.pk_layer, "+ - "frame.int_retries, "+ - "frame.int_version, "+ - "layer.str_name AS layer_name, " + - "layer.str_type AS layer_type, "+ - "layer.int_cores_min,"+ - "layer.int_cores_max,"+ - "layer.b_threadable,"+ - "layer.int_mem_min,"+ - "layer.int_gpu_min,"+ - "layer.str_cmd, "+ - "layer.str_range, "+ - "layer.int_chunk_size, "+ - "layer.str_services " + - "FROM " + - "job,"+ - "frame," + - "layer " + - "WHERE " + - "frame.pk_layer = layer.pk_layer " + - "AND " + - "layer.pk_job = job.pk_job " + - "AND " + - "layer.int_mem_min <= ? " + - "AND " + - "layer.int_gpu_min <= ? " + - "AND " + - "frame.str_state='WAITING' " + - "AND " + - "job.pk_job=? "+ - "AND " + - "layer.pk_layer IN (" + - "SELECT " + - "la.pk_layer " + - "FROM " + - "layer la " + - "LEFT JOIN layer_limit ON layer_limit.pk_layer = la.pk_layer " + - "LEFT JOIN limit_record ON limit_record.pk_limit_record = layer_limit.pk_limit_record " + - "LEFT JOIN (" + - "SELECT " + - "limit_record.pk_limit_record, " + - "SUM(layer_stat.int_running_count) AS int_sum_running " + - "FROM " + - "layer_limit " + - "LEFT JOIN limit_record ON layer_limit.pk_limit_record = limit_record.pk_limit_record " + - "LEFT JOIN layer_stat ON layer_stat.pk_layer = layer_limit.pk_layer " + - "GROUP BY limit_record.pk_limit_record) AS sum_running " + - "ON limit_record.pk_limit_record = sum_running.pk_limit_record " + - "WHERE " + - "sum_running.int_sum_running < limit_record.int_max_value " + - "OR sum_running.int_sum_running IS NULL " + - ") " + - ") WHERE LINENUM <= ?"; - - - /**** LAYER DISPATCHING **/ - - /** - * Finds the next frame in a job for a proc. - */ - public static final String FIND_DISPATCH_FRAME_BY_LAYER_AND_PROC = - - "SELECT "+ - "show_name, "+ - "job_name, " + - "pk_job,"+ - "pk_show,"+ - "pk_facility,"+ - "str_name,"+ - "str_shot,"+ - "str_user,"+ - "int_uid,"+ - "str_log_dir,"+ - "frame_name, "+ - "frame_state, "+ - "pk_frame, "+ - "pk_layer, "+ - "int_retries, "+ - "int_version, " + - "layer_name, " + - "layer_type, "+ - "b_threadable,"+ - "int_cores_min,"+ - "int_cores_max,"+ - "int_mem_min,"+ - "int_gpu_min,"+ - "str_cmd, "+ - "str_range,"+ - "int_chunk_size, "+ - "str_services " + - "FROM (SELECT " + - "ROW_NUMBER() OVER ( ORDER BY " + - "frame.int_dispatch_order ASC, " + - "frame.int_layer_order ASC " + - ") LINENUM, " + - "job.str_show AS show_name, "+ - "job.str_name AS job_name, " + - "job.pk_job,"+ - "job.pk_show,"+ - "job.pk_facility,"+ - "job.str_name,"+ - "job.str_shot,"+ - "job.str_user,"+ - "job.int_uid,"+ - "job.str_log_dir,"+ - "frame.str_name AS frame_name, "+ - "frame.str_state AS frame_state, "+ - "frame.pk_frame, "+ - "frame.pk_layer, "+ - "frame.int_retries, "+ - "frame.int_version, " + - "layer.str_name AS layer_name, " + - "layer.str_type AS layer_type, "+ - "layer.b_threadable,"+ - "layer.int_cores_min,"+ - "layer.int_cores_max,"+ - "layer.int_mem_min,"+ - "layer.int_gpu_min,"+ - "layer.str_cmd, "+ - "layer.str_range, "+ - "layer.int_chunk_size, "+ - "layer.str_services " + - "FROM " + - "job,"+ - "frame," + - "layer " + - "WHERE " + - "frame.pk_layer = layer.pk_layer " + - "AND " + - "layer.pk_job = job.pk_job " + - "AND " + - "layer.int_cores_min <= ? " + - "AND " + - "layer.int_mem_min <= ? " + - "AND " + - "layer.int_gpu_min = ? " + - "AND " + - "frame.str_state='WAITING' " + - "AND " + - "job.pk_layer=? "+ - "AND layer.pk_layer IN ( " + - "SELECT /*+ index (h i_str_host_tag) */ " + - "pk_layer " + - "FROM " + - "layer l,"+ - "host h " + - "WHERE " + - "l.pk_layer= ? " + - "AND " + - "CATSEARCH(h.str_tags, l.str_tags, ?) > 0 "+ - ") " + - "AND layer.pk_layer IN (" + - "SELECT " + - "la.pk_layer " + - "FROM " + - "layer la " + - "LEFT JOIN layer_limit ON layer_limit.pk_layer = la.pk_layer " + - "LEFT JOIN limit_record ON limit_record.pk_limit_record = layer_limit.pk_limit_record " + - "LEFT JOIN (" + - "SELECT " + - "limit_record.pk_limit_record, " + - "SUM(layer_stat.int_running_count) AS int_sum_running " + - "FROM " + - "layer_limit " + - "LEFT JOIN limit_record ON layer_limit.pk_limit_record = limit_record.pk_limit_record " + - "LEFT JOIN layer_stat ON layer_stat.pk_layer = layer_limit.pk_layer " + - "GROUP BY limit_record.pk_limit_record) AS sum_running " + - "ON limit_record.pk_limit_record = sum_running.pk_limit_record " + - "WHERE " + - "sum_running.int_sum_running < limit_record.int_max_value " + - "OR sum_running.int_sum_running IS NULL) " + - ") WHERE LINENUM <= ?"; - - /** - * Find the next frame in a job for a host. - */ - public static final String FIND_DISPATCH_FRAME_BY_LAYER_AND_HOST = - "SELECT " + - "show_name, "+ - "job_name, " + - "pk_job,"+ - "pk_show,"+ - "pk_facility,"+ - "str_name,"+ - "str_shot,"+ - "str_user,"+ - "int_uid,"+ - "str_log_dir,"+ - "frame_name, "+ - "frame_state, "+ - "pk_frame, "+ - "pk_layer, "+ - "int_retries, "+ - "int_version, " + - "layer_name, " + - "layer_type, "+ - "int_cores_min,"+ - "int_cores_max,"+ - "b_threadable,"+ - "int_mem_min,"+ - "int_gpu_min,"+ - "str_cmd, "+ - "str_range,"+ - "int_chunk_size, "+ - "str_services " + - "FROM (SELECT " + - "ROW_NUMBER() OVER ( ORDER BY " + - "frame.int_dispatch_order ASC, " + - "frame.int_layer_order ASC " + - ") LINENUM, " + - "job.str_show AS show_name, "+ - "job.str_name AS job_name, " + - "job.pk_job,"+ - "job.pk_show,"+ - "job.pk_facility,"+ - "job.str_name,"+ - "job.str_shot,"+ - "job.str_user,"+ - "job.int_uid,"+ - "job.str_log_dir,"+ - "frame.str_name AS frame_name, "+ - "frame.str_state AS frame_state, "+ - "frame.pk_frame, "+ - "frame.pk_layer, "+ - "frame.int_retries, "+ - "frame.int_version, "+ - "layer.str_name AS layer_name, " + - "layer.str_type AS layer_type, "+ - "layer.int_cores_min,"+ - "layer.int_cores_max,"+ - "layer.b_threadable,"+ - "layer.int_mem_min,"+ - "layer.int_gpu_min,"+ - "layer.str_cmd, "+ - "layer.str_range, "+ - "layer.int_chunk_size, "+ - "layer.str_services " + - "FROM " + - "job,"+ - "frame," + - "layer " + - "WHERE " + - "frame.pk_layer = layer.pk_layer " + - "AND " + - "layer.pk_job = job.pk_job " + - "AND " + - "layer.int_cores_min <= ? " + - "AND " + - "layer.int_mem_min <= ? " + - "AND " + - "layer.b_threadable >= ? " + - "AND " + - "layer.int_gpu_min <= ? " + - "AND " + - "frame.str_state='WAITING' " + - "AND " + - "layer.pk_layer=? "+ - "AND " + - "layer.pk_layer IN ( " + - "SELECT /*+ index (h i_str_host_tag) */ " + - "pk_layer " + - "FROM " + - "layer l,"+ - "host h " + - "WHERE " + - "l.pk_layer=? " + - "AND " + - "CATSEARCH(h.str_tags, l.str_tags,?) > 0 "+ - ") " + - "AND " + - "layer.pk_layer IN (" + - "SELECT " + - "la.pk_layer " + - "FROM " + - "layer la " + - "LEFT JOIN layer_limit ON layer_limit.pk_layer = la.pk_layer " + - "LEFT JOIN limit_record ON limit_record.pk_limit_record = layer_limit.pk_limit_record " + - "LEFT JOIN (" + - "SELECT " + - "limit_record.pk_limit_record, " + - "SUM(layer_stat.int_running_count) AS int_sum_running " + - "FROM " + - "layer_limit " + - "LEFT JOIN limit_record ON layer_limit.pk_limit_record = limit_record.pk_limit_record " + - "LEFT JOIN layer_stat ON layer_stat.pk_layer = layer_limit.pk_layer " + - "GROUP BY limit_record.pk_limit_record) AS sum_running " + - "ON limit_record.pk_limit_record = sum_running.pk_limit_record " + - "WHERE " + - "sum_running.int_sum_running < limit_record.int_max_value " + - "OR sum_running.int_sum_running IS NULL " + - ") " + - ") WHERE LINENUM <= ?"; - - - public static final String FIND_LOCAL_DISPATCH_FRAME_BY_LAYER_AND_PROC = - "SELECT "+ - "show_name, "+ - "job_name, " + - "pk_job,"+ - "pk_show,"+ - "pk_facility,"+ - "str_name,"+ - "str_shot,"+ - "str_user,"+ - "int_uid,"+ - "str_log_dir,"+ - "frame_name, "+ - "frame_state, "+ - "pk_frame, "+ - "pk_layer, "+ - "int_retries, "+ - "int_version, " + - "layer_name, " + - "layer_type, "+ - "b_threadable,"+ - "int_cores_min,"+ - "int_cores_max,"+ - "int_mem_min,"+ - "int_gpu_min,"+ - "str_cmd, "+ - "str_range,"+ - "int_chunk_size, "+ - "str_services " + - "FROM (SELECT " + - "ROW_NUMBER() OVER ( ORDER BY " + - "frame.int_dispatch_order ASC, " + - "frame.int_layer_order ASC " + - ") LINENUM, " + - "job.str_show AS show_name, "+ - "job.str_name AS job_name, " + - "job.pk_job,"+ - "job.pk_show,"+ - "job.pk_facility,"+ - "job.str_name,"+ - "job.str_shot,"+ - "job.str_user,"+ - "job.int_uid,"+ - "job.str_log_dir,"+ - "frame.str_name AS frame_name, "+ - "frame.str_state AS frame_state, "+ - "frame.pk_frame, "+ - "frame.pk_layer, "+ - "frame.int_retries, "+ - "frame.int_version, " + - "layer.str_name AS layer_name, " + - "layer.str_type AS layer_type, "+ - "layer.b_threadable,"+ - "layer.int_cores_min,"+ - "layer.int_mem_min,"+ - "layer.int_gpu_min,"+ - "layer.int_cores_max,"+ - "layer.str_cmd, "+ - "layer.str_range, "+ - "layer.int_chunk_size, "+ - "layer.str_services " + - "FROM " + - "job,"+ - "frame," + - "layer " + - "WHERE " + - "frame.pk_layer = layer.pk_layer " + - "AND " + - "layer.pk_job = job.pk_job " + - "AND " + - "layer.int_mem_min <= ? " + - "AND " + - "layer.int_gpu_min <= ? " + - "AND " + - "frame.str_state='WAITING' " + - "AND " + - "layer.pk_layer =? "+ - "AND " + - "layer.pk_layer IN (" + - "SELECT " + - "la.pk_layer " + - "FROM " + - "layer la " + - "LEFT JOIN layer_limit ON layer_limit.pk_layer = la.pk_layer " + - "LEFT JOIN limit_record ON limit_record.pk_limit_record = layer_limit.pk_limit_record " + - "LEFT JOIN (" + - "SELECT " + - "limit_record.pk_limit_record, " + - "SUM(layer_stat.int_running_count) AS int_sum_running " + - "FROM " + - "layer_limit " + - "LEFT JOIN limit_record ON layer_limit.pk_limit_record = limit_record.pk_limit_record " + - "LEFT JOIN layer_stat ON layer_stat.pk_layer = layer_limit.pk_layer " + - "GROUP BY limit_record.pk_limit_record) AS sum_running " + - "ON limit_record.pk_limit_record = sum_running.pk_limit_record " + - "WHERE " + - "sum_running.int_sum_running < limit_record.int_max_value " + - "OR sum_running.int_sum_running IS NULL " + - ") " + - ") WHERE LINENUM <= ?"; - - /** - * Find the next frame in a job for a host. - */ - public static final String FIND_LOCAL_DISPATCH_FRAME_BY_LAYER_AND_HOST = - "SELECT " + - "show_name, "+ - "job_name, " + - "pk_job,"+ - "pk_show,"+ - "pk_facility,"+ - "str_name,"+ - "str_shot,"+ - "str_user,"+ - "int_uid,"+ - "str_log_dir,"+ - "frame_name, "+ - "frame_state, "+ - "pk_frame, "+ - "pk_layer, "+ - "int_retries, "+ - "int_version, " + - "layer_name, " + - "layer_type, "+ - "int_cores_min,"+ - "int_cores_max,"+ - "b_threadable,"+ - "int_mem_min,"+ - "int_gpu_min,"+ - "str_cmd, "+ - "str_range,"+ - "int_chunk_size, "+ - "str_services " + - "FROM (SELECT " + - "ROW_NUMBER() OVER ( ORDER BY " + - "frame.int_dispatch_order ASC, " + - "frame.int_layer_order ASC " + - ") LINENUM, " + - "job.str_show AS show_name, "+ - "job.str_name AS job_name, " + - "job.pk_job,"+ - "job.pk_show,"+ - "job.pk_facility,"+ - "job.str_name,"+ - "job.str_shot,"+ - "job.str_user,"+ - "job.int_uid,"+ - "job.str_log_dir,"+ - "frame.str_name AS frame_name, "+ - "frame.str_state AS frame_state, "+ - "frame.pk_frame, "+ - "frame.pk_layer, "+ - "frame.int_retries, "+ - "frame.int_version, "+ - "layer.str_name AS layer_name, " + - "layer.str_type AS layer_type, "+ - "layer.int_cores_min,"+ - "layer.int_cores_max,"+ - "layer.b_threadable,"+ - "layer.int_mem_min,"+ - "layer.int_gpu_min,"+ - "layer.str_cmd, "+ - "layer.str_range, "+ - "layer.int_chunk_size, "+ - "layer.str_services "+ - "FROM " + - "job,"+ - "frame," + - "layer " + - "WHERE " + - "frame.pk_layer = layer.pk_layer " + - "AND " + - "layer.pk_job = job.pk_job " + - "AND " + - "layer.int_mem_min <= ? " + - "AND " + - "layer.int_gpu_min <= ? " + - "AND " + - "frame.str_state='WAITING' " + - "AND " + - "layer.pk_layer=? "+ - "AND " + - "layer.pk_layer IN (" + - "SELECT " + - "la.pk_layer " + - "FROM " + - "layer la " + - "LEFT JOIN layer_limit ON layer_limit.pk_layer = la.pk_layer " + - "LEFT JOIN limit_record ON limit_record.pk_limit_record = layer_limit.pk_limit_record " + - "LEFT JOIN (" + - "SELECT " + - "limit_record.pk_limit_record, " + - "SUM(layer_stat.int_running_count) AS int_sum_running " + - "FROM " + - "layer_limit " + - "LEFT JOIN limit_record ON layer_limit.pk_limit_record = limit_record.pk_limit_record " + - "LEFT JOIN layer_stat ON layer_stat.pk_layer = layer_limit.pk_layer " + - "GROUP BY limit_record.pk_limit_record) AS sum_running " + - "ON limit_record.pk_limit_record = sum_running.pk_limit_record " + - "WHERE " + - "sum_running.int_sum_running < limit_record.int_max_value " + - "OR sum_running.int_sum_running IS NULL " + - ") " + - ") WHERE LINENUM <= ?"; - - /** - * Looks for shows that are under their burst for a particular - * type of proc. The show has to be at least one whole proc - * under their burst to be considered for booking. - */ - public static final String FIND_SHOWS = - "SELECT " + - "vs_waiting.pk_show,"+ - "s.float_tier, " + - "s.int_burst " + - "FROM " + - "subscription s,"+ - "vs_waiting " + - "WHERE "+ - "vs_waiting.pk_show = s.pk_show " + - "AND " + - "s.pk_alloc = ? " + - "AND " + - "s.int_burst > 0 " + - "AND " + - "s.int_burst - s.int_cores >= 100 " + - "AND " + - "s.int_cores < s.int_burst "; - -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DispatcherDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DispatcherDaoJdbc.java deleted file mode 100644 index ff56fe35a..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/DispatcherDaoJdbc.java +++ /dev/null @@ -1,412 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.log4j.Logger; -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.AllocationInterface; -import com.imageworks.spcue.DispatchFrame; -import com.imageworks.spcue.DispatchHost; -import com.imageworks.spcue.GroupInterface; -import com.imageworks.spcue.JobDetail; -import com.imageworks.spcue.JobInterface; -import com.imageworks.spcue.LayerInterface; -import com.imageworks.spcue.ShowInterface; -import com.imageworks.spcue.SortableShow; -import com.imageworks.spcue.VirtualProc; -import com.imageworks.spcue.dao.DispatcherDao; -import com.imageworks.spcue.grpc.host.ThreadMode; -import com.imageworks.spcue.util.CueUtil; - -import static com.imageworks.spcue.dao.oracle.DispatchQuery.FIND_DISPATCH_FRAME_BY_JOB_AND_HOST; -import static com.imageworks.spcue.dao.oracle.DispatchQuery.FIND_DISPATCH_FRAME_BY_JOB_AND_PROC; -import static com.imageworks.spcue.dao.oracle.DispatchQuery.FIND_DISPATCH_FRAME_BY_LAYER_AND_HOST; -import static com.imageworks.spcue.dao.oracle.DispatchQuery.FIND_DISPATCH_FRAME_BY_LAYER_AND_PROC; -import static com.imageworks.spcue.dao.oracle.DispatchQuery.FIND_JOBS_BY_GROUP; -import static com.imageworks.spcue.dao.oracle.DispatchQuery.FIND_JOBS_BY_LOCAL; -import static com.imageworks.spcue.dao.oracle.DispatchQuery.FIND_JOBS_BY_SHOW; -import static com.imageworks.spcue.dao.oracle.DispatchQuery.FIND_LOCAL_DISPATCH_FRAME_BY_JOB_AND_HOST; -import static com.imageworks.spcue.dao.oracle.DispatchQuery.FIND_LOCAL_DISPATCH_FRAME_BY_JOB_AND_PROC; -import static com.imageworks.spcue.dao.oracle.DispatchQuery.FIND_LOCAL_DISPATCH_FRAME_BY_LAYER_AND_HOST; -import static com.imageworks.spcue.dao.oracle.DispatchQuery.FIND_LOCAL_DISPATCH_FRAME_BY_LAYER_AND_PROC; -import static com.imageworks.spcue.dao.oracle.DispatchQuery.FIND_SHOWS; -import static com.imageworks.spcue.dao.oracle.DispatchQuery.FIND_UNDER_PROCED_JOB_BY_FACILITY; -import static com.imageworks.spcue.dao.oracle.DispatchQuery.HIGHER_PRIORITY_JOB_BY_FACILITY_EXISTS; - - - -/** - * Dispatcher DAO - * - * @category DAO - */ -public class DispatcherDaoJdbc extends JdbcDaoSupport implements DispatcherDao { - - private static final Logger logger = Logger.getLogger(DispatcherDaoJdbc.class); - - public static final RowMapper PKJOB_MAPPER = - new RowMapper() { - public String mapRow(ResultSet rs, int rowNum) throws SQLException { - return rs.getString("pk_job"); - } - }; - - private static final RowMapper SHOW_MAPPER = new RowMapper() { - public SortableShow mapRow(ResultSet rs, int rowNum) throws SQLException { - return new SortableShow( - rs.getString("pk_show"), - rs.getFloat("float_tier")); - } - }; - - private int threadMode(int mode) { - if (mode == ThreadMode.ALL_VALUE) - return mode; - return ThreadMode.AUTO_VALUE; - } - - /** - * Number of milliseconds before the show cache expires and - * a new show cache is created. - */ - private static final long SHOW_CACHE_EXPIRE_TIME_SEC = 8000; - - /** - * Wraps a list of SortableShows along with an expiration time. - */ - private class ShowCache { - final private long expireTime = System.currentTimeMillis() + SHOW_CACHE_EXPIRE_TIME_SEC; - final private List shows; - - public ShowCache(List shows) { - this.shows = shows; - Collections.sort(this.shows); - } - - public boolean isExpired() { - return System.currentTimeMillis() > expireTime; - } - - public List getShows() { - return shows; - } - } - - /** - * A cache of SortableShows keyed on host tags. - */ - private final ConcurrentHashMap bookableShows = - new ConcurrentHashMap(); - - /** - * Returns a sorted list of shows that have pending jobs - * which could benefit from the specified allocation. - * - * @param alloc - * ' - * @return a sorted list of shows. - */ - private List getBookableShows(AllocationInterface alloc) { - String key = alloc.getAllocationId(); - - ShowCache cached = bookableShows.get(key); - if (cached == null) { - bookableShows.put(key, new ShowCache(getJdbcTemplate().query( - FIND_SHOWS, - SHOW_MAPPER, alloc.getAllocationId()))); - } - else if (cached.isExpired()) { - bookableShows.put(key, new ShowCache(getJdbcTemplate().query( - FIND_SHOWS, - SHOW_MAPPER, alloc.getAllocationId()))); - } - return bookableShows.get(key).shows; - } - - private Set findDispatchJobs(DispatchHost host, int numJobs, boolean shuffleShows) { - LinkedHashSet result = new LinkedHashSet(); - List shows = new LinkedList(getBookableShows(host)); - // shows were sorted. If we want it in random sequence, we need to shuffle it. - if (shuffleShows) { - if (!shows.isEmpty()) - shows.remove(0); - Collections.shuffle(shows); - } - - for (SortableShow s: shows) { - - if (s.isSkipped(host.tags, (long) host.cores, host.memory)) { - logger.info("skipping show " + s.getShowId()); - continue; - } - - if (s.isSkipped(host)) { - logger.info("skipping show " + s.getShowId() + ", over its subscription."); - continue; - } - - /** - * Check if the show is over its subscription because we're using - * cached SortableShows, we don't pull a fresh list of shows for - * a while. If the show is over its subscription the alloc - * gets add to the SortableShow skipped alloc set. - */ - if (getJdbcTemplate().queryForObject( - "SELECT int_burst - int_cores FROM subscription WHERE pk_show=? AND pk_alloc=?", - Integer.class, s.getShowId(), host.getAllocationId()) < 100) { - s.skip(host); - continue; - } - - result.addAll(getJdbcTemplate().query( - FIND_JOBS_BY_SHOW, - PKJOB_MAPPER, - s.getShowId(), host.getFacilityId(), host.os, - host.idleCores, host.idleMemory, - threadMode(host.threadMode), - (host.idleGpu > 0) ? 1: 0, host.idleGpu, - hostString(host.getName()), numJobs * 10)); - - if (result.size() < 1) { - if (host.gpu == 0) { - s.skip(host.tags, host.idleCores, host.idleMemory); - } - } - else { - return result; - } - } - return result; - - } - - @Override - public Set findDispatchJobsForAllShows(DispatchHost host, int numJobs) { - return findDispatchJobs(host, numJobs, true); - } - - @Override - public Set findDispatchJobs(DispatchHost host, int numJobs) { - return findDispatchJobs(host, numJobs, false); - } - - @Override - public Set findDispatchJobs(DispatchHost host, GroupInterface g) { - LinkedHashSet result = new LinkedHashSet(5); - result.addAll(getJdbcTemplate().query( - FIND_JOBS_BY_GROUP, - PKJOB_MAPPER, - g.getGroupId(),host.getFacilityId(), host.os, - host.idleCores, host.idleMemory, - threadMode(host.threadMode), - (host.idleGpu > 0) ? 1: 0, host.idleGpu, - hostString(host.getName()), 50)); - - return result; - } - - @Override - public List findNextDispatchFrames(JobInterface job, - VirtualProc proc, int limit) { - - if (proc.isLocalDispatch) { - return getJdbcTemplate().query( - FIND_LOCAL_DISPATCH_FRAME_BY_JOB_AND_PROC, - FrameDaoJdbc.DISPATCH_FRAME_MAPPER, - proc.memoryReserved, - proc.gpuReserved, - job.getJobId(), - limit); - } - else { - return getJdbcTemplate().query( - FIND_DISPATCH_FRAME_BY_JOB_AND_PROC, - FrameDaoJdbc.DISPATCH_FRAME_MAPPER, - proc.coresReserved, - proc.memoryReserved, - (proc.gpuReserved > 0) ? 1: 0, proc.gpuReserved, - job.getJobId(), job.getJobId(), - hostString(proc.hostName), limit); - } - } - - @Override - public List findNextDispatchFrames(JobInterface job, - DispatchHost host, int limit) { - - if (host.isLocalDispatch) { - return getJdbcTemplate().query( - FIND_LOCAL_DISPATCH_FRAME_BY_JOB_AND_HOST, - FrameDaoJdbc.DISPATCH_FRAME_MAPPER, - host.idleMemory, host.idleGpu, job.getJobId(), - limit); - - } else { - return getJdbcTemplate().query( - FIND_DISPATCH_FRAME_BY_JOB_AND_HOST, - FrameDaoJdbc.DISPATCH_FRAME_MAPPER, - host.idleCores, host.idleMemory, - threadMode(host.threadMode), - (host.idleGpu > 0) ? 1: 0, host.idleGpu, - job.getJobId(), job.getJobId(), - hostString(host.getName()), limit); - } - } - - - @Override - public List findNextDispatchFrames(LayerInterface layer, - VirtualProc proc, int limit) { - - if (proc.isLocalDispatch) { - return getJdbcTemplate().query( - FIND_LOCAL_DISPATCH_FRAME_BY_LAYER_AND_PROC, - FrameDaoJdbc.DISPATCH_FRAME_MAPPER, - proc.memoryReserved, proc.gpuReserved, - layer.getLayerId(), - limit); - } - else { - return getJdbcTemplate().query( - FIND_DISPATCH_FRAME_BY_LAYER_AND_PROC, - FrameDaoJdbc.DISPATCH_FRAME_MAPPER, - proc.coresReserved, proc.memoryReserved, - proc.gpuReserved, - layer.getLayerId(), layer.getLayerId(), - hostString(proc.hostName), limit); - } - } - - @Override - public List findNextDispatchFrames(LayerInterface layer, - DispatchHost host, int limit) { - - if (host.isLocalDispatch) { - return getJdbcTemplate().query( - FIND_LOCAL_DISPATCH_FRAME_BY_LAYER_AND_HOST, - FrameDaoJdbc.DISPATCH_FRAME_MAPPER, - host.idleMemory, host.idleGpu, layer.getLayerId(), - limit); - - } else { - return getJdbcTemplate().query( - FIND_DISPATCH_FRAME_BY_LAYER_AND_HOST, - FrameDaoJdbc.DISPATCH_FRAME_MAPPER, - host.idleCores, host.idleMemory, - threadMode(host.threadMode), - host.idleGpu, layer.getLayerId(), layer.getLayerId(), - hostString(host.getName()), limit); - } - } - - - @Override - public DispatchFrame findNextDispatchFrame(JobInterface job, VirtualProc proc) { - return findNextDispatchFrames(job, proc, 1).get(0); - } - - @Override - public DispatchFrame findNextDispatchFrame(JobInterface job, DispatchHost host) { - return findNextDispatchFrames(job, host, 1).get(0); - } - - @Override - public boolean findUnderProcedJob(JobInterface excludeJob, VirtualProc proc) { - long start = System.currentTimeMillis(); - try { - return getJdbcTemplate().queryForObject( - FIND_UNDER_PROCED_JOB_BY_FACILITY, - Integer.class, excludeJob.getShowId(), proc.getFacilityId(), - proc.os, excludeJob.getShowId(), - proc.getFacilityId(), proc.os, - proc.coresReserved, proc.memoryReserved, proc.gpuReserved, - hostString(proc.hostName)) > 0; - } catch (org.springframework.dao.EmptyResultDataAccessException e) { - return false; - } - finally { - logger.trace("findUnderProcedJob(Job excludeJob, VirtualProc proc) " + CueUtil.duration(start)); - } - } - - @Override - public boolean higherPriorityJobExists(JobDetail baseJob, VirtualProc proc) { - long start = System.currentTimeMillis(); - try { - return getJdbcTemplate().queryForObject( - HIGHER_PRIORITY_JOB_BY_FACILITY_EXISTS, - Boolean.class, baseJob.priority, proc.getFacilityId(), - proc.os, proc.getFacilityId(), proc.os, - proc.coresReserved, proc.memoryReserved, proc.gpuReserved, - hostString(proc.hostName)); - } catch (org.springframework.dao.EmptyResultDataAccessException e) { - return false; - } - finally { - logger.trace("higherPriorityJobExists(JobDetail baseJob, VirtualProc proc) " + CueUtil.duration(start)); - } - } - - @Override - public Set findDispatchJobs(DispatchHost host, - ShowInterface show, int numJobs) { - LinkedHashSet result = new LinkedHashSet(numJobs); - - result.addAll(getJdbcTemplate().query( - FIND_JOBS_BY_SHOW, - PKJOB_MAPPER, - show.getShowId(), host.getFacilityId(), host.os, - host.idleCores, host.idleMemory, - threadMode(host.threadMode), - (host.idleGpu > 0) ? 1: 0, host.idleGpu, - hostString(host.getName()), numJobs * 10)); - - return result; - } - - @Override - public Set findLocalDispatchJobs(DispatchHost host) { - LinkedHashSet result = new LinkedHashSet(5); - result.addAll(getJdbcTemplate().query( - FIND_JOBS_BY_LOCAL, - PKJOB_MAPPER, - host.getHostId(), host.getFacilityId(), - host.os, host.getHostId(), host.getFacilityId(), host.os)); - - return result; - } - - private static final String hostString(String name) { - return "str_name='" + name + "'"; - } -} - - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/FacilityDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/FacilityDaoJdbc.java deleted file mode 100644 index 1f77d6e3c..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/FacilityDaoJdbc.java +++ /dev/null @@ -1,88 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; - -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.FacilityEntity; -import com.imageworks.spcue.FacilityInterface; -import com.imageworks.spcue.dao.FacilityDao; -import com.imageworks.spcue.util.SqlUtil; - -public class FacilityDaoJdbc extends JdbcDaoSupport implements FacilityDao { - - public static final RowMapper FACILITY_MAPPER = new RowMapper() { - public FacilityInterface mapRow(ResultSet rs, int rowNum) throws SQLException { - FacilityEntity facility = new FacilityEntity(); - facility.id = rs.getString("pk_facility"); - facility.name = rs.getString("str_name"); - return facility; - } - }; - - public FacilityInterface getDefaultFacility() { - return getJdbcTemplate().queryForObject( - "SELECT pk_facility,str_name FROM facility WHERE b_default=1 AND ROWNUM < 2", - FACILITY_MAPPER); - } - - public FacilityInterface getFacility(String id) { - return getJdbcTemplate().queryForObject( - "SELECT pk_facility, str_name FROM facility WHERE pk_facility=? " + - "OR str_name=?", FACILITY_MAPPER, id, id); - } - - public boolean facilityExists(String name) { - return getJdbcTemplate().queryForObject( - "SELECT COUNT(1) FROM facility WHERE str_name=?", - Integer.class, name) > 0; - - } - - public FacilityInterface insertFacility(FacilityEntity facility) { - facility.id = SqlUtil.genKeyRandom(); - - getJdbcTemplate().update( - "INSERT INTO facility (pk_facility, str_name) VALUES (?,?)", - facility.getId(), facility.getName()); - - return facility; - } - - @Override - public int deleteFacility(FacilityInterface facility) { - return getJdbcTemplate().update( - "DELETE FROM facility WHERE pk_facility = ?", - facility.getFacilityId()); - } - - @Override - public int updateFacilityName(FacilityInterface facility, String name) { - return getJdbcTemplate().update( - "UPDATE facility SET str_name=? WHERE pk_facility = ?", - name, facility.getFacilityId()); - } - -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/FilterDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/FilterDaoJdbc.java deleted file mode 100644 index 3f7d99733..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/FilterDaoJdbc.java +++ /dev/null @@ -1,199 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.CallableStatement; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - -import org.springframework.jdbc.core.CallableStatementCreator; -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.SqlParameter; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.FilterEntity; -import com.imageworks.spcue.FilterInterface; -import com.imageworks.spcue.ShowInterface; -import com.imageworks.spcue.dao.FilterDao; -import com.imageworks.spcue.grpc.filter.FilterType; -import com.imageworks.spcue.util.SqlUtil; - -/** - * A DAO class for loading Filters, Actions, and Matchers. Part of the - * job filtering system. - * - * @category DAO - */ -public class FilterDaoJdbc extends JdbcDaoSupport implements FilterDao { - - private static final String GET_FILTER = - "SELECT " + - "filter.* " + - "FROM "+ - "filter "; - - private static final String GET_ACTIVE_FILTERS = - "SELECT " + - "filter.*" + - "FROM " + - "filter " + - "WHERE " + - "b_enabled = 1 "+ - "AND " + - "pk_show=? " + - "ORDER BY " + - "f_order ASC"; - - private static final String GET_FILTERS = - "SELECT " + - "filter.*" + - "FROM " + - "filter " + - "WHERE " + - "pk_show=? " + - "ORDER BY " + - "f_order ASC"; - - public static final RowMapper FILTER_DETAIL_MAPPER = new RowMapper() { - public FilterEntity mapRow(ResultSet rs, int rowNum) throws SQLException { - FilterEntity d = new FilterEntity(); - d.type = FilterType.valueOf(rs.getString("str_type")); - d.id = rs.getString("pk_filter"); - d.name = rs.getString("str_name"); - d.showId = rs.getString("pk_show"); - d.enabled = rs.getBoolean("b_enabled"); - d.order = rs.getFloat("f_order"); - return d; - } - }; - - public List getActiveFilters(ShowInterface show) { - return getJdbcTemplate().query( - GET_ACTIVE_FILTERS, FILTER_DETAIL_MAPPER, show.getShowId()); - } - - public List getFilters(ShowInterface show) { - return getJdbcTemplate().query( - GET_FILTERS, FILTER_DETAIL_MAPPER, show.getShowId()); - } - - public void deleteFilter(FilterInterface f) { - getJdbcTemplate().update( - "DELETE FROM action WHERE pk_filter=?",f.getFilterId()); - getJdbcTemplate().update( - "DELETE FROM matcher WHERE pk_filter=?",f.getFilterId()); - getJdbcTemplate().update( - "DELETE FROM filter WHERE pk_filter=?",f.getFilterId()); - reorderFilters(f); - } - - private static final String INSERT_FILTER = - "INSERT INTO " + - "filter "+ - "(" + - "pk_filter," + - "pk_show,"+ - "str_name,"+ - "str_type,"+ - "f_order "+ - ") VALUES (?,?,?,?,(SELECT COALESCE(MAX(f_order)+1,1.0) FROM filter WHERE pk_show=?))"; - - public void insertFilter(FilterEntity f) { - f.id = SqlUtil.genKeyRandom(); - getJdbcTemplate().update(INSERT_FILTER, - f.id, f.getShowId(),f.name, f.type.toString(), f.getShowId()); - reorderFilters(f); - } - - public void updateSetFilterEnabled(FilterInterface f, boolean enabled) { - getJdbcTemplate().update( - "UPDATE filter SET b_enabled=? WHERE pk_filter=?", - enabled, f.getFilterId()); - } - - public void updateSetFilterName(FilterInterface f, String name) { - getJdbcTemplate().update( - "UPDATE filter SET str_name=? WHERE pk_filter=?", - name, f.getFilterId()); - } - - public void updateSetFilterOrder(FilterInterface f, double order) { - getJdbcTemplate().update( - "UPDATE filter SET f_order=? - 0.1 WHERE pk_filter=?", - order, f.getFilterId()); - reorderFilters(f); - } - - public void lowerFilterOrder(FilterInterface f, int by) { - double lower_by = by + 0.1; - getJdbcTemplate().update( - "UPDATE filter SET f_order=f_order + ? WHERE pk_filter=?", - lower_by, f.getFilterId()); - reorderFilters(f); - } - - public void raiseFilterOrder(FilterInterface f, int by) { - double raise_by = (by * -1) - 0.1; - getJdbcTemplate().update( - "UPDATE filter SET f_order=f_order + ? WHERE pk_filter=?", - raise_by, f.getFilterId()); - reorderFilters(f); - } - - public void updateSetFilterType(FilterInterface f, FilterType type) { - getJdbcTemplate().update( - "UPDATE filter SET str_type=? WHERE pk_filter=?", - type.toString(), f.getFilterId()); - } - - public void reorderFilters(final ShowInterface s) { - getJdbcTemplate().update("LOCK TABLE filter IN SHARE MODE"); - getJdbcTemplate().call(new CallableStatementCreator() { - - public CallableStatement createCallableStatement(Connection con) throws SQLException { - CallableStatement c = con.prepareCall("{ call reorder_filters(?) }"); - c.setString(1, s.getShowId()); - return c; - } - }, new ArrayList()); - } - - public FilterEntity findFilter(ShowInterface show, String name) { - return getJdbcTemplate().queryForObject( - GET_FILTER + " WHERE pk_show=? AND str_name=?", - FILTER_DETAIL_MAPPER, show.getShowId(), name); - } - - public FilterEntity getFilter(String id) { - return getJdbcTemplate().queryForObject( - GET_FILTER + " WHERE pk_filter=?", - FILTER_DETAIL_MAPPER, id); - } - - public FilterEntity getFilter(FilterInterface filter) { - return getJdbcTemplate().queryForObject( - GET_FILTER + " WHERE pk_filter=?", - FILTER_DETAIL_MAPPER, filter.getFilterId()); - } -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/FrameDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/FrameDaoJdbc.java deleted file mode 100644 index fd3a421d1..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/FrameDaoJdbc.java +++ /dev/null @@ -1,1104 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.imageworks.spcue.dao.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.EnumSet; -import java.util.List; -import java.sql.Timestamp; -import java.util.Optional; - -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.DispatchFrame; -import com.imageworks.spcue.FrameDetail; -import com.imageworks.spcue.FrameEntity; -import com.imageworks.spcue.FrameInterface; -import com.imageworks.spcue.JobInterface; -import com.imageworks.spcue.LayerDetail; -import com.imageworks.spcue.LayerInterface; -import com.imageworks.spcue.LightweightDependency; -import com.imageworks.spcue.ResourceUsage; -import com.imageworks.spcue.VirtualProc; -import com.imageworks.spcue.dao.FrameDao; -import com.imageworks.spcue.dao.criteria.FrameSearchInterface; -import com.imageworks.spcue.dispatcher.Dispatcher; -import com.imageworks.spcue.dispatcher.FrameReservationException; -import com.imageworks.spcue.grpc.depend.DependType; -import com.imageworks.spcue.grpc.job.CheckpointState; -import com.imageworks.spcue.grpc.job.FrameExitStatus; -import com.imageworks.spcue.grpc.job.FrameState; -import com.imageworks.spcue.grpc.job.JobState; -import com.imageworks.spcue.grpc.job.LayerType; -import com.imageworks.spcue.util.CueUtil; -import com.imageworks.spcue.util.FrameSet; -import com.imageworks.spcue.util.SqlUtil; - -public class FrameDaoJdbc extends JdbcDaoSupport implements FrameDao { - - private static final String UPDATE_FRAME_STOPPED_NORSS = - "UPDATE "+ - "frame "+ - "SET " + - "str_state=?, "+ - "int_exit_status = ?, " + - "ts_stopped = systimestamp, " + - "ts_updated = systimestamp, " + - "int_version = int_version + 1, " + - "int_total_past_core_time = int_total_past_core_time + " + - "round(INTERVAL_TO_SECONDS(systimestamp - ts_started) * int_cores / 100) " + - "WHERE " + - "frame.pk_frame = ? " + - "AND " + - "frame.str_state = ? " + - "AND " + - "frame.int_version = ? "; - - @Override - public boolean updateFrameStopped(FrameInterface frame, FrameState state, - int exitStatus) { - return getJdbcTemplate().update(UPDATE_FRAME_STOPPED_NORSS, - state.toString(), exitStatus, frame.getFrameId(), - FrameState.RUNNING.toString(), frame.getVersion()) == 1; - } - - private static final String UPDATE_FRAME_STOPPED = - "UPDATE "+ - "frame "+ - "SET " + - "str_state=?, "+ - "int_exit_status = ?, " + - "ts_stopped = systimestamp + interval '1' second, " + - "ts_updated = systimestamp, " + - "int_mem_max_used = ?, " + - "int_version = int_version + 1, " + - "int_total_past_core_time = int_total_past_core_time + " + - "round(INTERVAL_TO_SECONDS(systimestamp + interval '1' second - ts_started) * int_cores / 100) " + - "WHERE " + - "frame.pk_frame = ? " + - "AND " + - "frame.str_state = ? " + - "AND " + - "frame.int_version = ? "; - - @Override - public boolean updateFrameStopped(FrameInterface frame, FrameState state, - int exitStatus, long maxRss) { - - - return getJdbcTemplate().update(UPDATE_FRAME_STOPPED, - state.toString(), exitStatus, maxRss, - frame.getFrameId(), FrameState.RUNNING.toString(), - frame.getVersion()) == 1; - } - - private static final String UPDATE_FRAME_CLEARED = - "UPDATE "+ - "frame "+ - "SET " + - "str_state = ?, "+ - "int_exit_status = ?, " + - "ts_stopped = systimestamp, " + - "ts_updated = systimestamp, " + - "int_version = int_version + 1 " + - "WHERE " + - "frame.pk_frame = ? " + - "AND " + - "frame.pk_frame NOT IN " + - "(SELECT proc.pk_frame FROM " + - "proc WHERE proc.pk_frame=?)"; - - @Override - public boolean updateFrameCleared(FrameInterface frame) { - - int result = getJdbcTemplate().update( - UPDATE_FRAME_CLEARED, - FrameState.WAITING.toString(), - Dispatcher.EXIT_STATUS_FRAME_CLEARED, - frame.getFrameId(), - frame.getFrameId()); - - return result > 0; - } - - private static final String UPDATE_FRAME_STARTED = - "UPDATE "+ - "frame "+ - "SET " + - "str_state = ?,"+ - "str_host=?, " + - "int_cores=?, "+ - "int_mem_reserved = ?, " + - "int_gpu_reserved = ?, " + - "ts_updated = systimestamp,"+ - "ts_started = systimestamp,"+ - "ts_stopped = null, "+ - "int_version = int_version + 1 "+ - "WHERE " + - "pk_frame = ? " + - "AND " + - "str_state = ? " + - "AND " + - "int_version = ? " + - "AND " + - "frame.pk_layer IN (" + - "SELECT " + - "layer.pk_layer " + - "FROM " + - "layer " + - "LEFT JOIN layer_limit ON layer_limit.pk_layer = layer.pk_layer " + - "LEFT JOIN limit_record ON limit_record.pk_limit_record = layer_limit.pk_limit_record " + - "LEFT JOIN (" + - "SELECT " + - "limit_record.pk_limit_record, " + - "SUM(layer_stat.int_running_count) AS int_sum_running " + - "FROM " + - "layer_limit " + - "LEFT JOIN limit_record ON layer_limit.pk_limit_record = limit_record.pk_limit_record " + - "LEFT JOIN layer_stat ON layer_stat.pk_layer = layer_limit.pk_layer " + - "GROUP BY limit_record.pk_limit_record) sum_running " + - "ON limit_record.pk_limit_record = sum_running.pk_limit_record " + - "WHERE " + - "sum_running.int_sum_running < limit_record.int_max_value " + - "OR sum_running.int_sum_running IS NULL " + - ")"; - - private static final String UPDATE_FRAME_RETRIES = - "UPDATE " + - "frame " + - "SET " + - "int_retries = int_retries + 1 " + - "WHERE " + - "pk_frame = ? " + - "AND " + - "int_exit_status NOT IN (?,?,?) "; - - @Override - public void updateFrameStarted(VirtualProc proc, FrameInterface frame) { - - lockFrameForUpdate(frame, FrameState.WAITING); - - int result = getJdbcTemplate().update(UPDATE_FRAME_STARTED, - FrameState.RUNNING.toString(), proc.hostName, proc.coresReserved, - proc.memoryReserved, proc.gpuReserved, frame.getFrameId(), - FrameState.WAITING.toString(), frame.getVersion()); - - if (result == 0) { - String error_msg = "the frame " + - frame + " was updated by another thread."; - throw new FrameReservationException(error_msg); - } - - /* - * Frames that were killed via nimby or hardware errors not attributed to - * the software do not increment the retry counter. - */ - getJdbcTemplate().update(UPDATE_FRAME_RETRIES, - frame.getFrameId(), -1, FrameExitStatus.SKIP_RETRY_VALUE, - Dispatcher.EXIT_STATUS_FRAME_CLEARED); - } - - private static final String UPDATE_FRAME_FIXED = - "UPDATE "+ - "frame "+ - "SET " + - "str_state = ?,"+ - "str_host=?, " + - "int_cores=?, "+ - "int_mem_reserved = ?, " + - "int_gpu_reserved = ?, " + - "ts_updated = systimestamp,"+ - "ts_started = systimestamp,"+ - "ts_stopped = null, "+ - "int_version = int_version + 1 " + - "WHERE " + - "pk_frame = ? " + - "AND " + - "str_state = 'RUNNING'"; - - @Override - public boolean updateFrameFixed(VirtualProc proc, FrameInterface frame) { - return getJdbcTemplate().update(UPDATE_FRAME_FIXED, - FrameState.RUNNING.toString(), proc.hostName, proc.coresReserved, - proc.memoryReserved, proc.gpuReserved, frame.getFrameId()) == 1; - } - - @Override - public DispatchFrame getDispatchFrame(String uuid) { - return getJdbcTemplate().queryForObject( - GET_DISPATCH_FRAME, DISPATCH_FRAME_MAPPER, uuid); - } - - static final RowMapper DISPATCH_FRAME_MAPPER = new RowMapper() { - public DispatchFrame mapRow(ResultSet rs, int rowNum) throws SQLException { - DispatchFrame frame = new DispatchFrame(); - frame.id = rs.getString("pk_frame"); - frame.name = rs.getString("frame_name"); - frame.layerId = rs.getString("pk_layer"); - frame.jobId = rs.getString("pk_job"); - frame.showId = rs.getString("pk_show"); - frame.facilityId = rs.getString("pk_facility"); - frame.retries = rs.getInt("int_retries"); - frame.state = FrameState.valueOf(rs.getString("frame_state")); - frame.command = rs.getString("str_cmd"); - frame.jobName = rs.getString("job_name"); - frame.layerName = rs.getString("layer_name"); - frame.chunkSize = rs.getInt("int_chunk_size"); - frame.range = rs.getString("str_range"); - frame.logDir = rs.getString("str_log_dir"); - frame.shot = rs.getString("str_shot"); - frame.show = rs.getString("show_name"); - frame.owner = rs.getString("str_user"); - int uid = rs.getInt("int_uid"); - frame.uid = rs.wasNull() ? Optional.empty() : Optional.of(uid); - frame.state = FrameState.valueOf(rs.getString("frame_state")); - frame.minCores = rs.getInt("int_cores_min"); - frame.maxCores = rs.getInt("int_cores_max"); - frame.threadable = rs.getBoolean("b_threadable"); - frame.minMemory = rs.getLong("int_mem_min"); - frame.minGpu = rs.getLong("int_gpu_min"); - frame.version = rs.getInt("int_version"); - frame.services = rs.getString("str_services"); - return frame; - } - }; - - private static final String GET_DISPATCH_FRAME = - "SELECT " + - "show.str_name AS show_name, "+ - "job.str_name AS job_name, " + - "job.pk_job,"+ - "job.pk_show,"+ - "job.pk_facility,"+ - "job.str_name,"+ - "job.str_shot,"+ - "job.str_user,"+ - "job.int_uid,"+ - "job.str_log_dir,"+ - "frame.str_name AS frame_name, "+ - "frame.str_state AS frame_state, "+ - "frame.pk_frame, "+ - "frame.pk_layer, "+ - "frame.int_retries, "+ - "frame.int_version, " + - "layer.str_name AS layer_name, " + - "layer.str_type AS layer_type, "+ - "layer.str_cmd, "+ - "layer.int_cores_min,"+ - "layer.int_cores_max,"+ - "layer.b_threadable,"+ - "layer.int_mem_min, "+ - "layer.int_gpu_min, "+ - "layer.str_range, "+ - "layer.int_chunk_size, " + - "layer.str_services " + - "FROM " + - "layer, " + - "job, "+ - "show, " + - "frame LEFT JOIN proc ON (proc.pk_frame = frame.pk_frame) " + - "WHERE " + - "job.pk_show = show.pk_show "+ - "AND " + - "frame.pk_job = job.pk_job " + - "AND " + - "frame.pk_layer = layer.pk_layer " + - "AND " + - "frame.pk_frame = ?"; - - private static final String GET_FRAME_DETAIL = - "SELECT " + - "frame.*, " + - "job.pk_facility," + - "job.pk_show " + - "FROM " + - "frame," + - "layer," + - "job," + - "show " + - "WHERE "+ - "frame.pk_job = job.pk_job " + - "AND " + - "frame.pk_layer = layer.pk_layer " + - "AND "+ - "job.pk_show = show.pk_show "; - - private static final String GET_MINIMAL_FRAME = - "SELECT " + - "frame.pk_frame," + - "frame.str_name, " + - "frame.pk_job, " + - "frame.pk_layer, "+ - "frame.str_state, " + - "frame.int_version, "+ - "job.pk_show, " + - "job.pk_facility "+ - "FROM " + - "frame," + - "layer," + - "job," + - "show " + - "WHERE "+ - "frame.pk_job = job.pk_job " + - "AND " + - "frame.pk_layer = layer.pk_layer " + - "AND " + - "job.pk_show = show.pk_show "; - - private static final RowMapper FRAME_MAPPER = - new RowMapper() { - public FrameEntity mapRow(ResultSet rs, - int rowNum) throws SQLException { - FrameEntity frame = new FrameEntity(); - frame.id = rs.getString("pk_frame"); - frame.name = rs.getString("str_name"); - frame.jobId = rs.getString("pk_job"); - frame.layerId = rs.getString("pk_layer"); - frame.showId = rs.getString("pk_show"); - frame.facilityId = rs.getString("pk_facility"); - frame.version = rs.getInt("int_version"); - return frame; - } - }; - - private static final RowMapper FRAME_DETAIL_MAPPER = new RowMapper() { - public FrameDetail mapRow(ResultSet rs, int rowNum) throws SQLException { - FrameDetail frame = new FrameDetail(); - frame.id = rs.getString("pk_frame"); - frame.dependCount = rs.getInt("int_depend_count"); - frame.exitStatus = rs.getInt("int_exit_status"); - frame.jobId = rs.getString("pk_job"); - frame.layerId = rs.getString("pk_layer"); - frame.showId = rs.getString("pk_show"); - frame.maxRss = rs.getInt("int_mem_max_used"); - frame.name = rs.getString("str_name"); - frame.number = rs.getInt("int_number"); - frame.dispatchOrder = rs.getInt("int_dispatch_order"); - frame.retryCount = rs.getInt("int_retries"); - frame.dateStarted = rs.getTimestamp("ts_started"); - frame.dateStopped = rs.getTimestamp("ts_stopped"); - frame.dateUpdated = rs.getTimestamp("ts_updated"); - frame.version = rs.getInt("int_version"); - - if (rs.getString("str_host") != null) { - frame.lastResource = String.format("%s/%d",rs.getString("str_host"),rs.getInt("int_cores")); - } - else { - frame.lastResource = ""; - } - frame.state = FrameState.valueOf(rs.getString("str_state")); - - return frame; - } - }; - - public static final String FIND_ORPHANED_FRAMES = - "SELECT " + - "frame.pk_frame, " + - "frame.pk_layer, " + - "frame.str_name, " + - "frame.int_version, " + - "job.pk_job, " + - "job.pk_show, " + - "job.pk_facility " + - "FROM " + - "frame, " + - "job " + - "WHERE " + - "job.pk_job = frame.pk_job " + - "AND " + - "frame.str_state='RUNNING' " + - "AND " + - "job.str_state = 'PENDING' " + - "AND " + - "(SELECT COUNT(1) FROM proc WHERE proc.pk_frame = frame.pk_frame) = 0 " + - "AND " + - "systimestamp - frame.ts_updated > interval '300' second"; - - @Override - public List getOrphanedFrames() { - return getJdbcTemplate().query(FIND_ORPHANED_FRAMES, - FRAME_MAPPER); - } - - private static final String IS_ORPHAN = - "SELECT " + - "COUNT(1) " + - "FROM " + - "frame " + - "WHERE " + - "frame.pk_frame = ? " + - "AND " + - "frame.str_state = 'RUNNING' " + - "AND " + - "(SELECT COUNT(1) FROM proc WHERE proc.pk_frame = frame.pk_frame) = 0 " + - "AND " + - "systimestamp - frame.ts_updated > interval '300' second"; - - @Override - public boolean isOrphan(FrameInterface frame) { - return getJdbcTemplate().queryForObject(IS_ORPHAN, Integer.class, - frame.getFrameId()) == 1; - } - - private static final String INSERT_FRAME = - "INSERT INTO " + - "frame " + - "("+ - "pk_frame, " + - "pk_layer, " + - "pk_job, " + - "str_name, " + - "str_state, " + - "int_number, " + - "int_dispatch_order, " + - "int_layer_order, "+ - "ts_updated "+ - ") " + - "VALUES (?,?,?,?,?,?,?,?,systimestamp)"; - - @Override - public void insertFrames(LayerDetail layer, List frames) { - - int count = 0; - for (int frame: frames) { - getJdbcTemplate().update(INSERT_FRAME, - SqlUtil.genKeyRandom(), - layer.getLayerId(), - layer.getJobId(), - CueUtil.buildFrameName(layer, frame), - FrameState.SETUP.toString(), - frame, - count, - layer.dispatchOrder); - count++; - } - } - - @Override - public List getDependentFrames(LightweightDependency depend) { - - /* - * Compound depends are handled in the DependManager. - */ - - String key = null; - StringBuilder sb = new StringBuilder(4096); - sb.append(GET_MINIMAL_FRAME); - sb.append(" AND frame.int_depend_count > 0 "); - - if (EnumSet.of( - DependType.JOB_ON_JOB, - DependType.JOB_ON_LAYER, - DependType.JOB_ON_FRAME).contains(depend.type)) { - sb.append("AND job.pk_job = ?"); - key = depend.dependErJobId; - } - else if (EnumSet.of( - DependType.LAYER_ON_FRAME, - DependType.LAYER_ON_LAYER, - DependType.LAYER_ON_JOB).contains(depend.type)) { - sb.append("AND layer.pk_layer = ?"); - key = depend.dependErLayerId; - } - else if (EnumSet.of( - DependType.FRAME_ON_JOB, - DependType.FRAME_ON_LAYER, - DependType.FRAME_ON_FRAME).contains(depend.type)) { - sb.append("AND frame.pk_frame = ?"); - key = depend.dependErFrameId; - } - else { - return new ArrayList(1); - } - - return getJdbcTemplate().query( - sb.toString(), FRAME_MAPPER, - new Object[] { key }); - } - - @Override - public FrameInterface findFrame(LayerInterface layer, int number) { - return getJdbcTemplate().queryForObject( - GET_MINIMAL_FRAME + " AND frame.pk_layer=? AND int_number=?", - FRAME_MAPPER, layer.getLayerId(), number); - } - - @Override - public FrameDetail getFrameDetail(FrameInterface frame) { - return getJdbcTemplate().queryForObject( - GET_FRAME_DETAIL + " AND pk_frame=?", - FRAME_DETAIL_MAPPER, frame.getFrameId()); - } - - @Override - public FrameDetail getFrameDetail(String id) { - return getJdbcTemplate().queryForObject( - GET_FRAME_DETAIL + " AND pk_frame=?", - FRAME_DETAIL_MAPPER, id); - } - - @Override - public FrameDetail findFrameDetail(JobInterface job, String name) { - //Uses C_FRAME_STR_NAME_UNQ - return getJdbcTemplate().queryForObject( - GET_FRAME_DETAIL + " AND frame.str_name=? AND frame.pk_job=?", - FRAME_DETAIL_MAPPER, name, job.getJobId()); - } - - @Override - public List findFrameDetails(FrameSearchInterface r) { - return getJdbcTemplate().query(r.getFilteredQuery(GET_FRAME_DETAIL), - FRAME_DETAIL_MAPPER, r.getValuesArray()); - } - - @Override - public List findFrames(FrameSearchInterface r) { - return getJdbcTemplate().query(r.getFilteredQuery(GET_MINIMAL_FRAME), - FRAME_MAPPER, r.getValuesArray()); - } - - private static final String FIND_LONGEST_FRAME = - "SELECT " + - "pk_frame " + - "FROM (" + - "SELECT " + - "pk_frame,"+ - "ts_stopped - ts_started AS duration " + - "FROM " + - "frame, " + - "layer " + - "WHERE " + - "frame.pk_layer = layer.pk_layer " + - "AND " + - "frame.pk_job = ? "+ - "AND " + - "str_state=? "+ - "AND " + - "layer.str_type=? " + - "ORDER BY "+ - "duration DESC "+ - ") WHERE ROWNUM = 1"; - - @Override - public FrameDetail findLongestFrame(JobInterface job) { - String pk_frame = getJdbcTemplate().queryForObject( - FIND_LONGEST_FRAME, String.class, job.getJobId(), - FrameState.SUCCEEDED.toString(), LayerType.RENDER.toString()); - return getFrameDetail(pk_frame); - } - - private static final String FIND_SHORTEST_FRAME = - "SELECT " + - "pk_frame " + - "FROM (" + - "SELECT " + - "pk_frame,"+ - "ts_stopped - ts_started AS duration " + - "FROM " + - "frame, " + - "layer " + - "WHERE " + - "frame.pk_layer = layer.pk_layer " + - "AND " + - "frame.pk_job = ? "+ - "AND " + - "frame.str_state=? "+ - "AND " + - "layer.str_type=? " + - "ORDER BY "+ - "duration ASC "+ - ") WHERE ROWNUM = 1"; - - @Override - public FrameDetail findShortestFrame(JobInterface job) { - String pk_frame = getJdbcTemplate().queryForObject( - FIND_SHORTEST_FRAME, String.class, job.getJobId(), - FrameState.SUCCEEDED.toString(),LayerType.RENDER.toString()); - return getFrameDetail(pk_frame); - } - - @Override - public FrameInterface getFrame(String id) { - return getJdbcTemplate().queryForObject( - GET_MINIMAL_FRAME + " AND frame.pk_frame=?", - FRAME_MAPPER, id); - } - - @Override - public FrameInterface findFrame(JobInterface job, String name) { - //Uses C_FRAME_STR_NAME_UNQ - return getJdbcTemplate().queryForObject( - GET_MINIMAL_FRAME + " AND frame.str_name=? AND frame.pk_job=?", - FRAME_MAPPER, name, job.getJobId()); - } - - @Override - public void checkRetries(FrameInterface frame) { - int max_retries = getJdbcTemplate().queryForObject( - "SELECT int_max_retries FROM job WHERE pk_job=?", Integer.class, - frame.getJobId()); - - if (getJdbcTemplate().queryForObject( - "SELECT int_retries FROM frame WHERE pk_frame=?", Integer.class, - frame.getFrameId()) >= max_retries) { - getJdbcTemplate().update( - "UPDATE frame SET str_state=? WHERE pk_frame=?", - FrameState.DEAD.toString(), frame.getFrameId()); - } - } - - public static final String GET_FRAME_ID = - "SELECT " + - "frame.pk_frame "+ - "FROM " + - "frame,"+ - "layer,"+ - "job "+ - "WHERE " + - "frame.pk_layer = layer.pk_layer " + - "AND " + - "frame.pk_job = job.pk_job "; - - - private static final String UPDATE_FRAME_STATE = - "UPDATE " + - "frame "+ - "SET " + - "str_state=?, " + - "ts_updated = systimestamp, " + - "int_version = int_version + 1 " + - "WHERE " + - "pk_frame = ? " + - "AND " + - "int_version = ? "; - - @Override - public boolean updateFrameState(FrameInterface frame, FrameState state) { - if (getJdbcTemplate().update(UPDATE_FRAME_STATE, - state.toString(), - frame.getFrameId(), - frame.getVersion()) == 1) { - logger.info("The frame " + frame + " state changed to " + - state.toString()); - return true; - } - logger.info("Failed to change the frame " + frame + " state to " + - state.toString()); - return false; - } - - private static final String MARK_AS_WAITING = - "UPDATE " + - "frame "+ - "SET " + - "str_state=?, " + - "ts_updated = systimestamp, " + - "int_depend_count = 0, " + - "int_version = int_version + 1 " + - "WHERE " + - "pk_frame = ? " + - "AND " + - "int_version = ? " + - "AND " + - "str_state = ? "; - - @Override - public void markFrameAsWaiting(FrameInterface frame) { - getJdbcTemplate().update( - MARK_AS_WAITING, - FrameState.WAITING.toString(), - frame.getFrameId(), - frame.getVersion(), - FrameState.DEPEND.toString()); - } - - private static final String MARK_AS_DEPEND = - "UPDATE " + - "frame "+ - "SET " + - "str_state=?, " + - "int_depend_count = ?, "+ - "ts_updated = systimestamp, " + - "int_version = int_version + 1 " + - "WHERE " + - "pk_frame = ? " + - "AND " + - "int_version = ? " + - "AND " + - "str_state = ? "; - - private static final String GET_FRAME_DEPEND_COUNT = - "SELECT " + - "COUNT(1) " + - "FROM " + - "depend " + - "WHERE " + - " ( " + - "(pk_job_depend_er = ? AND str_type LIKE 'JOB#_ON%' ESCAPE '#') " + - "OR " + - "pk_layer_depend_er=? " + - "OR " + - "pk_frame_depend_er=? " + - " ) " + - "AND " + - "depend.b_active = 1 " + - "AND " + - "depend.b_composite = 0 "; - - public void markFrameAsDepend(FrameInterface frame) { - // We need to full depend count in this case to reset the - // frames's depend count accurately. - int depend_count = getJdbcTemplate().queryForObject( - GET_FRAME_DEPEND_COUNT, Integer.class, - frame.getJobId(),frame.getLayerId(),frame.getFrameId()); - - if (depend_count > 0) { - getJdbcTemplate().update( - MARK_AS_DEPEND, - FrameState.DEPEND.toString(), - depend_count, - frame.getFrameId(), - frame.getVersion(), - FrameState.WAITING.toString()); - } - } - - private static final String FIND_HIGHEST_MEM_FRAME = - "SELECT " + - "pk_frame " + - "FROM (" + - "SELECT " + - "pk_frame " + - "FROM " + - "frame " + - "WHERE " + - "pk_job = ? "+ - "AND " + - "str_state=? "+ - "ORDER BY "+ - "int_mem_max_used DESC "+ - ") WHERE ROWNUM = 1"; - - @Override - public FrameDetail findHighestMemoryFrame(JobInterface job) { - String pk_frame = getJdbcTemplate().queryForObject( - FIND_HIGHEST_MEM_FRAME, String.class, job.getJobId(), - FrameState.SUCCEEDED.toString()); - return getFrameDetail(pk_frame); - } - - private static final String FIND_LOWEST_MEM_FRAME = - "SELECT " + - "pk_frame " + - "FROM (" + - "SELECT " + - "pk_frame " + - "FROM " + - "frame " + - "WHERE " + - "pk_job = ? "+ - "AND " + - "str_state=? "+ - "ORDER BY "+ - "int_mem_max_used ASC "+ - ") WHERE ROWNUM = 1"; - - @Override - public FrameDetail findLowestMemoryFrame(JobInterface job) { - String pk_frame = getJdbcTemplate().queryForObject( - FIND_LOWEST_MEM_FRAME, String.class, job.getJobId(), - FrameState.SUCCEEDED.toString()); - return getFrameDetail(pk_frame); - } - - @Override - public void reorderFramesFirst(LayerInterface layer, FrameSet frameSet) { - int start; - int size = frameSet.size(); - int min = getJdbcTemplate().queryForObject( - "SELECT MIN(int_dispatch_order) FROM frame WHERE pk_layer=?", Integer.class, - layer.getLayerId()); - - start = min - size; - for (int frameIdx=0; frameIdx < size; frameIdx++) { - getJdbcTemplate().update( - "UPDATE frame SET int_dispatch_order=? WHERE str_name=? AND pk_job=?", - start, CueUtil.buildFrameName(layer, frameSet.get(frameIdx)), layer.getJobId()); - - logger.info("reordering " + CueUtil.buildFrameName(layer, frameSet.get(frameIdx)) + " to " + - start); - start++; - } - } - - @Override - public void reorderFramesLast(LayerInterface layer, FrameSet frameSet) { - int start; - int size = frameSet.size(); - List frames = new ArrayList(size); - int max = getJdbcTemplate().queryForObject( - "SELECT MAX(int_dispatch_order) FROM frame WHERE pk_layer=?", Integer.class, - layer.getLayerId()); - - start = max + 1; - for (int i=0; i <= size; i++) { - frames.add(new Object[] { start + i, CueUtil.buildFrameName(layer, i), layer.getJobId() }); - } - - if (frames.size() > 0) { - getJdbcTemplate().batchUpdate( - "UPDATE frame SET int_dispatch_order=? WHERE str_name=? AND pk_job=?", frames); - } - } - - @Override - public void reorderLayerReverse(LayerInterface layer, FrameSet frameSet) { - - int size = frameSet.size(); - List frames = new ArrayList(size); - - for (int i=0; i< size; i++) { - if (i >= size - i -1) { break; } - try { - int a = getJdbcTemplate().queryForObject( - "SELECT int_dispatch_order FROM frame WHERE str_name=? AND pk_job=? AND pk_layer=?", Integer.class, - CueUtil.buildFrameName(layer,frameSet.get(i)), layer.getJobId(), layer.getLayerId()); - - int b = getJdbcTemplate().queryForObject( - "SELECT int_dispatch_order FROM frame WHERE str_name=? AND pk_job=? AND pk_layer=?", Integer.class, - CueUtil.buildFrameName(layer,frameSet.get(size-i-1)), layer.getJobId(), layer.getLayerId()); - - frames.add(new Object[] { a, layer.getLayerId(), CueUtil.buildFrameName(layer,frameSet.get(size-i-1)) }); - frames.add(new Object[] { b, layer.getLayerId(), CueUtil.buildFrameName(layer,frameSet.get(i)) }); - - } catch (Exception e) { - logger.info("frame not found while attempting to reverse layer, skipping"); - } - } - - if (frames.size() > 0) { - getJdbcTemplate().batchUpdate( - "UPDATE frame SET int_dispatch_order=? WHERE pk_layer=? and str_name=?", frames); - } - } - - @Override - public void staggerLayer(LayerInterface layer, String frameRange, int stagger) { - - /* - * If the layer is only 1 frame we don't stagger it. - */ - if (getJdbcTemplate().queryForObject( - "SELECT int_total_count FROM layer_stat WHERE pk_layer=?", Integer.class, - layer.getLayerId()) == 1) { - return; - } - - logger.info("staggering: " + layer.getName() + " range: " + frameRange - + " on " + stagger); - - FrameSet frameSet = null; - FrameSet range = null; - - try { - frameSet = new FrameSet(frameRange + ":" + stagger); - range = new FrameSet(frameRange); - } catch (Exception e) { - logger.warn("failed to stagger layer: " + layer.getName() + ", " + e); - return; - } - - /* - * Find the dispatch order of the first frame we're working with and base - * our other staggers of this value. - */ - int first = getJdbcTemplate().queryForObject( - "SELECT int_dispatch_order FROM frame WHERE str_name=? AND pk_job=? AND pk_layer=?", Integer.class, - CueUtil.buildFrameName(layer, range.get(0)), layer.getJobId(), layer.getLayerId()); - - int size = range.size(); - for (int i=0; i < size; i++) { - int frame = range.get(i); - int newDispatchOrder = frameSet.index(frame) + first; - - getJdbcTemplate().update( - "UPDATE frame SET int_dispatch_order=? WHERE pk_layer=? and str_name=?", - newDispatchOrder, layer.getLayerId(), CueUtil.buildFrameName(layer, frame)); - } - } - - @Override - public boolean isFrameComplete(FrameInterface f) { - - String state = getJdbcTemplate().queryForObject( - "SELECT str_state FROM frame WHERE pk_frame=?", - String.class, f.getFrameId()); - - if (state.equals(FrameState.SUCCEEDED.toString()) || - state.equals(FrameState.EATEN.toString())) { - return true; - } - - return false; - } - - private static final - RowMapper RESOURCE_USAGE_MAPPER = - new RowMapper() { - public ResourceUsage mapRow(ResultSet rs, - int rowNum) throws SQLException { - return new ResourceUsage( - rs.getLong("int_clock_time"), - rs.getInt("int_cores")); - } - - }; - - @Override - public ResourceUsage getResourceUsage(FrameInterface f) { - /* - * Using systimestamp = ts_started here because ts_stopped is not set. - * Stopping the frame allows it to be dispatched again, which could - * blow away the ts_stopped time. - */ - return getJdbcTemplate().queryForObject( - "SELECT " + - "NVL(interval_to_seconds(systimestamp - ts_started),1) " + - "AS int_clock_time, " + - "NVL(int_cores,100) AS int_cores " + - "FROM " + - "frame " + - "WHERE " + - "pk_frame = ?", RESOURCE_USAGE_MAPPER, f.getFrameId()); - } - - private static final String UPDATE_FRAME_IO_USAGE = - "UPDATE " + - "frame " + - "SET " + - "ts_updated = current_timestamp," + - "ts_llu = ? " + - "WHERE " + - "pk_frame = ? "; - - @Override - public void updateFrameUsage(FrameInterface f, long lluTime) { - getJdbcTemplate().update(UPDATE_FRAME_IO_USAGE, - new Timestamp(lluTime * 1000l), f.getFrameId()); - } - - private static final String UPDATE_FRAME_MEMORY_USAGE = - "UPDATE " + - "frame " + - "SET " + - "ts_updated = systimestamp," + - "int_mem_max_used = ?," + - "int_mem_used = ? " + - "WHERE " + - "pk_frame = ? "; - - @Override - public void updateFrameMemoryUsage(FrameInterface f, long maxRss, long rss) { - getJdbcTemplate().update(UPDATE_FRAME_MEMORY_USAGE, - maxRss, rss, f.getFrameId()); - } - - /** - * Attempt a SELECT FOR UPDATE NOWAIT on the frame record. If - * the frame is being modified by another transaction or if - * the version has been incremented a FrameReservationException - * is thrown. - * - * @param frame - * @param state - */ - @Override - public void lockFrameForUpdate(FrameInterface frame, FrameState state) { - try { - getJdbcTemplate().queryForObject( - "SELECT pk_frame FROM frame WHERE pk_frame=? AND " + - "str_state=? AND int_version =? FOR UPDATE NOWAIT", - String.class, frame.getFrameId(), - state.toString(), frame.getVersion()); - } catch (Exception e) { - String error_msg = "the frame " + - frame + " was updated by another thread."; - throw new FrameReservationException(error_msg, e); - } - } - - @Override - public boolean updateFrameCheckpointState(FrameInterface frame, CheckpointState state) { - - logger.info("Setting checkpoint state to: " + state.toString()); - - boolean result = false; - - if (state.equals(CheckpointState.COMPLETE)) { - /* - * Only update the checkpoint state to complete if the state - * is either Copying or Enabled. - */ - result = getJdbcTemplate().update( - "UPDATE frame SET str_checkpoint_state=?, " + - "int_checkpoint_count=int_checkpoint_count + 1 WHERE " + - "pk_frame=? AND str_checkpoint_state IN (?, ?)", - CheckpointState.COMPLETE.toString(), - frame.getFrameId(), - CheckpointState.COPYING.toString(), - CheckpointState.ENABLED.toString()) == 1; - } - else { - result = getJdbcTemplate().update( - "UPDATE frame SET str_checkpoint_state=? WHERE pk_frame=?", - state.toString(), frame.getFrameId()) == 1; - } - - /* - * If the checkpoint state is complete or disabled then set the frame - * state back to waiting, if and only if the frame state is currently - * in the checkpoint state. - */ - if ((state.equals(CheckpointState.DISABLED)) || - state.equals(CheckpointState.COMPLETE) && result) { - getJdbcTemplate().update( - "UPDATE frame SET str_state=? WHERE pk_frame=? AND str_state=?", - FrameState.WAITING.toString(), frame.getFrameId(), - FrameState.CHECKPOINT.toString()); - } - - return result; - } - - @Override - public List getStaleCheckpoints(int cutoffTimeSec) { - return getJdbcTemplate().query( - GET_MINIMAL_FRAME + - " AND job.str_state=? " + - " AND frame.str_state=? " + - " AND systimestamp - frame.ts_stopped > interval '" + cutoffTimeSec + "' second", - FRAME_MAPPER, - JobState.PENDING.toString(), - FrameState.CHECKPOINT.toString()); - } -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/GroupDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/GroupDaoJdbc.java deleted file mode 100644 index 1656da1f5..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/GroupDaoJdbc.java +++ /dev/null @@ -1,430 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.CallableStatement; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - -import org.springframework.jdbc.core.CallableStatementCreator; -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.SqlParameter; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.DepartmentInterface; -import com.imageworks.spcue.EntityCreationError; -import com.imageworks.spcue.EntityModificationError; -import com.imageworks.spcue.EntityRemovalError; -import com.imageworks.spcue.GroupDetail; -import com.imageworks.spcue.GroupInterface; -import com.imageworks.spcue.JobInterface; -import com.imageworks.spcue.ShowInterface; -import com.imageworks.spcue.dao.GroupDao; -import com.imageworks.spcue.grpc.job.JobState; -import com.imageworks.spcue.util.CueUtil; -import com.imageworks.spcue.util.SqlUtil; - -public class GroupDaoJdbc extends JdbcDaoSupport implements GroupDao { - - private static final int MAX_NESTING_LEVEL = 10; - - @Override - public String getRootGroupId(ShowInterface show) { - return getJdbcTemplate().queryForObject( - "SELECT pk_folder FROM folder WHERE pk_show=? AND pk_parent_folder IS NULL", - String.class, show.getShowId()); - } - - @Override - public void deleteGroup(GroupInterface group) { - - if (childGroupCount(group) > 0) { - throw new EntityRemovalError("failed to delete group " + group.getName() + - ", still has sub groups"); - } - - if (childJobCount(group) > 0) { - throw new EntityRemovalError("failed to delete group " + group.getName() + - ", still has sub jobs"); - } - - // reparent all jobs to root group - getJdbcTemplate().update( - "UPDATE job SET pk_folder=? WHERE pk_folder=?", - getRootGroupId(group), group.getId()); - - getJdbcTemplate().update( - "DELETE FROM folder WHERE pk_parent_folder IS NOT NULL AND pk_folder=?", group.getId()); - } - - public static final String INSERT_GROUP = - "INSERT INTO " + - "folder " + - "( " + - "pk_folder," + - "pk_parent_folder,"+ - "pk_show, " + - "pk_dept,"+ - "str_name " + - ") " + - "VALUES (?,?,?,?,?)"; - - @Override - public void insertGroup(GroupDetail group) { - group.id = SqlUtil.genKeyRandom(); - String parentId = group.parentId; - try { - getJdbcTemplate().update(INSERT_GROUP, - group.id, parentId, group.showId, group.deptId, group.name); - } catch (Exception e) { - throw new EntityCreationError("error creating group, " + e); - } - } - - @Override - public void insertGroup(GroupDetail group, GroupInterface parent) { - if (parent != null) { - group.parentId = parent.getGroupId(); - } - insertGroup(group); - } - - @Override - public void updateGroupParent(GroupInterface group, GroupInterface dest) { - - if (group.getGroupId().equals(dest.getGroupId())) { - throw new EntityModificationError("error moving group, " + - "cannot move group into itself"); - } - - if (!group.getShowId().equals(dest.getShowId())) { - throw new EntityModificationError("error moving group, " + - "cannot move groups between shows"); - } - - int recurse = 0; - String destParent = dest.getGroupId(); - while (true) { - destParent = getJdbcTemplate().queryForObject( - "SELECT pk_parent_folder FROM folder WHERE pk_folder=?", - String.class, destParent); - if (destParent == null) { break; } - if (destParent.equals(group.getGroupId())) { - throw new EntityModificationError("error moving group, you cannot move a group " + - "into one of its sub groups"); - } - recurse++; - if (recurse > MAX_NESTING_LEVEL) { - throw new EntityModificationError("error moving group, cannot tell " + - "if your moving a group into one of its sub groups"); - } - } - - int result = getJdbcTemplate().update( - "UPDATE folder SET pk_parent_folder=? WHERE pk_folder=? AND pk_parent_folder IS NOT NULL", - dest.getId(), group.getId()); - - recurseParentChange(group.getId(), dest.getId()); - if (result == 0) { - throw new EntityModificationError("error moving group, " - + group.getName() + ", the group does not exist or its the top level group"); - } - } - - @Override - public void updateName(GroupInterface group, String value) { - getJdbcTemplate().update( - "UPDATE folder SET str_name=? WHERE pk_folder=?", - value, group.getId()); - } - - @Override - public void updateDepartment(GroupInterface group, DepartmentInterface dept) { - getJdbcTemplate().update( - "UPDATE folder SET pk_dept=? WHERE pk_folder=?", - dept.getDepartmentId(), group.getId()); - } - - @Override - public void updateDefaultJobMaxCores(GroupInterface group, int value) { - if (value <= 0) { value = CueUtil.FEATURE_DISABLED; } - if (value < CueUtil.ONE_CORE && value != CueUtil.FEATURE_DISABLED) { - String msg = "The default max cores for a job must " + - "be greater than a single core"; - throw new IllegalArgumentException(msg); - } - getJdbcTemplate().update( - "UPDATE folder SET int_job_max_cores=? WHERE pk_folder=?", - value, group.getId()); - } - - @Override - public void updateDefaultJobMinCores(GroupInterface group, int value) { - if (value <= 0) { value = CueUtil.FEATURE_DISABLED; } - if (value < CueUtil.ONE_CORE && value != CueUtil.FEATURE_DISABLED) { - String msg = "The default min cores for a job must " + - "be greater than a single core"; - throw new IllegalArgumentException(msg); - } - getJdbcTemplate().update( - "UPDATE folder SET int_job_min_cores=? WHERE pk_folder=?", - value, group.getId()); - } - - @Override - public void updateMaxCores(GroupInterface group, int value) { - if (value < 0) { value = CueUtil.FEATURE_DISABLED; } - if (value < CueUtil.ONE_CORE && value != CueUtil.FEATURE_DISABLED) { - String msg = "The group max cores feature must " + - "be a whole core or greater, pass in: " + value; - throw new IllegalArgumentException(msg); - } - - getJdbcTemplate().update( - "UPDATE folder_resource SET int_max_cores=? WHERE pk_folder=?", - value, group.getId()); - } - - @Override - public void updateMinCores(GroupInterface group, int value) { - if (value < 0) { value = 0; } - getJdbcTemplate().update( - "UPDATE folder_resource SET int_min_cores=? WHERE pk_folder=?", - value, group.getId()); - } - - private static final String IS_OVER_MIN_CORES = - "SELECT " + - "COUNT(1) " + - "FROM " + - "job,"+ - "folder_resource fr "+ - "WHERE " + - "job.pk_folder = fr.pk_folder " + - "AND " + - "fr.int_cores > fr.int_min_cores " + - "AND "+ - "job.pk_job = ?"; - - @Override - public boolean isOverMinCores(JobInterface job) { - return getJdbcTemplate().queryForObject(IS_OVER_MIN_CORES, - Integer.class, job.getJobId()) > 0; - } - - @Override - public void updateDefaultJobPriority(GroupInterface group, int value) { - if (value < 0) { value = CueUtil.FEATURE_DISABLED; } - getJdbcTemplate().update( - "UPDATE folder SET int_job_priority=? WHERE pk_folder=?", - value, group.getId()); - if (value != CueUtil.FEATURE_DISABLED) { - getJdbcTemplate().update( - "UPDATE job_resource SET int_priority=? WHERE pk_job IN (" + - "SELECT pk_job from job WHERE pk_folder=?)", - value, group.getId()); - } - } - - private static final String GET_GROUP_DETAIL = - "SELECT " + - "folder.pk_folder, " + - "folder.int_job_max_cores,"+ - "folder.int_job_min_cores,"+ - "folder.int_job_priority,"+ - "folder.str_name,"+ - "folder.pk_parent_folder,"+ - "folder.pk_show,"+ - "folder.pk_dept,"+ - "folder_level.int_level, " + - "folder_resource.int_min_cores,"+ - "folder_resource.int_max_cores " + - "FROM " + - "folder, "+ - "folder_level, " + - "folder_resource " + - "WHERE " + - "folder.pk_folder = folder_level.pk_folder " + - "AND " + - "folder.pk_folder = folder_resource.pk_folder"; - - private static final String GET_GROUP_DETAIL_BY_JOB = - "SELECT " + - "folder.pk_folder, " + - "folder.int_job_max_cores,"+ - "folder.int_job_min_cores,"+ - "folder.int_job_priority,"+ - "folder.str_name,"+ - "folder.pk_parent_folder,"+ - "folder.pk_show,"+ - "folder.pk_dept,"+ - "folder_level.int_level, " + - "folder_resource.int_min_cores,"+ - "folder_resource.int_max_cores " + - "FROM " + - "folder, "+ - "folder_level, " + - "folder_resource, " + - "job "+ - "WHERE " + - "folder.pk_folder = folder_level.pk_folder " + - "AND " + - "folder.pk_folder = folder_resource.pk_folder " + - "AND " + - "job.pk_folder = folder.pk_folder " + - "AND " + - "job.pk_job = ?"; - - @Override - public GroupDetail getGroupDetail(String id) { - return getJdbcTemplate().queryForObject( - GET_GROUP_DETAIL + " AND folder.pk_folder=?", GROUP_DETAIL_MAPPER, id); - } - - @Override - public GroupDetail getGroupDetail(JobInterface job) { - return getJdbcTemplate().queryForObject(GET_GROUP_DETAIL_BY_JOB, - GROUP_DETAIL_MAPPER, job.getId()); - } - - @Override - public GroupDetail getRootGroupDetail(ShowInterface show) { - return getJdbcTemplate().queryForObject( - GET_GROUP_DETAIL + " AND folder.pk_show=? AND pk_parent_folder IS NULL", - GROUP_DETAIL_MAPPER, show.getShowId()); - } - - @Override - public GroupInterface getGroup(String id) { - return getJdbcTemplate().queryForObject( - "SELECT pk_show, pk_folder,str_name FROM folder WHERE pk_folder=?", - GROUP_MAPPER, id); - } - - @Override - public List getGroups(List idl) { - return getJdbcTemplate().query( - "SELECT pk_show, pk_folder, str_name FROM folder WHERE " + - SqlUtil.buildBindVariableArray("pk_folder", idl.size()), - GROUP_MAPPER, idl.toArray()); - } - - @Override - public List getChildrenRecursive(GroupInterface group) { - List groups = new ArrayList(32); - GroupInterface current = group; - for (GroupInterface g: getChildren(current)) { - current = g; - groups.add(current); - groups.addAll(getChildrenRecursive(current)); - } - return groups; - } - - @Override - public List getChildren(GroupInterface group) { - return getJdbcTemplate().query( - "SELECT pk_show, pk_folder, str_name FROM folder WHERE pk_parent_folder = ?", - GROUP_MAPPER, group.getGroupId()); - } - - private static final String IS_MANAGED = - "SELECT " + - "COUNT(1) " + - "FROM " + - "folder, " + - "point " + - "WHERE " + - "folder.pk_show = point.pk_show " + - "AND " + - "folder.pk_dept = point.pk_dept " + - "AND " + - "folder.b_exclude_managed = 0 " + - "AND " + - "point.b_managed = 1 " + - "AND " + - "folder.pk_folder = ?"; - - @Override - public boolean isManaged(GroupInterface group) { - return getJdbcTemplate().queryForObject(IS_MANAGED, - Integer.class, group.getGroupId()) > 0; - } - - public static final RowMapper GROUP_MAPPER = - new RowMapper() { - public GroupInterface mapRow(final ResultSet rs, int rowNum) throws SQLException { - return new GroupInterface() { - String id = rs.getString("pk_folder"); - String show = rs.getString("pk_show"); - String name = rs.getString("str_name"); - public String getGroupId() { return id; } - public String getShowId() { return show; } - public String getId() { return id; } - public String getName() { return name; } - }; - } - }; - - public static final RowMapper GROUP_DETAIL_MAPPER = - new RowMapper() { - public GroupDetail mapRow(ResultSet rs, int rowNum) throws SQLException { - GroupDetail group = new GroupDetail(); - group.id = rs.getString("pk_folder"); - group.jobMaxCores = rs.getInt("int_job_max_cores"); - group.jobMinCores = rs.getInt("int_job_min_cores"); - group.jobPriority = rs.getInt("int_job_priority"); - group.name = rs.getString("str_name"); - group.parentId = rs.getString("pk_parent_folder"); - group.showId = rs.getString("pk_show"); - group.deptId = rs.getString("pk_dept"); - return group; - } - }; - - - private int childGroupCount(GroupInterface group) { - return getJdbcTemplate().queryForObject( - "SELECT COUNT(*) FROM folder WHERE pk_parent_folder=?", - Integer.class, group.getId()); - } - - private int childJobCount(GroupInterface group) { - return getJdbcTemplate().queryForObject( - "SELECT COUNT(*) FROM job WHERE pk_folder=? AND str_state=?", - Integer.class, group.getId(), JobState.PENDING.toString()); - } - - private void recurseParentChange(final String folderId, final String newParentId) { - getJdbcTemplate().call(new CallableStatementCreator() { - - public CallableStatement createCallableStatement(Connection con) throws SQLException { - CallableStatement c = con.prepareCall("{ call recurse_folder_parent_change(?,?) }"); - c.setString(1, folderId); - c.setString(2, newParentId); - return c; - } - }, new ArrayList()); - } -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/HistoricalDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/HistoricalDaoJdbc.java deleted file mode 100644 index ec30f7700..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/HistoricalDaoJdbc.java +++ /dev/null @@ -1,52 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.util.List; - -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.JobInterface; -import com.imageworks.spcue.dao.HistoricalDao; -import com.imageworks.spcue.grpc.job.JobState; - -public class HistoricalDaoJdbc extends JdbcDaoSupport implements HistoricalDao { - - private static final String GET_FINISHED_JOBS = - JobDaoJdbc.GET_JOB + - "WHERE " + - "job.str_state=? " + - "AND " + - "systimestamp - job.ts_stopped > "; - - public List getFinishedJobs(int cutoffHours) { - String interval = "interval '" + cutoffHours + "' hour"; - return getJdbcTemplate().query(GET_FINISHED_JOBS + interval, - JobDaoJdbc.JOB_MAPPER, JobState.FINISHED.toString()); - } - - public void transferJob(JobInterface job) { - /** - * All of the historical transfer happens inside of triggers - */ - getJdbcTemplate().update("DELETE FROM job WHERE pk_job=?", job.getJobId()); - } -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/HostDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/HostDaoJdbc.java deleted file mode 100644 index 15c74278c..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/HostDaoJdbc.java +++ /dev/null @@ -1,711 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.sql.CallableStatement; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.springframework.dao.EmptyResultDataAccessException; -import org.springframework.jdbc.core.CallableStatementCreator; -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.SqlParameter; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.AllocationInterface; -import com.imageworks.spcue.DispatchHost; -import com.imageworks.spcue.EntityCreationError; -import com.imageworks.spcue.HostEntity; -import com.imageworks.spcue.HostInterface; -import com.imageworks.spcue.LocalHostAssignment; -import com.imageworks.spcue.Source; -import com.imageworks.spcue.dao.HostDao; -import com.imageworks.spcue.dispatcher.Dispatcher; -import com.imageworks.spcue.dispatcher.ResourceReservationFailureException; -import com.imageworks.spcue.grpc.host.HardwareState; -import com.imageworks.spcue.grpc.host.HostTagType; -import com.imageworks.spcue.grpc.host.LockState; -import com.imageworks.spcue.grpc.host.ThreadMode; -import com.imageworks.spcue.grpc.report.HostReport; -import com.imageworks.spcue.grpc.report.RenderHost; -import com.imageworks.spcue.util.CueUtil; -import com.imageworks.spcue.util.SqlUtil; - - -public class HostDaoJdbc extends JdbcDaoSupport implements HostDao { - - public static final RowMapper HOST_DETAIL_MAPPER = new RowMapper() { - public HostEntity mapRow(ResultSet rs, int rowNum) throws SQLException { - HostEntity host = new HostEntity(); - host.facilityId = rs.getString("pk_facility"); - host.allocId = rs.getString("pk_alloc"); - host.id = rs.getString("pk_host"); - host.lockState = LockState.valueOf(rs.getString("str_lock_state")); - host.name = rs.getString("str_name"); - host.nimbyEnabled = rs.getBoolean("b_nimby"); - host.state = HardwareState.valueOf(rs.getString("str_state")); - host.unlockAtBoot = rs.getBoolean("b_unlock_boot"); - host.cores = rs.getInt("int_cores"); - host.idleCores = rs.getInt("int_cores_idle"); - host.memory = rs.getInt("int_mem"); - host.idleMemory = rs.getInt("int_mem_idle"); - host.gpu = rs.getInt("int_gpu"); - host.idleGpu = rs.getInt("int_gpu_idle"); - host.dateBooted = rs.getDate("ts_booted"); - host.dateCreated = rs.getDate("ts_created"); - host.datePinged = rs.getDate("ts_ping"); - return host; - } - }; - - public static final RowMapper HOST_MAPPER = new RowMapper() { - public HostInterface mapRow(final ResultSet rs, int rowNum) throws SQLException { - return new HostInterface() { - final String id = rs.getString("pk_host"); - final String allocid = rs.getString("pk_alloc"); - final String name = rs.getString("str_name"); - final String facility = rs.getString("pk_facility"); - - public String getHostId() { return id; } - public String getAllocationId() { return allocid; } - public String getId() { return id; } - public String getName() { return name; } - public String getFacilityId() { return facility; }; - }; - } - }; - - private static final String GET_HOST_DETAIL = - "SELECT " + - "host.pk_host, " + - "host.pk_alloc,"+ - "host.str_lock_state,"+ - "host.b_nimby,"+ - "host.b_unlock_boot,"+ - "host.int_cores,"+ - "host.int_cores_idle,"+ - "host.int_mem,"+ - "host.int_mem_idle,"+ - "host.int_gpu,"+ - "host.int_gpu_idle,"+ - "host.ts_created,"+ - "host.str_name, " + - "host_stat.str_state,"+ - "host_stat.ts_ping,"+ - "host_stat.ts_booted, "+ - "alloc.pk_facility " + - "FROM " + - "host, " + - "alloc, " + - "host_stat " + - "WHERE " + - "host.pk_host = host_stat.pk_host " + - "AND " + - "host.pk_alloc = alloc.pk_alloc "; - - @Override - public void lockForUpdate(HostInterface host) { - try { - getJdbcTemplate().queryForObject( - "SELECT pk_host FROM host WHERE pk_host=? " + - "FOR UPDATE NOWAIT", - String.class, host.getHostId()); - } catch (Exception e) { - throw new ResourceReservationFailureException("unable to lock host " + - host.getName() + ", the host was locked by another thread.", e); - } - } - - @Override - public HostEntity getHostDetail(HostInterface host) { - return getJdbcTemplate().queryForObject(GET_HOST_DETAIL + " AND host.pk_host=?", - HOST_DETAIL_MAPPER, host.getHostId()); - } - - @Override - public HostEntity getHostDetail(String id) { - return getJdbcTemplate().queryForObject(GET_HOST_DETAIL + " AND host.pk_host=?", - HOST_DETAIL_MAPPER, id); - } - - @Override - public HostEntity findHostDetail(String name) { - return getJdbcTemplate().queryForObject(GET_HOST_DETAIL + " AND host.str_name=?", - HOST_DETAIL_MAPPER, name); - } - - private static final String GET_HOST= - "SELECT " + - "host.pk_host, " + - "host.pk_alloc,"+ - "host.str_name, " + - "alloc.pk_facility " + - "FROM " + - "host," + - "alloc " + - "WHERE " + - "host.pk_alloc = alloc.pk_alloc " ; - - @Override - public HostInterface getHost(String id) { - return getJdbcTemplate().queryForObject(GET_HOST + " AND host.pk_host=?", - HOST_MAPPER, id); - } - - @Override - public HostInterface getHost(LocalHostAssignment l) { - return getJdbcTemplate().queryForObject(GET_HOST + " AND host.pk_host = ("+ - "SELECT pk_host FROM host_local WHERE pk_host_local=?)", - HOST_MAPPER, l.getId()); - } - - @Override - public HostInterface findHost(String name) { - return getJdbcTemplate().queryForObject( - GET_HOST + " AND (host.str_name=? OR host.str_fqdn=?)", - HOST_MAPPER, name, name); - } - - public static final RowMapper DISPATCH_HOST_MAPPER = - new RowMapper() { - public DispatchHost mapRow(ResultSet rs, int rowNum) throws SQLException { - DispatchHost host = new DispatchHost(); - host.id = rs.getString("pk_host"); - host.allocationId = rs.getString("pk_alloc"); - host.facilityId = rs.getString("pk_facility"); - host.name = rs.getString("str_name"); - host.lockState = LockState.valueOf(rs.getString("str_lock_state")); - host.memory = rs.getInt("int_mem"); - host.cores = rs.getInt("int_cores"); - host.gpu= rs.getInt("int_gpu"); - host.idleMemory= rs.getInt("int_mem_idle"); - host.idleCores = rs.getInt("int_cores_idle"); - host.idleGpu= rs.getInt("int_gpu_idle"); - host.isNimby = rs.getBoolean("b_nimby"); - host.threadMode = rs.getInt("int_thread_mode"); - host.tags = rs.getString("str_tags"); - host.os = rs.getString("str_os"); - host.hardwareState = - HardwareState.valueOf(rs.getString("str_state")); - return host; - } - }; - - public static final String GET_DISPATCH_HOST = - "SELECT " + - "host.pk_host,"+ - "host.pk_alloc,"+ - "host.str_name," + - "host.str_lock_state, " + - "host.int_cores, "+ - "host.int_cores_idle, " + - "host.int_mem,"+ - "host.int_mem_idle, "+ - "host.int_gpu,"+ - "host.int_gpu_idle, "+ - "host.b_nimby, "+ - "host.int_thread_mode, "+ - "host.str_tags, " + - "host_stat.str_os, " + - "host_stat.str_state, " + - "alloc.pk_facility " + - "FROM " + - "host " + - "INNER JOIN host_stat " + - "ON (host.pk_host = host_stat.pk_host) " + - "INNER JOIN alloc " + - "ON (host.pk_alloc = alloc.pk_alloc) "; - - @Override - public DispatchHost findDispatchHost(String name) { - try { - return getJdbcTemplate().queryForObject( - GET_DISPATCH_HOST + - "WHERE (host.str_name=? OR host.str_fqdn=?)", - DISPATCH_HOST_MAPPER, name, name); - } catch (EmptyResultDataAccessException e) { - throw new EmptyResultDataAccessException( - "Failed to find host " + name, 1); - } - } - - @Override - public DispatchHost getDispatchHost(String id) { - return getJdbcTemplate().queryForObject( - GET_DISPATCH_HOST + - "WHERE host.pk_host=?", - DISPATCH_HOST_MAPPER, id); - } - - private static final String[] INSERT_HOST_DETAIL = - { - "INSERT INTO " + - "host " + - "("+ - "pk_host, " + - "pk_alloc, " + - "str_name, " + - "b_nimby, " + - "str_lock_state, " + - "int_procs,"+ - "int_cores, " + - "int_cores_idle, " + - "int_mem,"+ - "int_mem_idle,"+ - "int_gpu,"+ - "int_gpu_idle,"+ - "str_fqdn, " + - "int_thread_mode "+ - ") " + - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)", - - "INSERT INTO " + - "host_stat " + - "("+ - "pk_host_stat," + - "pk_host,"+ - "int_mem_total, " + - "int_mem_free,"+ - "int_gpu_total, " + - "int_gpu_free,"+ - "int_swap_total, " + - "int_swap_free,"+ - "int_mcp_total, " + - "int_mcp_free,"+ - "int_load, " + - "ts_booted, " + - "str_state, " + - "str_os " + - ") "+ - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)" - - }; - - @Override - public void insertRenderHost(RenderHost host, AllocationInterface a, boolean useLongNames) { - - ThreadMode threadMode = ThreadMode.AUTO; - if (host.getNimbyEnabled()) { - threadMode = ThreadMode.ALL; - } - - long memUnits = convertMemoryUnits(host); - if (memUnits < Dispatcher.MEM_RESERVED_MIN) { - throw new EntityCreationError("could not create host " + host.getName() + ", " + - " must have at least " + Dispatcher.MEM_RESERVED_MIN + " free memory."); - } - - String fqdn; - String name = host.getName(); - try { - fqdn = InetAddress.getByName(host.getName()).getCanonicalHostName(); - // If the provided host name matches the pinged name, use the pinged name. - // Otherwise use the provided name. - // If the host lookup fails, use the provided name. - // In all cases attempt to strip off the domain when setting the name. - if (fqdn.equals(host.getName())) { - name = getHostNameFromFQDN(fqdn, useLongNames); - } - else { - name = getHostNameFromFQDN(host.getName(), useLongNames); - fqdn = host.getName(); - } - } catch (UnknownHostException e) { - logger.warn(e); - fqdn = host.getName(); - name = getHostNameFromFQDN(name, useLongNames); - } - - String hid = SqlUtil.genKeyRandom(); - int coreUnits = host.getNumProcs() * host.getCoresPerProc(); - String os = host.getAttributesMap().get("SP_OS"); - if (os == null) { - os = Dispatcher.OS_DEFAULT; - } - - long totalGpu; - if (host.getAttributesMap().containsKey("totalGpu")) - totalGpu = Integer.parseInt(host.getAttributesMap().get("totalGpu")); - else - totalGpu = 0; - - long freeGpu; - if (host.getAttributesMap().containsKey("freeGpu")) - freeGpu = Integer.parseInt(host.getAttributesMap().get("freeGpu")); - else - freeGpu = 0; - - getJdbcTemplate().update(INSERT_HOST_DETAIL[0], - hid, a.getAllocationId(), name, host.getNimbyEnabled(), - LockState.OPEN.toString(), host.getNumProcs(), coreUnits, coreUnits, - memUnits, memUnits, totalGpu, totalGpu, - fqdn, threadMode.getNumber()); - - getJdbcTemplate().update(INSERT_HOST_DETAIL[1], - hid, hid, host.getTotalMem(), host.getFreeMem(), - totalGpu, freeGpu, - host.getTotalSwap(), host.getFreeSwap(), - host.getTotalMcp(), host.getFreeMcp(), - host.getLoad(), new Timestamp(host.getBootTime() * 1000l), - host.getState().toString(), os); - } - - @Override - public void recalcuateTags(final String id) { - getJdbcTemplate().call(new CallableStatementCreator() { - public CallableStatement createCallableStatement(Connection con) throws SQLException { - CallableStatement c = con.prepareCall("{ call recalculate_tags(?) }"); - c.setString(1, id); - return c; - } - }, new ArrayList()); - } - - private static final String UPDATE_RENDER_HOST = - "UPDATE " + - "host_stat " + - "SET " + - "int_mem_total=?, " + - "int_mem_free=?, " + - "int_swap_total=?, " + - "int_swap_free=?, "+ - "int_mcp_total=?, " + - "int_mcp_free=?, " + - "int_gpu_total=?, " + - "int_gpu_free=?, " + - "int_load=?," + - "ts_booted = ?, " + - "ts_ping = systimestamp, "+ - "str_os=? " + - "WHERE " + - "pk_host=?"; - - @Override - public void updateHostStats(HostInterface host, - long totalMemory, long freeMemory, - long totalSwap, long freeSwap, - long totalMcp, long freeMcp, - long totalGpu, long freeGpu, - int load, Timestamp bootTime, - String os) { - - if (os == null) { - os = Dispatcher.OS_DEFAULT; - } - - getJdbcTemplate().update(UPDATE_RENDER_HOST, - totalMemory, freeMemory, totalSwap, - freeSwap, totalMcp, freeMcp, totalGpu, freeGpu, load, - bootTime, os, host.getHostId()); - } - - @Override - public boolean hostExists(String hostname) { - try { - return getJdbcTemplate().queryForObject( - "SELECT 1 FROM host WHERE (str_fqdn=? OR str_name=?)", - Integer.class, hostname, hostname) > 0; - } catch (EmptyResultDataAccessException e) { - return false; - } - } - - @Override - public void updateHostResources(HostInterface host, HostReport report) { - - long memory = convertMemoryUnits(report.getHost()); - int cores = report.getHost().getNumProcs() * report.getHost().getCoresPerProc(); - - long totalGpu; - if (report.getHost().getAttributesMap().containsKey("totalGpu")) - totalGpu = Integer.parseInt(report.getHost().getAttributesMap().get("totalGpu")); - else - totalGpu = 0; - - getJdbcTemplate().update( - "UPDATE " + - "host " + - "SET " + - "b_nimby=?,"+ - "int_cores=?," + - "int_cores_idle=?," + - "int_mem=?," + - "int_mem_idle=?, " + - "int_gpu=?," + - "int_gpu_idle=? " + - "WHERE " + - "pk_host=? "+ - "AND " + - "int_cores = int_cores_idle " + - "AND " + - "int_mem = int_mem_idle", - report.getHost().getNimbyEnabled(), cores, cores, - memory, memory, totalGpu, totalGpu, host.getId()); - } - - @Override - public void updateHostLock(HostInterface host, LockState state, Source source) { - getJdbcTemplate().update( - "UPDATE host SET str_lock_state=?, str_lock_source=? WHERE pk_host=?", - state.toString(), source.toString(), host.getHostId()); - } - - @Override - public void updateHostRebootWhenIdle(HostInterface host, boolean enabled) { - getJdbcTemplate().update("UPDATE host SET b_reboot_idle=? WHERE pk_host=?", - enabled, host.getHostId()); - } - - @Override - public void deleteHost(HostInterface host) { - getJdbcTemplate().update( - "DELETE FROM comments WHERE pk_host=?",host.getHostId()); - getJdbcTemplate().update( - "DELETE FROM host WHERE pk_host=?",host.getHostId()); - } - - @Override - public void deleteDownHosts() { - // Not implemented. - } - - @Override - public void updateHostState(HostInterface host, HardwareState state) { - getJdbcTemplate().update( - "UPDATE host_stat SET str_state=? WHERE pk_host=?", - state.toString(), host.getHostId()); - } - - @Override - public void updateHostSetAllocation(HostInterface host, AllocationInterface alloc) { - - String tag = getJdbcTemplate().queryForObject( - "SELECT str_tag FROM alloc WHERE pk_alloc=?", - String.class, alloc.getAllocationId()); - getJdbcTemplate().update( - "UPDATE host SET pk_alloc=? WHERE pk_host=?", - alloc.getAllocationId(), host.getHostId()); - - removeTagsByType(host, HostTagType.ALLOC); - tagHost(host, tag, HostTagType.ALLOC); - } - - @Override - public boolean isHostLocked(HostInterface host) { - return getJdbcTemplate().queryForObject( - "SELECT COUNT(1) FROM host WHERE pk_host=? AND str_lock_state!=?", - Integer.class, host.getHostId(), LockState.OPEN.toString()) > 0; - } - - private static final String INSERT_TAG = - "INSERT INTO " + - "host_tag " + - "(" + - "pk_host_tag,"+ - "pk_host,"+ - "str_tag,"+ - "str_tag_type, " + - "b_constant " + - ") VALUES (?,?,?,?,?)"; - - - @Override - public void tagHost(String id, String tag, HostTagType type) { - boolean constant = false; - if (type.equals(HostTagType.ALLOC)) - constant = true; - - getJdbcTemplate().update(INSERT_TAG, - SqlUtil.genKeyRandom(), id, tag.trim(), type.toString(), constant); - } - - @Override - public void tagHost(HostInterface host, String tag, HostTagType type) { - tagHost(host.getHostId(), tag, type); - } - - @Override - public void removeTagsByType(HostInterface host, HostTagType type) { - getJdbcTemplate().update("DELETE FROM host_tag WHERE pk_host=? AND str_tag_type=?", - host.getHostId(), type.toString()); - } - - @Override - public void removeTag(HostInterface host, String tag) { - getJdbcTemplate().update( - "DELETE FROM host_tag WHERE pk_host=? AND str_tag=? AND b_constant=0", - host.getHostId(), tag); - } - - @Override - public void renameTag(HostInterface host, String oldTag, String newTag) { - getJdbcTemplate().update( - "UPDATE host_tag SET str_tag=? WHERE pk_host=? AND str_tag=? AND b_constant=0", - newTag, host.getHostId(), oldTag); - } - - @Override - public void updateThreadMode(HostInterface host, ThreadMode mode) { - getJdbcTemplate().update( - "UPDATE host SET int_thread_mode=? WHERE pk_host=?", - mode.getNumber(), host.getHostId()); - } - - @Override - public void updateHostOs(HostInterface host, String os) { - getJdbcTemplate().update( - "UPDATE host_stat SET str_os=? WHERE pk_host=?", - os, host.getHostId()); - } - - @Override - public boolean isKillMode(HostInterface h) { - return getJdbcTemplate().queryForObject( - "SELECT COUNT(1) FROM host_stat WHERE pk_host = ? " + - "AND int_swap_total - int_swap_free > ? AND int_mem_free < ?", - Integer.class, h.getHostId(), Dispatcher.KILL_MODE_SWAP_THRESHOLD, - Dispatcher.KILL_MODE_MEM_THRESHOLD) > 0; - } - - @Override - public int getStrandedCoreUnits(HostInterface h) { - try { - int idle_cores = getJdbcTemplate().queryForObject( - "SELECT int_cores_idle FROM host WHERE pk_host = ? AND int_mem_idle <= ?", - Integer.class, h.getHostId(), - Dispatcher.MEM_STRANDED_THRESHHOLD); - return (int) (Math.floor(idle_cores / 100.0)) * 100; - } catch (EmptyResultDataAccessException e) { - return 0; - } - } - - private static final String IS_HOST_UP = - "SELECT " + - "COUNT(1) " + - "FROM " + - "host_stat "+ - "WHERE " + - "host_stat.str_state = ? " + - "AND " + - "host_stat.pk_host = ? "; - - @Override - public boolean isHostUp(HostInterface host) { - return getJdbcTemplate().queryForObject(IS_HOST_UP, - Integer.class, HardwareState.UP.toString(), - host.getHostId()) == 1; - } - - private static final String IS_PREFER_SHOW = - "SELECT " + - "COUNT(1) " + - "FROM " + - "host," + - "owner," + - "deed "+ - "WHERE " + - "host.pk_host = deed.pk_host " + - "AND " + - "deed.pk_owner = owner.pk_owner " + - "AND " + - "host.pk_host = ?"; - - @Override - public boolean isPreferShow(HostInterface h) { - return getJdbcTemplate().queryForObject(IS_PREFER_SHOW, - Integer.class, h.getHostId()) > 0; - } - - @Override - public boolean isNimbyHost(HostInterface h) { - return getJdbcTemplate().queryForObject( - "SELECT COUNT(1) FROM host WHERE b_nimby=1 AND pk_host=?", - Integer.class, h.getHostId()) > 0; - } - - /** - * Checks if the passed in name looks like a fully qualified domain name. - * If so, returns the hostname without the domain. Otherwise returns the passed - * in name unchanged. - * @param fqdn - String - * @return String - hostname - */ - private String getHostNameFromFQDN(String fqdn, Boolean useLongNames) { - String hostName; - Pattern ipPattern = Pattern.compile("^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}$"); - Matcher ipMatcher = ipPattern.matcher(fqdn); - if (ipMatcher.matches()){ - hostName = fqdn; - } - else if (useLongNames) { - hostName = fqdn; - Pattern domainPattern = Pattern.compile( - ".*(\\.(.*)\\.(co(m|.[a-z]{2})|biz|edu|info|net|org|cn|de|eu|nl))$"); - Matcher domainMatcher = domainPattern.matcher(fqdn); - if (domainMatcher.matches()){ - hostName = fqdn.replace(domainMatcher.group(1), ""); - } - } - else { - hostName = fqdn.split("\\.")[0]; - } - return hostName; - - } - - /** - * Converts the amount of memory reported by the machine - * to a modificed value which takes into account the - * operating system and the possibility of user applications. - * - * @param host - * @return - */ - private long convertMemoryUnits(RenderHost host) { - - long memUnits; - if (host.getTagsList().contains("64bit")) { - memUnits = CueUtil.convertKbToFakeKb64bit(host.getTotalMem()); - } - else { - memUnits = CueUtil.convertKbToFakeKb32bit(host.getTotalMem()); - } - - /* - * If this is a desktop, we'll just cut the memory - * so we don't annoy the user. - */ - if (host.getNimbyEnabled()) { - memUnits = (long) (memUnits / 1.5) + Dispatcher.MEM_RESERVED_SYSTEM; - } - - return memUnits; - } - -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/JobDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/JobDaoJdbc.java deleted file mode 100644 index 0f591a350..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/JobDaoJdbc.java +++ /dev/null @@ -1,966 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.math.BigDecimal; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.springframework.dao.DataAccessException; -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.BuildableJob; -import com.imageworks.spcue.DepartmentInterface; -import com.imageworks.spcue.DispatchJob; -import com.imageworks.spcue.EntityModificationError; -import com.imageworks.spcue.ExecutionSummary; -import com.imageworks.spcue.FacilityInterface; -import com.imageworks.spcue.FrameStateTotals; -import com.imageworks.spcue.GroupDetail; -import com.imageworks.spcue.GroupInterface; -import com.imageworks.spcue.Inherit; -import com.imageworks.spcue.JobDetail; -import com.imageworks.spcue.JobInterface; -import com.imageworks.spcue.ResourceUsage; -import com.imageworks.spcue.ShowInterface; -import com.imageworks.spcue.TaskEntity; -import com.imageworks.spcue.dao.JobDao; -import com.imageworks.spcue.grpc.job.FrameState; -import com.imageworks.spcue.grpc.job.JobState; -import com.imageworks.spcue.util.CueUtil; -import com.imageworks.spcue.util.JobLogUtil; -import com.imageworks.spcue.util.SqlUtil; - -public class JobDaoJdbc extends JdbcDaoSupport implements JobDao { - private static final Pattern LAST_JOB_STRIP_PATTERN = Pattern.compile("_v*([_0-9]*$)"); - - /* - * Maps a row to a DispatchJob object - */ - public static final RowMapper DISPATCH_JOB_MAPPER = new RowMapper() { - public DispatchJob mapRow(ResultSet rs, int rowNum) throws SQLException { - DispatchJob job = new DispatchJob(); - job.id = rs.getString("pk_job"); - job.showId = rs.getString("pk_show"); - job.facilityId = rs.getString("pk_facility"); - job.name = rs.getString("str_name"); - job.state = JobState.valueOf(rs.getString("str_state")); - job.maxRetries = rs.getInt("int_max_retries"); - job.paused = rs.getBoolean("b_paused"); - job.autoEat = rs.getBoolean("b_autoeat"); - job.autoBook = rs.getBoolean("b_auto_book"); - job.autoUnbook = rs.getBoolean("b_auto_unbook"); - return job; - } - }; - - /* - * Maps a row to minimal job. - */ - public static final RowMapper JOB_MAPPER = new RowMapper() { - public JobInterface mapRow(final ResultSet rs, int rowNum) throws SQLException { - return new JobInterface() { - final String jobid = rs.getString("pk_job"); - final String showid = rs.getString("pk_show"); - final String name = rs.getString("str_name"); - final String facility = rs.getString("pk_facility"); - public String getJobId() { return jobid; } - public String getShowId() { return showid; } - public String getId() { return jobid; } - public String getName() { return name; } - public String getFacilityId() { return facility; } - }; - } - }; - - /* - * Maps a row to a JobDetail object - */ - private static final RowMapper JOB_DETAIL_MAPPER = - new RowMapper() { - public JobDetail mapRow(ResultSet rs, int rowNum) throws SQLException { - JobDetail job = new JobDetail(); - job.id = rs.getString("pk_job"); - job.showId = rs.getString("pk_show"); - job.facilityId = rs.getString("pk_show"); - job.deptId = rs.getString("pk_dept"); - job.groupId = rs.getString("pk_folder"); - job.logDir = rs.getString("str_log_dir"); - job.maxCoreUnits = rs.getInt("int_max_cores"); - job.minCoreUnits = rs.getInt("int_min_cores"); - job.name = rs.getString("str_name"); - job.priority = rs.getInt("int_priority"); - job.shot = rs.getString("str_shot"); - job.state = JobState.valueOf(rs.getString("str_state")); - int uid = rs.getInt("int_uid"); - job.uid = rs.wasNull() ? Optional.empty() : Optional.of(uid); - job.user = rs.getString("str_user"); - job.email = rs.getString("str_email"); - job.totalFrames = rs.getInt("int_frame_count"); - job.totalLayers = rs.getInt("int_layer_count"); - job.isPaused = rs.getBoolean("b_paused"); - job.maxRetries = rs.getInt("int_max_retries"); - job.showName = rs.getString("show_name"); - job.facilityName = rs.getString("facility_name"); - job.deptName = rs.getString("dept_name"); - return job; - } - }; - - private static final String GET_DISPATCH_JOB = - "SELECT " + - "job.pk_job, " + - "job.pk_facility, " + - "job.pk_show, " + - "job.str_name, "+ - "job.str_show, " + - "job.str_state, "+ - "job.b_paused, "+ - "job.int_max_retries, " + - "job.b_autoeat, " + - "job.b_auto_book,"+ - "job.b_auto_unbook " + - "FROM " + - "job "+ - "WHERE " + - "pk_job = ?"; - - @Override - public DispatchJob getDispatchJob(String uuid) { - return getJdbcTemplate().queryForObject( - GET_DISPATCH_JOB, DISPATCH_JOB_MAPPER, uuid); - } - - private static final String IS_JOB_COMPLETE = - "SELECT " + - "SUM (" + - "int_waiting_count + " + - "int_running_count + " + - "int_dead_count + " + - "int_depend_count + " + - "int_checkpoint_count " + - ") " + - "FROM " + - "job_stat " + - "WHERE " + - "pk_job=?"; - - @Override - public boolean isJobComplete(JobInterface job) { - if (isLaunching(job)) { - return false; - } - return getJdbcTemplate().queryForObject(IS_JOB_COMPLETE, - Integer.class, job.getJobId()) == 0; - } - - public static final String GET_JOB= - "SELECT " + - "job.pk_job, "+ - "job.pk_show, "+ - "job.pk_dept,"+ - "job.pk_facility,"+ - "job.str_name " + - "FROM " + - "job "; - - private static final String GET_JOB_DETAIL = - "SELECT " + - "job.pk_job,"+ - "job.pk_show,"+ - "job.pk_facility,"+ - "job.pk_dept,"+ - "job.pk_folder,"+ - "job.str_log_dir,"+ - "job.str_name,"+ - "job.str_shot,"+ - "job.str_state,"+ - "job.int_uid,"+ - "job.str_user,"+ - "job.str_email,"+ - "job.int_frame_count,"+ - "job.int_layer_count,"+ - "job.b_paused,"+ - "job.int_max_retries,"+ - "job_resource.int_max_cores,"+ - "job_resource.int_min_cores,"+ - "job_resource.int_priority,"+ - "show.str_name AS show_name, " + - "dept.str_name AS dept_name, "+ - "facility.str_name AS facility_name "+ - "FROM " + - "job, " + - "job_resource, "+ - "show, " + - "dept, "+ - "facility "+ - "WHERE " + - "job.pk_job = job_resource.pk_job " + - "AND " + - "job.pk_show = show.pk_show " + - "AND " + - "job.pk_dept = dept.pk_dept " + - "AND " + - "job.pk_facility = facility.pk_facility "; - - private static final String GET_JOB_BY_ID = - GET_JOB_DETAIL + "AND job.pk_job=?"; - - private static final String FIND_JOB_BY_NAME = - GET_JOB_DETAIL + "AND job.str_visible_name=? "; - - @Override - public JobDetail getJobDetail(String id) { - return getJdbcTemplate().queryForObject( - GET_JOB_BY_ID, JOB_DETAIL_MAPPER, id); - } - - @Override - public JobDetail findLastJob(String name) { - Matcher matcher = LAST_JOB_STRIP_PATTERN.matcher(name); - name = matcher.replaceAll("%"); - - return getJdbcTemplate().queryForObject( - "SELECT * FROM (" + GET_JOB_DETAIL + - " AND job.str_state = 'FINISHED' AND job.str_name LIKE ? ORDER BY job.ts_stopped DESC) " + - "WHERE ROWNUM = 1", JOB_DETAIL_MAPPER, name); - } - - @Override - public JobInterface getJob(String id) { - return getJdbcTemplate().queryForObject( - GET_JOB + " WHERE pk_job=?", JOB_MAPPER, id); - } - - - public static final String GET_JOBS_BY_TASK = - "SELECT " + - "job.pk_job, "+ - "job.pk_show, "+ - "job.pk_dept,"+ - "job.pk_facility,"+ - "job.str_name " + - "FROM " + - "job," + - "folder " + - "WHERE " + - "job.pk_folder = folder.pk_folder " + - "AND " + - "folder.b_exclude_managed = 0 " + - "AND " + - "job.str_state = ? " + - "AND " + - "job.pk_dept = ? " + - "AND " + - "job.str_shot = ? " + - "ORDER BY "+ - "ts_started ASC "; - - @Override - public List getJobs(TaskEntity t) { - return getJdbcTemplate().query(GET_JOBS_BY_TASK, - JOB_MAPPER, JobState.PENDING.toString(), t.deptId, t.shot); - } - - @Override - public JobDetail findJobDetail(String name) { - return getJdbcTemplate().queryForObject( - FIND_JOB_BY_NAME, JOB_DETAIL_MAPPER, name); - } - - @Override - public JobInterface findJob(String name) { - return getJdbcTemplate().queryForObject( - GET_JOB + " WHERE job.str_visible_name=?", JOB_MAPPER, name); - } - - @Override - public List findJobs(ShowInterface show) { - return getJdbcTemplate().query( - GET_JOB_DETAIL + " AND job.pk_show=?", JOB_DETAIL_MAPPER, show.getShowId()); - } - - @Override - public List findJobs(GroupInterface group) { - return getJdbcTemplate().query( - GET_JOB_DETAIL + " AND job.pk_folder=?", JOB_DETAIL_MAPPER, group.getId()); - } - - @Override - public void deleteJob(JobInterface j) { - /* See trigger before_delete_job */ - getJdbcTemplate().update("DELETE FROM job WHERE pk_job=?", j.getId()); - } - - @Override - public void updatePriority(JobInterface j, int v) { - getJdbcTemplate().update("UPDATE job_resource SET int_priority=? WHERE pk_job=?", - v, j.getJobId()); - } - - @Override - public void updatePriority(GroupInterface g, int v) { - getJdbcTemplate().update("UPDATE job_resource SET int_priority=? WHERE " + - "pk_job IN (SELECT pk_job FROM job WHERE job.pk_folder=?)", - v, g.getGroupId()); - } - - @Override - public void updateMinCores(GroupInterface g, int v) { - getJdbcTemplate().update("UPDATE job_resource SET int_min_cores=? WHERE " + - "pk_job IN (SELECT pk_job FROM job WHERE pk_folder=?)", - v, g.getGroupId()); - } - - @Override - public void updateMaxCores(GroupInterface g, int v) { - getJdbcTemplate().update("UPDATE job_resource SET int_max_cores=? WHERE " + - "pk_job IN (SELECT pk_job FROM job WHERE pk_folder=?)", - v, g.getGroupId()); - } - - @Override - public void updateMinCores(JobInterface j, int v) { - getJdbcTemplate().update("UPDATE job_resource SET int_min_cores=? WHERE pk_job=?", - v, j.getJobId()); - } - - @Override - public void updateMaxCores(JobInterface j, int v) { - getJdbcTemplate().update("UPDATE job_resource SET int_max_cores=? WHERE pk_job=?", - v, j.getJobId()); - } - - @Override - public void updatePaused(JobInterface j, boolean b) { - getJdbcTemplate().update("UPDATE job SET b_paused=? WHERE pk_job=?", - b, j.getJobId()); - } - - @Override - public void updateAutoEat(JobInterface j, boolean b) { - int maxRetries = 1; - if (b) { - maxRetries = 0; - } - getJdbcTemplate().update("UPDATE job SET b_autoeat=?, int_max_retries=? WHERE pk_job=?", - b, maxRetries, j.getJobId()); - } - - @Override - public void updateState(JobInterface job, JobState state) { - getJdbcTemplate().update("UPDATE job SET str_state=? WHERE pk_job=?", - state.toString(), job.getJobId()); - } - - @Override - public void updateLogPath(JobInterface job, String path) { - getJdbcTemplate().update("UPDATE job SET str_log_dir=? WHERE pk_job=?", - path, job.getJobId()); - } - - @Override - public void updateMaxRSS(JobInterface job, long value) { - getJdbcTemplate().update( - "UPDATE job_mem SET int_max_rss=? WHERE pk_job=? AND int_max_rss < ?", - value, job.getJobId(), value); - } - - private static final String UPDATE_JOB_FINISHED = - "UPDATE " + - "job " + - "SET " + - "str_state = ?, "+ - "str_visible_name=NULL, " + - "ts_stopped = systimestamp "+ - "WHERE " + - "str_state = 'PENDING'" + - "AND " + - "pk_job = ?"; - - @Override - public boolean updateJobFinished(JobInterface job) { - // Only return true if this thread was the one who actually - // set the job state to finished. - if(getJdbcTemplate().update(UPDATE_JOB_FINISHED, - JobState.FINISHED.toString(), job.getJobId()) == 1) { - return true; - } - return false; - } - - private static final String INSERT_JOB = - "INSERT INTO " + - "job " + - "(" + - "pk_job," + - "pk_show," + - "pk_folder,"+ - "pk_facility,"+ - "pk_dept,"+ - "str_name," + - "str_visible_name,"+ - "str_show,"+ - "str_shot," + - "str_user," + - "str_email,"+ - "str_state," + - "str_log_dir," + - "str_os, "+ - "int_uid," + - "b_paused," + - "b_autoeat,"+ - "int_max_retries " + - ") " + - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; - - @Override - public void insertJob(JobDetail j, JobLogUtil jobLogUtil) { - j.id = SqlUtil.genKeyRandom(); - j.logDir = jobLogUtil.getJobLogPath(j); - if (j.minCoreUnits < 100) { j.minCoreUnits = 100; } - - getJdbcTemplate().update(INSERT_JOB, - j.id, j.showId, j.groupId, j.facilityId, j.deptId, - j.name, j.name, j.showName, j.shot, j.user, j.email, j.state.toString(), - j.logDir, j.os, j.uid.orElse(null), j.isPaused, j.isAutoEat, j.maxRetries); - } - - private static final String JOB_EXISTS = - "SELECT " + - "1 " + - "FROM " + - "job " + - "WHERE " + - "str_name = ? " + - "AND " + - "str_state='PENDING' " + - "AND " + - "ROWNUM = 1"; - - @Override - public boolean exists(String name) { - try { - return (getJdbcTemplate().queryForObject(JOB_EXISTS, - Integer.class, name) >= 1); - } catch (Exception e) { - return false; - } - } - - private static final String IS_LAUNCHING = - "SELECT " + - "str_state " + - "FROM " + - "job " + - "WHERE " + - "pk_job=?"; - - @Override - public boolean isLaunching(JobInterface j) { - return getJdbcTemplate().queryForObject( - IS_LAUNCHING, String.class, j.getJobId()).equals( - JobState.STARTUP.toString()); - } - - @Override - public void activateJob(JobInterface job, JobState jobState) { - - int[] jobTotals = { 0, 0 }; // Depend, Waiting - - /* - * Sets all frames in the setup state to Waiting. Frames with a depend - * count > 0 are automatically updated to Depend via the - * update_frame_wait_to_dep trigger. - */ - getJdbcTemplate().update("UPDATE frame SET str_state=? WHERE pk_job=? AND str_state=?", - FrameState.WAITING.toString(), job.getId(), FrameState.SETUP.toString()); - - List> layers = getJdbcTemplate().queryForList( - "SELECT pk_layer, str_state, count(1) AS c FROM frame " + - "WHERE pk_job=? GROUP BY pk_layer, str_state", job.getId()); - - for (Map row: layers) { - String layer = (String) row.get("pk_layer"); - FrameState state = FrameState.valueOf((String) row.get("str_state")); - int count = ((BigDecimal) row.get("c")).intValue(); - - if (count == 0 || state == null) { continue; } - - switch (state) { - case DEPEND: - jobTotals[0] = jobTotals[0] + count; - getJdbcTemplate().update( - "UPDATE layer_stat SET int_depend_count=?,int_total_count=int_total_count + ? WHERE pk_layer=?", - count, count, layer); - break; - case WAITING: - jobTotals[1] = jobTotals[1] + count; - getJdbcTemplate().update( - "UPDATE layer_stat SET int_waiting_count=?,int_total_count=int_total_count + ? WHERE pk_layer=?", - count, count, layer); - break; - } - } - - getJdbcTemplate().update( - "UPDATE job_stat SET int_depend_count=?,int_waiting_count=? WHERE pk_job=?", - jobTotals[0], jobTotals[1], job.getJobId()); - - getJdbcTemplate().update( - "UPDATE job SET int_frame_count=?, int_layer_count=? WHERE pk_job=?", - jobTotals[0] + jobTotals[1], layers.size(), job.getJobId()); - - getJdbcTemplate().update( - "UPDATE show SET int_frame_insert_count=int_frame_insert_count+?, int_job_insert_count=int_job_insert_count+1 WHERE pk_show=?", - jobTotals[0] + jobTotals[1], job.getShowId()); - - updateState(job, jobState); - } - - private static final String HAS_PENDING_FRAMES = - "SELECT " + - "int_waiting_count " + - "FROM " + - "job,"+ - "job_stat " + - "WHERE " + - "job.pk_job = job_stat.pk_job " + - "AND " + - "job.str_state = 'PENDING' " + - "AND " + - "job.b_paused = 0 " + - "AND " + - "job.b_auto_book = 1 " + - "AND " + - "job.pk_job = ?"; - - @Override - public boolean hasPendingFrames(JobInterface job) { - try { - return getJdbcTemplate().queryForObject(HAS_PENDING_FRAMES, - Integer.class, job.getJobId()) > 0; - } catch (DataAccessException e) { - return false; - } - } - - private static final String IS_JOB_OVER_MIN_CORES = - "SELECT " + - "COUNT(1) " + - "FROM " + - "job_resource " + - "WHERE " + - "job_resource.pk_job = ? " + - "AND " + - "job_resource.int_cores > job_resource.int_min_cores"; - - @Override - public boolean isOverMinCores(JobInterface job) { - return getJdbcTemplate().queryForObject(IS_JOB_OVER_MIN_CORES, - Integer.class, job.getJobId()) > 0; - } - - private static final String IS_JOB_OVER_MAX_CORES = - "SELECT " + - "COUNT(1) " + - "FROM " + - "job_resource " + - "WHERE " + - "job_resource.pk_job = ? " + - "AND " + - "job_resource.int_cores + ? > job_resource.int_max_cores"; - - @Override - public boolean isOverMaxCores(JobInterface job) { - return getJdbcTemplate().queryForObject(IS_JOB_OVER_MAX_CORES, - Integer.class, job.getJobId(), 0) > 0; - } - - @Override - public boolean isOverMaxCores(JobInterface job, int coreUnits) { - return getJdbcTemplate().queryForObject(IS_JOB_OVER_MAX_CORES, - Integer.class, job.getJobId(), coreUnits) > 0; - } - - - private static final String IS_JOB_AT_MAX_CORES = - "SELECT " + - "COUNT(1) " + - "FROM " + - "job_resource " + - "WHERE " + - "job_resource.pk_job = ? " + - "AND " + - "job_resource.int_cores >= job_resource.int_max_cores "; - - @Override - public boolean isAtMaxCores(JobInterface job) { - return getJdbcTemplate().queryForObject(IS_JOB_AT_MAX_CORES, - Integer.class, job.getJobId()) > 0; - } - - @Override - public void updateMaxFrameRetries(JobInterface j, int max_retries) { - if (max_retries < 0) { - throw new IllegalArgumentException("max retries must be greater than 0"); - } - - int max_max_retries = getJdbcTemplate().queryForObject( - "SELECT int_value FROM config WHERE str_key=?", - Integer.class, "MAX_FRAME_RETRIES"); - - if (max_retries > max_max_retries) { - throw new IllegalArgumentException("max retries must be less than " - + max_max_retries); - } - - getJdbcTemplate().update( - "UPDATE job SET int_max_retries=? WHERE pk_job=?", - max_retries, j.getJobId()); - } - - private static final String GET_FRAME_STATE_TOTALS = - "SELECT " + - "job.int_frame_count," + - "job_stat.* " + - "FROM " + - "job," + - "job_stat " + - "WHERE " + - "job.pk_job = job_stat.pk_job " + - "AND " + - "job.pk_job=?"; - - public FrameStateTotals getFrameStateTotals(JobInterface job) { - return getJdbcTemplate().queryForObject( - GET_FRAME_STATE_TOTALS, - new RowMapper() { - public FrameStateTotals mapRow(ResultSet rs, int rowNum) throws SQLException { - FrameStateTotals t = new FrameStateTotals(); - t.dead = rs.getInt("int_dead_count"); - t.depend = rs.getInt("int_depend_count"); - t.eaten = rs.getInt("int_eaten_count"); - t.running = rs.getInt("int_running_count"); - t.succeeded = rs.getInt("int_succeeded_count"); - t.waiting = rs.getInt("int_waiting_count"); - t.total = rs.getInt("int_frame_count"); - return t; - } - },job.getJobId()); - } - - private static final String GET_EXECUTION_SUMMARY = - "SELECT " + - "job_usage.int_core_time_success,"+ - "job_usage.int_core_time_fail," + - "job_mem.int_max_rss " + - "FROM " + - "job," + - "job_usage, "+ - "job_mem " + - "WHERE " + - "job.pk_job = job_usage.pk_job "+ - "AND " + - "job.pk_job = job_mem.pk_job " + - "AND " + - "job.pk_job = ?"; - - public ExecutionSummary getExecutionSummary(JobInterface job) { - return getJdbcTemplate().queryForObject( - GET_EXECUTION_SUMMARY, - new RowMapper() { - public ExecutionSummary mapRow(ResultSet rs, int rowNum) throws SQLException { - ExecutionSummary e = new ExecutionSummary(); - e.coreTimeSuccess = rs.getLong("int_core_time_success"); - e.coreTimeFail = rs.getLong("int_core_time_fail"); - e.coreTime = e.coreTimeSuccess + e.coreTimeFail; - e.highMemoryKb = rs.getLong("int_max_rss"); - - return e; - } - }, job.getJobId()); - } - - private static final String INSERT_JOB_ENV = - "INSERT INTO " + - "job_env " + - "(" + - "pk_job_env, pk_job, str_key, str_value " + - ") " + - "VALUES (?,?,?,?)"; - - @Override - public void insertEnvironment(JobInterface job, Map env) { - for (Map.Entry e: env.entrySet()) { - String pk = SqlUtil.genKeyRandom(); - getJdbcTemplate().update(INSERT_JOB_ENV, - pk, job.getJobId(), e.getKey(), e.getValue()); - } - } - - @Override - public void insertEnvironment(JobInterface job, String key, String value) { - String pk = SqlUtil.genKeyRandom(); - getJdbcTemplate().update(INSERT_JOB_ENV, - pk, job.getJobId(), key, value); - } - - @Override - public Map getEnvironment(JobInterface job) { - Map result = new HashMap(); - - List> _result = getJdbcTemplate().queryForList( - "SELECT str_key, str_value FROM job_env WHERE pk_job=?", job.getJobId()); - - for (Map o: _result) { - result.put((String) o.get("str_key"), (String) o.get("str_value")); - } - - return result; - } - - @Override - public void updateParent(JobInterface job, GroupDetail dest) { - updateParent(job, dest, new Inherit[] { Inherit.All }); - } - - @Override - public void updateParent(JobInterface job, GroupDetail dest, Inherit[] inherits) { - - if (!job.getShowId().equals(dest.getShowId())) { - throw new EntityModificationError("error moving job, " + - "cannot move jobs between shows"); - } - - StringBuilder query = new StringBuilder(1024); - query.append("UPDATE job_resource SET "); - List values= new ArrayList(); - - Set inheritSet = new HashSet(inherits.length); - inheritSet.addAll(Arrays.asList(inherits)); - - for (Inherit i: inheritSet) { - switch(i) { - case Priority: - if (dest.jobPriority != CueUtil.FEATURE_DISABLED) { - query.append("int_priority=?,"); - values.add(dest.jobPriority); - } - break; - - case MinCores: - if (dest.jobMinCores != CueUtil.FEATURE_DISABLED) { - query.append("int_min_cores=?,"); - values.add(dest.jobMinCores); - } - break; - - case MaxCores: - if (dest.jobMaxCores != CueUtil.FEATURE_DISABLED) { - query.append("int_max_cores=?,"); - values.add(dest.jobMaxCores); - } - break; - - case All: - if (dest.jobPriority != CueUtil.FEATURE_DISABLED) { - query.append("int_priority=?,"); - values.add(dest.jobPriority); - } - - if (dest.jobMinCores != CueUtil.FEATURE_DISABLED) { - query.append("int_min_cores=?,"); - values.add(dest.jobMinCores); - } - - if (dest.jobMaxCores != CueUtil.FEATURE_DISABLED) { - query.append("int_max_cores=?,"); - values.add(dest.jobMaxCores); - } - break; - } - } - - getJdbcTemplate().update( - "UPDATE job SET pk_folder=?, pk_dept=? WHERE pk_job=?", - dest.getGroupId(), dest.getDepartmentId(), job.getJobId()); - - getJdbcTemplate().update( - "UPDATE job_history SET pk_dept=? WHERE pk_job=?", - dest.getDepartmentId(), job.getJobId()); - - if (values.size() > 0) { - query.deleteCharAt(query.length()-1); - query.append(" WHERE pk_job=?"); - values.add(job.getJobId()); - getJdbcTemplate().update(query.toString(), values.toArray()); - } - } - - private static final String HAS_PENDING_JOBS = - "SELECT " + - "job.pk_job " + - "FROM " + - "job,"+ - "job_stat, " + - "job_resource "+ - "WHERE " + - "job.pk_job = job_stat.pk_job " + - "AND " + - "job.pk_job = job_resource.pk_job " + - "AND " + - "job.str_state='PENDING' " + - "AND " + - "job.b_paused = 0 " + - "AND " + - "job.b_auto_book = 1 " + - "AND " + - "job_stat.int_waiting_count != 0" + - "AND " + - "job_resource.int_cores < job_resource.int_max_cores " + - "AND " + - "job.pk_facility = ? " + - "AND "+ - "ROWNUM = 1"; - - @Override - public boolean cueHasPendingJobs(FacilityInterface f) { - return getJdbcTemplate().queryForList( - HAS_PENDING_JOBS, f.getFacilityId()).size() > 0; - } - - @Override - public void enableAutoBooking(JobInterface job, boolean value) { - getJdbcTemplate().update( - "UPDATE job SET b_auto_book=? WHERE pk_job=?", value, job.getJobId()); - } - - @Override - public void enableAutoUnBooking(JobInterface job, boolean value) { - getJdbcTemplate().update( - "UPDATE job SET b_auto_unbook=? WHERE pk_job=?", value, job.getJobId()); - } - - public static final String MAP_POST_JOB = - "INSERT INTO " + - "job_post " + - "(pk_job_post, pk_job, pk_post_job) " + - "VALUES (?,?,?)"; - - @Override - public void mapPostJob(BuildableJob job) { - getJdbcTemplate().update(MAP_POST_JOB, - SqlUtil.genKeyRandom(), job.detail.id, job.getPostJob().detail.id); - } - - public static final String ACTIVATE_POST_JOB = - "UPDATE " + - "job " + - "SET " + - "str_state=? " + - "WHERE " + - "pk_job IN (SELECT pk_post_job FROM job_post WHERE pk_job = ?)"; - - @Override - public void activatePostJob(JobInterface job) { - getJdbcTemplate().update(ACTIVATE_POST_JOB, - JobState.PENDING.toString(), job.getJobId()); - getJdbcTemplate().update("DELETE FROM job_post WHERE pk_job=?",job.getJobId()); - } - - @Override - public void updateDepartment(GroupInterface group, DepartmentInterface dept) { - getJdbcTemplate().update("UPDATE job SET pk_dept=? WHERE pk_folder=?", - dept.getDepartmentId(), group.getGroupId()); - } - - @Override - public void updateDepartment(JobInterface job, DepartmentInterface dept) { - getJdbcTemplate().update("UPDATE job SET pk_dept=? WHERE pk_job=?", - dept.getDepartmentId(), job.getJobId()); - } - - - public void updateUsage(JobInterface job, ResourceUsage usage, int exitStatus) { - - if (exitStatus == 0) { - - getJdbcTemplate().update( - "UPDATE " + - "job_usage " + - "SET " + - "int_core_time_success = int_core_time_success + ?," + - "int_clock_time_success = int_clock_time_success + ?,"+ - "int_frame_success_count = int_frame_success_count + 1 " + - "WHERE " + - "pk_job = ? ", - usage.getCoreTimeSeconds(), - usage.getClockTimeSeconds(), - job.getJobId()); - - getJdbcTemplate().update( - "UPDATE " + - "job_usage " + - "SET " + - "int_clock_time_high = ? " + - "WHERE " + - "pk_job = ? " + - "AND " + - "int_clock_time_high < ?", - usage.getClockTimeSeconds(), - job.getJobId(), - usage.getClockTimeSeconds()); - } - else { - - getJdbcTemplate().update( - "UPDATE " + - "job_usage " + - "SET " + - "int_core_time_fail = int_core_time_fail + ?," + - "int_clock_time_fail = int_clock_time_fail + ?,"+ - "int_frame_fail_count = int_frame_fail_count + 1 " + - "WHERE " + - "pk_job = ? ", - usage.getCoreTimeSeconds(), - usage.getClockTimeSeconds(), - job.getJobId()); - } - } - - public void updateEmail(JobInterface job, String email) { - getJdbcTemplate().update( - "UPDATE job SET str_email=? WHERE pk_job=?", - email, job.getJobId()); - } -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/LayerDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/LayerDaoJdbc.java deleted file mode 100644 index f189af1d9..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/LayerDaoJdbc.java +++ /dev/null @@ -1,847 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; - -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import org.apache.commons.lang.StringUtils; -import org.springframework.dao.EmptyResultDataAccessException; -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.ExecutionSummary; -import com.imageworks.spcue.FrameStateTotals; -import com.imageworks.spcue.JobInterface; -import com.imageworks.spcue.LayerDetail; -import com.imageworks.spcue.LayerEntity; -import com.imageworks.spcue.LayerInterface; -import com.imageworks.spcue.LimitEntity; -import com.imageworks.spcue.LimitInterface; -import com.imageworks.spcue.ResourceUsage; -import com.imageworks.spcue.ThreadStats; -import com.imageworks.spcue.dao.LayerDao; -import com.imageworks.spcue.dispatcher.Dispatcher; -import com.imageworks.spcue.grpc.job.JobState; -import com.imageworks.spcue.grpc.job.LayerType; -import com.imageworks.spcue.util.CueUtil; -import com.imageworks.spcue.util.SqlUtil; - -public class LayerDaoJdbc extends JdbcDaoSupport implements LayerDao { - - private static final String INSERT_OUTPUT_PATH = - "INSERT INTO " + - "layer_output " + - "( " + - "pk_layer_output,"+ - "pk_layer,"+ - "pk_job,"+ - "str_filespec " + - ") VALUES (?,?,?,?)"; - - @Override - public void insertLayerOutput(LayerInterface layer, String filespec) { - getJdbcTemplate().update( - INSERT_OUTPUT_PATH, UUID.randomUUID().toString(), - layer.getLayerId(), layer.getJobId(), - filespec); - } - - private static final String GET_OUTPUT = - "SELECT " + - "str_filespec " + - "FROM " + - "layer_output " + - "WHERE " + - "pk_layer = ?"; - - private static final RowMapper OUTPUT_MAPPER = - new RowMapper() { - public String mapRow(ResultSet rs, int rowNum) throws SQLException { - return rs.getString("str_filespec"); - } - }; - - @Override - public List getLayerOutputs(LayerInterface layer) { - return getJdbcTemplate().query(GET_OUTPUT, - OUTPUT_MAPPER, layer.getLayerId()); - } - - private static final String IS_LAYER_DISPATCHABLE = - "SELECT " + - "int_waiting_count " + - "FROM " + - "layer_stat " + - "WHERE " + - "pk_layer=?"; - - @Override - public boolean isLayerDispatchable(LayerInterface l ) { - return getJdbcTemplate().queryForObject(IS_LAYER_DISPATCHABLE, - Integer.class, l.getLayerId()) > 0; - } - - private static final String IS_LAYER_COMPLETE = - "SELECT " + - "SUM ( " + - "int_waiting_count + " + - "int_running_count + " + - "int_dead_count + " + - "int_depend_count "+ - ") " + - "FROM " + - "layer_stat " + - "WHERE " + - "pk_layer=?"; - - public boolean isLayerComplete(LayerInterface l) { - if (isLaunching(l)) { - return false; - } - return getJdbcTemplate().queryForObject(IS_LAYER_COMPLETE, - Integer.class, l.getLayerId()) == 0; - } - - private static final String IS_LAUNCHING = - "SELECT " + - "str_state " + - "FROM " + - "job " + - "WHERE " + - "pk_job=?"; - - @Override - public boolean isLaunching(LayerInterface l) { - return getJdbcTemplate().queryForObject( - IS_LAUNCHING, String.class, l.getJobId()).equals( - JobState.STARTUP.toString()); - } - - private static final String IS_THREADABLE = - "SELECT " + - "b_threadable " + - "FROM " + - "layer " + - "WHERE " + - "pk_layer = ?"; - - @Override - public boolean isThreadable(LayerInterface l) { - return getJdbcTemplate().queryForObject(IS_THREADABLE, - Integer.class, l.getLayerId()) > 0; - } - - /** - * Query for layers table. Where clauses are appended later - */ - public static final String GET_LAYER_DETAIL = - "SELECT " + - "layer.*, " + - "job.pk_show, "+ - "job.pk_facility " + - "FROM " + - "layer," + - "job," + - "show " + - "WHERE " + - "layer.pk_job = job.pk_job " + - "AND " + - "job.pk_show = show.pk_show "; - - private static final String GET_LAYER = - "SELECT " + - "layer.pk_layer,"+ - "layer.pk_job,"+ - "job.pk_show,"+ - "job.pk_facility, " + - "layer.str_name "+ - "FROM " + - "layer," + - "job," + - "show " + - "WHERE " + - "layer.pk_job = job.pk_job " + - "AND " + - "job.pk_show = show.pk_show "; - - /** - * Maps a ResultSet to a LayerDetail - */ - public static final RowMapper LAYER_DETAIL_MAPPER = new RowMapper() { - public LayerDetail mapRow(ResultSet rs, int rowNum) throws SQLException { - LayerDetail layer = new LayerDetail(); - layer.chunkSize = rs.getInt("int_chunk_size"); - layer.command = rs.getString("str_cmd"); - layer.dispatchOrder = rs.getInt("int_dispatch_order"); - layer.id = rs.getString("pk_layer"); - layer.jobId = rs.getString("pk_job"); - layer.showId = rs.getString("pk_show"); - layer.facilityId =rs.getString("pk_facility"); - layer.name = rs.getString("str_name"); - layer.range = rs.getString("str_range"); - layer.minimumCores = rs.getInt("int_cores_min"); - layer.minimumMemory = rs.getLong("int_mem_min"); - layer.minimumGpu = rs.getLong("int_gpu_min"); - layer.type = LayerType.valueOf(rs.getString("str_type")); - layer.tags = Sets.newHashSet( - rs.getString("str_tags").replaceAll(" ", "").split("\\|")); - layer.services.addAll( - Lists.newArrayList(rs.getString("str_services").split(","))); - return layer; - } - }; - - /** - * Maps a ResultSet to a LayerDetail - */ - private static final RowMapper LAYER_MAPPER = new RowMapper() { - public LayerEntity mapRow(ResultSet rs, int rowNum) throws SQLException { - LayerEntity layer = new LayerEntity(); - layer.id = rs.getString("pk_layer"); - layer.jobId = rs.getString("pk_job"); - layer.showId = rs.getString("pk_show"); - layer.facilityId = rs.getString("pk_facility"); - layer.name = rs.getString("str_name"); - return layer; - } - }; - - - @Override - public LayerDetail getLayerDetail(String id) { - LayerDetail layerDetail = getJdbcTemplate().queryForObject(GET_LAYER_DETAIL + - " AND layer.pk_layer=?", LAYER_DETAIL_MAPPER, id); - layerDetail.limits.addAll(getLimitNames(layerDetail)); - return layerDetail; - } - - @Override - public LayerDetail getLayerDetail(LayerInterface layer) { - LayerDetail layerDetail = getJdbcTemplate().queryForObject(GET_LAYER_DETAIL + - " AND layer.pk_layer=?", LAYER_DETAIL_MAPPER, layer.getLayerId()); - layerDetail.limits.addAll(getLimitNames(layerDetail)); - return layerDetail; - } - - @Override - public LayerDetail findLayerDetail(JobInterface job, String name) { - LayerDetail layerDetail = getJdbcTemplate().queryForObject( - GET_LAYER_DETAIL + " AND layer.pk_job=? AND layer.str_name=?", - LAYER_DETAIL_MAPPER, job.getJobId(), name); - layerDetail.limits.addAll(getLimitNames(layerDetail)); - return layerDetail; - } - - @Override - public LayerInterface findLayer(JobInterface job, String name) { - try { - return getJdbcTemplate().queryForObject( - GET_LAYER + " AND layer.pk_job=? AND layer.str_name=?", - LAYER_MAPPER, job.getJobId(), name); - } catch (org.springframework.dao.EmptyResultDataAccessException e) { - throw new EmptyResultDataAccessException("The layer " + - name + " was not found in " + job.getName() + e, 0); - } - } - - @Override - public List getLayerDetails(JobInterface job) { - List layers = getJdbcTemplate().query( - GET_LAYER_DETAIL + " AND layer.pk_job=?", - LAYER_DETAIL_MAPPER, job.getJobId()); - layers.stream() - .forEach(layerDetail -> layerDetail.limits.addAll(getLimitNames(layerDetail))); - return layers; - } - - @Override - public List getLayers(JobInterface job) { - return getJdbcTemplate().query( - GET_LAYER + " AND layer.pk_job=?", - LAYER_MAPPER, job.getJobId()); - } - - @Override - public LayerInterface getLayer(String id) { - return getJdbcTemplate().queryForObject( - GET_LAYER + " AND layer.pk_layer=?", - LAYER_MAPPER, id); - } - - private static final String INSERT_LAYER = - "INSERT INTO " + - "layer " + - "("+ - "pk_layer, " + - "pk_job, "+ - "str_name, " + - "str_cmd, " + - "str_range, " + - "int_chunk_size, " + - "int_dispatch_order, " + - "str_tags, " + - "str_type," + - "int_cores_min, "+ - "int_cores_max, "+ - "b_threadable, " + - "int_mem_min, " + - "int_gpu_min, " + - "str_services " + - ") " + - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; - - @Override - public void insertLayerDetail(LayerDetail l) { - l.id = SqlUtil.genKeyRandom(); - getJdbcTemplate().update(INSERT_LAYER, - l.id, l.jobId, l.name, l.command, - l.range, l.chunkSize, l.dispatchOrder, - StringUtils.join(l.tags," | "), l.type.toString(), - l.minimumCores, l.maximumCores, l.isThreadable, - l.minimumMemory, l.minimumGpu, StringUtils.join(l.services,",")); - } - - @Override - public void updateLayerMinMemory(LayerInterface layer, long val) { - if (val < Dispatcher.MEM_RESERVED_MIN) { - val = Dispatcher.MEM_RESERVED_MIN; - } - getJdbcTemplate().update("UPDATE layer SET int_mem_min=? WHERE pk_layer=?", - val, layer.getLayerId()); - } - - @Override - public void updateLayerMinGpu(LayerInterface layer, long gpu) { - getJdbcTemplate().update("UPDATE layer SET int_gpu_min=? WHERE pk_layer=?", - gpu, layer.getLayerId()); - } - - private static final String BALANCE_MEM = - "UPDATE " + - "layer " + - "SET " + - "int_mem_min = ? " + - "WHERE " + - "pk_layer = ? " + - "AND " + - "int_mem_min > ? " + - "AND " + - "b_optimize = 1"; - - @Override - public boolean balanceLayerMinMemory(LayerInterface layer, long frameMaxRss) { - - /** - * Lowers the memory value on the frame when the maxrss is lower than - * the memory requirement. - */ - long maxrss = getJdbcTemplate().queryForObject( - "SELECT int_max_rss FROM layer_mem WHERE pk_layer=?", - Long.class, layer.getLayerId()); - - if (maxrss < frameMaxRss) { - maxrss = frameMaxRss; - } - if (maxrss < Dispatcher.MEM_RESERVED_MIN) { - maxrss = Dispatcher.MEM_RESERVED_MIN; - } else { - maxrss = maxrss + CueUtil.MB256; - } - - boolean result = getJdbcTemplate().update(BALANCE_MEM, - maxrss, layer.getLayerId(), maxrss) == 1; - if (result) { - logger.info(layer.getName() + " was balanced to " + maxrss); - } - return result; - } - - @Override - public void increaseLayerMinMemory(LayerInterface layer, long val) { - getJdbcTemplate().update("UPDATE layer SET int_mem_min=? WHERE pk_layer=? AND int_mem_min < ?", - val, layer.getLayerId(), val); - } - - @Override - public void increaseLayerMinGpu(LayerInterface layer, long gpu) { - getJdbcTemplate().update("UPDATE layer SET int_gpu_min=? WHERE pk_layer=? AND int_gpu_min < ?", - gpu, layer.getLayerId(), gpu); - } - - @Override - public void updateLayerMinCores(LayerInterface layer, int val) { - if (val < Dispatcher.CORE_POINTS_RESERVED_MIN) { - val = Dispatcher.CORE_POINTS_RESERVED_DEFAULT; - } - getJdbcTemplate().update("UPDATE layer SET int_cores_min=? WHERE pk_layer=?", - val, layer.getLayerId()); - } - - @Override - public void updateLayerMaxCores(LayerInterface layer, int val) { - getJdbcTemplate().update("UPDATE layer SET int_cores_max=? WHERE pk_layer=?", - val, layer.getLayerId()); - } - - private static final String UPDATE_LAYER_MAX_RSS = - "UPDATE " + - "layer_mem " + - "SET " + - "int_max_rss=? " + - "WHERE " + - "pk_layer=?"; - - @Override - public void updateLayerMaxRSS(LayerInterface layer, long val, boolean force) { - StringBuilder sb = new StringBuilder(UPDATE_LAYER_MAX_RSS); - Object[] options; - if (!force) { - options = new Object[] { val, layer.getLayerId(), val}; - sb.append (" AND int_max_rss < ?"); - } - else { - options = new Object[] { val, layer.getLayerId() }; - } - getJdbcTemplate().update(sb.toString(), options); - } - - @Override - public void updateLayerTags(LayerInterface layer, Set tags) { - if (tags.size() == 0) { - throw new IllegalArgumentException( - "Layers must have at least one tag."); - } - StringBuilder sb = new StringBuilder(128); - for (String t: tags) { - if (t == null) { continue; } - if (t.length() < 1) { continue; } - sb.append(t + " | "); - } - sb.delete(sb.length()-3, sb.length()); - if (sb.length() == 0) { - throw new IllegalArgumentException( - "Invalid layer tags, cannot contain null tags or " + - "tags of zero length."); - } - getJdbcTemplate().update( - "UPDATE layer SET str_tags=? WHERE pk_layer=?", - sb.toString(), layer.getLayerId()); - } - - @Override - public void appendLayerTags(LayerInterface layer, String val) { - String appendTag = " | " + val; - String matchTag = "%" + val + "%"; - - getJdbcTemplate().update("UPDATE layer SET str_tags = str_tags || ? " + - "WHERE pk_layer=? AND str_tags NOT LIKE ?", - appendTag, layer.getLayerId(), matchTag); - } - - public FrameStateTotals getFrameStateTotals(LayerInterface layer) { - return getJdbcTemplate().queryForObject( - "SELECT * FROM layer_stat WHERE pk_layer=?", - new RowMapper() { - public FrameStateTotals mapRow(ResultSet rs, int rowNum) throws SQLException { - FrameStateTotals t = new FrameStateTotals(); - t.dead = rs.getInt("int_dead_count"); - t.depend = rs.getInt("int_depend_count"); - t.eaten = rs.getInt("int_eaten_count"); - t.running = rs.getInt("int_running_count"); - t.succeeded = rs.getInt("int_succeeded_count"); - t.waiting = rs.getInt("int_waiting_count"); - t.total = rs.getInt("int_total_count"); - return t; - } - },layer.getLayerId()); - } - - private static final String GET_EXECUTION_SUMMARY = - "SELECT " + - "layer_usage.int_core_time_success,"+ - "layer_usage.int_core_time_fail," + - "layer_usage.int_clock_time_success," + - "layer_mem.int_max_rss " + - "FROM " + - "layer," + - "layer_usage, "+ - "layer_mem " + - "WHERE " + - "layer.pk_layer = layer_usage.pk_layer "+ - "AND " + - "layer.pk_layer = layer_mem.pk_layer " + - "AND " + - "layer.pk_layer = ?"; - - @Override - public ExecutionSummary getExecutionSummary(LayerInterface layer) { - return getJdbcTemplate().queryForObject( - GET_EXECUTION_SUMMARY, - new RowMapper() { - public ExecutionSummary mapRow(ResultSet rs, int rowNum) throws SQLException { - ExecutionSummary e = new ExecutionSummary(); - e.coreTimeSuccess = rs.getLong("int_core_time_success"); - e.coreTimeFail = rs.getLong("int_core_time_fail"); - e.coreTime = e.coreTimeSuccess + e.coreTimeFail; - e.highMemoryKb = rs.getLong("int_max_rss"); - return e; - } - }, layer.getLayerId()); - } - - private static final String INSERT_LAYER_ENV = - "INSERT INTO " + - "layer_env " + - "(" + - "pk_layer_env, pk_layer, pk_job, str_key, str_value " + - ") " + - "VALUES (?,?,?,?,?)"; - - @Override - public void insertLayerEnvironment(LayerInterface layer, Map env) { - for (Map.Entry e: env.entrySet()) { - String pk = SqlUtil.genKeyRandom(); - getJdbcTemplate().update(INSERT_LAYER_ENV, - pk, layer.getLayerId(), layer.getJobId(), e.getKey(), e.getValue()); - } - } - - @Override - public void insertLayerEnvironment(LayerInterface layer, String key, String value) { - String pk = SqlUtil.genKeyRandom(); - getJdbcTemplate().update(INSERT_LAYER_ENV, - pk, layer.getLayerId(), layer.getJobId(), key, value); - } - - @Override - public Map getLayerEnvironment(LayerInterface layer) { - Map result = new HashMap(); - List> _result = getJdbcTemplate().queryForList( - "SELECT str_key, str_value FROM layer_env WHERE pk_layer=?", layer.getLayerId()); - - for (Map o: _result) { - result.put((String) o.get("str_key"), (String) o.get("str_value")); - } - return result; - } - - private static final String FIND_PAST_MAX_RSS = - "SELECT "+ - "layer_mem.int_max_rss " + - "FROM " + - "layer, " + - "layer_mem, "+ - "layer_stat "+ - "WHERE " + - "layer.pk_layer = layer_stat.pk_layer " + - "AND " + - "layer.pk_layer = layer_mem.pk_layer " + - "AND " + - "layer.pk_job = ? " + - "AND " + - "layer.str_name = ? " + - "AND " + - "layer_stat.int_succeeded_count >= ceil(layer_stat.int_total_count * .5) "; - - @Override - public long findPastMaxRSS(JobInterface job, String name) { - try { - long maxRss = getJdbcTemplate().queryForObject(FIND_PAST_MAX_RSS, - Long.class, job.getJobId(), name); - if (maxRss >= Dispatcher.MEM_RESERVED_MIN) { - return maxRss; - } - else { - return Dispatcher.MEM_RESERVED_MIN; - } - } catch (EmptyResultDataAccessException e) { - // Actually want to return 0 here, which means - // there is no past history. - return 0; - } - } - - @Override - public void updateTags(JobInterface job, String tags, LayerType type) { - getJdbcTemplate().update( - "UPDATE layer SET str_tags=? WHERE pk_job=? AND str_type=?", - tags, job.getJobId(), type.toString()); - } - - @Override - public void updateMinMemory(JobInterface job, long mem, LayerType type) { - if (mem < Dispatcher.MEM_RESERVED_MIN) { - mem = Dispatcher.MEM_RESERVED_MIN; - } - getJdbcTemplate().update( - "UPDATE layer SET int_mem_min=? WHERE pk_job=? AND str_type=?", - mem, job.getJobId(), type.toString()); - } - - @Override - public void updateMinGpu(JobInterface job, long gpu, LayerType type) { - getJdbcTemplate().update( - "UPDATE layer SET int_gpu_min=? WHERE pk_job=? AND str_type=?", - gpu, job.getJobId(), type.toString()); - } - - @Override - public void updateMinCores(JobInterface job, int cores, LayerType type) { - getJdbcTemplate().update( - "UPDATE layer SET int_cores_min=? WHERE pk_job=? AND str_type=?", - cores, job.getJobId(), type.toString()); - } - - @Override - public void updateThreadable(LayerInterface layer, boolean threadable) { - getJdbcTemplate().update( - "UPDATE layer SET b_threadable=? WHERE pk_layer=?", - threadable, layer.getLayerId()); - } - - @Override - public void updateTimeout(LayerInterface layer, int timeout){ - getJdbcTemplate().update( - "UPDATE layer SET int_timeout=? WHERE pk_layer=?", - timeout, layer.getLayerId()); - } - - @Override - public void updateTimeoutLLU(LayerInterface layer, int timeout_llu){ - getJdbcTemplate().update( - "UPDATE layer SET int_timeout_llu=? WHERE pk_layer=?", - timeout_llu, layer.getLayerId()); - } - - @Override - public void enableMemoryOptimizer(LayerInterface layer, boolean value) { - getJdbcTemplate().update( - "UPDATE layer SET b_optimize=? WHERE pk_layer=?", - value, layer.getLayerId()); - } - - private static final String IS_OPTIMIZABLE = - "SELECT " + - "COUNT(1) "+ - "FROM " + - "layer, " + - "layer_stat, " + - "layer_usage " + - "WHERE " + - "layer.pk_layer = layer_stat.pk_layer " + - "AND " + - "layer.pk_layer = layer_usage.pk_layer " + - "AND " + - "layer.pk_layer = ? " + - "AND " + - "layer.int_cores_min = 100 " + - "AND " + - "str_tags LIKE '%general%' " + - "AND " + - "str_tags NOT LIKE '%util%' " + - "AND " + - "layer_stat.int_succeeded_count >= ? " + - "AND " + - "(layer_usage.int_core_time_success / layer_stat.int_succeeded_count) <= ?"; - - @Override - public boolean isOptimizable(LayerInterface l, int succeeded, float avg) { - if (succeeded < 1) { - throw new IllegalArgumentException("Succeeded frames option " + - "must be greater than zero"); - } - return getJdbcTemplate().queryForObject(IS_OPTIMIZABLE, - Integer.class, l.getLayerId(), succeeded, avg) > 0; - } - - private static final String THREAD_STATS = - "SELECT " + - "avg(interval_to_seconds(ts_stopped - ts_started)) AS avg, " + - "int_cores " + - "FROM " + - "frame " + - "WHERE " + - "frame.pk_layer = ? " + - "AND " + - "frame.int_checkpoint_count = 0 " + - "AND " + - "int_cores > 0 " + - "GROUP BY " + - "int_cores " + - "ORDER BY " + - "int_cores DESC "; - - @Override - public List getThreadStats(LayerInterface layer) { - - return getJdbcTemplate().query(THREAD_STATS, - new RowMapper() { - public ThreadStats mapRow(ResultSet rs, int rowNum) throws SQLException { - ThreadStats s = new ThreadStats(); - s.setThreads(rs.getInt("int_cores") / 100); - s.setAvgFrameTime(rs.getInt("avg")); - return s; - } - }, layer.getLayerId()); - } - - @Override - public void updateUsage(LayerInterface layer, ResourceUsage usage, int exitStatus) { - - if (exitStatus == 0) { - - getJdbcTemplate().update( - "UPDATE " + - "layer_usage " + - "SET " + - "int_core_time_success = int_core_time_success + ?," + - "int_clock_time_success = int_clock_time_success + ?,"+ - "int_frame_success_count = int_frame_success_count + 1 " + - "WHERE " + - "pk_layer = ? ", - usage.getCoreTimeSeconds(), - usage.getClockTimeSeconds(), - layer.getLayerId()); - - getJdbcTemplate().update( - "UPDATE " + - "layer_usage " + - "SET " + - "int_clock_time_high = ? " + - "WHERE " + - "pk_layer = ? " + - "AND " + - "int_clock_time_high < ?", - usage.getClockTimeSeconds(), - layer.getLayerId(), - usage.getClockTimeSeconds()); - - getJdbcTemplate().update( - "UPDATE " + - "layer_usage " + - "SET " + - "int_clock_time_low = ? " + - "WHERE " + - "pk_layer = ? " + - "AND " + - "(? < int_clock_time_low OR int_clock_time_low = 0)", - usage.getClockTimeSeconds(), - layer.getLayerId(), - usage.getClockTimeSeconds()); - } - else { - getJdbcTemplate().update( - "UPDATE " + - "layer_usage " + - "SET " + - "int_core_time_fail = int_core_time_fail + ?," + - "int_clock_time_fail = int_clock_time_fail + ?,"+ - "int_frame_fail_count = int_frame_fail_count + 1 " + - "WHERE " + - "pk_layer = ? ", - usage.getCoreTimeSeconds(), - usage.getClockTimeSeconds(), - layer.getLayerId()); - } - } - - private static final String INSERT_LIMIT = - "INSERT INTO " + - "layer_limit " + - "( " + - "pk_layer_limit,"+ - "pk_layer,"+ - "pk_limit_record,"+ - ") VALUES (?,?,?)"; - - private static final String GET_LIMITS = - "SELECT " + - "limit_record.pk_limit_record," + - "limit_record.str_name," + - "limit_record.int_max_value " + - "FROM " + - "layer_limit," + - "limit_record " + - "WHERE " + - "layer_limit.pk_layer = ? " + - "AND limit_record.pk_limit_record = layer_limit.pk_limit_record"; - - private static final String GET_LIMIT_NAMES = - "SELECT " + - "limit_record.str_name " + - "FROM " + - "layer_limit," + - "limit_record " + - "WHERE " + - "layer_limit.pk_layer = ? " + - "AND limit_record.pk_limit_record = layer_limit.pk_limit_record"; - - private static final RowMapper LIMIT_MAPPER = - new RowMapper() { - public LimitEntity mapRow(ResultSet rs, int rowNum) throws SQLException { - LimitEntity limit = new LimitEntity(); - limit.id = rs.getString("pk_limit_record"); - limit.name = rs.getString("str_name"); - limit.maxValue = rs.getInt("int_max_value"); - return limit; - } - }; - - private static final RowMapper LIMIT_NAME_MAPPER = - new RowMapper() { - public String mapRow(ResultSet rs, int rowNum) throws SQLException { - return rs.getString("str_name"); - } - }; - - @Override - public void addLimit(LayerInterface layer, String limitId) { - getJdbcTemplate().update(INSERT_LIMIT, UUID.randomUUID().toString(), layer.getLayerId(), - limitId); - } - - @Override - public void dropLimit(LayerInterface layer, String limitId) { - getJdbcTemplate().update( - "DELETE FROM layer_limit WHERE pk_limit_record = ? AND pk_layer = ?", - limitId, - layer.getLayerId()); - } - - @Override - public List getLimits(LayerInterface layer) { - return getJdbcTemplate().query(GET_LIMITS, - LIMIT_MAPPER, layer.getLayerId()); - } - - @Override - public List getLimitNames(LayerInterface layer) { - return getJdbcTemplate().query(GET_LIMIT_NAMES, - LIMIT_NAME_MAPPER, layer.getLayerId()); - } -} diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/LimitDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/LimitDaoJdbc.java deleted file mode 100644 index 96acf8af0..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/LimitDaoJdbc.java +++ /dev/null @@ -1,110 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; - -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.LimitEntity; -import com.imageworks.spcue.LimitInterface; -import com.imageworks.spcue.dao.LimitDao; -import com.imageworks.spcue.util.SqlUtil; - -public class LimitDaoJdbc extends JdbcDaoSupport implements LimitDao { - - public static final RowMapper LIMIT_MAPPER = new RowMapper() { - public LimitEntity mapRow(ResultSet rs, int rowNum) throws SQLException { - LimitEntity limit = new LimitEntity(); - limit.id = rs.getString("pk_limit_record"); - limit.name = rs.getString("str_name"); - limit.maxValue = rs.getInt("int_max_value"); - limit.currentRunning = rs.getInt("int_current_running"); - return limit; - } - }; - - @Override - public String createLimit(String name, int maxValue) { - String limitId = SqlUtil.genKeyRandom(); - getJdbcTemplate().update( - "INSERT INTO limit_record (pk_limit_record,str_name, int_max_value) VALUES (?,?,?)", - limitId, name, maxValue); - return limitId; - } - - @Override - public void deleteLimit(LimitInterface limit) { - getJdbcTemplate().update("DELETE FROM limit_record WHERE pk_limit_recor=?", - limit.getId()); - } - - @Override - public LimitEntity findLimit(String name){ - String findLimitQuery = GET_LIMIT_BASE + - "WHERE " + - "limit-record.str_name=? " + - "GROUP BY " + - "limit_record.str_name, " + - "limit_record.pk_limit_record, " + - "limit_record.int_max_value"; - return getJdbcTemplate().queryForObject(findLimitQuery, LIMIT_MAPPER, name); - } - - @Override - public LimitEntity getLimit(String id) { - String getLimitQuery = GET_LIMIT_BASE + - "WHERE " + - "limit_record.pk_limit_record=? " + - "GROUP BY " + - "limit_record.str_name, " + - "limit_record.pk_limit_record, " + - "limit_record.int_max_value"; - return getJdbcTemplate().queryForObject(getLimitQuery, LIMIT_MAPPER, id); - } - - @Override - public void setLimitName(LimitInterface limit, String name) { - getJdbcTemplate().update("UPDATE limit_record SET str_name = ? WHERE pk_limit_record = ?", - name, limit.getId()); - } - - public void setMaxValue(LimitInterface limit, int maxValue) { - getJdbcTemplate().update("UPDATE limit_record SET int_max_value = ? WHERE pk_limit_record = ?", - maxValue, limit.getId()); - } - - private static final String GET_LIMIT_BASE = - "SELECT " + - "limit_record.pk_limit_record, " + - "limit_record.str_name, " + - "limit_record.int_max_value," + - "SUM(layer_stat.int_running_count) AS int_current_running " + - "FROM " + - "limit_record " + - "LEFT JOIN " + - "layer_limit ON layer_limit.pk_limit_record = limit_record.pk_limit_record " + - "LEFT JOIN " + - "layer ON layer.pk_layer = layer_limit.pk_layer " + - "LEFT JOIN " + - "layer_stat ON layer_stat.pk_layer = layer.pk_layer "; -} diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/MaintenanceDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/MaintenanceDaoJdbc.java deleted file mode 100644 index 820804f50..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/MaintenanceDaoJdbc.java +++ /dev/null @@ -1,90 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.MaintenanceTask; -import com.imageworks.spcue.dao.MaintenanceDao; -import com.imageworks.spcue.grpc.host.HardwareState; - - -public class MaintenanceDaoJdbc extends JdbcDaoSupport implements MaintenanceDao { - - private static final String HOST_DOWN_INTERVAL = "interval '300' second"; - - private static final String UPDATE_HOSTS_DOWN = - "UPDATE " + - "host_stat " + - "SET " + - "str_state=? " + - "WHERE " + - "str_state='UP' " + - "AND " + - "systimestamp - ts_ping > " + HOST_DOWN_INTERVAL; - - public int setUpHostsToDown() { - return getJdbcTemplate().update(UPDATE_HOSTS_DOWN, - HardwareState.DOWN.toString()); - } - - public static final String LOCK_TASK = - "UPDATE " + - "task_lock " + - "SET " + - "int_lock = ?, " + - "ts_lastrun = systimestamp " + - "WHERE " + - "str_name= ? "+ - "AND " + - "(int_lock = ? OR ? - int_lock > int_timeout)"; - - public boolean lockTask(MaintenanceTask task) { - long now = System.currentTimeMillis(); - return getJdbcTemplate().update(LOCK_TASK, - now, task.toString(), 0, now) == 1; - } - - public static final String LOCK_TASK_MIN = - "UPDATE " + - "task_lock " + - "SET " + - "int_lock = ?, " + - "ts_lastrun = systimestamp " + - "WHERE " + - "str_name= ? "+ - "AND " + - "int_lock = ? " + - "AND " + - "interval_to_seconds(systimestamp - ts_lastrun) > ? "; - - public boolean lockTask(MaintenanceTask task, int minutes) { - long now = System.currentTimeMillis(); - return getJdbcTemplate().update(LOCK_TASK_MIN, - now, task.toString(), 0, minutes * 60) == 1; - } - - - public void unlockTask(MaintenanceTask task) { - getJdbcTemplate().update( - "UPDATE task_lock SET int_lock = 0 WHERE str_name=?", task.toString()); - } -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/MatcherDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/MatcherDaoJdbc.java deleted file mode 100644 index f55a0cdf1..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/MatcherDaoJdbc.java +++ /dev/null @@ -1,109 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; - -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.FilterInterface; -import com.imageworks.spcue.MatcherEntity; -import com.imageworks.spcue.MatcherInterface; -import com.imageworks.spcue.dao.MatcherDao; -import com.imageworks.spcue.grpc.filter.MatchSubject; -import com.imageworks.spcue.grpc.filter.MatchType; -import com.imageworks.spcue.util.SqlUtil; - -public class MatcherDaoJdbc extends JdbcDaoSupport implements MatcherDao { - - private static final String INSERT_MATCHER = - "INSERT INTO " + - "matcher " + - "( " + - "pk_matcher,pk_filter,str_subject,str_match,str_value"+ - ") VALUES (?,?,?,?,?)"; - - public void insertMatcher(MatcherEntity matcher) { - matcher.id = SqlUtil.genKeyRandom(); - - getJdbcTemplate().update(INSERT_MATCHER, - matcher.id, matcher.getFilterId(), matcher.subject.toString(), - matcher.type.toString(), matcher.value); - } - - public void deleteMatcher(MatcherInterface matcher) { - getJdbcTemplate().update( - "DELETE FROM matcher WHERE pk_matcher=?", - matcher.getMatcherId()); - } - - private static final String GET_MATCHER = - "SELECT " + - "matcher.*, " + - "filter.pk_show "+ - "FROM " + - "matcher, " + - "filter " + - "WHERE " + - "matcher.pk_filter = filter.pk_filter"; - - public MatcherEntity getMatcher(String id) { - return getJdbcTemplate().queryForObject( - GET_MATCHER + " AND matcher.pk_matcher=?", - MATCHER_DETAIL_MAPPER, id); - } - - public MatcherEntity getMatcher(MatcherInterface matcher) { - return getJdbcTemplate().queryForObject( - GET_MATCHER + " AND matcher.pk_matcher=?", MATCHER_DETAIL_MAPPER, - matcher.getMatcherId()); - } - - public List getMatchers(FilterInterface filter) { - return getJdbcTemplate().query( - GET_MATCHER + " AND filter.pk_filter=? ORDER BY ts_created ASC", - MATCHER_DETAIL_MAPPER, filter.getFilterId()); - } - - - public void updateMatcher(MatcherEntity matcher) { - getJdbcTemplate().update( - "UPDATE matcher SET str_subject=?,str_match=?,str_value=? WHERE pk_matcher=?", - matcher.subject.toString(), matcher.type.toString(), matcher.value, matcher.getMatcherId()); - } - - public static final RowMapper MATCHER_DETAIL_MAPPER = new RowMapper() { - public MatcherEntity mapRow(ResultSet rs, int rowNum) throws SQLException { - MatcherEntity matcher = new MatcherEntity(); - matcher.id = rs.getString("pk_matcher"); - matcher.showId = rs.getString("pk_show"); - matcher.filterId = rs.getString("pk_filter"); - matcher.name = null; - matcher.subject = MatchSubject.valueOf(rs.getString("str_subject")); - matcher.type = MatchType.valueOf(rs.getString("str_match")); - matcher.value = rs.getString("str_value"); - return matcher; - } - }; -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/NestedWhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/NestedWhiteboardDaoJdbc.java deleted file mode 100644 index 693d99fbf..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/NestedWhiteboardDaoJdbc.java +++ /dev/null @@ -1,485 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.ShowInterface; -import com.imageworks.spcue.dao.NestedWhiteboardDao; -import com.imageworks.spcue.grpc.host.NestedHost; -import com.imageworks.spcue.grpc.host.NestedHostSeq; -import com.imageworks.spcue.grpc.host.NestedProc; -import com.imageworks.spcue.grpc.job.GroupStats; -import com.imageworks.spcue.grpc.job.JobState; -import com.imageworks.spcue.grpc.job.JobStats; -import com.imageworks.spcue.grpc.job.NestedGroup; -import com.imageworks.spcue.grpc.job.NestedGroupSeq; -import com.imageworks.spcue.grpc.job.NestedJob; -import com.imageworks.spcue.util.Convert; -import com.imageworks.spcue.util.CueUtil; - -public class NestedWhiteboardDaoJdbc extends JdbcDaoSupport implements NestedWhiteboardDao { - - private class CachedJobWhiteboardMapper { - public final long time; - public NestedJobWhiteboardMapper mapper; - - public CachedJobWhiteboardMapper(NestedJobWhiteboardMapper result) { - this.mapper = result; - this.time = System.currentTimeMillis(); - } - } - - private static final int CACHE_TIMEOUT = 5000; - private final ConcurrentHashMap jobCache = - new ConcurrentHashMap(20); - - public static final String GET_NESTED_GROUPS = - "SELECT " + - "show.pk_show, " + - "show.str_name AS str_show, " + - "facility.str_name AS facility_name, " + - "dept.str_name AS dept_name, " + - "folder.pk_folder, " + - "folder.pk_parent_folder, " + - "folder.str_name AS group_name, " + - "folder.int_job_priority as int_def_job_priority, " + - "folder.int_job_min_cores as int_def_job_min_cores, " + - "folder.int_job_max_cores as int_def_job_max_cores, " + - "folder_resource.int_min_cores AS folder_min_cores, " + - "folder_resource.int_max_cores AS folder_max_cores, " + - "folder_level.int_level, " + - "job.pk_job, " + - "job.str_name, " + - "job.str_shot, " + - "job.str_user, " + - "job.str_state, " + - "job.str_log_dir, " + - "job.int_uid, " + - "job_resource.int_priority, " + - "job.ts_started, " + - "job.ts_stopped, " + - "job.ts_updated, " + - "job.b_paused, " + - "job.b_autoeat, " + - "job.b_comment, " + - "job.str_os, " + - "job.int_frame_count, " + - "job.int_layer_count, " + - "job_stat.int_waiting_count, " + - "job_stat.int_running_count, " + - "job_stat.int_dead_count, " + - "job_stat.int_eaten_count," + - "job_stat.int_depend_count, " + - "job_stat.int_succeeded_count, " + - "job_usage.int_core_time_success, " + - "job_usage.int_core_time_fail, " + - "job_usage.int_frame_success_count, " + - "job_usage.int_frame_fail_count, " + - "job_usage.int_clock_time_high, " + - "job_usage.int_clock_time_success, " + - "(job_resource.int_cores + job_resource.int_local_cores) AS int_cores, " + - "job_resource.int_min_cores, " + - "job_resource.int_max_cores, " + - "job_mem.int_max_rss " + - "FROM " + - "show, " + - "dept, " + - "folder_level, " + - "folder_resource, " + - "folder " + - "LEFT JOIN " + - "job " + - "ON " + - " (folder.pk_folder = job.pk_folder AND job.str_state='PENDING') " + - "LEFT JOIN " + - "facility " + - "ON " + - "(job.pk_facility = facility.pk_facility) " + - "LEFT JOIN " + - "job_stat " + - "ON " + - "(job.pk_job = job_stat.pk_job) " + - "LEFT JOIN " + - "job_resource " + - "ON " + - "(job.pk_job = job_resource.pk_job) " + - "LEFT JOIN " + - "job_usage " + - "ON " + - "(job.pk_job = job_usage.pk_job) " + - "LEFT JOIN " + - "job_mem " + - "ON " + - "(job.pk_job = job_mem.pk_job) " + - "WHERE " + - "show.pk_show = folder.pk_show " + - "AND " + - "folder.pk_folder = folder_level.pk_folder " + - "AND " + - "folder.pk_folder = folder_resource.pk_folder " + - "AND " + - "folder.pk_dept = dept.pk_dept "; - - class NestedJobWhiteboardMapper implements RowMapper { - - public Map groups = new HashMap(50); - public Map> childrenMap = new HashMap>(); - public String rootGroupID; - - @Override - public NestedGroup mapRow(ResultSet rs, int rowNum) throws SQLException { - - String groupId = rs.getString("pk_folder"); - NestedGroup group; - if (!groups.containsKey(groupId)) { - group = NestedGroup.newBuilder() - .setId(rs.getString("pk_folder")) - .setName(rs.getString("group_name")) - .setDefaultJobPriority(rs.getInt("int_def_job_priority")) - .setDefaultJobMinCores(Convert.coreUnitsToCores(rs.getInt("int_def_job_min_cores"))) - .setDefaultJobMaxCores(Convert.coreUnitsToCores(rs.getInt("int_def_job_max_cores"))) - .setMaxCores(Convert.coreUnitsToCores(rs.getInt("folder_max_cores"))) - .setMinCores(Convert.coreUnitsToCores(rs.getInt("folder_min_cores"))) - .setLevel(rs.getInt("int_level")) - .setDepartment(rs.getString("dept_name")) - .build(); - - String parentGroupId = rs.getString("pk_parent_folder"); - if (parentGroupId != null) { - List children = childrenMap.get(parentGroupId); - if (children == null) { - children = new ArrayList<>(); - childrenMap.put(parentGroupId, children); - } - children.add(groupId); - } - else { - rootGroupID = rs.getString("pk_folder"); - } - groups.put(groupId, group); - } - else { - group = groups.get(groupId); - } - if (rs.getString("pk_job") != null) { - GroupStats oldStats = group.getStats(); - JobStats jobStats = WhiteboardDaoJdbc.mapJobStats(rs); - GroupStats groupStats = GroupStats.newBuilder() - .setDeadFrames(oldStats.getDeadFrames() + jobStats.getDeadFrames()) - .setRunningFrames(oldStats.getRunningFrames() + jobStats.getRunningFrames()) - .setWaitingFrames(oldStats.getWaitingFrames() + jobStats.getWaitingFrames()) - .setDependFrames(oldStats.getDependFrames() + jobStats.getDependFrames()) - .setReservedCores(oldStats.getReservedCores() + jobStats.getReservedCores()) - .setPendingJobs(oldStats.getPendingJobs() + 1).build(); - - group = group.toBuilder() - .setStats(groupStats) - .addJobs(rs.getString("pk_job")) - .build(); - groups.put(groupId, group); - } - return group; - } - } - - private NestedJobWhiteboardMapper updateConnections(NestedJobWhiteboardMapper mapper) { - for (Map.Entry> entry : mapper.childrenMap.entrySet()) { - NestedGroup group = mapper.groups.get(entry.getKey()); - NestedGroupSeq.Builder childrenBuilder = NestedGroupSeq.newBuilder(); - for (String childId : entry.getValue()) { - NestedGroup child = mapper.groups.get(childId); - child = child.toBuilder().setParent(group).build(); - childrenBuilder.addNestedGroups(child); - mapper.groups.put(childId, child); - } - group = group.toBuilder() - .setGroups(childrenBuilder.build()) - .build(); - mapper.groups.put(entry.getKey(), group); - } - return mapper; - } - - public NestedGroup getJobWhiteboard(ShowInterface show) { - - CachedJobWhiteboardMapper cachedMapper = jobCache.get(show.getShowId()); - if (cachedMapper != null) { - if (System.currentTimeMillis() - cachedMapper.time < CACHE_TIMEOUT) { - return cachedMapper.mapper.groups.get(cachedMapper.mapper.rootGroupID); - } - } - - NestedJobWhiteboardMapper mapper = new NestedJobWhiteboardMapper(); - getJdbcTemplate().query( - GET_NESTED_GROUPS + " AND show.pk_show=? ORDER BY folder_level.int_level ASC", - mapper, show.getShowId()); - - mapper = updateConnections(mapper); - jobCache.put(show.getShowId(), new CachedJobWhiteboardMapper(mapper)); - return mapper.groups.get(mapper.rootGroupID); - } - - private static final NestedJob mapResultSetToJob(ResultSet rs) throws SQLException { - NestedJob.Builder jobBuilder = NestedJob.newBuilder() - .setId(rs.getString("pk_job")) - .setLogDir(rs.getString("str_log_dir")) - .setMaxCores(Convert.coreUnitsToCores(rs.getInt("int_max_cores"))) - .setMinCores(Convert.coreUnitsToCores(rs.getInt("int_min_cores"))) - .setName(rs.getString("str_name")) - .setPriority(rs.getInt("int_priority")) - .setShot(rs.getString("str_shot")) - .setShow(rs.getString("str_show")) - .setOs(rs.getString("str_os")) - .setFacility(rs.getString("facility_name")) - .setGroup(rs.getString("group_name")) - .setState(JobState.valueOf(rs.getString("str_state"))) - .setUser(rs.getString("str_user")) - .setIsPaused(rs.getBoolean("b_paused")) - .setHasComment(rs.getBoolean("b_comment")) - .setAutoEat(rs.getBoolean("b_autoeat")) - .setStartTime((int) (rs.getTimestamp("ts_started").getTime() / 1000)) - .setStats(WhiteboardDaoJdbc.mapJobStats(rs)); - - int uid = rs.getInt("int_uid"); - if (!rs.wasNull()) { - jobBuilder.setUid(uid); - } - - Timestamp ts = rs.getTimestamp("ts_stopped"); - if (ts != null) { - jobBuilder.setStopTime((int) (ts.getTime() / 1000)); - } - else { - jobBuilder.setStopTime(0); - } - return jobBuilder.build(); - } - - private static final String GET_HOSTS = - "SELECT " + - "alloc.str_name AS alloc_name, " + - "host.pk_host, " + - "host.str_name AS host_name, " + - "host_stat.str_state AS host_state, " + - "host.b_nimby, " + - "host_stat.ts_booted, " + - "host_stat.ts_ping, " + - "host.int_cores, " + - "host.int_cores_idle, " + - "host.int_gpu, " + - "host.int_gpu_idle, " + - "host.int_mem, " + - "host.int_mem_idle, " + - "host.str_lock_state, " + - "host.str_tags, " + - "host.b_comment, " + - "host.int_thread_mode, " + - "host_stat.str_os, " + - "host_stat.int_mem_total, " + - "host_stat.int_mem_free, " + - "host_stat.int_swap_total, " + - "host_stat.int_swap_free, " + - "host_stat.int_mcp_total, " + - "host_stat.int_mcp_free, " + - "host_stat.int_gpu_total, " + - "host_stat.int_gpu_free, " + - "host_stat.int_load, " + - "proc.pk_proc, " + - "proc.int_cores_reserved AS proc_cores, " + - "proc.int_mem_reserved AS proc_memory, " + - "proc.int_mem_used AS used_memory, " + - "proc.int_mem_max_used AS max_memory, " + - "proc.int_gpu_reserved AS proc_gpu, " + - "proc.ts_ping, " + - "proc.ts_booked, " + - "proc.ts_dispatched, " + - "proc.b_unbooked, " + - "redirect.str_name AS str_redirect, " + - "job.str_name AS job_name, " + - "job.str_log_dir, " + - "show.str_name AS show_name, " + - "frame.str_name AS frame_name " + - "FROM " + - "alloc, " + - "host_stat, " + - "host " + - "LEFT JOIN " + - "proc " + - "ON " + - "(proc.pk_host = host.pk_host) " + - "LEFT JOIN " + - "frame " + - "ON " + - "(proc.pk_frame = frame.pk_frame) " + - "LEFT JOIN " + - "job " + - "ON " + - "(proc.pk_job = job.pk_job) " + - "LEFT JOIN " + - "show " + - "ON " + - "(proc.pk_show = show.pk_show) " + - "LEFT JOIN " + - "redirect " + - "ON " + - "(proc.pk_proc = redirect.pk_proc) " + - "WHERE " + - "host.pk_alloc = alloc.pk_alloc " + - "AND " + - "host.pk_host = host_stat.pk_host "; - - /** - * Caches a the host whiteboard. This class is not - * thread safe so you have to synchronize calls to - * the "cache" method on your own. - */ - class CachedHostWhiteboard { - - /** - * Number of seconds till the cache expires - */ - private static final int CACHE_EXPIRE_TIME_MS = 10000; - - /** - * The host whiteboard we're caching - */ - private NestedHostSeq hostWhiteboard; - - /** - * The time in which the cache expires. - */ - private long expireTime = 0l; - - public void cache(List hostWhiteboard) { - this.hostWhiteboard = NestedHostSeq.newBuilder().addAllNestedHosts(hostWhiteboard).build(); - expireTime = System.currentTimeMillis() + CACHE_EXPIRE_TIME_MS; - } - - public NestedHostSeq get() { - return hostWhiteboard; - } - - public boolean isExpired() { - return System.currentTimeMillis() > expireTime; - } - } - - /** - * The CachedHostWhiteboard holds onto the result of the last - * host whiteboard query for about 10 seconds, returning the - * same result to all subsequent requests. - */ - private final CachedHostWhiteboard cachedHostWhiteboard = - new CachedHostWhiteboard(); - - public NestedHostSeq getHostWhiteboard() { - - if (!cachedHostWhiteboard.isExpired()) { - return cachedHostWhiteboard.get(); - } - - /* - * Ensures only 1 thread is doing the query, other threads will wait - * and then return the result of the thead that actually did - * the query. - */ - synchronized (cachedHostWhiteboard) { - - if (!cachedHostWhiteboard.isExpired()) { - return cachedHostWhiteboard.get(); - } - - final List result = new ArrayList(3000); - final Map hosts = new HashMap(3000); - final Map procs = new HashMap(8000); - - getJdbcTemplate().query( - GET_HOSTS, - new RowMapper() { - - public NestedHost mapRow(ResultSet rs, int row) throws SQLException { - NestedHost host; - String hid = rs.getString("pk_host"); - if (!hosts.containsKey(hid)) { - host = WhiteboardDaoJdbc.mapNestedHostBuilder(rs).build(); - hosts.put(hid, host); - result.add(host); - } - else { - host = hosts.get(hid); - } - - String pid = rs.getString("pk_proc"); - if (pid != null) { - NestedProc proc; - if (!procs.containsKey(pid)) { - proc = NestedProc.newBuilder() - .setId(pid) - .setName(CueUtil.buildProcName(host.getName(), - rs.getInt("proc_cores"))) - .setReservedCores(Convert.coreUnitsToCores( - rs.getInt("proc_cores"))) - .setReservedMemory(rs.getLong("proc_memory")) - .setUsedMemory(rs.getLong("used_memory")) - .setFrameName(rs.getString("frame_name")) - .setJobName(rs.getString("job_name")) - .setShowName(rs.getString("show_name")) - .setPingTime((int) (rs.getTimestamp("ts_ping").getTime() / 1000)) - .setBookedTime((int) (rs.getTimestamp("ts_booked").getTime() / 1000)) - .setDispatchTime((int) (rs.getTimestamp("ts_dispatched").getTime() / 1000)) - .setUnbooked(rs.getBoolean("b_unbooked")) - .setLogPath(String.format("%s/%s.%s.rqlog", - rs.getString("str_log_dir"),rs.getString("job_name"), - rs.getString("frame_name"))) - .setRedirectTarget(rs.getString("str_redirect")) - .setParent(host) - .build(); - - host = host.toBuilder().setProcs( - host.getProcs().toBuilder().addNestedProcs(proc).build()) - .build(); - procs.put(pid, proc); - } - else { - proc = procs.get(pid); - } - } - - return null; - } - }); - - cachedHostWhiteboard.cache(result); - } - return cachedHostWhiteboard.get(); - } -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/OwnerDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/OwnerDaoJdbc.java deleted file mode 100644 index ee94ba8ec..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/OwnerDaoJdbc.java +++ /dev/null @@ -1,127 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; - -import org.springframework.dao.EmptyResultDataAccessException; -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.Entity; -import com.imageworks.spcue.HostInterface; -import com.imageworks.spcue.OwnerEntity; -import com.imageworks.spcue.ShowInterface; -import com.imageworks.spcue.dao.OwnerDao; -import com.imageworks.spcue.util.SqlUtil; - -public class OwnerDaoJdbc extends JdbcDaoSupport implements OwnerDao { - - public static final RowMapper - OWNER_MAPPER = new RowMapper() { - public OwnerEntity mapRow(ResultSet rs, int rowNum) throws SQLException { - OwnerEntity o = new OwnerEntity(); - o.id = rs.getString("pk_owner"); - o.name = rs.getString("str_username"); - return o; - } - }; - - @Override - public boolean deleteOwner(Entity owner) { - return getJdbcTemplate().update( - "DELETE FROM owner WHERE pk_owner = ?", - owner.getId()) > 0; - } - - private static final String QUERY_FOR_OWNER = - "SELECT " + - "owner.pk_owner," + - "owner.str_username " + - "FROM " + - "owner "; - - @Override - public OwnerEntity findOwner(String name) { - try { - return getJdbcTemplate().queryForObject( - QUERY_FOR_OWNER + " WHERE str_username = ?", - OWNER_MAPPER, name); - } catch (EmptyResultDataAccessException e) { - throw new EmptyResultDataAccessException( - "Failed to find owner: " + name, 1); - } - } - - @Override - public OwnerEntity getOwner(String id) { - return getJdbcTemplate().queryForObject( - QUERY_FOR_OWNER + " WHERE pk_owner = ?", - OWNER_MAPPER, id); - } - - @Override - public OwnerEntity getOwner(HostInterface host) { - return getJdbcTemplate().queryForObject( - QUERY_FOR_OWNER + - "WHERE " + - "pk_owner = (" + - "SELECT "+ - "pk_owner " + - "FROM " + - "deed " + - "WHERE " + - "pk_host = ?)", - OWNER_MAPPER, host.getHostId()); - } - - public boolean isOwner(OwnerEntity owner, HostInterface host) { - return getJdbcTemplate().queryForObject( - "SELECT COUNT(1) FROM host, deed" + - " WHERE host.pk_host = deed.pk_host AND deed.pk_owner=?", - Integer.class, owner.getId()) > 0; - } - - private static final String INSERT_OWNER = - "INSERT INTO " + - "owner " + - "(" + - "pk_owner," + - "pk_show," + - "str_username " + - ") " + - "VALUES (?,?,?)"; - - @Override - public void insertOwner(OwnerEntity owner, ShowInterface show) { - owner.id = SqlUtil.genKeyRandom(); - getJdbcTemplate().update(INSERT_OWNER, - owner.id, show.getShowId(), owner.name); - } - - @Override - public void updateShow(Entity owner, ShowInterface show) { - getJdbcTemplate().update( - "UPDATE owner SET pk_show = ? WHERE pk_owner = ?", - show.getShowId(), owner.getId()); - } -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/PointDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/PointDaoJdbc.java deleted file mode 100644 index 6ce52d8aa..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/PointDaoJdbc.java +++ /dev/null @@ -1,222 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; - -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.DepartmentInterface; -import com.imageworks.spcue.JobInterface; -import com.imageworks.spcue.PointDetail; -import com.imageworks.spcue.PointInterface; -import com.imageworks.spcue.ShowInterface; -import com.imageworks.spcue.dao.PointDao; -import com.imageworks.spcue.util.SqlUtil; - -public class PointDaoJdbc extends JdbcDaoSupport implements PointDao { - - @Override - public void insertPointConf(PointDetail t) { - t.id = SqlUtil.genKeyRandom(); - getJdbcTemplate().update( - "INSERT INTO point (pk_point, pk_show, pk_dept) VALUES (?,?,?)", - t.id, t.getShowId(), t.getDepartmentId()); - } - - @Override - public boolean isManaged(ShowInterface show, DepartmentInterface dept) { - try { - return getJdbcTemplate().queryForObject( - "SELECT b_managed FROM point WHERE pk_show=? and pk_dept=?", - Integer.class, show.getShowId(), dept.getDepartmentId()) == 1; - } catch (org.springframework.dao.DataRetrievalFailureException e) { - return false; - } - } - - @Override - public PointDetail insertPointConf(ShowInterface show, DepartmentInterface dept) { - PointDetail r = new PointDetail(); - r.deptId = dept.getId(); - r.showId = show.getShowId(); - r.id = SqlUtil.genKeyRandom(); - getJdbcTemplate().update( - "INSERT INTO point (pk_point, pk_show, pk_dept) VALUES (?,?,?)", - r.id, r.getShowId(), r.getDepartmentId()); - return r; - } - - @Override - public boolean pointConfExists(ShowInterface show, DepartmentInterface dept) { - return getJdbcTemplate().queryForObject( - "SELECT COUNT(1) FROM point WHERE pk_show=? AND pk_dept=?", - Integer.class, show.getShowId(), dept.getDepartmentId()) > 0; - } - - private static final String UPDATE_TI_MANAGED = - "UPDATE " + - "point " + - "SET " + - "b_managed = 1,"+ - "str_ti_task=?, "+ - "int_min_cores=? " + - "WHERE " + - "pk_point=?"; - - @Override - public void updateEnableManaged(PointInterface p, String task, int coreUnits) { - getJdbcTemplate().update(UPDATE_TI_MANAGED, - task, coreUnits, p.getPointId()); - } - - private static final String UPDATE_DISABLE_TI_MANAGED = - "UPDATE " + - "point " + - "SET " + - "b_managed = 0,"+ - "str_ti_task=null, "+ - "int_min_cores=0 " + - "WHERE " + - "pk_point=?"; - - @Override - public void updateDisableManaged(PointInterface p) { - getJdbcTemplate().update(UPDATE_DISABLE_TI_MANAGED, p.getPointId()); - } - - private static final RowMapper DEPARTMENT_CONFIG_DETAIL_MAPPER = - new RowMapper() { - public PointDetail mapRow(ResultSet rs, int rowNum) throws SQLException { - PointDetail rpd = new PointDetail(); - rpd.deptId = rs.getString("pk_dept"); - rpd.showId = rs.getString("pk_show"); - rpd.id = rs.getString("pk_point"); - rpd.cores = rs.getInt("int_min_cores"); - rpd.tiTask = rs.getString("str_ti_task"); - return rpd; - } - }; - - private static final String GET_DEPARTMENT_CONFIG_DETAIL = - "SELECT "+ - "pk_point,"+ - "pk_dept,"+ - "pk_show,"+ - "str_ti_task,"+ - "int_min_cores "+ - "FROM " + - "point " + - "WHERE " + - "pk_point = ?"; - - @Override - public PointDetail getPointConfDetail(String id) { - return getJdbcTemplate().queryForObject(GET_DEPARTMENT_CONFIG_DETAIL, - DEPARTMENT_CONFIG_DETAIL_MAPPER, id); - } - - private static final String GET_POINT_CONFIG_DETAIL_BY_SHOW_DEPT = - "SELECT "+ - "pk_point,"+ - "pk_dept,"+ - "pk_show,"+ - "str_ti_task,"+ - "int_min_cores, "+ - "b_managed " + - "FROM " + - "point " + - "WHERE " + - "pk_show = ? " + - "AND " + - "pk_dept = ? "; - - @Override - public PointDetail getPointConfigDetail(ShowInterface show, DepartmentInterface dept) { - return getJdbcTemplate().queryForObject(GET_POINT_CONFIG_DETAIL_BY_SHOW_DEPT, - DEPARTMENT_CONFIG_DETAIL_MAPPER, show.getShowId(), dept.getDepartmentId()); - } - - private static final String UPDATE_TI_MANAGED_CORES = - "UPDATE " + - "point " + - "SET " + - "int_min_cores=? " + - "WHERE " + - "pk_point=?"; - - @Override - public void updateManagedCores(PointInterface cdept, int cores) { - getJdbcTemplate().update(UPDATE_TI_MANAGED_CORES, cores, - cdept.getPointId()); - - } - - private static final String GET_MANAGED_POINT_CONFS = - "SELECT " + - "pk_point,"+ - "pk_dept,"+ - "pk_show,"+ - "str_ti_task,"+ - "int_min_cores, "+ - "b_managed " + - "FROM " + - "point " + - "WHERE " + - "b_managed = 1 "; - - @Override - public List getManagedPointConfs() { - return getJdbcTemplate().query(GET_MANAGED_POINT_CONFS, - DEPARTMENT_CONFIG_DETAIL_MAPPER); - } - - @Override - public void updatePointConfUpdateTime(PointInterface t) { - getJdbcTemplate().update( - "UPDATE point SET ts_updated=systimestamp WHERE pk_point=?", - t.getPointId()); - } - - private static final String IS_OVER_MIN_CORES = - "SELECT " + - "COUNT(1) " + - "FROM " + - "job,"+ - "point p "+ - "WHERE " + - "job.pk_show = p.pk_show " + - "AND " + - "job.pk_dept = p.pk_dept " + - "AND " + - "p.int_cores > p.int_min_cores " + - "AND "+ - "job.pk_job = ?"; - - @Override - public boolean isOverMinCores(JobInterface job) { - return getJdbcTemplate().queryForObject(IS_OVER_MIN_CORES, - Integer.class, job.getJobId()) > 0; - } -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/ProcDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/ProcDaoJdbc.java deleted file mode 100644 index f363bbc0f..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/ProcDaoJdbc.java +++ /dev/null @@ -1,904 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.math.BigDecimal; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.springframework.dao.DataAccessException; -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.FrameInterface; -import com.imageworks.spcue.HostInterface; -import com.imageworks.spcue.JobInterface; -import com.imageworks.spcue.LayerInterface; -import com.imageworks.spcue.LocalHostAssignment; -import com.imageworks.spcue.ProcInterface; -import com.imageworks.spcue.Redirect; -import com.imageworks.spcue.VirtualProc; -import com.imageworks.spcue.dao.ProcDao; -import com.imageworks.spcue.dao.criteria.FrameSearchInterface; -import com.imageworks.spcue.dao.criteria.ProcSearchInterface; -import com.imageworks.spcue.dispatcher.Dispatcher; -import com.imageworks.spcue.dispatcher.ResourceDuplicationFailureException; -import com.imageworks.spcue.dispatcher.ResourceReservationFailureException; -import com.imageworks.spcue.grpc.host.HardwareState; -import com.imageworks.spcue.util.SqlUtil; - -public class ProcDaoJdbc extends JdbcDaoSupport implements ProcDao { - - private static final String VERIFY_RUNNING_PROC = - "SELECT " + - "proc.pk_frame " + - "FROM " + - "proc, " + - "job " + - "WHERE " + - "proc.pk_job = job.pk_job " + - "AND " + - "job.str_state = 'PENDING' " + - "AND " + - "proc.pk_proc= ? "; - - public boolean verifyRunningProc(String procId, String frameId) { - try { - String pk_frame = getJdbcTemplate().queryForObject( - VERIFY_RUNNING_PROC, String.class, procId); - if (pk_frame != null) { - return pk_frame.equals(frameId); - } - else { - return false; - } - } catch (org.springframework.dao.EmptyResultDataAccessException e) { - // EAT - } - return false; - } - - private static final String DELETE_VIRTUAL_PROC = - "DELETE FROM " + - "proc " + - "WHERE " + - "pk_proc=?"; - - public boolean deleteVirtualProc(VirtualProc proc) { - if(getJdbcTemplate().update(DELETE_VIRTUAL_PROC, proc.getProcId()) == 0) { - logger.warn("failed to delete " + proc + " , proc does not exist."); - return false; - } - // update all of the resource counts. - procDestroyed(proc); - return true; - } - - private static final String INSERT_VIRTUAL_PROC = - "INSERT INTO " + - "proc " + - "( " + - "pk_proc, " + - "pk_host, " + - "pk_show, "+ - "pk_layer,"+ - "pk_job," + - "pk_frame, "+ - "int_cores_reserved, " + - "int_mem_reserved, " + - "int_mem_pre_reserved, " + - "int_mem_used, "+ - "int_gpu_reserved, " + - "b_local " + - ") VALUES (?,?,?,?,?,?,?,?,?,?,?,?) "; - - public void insertVirtualProc(VirtualProc proc) { - proc.id = SqlUtil.genKeyRandom(); - int result = 0; - try { - result = getJdbcTemplate().update(INSERT_VIRTUAL_PROC, - proc.getProcId(), proc.getHostId(), proc.getShowId(), - proc.getLayerId(), proc.getJobId(), proc.getFrameId(), - proc.coresReserved, proc.memoryReserved, - proc.memoryReserved, Dispatcher.MEM_RESERVED_MIN, - proc.gpuReserved, proc.isLocalDispatch); - - // Update all of the resource counts - procCreated(proc); - } - catch (org.springframework.dao.DataIntegrityViolationException d) { - /* - * This means the frame is already running. If that is the - * case, don't delete it, just set pk_frame to null or - * the orphaned proc handler will catch it. - */ - throw new ResourceDuplicationFailureException("The frame " + - proc.getFrameId() + " is already assigned to a proc."); - } - catch (Exception e) { - String msg = "unable to book proc " + - proc.getName() + " on frame " + proc.getFrameId() + - " , " + e; - throw new ResourceReservationFailureException(msg,e); - } - - if (result == 0) { - String msg = "unable to book proc " + proc.id + - " the insert query succeeded but returned 0"; - throw new ResourceReservationFailureException(msg); - } - } - - private static final String UPDATE_VIRTUAL_PROC_ASSIGN = - "UPDATE " + - "proc " + - "SET " + - "pk_show=?," + - "pk_job=?,"+ - "pk_layer=?,"+ - "pk_frame=?, " + - "int_mem_used = 0,"+ - "int_mem_max_used = 0, "+ - "int_virt_used = 0,"+ - "int_virt_max_used = 0, "+ - "ts_dispatched=systimestamp " + - "WHERE " + - "pk_proc = ?"; - - public void updateVirtualProcAssignment(VirtualProc proc) { - - int result = 0; - try { - result = getJdbcTemplate().update( - UPDATE_VIRTUAL_PROC_ASSIGN, - proc.getShowId(), proc.getJobId(), proc.getLayerId(), - proc.getFrameId(), proc.getProcId()); - } - catch (org.springframework.dao.DataIntegrityViolationException d) { - throw new ResourceDuplicationFailureException("The frame " + - proc.getFrameId() + " is already assigned to " + - "the proc " + proc); - } - catch (Exception e) { - String msg = "unable to book proc " + - proc.id + ", " + e; - throw new ResourceReservationFailureException(msg, e); - } - - /* - * If the proc was not updated then it has disappeared. - */ - if (result == 0) { - String msg = "unable to book proc " + - proc.id + ", the proc no longer exists,"; - throw new ResourceReservationFailureException(msg); - } - } - - private static final String CLEAR_VIRTUAL_PROC_ASSIGN = - "UPDATE " + - "proc " + - "SET " + - "pk_frame = NULL " + - "WHERE " + - "pk_proc = ?"; - - public boolean clearVirtualProcAssignment(ProcInterface proc) { - return getJdbcTemplate().update(CLEAR_VIRTUAL_PROC_ASSIGN, - proc.getId()) == 1; - } - - private static final String CLEAR_VIRTUAL_PROC_ASSIGN_BY_FRAME = - "UPDATE " + - "proc " + - "SET " + - "pk_frame = NULL " + - "WHERE " + - "pk_frame = ?"; - - public boolean clearVirtualProcAssignment(FrameInterface frame) { - return getJdbcTemplate().update(CLEAR_VIRTUAL_PROC_ASSIGN_BY_FRAME, - frame.getFrameId()) == 1; - } - - private static final String UPDATE_PROC_MEMORY_USAGE = - "UPDATE " + - "proc " + - "SET " + - "int_mem_used = ?, " + - "int_mem_max_used = ?," + - "int_virt_used = ?, " + - "int_virt_max_used = ?, "+ - "ts_ping = systimestamp " + - "WHERE " + - "pk_frame=?"; - - @Override - public void updateProcMemoryUsage(FrameInterface f, long rss, long maxRss, - long vss, long maxVss) { - /* - * This method is going to repeat for a proc every 1 minute, so - * if the proc is being touched by another thread, then return - * quietly without updating memory usage. - * - * If another thread is accessing the proc record, that means - * the proc is probably being booked to another frame, which - * makes this update invalid anyway. - */ - try { - if (getJdbcTemplate().queryForObject( - "SELECT pk_frame FROM proc WHERE pk_frame=? FOR UPDATE", - String.class, f.getFrameId()).equals(f.getFrameId())) { - - getJdbcTemplate().update(UPDATE_PROC_MEMORY_USAGE, - rss, maxRss, vss, maxVss, f.getFrameId()); - } - } - catch (DataAccessException dae) { - logger.info("The proc for frame " + f + - " could not be updated with new memory stats: " + dae); - } - } - - /** - * Maps a row to a VirtualProc object. - */ - public static final RowMapper VIRTUAL_PROC_MAPPER = - new RowMapper() { - public VirtualProc mapRow(ResultSet rs, int rowNum) throws SQLException { - VirtualProc proc = new VirtualProc(); - proc.id = rs.getString("pk_proc"); - proc.hostId = rs.getString("pk_host"); - proc.showId = rs.getString("pk_show"); - proc.jobId= rs.getString("pk_job"); - proc.layerId = rs.getString("pk_layer"); - proc.frameId = rs.getString("pk_frame"); - proc.hostName = rs.getString("host_name"); - proc.allocationId = rs.getString("pk_alloc"); - proc.facilityId = rs.getString("pk_facility"); - proc.coresReserved =rs.getInt("int_cores_reserved"); - proc.memoryReserved = rs.getInt("int_mem_reserved"); - proc.memoryMax = rs.getInt("int_mem_max_used"); - proc.gpuReserved = rs.getInt("int_gpu_reserved"); - proc.virtualMemoryMax = rs.getLong("int_virt_max_used"); - proc.virtualMemoryUsed = rs.getLong("int_virt_used"); - proc.memoryUsed = rs.getInt("int_mem_used"); - proc.unbooked = rs.getBoolean("b_unbooked"); - proc.isLocalDispatch = rs.getBoolean("b_local"); - proc.os = rs.getString("str_os"); - return proc; - } - }; - - private static final String GET_VIRTUAL_PROC = - "SELECT " + - "proc.pk_proc," + - "proc.pk_host,"+ - "proc.pk_show,"+ - "proc.pk_job,"+ - "proc.pk_layer,"+ - "proc.pk_frame,"+ - "proc.b_unbooked,"+ - "proc.b_local,"+ - "host.pk_alloc, " + - "alloc.pk_facility,"+ - "proc.int_cores_reserved,"+ - "proc.int_mem_reserved,"+ - "proc.int_mem_max_used,"+ - "proc.int_mem_used,"+ - "proc.int_gpu_reserved,"+ - "proc.int_virt_max_used,"+ - "proc.int_virt_used,"+ - "host.str_name AS host_name, " + - "host_stat.str_os " + - "FROM " + - "proc," + - "host, " + - "host_stat, " + - "alloc " + - "WHERE " + - "proc.pk_host = host.pk_host " + - "AND " + - "host.pk_host = host_stat.pk_host " + - "AND " + - "host.pk_alloc = alloc.pk_alloc "; - - public VirtualProc getVirtualProc(String id) { - return getJdbcTemplate().queryForObject( - GET_VIRTUAL_PROC + " AND proc.pk_proc=? ", - VIRTUAL_PROC_MAPPER, id); - } - - public VirtualProc findVirtualProc(FrameInterface frame) { - return getJdbcTemplate().queryForObject( - GET_VIRTUAL_PROC + " AND proc.pk_frame=? ", - VIRTUAL_PROC_MAPPER, frame.getFrameId()); - } - - private static final String GET_VIRTUAL_PROC_LIST = - "SELECT " + - "proc.*, " + - "host.str_name AS host_name, " + - "host.pk_alloc, " + - "host_stat.str_os, " + - "alloc.pk_facility "+ - "FROM " + - "proc, " + - "frame, " + - "host," + - "host_stat, "+ - "alloc, " + - "layer," + - "job, " + - "folder, " + - "show "+ - "WHERE " + - "proc.pk_show = show.pk_show " + - "AND " + - "proc.pk_host = host.pk_host " + - "AND " + - "host.pk_alloc = alloc.pk_alloc " + - "AND " + - "host.pk_host = host_stat.pk_host " + - "AND " + - "proc.pk_job = job.pk_job " + - "AND " + - "proc.pk_layer = layer.pk_layer " + - "AND " + - "proc.pk_frame = frame.pk_frame " + - "AND " + - "job.pk_folder = folder.pk_folder "; - - public List findVirtualProcs(ProcSearchInterface r) { - return getJdbcTemplate().query(r.getFilteredQuery(GET_VIRTUAL_PROC_LIST), - VIRTUAL_PROC_MAPPER, r.getValuesArray()); - } - - @Override - public List findBookedVirtualProcs(ProcSearchInterface r) { - return getJdbcTemplate().query(r.getFilteredQuery(GET_VIRTUAL_PROC_LIST + - "AND proc.b_unbooked = 0"), VIRTUAL_PROC_MAPPER, r.getValuesArray()); - } - - public List findVirtualProcs(FrameSearchInterface r) { - return getJdbcTemplate().query(r.getFilteredQuery(GET_VIRTUAL_PROC_LIST), - VIRTUAL_PROC_MAPPER, r.getValuesArray()); - } - - public List findVirtualProcs(HostInterface host) { - return getJdbcTemplate().query(GET_VIRTUAL_PROC_LIST + " AND proc.pk_host=?", - VIRTUAL_PROC_MAPPER, host.getHostId()); - } - - public List findVirtualProcs(LayerInterface layer) { - return getJdbcTemplate().query(GET_VIRTUAL_PROC_LIST + " AND proc.pk_layer=?", - VIRTUAL_PROC_MAPPER, layer.getLayerId()); - } - - public List findVirtualProcs(JobInterface job) { - return getJdbcTemplate().query(GET_VIRTUAL_PROC_LIST + " AND proc.pk_job=?", - VIRTUAL_PROC_MAPPER, job.getJobId()); - } - - private static final String FIND_VIRTUAL_PROCS_LJA = - GET_VIRTUAL_PROC_LIST + - "AND proc.pk_job=( " + - "SELECT pk_job FROM host_local WHERE pk_host_local = ?) " + - "AND proc.pk_host=(" + - "SELECT pk_host FROM host_local WHERE pk_host_local = ?) "; - - @Override - public List findVirtualProcs(LocalHostAssignment l) { - return getJdbcTemplate().query( - FIND_VIRTUAL_PROCS_LJA, - VIRTUAL_PROC_MAPPER, - l.getId(), - l.getId()); - } - - public List findVirtualProcs(HardwareState state) { - return getJdbcTemplate().query(GET_VIRTUAL_PROC_LIST + " AND host_stat.str_state=?", - VIRTUAL_PROC_MAPPER, state.toString()); - } - - public void unbookVirtualProcs(List procs) { - List batchArgs = new ArrayList(procs.size()); - for (VirtualProc proc: procs) { - batchArgs.add(new Object[] { proc.id }); - } - - getJdbcTemplate().batchUpdate( - "UPDATE proc SET b_unbooked=1 WHERE pk_proc=?", batchArgs); - } - - @Override - public boolean setUnbookState(ProcInterface proc, boolean unbooked) { - return getJdbcTemplate().update( - "UPDATE proc SET b_unbooked=? WHERE pk_proc=?", - unbooked, proc.getProcId()) == 1; - } - - @Override - public boolean setRedirectTarget(ProcInterface p, Redirect r) { - String name = null; - boolean unbooked = false; - if (r != null) { - name = r.getDestinationName(); - unbooked = true; - } - return getJdbcTemplate().update( - "UPDATE proc SET str_redirect=?, b_unbooked=? WHERE pk_proc=?", - name, unbooked, p.getProcId()) == 1; - } - - public void unbookProc(ProcInterface proc) { - getJdbcTemplate().update("UPDATE proc SET b_unbooked=1 WHERE pk_proc=?", - proc.getProcId()); - } - - public String getCurrentShowId(ProcInterface p) { - return getJdbcTemplate().queryForObject("SELECT pk_show FROM proc WHERE pk_proc=?", - String.class, p.getProcId()); - } - - public String getCurrentJobId(ProcInterface p) { - return getJdbcTemplate().queryForObject("SELECT pk_job FROM proc WHERE pk_proc=?", - String.class, p.getProcId()); - } - - public String getCurrentLayerId(ProcInterface p) { - return getJdbcTemplate().queryForObject("SELECT pk_layer FROM proc WHERE pk_proc=?", - String.class, p.getProcId()); - } - - public String getCurrentFrameId(ProcInterface p) { - return getJdbcTemplate().queryForObject("SELECT pk_frame FROM proc WHERE pk_proc=?", - String.class, p.getProcId()); - } - - private static final String ORPHANED_PROC_INTERVAL = "interval '300' second"; - private static final String GET_ORPHANED_PROC_LIST = - "SELECT " + - "proc.*, " + - "host.str_name AS host_name, " + - "host_stat.str_os, " + - "host.pk_alloc, " + - "alloc.pk_facility " + - "FROM " + - "proc, " + - "host, " + - "host_stat,"+ - "alloc " + - "WHERE " + - "proc.pk_host = host.pk_host " + - "AND " + - "host.pk_host = host_stat.pk_host " + - "AND " + - "host.pk_alloc = alloc.pk_alloc " + - "AND " + - "systimestamp - proc.ts_ping > " + ORPHANED_PROC_INTERVAL; - - public List findOrphanedVirtualProcs() { - return getJdbcTemplate().query(GET_ORPHANED_PROC_LIST, VIRTUAL_PROC_MAPPER); - } - - public List findOrphanedVirtualProcs(int limit) { - return getJdbcTemplate().query( - GET_ORPHANED_PROC_LIST + " AND rownum <= " + limit, - VIRTUAL_PROC_MAPPER); - } - - private static final String IS_ORPHAN = - "SELECT " + - "COUNT(1) " + - "FROM " + - "proc " + - "WHERE " + - "proc.pk_proc = ? " + - "AND " + - "systimestamp - proc.ts_ping > " + ORPHANED_PROC_INTERVAL; - - @Override - public boolean isOrphan(ProcInterface proc) { - return getJdbcTemplate().queryForObject(IS_ORPHAN, - Integer.class, proc.getProcId()) == 1; - } - - - public boolean increaseReservedMemory(ProcInterface p, long value) { - try { - return getJdbcTemplate().update("UPDATE proc SET int_mem_reserved=? WHERE pk_proc=? AND int_mem_reserved < ?", - value, p.getProcId(), value) == 1; - } catch (Exception e) { - // check by trigger erify_host_resources - throw new ResourceReservationFailureException("failed to increase memory reserveration for proc " - + p.getProcId() + " to " + value + ", proc does not have that much memory to spare."); - } - } - - private static final String FIND_WORST_MEMORY_OFFENDER = - "SELECT " + - "pk_proc, " + - "pk_host, " + - "pk_show, "+ - "pk_job, "+ - "pk_layer,"+ - "pk_frame,"+ - "b_unbooked,"+ - "b_local, "+ - "pk_alloc, "+ - "pk_facility, " + - "int_cores_reserved,"+ - "int_mem_reserved," + - "int_mem_max_used,"+ - "int_mem_used,"+ - "int_gpu_reserved," + - "int_virt_max_used,"+ - "int_virt_used,"+ - "host_name, " + - "str_os " + - "FROM (" - + GET_VIRTUAL_PROC + " " + - "AND " + - "host.pk_host =? " + - "AND " + - "proc.int_mem_reserved != 0 " + - "ORDER BY " + - "proc.int_virt_used / proc.int_mem_pre_reserved DESC ) " + - "WHERE " + - "ROWNUM = 1"; - - @Override - public VirtualProc getWorstMemoryOffender(HostInterface host) { - return getJdbcTemplate().queryForObject(FIND_WORST_MEMORY_OFFENDER, - VIRTUAL_PROC_MAPPER, host.getHostId()); - } - - public long getReservedMemory(ProcInterface proc) { - return getJdbcTemplate().queryForObject( - "SELECT int_mem_reserved FROM proc WHERE pk_proc=?", - Long.class, proc.getProcId()); - } - - public long getReservedGpu(ProcInterface proc) { - return getJdbcTemplate().queryForObject( - "SELECT int_gpu_reserved FROM proc WHERE pk_proc=?", - Long.class, proc.getProcId()); - } - - private static final String FIND_UNDERUTILIZED_PROCS = - "SELECT " + - "proc.pk_proc," + - "proc.int_mem_reserved - layer_mem.int_max_rss AS free_mem " + - "FROM " + - "proc," + - "host, " + - "layer_mem " + - "WHERE " + - "proc.pk_host = host.pk_host " + - "AND " + - "proc.pk_layer = layer_mem.pk_layer " + - "AND " + - "layer_mem.int_max_rss > 0 " + - "AND " + - "host.pk_host = ? " + - "AND " + - "proc.pk_proc != ? " + - "AND " + - "proc.int_mem_reserved - layer_mem.int_max_rss > 0"; - - public boolean balanceUnderUtilizedProcs(ProcInterface targetProc, long targetMem) { - - List> result = getJdbcTemplate().queryForList(FIND_UNDERUTILIZED_PROCS, - targetProc.getHostId(), targetProc.getProcId()); - - if (result.size() == 0) { - logger.info("unable to find under utilized procs on host " + targetProc.getName()); - return false; - } - - final Map borrowMap = new HashMap(result.size()); - for (Map map: result) { - logger.info("creating borrow map for: " + (String) map.get("pk_proc")); - borrowMap.put((String) map.get("pk_proc"), 0l); - } - - long memBorrowedTotal = 0l; - int pass = 0; - int maxPasses = 3; - - while(true) { - // the amount of memory we're going to borrow per frame/proc - long memPerFrame = ((targetMem - memBorrowedTotal) / result.size()) + 1; - - // loop through all of our other running frames and try to borrow - // a little bit of memory from each one. - for (Map map: result) { - String pk_proc = (String) map.get("pk_proc"); - BigDecimal free_mem = (BigDecimal) map.get("free_mem"); - long available = free_mem.longValueExact() - borrowMap.get(pk_proc) - Dispatcher.MEM_RESERVED_MIN; - if (available > memPerFrame) { - borrowMap.put(pk_proc, borrowMap.get(pk_proc) + memPerFrame); - memBorrowedTotal = memBorrowedTotal + memPerFrame; - } - } - pass++; - - // If we were unable to borrow anything, just break - if (memBorrowedTotal == 0) { break; } - // If we got the memory we needed, break - if (memBorrowedTotal >= targetMem) { break; } - // If we've exceeded the number of tries in this loop, break - if (pass >= maxPasses) { break; } - } - - logger.info("attempted to borrow " + targetMem + " for host " - + targetProc.getName() + ", obtained " + memBorrowedTotal); - - if (memBorrowedTotal < targetMem) { - logger.warn("mem borrowed " + memBorrowedTotal + - " was less than the target memory of " + targetMem); - return false; - } - - /* - * This might fail... I'm not really sure if we should - * fail the whole operation or what. Just gonna let it ride for now. - */ - for (Map.Entry set: borrowMap.entrySet()) { - int success = getJdbcTemplate().update( - "UPDATE proc SET int_mem_reserved = int_mem_reserved - ? WHERE pk_proc=?", - set.getValue(), set.getKey()); - logger.info("transfering " + (set.getValue() * success) + " from " + set.getKey()); - } - - return true; - } - - public void updateReservedMemory(ProcInterface p, long value) { - getJdbcTemplate().update("UPDATE proc SET int_mem_reserved=? WHERE pk_proc=?", - value, p.getProcId()); - } - - /** - * Updates proc counts for the host, subscription, - * layer, job, folder, and proc point when a proc - * is destroyed. - * - * @param proc - */ - private void procDestroyed(VirtualProc proc) { - - - getJdbcTemplate().update( - "UPDATE " + - "host " + - "SET " + - "int_cores_idle = int_cores_idle + ?," + - "int_mem_idle = int_mem_idle + ?, " + - "int_gpu_idle = int_gpu_idle + ? " + - "WHERE " + - "pk_host = ?", - proc.coresReserved, proc.memoryReserved, proc.gpuReserved, proc.getHostId()); - - if (!proc.isLocalDispatch) { - getJdbcTemplate().update( - "UPDATE " + - "subscription " + - "SET " + - "int_cores = int_cores - ? " + - "WHERE " + - "pk_show = ? " + - "AND " + - "pk_alloc = ?", - proc.coresReserved, proc.getShowId(), - proc.getAllocationId()); - } - - getJdbcTemplate().update( - "UPDATE " + - "layer_resource " + - "SET " + - "int_cores = int_cores - ? " + - "WHERE " + - "pk_layer = ?", - proc.coresReserved, proc.getLayerId()); - - if (!proc.isLocalDispatch) { - - getJdbcTemplate().update( - "UPDATE " + - "job_resource " + - "SET " + - "int_cores = int_cores - ? " + - "WHERE " + - "pk_job = ?", - proc.coresReserved, proc.getJobId()); - - getJdbcTemplate().update( - "UPDATE " + - "folder_resource " + - "SET " + - "int_cores = int_cores - ? " + - "WHERE " + - "pk_folder = " + - "(SELECT pk_folder FROM job WHERE pk_job=?)", - proc.coresReserved, proc.getJobId()); - - getJdbcTemplate().update( - "UPDATE " + - "point " + - "SET " + - "int_cores = int_cores - ? " + - "WHERE " + - "pk_dept = " + - "(SELECT pk_dept FROM job WHERE pk_job=?) " + - "AND " + - "pk_show = " + - "(SELECT pk_show FROM job WHERE pk_job=?) ", - proc.coresReserved, proc.getJobId(), proc.getJobId()); - } - - if (proc.isLocalDispatch) { - - getJdbcTemplate().update( - "UPDATE " + - "job_resource " + - "SET " + - "int_local_cores = int_local_cores - ? " + - "WHERE " + - "pk_job = ?", - proc.coresReserved, proc.getJobId()); - - getJdbcTemplate().update( - "UPDATE " + - "host_local " + - "SET " + - "int_cores_idle = int_cores_idle + ?, " + - "int_mem_idle = int_mem_idle + ?, " + - "int_gpu_idle = int_gpu_idle + ? " + - "WHERE " + - "pk_job = ? " + - "AND " + - "pk_host = ? ", - proc.coresReserved, - proc.memoryReserved, - proc.gpuReserved, - proc.getJobId(), - proc.getHostId()); - } - } - - /** - * Updates proc counts for the host, subscription, - * layer, job, folder, and proc point when a new - * proc is created. - * - * @param proc - */ - private void procCreated(VirtualProc proc) { - - getJdbcTemplate().update( - "UPDATE " + - "host " + - "SET " + - "int_cores_idle = int_cores_idle - ?," + - "int_mem_idle = int_mem_idle - ?, " + - "int_gpu_idle = int_gpu_idle - ? " + - "WHERE " + - "pk_host = ?", - proc.coresReserved, proc.memoryReserved, proc.gpuReserved, proc.getHostId()); - - - /** - * Not keeping track of local cores this way. - */ - - if (!proc.isLocalDispatch) { - getJdbcTemplate().update( - "UPDATE " + - "subscription " + - "SET " + - "int_cores = int_cores + ? " + - "WHERE " + - "pk_show = ? " + - "AND " + - "pk_alloc = ?", - proc.coresReserved, proc.getShowId(), - proc.getAllocationId()); - } - - getJdbcTemplate().update( - "UPDATE " + - "layer_resource " + - "SET " + - "int_cores = int_cores + ? " + - "WHERE " + - "pk_layer = ?", - proc.coresReserved, proc.getLayerId()); - - if (!proc.isLocalDispatch) { - - getJdbcTemplate().update( - "UPDATE " + - "job_resource " + - "SET " + - "int_cores = int_cores + ? " + - "WHERE " + - "pk_job = ?", - proc.coresReserved, proc.getJobId()); - - getJdbcTemplate().update( - "UPDATE " + - "folder_resource " + - "SET " + - "int_cores = int_cores + ? " + - "WHERE " + - "pk_folder = " + - "(SELECT pk_folder FROM job WHERE pk_job=?)", - proc.coresReserved, proc.getJobId()); - - getJdbcTemplate().update( - "UPDATE " + - "point " + - "SET " + - "int_cores = int_cores + ? " + - "WHERE " + - "pk_dept = " + - "(SELECT pk_dept FROM job WHERE pk_job=?) " + - "AND " + - "pk_show = " + - "(SELECT pk_show FROM job WHERE pk_job=?) ", - proc.coresReserved, proc.getJobId(), proc.getJobId()); - } - - if (proc.isLocalDispatch) { - - getJdbcTemplate().update( - "UPDATE " + - "job_resource " + - "SET " + - "int_local_cores = int_local_cores + ? " + - "WHERE " + - "pk_job = ?", - proc.coresReserved, proc.getJobId()); - - getJdbcTemplate().update( - "UPDATE " + - "host_local " + - "SET " + - "int_cores_idle = int_cores_idle - ?, " + - "int_mem_idle = int_mem_idle - ? " + - "WHERE " + - "pk_job = ? " + - "AND " + - "pk_host = ?", - proc.coresReserved, - proc.memoryReserved, - proc.getJobId(), - proc.getHostId()); - } - } -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/RedirectDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/RedirectDaoJdbc.java deleted file mode 100644 index d5b62b877..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/RedirectDaoJdbc.java +++ /dev/null @@ -1,119 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; - -import org.springframework.dao.EmptyResultDataAccessException; -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.Redirect; -import com.imageworks.spcue.dao.RedirectDao; -import com.imageworks.spcue.grpc.host.RedirectType; - -public class RedirectDaoJdbc extends JdbcDaoSupport implements RedirectDao { - @Override - public boolean containsKey(String key) { - return getJdbcTemplate().queryForObject( - "SELECT count(1) FROM redirect WHERE pk_proc = ?", - Integer.class, - key) > 0; - } - - @Override - public int countRedirectsWithGroup(String groupId) { - return getJdbcTemplate().queryForObject( - "SELECT count(1) FROM redirect WHERE str_group_id = ?", - Integer.class, - groupId); - } - - @Override - public int deleteExpired() { - long cutoff = System.currentTimeMillis() - Redirect.EXPIRE_TIME; - return getJdbcTemplate().update( - "DELETE FROM redirect WHERE lng_creation_time < ?", - cutoff); - } - - @Override - public void put(String key, Redirect r) { - getJdbcTemplate().update( - "MERGE INTO redirect D " - + "USING (SELECT 1 FROM dual) S " - + "ON (D.pk_proc = ?) " - + "WHEN MATCHED THEN UPDATE SET " - + " D.str_group_id = ?, " - + " D.int_type = ?, " - + " D.str_destination_id = ?, " - + " D.str_name = ?, " - + " D.lng_creation_time = ? " - + "WHEN NOT MATCHED THEN INSERT (D.pk_proc, D.str_group_id, D.int_type, D.str_destination_id, D.str_name, D.lng_creation_time) " - + " VALUES ( ?, ?, ?, ?, ?, ?)", - key, - - r.getGroupId(), - r.getType().getNumber(), - r.getDestinationId(), - r.getDestinationName(), - r.getCreationTime(), - - key, - r.getGroupId(), - r.getType().getNumber(), - r.getDestinationId(), - r.getDestinationName(), - r.getCreationTime()); - } - - @Override - public Redirect remove(String key) { - Redirect r = null; - try { - r = getJdbcTemplate().queryForObject( - "SELECT str_group_id, int_type, str_destination_id, str_name, lng_creation_time " - + "FROM redirect " - + "WHERE pk_proc = ? " - + "FOR UPDATE", - new RowMapper() { - @Override - public Redirect mapRow(ResultSet rs, int rowNum) throws SQLException { - return new Redirect( - rs.getString("str_group_id"), - RedirectType.forNumber(rs.getInt("int_type")), - rs.getString("str_destination_id"), - rs.getString("str_name"), - rs.getLong("lng_creation_time")); - } - }, - key); - } - catch (EmptyResultDataAccessException e) { - return null; - } - - getJdbcTemplate().update( - "DELETE FROM redirect WHERE pk_proc = ?", - key); - - return r; - } -} diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/ServiceDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/ServiceDaoJdbc.java deleted file mode 100644 index 8b9225636..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/ServiceDaoJdbc.java +++ /dev/null @@ -1,256 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.LinkedHashSet; - -import com.google.common.collect.Sets; -import org.apache.commons.lang.StringUtils; -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.ServiceEntity; -import com.imageworks.spcue.ServiceOverrideEntity; -import com.imageworks.spcue.dao.ServiceDao; -import com.imageworks.spcue.util.SqlUtil; - -public class ServiceDaoJdbc extends JdbcDaoSupport implements ServiceDao { - - private static final String SPLITTER = " \\| "; - - private static final String JOINER = " | "; - - public static LinkedHashSet splitTags(String tags) { - LinkedHashSet set = Sets.newLinkedHashSet(); - for(String s: tags.split(SPLITTER)) { - set.add(s.replaceAll(" ", "")); - } - return set; - } - - public static String joinTags(LinkedHashSet tags) { - return StringUtils.join(tags, JOINER); - } - - public static final RowMapper SERVICE_MAPPER = - new RowMapper() { - public ServiceEntity mapRow(ResultSet rs, int rowNum) throws SQLException { - ServiceEntity s = new ServiceEntity(); - s.id = rs.getString("pk_service"); - s.name = rs.getString("str_name"); - s.minCores = rs.getInt("int_cores_min"); - s.maxCores = rs.getInt("int_cores_max"); - s.minMemory = rs.getLong("int_mem_min"); - s.minGpu = rs.getLong("int_gpu_min"); - s.threadable = rs.getBoolean("b_threadable"); - s.tags = splitTags(rs.getString("str_tags")); - return s; - } - }; - - public static final RowMapper SERVICE_OVERRIDE_MAPPER = - new RowMapper() { - public ServiceOverrideEntity mapRow(ResultSet rs, int rowNum) - throws SQLException { - ServiceOverrideEntity s = new ServiceOverrideEntity(); - s.id = rs.getString("pk_show_service"); - s.name = rs.getString("str_name"); - s.minCores = rs.getInt("int_cores_min"); - s.maxCores = rs.getInt("int_cores_max"); - s.minMemory = rs.getLong("int_mem_min"); - s.minGpu = rs.getLong("int_gpu_min"); - s.threadable = rs.getBoolean("b_threadable"); - s.tags = splitTags(rs.getString("str_tags")); - s.showId = rs.getString("pk_show"); - return s; - } - }; - - private static final String QUERY_FOR_SERVICE = - "SELECT " + - "service.pk_service," + - "service.str_name," + - "service.b_threadable," + - "service.int_cores_min," + - "service.int_cores_max," + - "service.int_mem_min," + - "service.int_gpu_min," + - "service.str_tags " + - "FROM " + - "service "; - - @Override - public ServiceEntity get(String id) { - return getJdbcTemplate().queryForObject( - QUERY_FOR_SERVICE + " WHERE (pk_service=? OR str_name=?)", - SERVICE_MAPPER, id, id); - } - - private static final String QUERY_FOR_SERVICE_OVER = - "SELECT " + - "show_service.pk_show_service," + - "show_service.str_name," + - "show_service.b_threadable," + - "show_service.int_cores_min," + - "show_service.int_cores_max, "+ - "show_service.int_mem_min," + - "show_service.int_gpu_min," + - "show_service.str_tags, " + - "show.pk_show " + - "FROM " + - "show_service," + - "show " + - "WHERE " + - "show_service.pk_show = show.pk_show "; - - @Override - public ServiceOverrideEntity getOverride(String id, String show) { - return getJdbcTemplate() - .queryForObject( - QUERY_FOR_SERVICE_OVER - + " AND (show_service.pk_show_service=? OR show_service.str_name=?)" - + " AND (show.str_name=? OR show.pk_show=?)", - SERVICE_OVERRIDE_MAPPER, id, id, show, show); - } - - @Override - public ServiceOverrideEntity getOverride(String id) { - return getJdbcTemplate().queryForObject( - QUERY_FOR_SERVICE_OVER + " AND (show_service.pk_show_service=? " + - "OR show_service.str_name=?)", - SERVICE_OVERRIDE_MAPPER, id, id); - } - - @Override - public boolean isOverridden(String service, String show) { - return getJdbcTemplate().queryForObject( - "SELECT COUNT(1) FROM show_service, show WHERE " - + "show_service.pk_show = show.pk_show = ? " - + "AND show_service.str_name=? and show.str_name=?", - Integer.class, service, show) > 0; - } - - private static final String INSERT_SERVICE = - "INSERT INTO " + - "service " + - "(" + - "pk_service," + - "str_name," + - "b_threadable," + - "int_cores_min," + - "int_cores_max, "+ - "int_mem_min," + - "int_gpu_min," + - "str_tags" + - ") VALUES (?,?,?,?,?,?,?,?)"; - - @Override - public void insert(ServiceEntity service) { - service.id = SqlUtil.genKeyRandom(); - getJdbcTemplate().update(INSERT_SERVICE, service.id, - service.name, service.threadable, service.minCores, - service.maxCores, service.minMemory, service.minGpu, - StringUtils.join(service.tags.toArray(), " | ")); - } - - private static final String INSERT_SERVICE_WITH_SHOW = - "INSERT INTO " + - "show_service " + - "(" + - "pk_show_service," + - "pk_show, " + - "str_name," + - "b_threadable," + - "int_cores_min," + - "int_cores_max," + - "int_mem_min," + - "int_gpu_min," + - "str_tags " + - ") VALUES (?,?,?,?,?,?,?,?,?)"; - - @Override - public void insert(ServiceOverrideEntity service) { - service.id = SqlUtil.genKeyRandom(); - getJdbcTemplate().update(INSERT_SERVICE_WITH_SHOW, service.id, - service.showId, service.name, service.threadable, - service.minCores, service.maxCores, service.minMemory, - service.minGpu, joinTags(service.tags)); - } - - private static final String UPDATE_SERVICE = - "UPDATE " + - "service " + - "SET " + - "str_name=?," + - "b_threadable=?," + - "int_cores_min=?," + - "int_cores_max=?,"+ - "int_mem_min=?," + - "int_gpu_min=?," + - "str_tags=? " + - "WHERE " + - "pk_service = ?"; - - @Override - public void update(ServiceEntity service) { - getJdbcTemplate().update(UPDATE_SERVICE, service.name, - service.threadable, service.minCores, service.maxCores, - service.minMemory, service.minGpu, joinTags(service.tags), - service.getId()); - } - - private static final String UPDATE_SERVICE_WITH_SHOW = - "UPDATE " + - "show_service " + - "SET " + - "str_name=?," + - "b_threadable=?," + - "int_cores_min=?," + - "int_cores_max=?," + - "int_mem_min=?," + - "int_gpu_min=?," + - "str_tags=? " + - "WHERE " + - "pk_show_service = ?"; - - @Override - public void update(ServiceOverrideEntity service) { - getJdbcTemplate().update(UPDATE_SERVICE_WITH_SHOW, service.name, - service.threadable, service.minCores, service.maxCores, - service.minMemory, service.minGpu, joinTags(service.tags), - service.getId()); - } - - @Override - public void delete(ServiceEntity service) { - getJdbcTemplate().update( - "DELETE FROM service WHERE pk_service=?", service.getId()); - } - - @Override - public void delete(ServiceOverrideEntity service) { - getJdbcTemplate().update( - "DELETE FROM show_service WHERE pk_show_service=?", - service.getId()); - } -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/ShowDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/ShowDaoJdbc.java deleted file mode 100644 index 0d6765e51..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/ShowDaoJdbc.java +++ /dev/null @@ -1,221 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; - -import org.apache.commons.lang.StringUtils; -import org.springframework.dao.DataAccessException; -import org.springframework.dao.EmptyResultDataAccessException; -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.HostInterface; -import com.imageworks.spcue.ShowEntity; -import com.imageworks.spcue.ShowInterface; -import com.imageworks.spcue.dao.ShowDao; -import com.imageworks.spcue.util.SqlUtil; - -public class ShowDaoJdbc extends JdbcDaoSupport implements ShowDao { - - private static final RowMapper SHOW_MAPPER = - new RowMapper() { - public ShowEntity mapRow(ResultSet rs, int rowNum) throws SQLException { - ShowEntity show = new ShowEntity(); - show.name = rs.getString("str_name"); - show.id = rs.getString("pk_show"); - show.defaultMaxCores = rs.getInt("int_default_max_cores"); - show.defaultMinCores = rs.getInt("int_default_min_cores"); - show.active = rs.getBoolean("b_active"); - - if (rs.getString("str_comment_email") != null) { - show.commentMail = rs.getString("str_comment_email").split(","); - } - else { - show.commentMail = new String[0]; - } - return show; - } - }; - - private static final String GET_SHOW = - "SELECT " + - "show.pk_show, " + - "show.int_default_max_cores, " + - "show.int_default_min_cores, " + - "show.str_name, " + - "show.b_active, " + - "show.str_comment_email " + - "FROM " + - "show "; - - private static final String GET_SHOW_BY_ALIAS = - "SELECT " + - "show.pk_show, " + - "show.int_default_max_cores, " + - "show.int_default_min_cores, " + - "show_alias.str_name, " + - "show.b_active, " + - "show.str_comment_email " + - "FROM " + - "show, " + - "show_alias " + - "WHERE " + - "show.pk_show = show_alias.pk_show " ; - - public ShowEntity findShowDetail(String name) { - try { - return getJdbcTemplate().queryForObject(GET_SHOW + "WHERE show.str_name=?", - SHOW_MAPPER, name); - } catch (EmptyResultDataAccessException e) { - return getJdbcTemplate().queryForObject(GET_SHOW_BY_ALIAS + "AND show_alias.str_name = ?", - SHOW_MAPPER, name); - } - } - - public ShowEntity getShowDetail(String id) { - return getJdbcTemplate().queryForObject( - GET_SHOW + "WHERE show.pk_show=?", SHOW_MAPPER, id); - } - - private static final String GET_PREFERRED_SHOW = - "SELECT " + - "show.pk_show, " + - "show.int_default_max_cores, " + - "show.int_default_min_cores, " + - "show.str_name, " + - "show.b_active, " + - "show.str_comment_email " + - "FROM " + - "show, "+ - "owner,"+ - "deed " + - "WHERE " + - "show.pk_show = owner.pk_show " + - "AND " + - "deed.pk_owner = owner.pk_owner " + - "AND " + - "deed.pk_host = ?"; - - public ShowEntity getShowDetail(HostInterface host) { - return getJdbcTemplate().queryForObject( - GET_PREFERRED_SHOW, SHOW_MAPPER, host.getHostId()); - } - - private static final String INSERT_SHOW = - "INSERT INTO show (pk_show,str_name) VALUES (?,?)"; - - public void insertShow(ShowEntity show) { - show.id = SqlUtil.genKeyRandom(); - getJdbcTemplate().update(INSERT_SHOW, show.id, show.name); - } - - private static final String SHOW_EXISTS = - "SELECT " + - "COUNT(show.pk_show) " + - "FROM " + - "show LEFT JOIN show_alias ON (show.pk_show = show_alias.pk_show )" + - "WHERE " + - "(show.str_name = ? OR show_alias.str_name = ?) "; - public boolean showExists(String name) { - try { - return getJdbcTemplate().queryForObject(SHOW_EXISTS, - Integer.class, name, name) >= 1; - } catch (DataAccessException e) { - return false; - } - } - - @Override - public void delete(ShowInterface s) { - getJdbcTemplate().update("DELETE FROM point WHERE pk_show=?", - s.getShowId()); - getJdbcTemplate().update("DELETE FROM folder WHERE pk_show=?", - s.getShowId()); - getJdbcTemplate().update("DELETE FROM folder WHERE pk_show=?", - s.getShowId()); - getJdbcTemplate().update("DELETE FROM show_alias WHERE pk_show=?", - s.getShowId()); - getJdbcTemplate().update("DELETE FROM show WHERE pk_show=?", - s.getShowId()); - } - - public void updateShowDefaultMinCores(ShowInterface s, int val) { - if (val < 0) { - String msg = "Invalid argument, default min cores " + val + - "must be greater tham 0"; - throw new IllegalArgumentException(msg); - } - getJdbcTemplate().update( - "UPDATE show SET int_default_min_cores=? WHERE pk_show=?", - val, s.getShowId()); - } - - public void updateShowDefaultMaxCores(ShowInterface s, int val) { - if (val < 0) { - String msg = "Invalid argument, default max cores " + val + - "must be greater tham 0"; - throw new IllegalArgumentException(msg); - } - getJdbcTemplate().update( - "UPDATE show SET int_default_max_cores=? WHERE pk_show=?", - val, s.getShowId()); - } - - @Override - public void updateBookingEnabled(ShowInterface s, boolean enabled) { - getJdbcTemplate().update( - "UPDATE show SET b_booking_enabled = ? WHERE pk_show=?", - enabled, s.getShowId()); - } - - @Override - public void updateDispatchingEnabled(ShowInterface s, boolean enabled) { - getJdbcTemplate().update( - "UPDATE show SET b_dispatch_enabled = ? WHERE pk_show=?", - enabled, s.getShowId()); - } - - @Override - public void updateActive(ShowInterface s, boolean enabled) { - getJdbcTemplate().update( - "UPDATE show SET b_active= ? WHERE pk_show=?", - enabled, s.getShowId()); - } - - @Override - public void updateShowCommentEmail(ShowInterface s, String[] email) { - getJdbcTemplate().update( - "UPDATE show SET str_comment_email = ? WHERE pk_show=?", - StringUtils.join(email, ","), s.getShowId()); - } - - @Override - public void updateFrameCounters(ShowInterface s, int exitStatus) { - String col = "int_frame_success_count = int_frame_success_count + 1"; - if (exitStatus > 0) { - col = "int_frame_fail_count = int_frame_fail_count + 1"; - } - getJdbcTemplate().update( - "UPDATE show SET " + col + " WHERE pk_show=?", s.getShowId()); - } -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/SubscriptionDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/SubscriptionDaoJdbc.java deleted file mode 100644 index 55548ae04..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/SubscriptionDaoJdbc.java +++ /dev/null @@ -1,229 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; - -import org.springframework.dao.DataAccessException; -import org.springframework.dao.EmptyResultDataAccessException; -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.AllocationInterface; -import com.imageworks.spcue.EntityModificationError; -import com.imageworks.spcue.ShowInterface; -import com.imageworks.spcue.SubscriptionEntity; -import com.imageworks.spcue.SubscriptionInterface; -import com.imageworks.spcue.VirtualProc; -import com.imageworks.spcue.dao.SubscriptionDao; -import com.imageworks.spcue.util.SqlUtil; - -public class SubscriptionDaoJdbc extends JdbcDaoSupport implements SubscriptionDao { - - private static final String IS_SHOW_OVER_SIZE = - "SELECT " + - "COUNT(1) " + - "FROM " + - "subscription s " + - "WHERE " + - "s.pk_show = ? " + - "AND " + - "s.pk_alloc = ? " + - "AND " + - "s.int_cores > s.int_size "; - - public boolean isShowOverSize(ShowInterface show, AllocationInterface alloc) { - try { - return getJdbcTemplate().queryForObject(IS_SHOW_OVER_SIZE, - Integer.class, show.getShowId(), alloc.getAllocationId()) > 0; - } catch (EmptyResultDataAccessException e) { - return false; - } - } - - public boolean isShowOverSize(VirtualProc proc) { - try { - return getJdbcTemplate().queryForObject(IS_SHOW_OVER_SIZE, - Integer.class, proc.getShowId(), proc.getAllocationId()) > 0; - } catch (EmptyResultDataAccessException e) { - return false; - } - } - - private static final String IS_SHOW_AT_OR_OVER_SIZE = - "SELECT " + - "COUNT(1) " + - "FROM " + - "subscription s " + - "WHERE " + - "s.pk_show = ? " + - "AND " + - "s.pk_alloc = ? " + - "AND " + - "s.int_cores >= s.int_size "; - - public boolean isShowAtOrOverSize(ShowInterface show, AllocationInterface alloc) { - try { - return getJdbcTemplate().queryForObject(IS_SHOW_AT_OR_OVER_SIZE, - Integer.class, show.getShowId(), alloc.getAllocationId()) > 0; - } catch (EmptyResultDataAccessException e) { - return false; - } - } - - private static final String IS_SHOW_OVER_BURST = - "SELECT " + - "COUNT(1) " + - "FROM " + - "subscription s " + - "WHERE " + - "s.pk_show = ? " + - "AND " + - "s.pk_alloc = ? " + - "AND " + - "s.int_cores + ? > s.int_burst"; - - @Override - public boolean isShowOverBurst(ShowInterface show, AllocationInterface alloc, int coreUnits) { - try { - return getJdbcTemplate().queryForObject(IS_SHOW_OVER_BURST, - Integer.class, show.getShowId(), alloc.getAllocationId(), - coreUnits) > 0; - } catch (EmptyResultDataAccessException e) { - return true; - } - } - - private static final String IS_SHOW_AT_OR_OVER_BURST = - "SELECT " + - "COUNT(1) " + - "FROM " + - "subscription s " + - "WHERE " + - "s.pk_show = ? " + - "AND " + - "s.pk_alloc = ? " + - "AND " + - "s.int_cores >= s.int_burst"; - - @Override - public boolean isShowAtOrOverBurst(ShowInterface show, AllocationInterface alloc) { - try { - return getJdbcTemplate().queryForObject(IS_SHOW_AT_OR_OVER_BURST, - Integer.class, show.getShowId(), alloc.getAllocationId()) > 0; - } catch (EmptyResultDataAccessException e) { - return true; - } - } - - private static final String GET_SUB = - "SELECT " + - "subscription.pk_alloc," + - "subscription.pk_show,"+ - "subscription.int_size,"+ - "subscription.int_burst,"+ - "subscription.pk_subscription,"+ - "(alloc.str_name || '.' || show.str_name) AS str_name " + - "FROM " + - "subscription," + - "alloc," + - "show," + - "facility " + - "WHERE " + - "subscription.pk_show = show.pk_show " + - "AND " + - "subscription.pk_alloc = alloc.pk_alloc " + - "AND " + - "alloc.pk_facility = facility.pk_facility "; - - public static RowMapper SUB_MAPPER = new RowMapper() { - public SubscriptionEntity mapRow(ResultSet rs, int rowNum) throws SQLException { - SubscriptionEntity s = new SubscriptionEntity(); - s.allocationId = rs.getString("pk_alloc"); - s.burst = rs.getInt("int_burst"); - s.size = rs.getInt("int_size"); - s.name = rs.getString("str_name"); - s.showId = rs.getString("pk_show"); - s.id = rs.getString("pk_subscription"); - return s; - } - }; - - public SubscriptionEntity getSubscriptionDetail(String id) { - return getJdbcTemplate().queryForObject( - GET_SUB + " AND pk_subscription=?", - SUB_MAPPER, id); - } - - private static final String INSERT_SUBSCRIPTION = - "INSERT INTO " + - "subscription " + - "( " + - "pk_subscription, pk_alloc, pk_show, int_size, int_burst"+ - ") " + - "VALUES (?,?,?,?,?)"; - - public void insertSubscription(SubscriptionEntity detail) { - detail.id = SqlUtil.genKeyRandom(); - getJdbcTemplate().update(INSERT_SUBSCRIPTION, - detail.id, detail.allocationId, detail.showId, detail.size, detail.burst); - } - private static final String HAS_RUNNING_PROCS = - "SELECT " + - "COUNT(1) " + - "FROM " + - "subscription s " + - "WHERE " + - "s.pk_subscription=? " + - "AND " + - "s.int_cores > 0 "; - - public boolean hasRunningProcs(SubscriptionInterface sub) { - try { - return getJdbcTemplate().queryForObject(HAS_RUNNING_PROCS, - Integer.class, sub.getSubscriptionId()) > 0; - } catch (DataAccessException e) { - return false; - } - } - - public void deleteSubscription(SubscriptionInterface sub) { - if (hasRunningProcs(sub)) { - throw new EntityModificationError("You cannot delete a subscription with running procs"); - } - getJdbcTemplate().update( - "DELETE FROM subscription WHERE pk_subscription=?", - sub.getSubscriptionId()); - } - - public void updateSubscriptionSize(SubscriptionInterface sub, int size) { - getJdbcTemplate().update( - "UPDATE subscription SET int_size=? WHERE pk_subscription=?", - size, sub.getSubscriptionId()); - } - - public void updateSubscriptionBurst(SubscriptionInterface sub, int size) { - getJdbcTemplate().update( - "UPDATE subscription SET int_burst=? WHERE pk_subscription=?", - size, sub.getSubscriptionId()); - } -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/TaskDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/TaskDaoJdbc.java deleted file mode 100644 index 059c67808..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/TaskDaoJdbc.java +++ /dev/null @@ -1,259 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Map; - -import org.springframework.dao.EmptyResultDataAccessException; -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.DepartmentInterface; -import com.imageworks.spcue.JobInterface; -import com.imageworks.spcue.PointInterface; -import com.imageworks.spcue.ShowInterface; -import com.imageworks.spcue.TaskEntity; -import com.imageworks.spcue.TaskInterface; -import com.imageworks.spcue.dao.TaskDao; -import com.imageworks.spcue.util.SqlUtil; - -/** - * DAO for managing department tasks - */ -public class TaskDaoJdbc extends JdbcDaoSupport implements TaskDao { - - @Override - public void deleteTasks(PointInterface cdept) { - getJdbcTemplate().update( - "DELETE FROM task WHERE pk_point=?", - cdept.getPointId()); - } - - @Override - public void deleteTasks(ShowInterface show, DepartmentInterface dept) { - getJdbcTemplate().update( - "DELETE FROM task WHERE pk_show=? AND pk_dept=?", - show.getShowId(), dept.getDepartmentId()); - } - - @Override - public void deleteTask(TaskInterface task) { - getJdbcTemplate().update( - "DELETE FROM task WHERE pk_task=?", - task.getId()); - } - - @Override - public boolean isManaged(TaskInterface t) { - try { - return getJdbcTemplate().queryForObject("SELECT b_managed FROM point WHERE pk_show=? and pk_dept=?", - Integer.class, t.getShowId(), t.getDepartmentId()) == 1; - } catch (org.springframework.dao.DataRetrievalFailureException e) { - return false; - } - } - - private static final String INSERT_TASK = - "INSERT INTO " + - "task " + - "( " + - "pk_task,"+ - "pk_point,"+ - "str_shot," + - "int_min_cores" + - ") " + - "VALUES (?,?,?,?)"; - - @Override - public void insertTask(TaskEntity task) { - task.id = SqlUtil.genKeyRandom(); - getJdbcTemplate().update(INSERT_TASK, - task.id, task.getPointId(), task.shot, task.minCoreUnits); - } - - private static final String GET_TASK_DETAIL = - "SELECT " + - "point.pk_dept,"+ - "point.pk_show,"+ - "point.pk_point,"+ - "task.pk_task," + - "task.int_min_cores + task.int_adjust_cores AS int_min_cores,"+ - "task.str_shot,"+ - "(task.str_shot || '.' || dept.str_name) AS str_name " + - "FROM " + - "point,"+ - "task,"+ - "dept, "+ - "show "+ - "WHERE " + - "point.pk_dept = dept.pk_dept "+ - "AND " + - "point.pk_show = show.pk_show " + - "AND " + - "point.pk_point = task.pk_point "; - - public static final RowMapper TASK_DETAIL_MAPPER = - new RowMapper() { - public TaskEntity mapRow(ResultSet rs, int row) throws SQLException { - TaskEntity t = new TaskEntity(); - t.pointId = rs.getString("pk_point"); - t.deptId = rs.getString("pk_dept"); - t.showId = rs.getString("pk_show"); - t.id = rs.getString("pk_task"); - t.minCoreUnits = rs.getInt("int_min_cores"); - t.name = rs.getString("str_name"); - t.shot = rs.getString("str_shot"); - return t; - } - }; - - @Override - public TaskEntity getTaskDetail(String id) { - return getJdbcTemplate().queryForObject( - GET_TASK_DETAIL + " AND task.pk_task=?", - TASK_DETAIL_MAPPER, id); - } - - @Override - public TaskEntity getTaskDetail(DepartmentInterface d, String shot) { - return getJdbcTemplate().queryForObject( - GET_TASK_DETAIL + " AND point.pk_dept = ? AND task.str_shot = ?", - TASK_DETAIL_MAPPER, d.getDepartmentId(), shot); - } - - @Override - public TaskEntity getTaskDetail(JobInterface j) { - Map map = getJdbcTemplate().queryForMap( - "SELECT pk_dept, str_shot FROM job WHERE job.pk_job=?", j.getJobId()); - - return getJdbcTemplate().queryForObject( - GET_TASK_DETAIL + " AND task.str_shot = ? AND point.pk_dept = ?", - TASK_DETAIL_MAPPER, map.get("str_shot").toString(), map.get("pk_dept").toString()); - } - - public void updateTaskMinCores(TaskInterface t, int value) { - if (value < 0) { - throw new IllegalArgumentException("min cores must be greater than or equal to 0"); - } - getJdbcTemplate().update( - "UPDATE task SET int_min_cores=? WHERE pk_task=?", - value, t.getTaskId()); - } - - @Override - public void adjustTaskMinCores(TaskInterface t, int value) { - if (value < 0) { - throw new IllegalArgumentException("min cores must be greater than or equal to 0"); - } - getJdbcTemplate().update( - "UPDATE task SET int_adjust_cores = ? - int_min_cores WHERE pk_task=?", - value, t.getTaskId()); - } - - @Override - public void mergeTask(TaskEntity t) { - String pkTask = null; - try { - pkTask = getJdbcTemplate().queryForObject( - "SELECT task.pk_task FROM task, point WHERE task.pk_point = point.pk_point AND " + - "task.str_shot = ? AND point.pk_point=?", String.class, - t.shot, t.getPointId()); - - } catch (EmptyResultDataAccessException dae) { - // Eat this, its possible that no task exists - } - - // No need to do anything with this task. - if (pkTask == null && t.minCoreUnits == 0) { - return; - } - - if (t.minCoreUnits == 0) { - getJdbcTemplate().update("DELETE FROM task WHERE pk_point=? AND str_shot=? ", - t.getPointId(), t.shot); - } - else if (getJdbcTemplate().update( - "UPDATE task SET int_min_cores=? WHERE pk_point=? AND str_shot=?", - t.minCoreUnits, t.getPointId(), t.shot) == 0) { - try { - insertTask(t); - } - catch (org.springframework.dao.DataIntegrityViolationException e) { - logger.warn("error inserting task " + t.shot + "," + e); - } - } - } - - private static final String CLEAR_TASK_ADJUSTMENTS = - "UPDATE " + - "task " + - "SET " + - "int_adjust_cores = 0 " + - "WHERE " + - "pk_show=? " + - "AND " + - "pk_dept = ? "; - - @Override - public void clearTaskAdjustments(PointInterface cdept) { - getJdbcTemplate().update(CLEAR_TASK_ADJUSTMENTS, - cdept.getShowId(), cdept.getDepartmentId()); - } - - private static final String CLEAR_TASK_ADJUSTMENT = - "UPDATE " + - "task " + - "SET " + - "int_adjust_cores = 0 " + - "WHERE " + - "pk_task=?"; - - @Override - public void clearTaskAdjustment(TaskInterface t) { - getJdbcTemplate().update(CLEAR_TASK_ADJUSTMENT, t.getTaskId()); - } - - private static final String IS_JOB_MANAGED = - "SELECT " + - "COUNT(1) " + - "FROM " + - "job,"+ - "task,"+ - "point " + - "WHERE " + - "job.pk_show = point.pk_show " + - "AND " + - "job.pk_dept = point.pk_dept " + - "AND " + - "task.pk_point = point.pk_point " + - "AND " + - "task.str_shot = job.str_shot " + - "AND " + - "job.pk_job = ?"; - - @Override - public boolean isManaged(JobInterface j) { - return getJdbcTemplate().queryForObject(IS_JOB_MANAGED, - Integer.class, j.getJobId()) > 0; - } -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/TrackitDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/TrackitDaoJdbc.java deleted file mode 100644 index 7ab2f0d6c..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/TrackitDaoJdbc.java +++ /dev/null @@ -1,108 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; - -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.TrackitTaskDetail; -import com.imageworks.spcue.dao.TrackitDao; - -public class TrackitDaoJdbc extends JdbcDaoSupport implements TrackitDao { - - public static final RowMapper TASK_DETAIL_MAPPER = - new RowMapper() { - public TrackitTaskDetail mapRow(ResultSet rs, int rowNum) throws SQLException { - TrackitTaskDetail t = new TrackitTaskDetail(); - t.show = rs.getString("str_show"); - t.shot = rs.getString("str_shot"); - t.status = rs.getString("str_status"); - t.startDate = rs.getDate("dt_start_date"); - t.endDate = rs.getDate("dt_est_end"); - t.frameCount = rs.getInt("int_frame_count"); - t.cgSup = rs.getString("str_cgsup"); - t.weeks = rs.getInt("int_weeks"); - return t; - } - }; - - private static final String GET_TASKS = - "SELECT DISTINCT "+ - "(CASE " + - "WHEN " + - "(asset_task.dt_est_end - next_day(sysdate,'monday')) / 7 < 1 THEN 1 " + - "ELSE " + - "CAST(((asset_task.dt_est_end - next_day(sysdate,'monday')) / 7) AS NUMERIC(6,0)) " + - "END) AS int_weeks,"+ - "show.str_show_id AS str_show, " + - "asset.str_name AS str_shot, "+ - "asset_task.str_prod_status AS str_status,"+ - "asset_task.dt_start_date, "+ - "asset_task.dt_est_end, "+ - "shot.lng_cut_length AS int_frame_count, "+ - "xmltype(asset.xml_asset_metadata).extract('/header/cgsup/text()').getstringval() AS str_cgsup " + - "FROM "+ - "element.asset,"+ - "pts.show,"+ - "pts.shot,"+ - "element.asset_task,"+ - "element.asset_pipeline,"+ - "element.x_asset_type,"+ - "element.x_show_task,"+ - "element.asset_task_entitys,"+ - "contact.entity entity, "+ - "element.x_asset_task_status "+ - "WHERE "+ - "asset.pk_asset = asset_task.pk_asset "+ - "AND " + - "asset.pk_show = show.pk_show "+ - "AND " + - "asset.str_remote_handle = shot.pk_shot " + - "AND " + - "asset.pk_x_asset_type = x_asset_type.pk_x_asset_type " + - "AND " + - "asset_task.pk_asset_pipeline = asset_pipeline.pk_asset_pipeline " + - "AND " + - "asset_task.pk_asset_task = asset_task_entitys.pk_asset_task (+) "+ - "AND " + - "asset_task_entitys.pk_entity = entity.pk_entity (+) " + - "AND " + - "asset_task.pk_x_asset_task_status = x_asset_task_status.pk_x_asset_task_status " + - "AND " + - "asset_pipeline.pk_x_show_task = x_show_task.pk_x_show_task " + - "AND " + - "str_group_type = 'Shot' " + - "AND " + - "trunc(asset_task.dt_est_end) != '31-DEC-69' " + - "AND " + - "show.str_show_id = ? " + - "AND " + - "x_show_task.str_value = ? "; - - @Override - public List getTasks(String show, String dept) { - return getJdbcTemplate().query(GET_TASKS, TASK_DETAIL_MAPPER, show, dept); - } -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/WhiteboardDaoJdbc.java deleted file mode 100644 index 75d631d4a..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/oracle/WhiteboardDaoJdbc.java +++ /dev/null @@ -1,2195 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.dao.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Timestamp; -import java.util.Arrays; -import java.util.List; -import java.util.Locale; - -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import org.apache.log4j.Logger; -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.support.JdbcDaoSupport; - -import com.imageworks.spcue.ActionInterface; -import com.imageworks.spcue.AllocationInterface; -import com.imageworks.spcue.DeedEntity; -import com.imageworks.spcue.DepartmentInterface; -import com.imageworks.spcue.DependInterface; -import com.imageworks.spcue.FilterInterface; -import com.imageworks.spcue.FrameInterface; -import com.imageworks.spcue.GroupInterface; -import com.imageworks.spcue.HostInterface; -import com.imageworks.spcue.JobInterface; -import com.imageworks.spcue.LayerInterface; -import com.imageworks.spcue.LocalHostAssignment; -import com.imageworks.spcue.MatcherInterface; -import com.imageworks.spcue.OwnerEntity; -import com.imageworks.spcue.ShowInterface; -import com.imageworks.spcue.dao.WhiteboardDao; -import com.imageworks.spcue.dao.criteria.FrameSearchInterface; -import com.imageworks.spcue.dao.criteria.FrameSearchFactory; -import com.imageworks.spcue.dao.criteria.HostSearchInterface; -import com.imageworks.spcue.dao.criteria.JobSearchInterface; -import com.imageworks.spcue.dao.criteria.ProcSearchInterface; -import com.imageworks.spcue.dao.criteria.ProcSearchFactory; -import com.imageworks.spcue.grpc.comment.Comment; -import com.imageworks.spcue.grpc.comment.CommentSeq; -import com.imageworks.spcue.grpc.department.Department; -import com.imageworks.spcue.grpc.department.DepartmentSeq; -import com.imageworks.spcue.grpc.depend.Depend; -import com.imageworks.spcue.grpc.depend.DependSeq; -import com.imageworks.spcue.grpc.depend.DependTarget; -import com.imageworks.spcue.grpc.depend.DependType; -import com.imageworks.spcue.grpc.facility.Allocation; -import com.imageworks.spcue.grpc.facility.AllocationSeq; -import com.imageworks.spcue.grpc.facility.AllocationStats; -import com.imageworks.spcue.grpc.facility.Facility; -import com.imageworks.spcue.grpc.facility.FacilitySeq; -import com.imageworks.spcue.grpc.filter.Action; -import com.imageworks.spcue.grpc.filter.ActionSeq; -import com.imageworks.spcue.grpc.filter.ActionType; -import com.imageworks.spcue.grpc.filter.ActionValueType; -import com.imageworks.spcue.grpc.filter.Filter; -import com.imageworks.spcue.grpc.filter.FilterSeq; -import com.imageworks.spcue.grpc.filter.FilterType; -import com.imageworks.spcue.grpc.filter.MatchSubject; -import com.imageworks.spcue.grpc.filter.MatchType; -import com.imageworks.spcue.grpc.filter.Matcher; -import com.imageworks.spcue.grpc.filter.MatcherSeq; -import com.imageworks.spcue.grpc.host.Deed; -import com.imageworks.spcue.grpc.host.DeedSeq; -import com.imageworks.spcue.grpc.host.HardwareState; -import com.imageworks.spcue.grpc.host.Host; -import com.imageworks.spcue.grpc.host.HostSeq; -import com.imageworks.spcue.grpc.host.LockState; -import com.imageworks.spcue.grpc.host.NestedHost; -import com.imageworks.spcue.grpc.host.Owner; -import com.imageworks.spcue.grpc.host.Proc; -import com.imageworks.spcue.grpc.host.ProcSeq; -import com.imageworks.spcue.grpc.host.ThreadMode; -import com.imageworks.spcue.grpc.job.CheckpointState; -import com.imageworks.spcue.grpc.job.Frame; -import com.imageworks.spcue.grpc.job.FrameSeq; -import com.imageworks.spcue.grpc.job.FrameState; -import com.imageworks.spcue.grpc.job.Group; -import com.imageworks.spcue.grpc.job.GroupSeq; -import com.imageworks.spcue.grpc.job.GroupStats; -import com.imageworks.spcue.grpc.job.Job; -import com.imageworks.spcue.grpc.job.JobSeq; -import com.imageworks.spcue.grpc.job.JobState; -import com.imageworks.spcue.grpc.job.JobStats; -import com.imageworks.spcue.grpc.job.Layer; -import com.imageworks.spcue.grpc.job.LayerSeq; -import com.imageworks.spcue.grpc.job.LayerStats; -import com.imageworks.spcue.grpc.job.LayerType; -import com.imageworks.spcue.grpc.job.UpdatedFrame; -import com.imageworks.spcue.grpc.job.UpdatedFrameCheckResult; -import com.imageworks.spcue.grpc.job.UpdatedFrameSeq; -import com.imageworks.spcue.grpc.limit.Limit; -import com.imageworks.spcue.grpc.renderpartition.RenderPartition; -import com.imageworks.spcue.grpc.renderpartition.RenderPartitionSeq; -import com.imageworks.spcue.grpc.renderpartition.RenderPartitionType; -import com.imageworks.spcue.grpc.service.Service; -import com.imageworks.spcue.grpc.service.ServiceOverride; -import com.imageworks.spcue.grpc.service.ServiceOverrideSeq; -import com.imageworks.spcue.grpc.service.ServiceSeq; -import com.imageworks.spcue.grpc.show.Show; -import com.imageworks.spcue.grpc.show.ShowSeq; -import com.imageworks.spcue.grpc.show.ShowStats; -import com.imageworks.spcue.grpc.subscription.Subscription; -import com.imageworks.spcue.grpc.subscription.SubscriptionSeq; -import com.imageworks.spcue.grpc.task.Task; -import com.imageworks.spcue.grpc.task.TaskSeq; -import com.imageworks.spcue.util.Convert; -import com.imageworks.spcue.util.CueUtil; -import com.imageworks.spcue.util.SqlUtil; - -public class WhiteboardDaoJdbc extends JdbcDaoSupport implements WhiteboardDao { - @SuppressWarnings("unused") - private static final Logger logger = Logger.getLogger(WhiteboardDaoJdbc.class); - - private FrameSearchFactory frameSearchFactory; - private ProcSearchFactory procSearchFactory; - - @Override - public Service getService(String id) { - return getJdbcTemplate().queryForObject( - GET_SERVICE + " WHERE (pk_service=? or str_name=?)", - SERVICE_MAPPER, id, id); - } - - @Override - public Service findService(String name) { - return getJdbcTemplate().queryForObject( - GET_SERVICE + " WHERE service.str_name=?", - SERVICE_MAPPER, name); - } - @Override - public ServiceSeq getDefaultServices() { - List services = getJdbcTemplate().query(GET_SERVICE, SERVICE_MAPPER); - return ServiceSeq.newBuilder().addAllServices(services).build(); - } - - @Override - public ServiceOverrideSeq getServiceOverrides(ShowInterface show) { - return ServiceOverrideSeq.newBuilder().addAllServiceOverrides(getJdbcTemplate().query( - GET_SERVICE_OVERRIDE + " AND show_service.pk_show = ?", - SERVICE_OVERRIDE_MAPPER, show.getId())).build(); - } - - @Override - public ServiceOverride getServiceOverride( - ShowInterface show, String name) { - return getJdbcTemplate().queryForObject ( - GET_SERVICE_OVERRIDE + - " AND show_service.pk_show=? AND (show_service.str_name=? OR" + - " show_service.pk_show_service=?)", - SERVICE_OVERRIDE_MAPPER, show.getId(), name, name); - } - - @Override - public Filter getFilter(FilterInterface filter) { - return getJdbcTemplate().queryForObject(GET_FILTER + " AND pk_filter=?", - FILTER_MAPPER, filter.getFilterId()); - } - - @Override - public Filter findFilter(ShowInterface show, String name) { - return getJdbcTemplate().queryForObject( - GET_FILTER + " AND filter.pk_show=? AND filter.str_name=?", - FILTER_MAPPER, show.getShowId(), name); - } - - @Override - public Filter findFilter(String show, String name) { - return getJdbcTemplate().queryForObject( - GET_FILTER + " AND show.str_name=? AND filter.str_name=?", - FILTER_MAPPER, show, name); - } - - @Override - public FilterSeq getFilters(ShowInterface show) { - return FilterSeq.newBuilder().addAllFilters( - getJdbcTemplate().query( - GET_FILTER + " AND show.pk_show=? ORDER BY f_order ASC", - FILTER_MAPPER, show.getShowId())).build(); - } - - @Override - public ActionSeq getActions(FilterInterface filter) { - return ActionSeq.newBuilder().addAllActions(getJdbcTemplate().query( - GET_ACTION + " AND filter.pk_filter=? ORDER BY b_stop ASC, ts_created ASC ", - ACTION_MAPPER, filter.getFilterId())).build(); - } - - @Override - public MatcherSeq getMatchers(FilterInterface filter) { - return MatcherSeq.newBuilder().addAllMatchers(getJdbcTemplate().query( - GET_MATCHER + " AND filter.pk_filter=? ORDER BY ts_created ASC", - MATCHER_MAPPER, filter.getFilterId())).build(); - } - - @Override - public Action getAction(ActionInterface action) { - return getJdbcTemplate().queryForObject( - GET_ACTION + " AND action.pk_action=?", - ACTION_MAPPER, action.getActionId()); - } - - @Override - public Matcher getMatcher(MatcherInterface matcher) { - return getJdbcTemplate().queryForObject( - GET_MATCHER + " AND matcher.pk_matcher=?", - MATCHER_MAPPER, matcher.getMatcherId()); - } - - @Override - public Show getShow(String id) { - return getJdbcTemplate().queryForObject( - GET_SHOW + " AND show.pk_show=?", - SHOW_MAPPER, id); - } - - @Override - public ShowSeq getShows() { - List shows = getJdbcTemplate().query(GET_SHOW, SHOW_MAPPER); - return ShowSeq.newBuilder().addAllShows(shows).build(); - } - - @Override - public ShowSeq getActiveShows() { - List shows = getJdbcTemplate().query(GET_SHOW + " AND b_active=?", - SHOW_MAPPER, true); - return ShowSeq.newBuilder().addAllShows(shows).build(); - } - - @Override - public Show findShow(String name) { - return getJdbcTemplate().queryForObject(GET_SHOW + " AND show.str_name=?", - SHOW_MAPPER, name); - } - - @Override - public Subscription getSubscription(String id) { - return getJdbcTemplate().queryForObject( - GET_SUBSCRIPTION + " AND subscription.pk_subscription=?", - SUBSCRIPTION_MAPPER, id); - } - - @Override - public Subscription findSubscription(String show, String alloc) { - return getJdbcTemplate().queryForObject( - GET_SUBSCRIPTION + - " AND show.str_name=? AND alloc.str_name=?", - SUBSCRIPTION_MAPPER, show, alloc); - } - - @Override - public SubscriptionSeq getSubscriptions(ShowInterface show) { - List subscriptions = getJdbcTemplate().query( - GET_SUBSCRIPTION + " AND show.pk_show=?", - SUBSCRIPTION_MAPPER, show.getShowId()); - return SubscriptionSeq.newBuilder().addAllSubscriptions(subscriptions).build(); - } - - @Override - public SubscriptionSeq getSubscriptions(AllocationInterface alloc) { - List subscriptions = getJdbcTemplate().query( - GET_SUBSCRIPTION + " AND subscription.pk_alloc=?", - SUBSCRIPTION_MAPPER, alloc.getAllocationId()); - return SubscriptionSeq.newBuilder().addAllSubscriptions(subscriptions).build(); - } - - @Override - public Allocation findAllocation(String name) { - return getJdbcTemplate().queryForObject( - GET_ALLOCATION + " AND alloc.str_name=?", - ALLOCATION_MAPPER, name); - } - - @Override - public Allocation getAllocation(String id) { - return getJdbcTemplate().queryForObject(GET_ALLOCATION + " AND alloc.pk_alloc=?", - ALLOCATION_MAPPER, id); - } - - @Override - public AllocationSeq getAllocations() { - return AllocationSeq.newBuilder().addAllAllocations(getJdbcTemplate().query( - GET_ALLOCATION + " ORDER BY alloc.str_name ", - ALLOCATION_MAPPER)).build(); - } - - @Override - public AllocationSeq getAllocations( - com.imageworks.spcue.FacilityInterface facility) { - return AllocationSeq.newBuilder().addAllAllocations(getJdbcTemplate().query( - GET_ALLOCATION + " AND alloc.pk_facility = ?", - ALLOCATION_MAPPER, facility.getFacilityId())).build(); - } - - @Override - public JobSeq getJobs(GroupInterface group) { - List jobs = getJdbcTemplate().query( - GET_PENDING_JOBS + " AND job.pk_folder=? ORDER BY job.str_name ASC", - JOB_MAPPER, group.getId()); - return JobSeq.newBuilder().addAllJobs(jobs).build(); - } - - @Override - public List getJobNames(JobSearchInterface r) { - return getJdbcTemplate().query(r.getFilteredQuery(GET_JOB_NAMES), - new RowMapper() { - public String mapRow(ResultSet rs, int rowNum) throws SQLException { - return SqlUtil.getString(rs, 1); - } - }, r.getValuesArray()); - } - - @Override - public JobSeq getJobs(JobSearchInterface r) { - List jobs = getJdbcTemplate().query( - r.getFilteredQuery(GET_JOB) + "ORDER BY job.str_name ASC", - JOB_MAPPER, - r.getValuesArray()); - return JobSeq.newBuilder().addAllJobs(jobs).build(); - } - - @Override - public Job findJob(String name) { - return getJdbcTemplate().queryForObject( - GET_PENDING_JOBS + " AND job.str_name=?", - JOB_MAPPER, name.toLowerCase()); - } - - @Override - public Job getJob(String id) { - return getJdbcTemplate().queryForObject( - GET_JOB + " AND job.pk_job=?", - JOB_MAPPER, id); - } - - @Override - public Layer getLayer(String id) { - return getJdbcTemplate().queryForObject( - GET_LAYER_WITH_LIMITS + " WHERE layer.pk_layer=?", - LAYER_MAPPER, id); - } - - @Override - public Layer findLayer(String job, String layer) { - return getJdbcTemplate().queryForObject( - GET_LAYER_WITH_LIMITS + " WHERE job.str_state='PENDING' AND job.str_name=? AND layer.str_name=?", - LAYER_MAPPER, job, layer); - } - - @Override - public LayerSeq getLayers(JobInterface job) { - String query = GET_LAYER_WITH_LIMITS + " WHERE layer.pk_job=? ORDER BY layer.int_dispatch_order ASC"; - List layers = getJdbcTemplate().query( - query, LAYER_MAPPER, job.getJobId()); - return LayerSeq.newBuilder().addAllLayers(layers).build(); - } - - public List getLimitNames(String layerId) { - return getJdbcTemplate().query(GET_LIMIT_NAMES, - LIMIT_NAME_MAPPER, layerId); - } - - @Override - public List getLimits(LayerInterface layer) { - List limits = getJdbcTemplate().query( - GET_LIMIT_FROM_LAYER_ID, LIMIT_MAPPER, layer.getLayerId()); - return limits; - } - - @Override - public GroupSeq getGroups(ShowInterface show) { - List groups = getJdbcTemplate().query( - GET_GROUPS + " AND folder.pk_show=? ORDER BY folder_level.int_level ASC, folder.str_name ASC ", - GROUP_MAPPER, show.getShowId()); - return GroupSeq.newBuilder().addAllGroups(groups).build(); - } - - @Override - public GroupSeq getGroups(GroupInterface group) { - List groups = getJdbcTemplate().query( - GET_GROUPS + " AND folder.pk_parent_folder=? ORDER BY folder_level.int_level ASC, folder.f_order DESC, folder.str_name ASC ", - GROUP_MAPPER, group.getGroupId()); - return GroupSeq.newBuilder().addAllGroups(groups).build(); - } - - @Override - public Group getGroup(String id) { - return getJdbcTemplate().queryForObject( - GET_GROUPS + " AND folder.pk_folder=?", - GROUP_MAPPER, id); - } - - @Override - public Group getRootGroup(ShowInterface show) { - return getJdbcTemplate().queryForObject( - GET_GROUPS + " AND show.pk_show=? AND folder.b_default=?", - GROUP_MAPPER, show.getShowId(), true); - } - - @Override - public Frame findFrame(String job, String layer, int frame) { - return getJdbcTemplate().queryForObject(FIND_FRAME, FRAME_MAPPER, job, layer, frame); - } - - @Override - public Frame getFrame(String id) { - return getJdbcTemplate().queryForObject( - GET_FRAME + " AND frame.pk_frame=?", FRAME_MAPPER, id); - } - - @Override - public FrameSeq getFrames(FrameSearchInterface r) { - List frames = getJdbcTemplate().query( - r.getSortedQuery(GET_FRAMES_CRITERIA), FRAME_MAPPER, r.getValuesArray()); - return FrameSeq.newBuilder().addAllFrames(frames).build(); - } - - @Override - public Depend getDepend(DependInterface depend) { - return getJdbcTemplate().queryForObject( - GET_DEPEND + " WHERE pk_depend=?",DEPEND_MAPPER, depend.getId()); - } - - @Override - public Depend getDepend(com.imageworks.spcue.depend.AbstractDepend depend) { - return getJdbcTemplate().queryForObject( - GET_DEPEND + " WHERE pk_depend=?",DEPEND_MAPPER, depend.getId()); - } - - @Override - public DependSeq getWhatDependsOnThis(JobInterface job) { - List depends = getJdbcTemplate().query( - GET_DEPEND + " WHERE pk_parent IS NULL AND pk_job_depend_on=?", - DEPEND_MAPPER, job.getJobId()); - return DependSeq.newBuilder().addAllDepends(depends).build(); - } - - @Override - public DependSeq getWhatDependsOnThis(LayerInterface layer) { - List depends = getJdbcTemplate().query( - GET_DEPEND + " WHERE pk_parent IS NULL AND pk_layer_depend_on=?", - DEPEND_MAPPER, layer.getLayerId()); - return DependSeq.newBuilder().addAllDepends(depends).build(); - } - - @Override - public DependSeq getWhatDependsOnThis(FrameInterface frame) { - List depends = getJdbcTemplate().query( - GET_DEPEND + " WHERE pk_frame_depend_on=?", - DEPEND_MAPPER, frame.getFrameId()); - return DependSeq.newBuilder().addAllDepends(depends).build(); - } - - @Override - public DependSeq getWhatThisDependsOn(JobInterface job) { - List depends = getJdbcTemplate().query( - GET_DEPEND + " WHERE pk_parent IS NULL AND pk_layer_depend_er IS NULL AND " + - "pk_frame_depend_er IS NULL AND pk_job_depend_er=?", - DEPEND_MAPPER, job.getJobId()); - return DependSeq.newBuilder().addAllDepends(depends).build(); - } - - @Override - public DependSeq getWhatThisDependsOn(LayerInterface layer) { - List depends = getJdbcTemplate().query( - GET_DEPEND + " WHERE pk_parent IS NULL AND pk_layer_depend_er=?", - DEPEND_MAPPER, layer.getLayerId()); - return DependSeq.newBuilder().addAllDepends(depends).build(); - } - - @Override - public DependSeq getWhatThisDependsOn(FrameInterface frame) { - /* - * This should show anything that is making the frame dependent. - */ - List depends = getJdbcTemplate().query( - GET_DEPEND + " WHERE " + - "(pk_job_depend_er=? AND str_type IN ('JOB_ON_JOB','JOB_ON_LAYER','JOB_ON_FRAME')) OR " + - "(pk_layer_depend_er=? AND str_type IN ('LAYER_ON_JOB','LAYER_ON_LAYER','LAYER_ON_FRAME')) " + - "OR (pk_frame_depend_er=?)", - DEPEND_MAPPER, frame.getJobId(), frame.getLayerId(), frame.getFrameId()); - return DependSeq.newBuilder().addAllDepends(depends).build(); - } - - @Override - public DependSeq getDepends(JobInterface job) { - List depends = getJdbcTemplate().query( - GET_DEPEND + " WHERE pk_job_depend_er=? AND str_type != 'FRAME_ON_FRAME'", - DEPEND_MAPPER, job.getJobId()); - return DependSeq.newBuilder().addAllDepends(depends).build(); - } - - @Override - public Depend getDepend(String id) { - return getJdbcTemplate().queryForObject( - GET_DEPEND + " WHERE pk_depend=?",DEPEND_MAPPER,id); - } - - @Override - public Group findGroup(String show, String group) { - return getJdbcTemplate().queryForObject( - GET_GROUPS + " AND show.str_name=? AND folder.str_name=?", - GROUP_MAPPER, show, group); - } - - @Override - public Host findHost(String name) { - return getJdbcTemplate().queryForObject( - GET_HOST + " AND host.str_name=?", HOST_MAPPER, name); - } - - @Override - public HostSeq getHosts(HostSearchInterface r) { - List hosts = getJdbcTemplate().query(r.getFilteredQuery(GET_HOST), HOST_MAPPER, - r.getValuesArray()); - return HostSeq.newBuilder().addAllHosts(hosts).build(); - } - - @Override - public Host getHost(String id) { - return getJdbcTemplate().queryForObject( - GET_HOST + " AND host.pk_host=?", HOST_MAPPER, id); - } - - @Override - public ProcSeq getProcs(HostInterface host) { - ProcSearchInterface r = procSearchFactory.create(); - r.filterByHost(host); - r.sortByHostName(); - r.sortByDispatchedTime(); - return ProcSeq.newBuilder().addAllProcs(getProcs(r).getProcsList()).build(); - } - - @Override - public ProcSeq getProcs(ProcSearchInterface p) { - p.sortByHostName(); - p.sortByDispatchedTime(); - List procs = getJdbcTemplate().query(p.getFilteredQuery(GET_PROC), - PROC_MAPPER, p.getValuesArray()); - return ProcSeq.newBuilder().addAllProcs(procs).build(); - } - - @Override - public CommentSeq getComments(HostInterface h) { - List comments = getJdbcTemplate().query( - GET_HOST_COMMENTS, COMMENT_MAPPER, h.getHostId()); - return CommentSeq.newBuilder().addAllComments(comments).build(); - } - - @Override - public CommentSeq getComments(JobInterface j) { - List comments = getJdbcTemplate().query( - GET_JOB_COMMENTS, COMMENT_MAPPER, j.getJobId()); - return CommentSeq.newBuilder().addAllComments(comments).build(); - } - - @Override - public UpdatedFrameCheckResult getUpdatedFrames(JobInterface job, - List layers, int epochSeconds) { - - if ((System.currentTimeMillis() / 1000) - epochSeconds > 60) { - long timeDiff = System.currentTimeMillis() - epochSeconds; - throw new IllegalArgumentException("the last update timestamp cannot be over " + - "a minute off the current time, difference was: " + timeDiff); - } - - UpdatedFrameCheckResult.Builder resultBuilder = UpdatedFrameCheckResult.newBuilder(); - resultBuilder.setState(JobState.valueOf(getJdbcTemplate().queryForObject( - "SELECT str_state FROM job WHERE pk_job=?", String.class, job.getJobId()))); - - FrameSearchInterface r = frameSearchFactory.create(job); - r.filterByLayers(layers); - r.filterByChangeDate(epochSeconds); - r.setMaxResults(100); - - List updatedFrameList = getJdbcTemplate().query( - r.getFilteredQuery(GET_UPDATED_FRAME), UPDATED_FRAME_MAPPER, r.getValuesArray()); - resultBuilder.setUpdatedFrames(UpdatedFrameSeq.newBuilder().addAllUpdatedFrames(updatedFrameList).build()); - resultBuilder.setServerTime((int) (System.currentTimeMillis() / 1000) - 1); - - return resultBuilder.build(); - } - - @Override - public Department getDepartment(ShowInterface show, String name) { - return getJdbcTemplate().queryForObject( - GET_DEPARTMENT, DEPARTMENT_MAPPER, - show.getShowId(), name); - } - - @Override - public DepartmentSeq getDepartments ( - ShowInterface show) { - List departments = getJdbcTemplate().query( - GET_DEPARTMENTS, DEPARTMENT_MAPPER, - show.getShowId()); - return DepartmentSeq.newBuilder().addAllDepartments(departments).build(); - } - - @Override - public List getDepartmentNames() { - return getJdbcTemplate().query("SELECT str_name FROM dept ORDER BY str_name ASC", - new RowMapper() { - public String mapRow(ResultSet rs, int row) throws SQLException { - return SqlUtil.getString(rs, "str_name"); - } - }); - } - - @Override - public Task getTask(ShowInterface show, DepartmentInterface dept, String shot) { - return getJdbcTemplate().queryForObject( - GET_TASK + " AND point.pk_show=? AND point.pk_dept=? AND task.str_shot=?", - TASK_MAPPER, show.getShowId(), dept.getDepartmentId(), shot); - } - - @Override - public TaskSeq getTasks(ShowInterface show, DepartmentInterface dept) { - if (dept == null) { - return TaskSeq.newBuilder().addAllTasks( - getJdbcTemplate().query( - GET_TASK + " AND point.pk_show=? ORDER BY task.str_shot", - TASK_MAPPER, show.getShowId())).build(); - } else { - return TaskSeq.newBuilder().addAllTasks(getJdbcTemplate().query( - GET_TASK + " AND point.pk_show=? AND point.pk_dept=? ORDER BY task.str_shot", - TASK_MAPPER, show.getShowId(), dept.getDepartmentId())).build(); - } - } - - - @Override - public DeedSeq getDeeds(OwnerEntity owner) { - List deeds = getJdbcTemplate().query( - QUERY_FOR_DEED + " AND owner.pk_owner=?", - DEED_MAPPER, owner.getId()); - return DeedSeq.newBuilder().addAllDeeds(deeds).build(); - } - - @Override - public DeedSeq getDeeds(ShowInterface show) { - List deeds = getJdbcTemplate().query( - QUERY_FOR_DEED + " AND show.pk_show=?", - DEED_MAPPER, show.getId()); - return DeedSeq.newBuilder().addAllDeeds(deeds).build(); - } - - @Override - public Host getHost(DeedEntity deed) { - return getJdbcTemplate().queryForObject( - GET_HOST + " AND host.pk_host=?", - HOST_MAPPER, deed.id); - } - - @Override - public Deed getDeed(HostInterface host) { - return getJdbcTemplate().queryForObject( - QUERY_FOR_DEED + " AND host.pk_host=?", - DEED_MAPPER, host.getHostId()); - } - - @Override - public HostSeq getHosts(OwnerEntity owner) { - StringBuilder sb = new StringBuilder(4096); - String query = GET_HOST; - query = query.replace("FROM " , "FROM owner, deed,"); - sb.append(query); - sb.append("AND deed.pk_host = host.pk_host "); - sb.append("AND deed.pk_owner = owner.pk_owner "); - sb.append("AND owner.pk_owner = ?"); - - List hosts = getJdbcTemplate().query( - sb.toString(), HOST_MAPPER, owner.getId()); - return HostSeq.newBuilder().addAllHosts(hosts).build(); - } - - @Override - public Owner getOwner(DeedEntity deed) { - return getJdbcTemplate().queryForObject( - QUERY_FOR_OWNER + " AND " + - "pk_owner = (SELECT deed.pk_owner FROM deed " + - "WHERE pk_deed=?)", OWNER_MAPPER, deed.getId()); - } - - @Override - public Owner getOwner(HostInterface host) { - return getJdbcTemplate().queryForObject( - QUERY_FOR_OWNER + " AND " + - "pk_owner = (SELECT deed.pk_owner FROM deed " + - "WHERE pk_host=?)", OWNER_MAPPER, host.getHostId()); - } - - @Override - public List getOwners(ShowInterface show) { - return getJdbcTemplate().query( - QUERY_FOR_OWNER + " AND owner.pk_show=?", OWNER_MAPPER, - show.getShowId()); - } - - - @Override - public RenderPartition getRenderPartition(LocalHostAssignment l) { - return getJdbcTemplate().queryForObject(QUERY_FOR_RENDER_PART + - "WHERE host_local.pk_host_local = ?", - RENDER_PARTION_MAPPER, l.getId()); - } - - - @Override - public RenderPartitionSeq getRenderPartitions(HostInterface host) { - List partitions = getJdbcTemplate().query(QUERY_FOR_RENDER_PART + - "WHERE host_local.pk_host = ?", - RENDER_PARTION_MAPPER, host.getHostId()); - return RenderPartitionSeq.newBuilder().addAllRenderPartitions(partitions).build(); - } - - - @Override - public Owner getOwner(String name) { - return getJdbcTemplate().queryForObject( - QUERY_FOR_OWNER + " AND " + - "(" + - "owner.str_username = ? " + - "OR " + - "owner.pk_owner = ?" + - ")", OWNER_MAPPER, name, name); - } - - @Override - public Facility getFacility(String name) { - return getJdbcTemplate().queryForObject( - QUERY_FOR_FACILITY + - " WHERE facility.pk_facility = ? OR facility.str_name = ?", - FACILITY_MAPPER, name, name); - } - - @Override - public FacilitySeq getFacilities() { - return FacilitySeq.newBuilder().addAllFacilities(getJdbcTemplate().query( - QUERY_FOR_FACILITY, FACILITY_MAPPER)).build(); - } - - @Override - public Limit findLimit(String name) { - String findLimitQuery = QUERY_FOR_LIMIT + - " WHERE limit_record.str_name = ? " + - "GROUP BY " + - "limit_record.str_name, " + - "limit_record.pk_limit_record, " + - "limit_record.int_max_value"; - return getJdbcTemplate().queryForObject(findLimitQuery, LIMIT_MAPPER, name); - } - - @Override - public Limit getLimit(String id) { - String getLimitQuery = QUERY_FOR_LIMIT + - " WHERE limit_record.pk_limit_record = ? "+ - "GROUP BY " + - "limit_record.str_name, " + - "limit_record.pk_limit_record, " + - "limit_record.int_max_value"; - return getJdbcTemplate().queryForObject(getLimitQuery, LIMIT_MAPPER, id); - } - - @Override - public List getLimits() { - String getLimitsQuery = QUERY_FOR_LIMIT + - " GROUP BY " + - "limit_record.str_name, " + - "limit_record.pk_limit_record, " + - "limit_record.int_max_value"; - return getJdbcTemplate().query(getLimitsQuery, LIMIT_MAPPER); - } - - /* - * Row Mappers - */ - - public static final RowMapper LIMIT_MAPPER = - new RowMapper() { - public Limit mapRow(ResultSet rs, int rowNum) throws SQLException { - return Limit.newBuilder() - .setId(SqlUtil.getString(rs, "pk_limit_record")) - .setName(SqlUtil.getString(rs, "str_name")) - .setMaxValue(rs.getInt("int_max_value")) - .setCurrentRunning(rs.getInt("int_current_running")) - .build(); - } - }; - - public static final RowMapper MATCHER_MAPPER = - new RowMapper() { - public Matcher mapRow(ResultSet rs, int rowNum) throws SQLException { - return Matcher.newBuilder() - .setId(SqlUtil.getString(rs, "pk_matcher")) - .setInput(SqlUtil.getString(rs, "str_value")) - .setSubject(MatchSubject.valueOf(SqlUtil.getString(rs, "str_subject"))) - .setType(MatchType.valueOf(SqlUtil.getString(rs, "str_match"))) - .build(); - } - }; - - public static final RowMapper FILTER_MAPPER = - new RowMapper() { - public Filter mapRow(ResultSet rs, int rowNum) throws SQLException { - return Filter.newBuilder() - .setId(SqlUtil.getString(rs, "pk_filter")) - .setType(FilterType.valueOf(SqlUtil.getString(rs, "str_type"))) - .setOrder(rs.getFloat("f_order")) - .setName(SqlUtil.getString(rs, "str_name")) - .setEnabled(rs.getBoolean("b_enabled")) - .build(); - } - }; - - public static final RowMapper ACTION_MAPPER = - new RowMapper() { - public Action mapRow(ResultSet rs, int rowNum) throws SQLException { - Action.Builder builder = Action.newBuilder() - .setId(SqlUtil.getString(rs, "pk_action")) - .setBooleanValue(false) - .setIntegerValue(0) - .setFloatValue(0f) - .setStringValue("") - .setType(ActionType.valueOf(SqlUtil.getString(rs, "str_action"))) - .setValueType(ActionValueType.valueOf(SqlUtil.getString(rs, "str_value_type"))); - - switch (builder.getValueType()) { - case GROUP_TYPE: - builder.setGroupValue(SqlUtil.getString(rs, "pk_folder")); - break; - case STRING_TYPE: - builder.setStringValue(SqlUtil.getString(rs, "str_value")); - break; - case INTEGER_TYPE: - builder.setIntegerValue(rs.getInt("int_value")); - break; - case FLOAT_TYPE: - builder.setFloatValue(rs.getFloat("float_value")); - break; - case BOOLEAN_TYPE: - builder.setBooleanValue(rs.getBoolean("b_value")); - break; - } - return builder.build(); - } - }; - - public static final RowMapper FACILITY_MAPPER = - new RowMapper() { - public Facility mapRow(ResultSet rs, int rowNum) throws SQLException { - return Facility.newBuilder() - .setName(rs.getString("str_name")) - .setId(rs.getString("pk_facility")) - .build(); - } - }; - - - public static final RowMapper DEED_MAPPER = - new RowMapper() { - public Deed mapRow(ResultSet rs, int rowNum) throws SQLException { - return Deed.newBuilder() - .setId(SqlUtil.getString(rs, "pk_deed")) - .setHost(SqlUtil.getString(rs, "str_host")) - .setOwner(SqlUtil.getString(rs, "str_username")) - .build(); - } - }; - - public static final RowMapper - RENDER_PARTION_MAPPER = new RowMapper() { - public RenderPartition mapRow(ResultSet rs, int rowNum) throws SQLException { - - RenderPartition.Builder builder = RenderPartition.newBuilder() - .setId(SqlUtil.getString(rs, "pk_host_local")) - .setCores(rs.getInt("int_cores_max") - rs.getInt("int_cores_idle")) - .setMaxCores(rs.getInt("int_cores_max")) - .setThreads(rs.getInt("int_threads")) - .setMaxMemory(rs.getLong("int_mem_max")) - .setMemory( rs.getLong("int_mem_max") - rs.getLong("int_mem_idle")) - .setMaxGpu(rs.getLong("int_gpu_max")) - .setHost(SqlUtil.getString(rs, "str_host_name")) - .setJob(SqlUtil.getString(rs, "str_job_name")) - .setRenderPartType(RenderPartitionType.valueOf(SqlUtil.getString(rs, "str_type"))) - .setLayer("") - .setFrame(""); - - if (SqlUtil.getString(rs, "str_layer_name") != null) { - builder.setLayer(SqlUtil.getString(rs, "str_layer_name")); - } - - if (SqlUtil.getString(rs, "str_frame_name") != null) { - builder.setFrame(SqlUtil.getString(rs, "str_frame_name")); - } - - return builder.build(); - - } - }; - - public static final RowMapper - OWNER_MAPPER = new RowMapper() { - public Owner mapRow(ResultSet rs, int rowNum) throws SQLException { - return Owner.newBuilder() - .setName(SqlUtil.getString(rs, "str_username")) - .setId(SqlUtil.getString(rs, "pk_owner")) - .setShow(SqlUtil.getString(rs, "str_show")) - .setHostCount(rs.getInt("host_count")) - .build(); - } - }; - - public static final RowMapper DEPARTMENT_MAPPER = - new RowMapper() { - public Department mapRow(ResultSet rs, int row) throws SQLException { - return Department.newBuilder() - .setId(SqlUtil.getString(rs, "pk_point")) - .setName(SqlUtil.getString(rs, "str_name")) - .setDept(SqlUtil.getString(rs, "str_dept")) - .setTiManaged(rs.getBoolean("b_managed")) - .setTiTask(SqlUtil.getString(rs, "str_ti_task")) - .setMinCores(Convert.coreUnitsToCores(rs.getInt("int_min_cores"))) - .build(); - } - }; - - public static final RowMapper PROC_MAPPER = - new RowMapper() { - public Proc mapRow(ResultSet rs, int row) throws SQLException { - return Proc.newBuilder() - .setId(SqlUtil.getString(rs, "pk_proc")) - .setName(CueUtil.buildProcName(SqlUtil.getString(rs, "host_name"), - rs.getInt("int_cores_reserved"))) - .setReservedCores(Convert.coreUnitsToCores(rs.getInt("int_cores_reserved"))) - .setReservedMemory(rs.getLong("int_mem_reserved")) - .setReservedGpu(rs.getLong("int_gpu_reserved")) - .setUsedMemory(rs.getLong("int_mem_used")) - .setFrameName(SqlUtil.getString(rs, "frame_name")) - .setJobName(SqlUtil.getString(rs, "job_name")) - .setGroupName(SqlUtil.getString(rs, "folder_name")) - .setShowName(SqlUtil.getString(rs, "show_name")) - .setPingTime((int) (rs.getTimestamp("ts_ping").getTime() / 1000)) - .setBookedTime((int) (rs.getTimestamp("ts_booked").getTime() / 1000)) - .setDispatchTime((int) (rs.getTimestamp("ts_dispatched").getTime() / 1000)) - .setUnbooked(rs.getBoolean("b_unbooked")) - .setLogPath(String.format("%s/%s.%s.rqlog", - SqlUtil.getString(rs, "str_log_dir"), SqlUtil.getString(rs, "job_name"), - SqlUtil.getString(rs, "frame_name"))) - .setRedirectTarget(SqlUtil.getString(rs, "str_redirect")) - .addAllServices(Arrays.asList(SqlUtil.getString(rs, "str_services").split(","))) - .build(); - } - }; - - public static final RowMapper TASK_MAPPER = - new RowMapper() { - public Task mapRow(ResultSet rs, int row) throws SQLException { - return Task.newBuilder() - .setId(SqlUtil.getString(rs, "pk_task")) - .setDept(SqlUtil.getString(rs, "str_dept")) - .setShot(SqlUtil.getString(rs, "str_shot")) - .setMinCores(Convert.coreUnitsToWholeCores(rs.getInt("int_min_cores"))) - .setAdjustCores(Convert.coreUnitsToWholeCores(rs.getInt("int_adjust_cores"))) - .build(); - } - }; - - public static final RowMapper COMMENT_MAPPER = - new RowMapper() { - - public Comment mapRow(ResultSet rs, int row) throws SQLException { - return Comment.newBuilder() - .setId(SqlUtil.getString(rs, "pk_comment")) - .setMessage(SqlUtil.getString(rs, "str_message")) - .setSubject(SqlUtil.getString(rs, "str_subject")) - .setTimestamp((int)(rs.getTimestamp("ts_created").getTime() / 1000)) - .setUser(SqlUtil.getString(rs, "str_user")) - .build(); - } - }; - - public static NestedHost.Builder mapNestedHostBuilder(ResultSet rs) throws SQLException { - NestedHost.Builder builder = NestedHost.newBuilder() - .setId(SqlUtil.getString(rs, "pk_host")) - .setName(SqlUtil.getString(rs, "host_name")) - .setAllocName(SqlUtil.getString(rs, "alloc_name")) - .setBootTime((int) (rs.getTimestamp("ts_booted").getTime() / 1000)) - .setFreeMcp(rs.getLong("int_mcp_free")) - .setFreeMemory(rs.getLong("int_mem_free")) - .setFreeSwap(rs.getLong("int_swap_free")) - .setFreeGpu(rs.getLong("int_gpu_free")) - .setLoad(rs.getInt("int_load")) - .setNimbyEnabled(rs.getBoolean("b_nimby")) - .setCores(Convert.coreUnitsToCores(rs.getInt("int_cores"))) - .setIdleCores(Convert.coreUnitsToCores(rs.getInt("int_cores_idle"))) - .setMemory(rs.getLong("int_mem")) - .setIdleMemory(rs.getLong("int_mem_idle")) - .setGpu(rs.getLong("int_gpu")) - .setIdleGpu(rs.getLong("int_gpu_idle")) - .setState(HardwareState.valueOf(SqlUtil.getString(rs, "host_state"))) - .setTotalMcp(rs.getLong("int_mcp_total")) - .setTotalMemory(rs.getLong("int_mem_total")) - .setTotalSwap(rs.getLong("int_swap_total")) - .setTotalGpu(rs.getLong("int_gpu_total")) - .setPingTime((int) (rs.getTimestamp("ts_ping").getTime() / 1000)) - .setLockState(LockState.valueOf(SqlUtil.getString(rs, "str_lock_state"))) - .setHasComment(rs.getBoolean("b_comment")) - .setThreadMode(ThreadMode.values()[rs.getInt("int_thread_mode")]) - .setOs(SqlUtil.getString(rs, "str_os")); - - String tags = SqlUtil.getString(rs, "str_tags"); - if (tags != null) - builder.addAllTags(Arrays.asList(tags.split(" "))); - return builder; - } - - public static Host.Builder mapHostBuilder(ResultSet rs) throws SQLException { - Host.Builder builder = Host.newBuilder(); - builder.setId(SqlUtil.getString(rs, "pk_host")); - builder.setName(SqlUtil.getString(rs, "host_name")); - builder.setAllocName(SqlUtil.getString(rs, "alloc_name")); - builder.setBootTime((int) (rs.getTimestamp("ts_booted").getTime() / 1000)); - builder.setFreeMcp(rs.getLong("int_mcp_free")); - builder.setFreeMemory(rs.getLong("int_mem_free")); - builder.setFreeSwap(rs.getLong("int_swap_free")); - builder.setFreeGpu(rs.getLong("int_gpu_free")); - builder.setLoad(rs.getInt("int_load")); - builder.setNimbyEnabled(rs.getBoolean("b_nimby")); - builder.setCores(Convert.coreUnitsToCores(rs.getInt("int_cores"))); - builder.setIdleCores(Convert.coreUnitsToCores(rs.getInt("int_cores_idle"))); - builder.setMemory(rs.getLong("int_mem")); - builder.setIdleMemory(rs.getLong("int_mem_idle")); - builder.setGpu(rs.getLong("int_gpu")); - builder.setIdleGpu(rs.getLong("int_gpu_idle")); - builder.setState(HardwareState.valueOf(SqlUtil.getString(rs, "host_state"))); - builder.setTotalMcp(rs.getLong("int_mcp_total")); - builder.setTotalMemory(rs.getLong("int_mem_total")); - builder.setTotalSwap(rs.getLong("int_swap_total")); - builder.setTotalGpu(rs.getLong("int_gpu_total")); - builder.setPingTime((int) (rs.getTimestamp("ts_ping").getTime() / 1000)); - builder.setLockState(LockState.valueOf(SqlUtil.getString(rs, "str_lock_state"))); - builder.setHasComment(rs.getBoolean("b_comment")); - builder.setThreadMode(ThreadMode.values()[rs.getInt("int_thread_mode")]); - builder.setOs(SqlUtil.getString(rs, "str_os")); - - String tags = SqlUtil.getString(rs, "str_tags"); - if (tags != null) - builder.addAllTags(Arrays.asList(tags.split(" "))); - return builder; - } - - public static final RowMapper HOST_MAPPER = - new RowMapper() { - public Host mapRow(ResultSet rs, int row) throws SQLException { - Host.Builder builder = mapHostBuilder(rs); - return builder.build(); - } - }; - - public static final RowMapper DEPEND_MAPPER = - new RowMapper() { - public Depend mapRow(ResultSet rs, int rowNum) throws SQLException { - return Depend.newBuilder() - .setId(SqlUtil.getString(rs, "pk_depend")) - .setActive(rs.getBoolean("b_active")) - .setAnyFrame(rs.getBoolean("b_any")) - .setDependErFrame(SqlUtil.getString(rs, "depend_er_frame")) - .setDependErLayer(SqlUtil.getString(rs, "depend_er_layer")) - .setDependErJob(SqlUtil.getString(rs, "depend_er_job")) - .setDependOnFrame(SqlUtil.getString(rs, "depend_on_frame")) - .setDependOnLayer(SqlUtil.getString(rs, "depend_on_layer")) - .setDependOnJob(SqlUtil.getString(rs, "depend_on_job")) - .setType(DependType.valueOf(SqlUtil.getString(rs, "str_type"))) - .setTarget(DependTarget.valueOf(SqlUtil.getString(rs, "str_target"))) - .build(); - } - }; - - public static final RowMapper ALLOCATION_MAPPER = - new RowMapper() { - public Allocation mapRow(ResultSet rs, int rowNum) throws SQLException { - return Allocation.newBuilder() - .setId(rs.getString("pk_alloc")) - .setName(SqlUtil.getString(rs, "str_name")) - .setFacility(SqlUtil.getString(rs, "facility_name")) - .setTag(SqlUtil.getString(rs, "str_tag")) - .setBillable(rs.getBoolean("b_billable")) - .setStats(AllocationStats.newBuilder() - .setCores(Convert.coreUnitsToCores(rs.getInt("int_cores"))) - .setAvailableCores(Convert.coreUnitsToCores(rs.getInt("int_available_cores"))) - .setIdleCores(Convert.coreUnitsToCores(rs.getInt("int_idle_cores"))) - .setRunningCores(Convert.coreUnitsToCores(rs.getInt("int_running_cores"))) - .setLockedCores(Convert.coreUnitsToCores(rs.getInt("int_locked_cores"))) - .setHosts(rs.getInt("int_hosts")) - .setDownHosts(rs.getInt("int_down_hosts")) - .setLockedHosts(rs.getInt("int_locked_hosts")) - .build()) - .build(); - } - }; - - private static final RowMapper GROUP_MAPPER = - new RowMapper() { - - public Group mapRow(ResultSet rs, int rowNum) throws SQLException { - GroupStats stats = GroupStats.newBuilder() - .setDeadFrames(rs.getInt("int_dead_count")) - .setRunningFrames(rs.getInt("int_running_count")) - .setWaitingFrames(rs.getInt("int_waiting_count")) - .setDependFrames(rs.getInt("int_depend_count")) - .setPendingJobs(rs.getInt("int_job_count")) - .setReservedCores(Convert.coreUnitsToCores(rs.getInt("int_cores"))) - .build(); - return Group.newBuilder() - .setId(SqlUtil.getString(rs, "pk_folder")) - .setName(SqlUtil.getString(rs, "group_name")) - .setDepartment(SqlUtil.getString(rs, "str_dept")) - .setDefaultJobPriority(rs.getInt("int_job_priority")) - .setDefaultJobMinCores(Convert.coreUnitsToCores(rs.getInt("int_job_min_cores"))) - .setDefaultJobMaxCores(Convert.coreUnitsToCores(rs.getInt("int_job_max_cores"))) - .setMaxCores(Convert.coreUnitsToCores(rs.getInt("int_max_cores"))) - .setMinCores(Convert.coreUnitsToCores(rs.getInt("int_min_cores"))) - .setLevel(rs.getInt("int_level")) - .setParentId(SqlUtil.getString(rs, "pk_parent_folder")) - .setGroupStats(stats) - .build(); - } - }; - - public static final RowMapper JOB_MAPPER = - new RowMapper() { - public Job mapRow(ResultSet rs, int rowNum) throws SQLException { - Job.Builder jobBuilder = Job.newBuilder() - .setLogDir(SqlUtil.getString(rs, "str_log_dir")) - .setMaxCores(Convert.coreUnitsToCores(rs.getInt("int_max_cores"))) - .setMinCores(Convert.coreUnitsToCores(rs.getInt("int_min_cores"))) - .setName(SqlUtil.getString(rs, "str_name")) - .setPriority(rs.getInt("int_priority")) - .setShot(SqlUtil.getString(rs, "str_shot")) - .setShow(SqlUtil.getString(rs, "str_show")) - .setFacility(SqlUtil.getString(rs, "facility_name")) - .setGroup(SqlUtil.getString(rs, "group_name")) - .setState(JobState.valueOf(SqlUtil.getString(rs, "str_state"))) - .setUser(SqlUtil.getString(rs, "str_user")) - .setIsPaused(rs.getBoolean("b_paused")) - .setHasComment(rs.getBoolean("b_comment")) - .setAutoEat(rs.getBoolean("b_autoeat")) - .setStartTime((int) (rs.getTimestamp("ts_started").getTime() / 1000)) - .setOs(SqlUtil.getString(rs, "str_os")); - - int uid = rs.getInt("int_uid"); - if (!rs.wasNull()) { - jobBuilder.setUid(uid); - } - - Timestamp ts = rs.getTimestamp("ts_stopped"); - if (ts != null) { - jobBuilder.setStopTime((int) (ts.getTime() / 1000)); - } - else { - jobBuilder.setStopTime(0); - } - - jobBuilder.setJobStats(mapJobStats(rs)); - return jobBuilder.build(); - } - }; - - public static JobStats mapJobStats(ResultSet rs) throws SQLException { - - JobStats.Builder statsBuilder = JobStats.newBuilder() - .setReservedCores(Convert.coreUnitsToCores(rs.getInt("int_cores"))) - .setMaxRss(rs.getLong("int_max_rss")) - .setTotalFrames(rs.getInt("int_frame_count")) - .setTotalLayers(rs.getInt("int_layer_count")) - .setWaitingFrames(rs.getInt("int_waiting_count")) - .setRunningFrames(rs.getInt("int_running_count")) - .setDeadFrames(rs.getInt("int_dead_count")) - .setSucceededFrames(rs.getInt("int_succeeded_count")) - .setEatenFrames(rs.getInt("int_eaten_count")) - .setDependFrames(rs.getInt("int_depend_count")) - .setPendingFrames(rs.getInt("int_waiting_count") + rs.getInt("int_depend_count")) - .setFailedCoreSec(rs.getLong("int_core_time_fail")) - .setRenderedCoreSec(rs.getLong("int_core_time_success")) - .setTotalCoreSec( rs.getLong("int_core_time_fail") + rs.getLong("int_core_time_success")) - .setRenderedFrameCount( rs.getLong("int_frame_success_count")) - .setFailedFrameCount(rs.getLong("int_frame_fail_count")) - .setHighFrameSec(rs.getInt("int_clock_time_high")); - - if (statsBuilder.getRenderedFrameCount() > 0) { - statsBuilder.setAvgCoreSec( - (int) (rs.getLong("int_clock_time_success") / statsBuilder.getRenderedFrameCount())); - statsBuilder.setAvgCoreSec( - (int) (statsBuilder.getRenderedCoreSec() / statsBuilder.getRenderedFrameCount())); - statsBuilder.setRemainingCoreSec( - (long) statsBuilder.getPendingFrames() * statsBuilder.getAvgCoreSec()); - } - else { - statsBuilder.setAvgFrameSec(0); - statsBuilder.setAvgCoreSec(0); - statsBuilder.setRemainingCoreSec(0); - } - return statsBuilder.build(); - } - - public static final RowMapper LAYER_MAPPER = - new RowMapper() { - public Layer mapRow(ResultSet rs, int rowNum) throws SQLException { - Layer.Builder builder = Layer.newBuilder() - .setId(SqlUtil.getString(rs, "pk_layer")) - .setParentId(SqlUtil.getString(rs, "pk_job")) - .setChunkSize(rs.getInt("int_chunk_size")) - .setDispatchOrder(rs.getInt("int_dispatch_order")) - .setName(SqlUtil.getString(rs, "str_name")) - .setRange(SqlUtil.getString(rs, "str_range")) - .setMinCores(Convert.coreUnitsToCores(rs.getInt("int_cores_min"))) - .setMaxCores(Convert.coreUnitsToCores(rs.getInt("int_cores_max"))) - .setIsThreadable(rs.getBoolean("b_threadable")) - .setMinMemory(rs.getLong("int_mem_min")) - .setMinGpu(rs.getLong("int_gpu_min")) - .setType(LayerType.valueOf(SqlUtil.getString(rs, "str_type"))) - .addAllTags(Sets.newHashSet( - SqlUtil.getString(rs, "str_tags"). - replaceAll(" ","").split("\\|"))) - .addAllServices(Arrays.asList(SqlUtil.getString(rs, "str_services").split(","))) - .addAllLimits(Arrays.asList(SqlUtil.getString(rs,"str_limit_names").split(","))) - .setMemoryOptimizerEnabled(rs.getBoolean("b_optimize")); - - LayerStats.Builder statsBuilder = LayerStats.newBuilder() - .setReservedCores(Convert.coreUnitsToCores(rs.getInt("int_cores"))) - .setMaxRss(rs.getLong("int_max_rss")) - .setTotalFrames(rs.getInt("int_total_count")) - .setWaitingFrames(rs.getInt("int_waiting_count")) - .setRunningFrames(rs.getInt("int_running_count")) - .setDeadFrames(rs.getInt("int_dead_count")) - .setSucceededFrames(rs.getInt("int_succeeded_count")) - .setEatenFrames(rs.getInt("int_eaten_count")) - .setDependFrames(rs.getInt("int_depend_count")) - .setPendingFrames( - rs.getInt("int_waiting_count") + rs.getInt("int_depend_count")) - .setFailedCoreSec(rs.getLong("int_core_time_fail")) - .setRenderedCoreSec(rs.getLong("int_core_time_success")) - .setTotalCoreSec( - rs.getLong("int_core_time_fail") + rs.getLong("int_core_time_success")) - .setRenderedFrameCount( rs.getLong("int_frame_success_count")) - .setFailedFrameCount(rs.getLong("int_frame_fail_count")) - .setHighFrameSec(rs.getInt("int_clock_time_high")) - .setLowFrameSec(rs.getInt("int_clock_time_low")); - - if (statsBuilder.getRenderedFrameCount() > 0) { - statsBuilder.setAvgFrameSec( - (int) (rs.getLong("int_clock_time_success") / statsBuilder.getRenderedFrameCount())); - statsBuilder.setAvgCoreSec( - (int) (statsBuilder.getRenderedCoreSec() / statsBuilder.getRenderedFrameCount())); - statsBuilder.setRemainingCoreSec( - (long) statsBuilder.getPendingFrames() * statsBuilder.getAvgCoreSec()); - } - else { - statsBuilder.setAvgFrameSec(0); - statsBuilder.setAvgCoreSec(0); - statsBuilder.setRemainingCoreSec(0); - } - builder.setLayerStats(statsBuilder.build()); - return builder.build(); - } - }; - - private static final RowMapper LIMIT_NAME_MAPPER = - new RowMapper() { - public String mapRow(ResultSet rs, int rowNum) throws SQLException { - return rs.getString("str_name"); - } - }; - - public static final RowMapper SUBSCRIPTION_MAPPER = - new RowMapper() { - public Subscription mapRow(ResultSet rs, int rowNum) throws SQLException { - return Subscription.newBuilder() - .setId(SqlUtil.getString(rs, "pk_subscription")) - .setBurst(rs.getInt("int_burst")) - .setName(SqlUtil.getString(rs, "name")) - .setReservedCores(rs.getInt("int_cores")) - .setSize(rs.getInt("int_size")) - .setAllocationName(SqlUtil.getString(rs, "alloc_name")) - .setShowName(SqlUtil.getString(rs, "show_name")) - .setFacility(SqlUtil.getString(rs, "facility_name")) - .build(); - } - }; - - public static final RowMapper UPDATED_FRAME_MAPPER = - new RowMapper() { - public UpdatedFrame mapRow(ResultSet rs, int rowNum) throws SQLException { - UpdatedFrame.Builder builder = UpdatedFrame.newBuilder() - .setId(SqlUtil.getString(rs, "pk_frame")) - .setExitStatus(rs.getInt("int_exit_status")) - .setMaxRss(rs.getInt("int_mem_max_used")) - .setRetryCount(rs.getInt("int_retries")) - .setState(FrameState.valueOf(SqlUtil.getString(rs, "str_state"))) - .setUsedMemory(rs.getInt("int_mem_used")); - - if (SqlUtil.getString(rs, "str_host") != null) { - builder.setLastResource(String.format(Locale.ROOT, "%s/%2.2f", - SqlUtil.getString(rs, "str_host"), - Convert.coreUnitsToCores(rs.getInt("int_cores")))); - }else { - builder.setLastResource(""); - } - - java.sql.Timestamp ts_started = rs.getTimestamp("ts_started"); - if (ts_started != null) { - builder.setStartTime((int) (rs.getTimestamp("ts_started").getTime() / 1000)); - } - else { - builder.setStartTime(0); - } - java.sql.Timestamp ts_stopped = rs.getTimestamp("ts_stopped"); - if (ts_stopped!= null) { - builder.setStopTime((int) (ts_stopped.getTime() / 1000)); - } - else { - builder.setStopTime(0); - } - - return builder.build(); - } - }; - - public static final RowMapper FRAME_MAPPER = - new RowMapper() { - public Frame mapRow(ResultSet rs, int rowNum) throws SQLException { - Frame.Builder builder = Frame.newBuilder() - .setId(SqlUtil.getString(rs, "pk_frame")) - .setName(SqlUtil.getString(rs, "str_name")) - .setExitStatus(rs.getInt("int_exit_status")) - .setMaxRss(rs.getLong("int_mem_max_used")) - .setNumber(rs.getInt("int_number")) - .setDispatchOrder(rs.getInt("int_dispatch_order")) - .setRetryCount(rs.getInt("int_retries")) - .setState(FrameState.valueOf(SqlUtil.getString(rs, "str_state"))) - .setLayerName(SqlUtil.getString(rs, "layer_name")) - .setUsedMemory(rs.getLong("int_mem_used")) - .setReservedMemory(rs.getLong("int_mem_reserved")) - .setReservedGpu(rs.getLong("int_gpu_reserved")) - .setCheckpointState(CheckpointState.valueOf( - SqlUtil.getString(rs, "str_checkpoint_state"))) - .setCheckpointCount(rs.getInt("int_checkpoint_count")); - - if (SqlUtil.getString(rs, "str_host") != null) { - builder.setLastResource(CueUtil.buildProcName(SqlUtil.getString(rs, "str_host"), - rs.getInt("int_cores"))); - } else { - builder.setLastResource(""); - } - - java.sql.Timestamp ts_started = rs.getTimestamp("ts_started"); - if (ts_started != null) { - builder.setStartTime((int) (rs.getTimestamp("ts_started").getTime() / 1000)); - } - else { - builder.setStartTime(0); - } - java.sql.Timestamp ts_stopped = rs.getTimestamp("ts_stopped"); - if (ts_stopped!= null) { - builder.setStopTime((int) (ts_stopped.getTime() / 1000)); - } - else { - builder.setStopTime(0); - } - - builder.setTotalCoreTime(rs.getInt("int_total_past_core_time")); - if (builder.getState() == FrameState.RUNNING) { - builder.setTotalCoreTime(builder.getTotalCoreTime() + - (int)(System.currentTimeMillis() / 1000 - builder.getStartTime()) * rs.getInt("int_cores") / 100); - } - return builder.build(); - } - }; - - private static final RowMapper SERVICE_MAPPER = - new RowMapper() { - public Service mapRow(ResultSet rs, int rowNum) throws SQLException { - return Service.newBuilder() - .setId(SqlUtil.getString(rs, "pk_service")) - .setName(SqlUtil.getString(rs, "str_name")) - .setThreadable(rs.getBoolean("b_threadable")) - .setMinCores(rs.getInt("int_cores_min")) - .setMaxCores(rs.getInt("int_cores_max")) - .setMinMemory(rs.getInt("int_mem_min")) - .setMinGpu(rs.getInt("int_gpu_min")) - .addAllTags(Lists.newArrayList(ServiceDaoJdbc.splitTags( - SqlUtil.getString(rs, "str_tags")))) - .build(); - } - }; - - private static final RowMapper SERVICE_OVERRIDE_MAPPER = - new RowMapper() { - public ServiceOverride mapRow(ResultSet rs, int rowNum) throws SQLException { - Service data = Service.newBuilder() - .setId(SqlUtil.getString(rs, "pk_show_service")) - .setName(SqlUtil.getString(rs, "str_name")) - .setThreadable(rs.getBoolean("b_threadable")) - .setMinCores(rs.getInt("int_cores_min")) - .setMaxCores(rs.getInt("int_cores_max")) - .setMinMemory(rs.getInt("int_mem_min")) - .setMinGpu(rs.getInt("int_gpu_min")) - .addAllTags(Lists.newArrayList(ServiceDaoJdbc.splitTags( - SqlUtil.getString(rs, "str_tags")))) - .build(); - return ServiceOverride.newBuilder() - .setId(SqlUtil.getString(rs, "pk_show_service")) - .setData(data) - .build(); - } - }; - - public static final RowMapper SHOW_MAPPER = - new RowMapper() { - public Show mapRow(ResultSet rs, int rowNum) throws SQLException { - ShowStats stats = ShowStats.newBuilder() - .setPendingFrames(rs.getInt("int_pending_count")) - .setRunningFrames(rs.getInt("int_running_count")) - .setDeadFrames(rs.getInt("int_dead_count")) - .setCreatedFrameCount(rs.getLong("int_frame_insert_count")) - .setCreatedJobCount(rs.getLong("int_job_insert_count")) - .setRenderedFrameCount(rs.getLong("int_frame_success_count")) - .setFailedFrameCount(rs.getLong("int_frame_fail_count")) - .setReservedCores(Convert.coreUnitsToCores(rs.getInt("int_cores"))) - .setPendingJobs(rs.getInt("int_job_count")) - .build(); - return Show.newBuilder() - .setId(SqlUtil.getString(rs, "pk_show")) - .setName(SqlUtil.getString(rs, "str_name")) - .setActive(rs.getBoolean("b_active")) - .setDefaultMaxCores(Convert.coreUnitsToCores(rs.getInt("int_default_max_cores"))) - .setDefaultMinCores(Convert.coreUnitsToCores(rs.getInt("int_default_min_cores"))) - .setBookingEnabled(rs.getBoolean("b_booking_enabled")) - .setDispatchEnabled(rs.getBoolean("b_dispatch_enabled")) - .setCommentEmail(SqlUtil.getString(rs, "str_comment_email")) - .setShowStats(stats) - .build(); - } - }; - /* - * Queries - */ - - private static final String GET_JOB_NAMES = - "SELECT " + - "job.str_name "+ - "FROM " + - "job," + - "show " + - "WHERE " + - "job.pk_show = show.pk_show " + - "AND " + - "job.str_state = 'PENDING' "; - - private static final String GET_HOST_COMMENTS = - "SELECT " + - "* " + - "FROM " + - "comments " + - "WHERE " + - "pk_host=? " + - "ORDER BY " + - "ts_created ASC"; - - private static final String GET_FILTER = - "SELECT " + - "filter.* " + - "FROM " + - "filter," + - "show " + - "WHERE " + - "filter.pk_show = show.pk_show"; - - private static final String GET_FRAME = - "SELECT " + - "frame.pk_frame, " + - "frame.int_exit_status,"+ - "frame.str_name,"+ - "frame.int_number,"+ - "frame.int_dispatch_order,"+ - "frame.ts_started,"+ - "frame.ts_stopped,"+ - "frame.int_retries,"+ - "frame.str_state,"+ - "frame.str_host,"+ - "frame.int_cores,"+ - "frame.int_mem_max_used," + - "frame.int_mem_used, " + - "frame.int_mem_reserved, " + - "frame.int_gpu_reserved, " + - "frame.str_checkpoint_state,"+ - "frame.int_checkpoint_count,"+ - "frame.int_total_past_core_time,"+ - "layer.str_name AS layer_name," + - "job.str_name AS job_name "+ - "FROM "+ - "job, " + - "layer, "+ - "frame " + - "WHERE " + - "frame.pk_layer = layer.pk_layer "+ - "AND "+ - "frame.pk_job= job.pk_job"; - - private static final String FIND_FRAME = GET_FRAME + " " + - "AND " + - "job.str_state='PENDING' " + - "AND " + - "job.str_name=? " + - "AND " + - "layer.str_name=? " + - "AND " + - "frame.int_number=?"; - - private static final String GET_PROC = - "SELECT " + - "host.str_name AS host_name,"+ - "job.str_name AS job_name,"+ - "job.str_log_dir,"+ - "folder.str_name as folder_name, "+ - "show.str_name AS show_name,"+ - "frame.str_name AS frame_name,"+ - "layer.str_services,"+ - "proc.pk_proc,"+ - "proc.pk_host,"+ - "proc.int_cores_reserved,"+ - "proc.int_mem_reserved, "+ - "proc.int_mem_used,"+ - "proc.int_mem_max_used,"+ - "proc.int_gpu_reserved, "+ - "proc.ts_ping,"+ - "proc.ts_booked,"+ - "proc.ts_dispatched,"+ - "proc.b_unbooked,"+ - "redirect.str_name AS str_redirect "+ - "FROM " + - "proc,"+ - "host, "+ - "alloc,"+ - "frame, "+ - "layer,"+ - "job,"+ - "folder, "+ - "show, " + - "redirect "+ - "WHERE " + - "proc.pk_host = host.pk_host " + - "AND " + - "host.pk_alloc = alloc.pk_alloc " + - "AND " + - "proc.pk_frame = frame.pk_frame " + - "AND " + - "proc.pk_layer = layer.pk_layer "+ - "AND " + - "proc.pk_job = job.pk_job " + - "AND " + - "job.pk_folder = folder.pk_folder " + - "AND " + - "proc.pk_show = show.pk_show " + - "AND " + - "proc.pk_proc = redirect.pk_proc (+) "; - - - private static final String GET_JOB_COMMENTS = - "SELECT " + - "* " + - "FROM " + - "comments " + - "WHERE " + - "pk_job=? " + - "ORDER BY " + - "ts_created ASC"; - - private static final String GET_UPDATED_FRAME = - "SELECT " + - "frame.pk_frame, " + - "frame.int_exit_status,"+ - "frame.ts_started,"+ - "frame.ts_stopped,"+ - "frame.int_retries,"+ - "frame.str_state,"+ - "frame.str_host,"+ - "frame.int_cores,"+ - "NVL(proc.int_mem_max_used, frame.int_mem_max_used) AS int_mem_max_used," + - "NVL(proc.int_mem_used, frame.int_mem_used) AS int_mem_used " + - "FROM "+ - "job, " + - "layer,"+ - "frame LEFT JOIN proc ON (proc.pk_frame = frame.pk_frame) " + - "WHERE " + - "frame.pk_layer = layer.pk_layer "+ - "AND "+ - "frame.pk_job= job.pk_job"; - - private static final String GET_ALLOCATION = - "SELECT " + - "alloc.pk_alloc, " + - "alloc.str_name, " + - "alloc.str_tag, " + - "alloc.b_billable,"+ - "facility.str_name AS facility_name,"+ - "vs_alloc_usage.int_cores,"+ - "vs_alloc_usage.int_idle_cores,"+ - "vs_alloc_usage.int_running_cores,"+ - "vs_alloc_usage.int_available_cores,"+ - "vs_alloc_usage.int_locked_cores,"+ - "vs_alloc_usage.int_hosts,"+ - "vs_alloc_usage.int_locked_hosts,"+ - "vs_alloc_usage.int_down_hosts "+ - "FROM " + - "alloc, " + - "facility, " + - "vs_alloc_usage " + - "WHERE " + - "alloc.pk_alloc = vs_alloc_usage.pk_alloc " + - "AND " + - "alloc.pk_facility = facility.pk_facility " + - "AND " + - "alloc.b_enabled = 1"; - - - private static final String GET_MATCHER = - "SELECT " + - "filter.pk_show," + - "matcher.* " + - "FROM " + - "filter,"+ - "matcher " + - "WHERE " + - "filter.pk_filter = matcher.pk_filter"; - - private static final String GET_DEPARTMENT = - "SELECT " + - "dept.str_name AS str_dept," + - "show.str_name || '.' || dept.str_name AS str_name, " + - "pk_point,"+ - "str_ti_task,"+ - "int_cores,"+ - "int_min_cores,"+ - "b_managed " + - "FROM " + - "point," + - "dept,"+ - "show " + - "WHERE " + - "point.pk_show = show.pk_show " + - "AND " + - "point.pk_dept = dept.pk_dept " + - "AND " + - "point.pk_show = ? " + - "AND " + - "dept.str_name = ?"; - - private static final String GET_DEPARTMENTS = - "SELECT " + - "dept.str_name AS str_dept," + - "show.str_name || '.' || dept.str_name AS str_name, " + - "pk_point,"+ - "str_ti_task,"+ - "int_cores,"+ - "int_min_cores,"+ - "b_managed " + - "FROM " + - "point," + - "dept,"+ - "show " + - "WHERE " + - "point.pk_show = show.pk_show " + - "AND " + - "point.pk_dept = dept.pk_dept " + - "AND " + - "point.pk_show = ? "; - - private static final String QUERY_FOR_OWNER = - "SELECT " + - "owner.pk_owner," + - "owner.str_username,"+ - "show.str_name AS str_show, " + - "(SELECT COUNT(1) FROM deed WHERE deed.pk_owner = owner.pk_owner) " + - " AS host_count " + - "FROM " + - "owner, " + - "show " + - "WHERE " + - "owner.pk_show = show.pk_show"; - - private static final String QUERY_FOR_RENDER_PART = - "SELECT " + - "host_local.pk_host_local,"+ - "host_local.int_cores_idle,"+ - "host_local.int_cores_max,"+ - "host_local.int_threads,"+ - "host_local.int_mem_idle,"+ - "host_local.int_mem_max,"+ - "host_local.int_gpu_idle,"+ - "host_local.int_gpu_max,"+ - "host_local.str_type,"+ - "(SELECT str_name FROM host WHERE host.pk_host = host_local.pk_host) " + - "AS str_host_name,"+ - "(SELECT str_name FROM job WHERE job.pk_job = host_local.pk_job) " + - "AS str_job_name,"+ - "(SELECT str_name FROM layer WHERE layer.pk_layer = host_local.pk_layer) " + - "AS str_layer_name,"+ - "(SELECT str_name FROM frame WHERE frame.pk_frame = host_local.pk_frame) " + - "AS str_frame_name " + - "FROM " + - "host_local "; - - private static final String QUERY_FOR_FACILITY = - "SELECT " + - "facility.pk_facility," + - "facility.str_name " + - "FROM " + - "facility "; - - private static final String QUERY_FOR_LIMIT = - "SELECT " + - "limit_record.pk_limit_record, " + - "limit_record.str_name, " + - "limit_record.int_max_value, " + - "SUM(layer_stat.int_running_count) AS int_current_running " + - "FROM " + - "limit_record " + - "LEFT JOIN " + - "layer_limit ON layer_limit.pk_limit_record = limit_record.pk_limit_record " + - "LEFT JOIN " + - "layer ON layer.pk_layer = layer_limit.pk_layer " + - "LEFT JOIN " + - "layer_stat ON layer_stat.pk_layer = layer.pk_layer "; - - private static final String GET_LIMIT_FROM_LAYER_ID = - "SELECT " + - "limit_record.pk_limit_record, " + - "limit_record.str_name, " + - "limit_record.int_max_value, " + - "SUM(layer_stat.int_running_count) AS int_current_running " + - "FROM " + - "limit_record " + - "LEFT JOIN " + - "layer_limit ON layer_limit.pk_limit_record = limit_record.pk_limit_record " + - "LEFT JOIN " + - "layer ON layer.pk_layer = layer_limit.pk_layer " + - "LEFT JOIN " + - "layer_stat ON layer_stat.pk_layer = layer.pk_layer " + - "WHERE " + - "layer_limit.pk_layer = ? " + - "GROUP BY " + - "limit_record.str_name, " + - "limit_record.pk_limit_record, " + - "limit_record.int_max_value"; - - public static final String GET_GROUPS = - "SELECT " + - "show.pk_show, " + - "show.str_name AS str_show," + - "dept.str_name AS str_dept," + - "folder.pk_folder," + - "folder.pk_parent_folder," + - "folder.str_name AS group_name," + - "folder.int_job_priority,"+ - "folder.int_job_min_cores," + - "folder.int_job_max_cores," + - "folder_resource.int_min_cores,"+ - "folder_resource.int_max_cores,"+ - "folder.b_default, " + - "folder_level.int_level, " + - "c.int_waiting_count, " + - "c.int_depend_count, " + - "c.int_running_count,"+ - "c.int_dead_count,"+ - "c.int_job_count,"+ - "c.int_cores " + - "FROM " + - "folder, " + - "folder_level," + - "folder_resource, "+ - "vs_folder_counts c, " + - "show," + - "dept " + - "WHERE " + - "show.pk_show = folder.pk_show "+ - "AND " + - "folder.pk_folder = folder_level.pk_folder " + - "AND " + - "folder.pk_folder = folder_resource.pk_folder " + - "AND " + - "folder.pk_folder = c.pk_folder " + - "AND " + - "folder.pk_dept = dept.pk_dept "; - - private static final String GET_ACTION = - "SELECT " + - "filter.pk_show," + - "action.* " + - "FROM " + - "filter,"+ - "action " + - "WHERE " + - "filter.pk_filter = action.pk_filter "; - - private static final String GET_JOB = - "SELECT " + - "job.pk_job,"+ - "job.str_log_dir," + - "job_resource.int_max_cores," + - "job_resource.int_min_cores," + - "job.str_name," + - "job.str_shot,"+ - "job.str_state,"+ - "job.int_uid,"+ - "job.str_user,"+ - "job.b_paused,"+ - "job.ts_started,"+ - "job.ts_stopped,"+ - "job.b_comment,"+ - "job.b_autoeat,"+ - "job.str_os,"+ - "job_resource.int_priority,"+ - "job.int_frame_count, " + - "job.int_layer_count, " + - "show.str_name as str_show," + - "show.pk_show as id_show,"+ - "facility.str_name AS facility_name,"+ - "folder.str_name AS group_name,"+ - "job_stat.int_waiting_count, "+ - "job_stat.int_running_count, "+ - "job_stat.int_dead_count, " + - "job_stat.int_eaten_count," + - "job_stat.int_depend_count, "+ - "job_stat.int_succeeded_count, "+ - "job_usage.int_core_time_success, "+ - "job_usage.int_core_time_fail, " + - "job_usage.int_frame_success_count, "+ - "job_usage.int_frame_fail_count, "+ - "job_usage.int_clock_time_high,"+ - "job_usage.int_clock_time_success,"+ - "job_mem.int_max_rss,"+ - "(job_resource.int_cores + job_resource.int_local_cores) AS int_cores " + - "FROM " + - "job,"+ - "folder,"+ - "show," + - "facility,"+ - "job_stat," + - "job_resource, " + - "job_mem, " + - "job_usage " + - "WHERE " + - "job.pk_show = show.pk_show " + - "AND " + - "job.pk_folder = folder.pk_folder " + - "AND " + - "job.pk_facility = facility.pk_facility " + - "AND " + - "job.pk_job = job_stat.pk_job " + - "AND " + - "job.pk_job = job_resource.pk_job " + - "AND " + - "job.pk_job = job_mem.pk_job " + - "AND " + - "job.pk_job = job_usage.pk_job "; - - private static final String GET_LAYER = - "SELECT " + - "layer.*," + - "layer_stat.int_total_count," + - "layer_stat.int_waiting_count," + - "layer_stat.int_running_count," + - "layer_stat.int_dead_count," + - "layer_stat.int_depend_count," + - "layer_stat.int_eaten_count," + - "layer_stat.int_succeeded_count," + - "layer_usage.int_core_time_success," + - "layer_usage.int_core_time_fail, "+ - "layer_usage.int_frame_success_count, "+ - "layer_usage.int_frame_fail_count, "+ - "layer_usage.int_clock_time_low, "+ - "layer_usage.int_clock_time_high," + - "layer_usage.int_clock_time_success," + - "layer_usage.int_clock_time_fail," + - "layer_mem.int_max_rss,"+ - "layer_resource.int_cores " + - "FROM " + - "layer, " + - "job," + - "layer_stat, " + - "layer_resource, " + - "layer_usage, " + - "layer_mem " + - "WHERE " + - "layer.pk_job = job.pk_job " + - "AND " + - "layer.pk_layer = layer_stat.pk_layer "+ - "AND " + - "layer.pk_layer = layer_resource.pk_layer " + - "AND " + - "layer.pk_layer = layer_usage.pk_layer " + - "AND " + - "layer.pk_layer = layer_mem.pk_layer "; - - private static final String GET_LAYER_WITH_LIMITS = - "SELECT " + - "layer.*, " + - "layer_stat.int_total_count, " + - "layer_stat.int_waiting_count, " + - "layer_stat.int_running_count, " + - "layer_stat.int_dead_count, " + - "layer_stat.int_depend_count, " + - "layer_stat.int_eaten_count, " + - "layer_stat.int_succeeded_count, " + - "layer_usage.int_core_time_success, " + - "layer_usage.int_core_time_fail, " + - "layer_usage.int_frame_success_count, " + - "layer_usage.int_frame_fail_count, " + - "layer_usage.int_clock_time_low, " + - "layer_usage.int_clock_time_high, " + - "layer_usage.int_clock_time_success, " + - "layer_usage.int_clock_time_fail, " + - "layer_mem.int_max_rss, " + - "layer_resource.int_cores, " + - "limit_names.str_limit_names " + - "FROM " + - "layer " + - "JOIN " + - "job ON layer.pk_job = job.pk_job " + - "JOIN " + - "layer_stat ON layer.pk_layer = layer_stat.pk_layer " + - "JOIN " + - "layer_resource ON layer.pk_layer = layer_resource.pk_layer " + - "JOIN " + - "layer_usage ON layer.pk_layer = layer_usage.pk_layer " + - "JOIN " + - "layer_mem ON layer.pk_layer = layer_mem.pk_layer " + - "LEFT JOIN " + - "(" + - "SELECT " + - "layer_limit.pk_layer, " + - "LISTAGG(limit_record.str_name, ',') AS str_limit_names " + - "FROM " + - "limit_record, " + - "layer_limit " + - "WHERE " + - "layer_limit.pk_limit_record = limit_record.pk_limit_record " + - "GROUP BY " + - "layer_limit.pk_layer) AS limit_names " + - "ON layer.pk_layer = limit_names.pk_layer "; - - private static final String GET_LIMIT_NAMES = - "SELECT " + - "limit_record.str_name " + - "FROM " + - "layer_limit, " + - "limit_record " + - "WHERE " + - "layer_limit.pk_layer = ? " + - "AND " + - "limit_record.pk_limit_record = layer_limit.pk_limit_record "; - - private static final String GET_SHOW = - "SELECT " + - "show.*," + - "NVL(vs_show_stat.int_pending_count,0) AS int_pending_count," + - "NVL(vs_show_stat.int_running_count,0) AS int_running_count," + - "NVL(vs_show_stat.int_dead_count,0) AS int_dead_count," + - "NVL(vs_show_resource.int_cores,0) AS int_cores, " + - "NVL(vs_show_stat.int_job_count,0) AS int_job_count " + - "FROM " + - "show " + - "LEFT JOIN vs_show_stat ON (vs_show_stat.pk_show = show.pk_show) "+ - "LEFT JOIN vs_show_resource ON (vs_show_resource.pk_show=show.pk_show) " + - "WHERE " + - "1 = 1 "; - - private static final String GET_SERVICE = - "SELECT " + - "service.pk_service,"+ - "service.str_name," + - "service.b_threadable," + - "service.int_cores_min," + - "service.int_cores_max," + - "service.int_mem_min," + - "service.int_gpu_min," + - "service.str_tags " + - "FROM "+ - "service "; - - private static final String GET_SERVICE_OVERRIDE = - "SELECT " + - "show_service.pk_show_service,"+ - "show_service.str_name," + - "show_service.b_threadable," + - "show_service.int_cores_min," + - "show_service.int_cores_max," + - "show_service.int_mem_min," + - "show_service.int_gpu_min," + - "show_service.str_tags " + - "FROM "+ - "show_service, " + - "show " + - "WHERE " + - "show_service.pk_show = show.pk_show "; - - private static final String GET_TASK = - "SELECT " + - "task.pk_task," + - "task.str_shot,"+ - "task.int_min_cores + task.int_adjust_cores AS int_min_cores, "+ - "task.int_adjust_cores, " + - "dept.str_name AS str_dept "+ - "FROM " + - "task,"+ - "dept, " + - "point "+ - "WHERE " + - "task.pk_point = point.pk_point " + - "AND " + - "point.pk_dept = dept.pk_dept "; - - private static final String GET_HOST = - "SELECT " + - "host.pk_host, "+ - "host.str_name AS host_name," + - "host_stat.str_state AS host_state,"+ - "host.b_nimby,"+ - "host_stat.ts_booted,"+ - "host_stat.ts_ping,"+ - "host.int_cores,"+ - "host.int_cores_idle,"+ - "host.int_mem,"+ - "host.int_mem_idle,"+ - "host.int_gpu,"+ - "host.int_gpu_idle,"+ - "host.str_tags,"+ - "host.str_lock_state,"+ - "host.b_comment,"+ - "host.int_thread_mode,"+ - "host_stat.str_os,"+ - "host_stat.int_mem_total,"+ - "host_stat.int_mem_free,"+ - "host_stat.int_swap_total,"+ - "host_stat.int_swap_free,"+ - "host_stat.int_mcp_total,"+ - "host_stat.int_mcp_free,"+ - "host_stat.int_gpu_total,"+ - "host_stat.int_gpu_free,"+ - "host_stat.int_load, " + - "alloc.str_name AS alloc_name " + - "FROM " + - "alloc," + - "facility, "+ - "host_stat,"+ - "host "+ - "WHERE " + - "host.pk_alloc = alloc.pk_alloc " + - "AND " + - "facility.pk_facility = alloc.pk_facility " + - "AND "+ - "host.pk_host = host_stat.pk_host "; - - private static final String GET_DEPEND = - "SELECT " + - "depend.pk_depend, "+ - "depend.str_type, "+ - "depend.b_active, "+ - "depend.b_any, "+ - "depend.str_target, "+ - "(SELECT str_name FROM job j WHERE j.pk_job = depend.pk_job_depend_on) AS depend_on_job, "+ - "(SELECT str_name FROM job j WHERE j.pk_job = depend.pk_job_depend_er) AS depend_er_job, "+ - "(SELECT str_name FROM layer l WHERE l.pk_layer = depend.pk_layer_depend_on) AS depend_on_layer, "+ - "(SELECT str_name FROM layer l WHERE l.pk_layer = depend.pk_layer_depend_er) AS depend_er_layer, "+ - "(SELECT str_name FROM frame f WHERE f.pk_frame = depend.pk_frame_depend_on) AS depend_on_frame, "+ - "(SELECT str_name FROM frame f WHERE f.pk_frame = depend.pk_frame_depend_er) AS depend_er_frame "+ - "FROM " + - "depend "; - - private static final String GET_SUBSCRIPTION = - "SELECT " + - "subscription.pk_subscription, " + - "(alloc.str_name || '.' || show.str_name) AS name, "+ - "subscription.int_burst, " + - "subscription.int_size, " + - "subscription.int_cores, " + - "show.str_name AS show_name, " + - "alloc.str_name AS alloc_name, " + - "facility.str_name AS facility_name " + - "FROM "+ - "show, " + - "alloc, " + - "facility,"+ - "subscription " + - "WHERE " + - "subscription.pk_show = show.pk_show " + - "AND " + - "subscription.pk_alloc = alloc.pk_alloc " + - "AND " + - "alloc.pk_facility = facility.pk_facility "; - - private static final String GET_PENDING_JOBS = - GET_JOB + - "AND " + - "job.str_state = 'PENDING' "; - - private static final String GET_FRAMES_CRITERIA = - - "SELECT " + - "frame.pk_frame, " + - "frame.int_exit_status,"+ - "frame.str_name,"+ - "frame.int_number,"+ - "frame.int_dispatch_order,"+ - "frame.ts_started,"+ - "frame.ts_stopped,"+ - "frame.int_retries,"+ - "frame.str_state,"+ - "frame.str_host,"+ - "frame.int_cores,"+ - "frame.int_mem_max_used," + - "frame.int_mem_used, " + - "frame.int_mem_reserved, " + - "frame.int_gpu_reserved, " + - "frame.str_checkpoint_state,"+ - "frame.int_checkpoint_count,"+ - "frame.int_total_past_core_time,"+ - "layer.str_name AS layer_name," + - "job.str_name AS job_name, "+ - "ROW_NUMBER() OVER " + - "(ORDER BY frame.int_dispatch_order ASC, layer.int_dispatch_order ASC) AS row_number " + - "FROM "+ - "job, " + - "layer,"+ - "frame " + - "WHERE " + - "frame.pk_layer = layer.pk_layer "+ - "AND "+ - "frame.pk_job= job.pk_job "; - - private static final String QUERY_FOR_DEED = - "SELECT " + - "host.str_name AS str_host,"+ - "show.str_name AS str_show,"+ - "owner.str_username," + - "deed.pk_deed " + - "FROM " + - "deed,"+ - "owner,"+ - "host,"+ - "show "+ - "WHERE " + - "deed.pk_host = host.pk_host " + - "AND " + - "deed.pk_owner = owner.pk_owner " + - "AND " + - "owner.pk_show = show.pk_show "; - - public FrameSearchFactory getFrameSearchFactory() { - return frameSearchFactory; - } - - public void setFrameSearchFactory(FrameSearchFactory frameSearchFactory) { - this.frameSearchFactory = frameSearchFactory; - } - - public ProcSearchFactory getProcSearchFactory() { - return procSearchFactory; - } - - public void setProcSearchFactory(ProcSearchFactory procSearchFactory) { - this.procSearchFactory = procSearchFactory; - } -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/DepartmentManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/DepartmentManagerService.java index 0d09ca44c..87d1ea158 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/DepartmentManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/DepartmentManagerService.java @@ -37,12 +37,10 @@ import com.imageworks.spcue.ShowInterface; import com.imageworks.spcue.TaskEntity; import com.imageworks.spcue.TaskInterface; -import com.imageworks.spcue.TrackitTaskDetail; import com.imageworks.spcue.dao.JobDao; import com.imageworks.spcue.dao.PointDao; import com.imageworks.spcue.dao.ShowDao; import com.imageworks.spcue.dao.TaskDao; -import com.imageworks.spcue.dao.TrackitDao; import com.imageworks.spcue.util.CueUtil; @@ -56,7 +54,6 @@ public class DepartmentManagerService implements DepartmentManager { private TaskDao taskDao; private ShowDao showDao; private JobDao jobDao; - private TrackitDao trackitDao; @Override public void createDepartmentConfig(PointDetail renderPoint) { @@ -176,61 +173,6 @@ public List getManagedPointConfs() { @Override @Transactional(propagation = Propagation.NOT_SUPPORTED) public void updateManagedTasks(PointInterface pd) { - if (env.getRequiredProperty("cue.trackit.enabled", Boolean.class)) { - - ShowInterface show = showDao.getShowDetail(pd.getShowId()); - PointDetail p = pointDao.getPointConfDetail(pd.getPointId()); - pointDao.updatePointConfUpdateTime(p); - - /* - * First calculate raw point ratios, which will be used to calculate - * the normalized proc point values - */ - float totalRawPoints = 0f; - float rawPoints = 0f; - - List tasks = trackitDao.getTasks(show.getName(), p.tiTask); - HashMap rawCache = new HashMap(tasks.size()); - - for (TrackitTaskDetail task : tasks) { - if (!IN_PROGRESS_TASK_STATUS.contains(task.status)) { - continue; - } - rawPoints = ((task.frameCount / 10f) / task.weeks); - rawCache.put(task.shot, rawPoints); - totalRawPoints = totalRawPoints + rawPoints; - } - - /* - * Now create TaskDetail objects which will be merged into - * the current data set. Tasks with a 0 minCores value will - * be deleted. - */ - float normalizedRawPoints = 0f; - if (totalRawPoints != 0) { - normalizedRawPoints = p.cores / totalRawPoints; - } - - for (TrackitTaskDetail task : tasks) { - - TaskEntity td = new TaskEntity(); - td.pointId = p.getPointId(); - td.deptId = p.getDepartmentId(); - td.showId = p.getShowId(); - td.shot = task.shot; - - if (!IN_PROGRESS_TASK_STATUS.contains(task.status)) { - td.minCoreUnits = 0; - } else { - td.minCoreUnits = (int) ((rawCache.get(task.shot) * normalizedRawPoints) + 0.5f); - if (td.minCoreUnits < CueUtil.ONE_CORE) { - td.minCoreUnits = CueUtil.ONE_CORE; - } - } - taskDao.mergeTask(td); - syncJobsWithTask(td); - } - } } @Override @@ -323,14 +265,6 @@ public void setTaskDao(TaskDao taskDao) { this.taskDao = taskDao; } - public TrackitDao getTrackitDao() { - return trackitDao; - } - - public void setTrackitDao(TrackitDao trackitDao) { - this.trackitDao = trackitDao; - } - public ShowDao getShowDao() { return showDao; } diff --git a/cuebot/src/main/resources/conf/ddl/oracle/demo_data.sql b/cuebot/src/main/resources/conf/ddl/oracle/demo_data.sql deleted file mode 100644 index fc97a477f..000000000 --- a/cuebot/src/main/resources/conf/ddl/oracle/demo_data.sql +++ /dev/null @@ -1,83 +0,0 @@ -Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000000','testing',2000000,100,0,0,0,0,1,1,1) - --- SPLIT HERE! -Insert into SHOW_ALIAS (PK_SHOW_ALIAS,PK_SHOW,STR_NAME) values ('00000000-0000-0000-0000-000000000001','00000000-0000-0000-0000-000000000000','test') - --- SPLIT HERE! -Insert into DEPT (PK_DEPT,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA0','Lighting',0) --- SPLIT HERE! -Insert into DEPT (PK_DEPT,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA1','Animation',0) --- SPLIT HERE! -Insert into DEPT (PK_DEPT,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA2','Hair',0) --- SPLIT HERE! -Insert into DEPT (PK_DEPT,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA3','Cloth',0) --- SPLIT HERE! -Insert into DEPT (PK_DEPT,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA4','Layout',0) --- SPLIT HERE! -Insert into DEPT (PK_DEPT,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA5','FX',0) --- SPLIT HERE! -Insert into DEPT (PK_DEPT,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA6','Pipeline',0) - --- SPLIT HERE! -Insert into FACILITY (PK_FACILITY,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA1','local',1) --- SPLIT HERE! -Insert into FACILITY (PK_FACILITY,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA0','cloud',0) - --- SPLIT HERE! -Insert into FOLDER (PK_FOLDER,PK_PARENT_FOLDER,PK_SHOW,STR_NAME,B_DEFAULT,PK_DEPT,INT_JOB_MIN_CORES,INT_JOB_MAX_CORES,INT_JOB_PRIORITY,F_ORDER,B_EXCLUDE_MANAGED) values ('A0000000-0000-0000-0000-000000000000',null,'00000000-0000-0000-0000-000000000000','testing',1,'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA6',-1,-1,-1,1,0) - --- SPLIT HERE! -Insert into POINT (PK_POINT,PK_DEPT,PK_SHOW,STR_TI_TASK,INT_CORES,B_MANAGED,INT_MIN_CORES,FLOAT_TIER) values ('FFEEDDCC-AAAA-AAAA-AAAA-AAAAAAAAAAA0','AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA6','00000000-0000-0000-0000-000000000000',null,0,0,0,0) - --- SPLIT HERE! -Insert into ALLOC (PK_ALLOC,STR_NAME,B_ALLOW_EDIT,B_DEFAULT,STR_TAG,PK_FACILITY,B_BILLABLE,B_ENABLED) values ('00000000-0000-0000-0000-000000000000','local.general',0,0,'general','AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA1',0,1) --- SPLIT HERE! -Insert into ALLOC (PK_ALLOC,STR_NAME,B_ALLOW_EDIT,B_DEFAULT,STR_TAG,PK_FACILITY,B_BILLABLE,B_ENABLED) values ('00000000-0000-0000-0000-000000000001','local.desktop',0,0,'desktop','AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA1',0,1) --- SPLIT HERE! -Insert into ALLOC (PK_ALLOC,STR_NAME,B_ALLOW_EDIT,B_DEFAULT,STR_TAG,PK_FACILITY,B_BILLABLE,B_ENABLED) values ('00000000-0000-0000-0000-000000000002','local.unassigned',0,1,'unassigned','AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA1',0,1) --- SPLIT HERE! -Insert into ALLOC (PK_ALLOC,STR_NAME,B_ALLOW_EDIT,B_DEFAULT,STR_TAG,PK_FACILITY,B_BILLABLE,B_ENABLED) values ('00000000-0000-0000-0000-000000000003','gcp.cloud',1,0,'cloud','AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA0',0,1) --- SPLIT HERE! -Insert into ALLOC (PK_ALLOC,STR_NAME,B_ALLOW_EDIT,B_DEFAULT,STR_TAG,PK_FACILITY,B_BILLABLE,B_ENABLED) values ('00000000-0000-0000-0000-000000000004','gcp.unassigned',1,0,'unassigned','AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA0',0,1) - --- SPLIT HERE! -Insert into SUBSCRIPTION (PK_SUBSCRIPTION,PK_ALLOC,PK_SHOW,INT_SIZE,INT_BURST,INT_CORES,FLOAT_TIER) values ('00000000-0000-0000-0000-000000000001','00000000-0000-0000-0000-000000000000','00000000-0000-0000-0000-000000000000',100000,100000,0,0) --- SPLIT HERE! -Insert into SUBSCRIPTION (PK_SUBSCRIPTION,PK_ALLOC,PK_SHOW,INT_SIZE,INT_BURST,INT_CORES,FLOAT_TIER) values ('00000000-0000-0000-0000-000000000002','00000000-0000-0000-0000-000000000001','00000000-0000-0000-0000-000000000000',100000,100000,0,0) --- SPLIT HERE! -Insert into SUBSCRIPTION (PK_SUBSCRIPTION,PK_ALLOC,PK_SHOW,INT_SIZE,INT_BURST,INT_CORES,FLOAT_TIER) values ('00000000-0000-0000-0000-000000000003','00000000-0000-0000-0000-000000000003','00000000-0000-0000-0000-000000000000',100000,100000,0,0) - --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA0','default',0,100,3355443,'general | desktop') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA1','prman',0,100,3355443,'general | desktop') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA2','arnold',1,100,3355443,'general | desktop | cloud') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA3','shell',0,100,3355443,'general | util') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA4','maya',0,100,2097152,'general | desktop') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA5','houdini',0,100,3355443,'general | desktop') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA7','katana',1,100,2097152,'general | desktop | util') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA8','shake',0,100,2097152,'general | desktop') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA9','nuke',0,100,2097152,'general | desktop') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAA11','preprocess',0,10,393216,'util') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAA12','postprocess',0,10,524288,'util') - --- SPLIT HERE! -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000005','MAX_FRAME_RETRIES',16,0,null,0) - --- SPLIT HERE! -Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000002','LOCK_HARDWARE_STATE_CHECK',0,30) --- SPLIT HERE! -Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000001','LOCK_HISTORICAL_TRANSFER',0,3600) --- SPLIT HERE! -Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000003','LOCK_ORPHANED_PROC_CHECK',0,30) --- SPLIT HERE! -Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000005','LOCK_TASK_UPDATE',1240618998852,3600) diff --git a/cuebot/src/main/resources/conf/ddl/oracle/migrations/.keep b/cuebot/src/main/resources/conf/ddl/oracle/migrations/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/cuebot/src/main/resources/conf/ddl/oracle/migrations/V2__Add_limit_table.sql b/cuebot/src/main/resources/conf/ddl/oracle/migrations/V2__Add_limit_table.sql deleted file mode 100644 index ee0e6d5ad..000000000 --- a/cuebot/src/main/resources/conf/ddl/oracle/migrations/V2__Add_limit_table.sql +++ /dev/null @@ -1,19 +0,0 @@ - -CREATE TABLE "LIMIT_RECORD" ( - "PK_LIMIT_RECORD" VARCHAR2(36 BYTE) NOT NULL, - "STR_NAME" VARCHAR2(255 BYTE) NOT NULL, - "INT_MAX_VALUE" NUMBER(38,0) DEFAULT 0 NOT NULL, - "B_HOST_LIMIT" NUMBER(1,0) DEFAULT 0 NOT NULL -); - -CREATE TABLE "LAYER_LIMIT" ( - "PK_LAYER_LIMIT" VARCHAR2(36 BYTE) NOT NULL, - "PK_LAYER" VARCHAR2(36 BYTE) NOT NULL, - "PK_LIMIT_RECORD" VARCHAR2(36 BYTE) NOT NULL -); - -CREATE TRIGGER "BEFORE_DELETE_LAYER_DROP_LIMIT" BEFORE DELETE ON layer -FOR EACH ROW -BEGIN - DELETE FROM layer_limit WHERE pk_layer=:old.pk_layer; -END; diff --git a/cuebot/src/main/resources/conf/ddl/oracle/schema.sql b/cuebot/src/main/resources/conf/ddl/oracle/schema.sql deleted file mode 100644 index dde36df9e..000000000 --- a/cuebot/src/main/resources/conf/ddl/oracle/schema.sql +++ /dev/null @@ -1,2737 +0,0 @@ -CREATE TABLE "HISTORY_PERIOD_BAK" - ( "PK" VARCHAR2(32 BYTE), - "DT_BEGIN" DATE NOT NULL, - "DT_END" DATE NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "FRAME_HISTORY" - ( "PK_FRAME_HISTORY" RAW(16) DEFAULT sys_guid() NOT NULL, - "PK_FRAME" VARCHAR2(36 BYTE) NOT NULL, - "PK_LAYER" VARCHAR2(36 BYTE) NOT NULL, - "PK_JOB" VARCHAR2(36 BYTE) NOT NULL, - "STR_NAME" VARCHAR2(256 BYTE) NOT NULL, - "STR_STATE" VARCHAR2(24 BYTE) NOT NULL, - "INT_MEM_RESERVED" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_MEM_MAX_USED" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_CORES" NUMBER(16,0) DEFAULT 100 NOT NULL, - "STR_HOST" VARCHAR2(64 BYTE) DEFAULT NULL, - "INT_EXIT_STATUS" NUMBER(8,0) DEFAULT -1 NOT NULL, - "PK_ALLOC" VARCHAR2(36 BYTE), - "INT_TS_STARTED" NUMBER(12,0) NOT NULL, - "INT_TS_STOPPED" NUMBER(12,0) DEFAULT 0 NOT NULL, - "INT_CHECKPOINT_COUNT" NUMBER(6,0) DEFAULT 0 NOT NULL, - "DT_LAST_MODIFIED" DATE NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "HISTORY_PERIOD" - ( "PK" VARCHAR2(32 BYTE) DEFAULT sys_guid(), - "DT_BEGIN" DATE DEFAULT to_date('01-JAN-2000','DD-MON-YYYY') NOT NULL, - "DT_END" DATE DEFAULT sysdate NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "DUPLICATE_CURSORS" - ( "DT_RECORDED" DATE, - "INST_ID" NUMBER, - "LNG_COUNT" NUMBER - ) --- SPLIT HERE! -CREATE TABLE "UNCOMMITTED_TRANSACTIONS_BAK" - ( "INST_ID" NUMBER, - "SID" NUMBER, - "SERIAL#" NUMBER, - "USERNAME" VARCHAR2(30 BYTE), - "MACHINE" VARCHAR2(64 BYTE), - "MODULE" VARCHAR2(48 BYTE), - "SERVICE_NAME" VARCHAR2(64 BYTE), - "DURATION" NUMBER, - "DT_RECORDED" DATE - ) --- SPLIT HERE! -CREATE TABLE "UNCOMMITTED_TRANSACTIONS" - ( "INST_ID" NUMBER, - "SID" NUMBER, - "SERIAL#" NUMBER, - "USERNAME" VARCHAR2(30 BYTE), - "MACHINE" VARCHAR2(64 BYTE), - "MODULE" VARCHAR2(48 BYTE), - "SERVICE_NAME" VARCHAR2(64 BYTE), - "DURATION" NUMBER, - "DT_RECORDED" DATE DEFAULT sysdate - ) --- SPLIT HERE! -CREATE TABLE "TEST" - ( "COL1" VARCHAR2(32 BYTE) - ) --- SPLIT HERE! -CREATE TABLE "LAYER_OUTPUT" - ( "PK_LAYER_OUTPUT" VARCHAR2(36 BYTE) NOT NULL, - "PK_LAYER" VARCHAR2(36 BYTE) NOT NULL, - "PK_JOB" VARCHAR2(36 BYTE) NOT NULL, - "STR_FILESPEC" VARCHAR2(2048 BYTE) NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "SHOW_SERVICE" - ( "PK_SHOW_SERVICE" VARCHAR2(36 BYTE) NOT NULL, - "PK_SHOW" VARCHAR2(36 BYTE) NOT NULL, - "STR_NAME" VARCHAR2(36 BYTE) NOT NULL, - "B_THREADABLE" NUMBER(1,0) NOT NULL, - "INT_CORES_MIN" NUMBER(8,0) NOT NULL, - "INT_MEM_MIN" NUMBER(16,0) NOT NULL, - "STR_TAGS" VARCHAR2(128 BYTE) NOT NULL, - "INT_CORES_MAX" NUMBER(10,0) DEFAULT 0 NOT NULL, - "INT_GPU_MIN" NUMBER(10,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "SQLN_EXPLAIN_PLAN" - ( "STATEMENT_ID" VARCHAR2(30 BYTE), - "TIMESTAMP" DATE, - "REMARKS" VARCHAR2(80 BYTE), - "OPERATION" VARCHAR2(30 BYTE), - "OPTIONS" VARCHAR2(30 BYTE), - "OBJECT_NODE" VARCHAR2(128 BYTE), - "OBJECT_OWNER" VARCHAR2(30 BYTE), - "OBJECT_NAME" VARCHAR2(30 BYTE), - "OBJECT_INSTANCE" NUMBER(*,0), - "OBJECT_TYPE" VARCHAR2(30 BYTE), - "OPTIMIZER" VARCHAR2(255 BYTE), - "SEARCH_COLUMNS" NUMBER(*,0), - "ID" NUMBER(*,0), - "PARENT_ID" NUMBER(*,0), - "POSITION" NUMBER(*,0), - "COST" NUMBER(*,0), - "CARDINALITY" NUMBER(*,0), - "BYTES" NUMBER(*,0), - "OTHER_TAG" VARCHAR2(255 BYTE), - "PARTITION_START" VARCHAR2(255 BYTE), - "PARTITION_STOP" VARCHAR2(255 BYTE), - "PARTITION_ID" NUMBER(*,0), - "OTHER" LONG, - "DISTRIBUTION" VARCHAR2(30 BYTE) - ) --- SPLIT HERE! -CREATE TABLE "DEED" - ( "PK_DEED" VARCHAR2(36 BYTE) NOT NULL, - "PK_OWNER" VARCHAR2(36 BYTE) NOT NULL, - "PK_HOST" VARCHAR2(36 BYTE) NOT NULL, - "B_BLACKOUT" NUMBER(1,0) DEFAULT 0 NOT NULL, - "INT_BLACKOUT_START" NUMBER(12,0), - "INT_BLACKOUT_STOP" NUMBER(12,0) - ) --- SPLIT HERE! -CREATE TABLE "OWNER" - ( "PK_OWNER" VARCHAR2(36 BYTE) NOT NULL, - "PK_SHOW" VARCHAR2(36 BYTE) NOT NULL, - "STR_USERNAME" VARCHAR2(64 BYTE) NOT NULL, - "TS_CREATED" TIMESTAMP (6) WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL, - "TS_UPDATED" TIMESTAMP (6) WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "HOST_LOCAL" - ( "PK_HOST_LOCAL" VARCHAR2(36 BYTE) NOT NULL, - "PK_JOB" VARCHAR2(36 BYTE) NOT NULL, - "PK_LAYER" VARCHAR2(36 BYTE), - "PK_FRAME" VARCHAR2(36 BYTE), - "PK_HOST" VARCHAR2(36 BYTE) NOT NULL, - "TS_CREATED" TIMESTAMP (6) WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL, - "TS_UPDATED" TIMESTAMP (6) WITH TIME ZONE, - "INT_MEM_MAX" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_MEM_IDLE" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_CORES_MAX" NUMBER(16,0) DEFAULT 100 NOT NULL, - "INT_CORES_IDLE" NUMBER(16,0) DEFAULT 100 NOT NULL, - "INT_THREADS" NUMBER(4,0) DEFAULT 1 NOT NULL, - "FLOAT_TIER" NUMBER(16,2) DEFAULT 0 NOT NULL, - "B_ACTIVE" NUMBER(1,0) DEFAULT 1 NOT NULL, - "STR_TYPE" VARCHAR2(36 BYTE) NOT NULL, - "INT_GPU_IDLE" NUMBER(10,0) DEFAULT 0 NOT NULL, - "INT_GPU_MAX" NUMBER(10,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "SERVICE" - ( "PK_SERVICE" VARCHAR2(36 BYTE) NOT NULL, - "STR_NAME" VARCHAR2(36 BYTE) NOT NULL, - "B_THREADABLE" NUMBER(1,0) NOT NULL, - "INT_CORES_MIN" NUMBER(8,0) NOT NULL, - "INT_MEM_MIN" NUMBER(16,0) NOT NULL, - "STR_TAGS" VARCHAR2(128 BYTE) NOT NULL, - "INT_CORES_MAX" NUMBER(10,0) DEFAULT 0 NOT NULL, - "INT_GPU_MIN" NUMBER(10,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "JOB_LOCAL" - ( "PK_JOB_LOCAL" VARCHAR2(36 BYTE) NOT NULL, - "PK_JOB" VARCHAR2(36 BYTE) NOT NULL, - "PK_HOST" VARCHAR2(36 BYTE) NOT NULL, - "STR_SOURCE" VARCHAR2(255 BYTE) NOT NULL, - "TS_CREATED" TIMESTAMP (6) WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL, - "INT_CORES" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_MAX_CORES" NUMBER(16,0) NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "TASK" - ( "PK_TASK" VARCHAR2(36 BYTE) NOT NULL, - "PK_POINT" VARCHAR2(36 BYTE) NOT NULL, - "STR_SHOT" VARCHAR2(36 BYTE) NOT NULL, - "INT_MIN_CORES" NUMBER(16,0) DEFAULT 100 NOT NULL, - "INT_ADJUST_CORES" NUMBER(16,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "POINT" - ( "PK_POINT" VARCHAR2(36 BYTE) NOT NULL, - "PK_DEPT" VARCHAR2(36 BYTE) NOT NULL, - "PK_SHOW" VARCHAR2(36 BYTE) NOT NULL, - "STR_TI_TASK" VARCHAR2(36 BYTE), - "INT_CORES" NUMBER(16,0) DEFAULT 0 NOT NULL, - "B_MANAGED" NUMBER(1,0) DEFAULT 0 NOT NULL, - "INT_MIN_CORES" NUMBER(16,0) DEFAULT 0 NOT NULL, - "FLOAT_TIER" NUMBER(16,2) DEFAULT 0 NOT NULL, - "TS_UPDATED" TIMESTAMP (6) WITH TIME ZONE DEFAULT systimestamp NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "LAYER_MEM" - ( "PK_LAYER_MEM" VARCHAR2(36 BYTE) NOT NULL, - "PK_JOB" VARCHAR2(36 BYTE) NOT NULL, - "PK_LAYER" VARCHAR2(36 BYTE) NOT NULL, - "INT_MAX_RSS" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_MAX_VSS" NUMBER(16,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "JOB_MEM" - ( "PK_JOB_MEM" VARCHAR2(36 BYTE) NOT NULL, - "PK_JOB" VARCHAR2(36 BYTE) NOT NULL, - "INT_MAX_RSS" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_MAX_VSS" NUMBER(16,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "FOLDER_RESOURCE" - ( "PK_FOLDER_RESOURCE" VARCHAR2(36 BYTE) NOT NULL, - "PK_FOLDER" VARCHAR2(36 BYTE) NOT NULL, - "INT_CORES" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_MAX_CORES" NUMBER(16,0) DEFAULT -1 NOT NULL, - "INT_MIN_CORES" NUMBER(16,0) DEFAULT 0 NOT NULL, - "FLOAT_TIER" NUMBER(16,2) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "SHOW_ALIAS" - ( "PK_SHOW_ALIAS" VARCHAR2(36 BYTE) NOT NULL, - "PK_SHOW" VARCHAR2(36 BYTE) NOT NULL, - "STR_NAME" VARCHAR2(16 BYTE) NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "DEPT" - ( "PK_DEPT" VARCHAR2(36 BYTE) NOT NULL, - "STR_NAME" VARCHAR2(36 BYTE) NOT NULL, - "B_DEFAULT" NUMBER(1,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "FACILITY" - ( "PK_FACILITY" VARCHAR2(36 BYTE) NOT NULL, - "STR_NAME" VARCHAR2(36 BYTE) NOT NULL, - "B_DEFAULT" NUMBER(1,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "JOB_POST" - ( "PK_JOB_POST" VARCHAR2(36 BYTE) NOT NULL, - "PK_JOB" VARCHAR2(36 BYTE) NOT NULL, - "PK_POST_JOB" VARCHAR2(36 BYTE) NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "LAYER_HISTORY" - ( "PK_LAYER" VARCHAR2(36 BYTE) NOT NULL, - "PK_JOB" VARCHAR2(36 BYTE) NOT NULL, - "STR_NAME" VARCHAR2(512 BYTE) NOT NULL, - "STR_TYPE" VARCHAR2(16 BYTE) NOT NULL, - "INT_CORES_MIN" NUMBER(38,0) DEFAULT 100 NOT NULL, - "INT_MEM_MIN" NUMBER(38,0) DEFAULT 4194304 NOT NULL, - "INT_CORE_TIME_SUCCESS" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_CORE_TIME_FAIL" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_FRAME_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_LAYER_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_WAITING_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_DEAD_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_DEPEND_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_EATEN_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_SUCCEEDED_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_RUNNING_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_MAX_RSS" NUMBER(38,0) DEFAULT 0 NOT NULL, - "B_ARCHIVED" NUMBER(1,0) DEFAULT 0 NOT NULL, - "DT_LAST_MODIFIED" DATE NOT NULL, - "STR_SERVICES" VARCHAR2(128) - ) --- SPLIT HERE! -CREATE TABLE "JOB_HISTORY" - ( "PK_JOB" VARCHAR2(36 BYTE) NOT NULL, - "PK_SHOW" VARCHAR2(36 BYTE) NOT NULL, - "STR_NAME" VARCHAR2(512 BYTE) NOT NULL, - "STR_SHOT" VARCHAR2(64 BYTE) NOT NULL, - "STR_USER" VARCHAR2(36 BYTE) NOT NULL, - "INT_CORE_TIME_SUCCESS" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_CORE_TIME_FAIL" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_FRAME_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_LAYER_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_WAITING_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_DEAD_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_DEPEND_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_EATEN_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_SUCCEEDED_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_RUNNING_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_MAX_RSS" NUMBER(38,0) DEFAULT 0 NOT NULL, - "B_ARCHIVED" NUMBER(1,0) DEFAULT 0 NOT NULL, - "PK_FACILITY" VARCHAR2(36 BYTE) NOT NULL, - "PK_DEPT" VARCHAR2(36 BYTE) NOT NULL, - "INT_TS_STARTED" NUMBER(12,0) NOT NULL, - "INT_TS_STOPPED" NUMBER(12,0) DEFAULT 0 NOT NULL, - "DT_LAST_MODIFIED" DATE NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "TASK_LOCK" - ( "PK_TASK_LOCK" VARCHAR2(36 BYTE) NOT NULL, - "STR_NAME" VARCHAR2(36 BYTE) NOT NULL, - "INT_LOCK" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_TIMEOUT" NUMBER(38,0) DEFAULT 30 NOT NULL, - "TS_LASTRUN" TIMESTAMP (6) DEFAULT systimestamp NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "MATTHEW_STATS_TAB" - ( "STATID" VARCHAR2(30 BYTE), - "TYPE" CHAR(1 BYTE), - "VERSION" NUMBER, - "FLAGS" NUMBER, - "C1" VARCHAR2(30 BYTE), - "C2" VARCHAR2(30 BYTE), - "C3" VARCHAR2(30 BYTE), - "C4" VARCHAR2(30 BYTE), - "C5" VARCHAR2(30 BYTE), - "N1" NUMBER, - "N2" NUMBER, - "N3" NUMBER, - "N4" NUMBER, - "N5" NUMBER, - "N6" NUMBER, - "N7" NUMBER, - "N8" NUMBER, - "N9" NUMBER, - "N10" NUMBER, - "N11" NUMBER, - "N12" NUMBER, - "D1" DATE, - "R1" RAW(32), - "R2" RAW(32), - "CH1" VARCHAR2(1000 BYTE) - ) --- SPLIT HERE! -CREATE TABLE "HOST_TAG" - ( "PK_HOST_TAG" VARCHAR2(36 BYTE) NOT NULL, - "PK_HOST" VARCHAR2(36 BYTE) NOT NULL, - "STR_TAG" VARCHAR2(36 BYTE) NOT NULL, - "STR_TAG_TYPE" VARCHAR2(24 BYTE) DEFAULT 'Hardware' NOT NULL, - "B_CONSTANT" NUMBER(1,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "JOB_USAGE" - ( "PK_JOB_USAGE" VARCHAR2(36 BYTE) NOT NULL, - "PK_JOB" VARCHAR2(36 BYTE) NOT NULL, - "INT_CORE_TIME_SUCCESS" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_CORE_TIME_FAIL" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_FRAME_SUCCESS_COUNT" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_FRAME_FAIL_COUNT" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_CLOCK_TIME_FAIL" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_CLOCK_TIME_HIGH" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_CLOCK_TIME_SUCCESS" NUMBER(16,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "JOB_RESOURCE" - ( "PK_JOB_RESOURCE" VARCHAR2(36 BYTE) NOT NULL, - "PK_JOB" VARCHAR2(36 BYTE) NOT NULL, - "INT_CORES" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_MAX_RSS" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_MAX_VSS" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_MIN_CORES" NUMBER(16,0) DEFAULT 100 NOT NULL, - "INT_MAX_CORES" NUMBER(16,0) DEFAULT 10000 NOT NULL, - "FLOAT_TIER" NUMBER(16,2) DEFAULT 0 NOT NULL, - "INT_PRIORITY" NUMBER(16,0) DEFAULT 1 NOT NULL, - "INT_LOCAL_CORES" NUMBER(16,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "JOB_STAT" - ( "PK_JOB_STAT" VARCHAR2(36 BYTE) NOT NULL, - "PK_JOB" VARCHAR2(36 BYTE) NOT NULL, - "INT_WAITING_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_RUNNING_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_DEAD_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_DEPEND_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_EATEN_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_SUCCEEDED_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_CHECKPOINT_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "SUBSCRIPTION" - ( "PK_SUBSCRIPTION" VARCHAR2(36 BYTE) NOT NULL, - "PK_ALLOC" VARCHAR2(36 BYTE) NOT NULL, - "PK_SHOW" VARCHAR2(36 BYTE) NOT NULL, - "INT_SIZE" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_BURST" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_CORES" NUMBER(16,0) DEFAULT 0 NOT NULL, - "FLOAT_TIER" NUMBER(16,2) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "SHOW" - ( "PK_SHOW" VARCHAR2(36 BYTE) NOT NULL, - "STR_NAME" VARCHAR2(36 BYTE) NOT NULL, - "B_PAUSED" NUMBER(1,0) DEFAULT 0 NOT NULL, - "INT_DEFAULT_MIN_CORES" NUMBER(16,0) DEFAULT 100 NOT NULL, - "INT_DEFAULT_MAX_CORES" NUMBER(16,0) DEFAULT 10000 NOT NULL, - "INT_FRAME_INSERT_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_JOB_INSERT_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_FRAME_SUCCESS_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_FRAME_FAIL_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "B_BOOKING_ENABLED" NUMBER(1,0) DEFAULT 1 NOT NULL, - "B_DISPATCH_ENABLED" NUMBER(1,0) DEFAULT 1 NOT NULL, - "B_ACTIVE" NUMBER(1,0) DEFAULT 1 NOT NULL, - "STR_COMMENT_EMAIL" VARCHAR2(1024 BYTE) - ) --- SPLIT HERE! -CREATE TABLE "PROC" - ( "PK_PROC" VARCHAR2(36 BYTE) NOT NULL, - "PK_HOST" VARCHAR2(36 BYTE) NOT NULL, - "PK_JOB" VARCHAR2(36 BYTE), - "PK_SHOW" VARCHAR2(36 BYTE), - "PK_LAYER" VARCHAR2(36 BYTE), - "PK_FRAME" VARCHAR2(36 BYTE), - "INT_CORES_RESERVED" NUMBER(38,0) NOT NULL, - "INT_MEM_RESERVED" NUMBER(38,0) NOT NULL, - "INT_MEM_USED" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_MEM_MAX_USED" NUMBER(38,0) DEFAULT 0 NOT NULL, - "B_UNBOOKED" NUMBER(1,0) DEFAULT 0 NOT NULL, - "INT_MEM_PRE_RESERVED" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_VIRT_USED" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_VIRT_MAX_USED" NUMBER(16,0) DEFAULT 0 NOT NULL, - "STR_REDIRECT" VARCHAR2(265 BYTE), - "B_LOCAL" NUMBER(1,0) DEFAULT 0 NOT NULL, - "TS_PING" TIMESTAMP (6) WITH TIME ZONE DEFAULT systimestamp NOT NULL, - "TS_BOOKED" TIMESTAMP (6) WITH TIME ZONE DEFAULT systimestamp NOT NULL, - "TS_DISPATCHED" TIMESTAMP (6) WITH TIME ZONE DEFAULT systimestamp NOT NULL, - "INT_GPU_RESERVED" NUMBER(10,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "MATCHER" - ( "PK_MATCHER" VARCHAR2(36 BYTE) NOT NULL, - "PK_FILTER" VARCHAR2(36 BYTE) NOT NULL, - "STR_SUBJECT" VARCHAR2(64 BYTE) NOT NULL, - "STR_MATCH" VARCHAR2(64 BYTE) NOT NULL, - "STR_VALUE" VARCHAR2(4000 BYTE) NOT NULL, - "TS_CREATED" TIMESTAMP (6) DEFAULT systimestamp NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "LAYER_USAGE" - ( "PK_LAYER_USAGE" VARCHAR2(36 BYTE) NOT NULL, - "PK_LAYER" VARCHAR2(36 BYTE) NOT NULL, - "PK_JOB" VARCHAR2(36 BYTE) NOT NULL, - "INT_CORE_TIME_SUCCESS" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_CORE_TIME_FAIL" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_FRAME_SUCCESS_COUNT" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_FRAME_FAIL_COUNT" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_CLOCK_TIME_FAIL" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_CLOCK_TIME_HIGH" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_CLOCK_TIME_LOW" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_CLOCK_TIME_SUCCESS" NUMBER(16,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "LAYER_STAT" - ( "PK_LAYER_STAT" VARCHAR2(36 BYTE) NOT NULL, - "PK_LAYER" VARCHAR2(36 BYTE) NOT NULL, - "PK_JOB" VARCHAR2(36 BYTE) NOT NULL, - "INT_TOTAL_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_WAITING_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_RUNNING_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_DEAD_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_DEPEND_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_EATEN_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_SUCCEEDED_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_CHECKPOINT_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "LAYER_RESOURCE" - ( "PK_LAYER_RESOURCE" VARCHAR2(36 BYTE) NOT NULL, - "PK_LAYER" VARCHAR2(36 BYTE) NOT NULL, - "PK_JOB" VARCHAR2(36 BYTE) NOT NULL, - "INT_CORES" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_MAX_RSS" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_MAX_VSS" NUMBER(16,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "LAYER_ENV" - ( "PK_LAYER_ENV" VARCHAR2(36 BYTE) NOT NULL, - "PK_LAYER" VARCHAR2(36 BYTE), - "PK_JOB" VARCHAR2(36 BYTE), - "STR_KEY" VARCHAR2(36 BYTE), - "STR_VALUE" VARCHAR2(2048 BYTE) - ) --- SPLIT HERE! -CREATE TABLE "LAYER" - ( "PK_LAYER" VARCHAR2(36 BYTE) NOT NULL, - "PK_JOB" VARCHAR2(36 BYTE) NOT NULL, - "STR_NAME" VARCHAR2(256 BYTE) NOT NULL, - "STR_CMD" VARCHAR2(4000 BYTE) NOT NULL, - "STR_RANGE" VARCHAR2(4000 BYTE) NOT NULL, - "INT_CHUNK_SIZE" NUMBER(38,0) DEFAULT 1 NOT NULL, - "INT_DISPATCH_ORDER" NUMBER(38,0) DEFAULT 1 NOT NULL, - "INT_CORES_MIN" NUMBER(38,0) DEFAULT 100 NOT NULL, - "INT_MEM_MIN" NUMBER(38,0) DEFAULT 4194304 NOT NULL, - "STR_TAGS" VARCHAR2(4000 BYTE) DEFAULT '' NOT NULL, - "STR_TYPE" VARCHAR2(16 BYTE) NOT NULL, - "B_THREADABLE" NUMBER(1,0) DEFAULT 1 NOT NULL, - "STR_SERVICES" VARCHAR2(128 BYTE) DEFAULT 'default' NOT NULL, - "B_OPTIMIZE" NUMBER(1,0) DEFAULT 1 NOT NULL, - "INT_CORES_MAX" NUMBER(10,0) DEFAULT 0 NOT NULL, - "INT_GPU_MIN" NUMBER(10,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "JOB_ENV" - ( "PK_JOB_ENV" VARCHAR2(36 BYTE) NOT NULL, - "PK_JOB" VARCHAR2(36 BYTE), - "STR_KEY" VARCHAR2(36 BYTE), - "STR_VALUE" VARCHAR2(2048 BYTE) - ) --- SPLIT HERE! -CREATE TABLE "JOB" - ( "PK_JOB" VARCHAR2(36 BYTE) NOT NULL, - "PK_FOLDER" VARCHAR2(36 BYTE) NOT NULL, - "PK_SHOW" VARCHAR2(36 BYTE) NOT NULL, - "STR_NAME" VARCHAR2(255 BYTE) NOT NULL, - "STR_VISIBLE_NAME" VARCHAR2(255 BYTE), - "STR_SHOT" VARCHAR2(64 BYTE) NOT NULL, - "STR_USER" VARCHAR2(32 BYTE) NOT NULL, - "STR_STATE" VARCHAR2(16 BYTE) NOT NULL, - "STR_LOG_DIR" VARCHAR2(4000 BYTE) DEFAULT '' NOT NULL, - "INT_UID" NUMBER(38,0) DEFAULT 0 NOT NULL, - "B_PAUSED" NUMBER(1,0) DEFAULT 0 NOT NULL, - "B_AUTOEAT" NUMBER(1,0) DEFAULT 0 NOT NULL, - "INT_FRAME_COUNT" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_LAYER_COUNT" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_MAX_RETRIES" NUMBER(4,0) DEFAULT 3 NOT NULL, - "B_AUTO_BOOK" NUMBER(1,0) DEFAULT 1 NOT NULL, - "B_AUTO_UNBOOK" NUMBER(1,0) DEFAULT 1 NOT NULL, - "B_COMMENT" NUMBER(1,0) DEFAULT 0 NOT NULL, - "STR_EMAIL" VARCHAR2(256 BYTE), - "PK_FACILITY" VARCHAR2(36 BYTE) NOT NULL, - "PK_DEPT" VARCHAR2(36 BYTE) NOT NULL, - "TS_STARTED" TIMESTAMP (6) WITH TIME ZONE DEFAULT systimestamp NOT NULL, - "TS_STOPPED" TIMESTAMP (6) WITH TIME ZONE, - "INT_MIN_CORES" NUMBER(16,0) DEFAULT 100 NOT NULL, - "INT_MAX_CORES" NUMBER(16,0) DEFAULT 20000 NOT NULL, - "STR_SHOW" VARCHAR2(32 BYTE) DEFAULT 'none' NOT NULL, - "TS_UPDATED" TIMESTAMP (6) WITH TIME ZONE DEFAULT systimestamp NOT NULL, - "STR_OS" VARCHAR2(12 BYTE) DEFAULT 'rhel40' NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "HOST_STAT" - ( "PK_HOST_STAT" VARCHAR2(36 BYTE) NOT NULL, - "PK_HOST" VARCHAR2(36 BYTE) NOT NULL, - "INT_MEM_TOTAL" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_MEM_FREE" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_SWAP_TOTAL" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_SWAP_FREE" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_MCP_TOTAL" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_MCP_FREE" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_LOAD" NUMBER(38,0) DEFAULT 0 NOT NULL, - "TS_PING" TIMESTAMP (6) WITH TIME ZONE DEFAULT systimestamp NOT NULL, - "TS_BOOTED" TIMESTAMP (6) WITH TIME ZONE DEFAULT systimestamp NOT NULL, - "STR_STATE" VARCHAR2(32 BYTE) DEFAULT 'Up' NOT NULL, - "STR_OS" VARCHAR2(12 BYTE) DEFAULT 'rhel40' NOT NULL, - "INT_GPU_TOTAL" NUMBER(10,0) DEFAULT 0 NOT NULL, - "INT_GPU_FREE" NUMBER(10,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "HOST" - ( "PK_HOST" VARCHAR2(36 BYTE) NOT NULL, - "PK_ALLOC" VARCHAR2(36 BYTE) NOT NULL, - "STR_NAME" VARCHAR2(30 BYTE) NOT NULL, - "STR_LOCK_STATE" VARCHAR2(36 BYTE) NOT NULL, - "B_NIMBY" NUMBER(1,0) DEFAULT 0 NOT NULL, - "TS_CREATED" TIMESTAMP (6) DEFAULT systimestamp NOT NULL, - "INT_CORES" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_PROCS" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_CORES_IDLE" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_MEM" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_MEM_IDLE" NUMBER(38,0) DEFAULT 0 NOT NULL, - "B_UNLOCK_BOOT" NUMBER(1,0) DEFAULT 0 NOT NULL, - "B_UNLOCK_IDLE" NUMBER(1,0) DEFAULT 0 NOT NULL, - "B_REBOOT_IDLE" NUMBER(1,0) DEFAULT 0 NOT NULL, - "STR_TAGS" VARCHAR2(128 BYTE), - "STR_FQDN" VARCHAR2(128 BYTE), - "B_COMMENT" NUMBER(1,0) DEFAULT 0 NOT NULL, - "INT_THREAD_MODE" NUMBER(1,0) DEFAULT 0 NOT NULL, - "STR_LOCK_SOURCE" VARCHAR2(128 BYTE), - "INT_GPU" NUMBER(10,0) DEFAULT 0 NOT NULL, - "INT_GPU_IDLE" NUMBER(10,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "FRAME" - ( "PK_FRAME" VARCHAR2(36 BYTE) NOT NULL, - "PK_LAYER" VARCHAR2(36 BYTE) NOT NULL, - "PK_JOB" VARCHAR2(36 BYTE) NOT NULL, - "STR_NAME" VARCHAR2(256 BYTE) NOT NULL, - "STR_STATE" VARCHAR2(24 BYTE) NOT NULL, - "INT_NUMBER" NUMBER(38,0) NOT NULL, - "INT_DEPEND_COUNT" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_EXIT_STATUS" NUMBER(38,0) DEFAULT -1 NOT NULL, - "INT_RETRIES" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_MEM_RESERVED" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_MEM_MAX_USED" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_MEM_USED" NUMBER(38,0) DEFAULT 0 NOT NULL, - "INT_DISPATCH_ORDER" NUMBER(38,0) DEFAULT 0 NOT NULL, - "STR_HOST" VARCHAR2(256 BYTE), - "INT_CORES" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_LAYER_ORDER" NUMBER(16,0) NOT NULL, - "TS_STARTED" TIMESTAMP (6) WITH TIME ZONE, - "TS_STOPPED" TIMESTAMP (6) WITH TIME ZONE, - "TS_LAST_RUN" TIMESTAMP (6) WITH TIME ZONE, - "TS_UPDATED" TIMESTAMP (6) WITH TIME ZONE, - "INT_VERSION" NUMBER(16,0) DEFAULT 0, - "STR_CHECKPOINT_STATE" VARCHAR2(12 BYTE) DEFAULT 'Disabled' NOT NULL, - "INT_CHECKPOINT_COUNT" NUMBER(6,0) DEFAULT 0 NOT NULL, - "INT_GPU_RESERVED" NUMBER(10,0) DEFAULT 0 NOT NULL, - "INT_TOTAL_PAST_CORE_TIME" NUMBER(16,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "FOLDER_LEVEL" - ( "PK_FOLDER_LEVEL" VARCHAR2(36 BYTE) NOT NULL, - "PK_FOLDER" VARCHAR2(36 BYTE) NOT NULL, - "INT_LEVEL" NUMBER(38,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "FOLDER" - ( "PK_FOLDER" VARCHAR2(36 BYTE) NOT NULL, - "PK_PARENT_FOLDER" VARCHAR2(36 BYTE), - "PK_SHOW" VARCHAR2(36 BYTE) NOT NULL, - "STR_NAME" VARCHAR2(36 BYTE) NOT NULL, - "INT_PRIORITY" NUMBER(38,0) DEFAULT 1 NOT NULL, - "B_DEFAULT" NUMBER(1,0) DEFAULT 0 NOT NULL, - "PK_DEPT" VARCHAR2(36 BYTE) NOT NULL, - "INT_JOB_MIN_CORES" NUMBER(16,0) DEFAULT -1 NOT NULL, - "INT_JOB_MAX_CORES" NUMBER(16,0) DEFAULT -1 NOT NULL, - "INT_JOB_PRIORITY" NUMBER(16,0) DEFAULT -1 NOT NULL, - "INT_MIN_CORES" NUMBER(16,0) DEFAULT 0 NOT NULL, - "INT_MAX_CORES" NUMBER(16,0) DEFAULT -1 NOT NULL, - "B_EXCLUDE_MANAGED" NUMBER(1,0) DEFAULT 0 NOT NULL, - "F_ORDER" NUMBER(16,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "FILTER" - ( "PK_FILTER" VARCHAR2(36 BYTE) NOT NULL, - "PK_SHOW" VARCHAR2(36 BYTE) NOT NULL, - "STR_NAME" VARCHAR2(128 BYTE) NOT NULL, - "STR_TYPE" VARCHAR2(16 BYTE) NOT NULL, - "F_ORDER" NUMBER(6,2) DEFAULT 0.0 NOT NULL, - "B_ENABLED" NUMBER(1,0) DEFAULT 1 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "DEPEND" - ( "PK_DEPEND" VARCHAR2(36 BYTE) NOT NULL, - "PK_PARENT" VARCHAR2(36 BYTE), - "PK_JOB_DEPEND_ON" VARCHAR2(36 BYTE) NOT NULL, - "PK_JOB_DEPEND_ER" VARCHAR2(36 BYTE) NOT NULL, - "PK_FRAME_DEPEND_ON" VARCHAR2(36 BYTE), - "PK_FRAME_DEPEND_ER" VARCHAR2(36 BYTE), - "PK_LAYER_DEPEND_ON" VARCHAR2(36 BYTE), - "PK_LAYER_DEPEND_ER" VARCHAR2(36 BYTE), - "STR_TYPE" VARCHAR2(36 BYTE) NOT NULL, - "B_ACTIVE" NUMBER(1,0) DEFAULT 1 NOT NULL, - "B_ANY" NUMBER(1,0) DEFAULT 0 NOT NULL, - "TS_CREATED" TIMESTAMP (6) DEFAULT systimestamp NOT NULL, - "TS_SATISFIED" TIMESTAMP (6), - "STR_TARGET" VARCHAR2(20 BYTE) DEFAULT 'Internal' NOT NULL, - "STR_SIGNATURE" VARCHAR2(36 BYTE) NOT NULL, - "B_COMPOSITE" NUMBER(1,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "CONFIG" - ( "PK_CONFIG" VARCHAR2(36 BYTE) NOT NULL, - "STR_KEY" VARCHAR2(36 BYTE) NOT NULL, - "INT_VALUE" NUMBER(38,0) DEFAULT 0, - "LONG_VALUE" NUMBER(38,0) DEFAULT 0, - "STR_VALUE" VARCHAR2(255 BYTE) DEFAULT '', - "B_VALUE" NUMBER(1,0) DEFAULT 0 - ) --- SPLIT HERE! -CREATE TABLE "COMMENTS" - ( "PK_COMMENT" VARCHAR2(36 BYTE) NOT NULL, - "PK_JOB" VARCHAR2(36 BYTE), - "PK_HOST" VARCHAR2(36 BYTE), - "TS_CREATED" TIMESTAMP (6) DEFAULT systimestamp NOT NULL, - "STR_USER" VARCHAR2(36 BYTE) NOT NULL, - "STR_SUBJECT" VARCHAR2(128 BYTE) NOT NULL, - "STR_MESSAGE" VARCHAR2(4000 BYTE) NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "ALLOC" - ( "PK_ALLOC" VARCHAR2(36 BYTE) NOT NULL, - "STR_NAME" VARCHAR2(36 BYTE) NOT NULL, - "B_ALLOW_EDIT" NUMBER(1,0) DEFAULT 1 NOT NULL, - "B_DEFAULT" NUMBER(1,0) DEFAULT 0 NOT NULL, - "STR_TAG" VARCHAR2(24 BYTE), - "B_BILLABLE" NUMBER(1,0) DEFAULT 1 NOT NULL, - "PK_FACILITY" VARCHAR2(36 BYTE) NOT NULL, - "B_ENABLED" NUMBER(1,0) DEFAULT 1 - ) --- SPLIT HERE! -CREATE TABLE "ACTION" - ( "PK_ACTION" VARCHAR2(36 BYTE) NOT NULL, - "PK_FILTER" VARCHAR2(36 BYTE) NOT NULL, - "PK_FOLDER" VARCHAR2(36 BYTE), - "STR_ACTION" VARCHAR2(24 BYTE) NOT NULL, - "STR_VALUE_TYPE" VARCHAR2(24 BYTE) NOT NULL, - "STR_VALUE" VARCHAR2(4000 BYTE), - "INT_VALUE" NUMBER(38,0), - "B_VALUE" NUMBER(1,0), - "TS_CREATED" TIMESTAMP (6) DEFAULT systimestamp NOT NULL, - "FLOAT_VALUE" NUMBER(6,2), - "B_STOP" NUMBER(1,0) DEFAULT 0 NOT NULL - ) --- SPLIT HERE! -CREATE TABLE "REDIRECT" - ( "PK_PROC" VARCHAR2(36 BYTE) NOT NULL, - "STR_GROUP_ID" VARCHAR(36 BYTE) NOT NULL, - "INT_TYPE" NUMBER(38,0) NOT NULL, - "STR_DESTINATION_ID" VARCHAR2(512 BYTE) NOT NULL, - "STR_NAME" VARCHAR2(512 BYTE) NOT NULL, - "LNG_CREATION_TIME" NUMBER(38,0) NOT NULL - ) - - --- SPLIT HERE! - COMMENT ON COLUMN "JOB_HISTORY"."INT_CORE_TIME_SUCCESS" IS 'seconds per core succeeded' --- SPLIT HERE! - COMMENT ON COLUMN "JOB_HISTORY"."INT_CORE_TIME_FAIL" IS 'seconds per core failed' --- SPLIT HERE! - COMMENT ON COLUMN "JOB_HISTORY"."INT_MAX_RSS" IS 'maximum kilobytes of rss memory used by a single frame' --- SPLIT HERE! - COMMENT ON COLUMN "LAYER_HISTORY"."INT_CORE_TIME_SUCCESS" IS 'seconds per core succeeded' --- SPLIT HERE! - COMMENT ON COLUMN "LAYER_HISTORY"."INT_CORE_TIME_FAIL" IS 'seconds per core failed' --- SPLIT HERE! - COMMENT ON COLUMN "LAYER_HISTORY"."INT_MAX_RSS" IS 'maximum kilobytes of rss memory used by a single frame' --- SPLIT HERE! - COMMENT ON COLUMN "FRAME_HISTORY"."INT_MEM_RESERVED" IS 'kilobytes of memory reserved' --- SPLIT HERE! - COMMENT ON COLUMN "FRAME_HISTORY"."INT_MEM_MAX_USED" IS 'maximum kilobytes of rss memory used' --- SPLIT HERE! - COMMENT ON COLUMN "FRAME_HISTORY"."INT_CORES" IS '100 cores per physical core' - --- SPLIT HERE! -CREATE PACKAGE HISTORY AS - - procedure period_shift; - procedure period_shift(piEndDate in date); - procedure period_clear; - -END HISTORY; --- SPLIT HERE! -CREATE FUNCTION CALCULATE_CORE_HOURS -(int_ts_started NUMERIC, int_ts_stopped NUMERIC, -int_start_report NUMERIC, int_stop_report NUMERIC, -int_job_stopped NUMERIC, int_cores NUMBER) -RETURN NUMBER IS - int_started NUMERIC(12,0); - int_stopped NUMERIC(12,0); -BEGIN - IF int_cores = 0 THEN - RETURN 0; - END IF; - - int_started := int_ts_started; - int_stopped := int_ts_stopped; - - IF int_stopped = 0 THEN - int_stopped := int_job_stopped; - END IF; - - IF int_stopped = 0 OR int_stopped > int_stop_report THEN - int_stopped := int_stop_report; - END IF; - - IF int_started < int_start_report THEN - int_started := int_start_report; - END IF; - RETURN ((int_stopped - int_started) * (int_cores / 100) / 3600); -END; --- SPLIT HERE! -CREATE FUNCTION "INTERVAL_TO_SECONDS" -( intrvl IN DSINTERVAL_UNCONSTRAINED -) RETURN NUMBER AS -BEGIN - RETURN EXTRACT(DAY FROM intrvl) * 86400 - + EXTRACT(HOUR FROM intrvl) * 3600 - + EXTRACT(MINUTE FROM intrvl) * 60 - + EXTRACT(SECOND FROM intrvl); -END INTERVAL_TO_SECONDS; --- SPLIT HERE! -CREATE FUNCTION "EPOCH" -( t IN TIMESTAMP WITH TIME ZONE -) RETURN NUMBER AS - epoch_date TIMESTAMP(0) WITH TIME ZONE := TIMESTAMP '1970-01-01 00:00:00.00 +00:00'; - epoch_sec NUMERIC(12,0); - delta INTERVAL DAY(9) TO SECOND(0); -BEGIN - delta := t - epoch_date; - RETURN INTERVAL_TO_SECONDS(delta); -END EPOCH; --- SPLIT HERE! -CREATE FUNCTION epoch_to_ts(seconds IN NUMBER) -RETURN TIMESTAMP AS -BEGIN - RETURN TO_TIMESTAMP('19700101000000','YYYYMMDDHH24MISS TZH:TZM') - + NUMTODSINTERVAL(seconds, 'SECOND'); -END; --- SPLIT HERE! -CREATE FUNCTION FIND_DURATION -(ts_started TIMESTAMP, ts_stopped TIMESTAMP) -RETURN NUMBER IS - t_interval INTERVAL DAY TO SECOND; - t_stopped TIMESTAMP(0); -BEGIN - - IF ts_started IS NULL THEN - RETURN 0; - END IF; - - IF ts_stopped IS NULL THEN - t_stopped := systimestamp; - ELSE - t_stopped := ts_stopped; - END IF; - - t_interval := t_stopped - ts_started; - - RETURN ROUND((EXTRACT(DAY FROM t_interval) * 86400 - + EXTRACT(HOUR FROM t_interval) * 3600 - + EXTRACT(MINUTE FROM t_interval) * 60 - + EXTRACT(SECOND FROM t_interval))); -END; --- SPLIT HERE! -CREATE FUNCTION genkey RETURN VARCHAR2 IS - str_result VARCHAR2(36); - guid VARCHAR2(36) := sys_guid(); -BEGIN - str_result := SUBSTR(guid, 0,8) || '-' || SUBSTR(guid,8,4) - || '-' || SUBSTR(guid,12,4) || '-' || SUBSTR(guid,16,4) || '-' || SUBSTR(guid,20,12); - RETURN str_result; -END; --- SPLIT HERE! - -CREATE FUNCTION render_weeks -(dt_end DATE) -RETURN NUMBER IS - int_weeks NUMERIC; -BEGIN - int_weeks := (dt_end - (next_day(sysdate,'sunday')+7)) / 7.0; - IF int_weeks < 1 THEN - RETURN 1; - ELSE - RETURN int_weeks; - END IF; -END; --- SPLIT HERE! -CREATE FUNCTION soft_tier(int_cores IN NUMERIC, int_min_cores IN NUMERIC) -RETURN NUMBER AS -BEGIN - IF int_cores IS NULL THEN - RETURN 0; - END IF; - IF int_min_cores = 0 OR int_cores >= int_min_cores THEN - RETURN 1; - ELSE - IF int_cores = 0 THEN - return int_min_cores * -1; - ELSE - RETURN int_cores / int_min_cores; - END IF; - END IF; -END; --- SPLIT HERE! -CREATE FUNCTION tier(int_cores IN NUMERIC, int_min_cores IN NUMERIC) -RETURN NUMBER AS -BEGIN - - IF int_min_cores = 0 THEN - RETURN (int_cores / 100) + 1; - ELSE - IF int_cores = 0 THEN - return int_min_cores * -1; - ELSE - RETURN int_cores / int_min_cores; - END IF; - END IF; -END; --- SPLIT HERE! -CREATE PROCEDURE recalculate_subs -IS -BEGIN - /** - * concatenates all tags in host_tag and sets host.str_tags - **/ - UPDATE subscription SET int_cores = 0; - for r in (select proc.pk_show, alloc.pk_alloc, sum(proc.int_cores_reserved) as c from proc, host, alloc - where proc.pk_host = host.pk_host AND host.pk_alloc = alloc.pk_alloc - group by proc.pk_show, alloc.pk_alloc) LOOP - UPDATE subscription SET int_cores = r.c WHERE pk_alloc=r.pk_alloc AND pk_show=r.pk_show; - - END LOOP; -END; --- SPLIT HERE! -CREATE PROCEDURE recalculate_tags(str_host_id IN VARCHAR2) -IS - str_tag VARCHAR2(256) := ''; -BEGIN - /** - * concatenates all tags in host_tag and sets host.str_tags - **/ - FOR tag IN (SELECT str_tag FROM host_tag WHERE pk_host=str_host_id ORDER BY str_tag_type ASC, str_tag ASC) LOOP - str_tag := str_tag || ' ' || tag.str_tag; - END LOOP; - - EXECUTE IMMEDIATE 'UPDATE host SET str_tags=trim(:1) WHERE pk_host=:2' - USING str_tag, str_host_id; -END; --- SPLIT HERE! -CREATE PROCEDURE recurse_folder_parent_change(str_folder_id IN VARCHAR2, str_parent_folder_id IN VARCHAR2) -IS - int_parent_level NUMBER(38); -BEGIN - SELECT int_level+1 INTO - int_parent_level - FROM - folder_level - WHERE - pk_folder = str_parent_folder_id; - - UPDATE - folder_level - SET - int_level = int_parent_level - WHERE - pk_folder = str_folder_id; - - FOR subfolder IN (SELECT pk_folder FROM folder WHERE pk_parent_folder = str_folder_id) LOOP - recurse_folder_parent_change(subfolder.pk_folder, str_folder_id); - END LOOP; -END; --- SPLIT HERE! -CREATE PROCEDURE rename_allocs -IS -BEGIN - FOR alloc IN (SELECT alloc.pk_alloc, alloc.str_name AS aname,facility.str_name AS fname FROM alloc,facility - WHERE alloc.pk_facility = facility.pk_facility) LOOP - EXECUTE IMMEDIATE 'UPDATE alloc SET str_name=:1 WHERE pk_alloc=:2' USING - alloc.fname || '.' || alloc.aname, alloc.pk_alloc; - END LOOP; -END; --- SPLIT HERE! -CREATE PROCEDURE reorder_filters(p_str_show_id IN VARCHAR2) IS - f_new_order NUMBER(16,0) := 1.0; -BEGIN - FOR r_filter IN (SELECT pk_filter FROM filter WHERE pk_show=p_str_show_id ORDER BY f_order ASC) LOOP - UPDATE filter SET f_order=f_new_order WHERE pk_filter = r_filter.pk_filter; - f_new_order := f_new_order + 1.0; - END LOOP; -END; --- SPLIT HERE! -CREATE PROCEDURE tmp_populate_folder IS -BEGIN - FOR t in (select pk_folder, pk_show, sum(int_cores) AS c from job, job_resource where job.pk_job = job_resource.pk_job GROUP by pk_folder, pk_show) LOOP - UPDATE folder_resource SET int_cores = t.c WHERE pk_folder = t.pk_folder; - COMMIT; - END LOOP; -END; --- SPLIT HERE! -CREATE PROCEDURE tmp_populate_point IS -BEGIN - FOR t in (select pk_dept, pk_show, sum(int_cores) AS c from job, job_resource where job.pk_job = job_resource.pk_job GROUP by pk_dept, pk_show) LOOP - UPDATE point SET int_cores = t.c WHERE pk_show = t.pk_show AND pk_dept = t.pk_dept; - END LOOP; -END; --- SPLIT HERE! -CREATE PROCEDURE tmp_populate_sub IS -BEGIN - FOR t in (select proc.pk_show, host.pk_alloc, sum(int_cores_reserved) AS c from proc,host where - proc.pk_host = host.pk_host GROUP BY proc.pk_show, host.pk_alloc) LOOP - UPDATE subscription SET int_cores = t.c WHERE pk_show = t.pk_show AND pk_alloc = t.pk_alloc; - END LOOP; -END; - --- SPLIT HERE! -CREATE UNIQUE INDEX "C_ACTION_PK" ON "ACTION" ("PK_ACTION") --- SPLIT HERE! -CREATE INDEX "I_ACTION_PK_FILTER" ON "ACTION" ("PK_FILTER") --- SPLIT HERE! -CREATE INDEX "I_ACTION_PK_GROUP" ON "ACTION" ("PK_FOLDER") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_ALLOC_PK" ON "ALLOC" ("PK_ALLOC") --- SPLIT HERE! -CREATE INDEX "I_ALLOC_PK_FACILITY" ON "ALLOC" ("PK_FACILITY") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_ALLOC_NAME_UNIQ" ON "ALLOC" ("STR_NAME") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_COMMENT_PK" ON "COMMENTS" ("PK_COMMENT") --- SPLIT HERE! -CREATE INDEX "I_COMMENT_PK_JOB" ON "COMMENTS" ("PK_JOB") --- SPLIT HERE! -CREATE INDEX "I_COMMENT_PK_HOST" ON "COMMENTS" ("PK_HOST") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_PK_PKCONFIG" ON "CONFIG" ("PK_CONFIG") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_SHOW_UK" ON "CONFIG" ("STR_KEY") --- SPLIT HERE! -CREATE UNIQUE INDEX "I_DEPEND_SIGNATURE" ON "DEPEND" ("STR_SIGNATURE") --- SPLIT HERE! -CREATE INDEX "I_DEPEND_ON_LAYER" ON "DEPEND" ("PK_LAYER_DEPEND_ON") --- SPLIT HERE! -CREATE INDEX "I_DEPEND_ER_LAYER" ON "DEPEND" ("PK_LAYER_DEPEND_ER") --- SPLIT HERE! -CREATE INDEX "I_DEPEND_STR_TARGET" ON "DEPEND" ("STR_TARGET") --- SPLIT HERE! -CREATE INDEX "I_DEPEND_ON_FRAME" ON "DEPEND" ("PK_FRAME_DEPEND_ON") --- SPLIT HERE! -CREATE INDEX "I_DEPEND_STR_TYPE" ON "DEPEND" ("STR_TYPE") --- SPLIT HERE! -CREATE INDEX "I_DEPEND_ER_FRAME" ON "DEPEND" ("PK_FRAME_DEPEND_ER") --- SPLIT HERE! -CREATE INDEX "I_DEPEND_B_COMPOSITE" ON "DEPEND" ("B_COMPOSITE") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_DEPEND_PK" ON "DEPEND" ("PK_DEPEND") --- SPLIT HERE! -CREATE INDEX "I_DEPEND_PKPARENT" ON "DEPEND" ("PK_PARENT") --- SPLIT HERE! -CREATE INDEX "I_DEPEND_PK_ON_JOB" ON "DEPEND" ("PK_JOB_DEPEND_ON") --- SPLIT HERE! -CREATE INDEX "I_DEPEND_PK_ER_JOB" ON "DEPEND" ("PK_JOB_DEPEND_ER") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_FILTER_PK" ON "FILTER" ("PK_FILTER") --- SPLIT HERE! -CREATE INDEX "I_FILTERS_PK_SHOW" ON "FILTER" ("PK_SHOW") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_FOLDER_UK" ON "FOLDER" ("PK_PARENT_FOLDER", "STR_NAME") --- SPLIT HERE! -CREATE INDEX "I_FOLDER_PKPARENTFOLDER" ON "FOLDER" ("PK_PARENT_FOLDER") --- SPLIT HERE! -CREATE INDEX "I_FOLDER_PKSHOW" ON "FOLDER" ("PK_SHOW") --- SPLIT HERE! -CREATE INDEX "I_FOLDER_STRNAME" ON "FOLDER" ("STR_NAME") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_FOLDER_PK" ON "FOLDER" ("PK_FOLDER") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_FOLDER_LEVEL_PK" ON "FOLDER_LEVEL" ("PK_FOLDER_LEVEL") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_FOLDER_LEVEL_UK" ON "FOLDER_LEVEL" ("PK_FOLDER") --- SPLIT HERE! -CREATE INDEX "I_FRAME_STATE_JOB" ON "FRAME" ("STR_STATE", "PK_JOB") --- SPLIT HERE! -CREATE INDEX "I_FRAME_DISPATCH_IDX" ON "FRAME" ("INT_DISPATCH_ORDER", "INT_LAYER_ORDER") --- SPLIT HERE! -CREATE INDEX "I_FRAME_PK_JOB" ON "FRAME" ("PK_JOB") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_FRAME_PK" ON "FRAME" ("PK_FRAME") --- SPLIT HERE! -CREATE INDEX "I_FRAME_PKJOBLAYER" ON "FRAME" ("PK_LAYER") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_FRAME_STR_NAME_UNQ" ON "FRAME" ("STR_NAME", "PK_JOB") --- SPLIT HERE! -CREATE INDEX "I_FRAME_INT_GPU_RESERVED" ON "FRAME" ("INT_GPU_RESERVED") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_STR_HOST_FQDN_UK" ON "HOST" ("STR_FQDN") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_HOST_PK" ON "HOST" ("PK_HOST") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_HOST_UK" ON "HOST" ("STR_NAME") --- SPLIT HERE! -CREATE INDEX "I_HOST_PKALLOC" ON "HOST" ("PK_ALLOC") --- SPLIT HERE! -CREATE INDEX "I_HOST_STRLOCKSTATE" ON "HOST" ("STR_LOCK_STATE") --- SPLIT HERE! -CREATE INDEX "I_HOST_INT_GPU" ON "HOST" ("INT_GPU") --- SPLIT HERE! -CREATE INDEX "I_HOST_INT_GPU_IDLE" ON "HOST" ("INT_GPU_IDLE") --- SPLIT HERE! -CREATE INDEX "I_HOST_STAT_INT_GPU_TOTAL" ON "HOST_STAT" ("INT_GPU_TOTAL") --- SPLIT HERE! -CREATE INDEX "I_HOST_STAT_INT_GPU_FREE" ON "HOST_STAT" ("INT_GPU_FREE") --- SPLIT HERE! -CREATE INDEX "I_HOST_STAT_STR_OS" ON "HOST_STAT" ("STR_OS") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_HOSTSTAT_PK" ON "HOST_STAT" ("PK_HOST_STAT") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_HOST_STAT_PK_HOST_UK" ON "HOST_STAT" ("PK_HOST") --- SPLIT HERE! -CREATE INDEX "I_BOOKING_3" ON "JOB" ("STR_STATE", "B_PAUSED", "PK_SHOW", "PK_FACILITY") --- SPLIT HERE! -CREATE INDEX "I_JOB_STR_OS" ON "JOB" ("STR_OS") --- SPLIT HERE! -CREATE INDEX "I_JOB_PK_DEPT" ON "JOB" ("PK_DEPT") --- SPLIT HERE! -CREATE INDEX "I_JOB_PK_FACILITY" ON "JOB" ("PK_FACILITY") --- SPLIT HERE! -CREATE INDEX "I_JOB_STR_SHOT" ON "JOB" ("STR_SHOT") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_JOB_PK" ON "JOB" ("PK_JOB") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_JOB_UK" ON "JOB" ("STR_VISIBLE_NAME") --- SPLIT HERE! -CREATE INDEX "I_JOB_PKGROUP" ON "JOB" ("PK_FOLDER") --- SPLIT HERE! -CREATE INDEX "I_JOB_PKSHOW" ON "JOB" ("PK_SHOW") --- SPLIT HERE! -CREATE INDEX "I_JOB_STR_NAME" ON "JOB" ("STR_NAME") --- SPLIT HERE! -CREATE INDEX "I_JOB_STR_STATE" ON "JOB" ("STR_STATE") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_JOB_ENV_PK" ON "JOB_ENV" ("PK_JOB_ENV") --- SPLIT HERE! -CREATE INDEX "I_JOB_ENV_PK_JOB" ON "JOB_ENV" ("PK_JOB") --- SPLIT HERE! -CREATE INDEX "I_LAYER_B_THREADABLE" ON "LAYER" ("B_THREADABLE") --- SPLIT HERE! -CREATE INDEX "I_LAYER_CORES_MEM" ON "LAYER" ("INT_CORES_MIN", "INT_MEM_MIN") --- SPLIT HERE! -CREATE INDEX "I_LAYER_CORES_MEM_THREAD" ON "LAYER" ("INT_CORES_MIN", "INT_MEM_MIN", "B_THREADABLE") --- SPLIT HERE! -CREATE INDEX "I_LAYER_MEM_MIN" ON "LAYER" ("INT_MEM_MIN") --- SPLIT HERE! -CREATE INDEX "I_LAYER_INT_DISPATCH_ORDER" ON "LAYER" ("INT_DISPATCH_ORDER") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_LAYER_PK" ON "LAYER" ("PK_LAYER") --- SPLIT HERE! -CREATE INDEX "I_LAYER_PKJOB" ON "LAYER" ("PK_JOB") --- SPLIT HERE! -CREATE INDEX "I_LAYER_STRNAME" ON "LAYER" ("STR_NAME") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_LAYER_STR_NAME_UNQ" ON "LAYER" ("STR_NAME", "PK_JOB") --- SPLIT HERE! -CREATE INDEX "I_LAYER_INT_GPU_MIN" ON "LAYER" ("INT_GPU_MIN") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_LAYER_ENV_PK" ON "LAYER_ENV" ("PK_LAYER_ENV") --- SPLIT HERE! -CREATE INDEX "I_LAYER_ENV_PK_JOB" ON "LAYER_ENV" ("PK_JOB") --- SPLIT HERE! -CREATE INDEX "I_LAYER_ENV_PK_LAYER" ON "LAYER_ENV" ("PK_LAYER") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_LAYERRESOURCE_PK" ON "LAYER_RESOURCE" ("PK_LAYER_RESOURCE") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_LAYERRESOURCE_UK" ON "LAYER_RESOURCE" ("PK_LAYER") --- SPLIT HERE! -CREATE INDEX "I_LAYER_RESOURCE_PK_JOB" ON "LAYER_RESOURCE" ("PK_JOB") --- SPLIT HERE! -CREATE UNIQUE INDEX "I_LAYER_STAT_PK_LAYER" ON "LAYER_STAT" ("PK_LAYER") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_LAYERSTAT_PK" ON "LAYER_STAT" ("PK_LAYER_STAT") --- SPLIT HERE! -CREATE INDEX "I_LAYERSTAT_PKJOB" ON "LAYER_STAT" ("PK_JOB") --- SPLIT HERE! -CREATE INDEX "I_LAYERSTAT_INT_WAITING_COUNT" ON "LAYER_STAT" (CASE WHEN "INT_WAITING_COUNT" > 0 THEN 1 ELSE NULL END, CASE WHEN "INT_WAITING_COUNT" > 0 THEN "PK_LAYER" ELSE NULL END) --- SPLIT HERE! -CREATE INDEX "I_LAYER_USAGE_PK_JOB" ON "LAYER_USAGE" ("PK_JOB") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_LAYER_USAGE_PK" ON "LAYER_USAGE" ("PK_LAYER_USAGE") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_LAYER_USAGE_PK_LAYER_UK" ON "LAYER_USAGE" ("PK_LAYER") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_MATCHER_PK" ON "MATCHER" ("PK_MATCHER") --- SPLIT HERE! -CREATE INDEX "I_MATCHER_PK_FILTER" ON "MATCHER" ("PK_FILTER") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_PROC_PK" ON "PROC" ("PK_PROC") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_PROC_UK" ON "PROC" ("PK_FRAME") --- SPLIT HERE! -CREATE INDEX "I_PROC_PKHOST" ON "PROC" ("PK_HOST") --- SPLIT HERE! -CREATE INDEX "I_PROC_PKJOB" ON "PROC" ("PK_JOB") --- SPLIT HERE! -CREATE INDEX "I_PROC_PKLAYER" ON "PROC" ("PK_LAYER") --- SPLIT HERE! -CREATE INDEX "I_PROC_PKSHOW" ON "PROC" ("PK_SHOW") --- SPLIT HERE! -CREATE INDEX "I_PROC_INT_GPU_RESERVED" ON "PROC" ("INT_GPU_RESERVED") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_SHOW_PK" ON "SHOW" ("PK_SHOW") --- SPLIT HERE! -CREATE INDEX "I_SUB_TIER" ON "SUBSCRIPTION" ("FLOAT_TIER") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_SUBSCRIPTION_PK" ON "SUBSCRIPTION" ("PK_SUBSCRIPTION") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_SUBSCRIPTION_UK" ON "SUBSCRIPTION" ("PK_SHOW", "PK_ALLOC") --- SPLIT HERE! -CREATE INDEX "I_SUBSCRIPTION_PKALLOC" ON "SUBSCRIPTION" ("PK_ALLOC") --- SPLIT HERE! -CREATE INDEX "I_JOB_STAT_INT_WAITING_COUNT" ON "JOB_STAT" ("INT_WAITING_COUNT") --- SPLIT HERE! -CREATE UNIQUE INDEX "I_JOB_STAT_PK_JOB" ON "JOB_STAT" ("PK_JOB") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_JOB_STAT_PK" ON "JOB_STAT" ("PK_JOB_STAT") --- SPLIT HERE! -CREATE INDEX "I_JOB_RESOURCE_MIN_MAX" ON "JOB_RESOURCE" ("INT_MIN_CORES", "INT_MAX_CORES") --- SPLIT HERE! -CREATE INDEX "I_JOB_TIER" ON "JOB_RESOURCE" ("FLOAT_TIER") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_JOB_RESOURCE_PK" ON "JOB_RESOURCE" ("PK_JOB_RESOURCE") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_JOB_RESOURCE_UK" ON "JOB_RESOURCE" ("PK_JOB") --- SPLIT HERE! -CREATE INDEX "I_JOB_RESOURCE_CORES" ON "JOB_RESOURCE" ("INT_CORES") --- SPLIT HERE! -CREATE INDEX "I_JOB_RESOURCE_MAX_C" ON "JOB_RESOURCE" ("INT_MAX_CORES") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_JOB_USAGE_PK" ON "JOB_USAGE" ("PK_JOB_USAGE") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_JOB_USAGE_PK_JOB_UNIQ" ON "JOB_USAGE" ("PK_JOB") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_HOST_TAG_PK" ON "HOST_TAG" ("PK_HOST_TAG") --- SPLIT HERE! -CREATE INDEX "I_HOST_TAG_PK_HOST" ON "HOST_TAG" ("PK_HOST") --- SPLIT HERE! -CREATE INDEX "I_HOST_STR_TAG_TYPE" ON "HOST_TAG" ("STR_TAG_TYPE") --- SPLIT HERE! -CREATE INDEX "MATTHEW_STATS_TAB" ON "MATTHEW_STATS_TAB" ("STATID", "TYPE", "C5", "C1", "C2", "C3", "C4", "VERSION") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_TASK_LOCK_PK" ON "TASK_LOCK" ("PK_TASK_LOCK") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_JOB_HISTORY_PK" ON "JOB_HISTORY" ("PK_JOB") --- SPLIT HERE! -CREATE INDEX "I_JOB_HISTORY_PK_SHOW" ON "JOB_HISTORY" ("PK_SHOW") --- SPLIT HERE! -CREATE INDEX "I_JOB_HISTORY_B_ARCHIVED" ON "JOB_HISTORY" ("B_ARCHIVED") --- SPLIT HERE! -CREATE INDEX "I_JOB_HISTORY_TS_START_STOP" ON "JOB_HISTORY" ("INT_TS_STARTED", "INT_TS_STOPPED") --- SPLIT HERE! -CREATE INDEX "I_JOB_HISTORY_STR_NAME" ON "JOB_HISTORY" ("STR_NAME") --- SPLIT HERE! -CREATE INDEX "I_JOB_HISTORY_STR_SHOT" ON "JOB_HISTORY" ("STR_SHOT") --- SPLIT HERE! -CREATE INDEX "I_JOB_HISTORY_STR_USER" ON "JOB_HISTORY" ("STR_USER") --- SPLIT HERE! -CREATE INDEX "I_JOB_HISTORY_PK_DEPT" ON "JOB_HISTORY" ("PK_DEPT") --- SPLIT HERE! -CREATE INDEX "I_JOB_HISTORY_PK_FACILITY" ON "JOB_HISTORY" ("PK_FACILITY") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_LAYER_HISTORY_PK" ON "LAYER_HISTORY" ("PK_LAYER") --- SPLIT HERE! -CREATE INDEX "I_LAYER_HISTORY_STR_NAME" ON "LAYER_HISTORY" ("STR_NAME") --- SPLIT HERE! -CREATE INDEX "I_LAYER_HISTORY_STR_TYPE" ON "LAYER_HISTORY" ("STR_TYPE") --- SPLIT HERE! -CREATE INDEX "I_LAYER_HISTORY_PK_JOB" ON "LAYER_HISTORY" ("PK_JOB") --- SPLIT HERE! -CREATE INDEX "I_LAYER_HISTORY_B_ARCHIVED" ON "LAYER_HISTORY" ("B_ARCHIVED") --- SPLIT HERE! -CREATE INDEX "I_JOB_POST_PK_POST_JOB" ON "JOB_POST" ("PK_POST_JOB") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_JOB_POST_PK" ON "JOB_POST" ("PK_JOB_POST") --- SPLIT HERE! -CREATE INDEX "I_JOB_POST_PK_JOB" ON "JOB_POST" ("PK_JOB") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_FACILITY_PK" ON "FACILITY" ("PK_FACILITY") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_DEPT_PK" ON "DEPT" ("PK_DEPT") --- SPLIT HERE! -CREATE INDEX "I_SHOW_ALIAS_PK_SHOW" ON "SHOW_ALIAS" ("PK_SHOW") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_SHOW_ALIAS_PK" ON "SHOW_ALIAS" ("PK_SHOW_ALIAS") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_FOLDER_RESOURCE_PK" ON "FOLDER_RESOURCE" ("PK_FOLDER_RESOURCE") --- SPLIT HERE! -CREATE INDEX "I_FOLDER_RES_INT_MAX_CORES" ON "FOLDER_RESOURCE" ("INT_MAX_CORES") --- SPLIT HERE! -CREATE INDEX "I_FOLDER_RESOURCE_FL_TIER" ON "FOLDER_RESOURCE" ("FLOAT_TIER") --- SPLIT HERE! -CREATE INDEX "I_FOLDERRESOURCE_PKFOLDER" ON "FOLDER_RESOURCE" ("PK_FOLDER") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_JOB_MEM_PK" ON "JOB_MEM" ("PK_JOB_MEM") --- SPLIT HERE! -CREATE UNIQUE INDEX "I_JOB_MEM_PK_JOB" ON "JOB_MEM" ("PK_JOB") --- SPLIT HERE! -CREATE INDEX "I_JOB_MEM_INT_MAX_RSS" ON "JOB_MEM" ("INT_MAX_RSS") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_LAYER_MEM_PK" ON "LAYER_MEM" ("PK_LAYER_MEM") --- SPLIT HERE! -CREATE UNIQUE INDEX "I_LAYER_MEM_PK_LAYER" ON "LAYER_MEM" ("PK_LAYER") --- SPLIT HERE! -CREATE INDEX "I_LAYER_MEM_PK_JOB" ON "LAYER_MEM" ("PK_JOB") --- SPLIT HERE! -CREATE INDEX "I_LAYER_MEM_INT_MAX_RSS" ON "LAYER_MEM" ("INT_MAX_RSS") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_POINT_PK" ON "POINT" ("PK_POINT") --- SPLIT HERE! -CREATE INDEX "I_POINT_PK_DEPT" ON "POINT" ("PK_DEPT") --- SPLIT HERE! -CREATE INDEX "I_POINT_PK_SHOW" ON "POINT" ("PK_SHOW") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_POINT_PK_SHOW_DEPT" ON "POINT" ("PK_SHOW", "PK_DEPT") --- SPLIT HERE! -CREATE INDEX "I_POINT_TIER" ON "POINT" ("FLOAT_TIER") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_TASK_PK" ON "TASK" ("PK_TASK") --- SPLIT HERE! -CREATE INDEX "I_TASK_PK_POINT" ON "TASK" ("PK_POINT") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_TASK_UNIQ" ON "TASK" ("STR_SHOT", "PK_POINT") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_PK_JOB_LOCAL" ON "JOB_LOCAL" ("PK_JOB_LOCAL") --- SPLIT HERE! -CREATE UNIQUE INDEX "I_JOB_LOCAL_PK_JOB" ON "JOB_LOCAL" ("PK_JOB") --- SPLIT HERE! -CREATE UNIQUE INDEX "I_JOB_LOCAL_PK_HOST" ON "JOB_LOCAL" ("PK_HOST") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_PK_SERVICE" ON "SERVICE" ("PK_SERVICE") --- SPLIT HERE! -CREATE UNIQUE INDEX "I_SERVICE_STR_NAME" ON "SERVICE" ("STR_NAME") --- SPLIT HERE! -CREATE INDEX "I_SERVICE_INT_GPU_MIN" ON "SERVICE" ("INT_GPU_MIN") --- SPLIT HERE! -CREATE INDEX "I_HOST_LOCAL" ON "HOST_LOCAL" ("PK_HOST") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_PK_HOST_LOCAL" ON "HOST_LOCAL" ("PK_HOST_LOCAL") --- SPLIT HERE! -CREATE INDEX "I_HOST_LOCAL_PK_JOB" ON "HOST_LOCAL" ("PK_JOB") --- SPLIT HERE! -CREATE UNIQUE INDEX "I_HOST_LOCAL_UNIQUE" ON "HOST_LOCAL" ("PK_HOST", "PK_JOB") --- SPLIT HERE! -CREATE INDEX "I_HOST_LOCAL_INT_GPU_IDLE" ON "HOST_LOCAL" ("INT_GPU_IDLE") --- SPLIT HERE! -CREATE INDEX "I_HOST_LOCAL_INT_GPU_MAX" ON "HOST_LOCAL" ("INT_GPU_MAX") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_PK_OWNER" ON "OWNER" ("PK_OWNER") --- SPLIT HERE! -CREATE INDEX "I_OWNER_PK_SHOW" ON "OWNER" ("PK_SHOW") --- SPLIT HERE! -CREATE UNIQUE INDEX "I_OWNER_STR_USERNAME" ON "OWNER" ("STR_USERNAME") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_PK_DEED" ON "DEED" ("PK_DEED") --- SPLIT HERE! -CREATE UNIQUE INDEX "I_DEED_PK_HOST" ON "DEED" ("PK_HOST") --- SPLIT HERE! -CREATE INDEX "I_DEED_PK_OWNER" ON "DEED" ("PK_OWNER") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_PK_SHOW_SERVICE" ON "SHOW_SERVICE" ("PK_SHOW_SERVICE") --- SPLIT HERE! -CREATE UNIQUE INDEX "I_SHOW_SERVICE_STR_NAME" ON "SHOW_SERVICE" ("STR_NAME", "PK_SHOW") --- SPLIT HERE! -CREATE INDEX "I_SHOW_SERVICE_INT_GPU_MIN" ON "SHOW_SERVICE" ("INT_GPU_MIN") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_PK_LAYER_OUTPUT" ON "LAYER_OUTPUT" ("PK_LAYER_OUTPUT") --- SPLIT HERE! -CREATE INDEX "I_LAYER_OUTPUT_PK_LAYER" ON "LAYER_OUTPUT" ("PK_LAYER") --- SPLIT HERE! -CREATE INDEX "I_LAYER_OUTPUT_PK_JOB" ON "LAYER_OUTPUT" ("PK_JOB") --- SPLIT HERE! -CREATE UNIQUE INDEX "I_LAYER_OUTPUT_UNIQUE" ON "LAYER_OUTPUT" ("PK_LAYER", "STR_FILESPEC") --- SPLIT HERE! - -CREATE INDEX "I_FRAME_HISTORY_TS_START_STOP" ON "FRAME_HISTORY" ("INT_TS_STARTED", "INT_TS_STOPPED") --- SPLIT HERE! - -CREATE INDEX "I_FRAME_HISTORY_INT_EXIT_STAT" ON "FRAME_HISTORY" ("INT_EXIT_STATUS") --- SPLIT HERE! -CREATE INDEX "I_FRAME_HISTORY_INT_TS_STOPPED" ON "FRAME_HISTORY" ("INT_TS_STOPPED") --- SPLIT HERE! -CREATE INDEX "I_FRAME_HISTORY_PK_ALLOC" ON "FRAME_HISTORY" ("PK_ALLOC") --- SPLIT HERE! -CREATE INDEX "I_FRAME_HISTORY_PK_FRAME" ON "FRAME_HISTORY" ("PK_FRAME") --- SPLIT HERE! -CREATE INDEX "I_FRAME_HISTORY_PK_JOB" ON "FRAME_HISTORY" ("PK_JOB") --- SPLIT HERE! -CREATE INDEX "I_FRAME_HISTORY_PK_LAYER" ON "FRAME_HISTORY" ("PK_LAYER") --- SPLIT HERE! -CREATE INDEX "I_FRAME_HISTORY_STR_STATE" ON "FRAME_HISTORY" ("STR_STATE") --- SPLIT HERE! -CREATE UNIQUE INDEX "C_REDIRECT_PK" ON "REDIRECT" ("PK_PROC") --- SPLIT HERE! -CREATE INDEX "I_REDIRECT_GROUP" ON "REDIRECT" ("STR_GROUP_ID") --- SPLIT HERE! -CREATE INDEX "I_REDIRECT_CREATE" ON "REDIRECT" ("LNG_CREATION_TIME") --- SPLIT HERE! - -ALTER TABLE "ACTION" ADD CONSTRAINT "C_ACTION_PK" PRIMARY KEY ("PK_ACTION") - USING INDEX "C_ACTION_PK" --- SPLIT HERE! -ALTER TABLE "ALLOC" ADD CONSTRAINT "C_ALLOC_NAME_UNIQ" UNIQUE ("STR_NAME") - USING INDEX "C_ALLOC_NAME_UNIQ" --- SPLIT HERE! -ALTER TABLE "ALLOC" ADD CONSTRAINT "C_ALLOC_PK" PRIMARY KEY ("PK_ALLOC") - USING INDEX "C_ALLOC_PK" --- SPLIT HERE! -ALTER TABLE "COMMENTS" ADD CONSTRAINT "C_COMMENT_PK" PRIMARY KEY ("PK_COMMENT") - USING INDEX "C_COMMENT_PK" --- SPLIT HERE! -ALTER TABLE "CONFIG" ADD CONSTRAINT "C_PK_PKCONFIG" PRIMARY KEY ("PK_CONFIG") - USING INDEX "C_PK_PKCONFIG" --- SPLIT HERE! -ALTER TABLE "CONFIG" ADD CONSTRAINT "C_SHOW_UK" UNIQUE ("STR_KEY") - USING INDEX "C_SHOW_UK" --- SPLIT HERE! -ALTER TABLE "DEPEND" ADD CONSTRAINT "C_DEPEND_PK" PRIMARY KEY ("PK_DEPEND") - USING INDEX "C_DEPEND_PK" --- SPLIT HERE! -ALTER TABLE "FILTER" ADD CONSTRAINT "C_FILTER_PK" PRIMARY KEY ("PK_FILTER") - USING INDEX "C_FILTER_PK" --- SPLIT HERE! -ALTER TABLE "FOLDER" ADD CONSTRAINT "C_FOLDER_UK" UNIQUE ("PK_PARENT_FOLDER", "STR_NAME") - USING INDEX "C_FOLDER_UK" --- SPLIT HERE! -ALTER TABLE "FOLDER" ADD CONSTRAINT "C_FOLDER_PK" PRIMARY KEY ("PK_FOLDER") - USING INDEX "C_FOLDER_PK" --- SPLIT HERE! -ALTER TABLE "FOLDER_LEVEL" ADD CONSTRAINT "C_FOLDER_LEVEL_PK" PRIMARY KEY ("PK_FOLDER_LEVEL") - USING INDEX "C_FOLDER_LEVEL_PK" --- SPLIT HERE! -ALTER TABLE "FOLDER_LEVEL" ADD CONSTRAINT "C_FOLDER_LEVEL_UK" UNIQUE ("PK_FOLDER") - USING INDEX "C_FOLDER_LEVEL_UK" --- SPLIT HERE! -ALTER TABLE "FRAME" ADD CONSTRAINT "C_FRAME_PK" PRIMARY KEY ("PK_FRAME") - USING INDEX "C_FRAME_PK" --- SPLIT HERE! -ALTER TABLE "FRAME" ADD CONSTRAINT "C_FRAME_STR_NAME_UNQ" UNIQUE ("STR_NAME", "PK_JOB") - USING INDEX "C_FRAME_STR_NAME_UNQ" --- SPLIT HERE! -ALTER TABLE "HOST" ADD CONSTRAINT "C_STR_HOST_FQDN_UK" UNIQUE ("STR_FQDN") - USING INDEX "C_STR_HOST_FQDN_UK" --- SPLIT HERE! -ALTER TABLE "HOST" ADD CONSTRAINT "C_HOST_PK" PRIMARY KEY ("PK_HOST") - USING INDEX "C_HOST_PK" --- SPLIT HERE! -ALTER TABLE "HOST" ADD CONSTRAINT "C_HOST_UK" UNIQUE ("STR_NAME") - USING INDEX "C_HOST_UK" --- SPLIT HERE! -ALTER TABLE "HOST_STAT" ADD CONSTRAINT "C_HOSTSTAT_PK" PRIMARY KEY ("PK_HOST_STAT") - USING INDEX "C_HOSTSTAT_PK" --- SPLIT HERE! -ALTER TABLE "HOST_STAT" ADD CONSTRAINT "C_HOST_STAT_PK_HOST_UK" UNIQUE ("PK_HOST") - USING INDEX "C_HOST_STAT_PK_HOST_UK" --- SPLIT HERE! -ALTER TABLE "JOB" ADD CONSTRAINT "C_JOB_PK" PRIMARY KEY ("PK_JOB") - USING INDEX "C_JOB_PK" --- SPLIT HERE! -ALTER TABLE "JOB" ADD CONSTRAINT "C_JOB_UK" UNIQUE ("STR_VISIBLE_NAME") - USING INDEX "C_JOB_UK" --- SPLIT HERE! -ALTER TABLE "JOB_ENV" ADD CONSTRAINT "C_JOB_ENV_PK" PRIMARY KEY ("PK_JOB_ENV") - USING INDEX "C_JOB_ENV_PK" --- SPLIT HERE! -ALTER TABLE "JOB_LOCAL" ADD CONSTRAINT "C_PK_JOB_LOCAL" PRIMARY KEY ("PK_JOB_LOCAL") - USING INDEX "C_PK_JOB_LOCAL" --- SPLIT HERE! -ALTER TABLE "SERVICE" ADD CONSTRAINT "C_PK_SERVICE" PRIMARY KEY ("PK_SERVICE") - USING INDEX "C_PK_SERVICE" --- SPLIT HERE! -ALTER TABLE "HOST_LOCAL" ADD CONSTRAINT "C_PK_HOST_LOCAL" PRIMARY KEY ("PK_HOST_LOCAL") - USING INDEX "C_PK_HOST_LOCAL" --- SPLIT HERE! -ALTER TABLE "OWNER" ADD CONSTRAINT "C_PK_OWNER" PRIMARY KEY ("PK_OWNER") - USING INDEX "C_PK_OWNER" --- SPLIT HERE! -ALTER TABLE "DEED" ADD CONSTRAINT "C_PK_DEED" PRIMARY KEY ("PK_DEED") - USING INDEX "C_PK_DEED" --- SPLIT HERE! -ALTER TABLE "SHOW_SERVICE" ADD CONSTRAINT "C_PK_SHOW_SERVICE" PRIMARY KEY ("PK_SHOW_SERVICE") - USING INDEX "C_PK_SHOW_SERVICE" --- SPLIT HERE! -ALTER TABLE "LAYER_OUTPUT" ADD CONSTRAINT "C_PK_LAYER_OUTPUT" PRIMARY KEY ("PK_LAYER_OUTPUT") - USING INDEX "C_PK_LAYER_OUTPUT" --- SPLIT HERE! -ALTER TABLE "LAYER" ADD CONSTRAINT "C_LAYER_PK" PRIMARY KEY ("PK_LAYER") - USING INDEX "C_LAYER_PK" --- SPLIT HERE! -ALTER TABLE "LAYER" ADD CONSTRAINT "C_LAYER_STR_NAME_UNQ" UNIQUE ("STR_NAME", "PK_JOB") - USING INDEX "C_LAYER_STR_NAME_UNQ" --- SPLIT HERE! -ALTER TABLE "LAYER_ENV" ADD CONSTRAINT "C_LAYER_ENV_PK" PRIMARY KEY ("PK_LAYER_ENV") - USING INDEX "C_LAYER_ENV_PK" --- SPLIT HERE! -ALTER TABLE "LAYER_RESOURCE" ADD CONSTRAINT "C_LAYERRESOURCE_PK" PRIMARY KEY ("PK_LAYER_RESOURCE") - USING INDEX "C_LAYERRESOURCE_PK" --- SPLIT HERE! -ALTER TABLE "LAYER_RESOURCE" ADD CONSTRAINT "C_LAYERRESOURCE_UK" UNIQUE ("PK_LAYER") - USING INDEX "C_LAYERRESOURCE_UK" --- SPLIT HERE! -ALTER TABLE "LAYER_STAT" ADD CONSTRAINT "C_LAYERSTAT_PK" PRIMARY KEY ("PK_LAYER_STAT") - USING INDEX "C_LAYERSTAT_PK" --- SPLIT HERE! -ALTER TABLE "LAYER_USAGE" ADD CONSTRAINT "C_LAYER_USAGE_PK" PRIMARY KEY ("PK_LAYER_USAGE") - USING INDEX "C_LAYER_USAGE_PK" --- SPLIT HERE! -ALTER TABLE "LAYER_USAGE" ADD CONSTRAINT "C_LAYER_USAGE_PK_LAYER_UK" UNIQUE ("PK_LAYER") - USING INDEX "C_LAYER_USAGE_PK_LAYER_UK" --- SPLIT HERE! -ALTER TABLE "MATCHER" ADD CONSTRAINT "C_MATCHER_PK" PRIMARY KEY ("PK_MATCHER") - USING INDEX "C_MATCHER_PK" --- SPLIT HERE! -ALTER TABLE "PROC" ADD CONSTRAINT "C_PROC_PK" PRIMARY KEY ("PK_PROC") - USING INDEX "C_PROC_PK" --- SPLIT HERE! -ALTER TABLE "PROC" ADD CONSTRAINT "C_PROC_UK" UNIQUE ("PK_FRAME") - USING INDEX "C_PROC_UK" --- SPLIT HERE! -ALTER TABLE "SHOW" ADD CONSTRAINT "C_SHOW_PK" PRIMARY KEY ("PK_SHOW") - USING INDEX "C_SHOW_PK" --- SPLIT HERE! -ALTER TABLE "SUBSCRIPTION" ADD CONSTRAINT "C_SUBSCRIPTION_PK" PRIMARY KEY ("PK_SUBSCRIPTION") - USING INDEX "C_SUBSCRIPTION_PK" --- SPLIT HERE! -ALTER TABLE "SUBSCRIPTION" ADD CONSTRAINT "C_SUBSCRIPTION_UK" UNIQUE ("PK_SHOW", "PK_ALLOC") - USING INDEX "C_SUBSCRIPTION_UK" --- SPLIT HERE! -ALTER TABLE "JOB_STAT" ADD CONSTRAINT "C_JOB_STAT_PK" PRIMARY KEY ("PK_JOB_STAT") - USING INDEX "C_JOB_STAT_PK" --- SPLIT HERE! -ALTER TABLE "JOB_RESOURCE" ADD CONSTRAINT "C_JOB_RESOURCE_PK" PRIMARY KEY ("PK_JOB_RESOURCE") - USING INDEX "C_JOB_RESOURCE_PK" --- SPLIT HERE! -ALTER TABLE "JOB_RESOURCE" ADD CONSTRAINT "C_JOB_RESOURCE_UK" UNIQUE ("PK_JOB") - USING INDEX "C_JOB_RESOURCE_UK" --- SPLIT HERE! -ALTER TABLE "JOB_USAGE" ADD CONSTRAINT "C_JOB_USAGE_PK" PRIMARY KEY ("PK_JOB_USAGE") - USING INDEX "C_JOB_USAGE_PK" --- SPLIT HERE! -ALTER TABLE "JOB_USAGE" ADD CONSTRAINT "C_JOB_USAGE_PK_JOB_UNIQ" UNIQUE ("PK_JOB") - USING INDEX "C_JOB_USAGE_PK_JOB_UNIQ" --- SPLIT HERE! -ALTER TABLE "HOST_TAG" ADD CONSTRAINT "C_HOST_TAG_PK" PRIMARY KEY ("PK_HOST_TAG") - USING INDEX "C_HOST_TAG_PK" --- SPLIT HERE! -ALTER TABLE "TASK_LOCK" ADD CONSTRAINT "C_TASK_LOCK_PK" PRIMARY KEY ("PK_TASK_LOCK") - USING INDEX "C_TASK_LOCK_PK" --- SPLIT HERE! -ALTER TABLE "JOB_HISTORY" ADD CONSTRAINT "C_JOB_HISTORY_PK" PRIMARY KEY ("PK_JOB") - USING INDEX "C_JOB_HISTORY_PK" --- SPLIT HERE! -ALTER TABLE "LAYER_HISTORY" ADD CONSTRAINT "C_LAYER_HISTORY_PK" PRIMARY KEY ("PK_LAYER") - USING INDEX "C_LAYER_HISTORY_PK" --- SPLIT HERE! -ALTER TABLE "JOB_POST" ADD CONSTRAINT "C_JOB_POST_PK" PRIMARY KEY ("PK_JOB_POST") - USING INDEX "C_JOB_POST_PK" --- SPLIT HERE! -ALTER TABLE "FACILITY" ADD CONSTRAINT "C_FACILITY_PK" PRIMARY KEY ("PK_FACILITY") - USING INDEX "C_FACILITY_PK" --- SPLIT HERE! -ALTER TABLE "DEPT" ADD CONSTRAINT "C_DEPT_PK" PRIMARY KEY ("PK_DEPT") - USING INDEX "C_DEPT_PK" --- SPLIT HERE! -ALTER TABLE "SHOW_ALIAS" ADD CONSTRAINT "C_SHOW_ALIAS_PK" PRIMARY KEY ("PK_SHOW_ALIAS") - USING INDEX "C_SHOW_ALIAS_PK" --- SPLIT HERE! -ALTER TABLE "FOLDER_RESOURCE" ADD CONSTRAINT "C_FOLDER_RESOURCE_PK" PRIMARY KEY ("PK_FOLDER_RESOURCE") - USING INDEX "C_FOLDER_RESOURCE_PK" --- SPLIT HERE! -ALTER TABLE "JOB_MEM" ADD CONSTRAINT "C_JOB_MEM_PK" PRIMARY KEY ("PK_JOB_MEM") - USING INDEX "C_JOB_MEM_PK" --- SPLIT HERE! -ALTER TABLE "LAYER_MEM" ADD CONSTRAINT "C_LAYER_MEM_PK" PRIMARY KEY ("PK_LAYER_MEM") - USING INDEX "C_LAYER_MEM_PK" --- SPLIT HERE! -ALTER TABLE "POINT" ADD CONSTRAINT "C_POINT_PK" PRIMARY KEY ("PK_POINT") - USING INDEX "C_POINT_PK" --- SPLIT HERE! -ALTER TABLE "POINT" ADD CONSTRAINT "C_POINT_PK_SHOW_DEPT" UNIQUE ("PK_SHOW", "PK_DEPT") - USING INDEX "C_POINT_PK_SHOW_DEPT" --- SPLIT HERE! -ALTER TABLE "TASK" ADD CONSTRAINT "C_TASK_PK" PRIMARY KEY ("PK_TASK") - USING INDEX "C_TASK_PK" --- SPLIT HERE! -ALTER TABLE "TASK" ADD CONSTRAINT "C_TASK_UNIQ" UNIQUE ("STR_SHOT", "PK_POINT") - USING INDEX "C_TASK_UNIQ" --- SPLIT HERE! -ALTER TABLE "HISTORY_PERIOD" ADD CONSTRAINT "C_HISTORY_PERIOD_PK" PRIMARY KEY ("PK") - USING INDEX (CREATE UNIQUE INDEX "C_HISTORY_PERIOD_PK" ON "HISTORY_PERIOD" ("PK")) --- SPLIT HERE! -ALTER TABLE "FRAME_HISTORY" ADD CONSTRAINT "C_FRAME_HISTORY_PK" PRIMARY KEY ("PK_FRAME_HISTORY") - USING INDEX (CREATE UNIQUE INDEX "C_FRAME_HISTORY_PK" ON "FRAME_HISTORY" ("PK_FRAME_HISTORY")) --- SPLIT HERE! -ALTER TABLE "REDIRECT" ADD CONSTRAINT "C_REDIRECT_PK" PRIMARY KEY ("PK_PROC") - USING INDEX "C_REDIRECT_PK" --- SPLIT HERE! - -CREATE VIEW "VS_SHOW_RESOURCE" ("PK_SHOW", "INT_CORES") AS - SELECT - job.pk_show, - SUM(int_cores) AS int_cores - FROM - job, - job_resource - WHERE - job.pk_job = job_resource.pk_job - AND - job.str_state='Pending' - GROUP BY - job.pk_show - --- SPLIT HERE! -CREATE VIEW "VS_SHOW_STAT" ("PK_SHOW", "INT_PENDING_COUNT", "INT_RUNNING_COUNT", "INT_DEAD_COUNT", "INT_JOB_COUNT") AS - SELECT - job.pk_show, - SUM(int_waiting_count+int_depend_count) AS int_pending_count, - SUM(int_running_count) AS int_running_count, - SUM(int_dead_count) AS int_dead_count, - COUNT(1) AS int_job_count - FROM - job_stat, - job - WHERE - job_stat.pk_job = job.pk_job - AND - job.str_state = 'Pending' - GROUP BY job.pk_show - --- SPLIT HERE! -CREATE VIEW "VS_JOB_RESOURCE" ("PK_JOB", "INT_PROCS", "INT_CORES", "INT_MEM_RESERVED") AS - SELECT - job.pk_job, - COUNT(proc.pk_proc) AS int_procs, - COALESCE(SUM(int_cores_reserved),0) AS int_cores, - COALESCE(SUM(int_mem_reserved),0) AS int_mem_reserved - FROM - job LEFT JOIN proc ON (proc.pk_job = job.pk_job) - GROUP BY - job.pk_job - --- SPLIT HERE! -CREATE VIEW "VS_ALLOC_USAGE" ("PK_ALLOC", "INT_CORES", "INT_IDLE_CORES", "INT_RUNNING_CORES", "INT_LOCKED_CORES", "INT_AVAILABLE_CORES", "INT_HOSTS", "INT_LOCKED_HOSTS", "INT_DOWN_HOSTS") AS - SELECT - alloc.pk_alloc, - NVL(SUM(host.int_cores),0) AS int_cores, - NVL(SUM(host.int_cores_idle),0) AS int_idle_cores, - NVL(SUM(host.int_cores - host.int_cores_idle),0) as int_running_cores, - NVL((SELECT SUM(int_cores) FROM host WHERE host.pk_alloc=alloc.pk_alloc AND (str_lock_state='NimbyLocked' OR str_lock_state='Locked')),0) AS int_locked_cores, - NVL((SELECT SUM(int_cores_idle) FROM host h,host_stat hs WHERE h.pk_host = hs.pk_host AND h.pk_alloc=alloc.pk_alloc AND h.str_lock_state='Open' AND hs.str_state ='Up'),0) AS int_available_cores, - COUNT(host.pk_host) AS int_hosts, - (SELECT COUNT(*) FROM host WHERE host.pk_alloc=alloc.pk_alloc AND str_lock_state='Locked') AS int_locked_hosts, - (SELECT COUNT(*) FROM host h,host_stat hs WHERE h.pk_host = hs.pk_host AND h.pk_alloc=alloc.pk_alloc AND hs.str_state='Down') AS int_down_hosts - FROM - alloc LEFT JOIN host ON (alloc.pk_alloc = host.pk_alloc) - GROUP BY - alloc.pk_alloc - --- SPLIT HERE! -CREATE VIEW "VS_FOLDER_COUNTS" ("PK_FOLDER", "INT_DEPEND_COUNT", "INT_WAITING_COUNT", "INT_RUNNING_COUNT", "INT_DEAD_COUNT", "INT_CORES", "INT_JOB_COUNT") AS - SELECT - folder.pk_folder, - NVL(SUM(int_depend_count),0) AS int_depend_count, - NVL(SUM(int_waiting_count),0) AS int_waiting_count, - NVL(SUM(int_running_count),0) AS int_running_count, - NVL(SUM(int_dead_count),0) AS int_dead_count, - NVL(SUM(int_cores),0) AS int_cores, - NVL(COUNT(job.pk_job),0) AS int_job_count -FROM - folder - LEFT JOIN - job ON (folder.pk_folder = job.pk_folder AND job.str_state='Pending') - LEFT JOIN - job_stat ON (job.pk_job = job_stat.pk_job) - LEFT JOIN - job_resource ON (job.pk_job = job_resource.pk_job) - GROUP BY - folder.pk_folder - --- SPLIT HERE! -CREATE VIEW "VS_WAITING" ("PK_SHOW") AS - SELECT - job.pk_show - FROM - job_resource jr, - job_stat, - job - WHERE - job_stat.pk_job = job.pk_job - AND - jr.pk_job = job.pk_job - AND - job.str_state = 'Pending' - AND - job.b_paused = 0 - AND - jr.int_max_cores - jr.int_cores >= 100 - AND - job_stat.int_waiting_count != 0 - - GROUP BY job.pk_show - --- SPLIT HERE! -CREATE VIEW "V_HISTORY_FRAME" ("PK_FRAME_HISTORY", "PK_FRAME", "PK_LAYER", "PK_JOB", "STR_NAME", "STR_STATE", "INT_MEM_RESERVED", "INT_MEM_MAX_USED", "INT_CORES", "STR_HOST", "INT_EXIT_STATUS", "STR_ALLOC_NAME", "B_ALLOC_BILLABLE", "STR_FACILITY_NAME", "INT_TS_STARTED", "INT_TS_STOPPED", "INT_CHECKPOINT_COUNT", "STR_SHOW_NAME", "DT_LAST_MODIFIED") AS - select -fh.PK_FRAME_HISTORY, -fh.PK_FRAME, -fh.PK_LAYER, -fh.PK_JOB, -fh.STR_NAME, -fh.STR_STATE, -fh.INT_MEM_RESERVED, -fh.INT_MEM_MAX_USED, -fh.INT_CORES, -fh.STR_HOST, -fh.INT_EXIT_STATUS, -a.STR_NAME STR_ALLOC_NAME, -a.B_BILLABLE B_ALLOC_BILLABLE, -f.STR_NAME STR_FACILITY_NAME, -fh.INT_TS_STARTED, -fh.INT_TS_STOPPED, -fh.INT_CHECKPOINT_COUNT, -null str_show_name, -fh.dt_last_modified -from frame_history fh, job_history jh, alloc a, facility f -where fh.pk_job = jh.pk_job -and fh.pk_alloc = a.pk_alloc (+) -and a.pk_facility = f.pk_facility (+) -and fh.dt_last_modified >= ( - select dt_begin - from history_period -) -and fh.dt_last_modified < ( - select dt_end - from history_period -) --- SPLIT HERE! -CREATE VIEW "V_HISTORY_JOB" ("PK_JOB", "STR_NAME", "STR_SHOT", "STR_USER", "INT_CORE_TIME_SUCCESS", "INT_CORE_TIME_FAIL", "INT_FRAME_COUNT", "INT_LAYER_COUNT", "INT_WAITING_COUNT", "INT_DEAD_COUNT", "INT_DEPEND_COUNT", "INT_EATEN_COUNT", "INT_SUCCEEDED_COUNT", "INT_RUNNING_COUNT", "INT_MAX_RSS", "B_ARCHIVED", "STR_FACILITY_NAME", "STR_DEPT_NAME", "INT_TS_STARTED", "INT_TS_STOPPED", "STR_SHOW_NAME", "DT_LAST_MODIFIED") AS - select -jh.PK_JOB, -jh.STR_NAME, -jh.STR_SHOT, -jh.STR_USER, -jh.INT_CORE_TIME_SUCCESS, -jh.INT_CORE_TIME_FAIL, -jh.INT_FRAME_COUNT, -jh.INT_LAYER_COUNT, -jh.INT_WAITING_COUNT, -jh.INT_DEAD_COUNT, -jh.INT_DEPEND_COUNT, -jh.INT_EATEN_COUNT, -jh.INT_SUCCEEDED_COUNT, -jh.INT_RUNNING_COUNT, -jh.INT_MAX_RSS, -jh.B_ARCHIVED, -f.str_name STR_FACILITY_NAME, -d.str_name str_dept_name, -jh.INT_TS_STARTED, -jh.INT_TS_STOPPED, -s.str_name str_show_name, -jh.dt_last_modified -from job_history jh, show s, facility f, dept d -where jh.pk_show = s.pk_show -and jh.pk_facility = f.pk_facility -and jh.pk_dept = d.pk_dept -and ( - jh.dt_last_modified >= ( - select dt_begin - from history_period - ) - or - jh.int_ts_stopped = 0 -) --- SPLIT HERE! -CREATE VIEW "V_HISTORY_LAYER" ("PK_LAYER", "PK_JOB", "STR_NAME", "STR_TYPE", "INT_CORES_MIN", "INT_MEM_MIN", "INT_CORE_TIME_SUCCESS", "INT_CORE_TIME_FAIL", "INT_FRAME_COUNT", "INT_LAYER_COUNT", "INT_WAITING_COUNT", "INT_DEAD_COUNT", "INT_DEPEND_COUNT", "INT_EATEN_COUNT", "INT_SUCCEEDED_COUNT", "INT_RUNNING_COUNT", "INT_MAX_RSS", "B_ARCHIVED", "STR_SERVICES", "STR_SHOW_NAME", "DT_LAST_MODIFIED") AS - select -lh.PK_LAYER, -lh.PK_JOB, -lh.STR_NAME, -lh.STR_TYPE, -lh.INT_CORES_MIN, -lh.INT_MEM_MIN, -lh.INT_CORE_TIME_SUCCESS, -lh.INT_CORE_TIME_FAIL, -lh.INT_FRAME_COUNT, -lh.INT_LAYER_COUNT, -lh.INT_WAITING_COUNT, -lh.INT_DEAD_COUNT, -lh.INT_DEPEND_COUNT, -lh.INT_EATEN_COUNT, -lh.INT_SUCCEEDED_COUNT, -lh.INT_RUNNING_COUNT, -lh.INT_MAX_RSS, -lh.B_ARCHIVED, -lh.STR_SERVICES, -s.str_name str_show_name, -lh.dt_last_modified -from layer_history lh, job_history jh, show s -where lh.pk_job = jh.pk_job -and jh.pk_show = s.pk_show -and jh.dt_last_modified >= ( - select dt_begin - from history_period -) -and jh.dt_last_modified < ( - select dt_end - from history_period -) --- SPLIT HERE! -CREATE PACKAGE BODY HISTORY AS - - procedure period_shift as - vTemp date - begin - - begin - - select dt_end - into vTemp - from history_period - - update history_period - set dt_begin = vTemp, - dt_end = ( - select sysdate from dual - ) - - exception - when no_data_found then - insert into history_period (pk) values (sys_guid()) - select dt_end - into vTemp - from history_period - when others then - raise - end - - end period_shift - - procedure period_shift(piEndDate in date) as - vTemp date - begin - - begin - - select dt_end - into vTemp - from history_period - - update history_period - set dt_begin = vTemp, - dt_end = ( - select nvl(piEndDate, sysdate) from dual - ) - - exception - when no_data_found then - insert into history_period (pk) values (sys_guid()) - select dt_end - into vTemp - from history_period - when others then - raise - end - - end period_shift - - - procedure period_clear as - begin - - delete from history_period - insert into history_period (pk) values (sys_guid()) - - end period_clear - -END HISTORY --- SPLIT HERE! - - -ALTER TABLE "ACTION" ADD CONSTRAINT "C_ACTION_PK_FILTER" FOREIGN KEY ("PK_FILTER") - REFERENCES "FILTER" ("PK_FILTER") --- SPLIT HERE! -ALTER TABLE "ACTION" ADD CONSTRAINT "C_ACTION_PK_FOLDER" FOREIGN KEY ("PK_FOLDER") - REFERENCES "FOLDER" ("PK_FOLDER") --- SPLIT HERE! -ALTER TABLE "ALLOC" ADD CONSTRAINT "C_ALLOC_PK_FACILITY" FOREIGN KEY ("PK_FACILITY") - REFERENCES "FACILITY" ("PK_FACILITY") --- SPLIT HERE! -ALTER TABLE "COMMENTS" ADD CONSTRAINT "C_COMMENT_PK_HOST" FOREIGN KEY ("PK_HOST") - REFERENCES "HOST" ("PK_HOST") --- SPLIT HERE! -ALTER TABLE "COMMENTS" ADD CONSTRAINT "C_COMMENT_PK_JOB" FOREIGN KEY ("PK_JOB") - REFERENCES "JOB" ("PK_JOB") --- SPLIT HERE! -ALTER TABLE "FILTER" ADD CONSTRAINT "C_FILTER_PK_SHOW" FOREIGN KEY ("PK_SHOW") - REFERENCES "SHOW" ("PK_SHOW") --- SPLIT HERE! -ALTER TABLE "FOLDER" ADD CONSTRAINT "C_FOLDER_PK_SHOW" FOREIGN KEY ("PK_SHOW") - REFERENCES "SHOW" ("PK_SHOW") --- SPLIT HERE! -ALTER TABLE "FOLDER" ADD CONSTRAINT "C_FOLDER_PK_DEPT" FOREIGN KEY ("PK_DEPT") - REFERENCES "DEPT" ("PK_DEPT") --- SPLIT HERE! -ALTER TABLE "FOLDER_LEVEL" ADD CONSTRAINT "C_FOLDER_LEVEL_PK_FOLDER" FOREIGN KEY ("PK_FOLDER") - REFERENCES "FOLDER" ("PK_FOLDER") --- SPLIT HERE! -ALTER TABLE "FRAME" ADD CONSTRAINT "C_FRAME_PK_JOB" FOREIGN KEY ("PK_JOB") - REFERENCES "JOB" ("PK_JOB") --- SPLIT HERE! -ALTER TABLE "FRAME" ADD CONSTRAINT "C_FRAME_PK_LAYER" FOREIGN KEY ("PK_LAYER") - REFERENCES "LAYER" ("PK_LAYER") --- SPLIT HERE! -ALTER TABLE "HOST" ADD CONSTRAINT "C_HOST_PK_ALLOC" FOREIGN KEY ("PK_ALLOC") - REFERENCES "ALLOC" ("PK_ALLOC") --- SPLIT HERE! -ALTER TABLE "HOST_STAT" ADD CONSTRAINT "C_HOST_STAT_PK_HOST" FOREIGN KEY ("PK_HOST") - REFERENCES "HOST" ("PK_HOST") --- SPLIT HERE! -ALTER TABLE "JOB" ADD CONSTRAINT "C_JOB_PK_SHOW" FOREIGN KEY ("PK_SHOW") - REFERENCES "SHOW" ("PK_SHOW") --- SPLIT HERE! -ALTER TABLE "JOB" ADD CONSTRAINT "C_JOB_PK_FOLDER" FOREIGN KEY ("PK_FOLDER") - REFERENCES "FOLDER" ("PK_FOLDER") --- SPLIT HERE! -ALTER TABLE "JOB" ADD CONSTRAINT "C_JOB_PK_FACILITY" FOREIGN KEY ("PK_FACILITY") - REFERENCES "FACILITY" ("PK_FACILITY") --- SPLIT HERE! -ALTER TABLE "JOB" ADD CONSTRAINT "C_JOB_PK_DEPT" FOREIGN KEY ("PK_DEPT") - REFERENCES "DEPT" ("PK_DEPT") --- SPLIT HERE! -ALTER TABLE "JOB_ENV" ADD CONSTRAINT "C_JOB_ENV_PK_JOB" FOREIGN KEY ("PK_JOB") - REFERENCES "JOB" ("PK_JOB") --- SPLIT HERE! -ALTER TABLE "LAYER" ADD CONSTRAINT "C_LAYER_PK_JOB" FOREIGN KEY ("PK_JOB") - REFERENCES "JOB" ("PK_JOB") --- SPLIT HERE! -ALTER TABLE "LAYER_ENV" ADD CONSTRAINT "C_LAYER_ENV_PK_LAYER" FOREIGN KEY ("PK_LAYER") - REFERENCES "LAYER" ("PK_LAYER") --- SPLIT HERE! -ALTER TABLE "LAYER_ENV" ADD CONSTRAINT "C_LAYER_ENV_PK_JOB" FOREIGN KEY ("PK_JOB") - REFERENCES "JOB" ("PK_JOB") --- SPLIT HERE! -ALTER TABLE "LAYER_RESOURCE" ADD CONSTRAINT "C_LAYER_RESOURCE_PK_JOB" FOREIGN KEY ("PK_JOB") - REFERENCES "JOB" ("PK_JOB") --- SPLIT HERE! -ALTER TABLE "LAYER_RESOURCE" ADD CONSTRAINT "C_LAYER_RESOURCE_PK_LAYER" FOREIGN KEY ("PK_LAYER") - REFERENCES "LAYER" ("PK_LAYER") --- SPLIT HERE! -ALTER TABLE "LAYER_STAT" ADD CONSTRAINT "C_LAYER_STAT_PK_JOB" FOREIGN KEY ("PK_JOB") - REFERENCES "JOB" ("PK_JOB") --- SPLIT HERE! -ALTER TABLE "LAYER_STAT" ADD CONSTRAINT "C_LAYER_STAT_PK_LAYER" FOREIGN KEY ("PK_LAYER") - REFERENCES "LAYER" ("PK_LAYER") --- SPLIT HERE! -ALTER TABLE "LAYER_USAGE" ADD CONSTRAINT "C_LAYER_USAGE_PK_JOB" FOREIGN KEY ("PK_JOB") - REFERENCES "JOB" ("PK_JOB") --- SPLIT HERE! -ALTER TABLE "LAYER_USAGE" ADD CONSTRAINT "C_LAYER_USAGE_PK_LAYER" FOREIGN KEY ("PK_LAYER") - REFERENCES "LAYER" ("PK_LAYER") --- SPLIT HERE! -ALTER TABLE "MATCHER" ADD CONSTRAINT "C_MATCHER_PK_FILTER" FOREIGN KEY ("PK_FILTER") - REFERENCES "FILTER" ("PK_FILTER") --- SPLIT HERE! -ALTER TABLE "PROC" ADD CONSTRAINT "C_PROC_PK_FRAME" FOREIGN KEY ("PK_FRAME") - REFERENCES "FRAME" ("PK_FRAME") --- SPLIT HERE! -ALTER TABLE "PROC" ADD CONSTRAINT "C_PROC_PK_HOST" FOREIGN KEY ("PK_HOST") - REFERENCES "HOST" ("PK_HOST") --- SPLIT HERE! -ALTER TABLE "SUBSCRIPTION" ADD CONSTRAINT "C_SUBSCRIPTION_PK_ALLOC" FOREIGN KEY ("PK_ALLOC") - REFERENCES "ALLOC" ("PK_ALLOC") --- SPLIT HERE! -ALTER TABLE "SUBSCRIPTION" ADD CONSTRAINT "C_SUBSCRIPTION_PK_SHOW" FOREIGN KEY ("PK_SHOW") - REFERENCES "SHOW" ("PK_SHOW") --- SPLIT HERE! -ALTER TABLE "JOB_STAT" ADD CONSTRAINT "C_JOB_STAT_PK_JOB" FOREIGN KEY ("PK_JOB") - REFERENCES "JOB" ("PK_JOB") --- SPLIT HERE! -ALTER TABLE "JOB_RESOURCE" ADD CONSTRAINT "C_JOB_RESOURCE_PK_JOB" FOREIGN KEY ("PK_JOB") - REFERENCES "JOB" ("PK_JOB") --- SPLIT HERE! -ALTER TABLE "JOB_USAGE" ADD CONSTRAINT "C_JOB_USAGE_PK_JOB" FOREIGN KEY ("PK_JOB") - REFERENCES "JOB" ("PK_JOB") --- SPLIT HERE! -ALTER TABLE "JOB_HISTORY" ADD CONSTRAINT "C_JOB_HISTORY_PK_FACILITY" FOREIGN KEY ("PK_FACILITY") - REFERENCES "FACILITY" ("PK_FACILITY") --- SPLIT HERE! -ALTER TABLE "JOB_HISTORY" ADD CONSTRAINT "C_JOB_HISTORY_PK_DEPT" FOREIGN KEY ("PK_DEPT") - REFERENCES "DEPT" ("PK_DEPT") --- SPLIT HERE! -ALTER TABLE "JOB_HISTORY" ADD CONSTRAINT "C_JOB_HISTORY_PK_SHOW" FOREIGN KEY ("PK_SHOW") - REFERENCES "SHOW" ("PK_SHOW") --- SPLIT HERE! -ALTER TABLE "LAYER_HISTORY" ADD CONSTRAINT "C_LAYER_HISTORY_PK_JOB" FOREIGN KEY ("PK_JOB") - REFERENCES "JOB_HISTORY" ("PK_JOB") ON DELETE CASCADE --- SPLIT HERE! -ALTER TABLE "JOB_POST" ADD CONSTRAINT "C_JOB_POST_PK_JOB" FOREIGN KEY ("PK_JOB") - REFERENCES "JOB" ("PK_JOB") --- SPLIT HERE! -ALTER TABLE "JOB_POST" ADD CONSTRAINT "C_JOB_POST_PK_POST_JOB" FOREIGN KEY ("PK_POST_JOB") - REFERENCES "JOB" ("PK_JOB") --- SPLIT HERE! -ALTER TABLE "SHOW_ALIAS" ADD CONSTRAINT "C_SHOW_ALIAS_PK_SHOW" FOREIGN KEY ("PK_SHOW") - REFERENCES "SHOW" ("PK_SHOW") --- SPLIT HERE! -ALTER TABLE "FOLDER_RESOURCE" ADD CONSTRAINT "C_FOLDER_RESOURCE_PK_FOLDER" FOREIGN KEY ("PK_FOLDER") - REFERENCES "FOLDER" ("PK_FOLDER") --- SPLIT HERE! -ALTER TABLE "JOB_MEM" ADD CONSTRAINT "C_JOB_MEM_PK_JOB" FOREIGN KEY ("PK_JOB") - REFERENCES "JOB" ("PK_JOB") --- SPLIT HERE! -ALTER TABLE "LAYER_MEM" ADD CONSTRAINT "C_LAYER_MEM_PK_JOB" FOREIGN KEY ("PK_JOB") - REFERENCES "JOB" ("PK_JOB") --- SPLIT HERE! -ALTER TABLE "LAYER_MEM" ADD CONSTRAINT "C_LAYER_MEM_PK_LAYER" FOREIGN KEY ("PK_LAYER") - REFERENCES "LAYER" ("PK_LAYER") --- SPLIT HERE! -ALTER TABLE "POINT" ADD CONSTRAINT "C_POINT_PK_DEPT" FOREIGN KEY ("PK_DEPT") - REFERENCES "DEPT" ("PK_DEPT") --- SPLIT HERE! -ALTER TABLE "POINT" ADD CONSTRAINT "C_POINT_PK_SHOW" FOREIGN KEY ("PK_SHOW") - REFERENCES "SHOW" ("PK_SHOW") --- SPLIT HERE! -ALTER TABLE "TASK" ADD CONSTRAINT "C_TASK_PK_POINT" FOREIGN KEY ("PK_POINT") - REFERENCES "POINT" ("PK_POINT") --- SPLIT HERE! -ALTER TABLE "JOB_LOCAL" ADD CONSTRAINT "C_JOB_LOCAL_PK_JOB" FOREIGN KEY ("PK_JOB") - REFERENCES "JOB" ("PK_JOB") --- SPLIT HERE! -ALTER TABLE "JOB_LOCAL" ADD CONSTRAINT "C_JOB_LOCAL_PK_HOST" FOREIGN KEY ("PK_HOST") - REFERENCES "HOST" ("PK_HOST") --- SPLIT HERE! -ALTER TABLE "HOST_LOCAL" ADD CONSTRAINT "C_HOST_LOCAL_PK_JOB" FOREIGN KEY ("PK_JOB") - REFERENCES "JOB" ("PK_JOB") --- SPLIT HERE! -ALTER TABLE "HOST_LOCAL" ADD CONSTRAINT "C_HOST_LOCAL_PK_HOST" FOREIGN KEY ("PK_HOST") - REFERENCES "HOST" ("PK_HOST") --- SPLIT HERE! -ALTER TABLE "OWNER" ADD CONSTRAINT "C_OWNER_PK_SHOW" FOREIGN KEY ("PK_SHOW") - REFERENCES "SHOW" ("PK_SHOW") --- SPLIT HERE! -ALTER TABLE "DEED" ADD CONSTRAINT "C_DEED_PK_HOST" FOREIGN KEY ("PK_HOST") - REFERENCES "HOST" ("PK_HOST") --- SPLIT HERE! -ALTER TABLE "SHOW_SERVICE" ADD CONSTRAINT "C_SHOW_SERVICE_PK_SHOW" FOREIGN KEY ("PK_SHOW") - REFERENCES "SHOW" ("PK_SHOW") --- SPLIT HERE! -ALTER TABLE "LAYER_OUTPUT" ADD CONSTRAINT "C_LAYER_OUTPUT_PK_LAYER" FOREIGN KEY ("PK_LAYER") - REFERENCES "LAYER" ("PK_LAYER") --- SPLIT HERE! -ALTER TABLE "LAYER_OUTPUT" ADD CONSTRAINT "C_LAYER_OUTPUT_PK_JOB" FOREIGN KEY ("PK_JOB") - REFERENCES "JOB" ("PK_JOB") --- SPLIT HERE! -ALTER TABLE "FRAME_HISTORY" ADD CONSTRAINT "C_FRAME_HISTORY_PK_JOB" FOREIGN KEY ("PK_JOB") - REFERENCES "JOB_HISTORY" ("PK_JOB") ON DELETE CASCADE --- SPLIT HERE! -ALTER TABLE "FRAME_HISTORY" ADD CONSTRAINT "C_FRAME_HISTORY_PK_LAYER" FOREIGN KEY ("PK_LAYER") - REFERENCES "LAYER_HISTORY" ("PK_LAYER") ON DELETE CASCADE --- SPLIT HERE! -ALTER TABLE "FRAME_HISTORY" ADD CONSTRAINT "C_FRAME_HISTORY_PK_ALLOC" FOREIGN KEY ("PK_ALLOC") - REFERENCES "ALLOC" ("PK_ALLOC") --- SPLIT HERE! - -CREATE TRIGGER tbiu_layer_history -before insert or update -on layer_history -referencing new as new old as old -for each row -begin - :new.dt_last_modified := sysdate; -end; --- SPLIT HERE! - -CREATE TRIGGER "AFTER_JOB_MOVED" AFTER UPDATE ON job -FOR EACH ROW - WHEN (NEW.pk_folder != OLD.pk_folder) DECLARE - int_core_count NUMERIC(16,0); -BEGIN - SELECT int_cores INTO int_core_count - FROM job_resource WHERE pk_job = :new.pk_job; - - IF int_core_count > 0 THEN - UPDATE folder_resource SET int_cores = int_cores + int_core_count - WHERE pk_folder = :new.pk_folder; - - UPDATE folder_resource SET int_cores = int_cores - int_core_count - WHERE pk_folder = :old.pk_folder; - END IF; -END; --- SPLIT HERE! - -CREATE TRIGGER "BEFORE_DELETE_JOB" BEFORE DELETE ON job -FOR EACH ROW -DECLARE - TYPE StatType IS RECORD ( - int_core_time_success NUMERIC(38), - int_core_time_fail NUMERIC(38), - int_waiting_count NUMERIC(38), - int_dead_count NUMERIC(38), - int_depend_count NUMERIC(38), - int_eaten_count NUMERIC(38), - int_succeeded_count NUMERIC(38), - int_running_count NUMERIC(38), - int_max_rss NUMERIC(38) - ); - js StatType; - -BEGIN - SELECT - job_usage.int_core_time_success, - job_usage.int_core_time_fail, - job_stat.int_waiting_count, - job_stat.int_dead_count, - job_stat.int_depend_count, - job_stat.int_eaten_count, - job_stat.int_succeeded_count, - job_stat.int_running_count, - job_mem.int_max_rss - INTO - js - FROM - job_mem, - job_usage, - job_stat - WHERE - job_usage.pk_job = job_mem.pk_job - AND - job_stat.pk_job = job_mem.pk_job - AND - job_mem.pk_job = :old.pk_job; - - UPDATE - job_history - SET - pk_dept = :old.pk_dept, - int_core_time_success = js.int_core_time_success, - int_core_time_fail = js.int_core_time_fail, - int_frame_count = :old.int_frame_count, - int_layer_count = :old.int_layer_count, - int_waiting_count = js.int_waiting_count, - int_dead_count = js.int_dead_count, - int_depend_count = js.int_depend_count, - int_eaten_count = js.int_eaten_count, - int_succeeded_count = js.int_succeeded_count, - int_running_count = js.int_running_count, - int_max_rss = js.int_max_rss, - b_archived = 1, - int_ts_stopped = nvl(epoch(:old.ts_stopped), epoch(systimestamp)) - WHERE - pk_job = :old.pk_job; - - delete from depend where pk_job_depend_on=:old.pk_job or pk_job_depend_er=:old.pk_job; - delete from frame where pk_job=:old.pk_job; - delete from layer where pk_job=:old.pk_job; - delete from job_env WHERE pk_job=:old.pk_job; - delete from job_stat WHERE pk_job=:old.pk_job; - delete from job_resource WHERE pk_job=:old.pk_job; - delete from job_usage WHERE pk_job=:old.pk_job; - delete from job_mem WHERE pk_job=:old.pk_job; - delete from comments WHERE pk_job=:old.pk_job; - -END; --- SPLIT HERE! - -CREATE TRIGGER "AFTER_JOB_FINISHED" AFTER UPDATE ON job -FOR EACH ROW - WHEN (old.str_state = 'Pending' AND new.str_state = 'Finished') DECLARE - ts NUMERIC(12,0) := epoch(systimestamp); - TYPE StatType IS RECORD ( - int_core_time_success NUMERIC(38), - int_core_time_fail NUMERIC(38), - int_waiting_count NUMERIC(38), - int_dead_count NUMERIC(38), - int_depend_count NUMERIC(38), - int_eaten_count NUMERIC(38), - int_succeeded_count NUMERIC(38), - int_running_count NUMERIC(38), - int_max_rss NUMERIC(38) - ); - - TYPE LayerStatType IS RECORD ( - int_core_time_success NUMERIC(38), - int_core_time_fail NUMERIC(38), - int_total_count NUMERIC(38), - int_waiting_count NUMERIC(38), - int_dead_count NUMERIC(38), - int_depend_count NUMERIC(38), - int_eaten_count NUMERIC(38), - int_succeeded_count NUMERIC(38), - int_running_count NUMERIC(38), - int_max_rss NUMERIC(38) - ); - js StatType; - ls LayerStatType; -BEGIN - SELECT - job_usage.int_core_time_success, - job_usage.int_core_time_fail, - job_stat.int_waiting_count, - job_stat.int_dead_count, - job_stat.int_depend_count, - job_stat.int_eaten_count, - job_stat.int_succeeded_count, - job_stat.int_running_count, - job_mem.int_max_rss - INTO - js - FROM - job_mem, - job_usage, - job_stat - WHERE - job_usage.pk_job = job_mem.pk_job - AND - job_stat.pk_job = job_mem.pk_job - AND - job_mem.pk_job = :new.pk_job; - - UPDATE - job_history - SET - pk_dept = :new.pk_dept, - int_core_time_success = js.int_core_time_success, - int_core_time_fail = js.int_core_time_fail, - int_frame_count = :new.int_frame_count, - int_layer_count = :new.int_layer_count, - int_waiting_count = js.int_waiting_count, - int_dead_count = js.int_dead_count, - int_depend_count = js.int_depend_count, - int_eaten_count = js.int_eaten_count, - int_succeeded_count = js.int_succeeded_count, - int_running_count = js.int_running_count, - int_max_rss = js.int_max_rss, - int_ts_stopped = ts - WHERE - pk_job = :new.pk_job; - - FOR one_layer in (SELECT pk_layer from layer where pk_job = :new.pk_job) LOOP - SELECT - layer_usage.int_core_time_success, - layer_usage.int_core_time_fail, - layer_stat.int_total_count, - layer_stat.int_waiting_count, - layer_stat.int_dead_count, - layer_stat.int_depend_count, - layer_stat.int_eaten_count, - layer_stat.int_succeeded_count, - layer_stat.int_running_count, - layer_mem.int_max_rss - INTO - ls - FROM - layer_mem, - layer_usage, - layer_stat - WHERE - layer_usage.pk_layer = layer_mem.pk_layer - AND - layer_stat.pk_layer = layer_mem.pk_layer - AND - layer_mem.pk_layer = one_layer.pk_layer; - - UPDATE - layer_history - SET - int_core_time_success = ls.int_core_time_success, - int_core_time_fail = ls.int_core_time_fail, - int_frame_count = ls.int_total_count, - int_waiting_count = ls.int_waiting_count, - int_dead_count = ls.int_dead_count, - int_depend_count = ls.int_depend_count, - int_eaten_count = ls.int_eaten_count, - int_succeeded_count = ls.int_succeeded_count, - int_running_count = ls.int_running_count, - int_max_rss = ls.int_max_rss - WHERE - pk_layer = one_layer.pk_layer; - END LOOP; - - /** - * Delete any local core assignements from this job. - **/ - DELETE FROM job_local WHERE pk_job=:new.pk_job; - -END; --- SPLIT HERE! - -CREATE TRIGGER "AFTER_INSERT_JOB" AFTER INSERT ON job -FOR EACH ROW -BEGIN - INSERT INTO job_stat (pk_job_stat,pk_job) VALUES(:new.pk_job,:new.pk_job); - INSERT INTO job_resource (pk_job_resource,pk_job) VALUES(:new.pk_job,:new.pk_job); - INSERT INTO job_usage (pk_job_usage,pk_job) VALUES(:new.pk_job,:new.pk_job); - INSERT INTO job_mem (pk_job_mem,pk_job) VALUES (:new.pk_job,:new.pk_job); - - INSERT INTO job_history - (pk_job, pk_show, pk_facility, pk_dept, str_name, str_shot, str_user, int_ts_started) - VALUES - (:new.pk_job, :new.pk_show, :new.pk_facility, :new.pk_dept, - :new.str_name, :new.str_shot, :new.str_user, epoch(systimestamp)); -END; --- SPLIT HERE! - -CREATE TRIGGER "AFTER_JOB_DEPT_UPDATE" AFTER UPDATE ON job -FOR EACH ROW - WHEN (NEW.pk_dept != OLD.pk_dept AND new.str_state='Pending') DECLARE - int_running_cores NUMERIC(16,0); -BEGIN - /** - * Handles the accounting for moving a job between departments. - **/ - SELECT int_cores INTO int_running_cores - FROM job_resource WHERE pk_job = :new.pk_job; - - IF int_running_cores > 0 THEN - UPDATE point SET int_cores = int_cores + int_running_cores - WHERE pk_dept = :new.pk_dept AND pk_show = :new.pk_show; - - UPDATE point SET int_cores = int_cores - int_running_cores - WHERE pk_dept = :old.pk_dept AND pk_show = :old.pk_show; - END IF; - -END; --- SPLIT HERE! - -CREATE TRIGGER "VERIFY_HOST_LOCAL" BEFORE UPDATE ON host_local -FOR EACH ROW - WHEN ((NEW.int_cores_max = OLD.int_cores_max AND NEW.int_mem_max = OLD.int_mem_max) AND -(NEW.int_cores_idle != OLD.int_cores_idle OR NEW.int_mem_idle != OLD.int_mem_idle)) BEGIN - /** - * Check to see if the new cores exceeds max cores. This check is only - * done if NEW.int_max_cores is equal to OLD.int_max_cores and - * NEW.int_cores > OLD.int_cores, otherwise this error will be thrown - * when people lower the max. - **/ - IF :NEW.int_cores_idle < 0 THEN - Raise_application_error(-20021, 'host local doesnt have enough idle cores.'); - END IF; - - IF :NEW.int_mem_idle < 0 THEN - Raise_application_error(-20021, 'host local doesnt have enough idle memory'); - END IF; - -END; --- SPLIT HERE! - -CREATE TRIGGER "TIER_HOST_LOCAL" BEFORE UPDATE ON host_local -FOR EACH ROW -BEGIN - :new.float_tier := tier(:new.int_cores_max - :new.int_cores_idle,:new.int_cores_max); -END; --- SPLIT HERE! - -CREATE TRIGGER "AFTER_INSERT_LAYER" AFTER INSERT ON layer -FOR EACH ROW -BEGIN - - INSERT INTO layer_stat (pk_layer_stat, pk_layer, pk_job) VALUES (:new.pk_layer, :new.pk_layer, :new.pk_job); - INSERT INTO layer_resource (pk_layer_resource, pk_layer, pk_job) VALUES (:new.pk_layer, :new.pk_layer, :new.pk_job); - INSERT INTO layer_usage (pk_layer_usage, pk_layer, pk_job) VALUES (:new.pk_layer, :new.pk_layer, :new.pk_job); - INSERT INTO layer_mem (pk_layer_mem, pk_layer, pk_job) VALUES (:new.pk_layer, :new.pk_layer, :new.pk_job); - - INSERT INTO layer_history - (pk_layer, pk_job, str_name, str_type, int_cores_min, int_mem_min, b_archived, str_services) - VALUES - (:new.pk_layer, :new.pk_job, :new.str_name, :new.str_type, :new.int_cores_min, :new.int_mem_min, 0, :new.str_services); -END; --- SPLIT HERE! - -CREATE TRIGGER "BEFORE_DELETE_LAYER" BEFORE DELETE ON layer -FOR EACH ROW -DECLARE - TYPE StatType IS RECORD ( - int_core_time_success NUMERIC(38), - int_core_time_fail NUMERIC(38), - int_total_count NUMERIC(38), - int_waiting_count NUMERIC(38), - int_dead_count NUMERIC(38), - int_depend_count NUMERIC(38), - int_eaten_count NUMERIC(38), - int_succeeded_count NUMERIC(38), - int_running_count NUMERIC(38), - int_max_rss NUMERIC(38) - ); - js StatType; - -BEGIN - SELECT - layer_usage.int_core_time_success, - layer_usage.int_core_time_fail, - layer_stat.int_total_count, - layer_stat.int_waiting_count, - layer_stat.int_dead_count, - layer_stat.int_depend_count, - layer_stat.int_eaten_count, - layer_stat.int_succeeded_count, - layer_stat.int_running_count, - layer_mem.int_max_rss - INTO - js - FROM - layer_mem, - layer_usage, - layer_stat - WHERE - layer_usage.pk_layer = layer_mem.pk_layer - AND - layer_stat.pk_layer = layer_mem.pk_layer - AND - layer_mem.pk_layer = :old.pk_layer; - - UPDATE - layer_history - SET - int_core_time_success = js.int_core_time_success, - int_core_time_fail = js.int_core_time_fail, - int_frame_count = js.int_total_count, - int_waiting_count = js.int_waiting_count, - int_dead_count = js.int_dead_count, - int_depend_count = js.int_depend_count, - int_eaten_count = js.int_eaten_count, - int_succeeded_count = js.int_succeeded_count, - int_running_count = js.int_running_count, - int_max_rss = js.int_max_rss, - b_archived = 1 - WHERE - pk_layer = :old.pk_layer; - - delete from layer_resource where pk_layer=:old.pk_layer; - delete from layer_stat where pk_layer=:old.pk_layer; - delete from layer_usage where pk_layer=:old.pk_layer; - delete from layer_env where pk_layer=:old.pk_layer; - delete from layer_mem where pk_layer=:old.pk_layer; - delete from layer_output where pk_layer=:old.pk_layer; -END; --- SPLIT HERE! - -CREATE TRIGGER tbiu_job_history -before insert or update -on job_history -referencing new as new old as old -for each row -begin - :new.dt_last_modified := sysdate; -end; --- SPLIT HERE! - -CREATE TRIGGER "VERIFY_HOST_RESOURCES" BEFORE UPDATE ON host -FOR EACH ROW - WHEN (new.int_cores_idle != old.int_cores_idle OR new.int_mem_idle != old.int_mem_idle) BEGIN - IF :new.int_cores_idle < 0 THEN - Raise_application_error(-20011, 'unable to allocate additional core units'); - END IF; - - If :new.int_mem_idle < 0 THEN - Raise_application_error(-20012, 'unable to allocate additional memory'); - END IF; - - If :new.int_gpu_idle < 0 THEN - Raise_application_error(-20013, 'unable to allocate additional gpu memory'); - END IF; - -END; --- SPLIT HERE! - -CREATE TRIGGER "BEFORE_DELETE_HOST" BEFORE DELETE ON host -FOR EACH ROW -BEGIN - delete from host_stat WHERE pk_host = :old.pk_host; - delete from host_tag WHERE pk_host = :old.pk_host; - delete from deed WHERE pk_host = :old.pk_host; -END; --- SPLIT HERE! - -CREATE TRIGGER "VERIFY_JOB_RESOURCES" BEFORE UPDATE ON job_resource -FOR EACH ROW - WHEN ( NEW.int_max_cores = OLD.int_max_cores AND NEW.int_cores > OLD.int_cores) BEGIN - /** - * Check to see if the new cores exceeds max cores. This check is only - * done if NEW.int_max_cores is equal to OLD.int_max_cores and - * NEW.int_cores > OLD.int_cores, otherwise this error will be thrown - * at the wrong time. - **/ - IF :NEW.int_cores > :NEW.int_max_cores THEN - Raise_application_error(-20021, 'job has exceeded max cores'); - END IF; -END; --- SPLIT HERE! - -CREATE TRIGGER "TIER_JOB" BEFORE UPDATE ON job_resource -FOR EACH ROW -BEGIN - /** calculates new tier **/ - :new.float_tier := tier(:new.int_cores,:new.int_min_cores); -END; --- SPLIT HERE! - -CREATE TRIGGER "VERIFY_JOB_LOCAL" BEFORE UPDATE ON job_local -FOR EACH ROW - WHEN ( NEW.int_max_cores = OLD.int_max_cores AND NEW.int_cores > OLD.int_cores) BEGIN - /** - * Check to see if the new cores exceeds max cores. This check is only - * done if NEW.int_max_cores is equal to OLD.int_max_cores and - * NEW.int_cores > OLD.int_cores, otherwise this error will be thrown - * when people lower the max. - **/ - IF :NEW.int_cores > :NEW.int_max_cores THEN - Raise_application_error(-20021, 'job local has exceeded max cores'); - END IF; -END; --- SPLIT HERE! - -CREATE TRIGGER "TIER_FOLDER" BEFORE UPDATE ON folder_resource -FOR EACH ROW -BEGIN - /** calculates new tier **/ - :new.float_tier := soft_tier(:new.int_cores,:new.int_min_cores); -END; --- SPLIT HERE! - -CREATE TRIGGER "BEFORE_DELETE_FOLDER" BEFORE DELETE ON folder -FOR EACH ROW -BEGIN - DELETE FROM folder_level WHERE pk_folder = :old.pk_folder; - DELETE FROM folder_resource WHERE pk_folder = :old.pk_folder; -END; --- SPLIT HERE! - -CREATE TRIGGER "AFTER_INSERT_FOLDER" AFTER INSERT ON folder -FOR EACH ROW -DECLARE - int_level NUMERIC(16,0) :=0; -BEGIN - IF :new.pk_parent_folder IS NOT NULL THEN - SELECT folder_level.int_level + 1 INTO int_level FROM folder_level WHERE pk_folder = :new.pk_parent_folder; - END IF; - INSERT INTO folder_level (pk_folder_level,pk_folder,int_level) VALUES (:new.pk_folder, :new.pk_folder, int_level); - INSERT INTO folder_resource (pk_folder_resource,pk_folder) VALUES (:new.pk_folder, :new.pk_folder); -END; --- SPLIT HERE! - -CREATE TRIGGER "BEFORE_INSERT_FOLDER" BEFORE INSERT ON folder -FOR EACH ROW -BEGIN - IF :new.pk_parent_folder IS NULL THEN - :new.b_default := 1; - END IF; -END; --- SPLIT HERE! - -CREATE TRIGGER "BEFORE_INSERT_PROC" BEFORE INSERT ON proc -FOR EACH ROW -BEGIN - IF :new.int_cores_reserved <= 0 THEN - Raise_application_error(-20010, 'failed to allocate proc, tried to allocate 0 cores'); - END IF; -END; --- SPLIT HERE! - -CREATE TRIGGER "UPDATE_PROC_UPDATE_LAYER" AFTER UPDATE ON proc -FOR EACH ROW - WHEN (new.pk_layer != old.pk_layer) BEGIN - FOR lr IN ( - SELECT - pk_layer - FROM - layer_stat - WHERE - pk_layer IN (:old.pk_layer,:new.pk_layer) - ORDER BY layer_stat.pk_layer DESC - ) LOOP - - IF lr.pk_layer = :old.pk_layer THEN - - UPDATE layer_resource SET - int_cores = int_cores - :old.int_cores_reserved - WHERE - pk_layer = :old.pk_layer; - - ELSE - - UPDATE layer_resource SET - int_cores = int_cores + :new.int_cores_reserved - WHERE - pk_layer = :new.pk_layer; - END IF; - - END LOOP; -END; --- SPLIT HERE! - -CREATE TRIGGER "UPGRADE_PROC_MEMORY_USAGE" AFTER UPDATE ON proc -FOR EACH ROW - WHEN (NEW.int_mem_reserved != OLD.int_mem_reserved) BEGIN - UPDATE host SET - int_mem_idle = int_mem_idle - (:new.int_mem_reserved - :old.int_mem_reserved) - WHERE - pk_host = :new.pk_host; -END; --- SPLIT HERE! - -CREATE TRIGGER "UPDATE_FRAME_WAIT_TO_DEP" BEFORE UPDATE ON frame -FOR EACH ROW - WHEN (NEW.int_depend_count > 0 AND NEW.str_state IN ('Dead','Succeeded','Waiting','Checkpoint')) BEGIN - :NEW.str_state := 'Depend'; - :NEW.ts_updated := systimestamp; - :NEW.int_version := :NEW.int_version + 1; -END; --- SPLIT HERE! - -CREATE TRIGGER "UPDATE_FRAME_EATEN" BEFORE UPDATE ON frame -FOR EACH ROW - WHEN (NEW.str_state='Eaten' AND OLD.str_state='Succeeded') BEGIN - :NEW.str_state :='Succeeded'; -END; --- SPLIT HERE! - -CREATE TRIGGER "UPDATE_FRAME_DEP_TO_WAIT" BEFORE UPDATE ON frame -FOR EACH ROW - WHEN (OLD.int_depend_count > 0 AND NEW.int_depend_count < 1 AND OLD.str_state='Depend') BEGIN - :NEW.str_state := 'Waiting'; - :NEW.ts_updated := systimestamp; - :NEW.int_version := :NEW.int_version + 1; -END; --- SPLIT HERE! - -CREATE TRIGGER "FRAME_HISTORY_OPEN" AFTER UPDATE ON frame -FOR EACH ROW - WHEN (NEW.str_state != OLD.str_state) DECLARE - str_pk_alloc VARCHAR2(36) := null; - int_checkpoint integer := 0; -BEGIN - - IF :old.str_state = 'Running' THEN - - IF :new.int_exit_status = 299 THEN - - EXECUTE IMMEDIATE - 'DELETE FROM - frame_history - WHERE - int_ts_stopped = 0 AND pk_frame=:1' - USING - :new.pk_frame; - - ELSE - If :new.str_state = 'Checkpoint' THEN - int_checkpoint := 1; - END IF; - - EXECUTE IMMEDIATE - 'UPDATE - frame_history - SET - int_mem_max_used=:1, - int_ts_stopped=:2, - int_exit_status=:3, - int_checkpoint_count=:4 - WHERE - int_ts_stopped = 0 AND pk_frame=:5' - USING - :new.int_mem_max_used, - epoch(systimestamp), - :new.int_exit_status, - int_checkpoint, - :new.pk_frame; - END IF; - END IF; - - IF :new.str_state = 'Running' THEN - - SELECT pk_alloc INTO str_pk_alloc FROM host WHERE str_name=:new.str_host; - - EXECUTE IMMEDIATE - 'INSERT INTO - frame_history - ( - pk_frame, - pk_layer, - pk_job, - str_name, - str_state, - int_cores, - int_mem_reserved, - str_host, - int_ts_started, - pk_alloc - ) - VALUES - (:1,:2,:3,:4,:5,:6,:7,:8,:9,:10)' - USING :new.pk_frame, - :new.pk_layer, - :new.pk_job, - :new.str_name, - 'Running', - :new.int_cores, - :new.int_mem_reserved, - :new.str_host, - epoch(systimestamp), - str_pk_alloc; - END IF; - - -EXCEPTION - /** - * When we first roll this out then job won't be in the historical - * table, so frames on existing jobs will fail unless we catch - * and eat the exceptions. - **/ - WHEN OTHERS THEN - NULL; -END; --- SPLIT HERE! - -CREATE TRIGGER "UPDATE_FRAME_CHECKPOINT_STATE" BEFORE UPDATE ON frame -FOR EACH ROW - WHEN (NEW.str_state='Waiting' AND OLD.str_state='Running' AND NEW.str_checkpoint_state IN ('Enabled', 'Copying')) BEGIN - :NEW.str_state :='Checkpoint'; -END; --- SPLIT HERE! - -CREATE TRIGGER "UPDATE_FRAME_STATUS_COUNTS" AFTER UPDATE ON frame -FOR EACH ROW - WHEN (old.str_state != 'Setup' AND old.str_state != new.str_state) DECLARE - s_old_status_col VARCHAR2(32); - s_new_status_col VARCHAR2(32); -BEGIN - s_old_status_col := 'int_' || :old.str_state || '_count'; - s_new_status_col := 'int_' || :new.str_state || '_count'; - - EXECUTE IMMEDIATE 'UPDATE layer_stat SET ' || s_old_status_col || '=' || s_old_status_col || ' -1, ' - || s_new_status_col || ' = ' || s_new_status_col || '+1 WHERE pk_layer=:1' USING :new.pk_layer; - - EXECUTE IMMEDIATE 'UPDATE job_stat SET ' || s_old_status_col || '=' || s_old_status_col || ' -1, ' - || s_new_status_col || ' = ' || s_new_status_col || '+1 WHERE pk_job=:1' USING :new.pk_job; -END; --- SPLIT HERE! - -CREATE TRIGGER "VERIFY_SUBSCRIPTION" BEFORE UPDATE ON subscription -FOR EACH ROW - WHEN ( NEW.int_burst = OLD.int_burst AND NEW.int_cores > OLD.int_cores) BEGIN - /** - * Check to see if adding more procs will push the show over - * its subscription size. This check is only done when - * new.int_burst = old.int_burst and new.int_cores > old.int cores, - * otherwise this error would be thrown at the wrong time. - **/ - IF :NEW.int_cores > :NEW.int_burst THEN - Raise_application_error(-20022, 'subscription has exceeded burst size'); - END IF; -END; --- SPLIT HERE! - -CREATE TRIGGER "TIER_SUBSCRIPTION" BEFORE UPDATE ON subscription -FOR EACH ROW -BEGIN - /* calcultes a soft tier */ - :new.float_tier := tier(:new.int_cores, :new.int_size); -END; --- SPLIT HERE! - -CREATE TRIGGER "POINT_TIER" BEFORE UPDATE ON point -FOR EACH ROW -BEGIN - /* calcultes a soft tier */ - :new.float_tier := soft_tier(:new.int_cores, :new.int_min_cores); -END; --- SPLIT HERE! - -CREATE TRIGGER TBIU_FRAME_HISTORY -before INSERT OR UPDATE -ON frame_history -REFERENCING NEW AS NEW OLD AS OLD -FOR EACH row -BEGIN - :new.dt_last_modified := sysdate; -END; --- SPLIT HERE! - -CALL ctx_ddl.create_index_set('tag_set') --- SPLIT HERE! -CALL ctx_ddl.add_index('tag_set','str_name') --- SPLIT HERE! -create index i_host_str_tags ON host (str_tags) INDEXTYPE IS ctxsys.ctxcat parameters ('INDEX SET tag_set') diff --git a/cuebot/src/main/resources/conf/spring/applicationContext-dao-oracle.xml b/cuebot/src/main/resources/conf/spring/applicationContext-dao-oracle.xml deleted file mode 100644 index afe238bcf..000000000 --- a/cuebot/src/main/resources/conf/spring/applicationContext-dao-oracle.xml +++ /dev/null @@ -1,147 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml index b7f6e3d14..24bd94195 100644 --- a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml +++ b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml @@ -109,7 +109,6 @@ - diff --git a/cuebot/src/main/resources/conf/spring/applicationContext-trackit.xml b/cuebot/src/main/resources/conf/spring/applicationContext-trackit.xml index 0d8a913e8..a6f8e0f48 100644 --- a/cuebot/src/main/resources/conf/spring/applicationContext-trackit.xml +++ b/cuebot/src/main/resources/conf/spring/applicationContext-trackit.xml @@ -21,8 +21,4 @@ xsi:schemaLocation=" http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd"> - - - - diff --git a/cuebot/src/main/resources/log4j.properties b/cuebot/src/main/resources/log4j.properties index 0eb102e58..138d514ae 100644 --- a/cuebot/src/main/resources/log4j.properties +++ b/cuebot/src/main/resources/log4j.properties @@ -33,7 +33,6 @@ log4j.logger.com.imageworks.spcue=DEBUG log4j.logger.com.imageworks.spcue.dispatcher.RqdReportManagerService=DEBUG log4j.logger.com.imageworks.spcue.service.HostManagerService=TRACE log4j.logger.com.imageworks.spcue.dispatcher=TRACE -log4j.logger.com.imageworks.spcue.dao.oracle.DispatcherDaoJdbc=DEBUG #log4j.logger.org.springframework=DEBUG diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index f77e10273..82404ee1c 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -1,10 +1,6 @@ cue.proxy = tcp -h cuetest01-vm -p 9019 -t 10000:tcp -h cuetest02-vm -p 9019 -t 10000:tcp -h cuetest03-vm -p 9019 -t 10000 spring.velocity.checkTemplateLocation=false -# Oracle versions of config values will look like: -# datasource.cueDataSource.driverClassName=oracle.jdbc.OracleDriver -# datasource.cueDataSource.jdbc-url=jdbc:oracle:oci:@dbname - datasource.cue-data-source.driver-class-name=org.postgresql.Driver datasource.cue-data-source.jdbc-url=jdbc:postgresql://dbhost/dbname datasource.cue-data-source.username=cue @@ -13,16 +9,6 @@ datasource.cue-data-source.password=password # connection rebalancing. datasource.cue-data-source.maxAge=21600000 -cue.trackit.enabled=false -# If using Oracle trackit, ensure the drivers are installed and use the following config value: -# datasource.trackit-data-source.driver-class-name=oracle.jdbc.OracleDriver -datasource.trackit-data-source.jdbc-url=jdbc:oracle:oci:@dbname -datasource.trackit-data-source.username=element_ro -datasource.trackit-data-source.password=password -# Discard connections after 6 hours, this allows for gradual -# connection rebalancing. -datasource.trackit-data-source.max-age=21600000 - grpc.cue_port=${CUEBOT_GRPC_CUE_PORT:8443} grpc.rqd_server_port=${CUEBOT_GRPC_RQD_SERVER_PORT:8444} grpc.max_message_bytes=104857600 diff --git a/cuebot/src/test/java/com/imageworks/spcue/config/TestAppConfig.java b/cuebot/src/test/java/com/imageworks/spcue/config/TestAppConfig.java index afbc6c6b5..e86126839 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/config/TestAppConfig.java +++ b/cuebot/src/test/java/com/imageworks/spcue/config/TestAppConfig.java @@ -37,26 +37,11 @@ "classpath:conf/spring/applicationContext-grpcServer.xml", "classpath:conf/spring/applicationContext-service.xml", "classpath:conf/spring/applicationContext-jms.xml", - "classpath:conf/spring/applicationContext-trackit.xml", "classpath:conf/spring/applicationContext-criteria.xml"}) @EnableConfigurationProperties @PropertySource({"classpath:opencue.properties"}) public class TestAppConfig { - @Bean - @ConfigurationProperties(prefix="datasource.trackit-data-source") - public DataSource trackitDataSource() { - return DataSourceBuilder.create().build(); - } - - @Configuration - @Conditional(OracleDatabaseCondition.class) - @ImportResource({ - "classpath:conf/spring/applicationContext-oracle-datasource.xml", - "classpath:conf/spring/applicationContext-dao-oracle.xml" - }) - static class OracleEngineConfig {} - @Configuration @Conditional(PostgresDatabaseCondition.class) @ImportResource({ diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/AssumingOracleEngine.java b/cuebot/src/test/java/com/imageworks/spcue/test/AssumingOracleEngine.java deleted file mode 100644 index d705a7240..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/AssumingOracleEngine.java +++ /dev/null @@ -1,58 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.imageworks.spcue.test; - -import org.junit.AssumptionViolatedException; -import org.junit.rules.TestRule; -import org.junit.runner.Description; -import org.junit.runners.model.Statement; - -import com.imageworks.spcue.config.DatabaseEngine; - - -public class AssumingOracleEngine implements TestRule { - - private DatabaseEngine dbEngine; - - public AssumingOracleEngine() {} - - @Override - public Statement apply(Statement base, Description description) { - return new Statement() { - @Override - public void evaluate() throws Throwable { - if (dbEngine == DatabaseEngine.ORACLE) { - base.evaluate(); - } else { - throw new AssumptionViolatedException( - "Current database engine is " + dbEngine.toString() + - ", test requires ORACLE. Skipping"); - } - } - }; - } - - public DatabaseEngine getDbEngine() { - return dbEngine; - } - - public void setDbEngine(DatabaseEngine dbEngine) { - this.dbEngine = dbEngine; - } -} diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/AssumingTrackitEnabled.java b/cuebot/src/test/java/com/imageworks/spcue/test/AssumingTrackitEnabled.java deleted file mode 100644 index 6469ef710..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/AssumingTrackitEnabled.java +++ /dev/null @@ -1,50 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.imageworks.spcue.test; - -import org.junit.AssumptionViolatedException; -import org.junit.rules.TestRule; -import org.junit.runner.Description; -import org.junit.runners.model.Statement; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.env.Environment; - - -public class AssumingTrackitEnabled implements TestRule { - - @Autowired - private Environment env; - - public AssumingTrackitEnabled() {} - - @Override - public Statement apply(Statement base, Description description) { - return new Statement() { - @Override - public void evaluate() throws Throwable { - if (env.getRequiredProperty("cue.trackit.enabled", Boolean.class)) { - base.evaluate(); - } else { - throw new AssumptionViolatedException( - "Trackit is not enabled in this environment. Skipping"); - } - } - }; - } -} diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/TestDatabaseSetup.java b/cuebot/src/test/java/com/imageworks/spcue/test/TestDatabaseSetup.java deleted file mode 100644 index b46977064..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/TestDatabaseSetup.java +++ /dev/null @@ -1,217 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test; - -import java.io.File; -import java.io.FileNotFoundException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Scanner; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public final class TestDatabaseSetup { - private static final String USERNAME = "ct" + System.currentTimeMillis(); - private static final String PASSWORD = "password"; - private String sysPwd; - private String dbTns = "oraxelocal"; - private static AtomicBoolean setupComplete = new AtomicBoolean(false); - - public TestDatabaseSetup() { - String tns = System.getenv("CUEBOT_DB_TNS"); - if (tns != null) { - setDbTns(tns); - } - String pwd = System.getenv("CUEBOT_DB_SYS_PWD"); - if (pwd == null) { - throw new RuntimeException("CUEBOT_DB_SYS_PWD must be set in your environment"); - } - setSysPwd(pwd); - } - - private String getDbTns() { - return dbTns; - } - - private void setDbTns(String dbTns) { - this.dbTns = dbTns; - } - - private String getSysPwd() { - return sysPwd; - } - - private void setSysPwd(String sysPwd) { - this.sysPwd = sysPwd; - } - - public String getUsername() { - return USERNAME; - } - - public String getPassword() { - return PASSWORD; - } - - public String getUrl() { - return "jdbc:oracle:oci:@" + getDbTns(); - } - - public void create() throws Exception { - if (!setupComplete.compareAndSet(false, true)) { - return; - } - - if (System.getenv("TNS_ADMIN") == null) { - throw new RuntimeException("TNS_ADMIN must be set in your environment"); - } - System.setProperty("oracle.net.tns_admin", System.getenv("TNS_ADMIN")); - System.out.println("CREATING CUE3 TEST USER"); - Connection sysConn = DriverManager.getConnection( - "jdbc:oracle:oci:@" + getDbTns(), - "sys as sysdba", - getSysPwd() - ); - - purgeOldUsers(sysConn); - - Statement stmt = null; - try { - stmt = sysConn.createStatement(); - stmt.execute("CREATE USER " + USERNAME + " IDENTIFIED BY " + PASSWORD); - stmt.execute("GRANT CONNECT, RESOURCE, DBA TO " + USERNAME); - } finally { - if (stmt != null) { - stmt.close(); - } - - if (sysConn != null) { - sysConn.close(); - } - } - - // The spring junit runner doesn't want to call the destroy-method on this bean, even if we tell it to in the XML. As such, - // we're adding a shutdown hook here to ensure that the database gets cleaned up. Newer version of spring have a class-level - // @DirtiesContext annotation that you can use to tell spring to destroy everything after the test class runs. - Runtime.getRuntime().addShutdownHook(new Thread() { - @Override - public void run() { - try { - TestDatabaseSetup.this.destroy(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - }); - - System.out.println("CREATING CUE3 TEST DATABASE " + USERNAME); - Connection conn = DriverManager.getConnection( - "jdbc:oracle:oci:@" + getDbTns(), - USERNAME, - PASSWORD - ); - stmt = null; - try { - stmt = conn.createStatement(); - - ClassLoader classLoader = getClass().getClassLoader(); - applySqlFile(new File(classLoader.getResource("conf/ddl/oracle/schema.sql").getFile()), stmt); - applySqlFile(new File(classLoader.getResource("conf/ddl/oracle/test_data.sql").getFile()), stmt); - } finally { - if (stmt != null) { - stmt.close(); - } - - if (conn != null) { - conn.close(); - } - } - } - - public void destroy() throws Exception { - System.out.println("DESTROYING CUE3 TEST DATABASE " + USERNAME); - try (Connection conn = DriverManager.getConnection( - "jdbc:oracle:oci:@" + getDbTns(), - "sys as sysdba", - getSysPwd() - )) { - purgeUser(conn, USERNAME); - } - } - - private void purgeOldUsers(Connection conn) throws SQLException { - long EXPIRE_TIME = TimeUnit.MILLISECONDS.convert(6, TimeUnit.HOURS); - long now = System.currentTimeMillis(); - Pattern ct_re = Pattern.compile("^CT(\\d+)$"); - - try ( - Statement stmt = conn.createStatement(); - ResultSet rs = stmt.executeQuery( - "SELECT username FROM dba_users WHERE username LIKE 'CT%'"); - ) { - while (rs.next()) { - String username = rs.getString(1); - Matcher m = ct_re.matcher(username); - if (!m.matches()) { - continue; - } - - long ts = Long.valueOf(m.group(1)); - if (ts >= now - EXPIRE_TIME) { - System.out.println("FOUND NON-EXPIRED USER " + username); - continue; - } - - System.out.println("REMOVING EXPIRED USER " + username); - purgeUser(conn, username); - } - } - } - - private void purgeUser(Connection conn, String username) throws SQLException { - try (Statement rmstmt = conn.createStatement()) { - rmstmt.execute("DROP USER " + username + " CASCADE"); - } - } - - private void applySqlFile(File sqlFile, Statement stmt) throws FileNotFoundException, SQLException { - // http://stackoverflow.com/a/18897411 - String dbCreateScript = new Scanner(sqlFile, "UTF-8").useDelimiter("\\A").next(); - String[] dbCreateScriptPieces = dbCreateScript.split("-- SPLIT HERE!"); - - for (String dbCreateScriptPiece : dbCreateScriptPieces) { - System.out.print("."); - try { - stmt.execute(dbCreateScriptPiece); - } catch (Exception e) { - System.out.println(dbCreateScriptPiece); - throw e; - } - } - System.out.println(); - } -} - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/ActionDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/ActionDaoTests.java deleted file mode 100644 index 6fa99a270..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/ActionDaoTests.java +++ /dev/null @@ -1,208 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import javax.annotation.Resource; - -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.ActionEntity; -import com.imageworks.spcue.FilterEntity; -import com.imageworks.spcue.ShowInterface; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.ActionDao; -import com.imageworks.spcue.dao.FilterDao; -import com.imageworks.spcue.dao.GroupDao; -import com.imageworks.spcue.dao.ShowDao; -import com.imageworks.spcue.grpc.filter.ActionType; -import com.imageworks.spcue.grpc.filter.ActionValueType; -import com.imageworks.spcue.grpc.filter.FilterType; -import com.imageworks.spcue.service.JobManager; -import com.imageworks.spcue.test.AssumingOracleEngine; - -import static org.junit.Assert.assertEquals; - - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class ActionDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - ActionDao actionDao; - - @Resource - FilterDao filterDao; - - @Resource - ShowDao showDao; - - @Resource - GroupDao groupDao; - - @Resource - JobManager jobManager; - - private static String FILTER_NAME = "test_filter"; - - public ShowInterface getShow() { - return showDao.getShowDetail("00000000-0000-0000-0000-000000000000"); - } - - public FilterEntity buildFilter() { - FilterEntity filter = new FilterEntity(); - filter.name = FILTER_NAME; - filter.showId = "00000000-0000-0000-0000-000000000000"; - filter.type = FilterType.MATCH_ANY; - filter.enabled = true; - - return filter; - } - - @Test - @Transactional - @Rollback(true) - public void testCreateAction() { - - FilterEntity f = buildFilter(); - filterDao.insertFilter(f); - - ActionEntity a1 = new ActionEntity(); - a1.type = ActionType.PAUSE_JOB; - a1.filterId = f.getFilterId(); - a1.booleanValue = true; - a1.valueType = ActionValueType.BOOLEAN_TYPE; - actionDao.createAction(a1); - - ActionEntity a2 = new ActionEntity(); - a2.type = ActionType.MOVE_JOB_TO_GROUP; - a2.filterId = f.getFilterId(); - a2.groupValue = groupDao.getRootGroupId(getShow()); - a2.valueType = ActionValueType.GROUP_TYPE; - actionDao.createAction(a2); - - ActionEntity a3 = new ActionEntity(); - a3.type = ActionType.SET_JOB_MAX_CORES; - a3.filterId = f.getFilterId(); - a3.floatValue = 1f; - a3.valueType = ActionValueType.FLOAT_TYPE; - actionDao.createAction(a3); - - ActionEntity a4 = new ActionEntity(); - a4.type = ActionType.SET_JOB_MIN_CORES; - a4.filterId = f.getFilterId(); - a4.floatValue = 1; - a4.valueType = ActionValueType.FLOAT_TYPE; - actionDao.createAction(a4); - - ActionEntity a5 = new ActionEntity(); - a5.type = ActionType.STOP_PROCESSING; - a5.filterId = f.getFilterId(); - a5.valueType = ActionValueType.NONE_TYPE; - actionDao.createAction(a5); - } - - @Test - @Transactional - @Rollback(true) - public void testDeleteAction() { - - FilterEntity f = buildFilter(); - filterDao.insertFilter(f); - - ActionEntity a = new ActionEntity(); - a.type = ActionType.STOP_PROCESSING; - a.filterId = f.getFilterId(); - a.valueType = ActionValueType.NONE_TYPE; - actionDao.createAction(a); - actionDao.deleteAction(a); - } - - @Test - @Transactional - @Rollback(true) - public void testGetAction() { - FilterEntity f = buildFilter(); - filterDao.insertFilter(f); - - ActionEntity a = new ActionEntity(); - a.type = ActionType.STOP_PROCESSING; - a.filterId = f.getFilterId(); - a.valueType = ActionValueType.NONE_TYPE; - actionDao.createAction(a); - actionDao.getAction(a); - actionDao.getAction(a.getActionId()); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateAction() { - FilterEntity f = buildFilter(); - filterDao.insertFilter(f); - - ActionEntity a = new ActionEntity(); - a.type = ActionType.STOP_PROCESSING; - a.filterId = f.getFilterId(); - a.name = null; - a.valueType = ActionValueType.NONE_TYPE; - actionDao.createAction(a); - - a.floatValue = 1f; - a.type = ActionType.SET_JOB_MIN_CORES; - a.valueType = ActionValueType.FLOAT_TYPE; - - actionDao.updateAction(a); - - assertEquals(Integer.valueOf(1), - jdbcTemplate.queryForObject( - "SELECT float_value FROM action WHERE pk_action=?", - Integer.class, a.id)); - } - - @Test - @Transactional - @Rollback(true) - public void testGetActions() { - FilterEntity f = buildFilter(); - filterDao.insertFilter(f); - - ActionEntity a = new ActionEntity(); - a.type = ActionType.STOP_PROCESSING; - a.filterId = f.getFilterId(); - a.name = null; - a.valueType = ActionValueType.NONE_TYPE; - actionDao.createAction(a); - - actionDao.getActions(f); - } -} - - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/AllocationDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/AllocationDaoTests.java deleted file mode 100644 index 98dacfc5e..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/AllocationDaoTests.java +++ /dev/null @@ -1,185 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import javax.annotation.Resource; - -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.AllocationEntity; -import com.imageworks.spcue.FacilityInterface; -import com.imageworks.spcue.ShowEntity; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.AllocationDao; -import com.imageworks.spcue.dao.FacilityDao; -import com.imageworks.spcue.service.AdminManager; -import com.imageworks.spcue.test.AssumingOracleEngine; - -import static org.junit.Assert.assertEquals; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class AllocationDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - AllocationDao allocDao; - - @Resource - FacilityDao facilityDao; - - @Resource - AdminManager adminManager; - - public static final String ALLOC_FQN = "spi.test_alloc"; - public static final String ALLOC_NAME = "test_alloc"; - public static final String ALLOC_TAG = "test"; - - private AllocationEntity alloc; - - @Before - public void before() { - - alloc = new AllocationEntity(); - alloc.name = ALLOC_NAME; - alloc.tag = ALLOC_TAG; - - allocDao.insertAllocation( - facilityDao.getFacility("spi"), alloc); - } - - @Test - @Transactional - @Rollback(true) - public void testGetAllocation() { - allocDao.getAllocationEntity(alloc.getId()); - } - - @Test - @Transactional - @Rollback(true) - public void testFindAllocation() { - FacilityInterface f = facilityDao.getFacility("spi"); - allocDao.findAllocationEntity(f.getName(), ALLOC_NAME); - } - - @Test - @Transactional - @Rollback(true) - public void testFindAllocation2() { - FacilityInterface f = facilityDao.getFacility("spi"); - allocDao.findAllocationEntity(ALLOC_FQN); - } - - - @Test - @Transactional - @Rollback(true) - public void testDeleteAllocation() { - allocDao.deleteAllocation(alloc); - } - - @Test - @Transactional - @Rollback(true) - public void testDeleteAllocationWithProc() { - - // Use the alloc so deleting triggers it just to be disaled. - ShowEntity show = adminManager.getShowEntity( - "00000000-0000-0000-0000-000000000000"); - adminManager.createSubscription(show, alloc, 10, 10); - allocDao.deleteAllocation(alloc); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT COUNT(1) FROM alloc WHERE pk_alloc=? AND b_enabled = 0", - Integer.class, alloc.getAllocationId())); - - assertEquals(ALLOC_FQN, jdbcTemplate.queryForObject( - "SELECT str_name FROM alloc WHERE pk_alloc=? AND b_enabled = 0", - String.class, alloc.getAllocationId())); - - // Now re-enable it. - allocDao.insertAllocation(facilityDao.getDefaultFacility(), alloc); - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT COUNT(1) FROM alloc WHERE pk_alloc=? AND b_enabled = 1", - Integer.class, alloc.getAllocationId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateAllocationName() { - allocDao.updateAllocationName(alloc, "frickjack"); - assertEquals("spi.frickjack", jdbcTemplate.queryForObject( - "SELECT str_name FROM alloc WHERE pk_alloc=?", - String.class, - alloc.getId())); - } - - @Test(expected = IllegalArgumentException.class) - @Transactional - @Rollback(true) - public void testUpdateAllocationNameBad() { - allocDao.updateAllocationName(alloc, "spi.frickjack"); - assertEquals("spi.frickjack", jdbcTemplate.queryForObject( - "SELECT str_name FROM alloc WHERE pk_alloc=?", - String.class, alloc.getId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateAllocationTag() { - allocDao.updateAllocationTag(alloc, "foo"); - assertEquals("foo",jdbcTemplate.queryForObject( - "SELECT str_tag FROM alloc WHERE pk_alloc=?", - String.class, alloc.getId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateAllocationBillable() { - allocDao.updateAllocationBillable(alloc, false); - - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT b_billable FROM alloc WHERE pk_alloc=?", - Integer.class, alloc.getId())); - - allocDao.updateAllocationBillable(alloc, true); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT b_billable FROM alloc WHERE pk_alloc=?", - Integer.class, alloc.getId())); - } -} - - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/BookingDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/BookingDaoTests.java deleted file mode 100644 index 58b47bfdb..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/BookingDaoTests.java +++ /dev/null @@ -1,456 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import java.io.File; -import javax.annotation.Resource; - -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.DispatchHost; -import com.imageworks.spcue.FrameInterface; -import com.imageworks.spcue.JobDetail; -import com.imageworks.spcue.LayerInterface; -import com.imageworks.spcue.LocalHostAssignment; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.BookingDao; -import com.imageworks.spcue.dao.DispatcherDao; -import com.imageworks.spcue.dao.HostDao; -import com.imageworks.spcue.dao.ProcDao; -import com.imageworks.spcue.grpc.host.HardwareState; -import com.imageworks.spcue.grpc.renderpartition.RenderPartition; -import com.imageworks.spcue.grpc.renderpartition.RenderPartitionType; -import com.imageworks.spcue.grpc.report.RenderHost; -import com.imageworks.spcue.service.AdminManager; -import com.imageworks.spcue.service.HostManager; -import com.imageworks.spcue.service.JobLauncher; -import com.imageworks.spcue.service.JobManager; -import com.imageworks.spcue.service.Whiteboard; -import com.imageworks.spcue.test.AssumingOracleEngine; -import com.imageworks.spcue.util.CueUtil; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class BookingDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - HostManager hostManager; - - @Resource - AdminManager adminManager; - - @Resource - JobLauncher jobLauncher; - - @Resource - JobManager jobManager; - - @Resource - HostDao hostDao; - - @Resource - BookingDao bookingDao; - - @Resource - DispatcherDao dispatcherDao; - - @Resource - ProcDao procDao; - - @Resource - Whiteboard whiteboard; - - public DispatchHost createHost() { - RenderHost host = RenderHost.newBuilder() - .setName("test_host") - .setBootTime(1192369572) - .setFreeMcp(76020) - .setFreeMem(53500) - .setFreeSwap(20760) - .setLoad(1) - .setTotalMcp(195430) - .setTotalMem((int) CueUtil.GB16) - .setTotalSwap((int) CueUtil.GB16) - .setNimbyEnabled(false) - .setNumProcs(2) - .setCoresPerProc(100) - .setState(HardwareState.UP) - .setFacility("spi") - .addTags("general") - .putAttributes("freeGpu", String.format("%d", CueUtil.MB512)) - .putAttributes("totalGpu", String.format("%d", CueUtil.MB512)) - .build(); - DispatchHost dh = hostManager.createHost(host); - hostManager.setAllocation(dh, - adminManager.findAllocationDetail("spi", "general")); - - return dh; - } - - public JobDetail launchJob() { - jobLauncher.testMode = true; - jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec.xml")); - JobDetail d = jobManager.findJobDetail("pipe-dev.cue-testuser_shell_v1"); - jobManager.setJobPaused(d, false); - return d; - } - - @Test - @Transactional - @Rollback(true) - public void insertLocalJobAssignment() { - - DispatchHost h = createHost(); - JobDetail j = launchJob(); - - LocalHostAssignment lja = new LocalHostAssignment(); - lja.setMaxCoreUnits(200); - lja.setMaxMemory(CueUtil.GB4); - lja.setMaxGpu(1); - lja.setThreads(2); - - bookingDao.insertLocalHostAssignment(h, j, lja); - - - assertEquals(Integer.valueOf(2), jdbcTemplate.queryForObject( - "SELECT int_threads FROM host_local WHERE pk_job=?", - Integer.class, j.getJobId())); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT int_gpu_max FROM host_local WHERE pk_job=?", - Integer.class, j.getJobId())); - - assertEquals(Integer.valueOf(200), jdbcTemplate.queryForObject( - "SELECT int_cores_max FROM host_local WHERE pk_job=?", - Integer.class, j.getJobId())); - - assertEquals(Long.valueOf(CueUtil.GB4), jdbcTemplate.queryForObject( - "SELECT int_mem_max FROM host_local WHERE pk_job=?", - Long.class, j.getJobId())); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT int_gpu_max FROM host_local WHERE pk_job=?", - Integer.class, j.getJobId())); - - assertEquals(Integer.valueOf(200), jdbcTemplate.queryForObject( - "SELECT int_cores_idle FROM host_local WHERE pk_job=?", - Integer.class, j.getJobId())); - - assertEquals(Long.valueOf(CueUtil.GB4), jdbcTemplate.queryForObject( - "SELECT int_mem_idle FROM host_local WHERE pk_job=?", - Long.class, j.getJobId())); - } - - @Test - @Transactional - @Rollback(true) - public void insertLocalLayerAssignment() { - - DispatchHost h = createHost(); - JobDetail j = launchJob(); - LayerInterface layer = jobManager.getLayers(j).get(0); - - LocalHostAssignment lja = new LocalHostAssignment(); - lja.setMaxCoreUnits(200); - lja.setMaxMemory(CueUtil.GB4); - lja.setMaxGpu(1); - lja.setThreads(2); - - bookingDao.insertLocalHostAssignment(h, layer, lja); - - assertEquals(layer.getLayerId(), jdbcTemplate.queryForObject( - "SELECT pk_layer FROM host_local WHERE pk_host_local=?", - String.class, lja.getId())); - - assertEquals(RenderPartitionType.LAYER_PARTITION.toString(), - jdbcTemplate.queryForObject( - "SELECT str_type FROM host_local WHERE pk_host_local=?", - String.class, lja.getId())); - - assertEquals(Integer.valueOf(2), jdbcTemplate.queryForObject( - "SELECT int_threads FROM host_local WHERE pk_job=?", - Integer.class, j.getJobId())); - - assertEquals(Integer.valueOf(200), jdbcTemplate.queryForObject( - "SELECT int_cores_max FROM host_local WHERE pk_job=?", - Integer.class, j.getJobId())); - - assertEquals(Long.valueOf(CueUtil.GB4), jdbcTemplate.queryForObject( - "SELECT int_mem_max FROM host_local WHERE pk_job=?", - Long.class, j.getJobId())); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT int_gpu_max FROM host_local WHERE pk_job=?", - Integer.class, j.getJobId())); - - assertEquals(Integer.valueOf(200), jdbcTemplate.queryForObject( - "SELECT int_cores_idle FROM host_local WHERE pk_job=?", - Integer.class, j.getJobId())); - - assertEquals(Long.valueOf(CueUtil.GB4), jdbcTemplate.queryForObject( - "SELECT int_mem_idle FROM host_local WHERE pk_job=?", - Long.class, j.getJobId())); - } - - @Test - @Transactional - @Rollback(true) - public void insertLocalFrameAssignment() { - - DispatchHost h = createHost(); - JobDetail j = launchJob(); - LayerInterface layer = jobManager.getLayers(j).get(0); - FrameInterface frame = jobManager.findFrame(layer, 1); - - LocalHostAssignment lja = new LocalHostAssignment(); - lja.setMaxCoreUnits(200); - lja.setMaxMemory(CueUtil.GB4); - lja.setMaxGpu(1); - lja.setThreads(2); - - bookingDao.insertLocalHostAssignment(h, frame, lja); - - assertEquals(frame.getFrameId(), jdbcTemplate.queryForObject( - "SELECT pk_frame FROM host_local WHERE pk_host_local=?", - String.class, lja.getId())); - - assertEquals(RenderPartitionType.FRAME_PARTITION.toString(), - jdbcTemplate.queryForObject( - "SELECT str_type FROM host_local WHERE pk_host_local=?", - String.class, lja.getId())); - - assertEquals(Integer.valueOf(2), jdbcTemplate.queryForObject( - "SELECT int_threads FROM host_local WHERE pk_job=?", - Integer.class, j.getJobId())); - - assertEquals(Integer.valueOf(200), jdbcTemplate.queryForObject( - "SELECT int_cores_max FROM host_local WHERE pk_job=?", - Integer.class, j.getJobId())); - - assertEquals(Long.valueOf(CueUtil.GB4), jdbcTemplate.queryForObject( - "SELECT int_mem_max FROM host_local WHERE pk_job=?", - Long.class, j.getJobId())); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT int_gpu_max FROM host_local WHERE pk_job=?", - Integer.class, j.getJobId())); - - assertEquals(Integer.valueOf(200), jdbcTemplate.queryForObject( - "SELECT int_cores_idle FROM host_local WHERE pk_job=?", - Integer.class, j.getJobId())); - - assertEquals(Long.valueOf(CueUtil.GB4), jdbcTemplate.queryForObject( - "SELECT int_mem_idle FROM host_local WHERE pk_job=?", - Long.class, j.getJobId())); - } - - @Test - @Transactional - @Rollback(true) - public void testGetLocalJobAssignment() { - - DispatchHost h = createHost(); - JobDetail j = launchJob(); - - LocalHostAssignment lja = new LocalHostAssignment(); - lja.setMaxCoreUnits(200); - lja.setMaxMemory(CueUtil.GB4); - lja.setThreads(2); - lja.setMaxGpu(1); - - bookingDao.insertLocalHostAssignment(h, j, lja); - - LocalHostAssignment lja2 = bookingDao.getLocalJobAssignment(h.getHostId(), - j.getJobId()); - - assertEquals(lja.getMaxCoreUnits(), lja2.getMaxCoreUnits()); - assertEquals(lja.getMaxMemory(), lja2.getMaxMemory()); - assertEquals(lja.getMaxGpu(), lja2.getMaxGpu()); - assertEquals(lja.getThreads(), lja2.getThreads()); - - } - - @Test - @Transactional - @Rollback(true) - public void testGetRenderPartition() { - - DispatchHost h = createHost(); - JobDetail j = launchJob(); - - LocalHostAssignment lja = new LocalHostAssignment(); - lja.setMaxCoreUnits(200); - lja.setMaxMemory(CueUtil.GB4); - lja.setThreads(2); - lja.setMaxGpu(1); - - bookingDao.insertLocalHostAssignment(h, j, lja); - - LocalHostAssignment lja2 = bookingDao.getLocalJobAssignment(h.getHostId(), - j.getJobId()); - - assertEquals(lja.getMaxCoreUnits(), lja2.getMaxCoreUnits()); - assertEquals(lja.getMaxMemory(), lja2.getMaxMemory()); - assertEquals(lja.getThreads(), lja2.getThreads()); - assertEquals(lja.getMaxGpu(), lja2.getMaxGpu()); - - RenderPartition rp = whiteboard.getRenderPartition(lja2); - - assertEquals(lja2.getMaxCoreUnits(), rp.getMaxCores()); - assertEquals(lja2.getMaxMemory(), rp.getMaxMemory()); - assertEquals(lja2.getThreads(), rp.getThreads()); - logger.info("--------------------"); - logger.info(lja2.getMaxGpu()); - logger.info(rp.getMaxGpu()); - assertEquals(lja2.getMaxGpu(), rp.getMaxGpu()); - assertEquals(h.getName(), rp.getHost()); - assertEquals(j.getName(), rp.getJob()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetProcs() { - - DispatchHost h = createHost(); - JobDetail j = launchJob(); - - LocalHostAssignment lja = new LocalHostAssignment(); - lja.setMaxCoreUnits(200); - lja.setMaxMemory(CueUtil.GB4); - lja.setThreads(2); - lja.setMaxGpu(1); - - bookingDao.insertLocalHostAssignment(h, j, lja); - - assertEquals(0, procDao.findVirtualProcs(lja).size()); - } - - @Test - @Transactional - @Rollback(true) - public void updateMaxCores() { - - DispatchHost h = createHost(); - JobDetail j = launchJob(); - - LocalHostAssignment lja = new LocalHostAssignment(); - lja.setMaxCoreUnits(200); - lja.setMaxMemory(CueUtil.GB4); - lja.setThreads(2); - lja.setMaxGpu(1); - - bookingDao.insertLocalHostAssignment(h, j, lja); - assertTrue(bookingDao.updateMaxCores(lja, 100)); - assertEquals(Integer.valueOf(100), jdbcTemplate.queryForObject( - "SELECT int_cores_max FROM host_local WHERE pk_host=?", - Integer.class, h.getHostId())); - - LocalHostAssignment lj2 = bookingDao.getLocalJobAssignment(lja.id); - - assertEquals(100, lj2.getIdleCoreUnits()); - assertEquals(100, lj2.getMaxCoreUnits()); - - bookingDao.updateMaxCores(lja, 200); - - lj2 = bookingDao.getLocalJobAssignment(lja.id); - - assertEquals(200, lj2.getIdleCoreUnits()); - assertEquals(200, lj2.getMaxCoreUnits()); - } - - @Test - @Transactional - @Rollback(true) - public void updateMaxMemory() { - - DispatchHost h = createHost(); - JobDetail j = launchJob(); - - LocalHostAssignment lja = new LocalHostAssignment(); - lja.setMaxCoreUnits(200); - lja.setMaxMemory(CueUtil.GB4); - lja.setThreads(2); - lja.setMaxGpu(1); - - bookingDao.insertLocalHostAssignment(h, j, lja); - bookingDao.updateMaxMemory(lja, CueUtil.GB2); - - LocalHostAssignment lj2 = bookingDao.getLocalJobAssignment(lja.id); - - assertEquals(CueUtil.GB2, lj2.getIdleMemory()); - assertEquals(CueUtil.GB2, lj2.getMaxMemory()); - - bookingDao.updateMaxMemory(lja, CueUtil.GB4); - - lj2 = bookingDao.getLocalJobAssignment(lja.id); - - assertEquals(CueUtil.GB4, lj2.getIdleMemory()); - assertEquals(CueUtil.GB4, lj2.getMaxMemory()); -} - - @Test - @Transactional - @Rollback(true) - public void updateMaxGpu() { - - DispatchHost h = createHost(); - JobDetail j = launchJob(); - - LocalHostAssignment lja = new LocalHostAssignment(); - lja.setMaxCoreUnits(200); - lja.setMaxMemory(CueUtil.GB4); - lja.setThreads(2); - lja.setMaxGpu(1); - - bookingDao.insertLocalHostAssignment(h, j, lja); - bookingDao.updateMaxMemory(lja, CueUtil.GB2); - - LocalHostAssignment lj2 = bookingDao.getLocalJobAssignment(lja.id); - - assertEquals(CueUtil.GB2, lj2.getIdleMemory()); - assertEquals(CueUtil.GB2, lj2.getMaxMemory()); - assertEquals(1, lj2.getMaxGpu()); - - bookingDao.updateMaxGpu(lja, 2); - - lj2 = bookingDao.getLocalJobAssignment(lja.id); - - assertEquals(CueUtil.GB2, lj2.getIdleMemory()); - assertEquals(CueUtil.GB2, lj2.getMaxMemory()); - assertEquals(2, lj2.getMaxGpu()); - } -} - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/CommentDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/CommentDaoTests.java deleted file mode 100644 index eecd2b8e2..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/CommentDaoTests.java +++ /dev/null @@ -1,238 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import java.io.File; -import javax.annotation.Resource; - -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.CommentDetail; -import com.imageworks.spcue.DispatchHost; -import com.imageworks.spcue.JobDetail; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.CommentDao; -import com.imageworks.spcue.grpc.host.HardwareState; -import com.imageworks.spcue.grpc.report.RenderHost; -import com.imageworks.spcue.service.HostManager; -import com.imageworks.spcue.service.JobLauncher; -import com.imageworks.spcue.service.JobManager; -import com.imageworks.spcue.test.AssumingOracleEngine; -import com.imageworks.spcue.util.CueUtil; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class CommentDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - CommentDao commentDao; - - @Resource - JobManager jobManager; - - @Resource - JobLauncher jobLauncher; - - @Resource - HostManager hostManager; - - @Before - public void testMode() { - jobLauncher.testMode = true; - jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec.xml")); - } - - @Test - @Transactional - @Rollback(true) - public void testDeleteComment() { - - JobDetail job = jobManager.findJobDetail("pipe-dev.cue-testuser_shell_v1"); - - CommentDetail d = new CommentDetail(); - d.message = "a message"; - d.subject = "a subject"; - d.user = "user"; - - commentDao.insertComment(job, d); - commentDao.deleteComment(d.getId()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetComment() { - - JobDetail job = jobManager.findJobDetail("pipe-dev.cue-testuser_shell_v1"); - - CommentDetail d = new CommentDetail(); - d.message = "a message"; - d.subject = "a subject"; - d.user = "user"; - - commentDao.insertComment(job, d); - - CommentDetail nd = commentDao.getCommentDetail(d.getId()); - - assertEquals(d.message,nd.message); - assertEquals(d.subject,nd.subject); - assertEquals(d.user,nd.user); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertCommentOnJob() { - - JobDetail job = jobManager.findJobDetail("pipe-dev.cue-testuser_shell_v1"); - - CommentDetail d = new CommentDetail(); - d.message = "a message"; - d.subject = "a subject"; - d.user = "user"; - - commentDao.insertComment(job, d); - - CommentDetail nd = commentDao.getCommentDetail(d.getId()); - - assertEquals(d.message,nd.message); - assertEquals(d.subject,nd.subject); - assertEquals(d.user,nd.user); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertCommentOnHost() { - - RenderHost host = RenderHost.newBuilder() - .setName("boo") - .setBootTime(1192369572) - .setFreeMcp(76020) - .setFreeMem(15290520) - .setFreeSwap(2076) - .setLoad(1) - .setTotalMcp(19543) - .setTotalMem(15290520) - .setTotalSwap(2096) - .setNimbyEnabled(false) - .setNumProcs(2) - .setCoresPerProc(400) - .addTags("linux") - .setState(HardwareState.UP) - .setFacility("spi") - .putAttributes("freeGpu", String.format("%d", CueUtil.MB512)) - .putAttributes("totalGpu", String.format("%d", CueUtil.MB512)) - .build(); - - CommentDetail d = new CommentDetail(); - d.message = "a message"; - d.subject = "a subject"; - d.user = "user"; - - DispatchHost h = hostManager.createHost(host); - commentDao.insertComment(h, d); - - assertNotNull(d.id); - - CommentDetail nd = commentDao.getCommentDetail(d.getId()); - - assertEquals(d.message,nd.message); - assertEquals(d.subject,nd.subject); - assertEquals(d.user,nd.user); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateComment() { - - JobDetail job = jobManager.findJobDetail("pipe-dev.cue-testuser_shell_v1"); - - CommentDetail d = new CommentDetail(); - d.message = "a message"; - d.subject = "a subject"; - d.user = "user"; - - commentDao.insertComment(job, d); - - d.message = "no"; - d.subject = "no"; - - commentDao.updateComment(d); - - CommentDetail nd = commentDao.getCommentDetail(d.getId()); - - assertEquals("no",nd.message); - assertEquals("no",nd.subject); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateCommentMessage() { - - JobDetail job = jobManager.findJobDetail("pipe-dev.cue-testuser_shell_v1"); - - CommentDetail d = new CommentDetail(); - d.message = "a message"; - d.subject = "a subject"; - d.user = "user"; - - commentDao.insertComment(job, d); - commentDao.updateCommentMessage(d.getId(), "no"); - CommentDetail nd = commentDao.getCommentDetail(d.getId()); - assertEquals("no",nd.message); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateCommentSubject() { - - JobDetail job = jobManager.findJobDetail("pipe-dev.cue-testuser_shell_v1"); - - CommentDetail d = new CommentDetail(); - d.message = "a message"; - d.subject = "a subject"; - d.user = "user"; - - commentDao.insertComment(job, d); - commentDao.updateCommentSubject(d.getId(), "no"); - CommentDetail nd = commentDao.getCommentDetail(d.getId()); - assertEquals("no",nd.subject); - } -} - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/DeedDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/DeedDaoTests.java deleted file mode 100644 index 788fe040c..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/DeedDaoTests.java +++ /dev/null @@ -1,169 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import javax.annotation.Resource; - -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.DeedEntity; -import com.imageworks.spcue.DispatchHost; -import com.imageworks.spcue.OwnerEntity; -import com.imageworks.spcue.ShowInterface; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.DeedDao; -import com.imageworks.spcue.grpc.host.HardwareState; -import com.imageworks.spcue.grpc.report.RenderHost; -import com.imageworks.spcue.service.AdminManager; -import com.imageworks.spcue.service.HostManager; -import com.imageworks.spcue.service.OwnerManager; -import com.imageworks.spcue.test.AssumingOracleEngine; -import com.imageworks.spcue.util.CueUtil; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class DeedDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - OwnerManager ownerManager; - - @Resource - DeedDao deedDao; - - @Resource - AdminManager adminManager; - - @Resource - HostManager hostManager; - - public DispatchHost createHost() { - - RenderHost host = RenderHost.newBuilder() - .setName("test_host") - .setBootTime(1192369572) - .setFreeMcp(76020) - .setFreeMem(15290520) - .setFreeSwap(2076) - .setLoad(1) - .setTotalMcp(19543) - .setTotalMem((int) CueUtil.GB16) - .setTotalSwap((int) CueUtil.GB16) - .setNimbyEnabled(false) - .setNumProcs(2) - .setCoresPerProc(100) - .addTags("general") - .setState(HardwareState.UP) - .setFacility("spi") - .putAttributes("freeGpu", String.format("%d", CueUtil.MB512)) - .putAttributes("totalGpu", String.format("%d", CueUtil.MB512)) - .build(); - - DispatchHost dh = hostManager.createHost(host); - hostManager.setAllocation(dh, - adminManager.findAllocationDetail("spi", "general")); - - return dh; - } - - @Test - @Transactional - @Rollback(true) - public void testInsertDeed() { - - DispatchHost host = createHost(); - ShowInterface s = adminManager.findShowEntity("pipe"); - OwnerEntity o = ownerManager.createOwner("squarepants", s); - DeedEntity d = deedDao.insertDeed(o, host); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT COUNT(1) FROM deed WHERE pk_deed=?", - Integer.class, d.getId())); - - assertEquals(host.getName(), d.host); - } - - @Test - @Transactional - @Rollback(true) - public void tesDeleteDeed() { - - DispatchHost host = createHost(); - ShowInterface s = adminManager.findShowEntity("pipe"); - OwnerEntity o = ownerManager.createOwner("squarepants", s); - DeedEntity d = deedDao.insertDeed(o, host); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT COUNT(1) FROM deed WHERE pk_deed=?", - Integer.class, d.getId())); - - assertTrue(deedDao.deleteDeed(d)); - - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT COUNT(1) FROM deed WHERE pk_deed=?", - Integer.class, d.getId())); - - assertFalse(deedDao.deleteDeed(d)); - } - - @Test - @Transactional - @Rollback(true) - public void tesGetDeed() { - - DispatchHost host = createHost(); - ShowInterface s = adminManager.findShowEntity("pipe"); - OwnerEntity o = ownerManager.createOwner("squarepants", s); - DeedEntity d = deedDao.insertDeed(o, host); - - DeedEntity d2 = deedDao.getDeed(d.id); - - assertEquals(d, d2); - } - - @Test - @Transactional - @Rollback(true) - public void tesGetDeeds() { - - DispatchHost host = createHost(); - ShowInterface s = adminManager.findShowEntity("pipe"); - OwnerEntity o = ownerManager.createOwner("squarepants", s); - DeedEntity d = deedDao.insertDeed(o, host); - - assertEquals(1, deedDao.getDeeds(o).size()); - assertEquals(d, deedDao.getDeeds(o).get(0)); - } -} - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/DepartmentDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/DepartmentDaoTests.java deleted file mode 100644 index e309d1630..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/DepartmentDaoTests.java +++ /dev/null @@ -1,107 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import javax.annotation.Resource; - -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.DepartmentInterface; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.DepartmentDao; -import com.imageworks.spcue.test.AssumingOracleEngine; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class DepartmentDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - DepartmentDao departmentDao; - - @Test - @Transactional - @Rollback(true) - public void testGetDepartment() { - String dept= "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA0"; - assertEquals(dept, departmentDao.getDepartment(dept).getId()); - assertEquals(dept, departmentDao.getDepartment(dept).getDepartmentId()); - } - - @Test - @Transactional - @Rollback(true) - public void testFindDepartment() { - String dept= "Hair"; - assertEquals(dept, departmentDao.findDepartment(dept).getName()); - } - - @Test - @Transactional - @Rollback(true) - public void testgetDefaultDepartment() { - assertEquals(jdbcTemplate.queryForObject( - "SELECT pk_dept FROM dept WHERE b_default=1", - String.class),departmentDao.getDefaultDepartment().getId()); - } - - @Test - @Transactional - @Rollback(true) - public void testDepartmentExists() { - String dept= "Cloth"; - assertTrue(departmentDao.departmentExists(dept)); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertDepartment() { - String deptName = "TestDept"; - departmentDao.insertDepartment(deptName); - DepartmentInterface d = departmentDao.findDepartment(deptName); - assertEquals(d.getName(), deptName); - } - - @Test - @Transactional - @Rollback(true) - public void testDeleteDepartment() { - String deptName = "TestDept"; - departmentDao.insertDepartment(deptName); - DepartmentInterface d = departmentDao.findDepartment(deptName); - assertEquals(d.getName(), deptName); - departmentDao.deleteDepartment(d); - } -} - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/DependDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/DependDaoTests.java deleted file mode 100644 index e01ed32d4..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/DependDaoTests.java +++ /dev/null @@ -1,455 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import java.io.File; -import javax.annotation.Resource; - -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.FrameDetail; -import com.imageworks.spcue.JobDetail; -import com.imageworks.spcue.LayerInterface; -import com.imageworks.spcue.LightweightDependency; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.DependDao; -import com.imageworks.spcue.dao.FrameDao; -import com.imageworks.spcue.dao.LayerDao; -import com.imageworks.spcue.depend.FrameByFrame; -import com.imageworks.spcue.depend.FrameOnFrame; -import com.imageworks.spcue.depend.FrameOnJob; -import com.imageworks.spcue.depend.FrameOnLayer; -import com.imageworks.spcue.depend.JobOnFrame; -import com.imageworks.spcue.depend.JobOnJob; -import com.imageworks.spcue.depend.JobOnLayer; -import com.imageworks.spcue.depend.LayerOnFrame; -import com.imageworks.spcue.depend.LayerOnJob; -import com.imageworks.spcue.depend.LayerOnLayer; -import com.imageworks.spcue.depend.PreviousFrame; -import com.imageworks.spcue.grpc.depend.DependTarget; -import com.imageworks.spcue.grpc.depend.DependType; -import com.imageworks.spcue.service.DependManager; -import com.imageworks.spcue.service.JobLauncher; -import com.imageworks.spcue.service.JobManager; -import com.imageworks.spcue.service.JobManagerSupport; -import com.imageworks.spcue.test.AssumingOracleEngine; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class DependDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - DependDao dependDao; - - @Resource - FrameDao frameDao; - - @Resource - LayerDao layerDao; - - @Resource - JobManager jobManager; - - @Resource - DependManager dependManager; - - @Resource - JobManagerSupport jobManagerSupport; - - @Resource - JobLauncher jobLauncher; - - @Before - public void launchTestJobs() { - jobLauncher.testMode = true; - jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec_depend_test.xml")); - } - - public JobDetail getJobA() { - return jobManager.findJobDetail("pipe-dev.cue-testuser_depend_test_a"); - } - - public JobDetail getJobB() { - return jobManager.findJobDetail("pipe-dev.cue-testuser_depend_test_b"); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertJobOnJob() { - - JobDetail job_a = getJobA(); - JobDetail job_b = getJobB(); - - JobOnJob depend = new JobOnJob(job_a, job_b); - dependDao.insertDepend(depend); - - LightweightDependency lwd = dependDao.getDepend(depend.getId()); - assertEquals(depend.getId(), lwd.getId()); - assertEquals(DependType.JOB_ON_JOB, lwd.type); - assertEquals(DependTarget.EXTERNAL, lwd.target); - assertTrue(lwd.active); - assertFalse(lwd.anyFrame); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertJobOnLayer() { - - JobDetail job_a = getJobA(); - JobDetail job_b = getJobB(); - - LayerInterface layer = layerDao.findLayer(job_b, "pass_1"); - JobOnLayer depend = new JobOnLayer(job_a, layer); - dependDao.insertDepend(depend); - - LightweightDependency lwd = dependDao.getDepend(depend.getId()); - assertEquals(depend.getId(), lwd.getId()); - assertEquals(DependType.JOB_ON_LAYER, lwd.type); - assertEquals(DependTarget.EXTERNAL, lwd.target); - assertTrue(lwd.active); - assertFalse(lwd.anyFrame); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertJobOnFrame() { - - JobDetail job_a = getJobA(); - JobDetail job_b = getJobB(); - - FrameDetail frame = frameDao.findFrameDetail(job_b, "0001-pass_1"); - JobOnFrame depend = new JobOnFrame(job_a, frame); - dependDao.insertDepend(depend); - - LightweightDependency lwd = dependDao.getDepend(depend.getId()); - assertEquals(depend.getId(), lwd.getId()); - assertEquals(DependType.JOB_ON_FRAME, lwd.type); - assertEquals(DependTarget.EXTERNAL, lwd.target); - assertTrue(lwd.active); - assertFalse(lwd.anyFrame); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertLayerOnJob() { - - JobDetail job_a = getJobA(); - JobDetail job_b = getJobB(); - LayerInterface layer = layerDao.findLayer(job_b, "pass_1"); - - LayerOnJob depend = new LayerOnJob(layer, job_a); - dependDao.insertDepend(depend); - - LightweightDependency lwd = dependDao.getDepend(depend.getId()); - assertEquals(depend.getId(), lwd.getId()); - assertEquals(DependType.LAYER_ON_JOB, lwd.type); - assertEquals(DependTarget.EXTERNAL, lwd.target); - assertTrue(lwd.active); - assertFalse(lwd.anyFrame); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertLayerOnLayer() { - - JobDetail job_a = getJobA(); - JobDetail job_b = getJobB(); - LayerInterface layer_a = layerDao.findLayer(job_a, "pass_1"); - LayerInterface layer_b = layerDao.findLayer(job_b, "pass_1"); - - LayerOnLayer depend = new LayerOnLayer(layer_a, layer_b); - dependDao.insertDepend(depend); - - LightweightDependency lwd = dependDao.getDepend(depend.getId()); - assertEquals(depend.getId(), lwd.getId()); - assertEquals(DependType.LAYER_ON_LAYER, lwd.type); - assertEquals(DependTarget.EXTERNAL, lwd.target); - assertTrue(lwd.active); - assertFalse(lwd.anyFrame); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertLayerOnFrame() { - - JobDetail job_a = getJobA(); - JobDetail job_b = getJobB(); - LayerInterface layer = layerDao.findLayer(job_a, "pass_1"); - FrameDetail frame = frameDao.findFrameDetail(job_b, "0001-pass_1"); - - LayerOnFrame depend = new LayerOnFrame(layer, frame); - dependDao.insertDepend(depend); - - LightweightDependency lwd = dependDao.getDepend(depend.getId()); - assertEquals(depend.getId(), lwd.getId()); - assertEquals(DependType.LAYER_ON_FRAME, lwd.type); - assertEquals(DependTarget.EXTERNAL, lwd.target); - assertTrue(lwd.active); - assertFalse(lwd.anyFrame); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertFrameOnJob() { - - JobDetail job_a = getJobA(); - JobDetail job_b = getJobB(); - FrameDetail frame = frameDao.findFrameDetail(job_b, "0001-pass_1"); - - FrameOnJob depend = new FrameOnJob(frame, job_a); - dependDao.insertDepend(depend); - - LightweightDependency lwd = dependDao.getDepend(depend.getId()); - assertEquals(depend.getId(), lwd.getId()); - assertEquals(DependType.FRAME_ON_JOB, lwd.type); - assertEquals(DependTarget.EXTERNAL, lwd.target); - assertTrue(lwd.active); - assertFalse(lwd.anyFrame); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertFrameOnLayer() { - - JobDetail job_a = getJobA(); - JobDetail job_b = getJobB(); - LayerInterface layer = layerDao.findLayer(job_a, "pass_1"); - FrameDetail frame = frameDao.findFrameDetail(job_b, "0001-pass_1"); - - FrameOnLayer depend = new FrameOnLayer(frame,layer); - dependDao.insertDepend(depend); - - LightweightDependency lwd = dependDao.getDepend(depend.getId()); - assertEquals(depend.getId(), lwd.getId()); - assertEquals(DependType.FRAME_ON_LAYER, lwd.type); - assertEquals(DependTarget.EXTERNAL, lwd.target); - assertTrue(lwd.active); - assertFalse(lwd.anyFrame); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertFrameOnFrame() { - - JobDetail job_a = getJobA(); - JobDetail job_b = getJobB(); - - FrameDetail frame_a = frameDao.findFrameDetail(job_a, "0001-pass_1"); - FrameDetail frame_b = frameDao.findFrameDetail(job_b, "0001-pass_1"); - - FrameOnFrame depend = new FrameOnFrame(frame_a, frame_b); - dependDao.insertDepend(depend); - - LightweightDependency lwd = dependDao.getDepend(depend.getId()); - assertEquals(depend.getId(), lwd.getId()); - assertEquals(DependType.FRAME_ON_FRAME, lwd.type); - assertEquals(DependTarget.EXTERNAL, lwd.target); - assertTrue(lwd.active); - assertFalse(lwd.anyFrame); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertFrameByFrame() { - - JobDetail job_a = getJobA(); - JobDetail job_b = getJobB(); - LayerInterface layer_a = layerDao.findLayer(job_a, "pass_1"); - LayerInterface layer_b = layerDao.findLayer(job_b, "pass_1"); - - FrameByFrame depend = new FrameByFrame(layer_a, layer_b); - dependDao.insertDepend(depend); - - LightweightDependency lwd = dependDao.getDepend(depend.getId()); - assertEquals(depend.getId(), lwd.getId()); - assertEquals(DependType.FRAME_BY_FRAME, lwd.type); - assertEquals(DependTarget.EXTERNAL, lwd.target); - assertTrue(lwd.active); - assertFalse(lwd.anyFrame); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertPreviousFrame() { - - JobDetail job_a = getJobA(); - JobDetail job_b = getJobB(); - LayerInterface layer_a = layerDao.findLayer(job_a, "pass_1"); - LayerInterface layer_b = layerDao.findLayer(job_b, "pass_1"); - - PreviousFrame depend = new PreviousFrame(layer_a, layer_b); - dependDao.insertDepend(depend); - - LightweightDependency lwd = dependDao.getDepend(depend.getId()); - assertEquals(depend.getId(), lwd.getId()); - assertEquals(DependType.PREVIOUS_FRAME, lwd.type); - assertEquals(DependTarget.EXTERNAL, lwd.target); - assertTrue(lwd.active); - assertFalse(lwd.anyFrame); - } - - @Test - @Transactional - @Rollback(true) - public void testReinsertFrameOnFrame() { - - JobDetail job_a = getJobA(); - JobDetail job_b = getJobB(); - - FrameDetail frame_a = frameDao.findFrameDetail(job_a, "0001-pass_1"); - FrameDetail frame_b = frameDao.findFrameDetail(job_b, "0001-pass_1"); - - FrameOnFrame depend = new FrameOnFrame(frame_a, frame_b); - dependDao.insertDepend(depend); - - LightweightDependency lwd = dependDao.getDepend(depend.getId()); - assertEquals(depend.getId(), lwd.getId()); - assertEquals(DependType.FRAME_ON_FRAME, lwd.type); - assertEquals(DependTarget.EXTERNAL, lwd.target); - assertTrue(lwd.active); - assertFalse(lwd.anyFrame); - - dependDao.setInactive(lwd); - - // Try to reinsert it now that the original is inactive. - depend = new FrameOnFrame(frame_a, frame_b); - dependDao.insertDepend(depend); - } - - @Test - @Transactional - @Rollback(true) - public void testGetWhatDependsOnJob() { - JobDetail job_a = getJobA(); - JobDetail job_b = getJobB(); - - JobOnJob depend = new JobOnJob(job_a, job_b); - dependDao.insertDepend(depend); - - assertEquals(1, dependDao.getWhatDependsOn(job_b).size()); - assertEquals(0, dependDao.getWhatDependsOn(job_a).size()); - } - - - @Test - @Transactional - @Rollback(true) - public void testGetWhatDependsOnLayer() { - - JobDetail job_a = getJobA(); - JobDetail job_b = getJobB(); - - LayerInterface layer_a = layerDao.findLayer(job_a, "pass_1"); - LayerInterface layer_b = layerDao.findLayer(job_b, "pass_1"); - - LayerOnLayer depend = new LayerOnLayer(layer_a, layer_b); - dependDao.insertDepend(depend); - - assertEquals(1, dependDao.getWhatDependsOn(layer_b).size()); - assertEquals(0, dependDao.getWhatDependsOn(layer_a).size()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetWhatDependsOnLayerInactive() { - - JobDetail job_a = getJobA(); - JobDetail job_b = getJobB(); - - LayerInterface layer_a = layerDao.findLayer(job_a, "pass_1"); - LayerInterface layer_b = layerDao.findLayer(job_b, "pass_1"); - - LayerOnLayer depend = new LayerOnLayer(layer_a, layer_b); - dependDao.insertDepend(depend); - - dependDao.setInactive(dependDao.getDepend(depend.getId())); - - assertEquals(1, dependDao.getWhatDependsOn(layer_b, false).size()); - assertEquals(0, dependDao.getWhatDependsOn(layer_b, true).size()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetWhatDependsOnFrame() { - - JobDetail job_a = getJobA(); - JobDetail job_b = getJobB(); - - FrameDetail frame_a = frameDao.findFrameDetail(job_a, "0001-pass_1"); - FrameDetail frame_b = frameDao.findFrameDetail(job_b, "0001-pass_1"); - - FrameOnFrame depend = new FrameOnFrame(frame_a, frame_b); - dependDao.insertDepend(depend); - - assertEquals(1, dependDao.getWhatDependsOn(frame_b).size()); - assertEquals(0, dependDao.getWhatDependsOn(frame_a).size()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetWhatDependsOnFrameInactive() { - - JobDetail job_a = getJobA(); - JobDetail job_b = getJobB(); - - FrameDetail frame_a = frameDao.findFrameDetail(job_a, "0001-pass_1"); - FrameDetail frame_b = frameDao.findFrameDetail(job_b, "0001-pass_1"); - - FrameOnFrame depend = new FrameOnFrame(frame_a, frame_b); - dependDao.insertDepend(depend); - - dependDao.setInactive(dependDao.getDepend(depend.getId())); - - assertEquals(1, dependDao.getWhatDependsOn(frame_b, false).size()); - assertEquals(0, dependDao.getWhatDependsOn(frame_b, true).size()); - assertEquals(0, dependDao.getWhatDependsOn(frame_a, true).size()); - } -} - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/DispatcherDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/DispatcherDaoTests.java deleted file mode 100644 index ef35e9558..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/DispatcherDaoTests.java +++ /dev/null @@ -1,522 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import java.io.File; -import java.util.List; -import java.util.Set; -import javax.annotation.Resource; - -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.DispatchFrame; -import com.imageworks.spcue.DispatchHost; -import com.imageworks.spcue.JobDetail; -import com.imageworks.spcue.LayerDetail; -import com.imageworks.spcue.LayerInterface; -import com.imageworks.spcue.LocalHostAssignment; -import com.imageworks.spcue.VirtualProc; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.AllocationDao; -import com.imageworks.spcue.dao.BookingDao; -import com.imageworks.spcue.dao.DispatcherDao; -import com.imageworks.spcue.dao.HostDao; -import com.imageworks.spcue.dao.JobDao; -import com.imageworks.spcue.dao.LayerDao; -import com.imageworks.spcue.dao.ProcDao; -import com.imageworks.spcue.dispatcher.DispatchSupport; -import com.imageworks.spcue.dispatcher.Dispatcher; -import com.imageworks.spcue.grpc.host.HardwareState; -import com.imageworks.spcue.grpc.job.JobState; -import com.imageworks.spcue.grpc.report.RenderHost; -import com.imageworks.spcue.service.AdminManager; -import com.imageworks.spcue.service.GroupManager; -import com.imageworks.spcue.service.HostManager; -import com.imageworks.spcue.service.JobLauncher; -import com.imageworks.spcue.service.JobManager; -import com.imageworks.spcue.test.AssumingOracleEngine; -import com.imageworks.spcue.util.CueUtil; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class DispatcherDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - DispatcherDao dispatcherDao; - - @Resource - HostDao hostDao; - - @Resource - ProcDao procDao; - - @Resource - LayerDao layerDao; - - @Resource - JobDao jobDao; - - @Resource - AllocationDao allocationDao; - - @Resource - JobManager jobManager; - - @Resource - DispatchSupport dispatchSupport; - - @Resource - HostManager hostManager; - - @Resource - AdminManager adminManager; - - @Resource - GroupManager groupManager; - - @Resource - Dispatcher dispatcher; - - @Resource - JobLauncher jobLauncher; - - @Resource - BookingDao bookingDao; - - private static final String HOSTNAME="beta"; - - public DispatchHost getHost() { - return hostDao.findDispatchHost(HOSTNAME); - } - - public JobDetail getJob1() { - return jobManager.findJobDetail( - "pipe-dev.cue-testuser_shell_dispatch_test_v1"); - } - - public JobDetail getJob2() { - return jobManager.findJobDetail( - "pipe-dev.cue-testuser_shell_dispatch_test_v2"); - } - - @Before - public void launchJob() { - dispatcher.setTestMode(true); - jobLauncher.testMode = true; - jobLauncher.launch( - new File("src/test/resources/conf/jobspec/jobspec_dispatch_test.xml")); - } - - @Before - public void createHost() { - RenderHost host = RenderHost.newBuilder() - .setName(HOSTNAME) - .setBootTime(1192369572) - .setFreeMcp(76020) - .setFreeMem(53500) - .setFreeSwap(20760) - .setLoad(1) - .setTotalMcp(195430) - .setTotalMem(8173264) - .setTotalSwap(20960) - .setNimbyEnabled(false) - .setNumProcs(2) - .setCoresPerProc(100) - .addTags("test") - .setState(HardwareState.UP) - .setFacility("spi") - .putAttributes("SP_OS", "Linux") - .build(); - - hostManager.createHost(host, - adminManager.findAllocationDetail("spi", "general")); - } - - @Test - @Transactional - @Rollback(true) - public void testFindNextDispatchFrameByHost() { - DispatchHost host = getHost(); - JobDetail job = getJob1(); - - for (LayerDetail layer: layerDao.getLayerDetails(job)) { - assertTrue(layer.tags.contains("general")); - } - - assertTrue(jdbcTemplate.queryForObject( - "SELECT str_tags FROM host WHERE pk_host=?",String.class, - host.id).contains("general")); - - DispatchFrame frame = dispatcherDao.findNextDispatchFrame(job, host); - assertNotNull(frame); - assertEquals(frame.name, "0001-pass_1"); - } - - - @Test - @Transactional - @Rollback(true) - public void testFindNextDispatchFrameByProc() { - DispatchHost host = getHost(); - JobDetail job = getJob1(); - - // TODO: fix the fact you can book the same proc on multiple frames - // probably just need to make sure you can't update a proc's frame - // assignment unless the frame id is null. - - DispatchFrame frame = dispatcherDao.findNextDispatchFrame(job, host); - assertNotNull(frame); - assertEquals("0001-pass_1", frame.name); - - VirtualProc proc = VirtualProc.build(host, frame); - proc.coresReserved = 100; - dispatcher.dispatch(frame, proc); - - frame = dispatcherDao.findNextDispatchFrame(job, proc); - assertNotNull(frame); - assertEquals("0001-pass_2", frame.name); - dispatcher.dispatch(frame, proc); - - frame = dispatcherDao.findNextDispatchFrame(job, proc); - assertNotNull(frame); - assertEquals("0002-pass_1", frame.name); - dispatcher.dispatch(frame, proc); - } - - @Test - @Transactional - @Rollback(true) - public void testFindNextDispatchFramesByProc() { - DispatchHost host = getHost(); - JobDetail job = getJob1(); - - // TODO: fix the fact you can book the same proc on multiple frames - // probably just need to make sure you can't update a proc's frame - // assignment unless the frame id is null. - - List frames = - dispatcherDao.findNextDispatchFrames(job, host,10); - assertEquals(10, frames.size()); - - DispatchFrame frame = frames.get(0); - - VirtualProc proc = VirtualProc.build(host, frame); - proc.coresReserved = 100; - dispatcher.dispatch(frame, proc); - - frame = dispatcherDao.findNextDispatchFrame(job, proc); - assertNotNull(frame); - assertEquals(frame.name,"0001-pass_2"); - dispatcher.dispatch(frame, proc); - - frame = dispatcherDao.findNextDispatchFrame(job, proc); - assertNotNull(frame); - assertEquals(frame.name,"0002-pass_1"); - dispatcher.dispatch(frame, proc); - } - - @Test - @Transactional - @Rollback(true) - public void testFindNextDispatchFramesByHostAndJobLocal() { - DispatchHost host = getHost(); - JobDetail job = getJob1(); - host.isLocalDispatch = true; - List frames = - dispatcherDao.findNextDispatchFrames(job, host, 10); - assertEquals(10, frames.size()); - } - - @Test - @Transactional - @Rollback(true) - public void testFindNextDispatchFramesByHostAndLayerLocal() { - DispatchHost host = getHost(); - JobDetail job = getJob1(); - LayerInterface layer = jobManager.getLayers(job).get(0); - host.isLocalDispatch = true; - - List frames = - dispatcherDao.findNextDispatchFrames(layer, host, 10); - assertEquals(10, frames.size()); - } - - @Test - @Transactional - @Rollback(true) - public void testFindNextDispatchFramesByProcAndJobLocal() { - DispatchHost host = getHost(); - JobDetail job = getJob1(); - host.isLocalDispatch = true; - List frames = - dispatcherDao.findNextDispatchFrames(job, host, 10); - assertEquals(10, frames.size()); - - DispatchFrame frame = frames.get(0); - VirtualProc proc = VirtualProc.build(host, frame); - proc.coresReserved = 100; - proc.isLocalDispatch = true; - - frames = dispatcherDao.findNextDispatchFrames(job, proc, 10); - assertEquals(10, frames.size()); - } - - @Test - @Transactional - @Rollback(true) - public void testFindNextDispatchFramesByProcAndLayerLocal() { - DispatchHost host = getHost(); - JobDetail job = getJob1(); - LayerInterface layer = jobManager.getLayers(job).get(0); - host.isLocalDispatch = true; - - List frames = - dispatcherDao.findNextDispatchFrames(layer, host, 10); - assertEquals(10, frames.size()); - - DispatchFrame frame = frames.get(0); - VirtualProc proc = VirtualProc.build(host, frame); - proc.coresReserved = 100; - proc.isLocalDispatch = true; - - frames = dispatcherDao.findNextDispatchFrames(layer, proc, 10); - assertEquals(10, frames.size()); - } - - - @Test - @Transactional - @Rollback(true) - public void testFindDispatchJobs() { - DispatchHost host = getHost(); - - assertTrue(jdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM job WHERE str_state='PENDING'", Integer.class) > 0); - - Set jobs = dispatcherDao.findDispatchJobs(host, 10); - assertTrue(jobs.size() > 0); - } - - @Test - @Transactional - @Rollback(true) - public void testFindDispatchJobsByGroup() { - DispatchHost host = getHost(); - final JobDetail job = getJob1(); - - assertNotNull(job); - assertNotNull(job.groupId); - - Set jobs = dispatcherDao.findDispatchJobs(host, - groupManager.getGroupDetail(job)); - assertTrue(jobs.size() > 0); - } - - @Test - @Transactional - @Rollback(true) - public void testFindDispatchJobsByShow() { - DispatchHost host = getHost(); - final JobDetail job = getJob1(); - assertNotNull(job); - - Set jobs = dispatcherDao.findDispatchJobs(host, - adminManager.findShowEntity("pipe"), 5); - assertTrue(jobs.size() > 0); - } - - @Test - @Transactional - @Rollback(true) - public void testFindDispatchJobsByLocal() { - DispatchHost host = getHost(); - final JobDetail job = getJob1(); - assertNotNull(job); - - Set jobs = dispatcherDao.findLocalDispatchJobs(host); - assertEquals(0, jobs.size()); - - LocalHostAssignment lja = new LocalHostAssignment(); - lja.setThreads(1); - lja.setMaxMemory(CueUtil.GB16); - lja.setMaxCoreUnits(200); - lja.setMaxGpu(1); - bookingDao.insertLocalHostAssignment(host, job, lja); - - jobs = dispatcherDao.findLocalDispatchJobs(host); - assertTrue(jobs.size() > 0); - } - - @Test - @Transactional - @Rollback(true) - public void testfindUnderProcedJob() { - DispatchHost host = getHost(); - JobDetail job1 = getJob1(); - JobDetail job2 = getJob2(); - - jobDao.updateMinCores(job1, 0); - jobDao.updateMinCores(job2, 1000); - - DispatchFrame frame = dispatcherDao.findNextDispatchFrame(job1, host); - assertNotNull(frame); - - assertEquals(JobState.PENDING.toString(), - jdbcTemplate.queryForObject( - "SELECT str_state FROM job WHERE pk_job=?", - String.class, job1.id)); - - assertEquals(JobState.PENDING.toString(), - jdbcTemplate.queryForObject( - "SELECT str_state FROM job WHERE pk_job=?", - String.class, job2.id)); - - VirtualProc proc = VirtualProc.build(host, frame); - proc.coresReserved = 100; - dispatcher.dispatch(frame, proc); - - boolean under = dispatcherDao.findUnderProcedJob(job1, proc); - assertTrue(under); - } - - @Test - @Transactional - @Rollback(true) - public void testHigherPriorityJobExistsTrue() { - DispatchHost host = getHost(); - JobDetail job1 = getJob1(); - JobDetail job2 = getJob2(); - job1.priority = 100; - job2.priority = 200; - - jobDao.updateMinCores(job1, 0); - jobDao.updateMinCores(job2, 0); - jobDao.updatePriority(job1, 100); - jobDao.updatePriority(job2, 200); - - DispatchFrame frame = dispatcherDao.findNextDispatchFrame(job1, host); - assertNotNull(frame); - - assertEquals(JobState.PENDING.toString(), - jdbcTemplate.queryForObject( - "SELECT str_state FROM job WHERE pk_job=?", - String.class, job1.id)); - - assertEquals(JobState.PENDING.toString(), - jdbcTemplate.queryForObject( - "SELECT str_state FROM job WHERE pk_job=?", - String.class, job2.id)); - - VirtualProc proc = VirtualProc.build(host, frame); - proc.coresReserved = 100; - dispatcher.dispatch(frame, proc); - - boolean isHigher = dispatcherDao.higherPriorityJobExists(job1, proc); - assertTrue(isHigher); - } - - @Test - @Transactional - @Rollback(true) - public void testHigherPriorityJobExistsFalse() { - DispatchHost host = getHost(); - JobDetail job1 = getJob1(); - JobDetail job2 = getJob2(); - job1.priority = 20000; - job2.priority = 100; - - jobDao.updateMinCores(job1, 0); - jobDao.updateMinCores(job2, 0); - jobDao.updatePriority(job1, 20000); - jobDao.updatePriority(job2, 100); - - DispatchFrame frame = dispatcherDao.findNextDispatchFrame(job1, host); - assertNotNull(frame); - - assertEquals(JobState.PENDING.toString(), - jdbcTemplate.queryForObject( - "SELECT str_state FROM job WHERE pk_job=?", - String.class, job1.id)); - - assertEquals(JobState.PENDING.toString(), - jdbcTemplate.queryForObject( - "SELECT str_state FROM job WHERE pk_job=?", - String.class, job2.id)); - - VirtualProc proc = VirtualProc.build(host, frame); - proc.coresReserved = 100; - dispatcher.dispatch(frame, proc); - - boolean isHigher = dispatcherDao.higherPriorityJobExists(job1, proc); - assertFalse(isHigher); - } - - @Test - @Transactional - @Rollback(true) - public void testHigherPriorityJobExistsMaxProcBound() { - DispatchHost host = getHost(); - JobDetail job1 = getJob1(); - JobDetail job2 = getJob2(); - job1.priority = 100; - job2.priority = 200; - - jobDao.updateMaxCores(job2, 0); - jobDao.updatePriority(job1, 100); - jobDao.updatePriority(job2, 200); - - DispatchFrame frame = dispatcherDao.findNextDispatchFrame(job1, host); - assertNotNull(frame); - - assertEquals(JobState.PENDING.toString(), - jdbcTemplate.queryForObject( - "SELECT str_state FROM job WHERE pk_job=?", - String.class, job1.id)); - - assertEquals(JobState.PENDING.toString(), - jdbcTemplate.queryForObject( - "SELECT str_state FROM job WHERE pk_job=?", - String.class, job2.id)); - - VirtualProc proc = VirtualProc.build(host, frame); - proc.coresReserved = 100; - dispatcher.dispatch(frame, proc); - - boolean isHigher = dispatcherDao.higherPriorityJobExists(job1, proc); - assertFalse(isHigher); - } -} diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/FacilityInterfaceDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/FacilityInterfaceDaoTests.java deleted file mode 100644 index 84c7c4a16..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/FacilityInterfaceDaoTests.java +++ /dev/null @@ -1,78 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import javax.annotation.Resource; - -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.FacilityDao; -import com.imageworks.spcue.test.AssumingOracleEngine; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class FacilityInterfaceDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - FacilityDao facilityDao; - - @Test - @Transactional - @Rollback(true) - public void testGetDetaultFacility() { - assertEquals(jdbcTemplate.queryForObject( - "SELECT pk_facility FROM facility WHERE b_default=1", - String.class),facilityDao.getDefaultFacility().getId()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetFacility() { - String id = "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA0"; - assertEquals(id, facilityDao.getFacility(id).getId()); - assertEquals(id, facilityDao.getFacility("spi").getId()); - } - - @Test - @Transactional - @Rollback(true) - public void testFacilityExists() { - assertTrue(facilityDao.facilityExists("spi")); - assertFalse(facilityDao.facilityExists("rambo")); - } -} - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/FilterDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/FilterDaoTests.java deleted file mode 100644 index 8e2a1c3be..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/FilterDaoTests.java +++ /dev/null @@ -1,302 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import javax.annotation.Resource; - -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.FilterEntity; -import com.imageworks.spcue.ShowEntity; -import com.imageworks.spcue.ShowInterface; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.FilterDao; -import com.imageworks.spcue.dao.ShowDao; -import com.imageworks.spcue.grpc.filter.FilterType; -import com.imageworks.spcue.service.AdminManager; -import com.imageworks.spcue.test.AssumingOracleEngine; - -import static org.junit.Assert.assertEquals; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class FilterDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - FilterDao filterDao; - - @Resource - ShowDao showDao; - - @Resource - AdminManager adminManager; - - private static String FILTER_NAME = "test_filter"; - - public ShowInterface createShow() { - ShowEntity show = new ShowEntity(); - show.name = "testtest"; - adminManager.createShow(show); - return show; - } - - public ShowInterface getShow() { - return showDao.findShowDetail("testtest"); - } - - public FilterEntity buildFilter(ShowInterface show) { - FilterEntity filter = new FilterEntity(); - filter.name = FILTER_NAME; - filter.showId = show.getId(); - filter.type = FilterType.MATCH_ANY; - filter.enabled = true; - - return filter; - } - - @Test - @Transactional - @Rollback(true) - public void testGetActiveFilters() { - filterDao.getActiveFilters(createShow()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetFilters() { - filterDao.getFilters(createShow()); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateSetFilterEnabled() { - FilterEntity f = buildFilter(createShow()); - filterDao.insertFilter(f); - filterDao.updateSetFilterEnabled(f, false); - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT b_enabled FROM filter WHERE pk_filter=?", - Integer.class, f.getFilterId())); - filterDao.updateSetFilterEnabled(f, true); - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT b_enabled FROM filter WHERE pk_filter=?", - Integer.class, f.getFilterId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateSetFilterName() { - FilterEntity f = buildFilter(createShow()); - filterDao.insertFilter(f); - assertEquals(FILTER_NAME, jdbcTemplate.queryForObject( - "SELECT str_name FROM filter WHERE pk_filter=?", - String.class, - f.getFilterId())); - filterDao.updateSetFilterName(f, "TEST"); - assertEquals("TEST", jdbcTemplate.queryForObject( - "SELECT str_name FROM filter WHERE pk_filter=?", - String.class, - f.getFilterId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateSetFilterType() { - FilterEntity f = buildFilter(createShow()); - filterDao.insertFilter(f); - assertEquals(FilterType.MATCH_ANY.toString(), jdbcTemplate.queryForObject( - "SELECT str_type FROM filter WHERE pk_filter=?", - String.class, - f.getFilterId())); - filterDao.updateSetFilterType(f, FilterType.MATCH_ALL); - assertEquals(FilterType.MATCH_ALL.toString(), jdbcTemplate.queryForObject( - "SELECT str_type FROM filter WHERE pk_filter=?", - String.class, - f.getFilterId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateSetFilterOrder() { - - ShowInterface show = createShow(); - int currentFilters = jdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM filter WHERE pk_show=?", - Integer.class, show.getShowId()); - - FilterEntity f1 = buildFilter(show); - filterDao.insertFilter(f1); - - FilterEntity f2 = buildFilter(show); - f2.name = "TEST"; - filterDao.insertFilter(f2); - - assertEquals(Integer.valueOf(currentFilters+1), jdbcTemplate.queryForObject( - "SELECT f_order FROM filter WHERE pk_filter=?", - Integer.class, f1.getFilterId())); - - assertEquals(Integer.valueOf(currentFilters+2), jdbcTemplate.queryForObject( - "SELECT f_order FROM filter WHERE pk_filter=?", - Integer.class, f2.getFilterId())); - - filterDao.updateSetFilterOrder(f2,1); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT f_order FROM filter WHERE pk_filter=?", - Integer.class, f2.getFilterId())); - } - - @Test - @Transactional - @Rollback(true) - public void testDeleteFilter() { - FilterEntity f = buildFilter(createShow()); - filterDao.insertFilter(f); - filterDao.deleteFilter(f); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertFilter() { - FilterEntity f = buildFilter(createShow()); - filterDao.insertFilter(f); - } - - @Test - @Transactional - @Rollback(true) - public void testReorderFilters() { - buildFilter(createShow()); - filterDao.reorderFilters(getShow()); - } - - @Test - @Transactional - @Rollback(true) - public void testLowerFilterOrder() { - - ShowInterface show = createShow(); - - FilterEntity f1 = buildFilter(show); - filterDao.insertFilter(f1); - - FilterEntity f2 = buildFilter(show); - f2.name = "TEST"; - filterDao.insertFilter(f2); - - - /** - * These could fail if the test DB has other filters. - */ - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT f_order FROM filter WHERE pk_filter=?", - Integer.class, f1.getFilterId())); - - assertEquals(Integer.valueOf(2), jdbcTemplate.queryForObject( - "SELECT f_order FROM filter WHERE pk_filter=?", - Integer.class, f2.getFilterId())); - - filterDao.lowerFilterOrder(f2,1); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT f_order FROM filter WHERE pk_filter=?", - Integer.class, f1.getFilterId())); - - assertEquals(Integer.valueOf(2), jdbcTemplate.queryForObject( - "SELECT f_order FROM filter WHERE pk_filter=?", - Integer.class, f2.getFilterId())); - } - - @Test - @Transactional - @Rollback(true) - public void testRaiseFilterOrder() { - - ShowInterface show = createShow(); - - FilterEntity f1 = buildFilter(show); - filterDao.insertFilter(f1); - - FilterEntity f2 = buildFilter(show); - f2.name = "TEST"; - filterDao.insertFilter(f2); - - /** - * These could fail if the test DB has other filters. - */ - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT f_order FROM filter WHERE pk_filter=?", - Integer.class, f1.getFilterId())); - - assertEquals(Integer.valueOf(2), jdbcTemplate.queryForObject( - "SELECT f_order FROM filter WHERE pk_filter=?", - Integer.class, f2.getFilterId())); - - filterDao.raiseFilterOrder(f1, 1); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT f_order FROM filter WHERE pk_filter=?", - Integer.class, f1.getFilterId())); - - assertEquals(Integer.valueOf(2), jdbcTemplate.queryForObject( - "SELECT f_order FROM filter WHERE pk_filter=?", - Integer.class, f2.getFilterId())); - } - - @Test - @Transactional - @Rollback(true) - public void testGetFilter() { - FilterEntity f = buildFilter(createShow()); - filterDao.insertFilter(f); - - filterDao.getFilter(f); - filterDao.getFilter(f.getId()); - } - - @Test - @Transactional - @Rollback(true) - public void testFindFilter() { - FilterEntity f = buildFilter(createShow()); - filterDao.insertFilter(f); - - filterDao.findFilter(getShow(), FILTER_NAME); - } - -} - - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/FrameDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/FrameDaoTests.java deleted file mode 100644 index f1532bcec..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/FrameDaoTests.java +++ /dev/null @@ -1,659 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import java.io.File; -import java.math.BigDecimal; -import java.util.Map; -import javax.annotation.Resource; - -import com.google.common.collect.ImmutableList; -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.test.context.transaction.AfterTransaction; -import org.springframework.test.context.transaction.BeforeTransaction; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.DispatchFrame; -import com.imageworks.spcue.DispatchHost; -import com.imageworks.spcue.FrameDetail; -import com.imageworks.spcue.FrameInterface; -import com.imageworks.spcue.JobDetail; -import com.imageworks.spcue.LayerInterface; -import com.imageworks.spcue.VirtualProc; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.AllocationDao; -import com.imageworks.spcue.dao.FrameDao; -import com.imageworks.spcue.dao.HostDao; -import com.imageworks.spcue.dao.LayerDao; -import com.imageworks.spcue.dao.ProcDao; -import com.imageworks.spcue.dao.criteria.FrameSearchFactory; -import com.imageworks.spcue.dao.criteria.FrameSearchInterface; -import com.imageworks.spcue.depend.FrameOnFrame; -import com.imageworks.spcue.dispatcher.DispatchSupport; -import com.imageworks.spcue.grpc.host.HardwareState; -import com.imageworks.spcue.grpc.job.CheckpointState; -import com.imageworks.spcue.grpc.job.FrameSearchCriteria; -import com.imageworks.spcue.grpc.job.FrameState; -import com.imageworks.spcue.grpc.report.RenderHost; -import com.imageworks.spcue.service.DependManager; -import com.imageworks.spcue.service.HostManager; -import com.imageworks.spcue.service.JobLauncher; -import com.imageworks.spcue.service.JobManager; -import com.imageworks.spcue.test.AssumingOracleEngine; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class FrameDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - FrameDao frameDao; - - @Resource - LayerDao layerDao; - - @Resource - JobManager jobManager; - - @Resource - JobLauncher jobLauncher; - - @Resource - HostDao hostDao; - - @Resource - ProcDao procDao; - - @Resource - AllocationDao allocationDao; - - @Resource - HostManager hostManager; - - @Resource - DependManager dependManager; - - @Resource - DispatchSupport dispatchSupport; - - @Resource - FrameSearchFactory frameSearchFactory; - - private static final String HOST = "beta"; - - public DispatchHost createHost() { - return hostDao.findDispatchHost(HOST); - } - - @BeforeTransaction - public void create() { - - RenderHost host = RenderHost.newBuilder() - .setName(HOST) - .setBootTime(1192369572) - .setFreeMcp(76020) - .setFreeMem(53500) - .setFreeSwap(20760) - .setLoad(1) - .setTotalMcp(195430) - .setTotalMem(8173264) - .setTotalSwap(20960) - .setNimbyEnabled(false) - .setNumProcs(1) - .setCoresPerProc(100) - .addAllTags(ImmutableList.of("mcore", "4core", "8g")) - .setState(HardwareState.UP) - .setFacility("spi") - .putAttributes("freeGpu", "512") - .putAttributes("totalGpu", "512") - .build(); - - hostManager.createHost(host); - } - - @AfterTransaction - public void destroy() { - jdbcTemplate.update( - "DELETE FROM host WHERE str_name=?",HOST); - } - - public JobDetail launchJob() { - jobLauncher.testMode = true; - jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec.xml")); - return jobManager.findJobDetail("pipe-dev.cue-testuser_shell_v1"); - } - - @Test - @Transactional - @Rollback(true) - public void testCheckRetries() { - JobDetail job = launchJob(); - frameDao.checkRetries(frameDao.findFrame(job,"0001-pass_1")); - // TODO: check to see if it actually works - } - - @Test - @Transactional - @Rollback(true) - public void testGetFrameDetail() { - JobDetail job = launchJob(); - FrameInterface f = frameDao.findFrame(job, "0001-pass_1"); - FrameDetail frame = frameDao.getFrameDetail(f); - frame = frameDao.getFrameDetail(f.getFrameId()); - assertEquals("0001-pass_1", frame.name); - } - - @Test - @Transactional - @Rollback(true) - public void testFindFrameDetail() { - JobDetail job = launchJob(); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1"); - assertEquals("0001-pass_1", frame.name); - } - - @Test - @Transactional - @Rollback(true) - public void testGetFrame() { - JobDetail job = launchJob(); - FrameInterface f = frameDao.findFrame(job, "0001-pass_1"); - FrameInterface frame = frameDao.getFrame(f.getFrameId()); - assertEquals("0001-pass_1", frame.getName()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetFrameByLayer() { - JobDetail job = launchJob(); - FrameInterface f = frameDao.findFrame(job, "0001-pass_1"); - FrameInterface f2 = frameDao.findFrame((LayerInterface)f, 1); - - assertEquals(f.getFrameId(), f2.getFrameId()); - assertEquals(f.getLayerId(), f2.getLayerId()); - assertEquals(f.getJobId(), f2.getJobId()); - } - - @Test - @Transactional - @Rollback(true) - public void testFindFrame() { - JobDetail job = launchJob(); - FrameInterface f = frameDao.findFrame(job, "0001-pass_1"); - assertEquals(f.getName(),"0001-pass_1"); - } - - @Test - @Transactional - @Rollback(true) - public void testFindFrames() { - JobDetail job = launchJob(); - FrameSearchInterface r = frameSearchFactory.create(job); - FrameSearchCriteria criteria = r.getCriteria(); - r.setCriteria(criteria.toBuilder() - .addFrames("0001-pass_1") - .build()); - assertEquals(1, frameDao.findFrames(r).size()); - } - - @Test - @Transactional - @Rollback(true) - public void testFindFrameDetails() { - JobDetail job = launchJob(); - FrameSearchInterface r = frameSearchFactory.create(job); - FrameSearchCriteria criteria = r.getCriteria(); - r.setCriteria(criteria.toBuilder() - .addFrames("0001-pass_1") - .build()); - assertEquals(1, frameDao.findFrameDetails(r).size()); - } - - @Test - @Transactional - @Rollback(true) - public void testgetOrphanedFrames() { - assertEquals(0, frameDao.getOrphanedFrames().size()); - - JobDetail job = launchJob(); - FrameInterface f = frameDao.findFrame(job, "0001-pass_1"); - - /* - * Update the first frame to the orphan state, which is a frame - * that is in the running state, has no corresponding proc entry - * and has not been udpated in the last 5 min. - */ - jdbcTemplate.update( - "UPDATE frame SET str_state='RUNNING', " + - "ts_updated=systimestamp - interval '301' second WHERE pk_frame=?", - f.getFrameId()); - - assertEquals(1, frameDao.getOrphanedFrames().size()); - assertTrue(frameDao.isOrphan(f)); - - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateFrameState() { - JobDetail job = launchJob(); - FrameInterface f = frameDao.findFrame(job, "0001-pass_1"); - assertTrue(frameDao.updateFrameState(f, FrameState.RUNNING)); - - assertEquals(FrameState.RUNNING.toString(), - jdbcTemplate.queryForObject( - "SELECT str_state FROM frame WHERE pk_frame=?", - String.class, - f.getFrameId())); - } - - @Test - @Transactional - @Rollback(true) - public void testFailUpdateFrameState() { - JobDetail job = launchJob(); - FrameInterface f = frameDao.findFrame(job, "0001-pass_1"); - - /** Change the version so the update fails **/ - jdbcTemplate.update( - "UPDATE frame SET int_version = int_version + 1 WHERE pk_frame=?", - f.getFrameId()); - - assertEquals(false, frameDao.updateFrameState(f, FrameState.RUNNING)); - } - - - @Test - @Transactional - @Rollback(true) - public void testUpdateFrameStarted() { - - DispatchHost host = createHost(); - JobDetail job = launchJob(); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1_preprocess"); - DispatchFrame fd = frameDao.getDispatchFrame(frame.getId()); - VirtualProc proc = new VirtualProc(); - proc.allocationId = host.allocationId; - proc.coresReserved = 100; - proc.hostId = host.id; - proc.hostName = host.name; - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - - assertEquals(FrameState.WAITING, frame.state); - - procDao.insertVirtualProc(proc); - procDao.verifyRunningProc(proc.getId(), frame.getId()); - frameDao.updateFrameStarted(proc, fd); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateFrameStopped() { - - DispatchHost host = createHost(); - JobDetail job = launchJob(); - - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1_preprocess"); - DispatchFrame fd = frameDao.getDispatchFrame(frame.getId()); - - assertEquals("0001-pass_1_preprocess",frame.getName()); - assertEquals(FrameState.WAITING, frame.state); - - VirtualProc proc = new VirtualProc(); - proc.allocationId = host.allocationId; - proc.coresReserved = 100; - proc.hostId = host.id; - proc.hostName = host.name; - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - - procDao.insertVirtualProc(proc); - procDao.verifyRunningProc(proc.getId(), frame.getId()); - - frameDao.updateFrameStarted(proc, fd); - - try { - Thread.sleep(1001); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - DispatchFrame fd2 = frameDao.getDispatchFrame(frame.getId()); - assertTrue(frameDao.updateFrameStopped(fd2, FrameState.DEAD, 1, 1000l)); - - assertEquals(FrameState.DEAD.toString(),jdbcTemplate.queryForObject( - "SELECT str_state FROM frame WHERE pk_frame=?", - String.class, frame.getFrameId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateFrameFixed() { - - DispatchHost host = createHost(); - JobDetail job = launchJob(); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1_preprocess"); - DispatchFrame fd = frameDao.getDispatchFrame(frame.getId()); - - assertEquals("0001-pass_1_preprocess",frame.getName()); - assertEquals(FrameState.WAITING, frame.state); - - VirtualProc proc = new VirtualProc(); - proc.allocationId = host.allocationId; - proc.coresReserved = 100; - proc.hostId = host.id; - proc.hostName = host.name; - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - - procDao.insertVirtualProc(proc); - procDao.verifyRunningProc(proc.getId(), frame.getId()); - frameDao.updateFrameStarted(proc, fd); - - try { - Thread.sleep(1001); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - frameDao.updateFrameState(frame, FrameState.WAITING); - frameDao.updateFrameFixed(proc, frame); - - assertEquals(FrameState.RUNNING.toString(), - jdbcTemplate.queryForObject( - "SELECT str_state FROM frame WHERE pk_frame=?", - String.class, frame.getFrameId())); - } - - @Test - @Transactional - @Rollback(true) - public void testGetDispatchFrame() { - DispatchHost host = createHost(); - JobDetail job = launchJob(); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1"); - - VirtualProc proc = new VirtualProc(); - proc.allocationId = host.allocationId; - proc.coresReserved = 100; - proc.hostId = host.id; - proc.hostName = host.name; - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - - procDao.insertVirtualProc(proc); - procDao.verifyRunningProc(proc.getId(), frame.getId()); - - DispatchFrame dframe = frameDao.getDispatchFrame(frame.id); - assertEquals(dframe.id, frame.id); - } - - @Test - @Transactional - @Rollback(true) - public void testMarkFrameAsWaiting() { - JobDetail job = launchJob(); - - FrameInterface f = frameDao.findFrameDetail(job, "0001-pass_1"); - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT int_depend_count FROM frame WHERE pk_frame=?", - Integer.class, f.getFrameId())); - - frameDao.markFrameAsWaiting(f); - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT int_depend_count FROM frame WHERE pk_frame=?", - Integer.class, f.getFrameId())); - } - - @Test - @Transactional - @Rollback(true) - public void testMarkFrameAsDepend() { - JobDetail job = launchJob(); - - FrameInterface f = frameDao.findFrameDetail(job, "0001-pass_1"); - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT int_depend_count FROM frame WHERE pk_frame=?", - Integer.class, f.getFrameId())); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT b_active FROM depend WHERE pk_layer_depend_er=?", - Integer.class, f.getLayerId())); - - frameDao.markFrameAsWaiting(f); - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT int_depend_count FROM frame WHERE pk_frame=?", - Integer.class, f.getFrameId())); - - /* - * Need to grab new version of frame - * object once the state has changed. - */ - f = frameDao.findFrameDetail(job, "0001-pass_1"); - - frameDao.markFrameAsDepend(f); - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT int_depend_count FROM frame WHERE pk_frame=?", - Integer.class, f.getFrameId())); - } - - @Test(expected=org.springframework.dao.EmptyResultDataAccessException.class) - @Transactional - @Rollback(true) - public void testFindLongestFrame() { - JobDetail job = launchJob(); - frameDao.findLongestFrame(job); - } - - @Test(expected=org.springframework.dao.EmptyResultDataAccessException.class) - @Transactional - @Rollback(true) - public void testFindShortestFrame() { - JobDetail job = launchJob(); - frameDao.findShortestFrame(job); - } - - @Test(expected=org.springframework.dao.EmptyResultDataAccessException.class) - @Transactional - @Rollback(true) - public void findHighestMemoryFrame() { - JobDetail job = launchJob(); - frameDao.findHighestMemoryFrame(job); - } - - @Test(expected=org.springframework.dao.EmptyResultDataAccessException.class) - @Transactional - @Rollback(true) - public void findLowestMemoryFrame() { - JobDetail job = launchJob(); - frameDao.findLowestMemoryFrame(job); - } - - @Test - @Transactional - @Rollback(true) - public void testGetDependentFrames() { - JobDetail job = launchJob(); - FrameInterface frame_a = frameDao.findFrame(job, "0001-pass_1"); - FrameInterface frame_b = frameDao.findFrame(job, "0002-pass_1"); - - dependManager.createDepend(new FrameOnFrame( - frame_a, frame_b)); - - assertEquals(1, frameDao.getDependentFrames( - dependManager.getWhatDependsOn(frame_b).get(0)).size(),1); - } - - @Test - @Transactional - @Rollback(true) - public void testGetResourceUsage() { - DispatchHost host = createHost(); - JobDetail job = launchJob(); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1"); - - VirtualProc proc = new VirtualProc(); - proc.allocationId = host.allocationId; - proc.coresReserved = 100; - proc.hostId = host.id; - proc.hostName = host.name; - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - - procDao.insertVirtualProc(proc); - procDao.verifyRunningProc(proc.getId(), frame.getId()); - - DispatchFrame dframe = frameDao.getDispatchFrame(frame.id); - frameDao.getResourceUsage(dframe); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateFrameCleared() { - DispatchHost host = createHost(); - JobDetail job = launchJob(); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1"); - - VirtualProc proc = new VirtualProc(); - proc.allocationId = host.allocationId; - proc.coresReserved = 100; - proc.hostId = host.id; - proc.hostName = host.name; - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - - procDao.insertVirtualProc(proc); - procDao.verifyRunningProc(proc.getId(), frame.getId()); - - /* - * Only frames without active procs can be cleared. - */ - - DispatchFrame dframe = frameDao.getDispatchFrame(frame.id); - assertFalse(frameDao.updateFrameCleared(dframe)); - - dispatchSupport.unbookProc(proc); - assertTrue(frameDao.updateFrameCleared(dframe)); - - } - - @Test - @Transactional - @Rollback(true) - public void testGetStaleCheckpoints() { - - DispatchHost host = createHost(); - JobDetail job = launchJob(); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1_preprocess"); - - assertEquals(0, frameDao.getStaleCheckpoints(300).size()); - jdbcTemplate.update("UPDATE frame SET str_state=?, " + - "ts_stopped=systimestamp - interval '400' second WHERE pk_frame=?", - FrameState.CHECKPOINT.toString(), frame.getFrameId()); - assertEquals(1, frameDao.getStaleCheckpoints(300).size()); - } - - @Test - @Transactional - @Rollback(true) - public void testSetCheckpointState() { - - DispatchHost host = createHost(); - JobDetail job = launchJob(); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1_preprocess"); - - frameDao.updateFrameCheckpointState(frame, CheckpointState.ENABLED); - - String state = jdbcTemplate.queryForObject( - "SELECT str_checkpoint_state FROM frame WHERE pk_frame=?", - String.class, frame.getFrameId()); - - assertEquals(CheckpointState.ENABLED.toString(), state); - - /** - * To set a checkpoint complete the frame state must be in the checkpoint state. - */ - frameDao.updateFrameState(frame, FrameState.CHECKPOINT); - jdbcTemplate.update( - "UPDATE frame SET ts_started=systimestamp, ts_stopped=systimestamp + INTERVAL '20' second WHERE pk_frame=?", - frame.getFrameId()); - - assertTrue(frameDao.updateFrameCheckpointState(frame, CheckpointState.COMPLETE)); - Map result = jdbcTemplate.queryForMap( - "SELECT int_checkpoint_count FROM frame WHERE pk_frame=?", - frame.getFrameId()); - - BigDecimal checkPointCount = (BigDecimal) result.get("int_checkpoint_count"); - assertEquals(1, checkPointCount.intValue()); - } - - @Test - @Transactional - @Rollback(true) - public void testIsFrameComplete() { - - DispatchHost host = createHost(); - JobDetail job = launchJob(); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1_preprocess"); - - frameDao.updateFrameState(frame, FrameState.EATEN); - assertTrue(frameDao.isFrameComplete(frame)); - - frame = frameDao.findFrameDetail(job, "0001-pass_1_preprocess"); - frameDao.updateFrameState(frame, FrameState.SUCCEEDED); - assertTrue(frameDao.isFrameComplete(frame)); - - frame = frameDao.findFrameDetail(job, "0001-pass_1_preprocess"); - frameDao.updateFrameState(frame, FrameState.WAITING); - assertFalse(frameDao.isFrameComplete(frame)); - } -} diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/GroupDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/GroupDaoTests.java deleted file mode 100644 index 1a83971f3..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/GroupDaoTests.java +++ /dev/null @@ -1,393 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; -import javax.annotation.Resource; - -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.GroupDetail; -import com.imageworks.spcue.GroupInterface; -import com.imageworks.spcue.JobDetail; -import com.imageworks.spcue.ShowInterface; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.DepartmentDao; -import com.imageworks.spcue.dao.GroupDao; -import com.imageworks.spcue.dao.ShowDao; -import com.imageworks.spcue.service.JobLauncher; -import com.imageworks.spcue.service.JobManager; -import com.imageworks.spcue.test.AssumingOracleEngine; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class GroupDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - GroupDao groupDao; - - @Resource - ShowDao showDao; - - @Resource - DepartmentDao departmentDao; - - @Resource - JobManager jobManager; - - @Resource - JobLauncher jobLauncher; - - @Before - public void before() { - jobLauncher.testMode = true; - } - - public ShowInterface getShow() { - return showDao.getShowDetail("00000000-0000-0000-0000-000000000000"); - } - - public JobDetail launchJob() { - jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec.xml")); - return jobManager.findJobDetail("pipe-dev.cue-testuser_shell_v1"); - } - - public GroupDetail createGroup() { - GroupDetail group = new GroupDetail(); - group.name = "Shit"; - group.parentId = groupDao.getRootGroupId(getShow()); - group.showId = getShow().getId(); - group.deptId = departmentDao.getDefaultDepartment().getId(); - groupDao.insertGroup(group, groupDao.getRootGroupDetail(getShow())); - return group; - } - - public GroupDetail createSubGroup(GroupDetail parent) { - GroupDetail group = new GroupDetail(); - group.name = "SubShit"; - group.parentId = parent.id; - group.showId = getShow().getId(); - group.deptId = departmentDao.getDefaultDepartment().getId(); - groupDao.insertGroup(group, groupDao.getGroup(parent.id)); - return group; - } - - @Test - @Transactional - @Rollback(true) - public void testGetGroup() { - GroupDetail group = createGroup(); - GroupInterface g = groupDao.getGroup(group.id); - assertEquals(group.id,g.getGroupId()); - assertEquals(group.id,g.getId()); - assertEquals(group.name, g.getName()); - assertEquals(group.showId, g.getShowId()); - } - - - @Test - @Transactional - @Rollback(true) - public void testGetGroups() { - GroupDetail group = createGroup(); - List l = new ArrayList(); - l.add(group.id); - List g = groupDao.getGroups(l); - assertEquals(1, g.size()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetRootGroupId() { - groupDao.getRootGroupId(getShow()); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertGroup() { - GroupDetail group = createGroup(); - assertFalse(group.isNew()); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertGroupAlternateMethod() { - GroupDetail group = new GroupDetail(); - group.name = "Shit"; - group.parentId = groupDao.getRootGroupId(getShow()); - group.showId = getShow().getId(); - group.deptId = departmentDao.getDefaultDepartment().getId(); - groupDao.insertGroup(group); - } - - @Test - @Transactional - @Rollback(true) - public void testDeleteGroup() { - // Can't delete groups yet, will fail - GroupDetail group = createGroup(); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM folder WHERE pk_folder=?", - Integer.class, group.getId())); - - groupDao.deleteGroup(group); - - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM folder WHERE pk_folder=?", - Integer.class, group.getId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateGroupParent() { - GroupDetail group = createGroup(); - GroupDetail subgroup = createSubGroup(group); - groupDao.updateGroupParent(subgroup, - groupDao.getGroupDetail( - groupDao.getRootGroupId(getShow()))); - - assertEquals(Integer.valueOf(1),jdbcTemplate.queryForObject( - "SELECT int_level FROM folder_level WHERE pk_folder=?", - Integer.class, subgroup.getId())); - - groupDao.updateGroupParent(subgroup, group); - - assertEquals(Integer.valueOf(2),jdbcTemplate.queryForObject( - "SELECT int_level FROM folder_level WHERE pk_folder=?", - Integer.class, subgroup.getId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateDefaultJobMaxCores() { - GroupDetail group = createGroup(); - assertEquals(Integer.valueOf(-1), jdbcTemplate.queryForObject( - "SELECT int_job_max_cores FROM folder WHERE pk_folder=?", - Integer.class, group.getGroupId())); - groupDao.updateDefaultJobMaxCores(group, 100); - assertEquals(Integer.valueOf(100), jdbcTemplate.queryForObject( - "SELECT int_job_max_cores FROM folder WHERE pk_folder=?", - Integer.class, group.getGroupId())); - groupDao.updateDefaultJobMaxCores(group, -1); - assertEquals(Integer.valueOf(-1), jdbcTemplate.queryForObject( - "SELECT int_job_max_cores FROM folder WHERE pk_folder=?", - Integer.class, group.getGroupId())); - - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateDefaultJobMinCores() { - GroupDetail group = createGroup(); - assertEquals(Integer.valueOf(-1), jdbcTemplate.queryForObject( - "SELECT int_job_min_cores FROM folder WHERE pk_folder=?", - Integer.class, group.getGroupId())); - groupDao.updateDefaultJobMinCores(group, 100); - assertEquals(Integer.valueOf(100), jdbcTemplate.queryForObject( - "SELECT int_job_min_cores FROM folder WHERE pk_folder=?", - Integer.class, group.getGroupId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateDefaultJobPriority() { - GroupDetail group = createGroup(); - assertEquals(Integer.valueOf(-1), jdbcTemplate.queryForObject( - "SELECT int_job_priority FROM folder WHERE pk_folder=?", - Integer.class, group.getGroupId())); - groupDao.updateDefaultJobPriority(group, 1000); - assertEquals(Integer.valueOf(1000), jdbcTemplate.queryForObject( - "SELECT int_job_priority FROM folder WHERE pk_folder=?", - Integer.class, group.getGroupId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateMinCores() { - GroupDetail group = createGroup(); - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT int_min_cores FROM folder_resource WHERE pk_folder=?", - Integer.class, group.getGroupId())); - groupDao.updateMinCores(group, 10); - assertEquals(Integer.valueOf(10), jdbcTemplate.queryForObject( - "SELECT int_min_cores FROM folder_resource WHERE pk_folder=?", - Integer.class, group.getGroupId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateMaxCores() { - GroupDetail group = createGroup(); - assertEquals(Integer.valueOf(-1), jdbcTemplate.queryForObject( - "SELECT int_max_cores FROM folder_resource WHERE pk_folder=?", - Integer.class, group.getGroupId())); - groupDao.updateMaxCores(group, 100); - assertEquals(Integer.valueOf(100), jdbcTemplate.queryForObject( - "SELECT int_max_cores FROM folder_resource WHERE pk_folder=?", - Integer.class, group.getGroupId())); - groupDao.updateMaxCores(group, -5); - assertEquals(Integer.valueOf(-1), jdbcTemplate.queryForObject( - "SELECT int_max_cores FROM folder_resource WHERE pk_folder=?", - Integer.class, group.getGroupId())); - } - - @Test - @Transactional - @Rollback(true) - public void testIsManaged() { - GroupDetail group = createGroup(); - assertEquals(false, groupDao.isManaged(group)); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateName() { - GroupDetail group = createGroup(); - groupDao.updateName(group, "NewName"); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateDepartment() { - GroupDetail group = createGroup(); - groupDao.updateDepartment(group, departmentDao.findDepartment("Lighting")); - } - - @Test - @Transactional - @Rollback(true) - public void testGetGroupDetail() { - GroupDetail group = createGroup(); - GroupDetail group2 = groupDao.getGroupDetail(group.id); - } - - @Test - @Transactional - @Rollback(true) - public void testGetChildrenRecursive() { - boolean is_test2 = false; - boolean is_test3 = false; - - GroupDetail g1 = new GroupDetail(); - g1.name = "Test1"; - g1.showId = getShow().getId(); - g1.deptId = departmentDao.getDefaultDepartment().getId(); - groupDao.insertGroup(g1, groupDao.getRootGroupDetail(getShow())); - - GroupDetail g2 = new GroupDetail(); - g2.name = "Test2"; - g2.showId = getShow().getId(); - g2.deptId = departmentDao.getDefaultDepartment().getId(); - groupDao.insertGroup(g2, groupDao.getRootGroupDetail(getShow())); - - for ( GroupInterface g: groupDao.getChildrenRecursive(groupDao.getGroup("A0000000-0000-0000-0000-000000000000"))) { - if (g.getName().equals("Test1")) { - is_test2 = true; - } - if (g.getName().equals("Test2")) { - is_test3 = true; - } - } - assertTrue(is_test2); - assertTrue(is_test3); - } - - @Test - @Transactional - @Rollback(true) - public void testGetChildren() { - boolean is_testuserA = false; - boolean is_testuserB = false; - - GroupDetail g1 = new GroupDetail(); - g1.name = "testuserA"; - g1.showId = getShow().getId(); - g1.deptId = departmentDao.getDefaultDepartment().getId(); - groupDao.insertGroup(g1, groupDao.getRootGroupDetail(getShow())); - - GroupDetail g2 = new GroupDetail(); - g2.name = "testuserB"; - g2.showId = getShow().getId(); - g2.deptId = departmentDao.getDefaultDepartment().getId(); - groupDao.insertGroup(g2, groupDao.getRootGroupDetail(getShow())); - - List groups = groupDao.getChildren(groupDao.getGroup("A0000000-0000-0000-0000-000000000000")); - for (GroupInterface g : groups) { - if (g.getName().equals("testuserA")) { - is_testuserA = true; - } - if (g.getName().equals("testuserB")) { - is_testuserB = true; - } - } - assertTrue(is_testuserA); - assertTrue(is_testuserB); - } - - @Test - @Transactional - @Rollback(true) - public void testIsOverMinCores() { - - JobDetail job = launchJob(); - assertFalse(groupDao.isOverMinCores(job)); - - String groupid = jdbcTemplate.queryForObject("SELECT pk_folder FROM job WHERE pk_job=?", - String.class, job.getJobId()); - - // Now update some values so it returns true. - jdbcTemplate.update("UPDATE folder_resource SET int_cores = int_min_cores + 1 WHERE pk_folder=?", - groupid); - - assertTrue(groupDao.isOverMinCores(job)); - } -} - - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/HistoricalDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/HistoricalDaoTests.java deleted file mode 100644 index 993e03de3..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/HistoricalDaoTests.java +++ /dev/null @@ -1,82 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import java.io.File; -import javax.annotation.Resource; - -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.JobDetail; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.HistoricalDao; -import com.imageworks.spcue.service.JobLauncher; -import com.imageworks.spcue.service.JobManager; -import com.imageworks.spcue.test.AssumingOracleEngine; - -import static org.junit.Assert.assertEquals; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class HistoricalDaoTests extends - AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - private JobManager jobManager; - - @Resource - private JobLauncher jobLauncher; - - @Resource - private HistoricalDao historicalDao; - - @Test - @Transactional - @Rollback(true) - public void testGetFinishedJobs() { - historicalDao.getFinishedJobs(24); - } - - @Test - @Transactional - @Rollback(true) - public void testTransferJob() { - jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec.xml")); - JobDetail j = jobManager.findJobDetail("pipe-dev.cue-testuser_shell_v1"); - jobManager.shutdownJob(j); - historicalDao.transferJob(j); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM job_history WHERE pk_job=?", - Integer.class, j.getJobId())); - } -} - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/HostDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/HostDaoTests.java deleted file mode 100644 index 37ffb61ab..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/HostDaoTests.java +++ /dev/null @@ -1,571 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import java.math.BigDecimal; -import java.sql.Timestamp; -import java.util.Map; -import javax.annotation.Resource; - -import com.google.common.collect.ImmutableList; -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.test.context.transaction.AfterTransaction; -import org.springframework.test.context.transaction.BeforeTransaction; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.DispatchHost; -import com.imageworks.spcue.HostEntity; -import com.imageworks.spcue.HostInterface; -import com.imageworks.spcue.Source; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.AllocationDao; -import com.imageworks.spcue.dao.FacilityDao; -import com.imageworks.spcue.dao.HostDao; -import com.imageworks.spcue.dispatcher.Dispatcher; -import com.imageworks.spcue.grpc.host.HardwareState; -import com.imageworks.spcue.grpc.host.HostTagType; -import com.imageworks.spcue.grpc.host.LockState; -import com.imageworks.spcue.grpc.host.ThreadMode; -import com.imageworks.spcue.grpc.report.HostReport; -import com.imageworks.spcue.grpc.report.RenderHost; -import com.imageworks.spcue.service.HostManager; -import com.imageworks.spcue.test.AssumingOracleEngine; -import com.imageworks.spcue.util.CueUtil; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class HostDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - private static final String TEST_HOST = "beta"; - - @Resource - protected AllocationDao allocationDao; - - @Resource - protected HostDao hostDao; - - @Resource - protected HostManager hostManager; - - @Resource - protected FacilityDao facilityDao; - - public HostDaoTests() { } - - public static RenderHost buildRenderHost(String name) { - RenderHost host = RenderHost.newBuilder() - .setName(name) - .setBootTime(1192369572) - .setFreeMcp(7602) - .setFreeMem(15290520) - .setFreeSwap((int) CueUtil.MB512) - .setLoad(1) - .setNimbyEnabled(false) - .setTotalMcp(19543) - .setTotalMem((int) CueUtil.GB16) - .setTotalSwap((int) CueUtil.GB2) - .setNimbyEnabled(false) - .setNumProcs(2) - .setCoresPerProc(400) - .addAllTags(ImmutableList.of("linux", "64bit")) - .setState(HardwareState.UP) - .setFacility("spi") - .putAttributes("freeGpu", String.format("%d", CueUtil.MB512)) - .putAttributes("totalGpu", String.format("%d", CueUtil.MB512)) - .build(); - - return host; - } - - @Test - public void testInit() { } - - @BeforeTransaction - public void clear() { - jdbcTemplate.update( - "DELETE FROM host WHERE str_name=?", TEST_HOST); - } - - @AfterTransaction - public void destroy() { - jdbcTemplate.update( - "DELETE FROM host WHERE str_name=?", TEST_HOST); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertHost() { - hostDao.insertRenderHost(buildRenderHost(TEST_HOST), - hostManager.getDefaultAllocationDetail(), - false); - - assertEquals(Long.valueOf(CueUtil.GB16 - Dispatcher.MEM_RESERVED_SYSTEM), jdbcTemplate.queryForObject( - "SELECT int_mem FROM host WHERE str_name=?", - Long.class, TEST_HOST)); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertHostFQDN1() { - String TEST_HOST_NEW = "ice-ns1.yvr"; - String FQDN_HOST = TEST_HOST_NEW + ".spimageworks.com"; - hostDao.insertRenderHost(buildRenderHost(FQDN_HOST), - hostManager.getDefaultAllocationDetail(), - true); - - HostEntity hostDetail = hostDao.findHostDetail(TEST_HOST_NEW); - assertEquals(TEST_HOST_NEW, hostDetail.name); - - HostInterface host = hostDao.findHost(FQDN_HOST); - HostEntity hostDetail2 = hostDao.getHostDetail(host); - assertEquals(TEST_HOST_NEW, hostDetail2.name); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertHostFQDN2() { - String TEST_HOST_NEW = "compile21"; - String FQDN_HOST = TEST_HOST_NEW + ".spimageworks.com"; - hostDao.insertRenderHost(buildRenderHost(FQDN_HOST), - hostManager.getDefaultAllocationDetail(), - false); - - HostEntity hostDetail = hostDao.findHostDetail(TEST_HOST_NEW); - assertEquals(TEST_HOST_NEW, hostDetail.name); - - HostInterface host = hostDao.findHost(FQDN_HOST); - HostEntity hostDetail2 = hostDao.getHostDetail(host); - assertEquals(TEST_HOST_NEW, hostDetail2.name); - - } - - @Test - @Transactional - @Rollback(true) - public void testInsertHostFQDN3() { - String TEST_HOST_NEW = "hostname"; - String FQDN_HOST = TEST_HOST_NEW + ".fake.co.uk"; - hostDao.insertRenderHost(buildRenderHost(FQDN_HOST), - hostManager.getDefaultAllocationDetail(), - false); - - HostEntity hostDetail = hostDao.findHostDetail(TEST_HOST_NEW); - assertEquals(TEST_HOST_NEW, hostDetail.name); - - HostInterface host = hostDao.findHost(FQDN_HOST); - HostEntity hostDetail2 = hostDao.getHostDetail(host); - assertEquals(TEST_HOST_NEW, hostDetail2.name); - - } - - @Test - @Transactional - @Rollback(true) - public void testInsertHostAlternateOS() { - - RenderHost host = buildRenderHost(TEST_HOST).toBuilder() - .putAttributes("SP_OS", "spinux1") - .build(); - - hostDao.insertRenderHost(host, - hostManager.getDefaultAllocationDetail(), - false); - - assertEquals("spinux1",jdbcTemplate.queryForObject( - "SELECT str_os FROM host_stat, host " + - "WHERE host.pk_host = host_stat.pk_host " + - "AND host.str_name=?",String.class, TEST_HOST), "spinux1"); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertHostDesktop() { - - RenderHost host = buildRenderHost(TEST_HOST); - hostDao.insertRenderHost(host, - hostManager.getDefaultAllocationDetail(), - false); - - assertEquals(Long.valueOf(CueUtil.GB16 - Dispatcher.MEM_RESERVED_SYSTEM), jdbcTemplate.queryForObject( - "SELECT int_mem FROM host WHERE str_name=?", - Long.class, TEST_HOST)); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateThreadMode() { - - RenderHost host = buildRenderHost(TEST_HOST); - host.toBuilder().setNimbyEnabled(true).build(); - hostDao.insertRenderHost(host, - hostManager.getDefaultAllocationDetail(), - false); - - HostEntity d = hostDao.findHostDetail(TEST_HOST); - hostDao.updateThreadMode(d, ThreadMode.AUTO); - - assertEquals(Integer.valueOf(ThreadMode.AUTO_VALUE), jdbcTemplate.queryForObject( - "SELECT int_thread_mode FROM host WHERE pk_host=?", - Integer.class, d.id)); - - hostDao.updateThreadMode(d, ThreadMode.ALL); - - assertEquals(Integer.valueOf(ThreadMode.ALL_VALUE), jdbcTemplate.queryForObject( - "SELECT int_thread_mode FROM host WHERE pk_host=?", - Integer.class, d.id)); - } - - @Test - @Transactional - @Rollback(true) - public void testGetHostDetail() { - - hostDao.insertRenderHost(buildRenderHost(TEST_HOST), - hostManager.getDefaultAllocationDetail(), - false); - - HostEntity host = hostDao.findHostDetail(TEST_HOST); - hostDao.getHostDetail(host); - hostDao.getHostDetail(host.getHostId()); - } - - @Test - @Transactional - @Rollback(true) - public void testIsHostLocked() { - hostDao.insertRenderHost(buildRenderHost(TEST_HOST), - hostManager.getDefaultAllocationDetail(), - false); - - HostEntity host = hostDao.findHostDetail(TEST_HOST); - assertEquals(hostDao.isHostLocked(host),false); - - hostDao.updateHostLock(host, LockState.LOCKED, new Source("TEST")); - assertEquals(hostDao.isHostLocked(host),true); - } - - @Test - @Transactional - @Rollback(true) - public void testIsKillMode() { - hostDao.insertRenderHost(buildRenderHost(TEST_HOST), - hostManager.getDefaultAllocationDetail(), - false); - - HostEntity host = hostDao.findHostDetail(TEST_HOST); - assertFalse(hostDao.isKillMode(host)); - - jdbcTemplate.update( - "UPDATE host_stat SET int_swap_free = ?, int_mem_free = ? WHERE pk_host = ?", - CueUtil.MB256, CueUtil.MB256, host.getHostId()); - - assertTrue(hostDao.isKillMode(host)); - } - - @Test - @Transactional - @Rollback(true) - public void testIsHostUp() { - hostDao.insertRenderHost(buildRenderHost(TEST_HOST), - hostManager.getDefaultAllocationDetail(), - false); - - assertTrue(hostDao.isHostUp(hostDao.findHostDetail(TEST_HOST))); - - hostDao.updateHostState(hostDao.findHostDetail(TEST_HOST), - HardwareState.DOWN); - assertFalse(hostDao.isHostUp(hostDao.findHostDetail(TEST_HOST))); - } - - @Test - @Transactional - @Rollback(true) - public void testHostExists() { - hostDao.insertRenderHost(buildRenderHost(TEST_HOST), - hostManager.getDefaultAllocationDetail(), - false); - - assertEquals(hostDao.hostExists(TEST_HOST),true); - assertEquals(hostDao.hostExists("frickjack"),false); - } - - @Test - @Transactional - @Rollback(true) - public void testDeleteHost() { - hostDao.insertRenderHost(buildRenderHost(TEST_HOST), - hostManager.getDefaultAllocationDetail(), - false); - - HostEntity host = hostDao.findHostDetail(TEST_HOST); - assertEquals(hostDao.hostExists(TEST_HOST),true); - hostDao.deleteHost(host); - assertEquals(hostDao.hostExists(TEST_HOST),false); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateHostRebootWhenIdle() { - hostDao.insertRenderHost(buildRenderHost(TEST_HOST), - hostManager.getDefaultAllocationDetail(), - false); - - HostEntity host = hostDao.findHostDetail(TEST_HOST); - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT b_reboot_idle FROM host WHERE pk_host=?", - Integer.class, host.getHostId())); - hostDao.updateHostRebootWhenIdle(host, true); - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT b_reboot_idle FROM host WHERE pk_host=?", - Integer.class, host.getHostId())); - } - - @Test - @Transactional - @Rollback(true) - public void updateHostStats() { - - hostDao.insertRenderHost(buildRenderHost(TEST_HOST), - hostManager.getDefaultAllocationDetail(), - false); - - DispatchHost dispatchHost = hostDao.findDispatchHost(TEST_HOST); - hostDao.updateHostStats(dispatchHost, - CueUtil.GB8, - CueUtil.GB8, - CueUtil.GB8, - CueUtil.GB8, - CueUtil.GB8, - CueUtil.GB8, - 1, - 1, - 100, - new Timestamp(1247526000 * 1000l), - "spinux1"); - - Map result = jdbcTemplate.queryForMap( - "SELECT * FROM host_stat WHERE pk_host=?", - dispatchHost.getHostId()); - - assertEquals(CueUtil.GB8, ((BigDecimal) - (result.get("int_mem_total"))).longValue()); - assertEquals(CueUtil.GB8, ((BigDecimal) - (result.get("int_mem_free"))).longValue()); - assertEquals(CueUtil.GB8, ((BigDecimal) - (result.get("int_swap_total"))).longValue()); - assertEquals(CueUtil.GB8, ((BigDecimal) - (result.get("int_swap_free"))).longValue()); - assertEquals(CueUtil.GB8, ((BigDecimal) - (result.get("int_mcp_total"))).longValue()); - assertEquals(CueUtil.GB8, ((BigDecimal) - (result.get("int_mcp_free"))).longValue()); - assertEquals(100, ((BigDecimal) - (result.get("int_load"))).intValue()); - assertEquals(new Timestamp(1247526000 * 1000l), - (Timestamp) result.get("ts_booted")); - - } - - @Test - @Transactional - @Rollback(true) - public void updateHostResources() { - - hostDao.insertRenderHost(buildRenderHost(TEST_HOST), - hostManager.getDefaultAllocationDetail(), - false); - - DispatchHost dispatchHost = hostDao.findDispatchHost(TEST_HOST); - HostReport report = HostReport.newBuilder() - .setHost( - buildRenderHost(TEST_HOST).toBuilder() - .setCoresPerProc(1200) - .setNumProcs(2) - .setTotalMem((int) CueUtil.GB32) - ).build(); - hostDao.updateHostResources(dispatchHost, report); - - // Verify what the original values are - assertEquals(800, dispatchHost.cores); - assertEquals(800, dispatchHost.idleCores); - assertEquals(CueUtil.GB16 - Dispatcher.MEM_RESERVED_SYSTEM, - dispatchHost.idleMemory); - assertEquals(CueUtil.GB16- Dispatcher.MEM_RESERVED_SYSTEM, - dispatchHost.memory); - - dispatchHost = hostDao.findDispatchHost(TEST_HOST); - - // Now verify they've changed. - assertEquals(2400, dispatchHost.cores); - assertEquals(2400, dispatchHost.idleCores); - assertEquals(CueUtil.GB32 - Dispatcher.MEM_RESERVED_SYSTEM, - dispatchHost.idleMemory); - assertEquals(CueUtil.GB32- Dispatcher.MEM_RESERVED_SYSTEM, - dispatchHost.memory); - } - - @Test - @Transactional - @Rollback(true) - public void testGetDispatchHost() { - hostDao.insertRenderHost(buildRenderHost(TEST_HOST), - hostManager.getDefaultAllocationDetail(), - false); - - HostEntity hostDetail = hostDao.findHostDetail(TEST_HOST); - DispatchHost dispatchHost = hostDao.findDispatchHost(TEST_HOST); - - assertEquals(dispatchHost.name, TEST_HOST); - assertEquals(dispatchHost.allocationId, hostDetail.getAllocationId()); - assertEquals(dispatchHost.id, hostDetail.getHostId()); - assertEquals(dispatchHost.cores, hostDetail.cores); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateHostSetAllocation() { - - hostDao.insertRenderHost(buildRenderHost(TEST_HOST), - hostManager.getDefaultAllocationDetail(), - false); - - HostEntity hostDetail = hostDao.findHostDetail(TEST_HOST); - - hostDao.updateHostSetAllocation(hostDetail, - hostManager.getDefaultAllocationDetail()); - - hostDetail = hostDao.findHostDetail(TEST_HOST); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateHostSetManualTags() { - DispatchHost host = hostManager.createHost(buildRenderHost(TEST_HOST)); - - hostDao.tagHost(host,"frick", HostTagType.MANUAL); - hostDao.tagHost(host,"jack", HostTagType.MANUAL); - hostDao.recalcuateTags(host.id); - - String tag = jdbcTemplate.queryForObject( - "SELECT str_tags FROM host WHERE pk_host=?",String.class, host.id); - assertEquals("unassigned beta frick jack", tag); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateHostSetOS() { - DispatchHost host = hostManager.createHost(buildRenderHost(TEST_HOST)); - hostDao.updateHostOs(host, "foo"); - String tag = jdbcTemplate.queryForObject( - "SELECT str_os FROM host_stat WHERE pk_host=?",String.class, host.id); - assertEquals("foo", tag); - } - - @Test - @Transactional - @Rollback(true) - public void testChangeTags() { - DispatchHost host = hostManager.createHost(buildRenderHost(TEST_HOST)); - - String tag = jdbcTemplate.queryForObject( - "SELECT str_tags FROM host WHERE pk_host=?",String.class, host.id); - assertEquals("unassigned beta", tag); - - hostDao.removeTag(host, "linux"); - hostDao.recalcuateTags(host.id); - - assertEquals("unassigned beta", jdbcTemplate.queryForObject( - "SELECT str_tags FROM host WHERE pk_host=?",String.class, host.id)); - - hostDao.tagHost(host, "32bit",HostTagType.MANUAL); - hostDao.recalcuateTags(host.id); - - assertEquals("unassigned beta 32bit", jdbcTemplate.queryForObject( - "SELECT str_tags FROM host WHERE pk_host=?",String.class, host.id)); - } - - @Test - @Transactional - @Rollback(true) - public void testGetStrandedCoreUnits() { - DispatchHost host = hostManager.createHost(buildRenderHost(TEST_HOST)); - - jdbcTemplate.update( - "UPDATE host SET int_mem_idle = ? WHERE pk_host = ?", - CueUtil.GB, host.getHostId()); - - assertEquals(host.idleCores, hostDao.getStrandedCoreUnits(host)); - - jdbcTemplate.update( - "UPDATE host SET int_mem_idle = ? WHERE pk_host = ?", - CueUtil.GB2, host.getHostId()); - - assertEquals(0, hostDao.getStrandedCoreUnits(host)); - - // Check to see if fractional cores is rounded to the lowest - // whole core properly. - jdbcTemplate.update( - "UPDATE host SET int_cores_idle=150, int_mem_idle = ? WHERE pk_host = ?", - CueUtil.GB, host.getHostId()); - - assertEquals(100, hostDao.getStrandedCoreUnits(host)); - } - - @Test - @Transactional - @Rollback(true) - public void testIsPreferShow() { - DispatchHost host = hostManager.createHost(buildRenderHost(TEST_HOST)); - assertFalse(hostDao.isPreferShow(host)); - } - - @Test - @Transactional - @Rollback(true) - public void testIsNimby() { - DispatchHost host = hostManager.createHost(buildRenderHost(TEST_HOST)); - assertFalse(hostDao.isNimbyHost(host)); - } -} - - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/JobDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/JobDaoTests.java deleted file mode 100644 index 31f8dca66..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/JobDaoTests.java +++ /dev/null @@ -1,681 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import java.io.File; -import java.util.HashMap; -import java.util.Map; -import javax.annotation.Resource; - -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.DispatchJob; -import com.imageworks.spcue.ExecutionSummary; -import com.imageworks.spcue.FacilityEntity; -import com.imageworks.spcue.GroupDetail; -import com.imageworks.spcue.GroupInterface; -import com.imageworks.spcue.JobDetail; -import com.imageworks.spcue.JobInterface; -import com.imageworks.spcue.PointInterface; -import com.imageworks.spcue.ResourceUsage; -import com.imageworks.spcue.TaskEntity; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.DepartmentDao; -import com.imageworks.spcue.dao.FacilityDao; -import com.imageworks.spcue.dao.GroupDao; -import com.imageworks.spcue.dao.JobDao; -import com.imageworks.spcue.dao.PointDao; -import com.imageworks.spcue.dao.ShowDao; -import com.imageworks.spcue.dao.TaskDao; -import com.imageworks.spcue.grpc.job.JobState; -import com.imageworks.spcue.service.JobLauncher; -import com.imageworks.spcue.service.JobManager; -import com.imageworks.spcue.service.JobSpec; -import com.imageworks.spcue.test.AssumingOracleEngine; -import com.imageworks.spcue.util.JobLogUtil; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class JobDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - JobManager jobManager; - - @Resource - JobLauncher jobLauncher; - - @Resource - JobDao jobDao; - - @Resource - PointDao pointDao; - - @Resource - ShowDao showDao; - - @Resource - TaskDao taskDao; - - @Resource - GroupDao groupDao; - - @Resource - FacilityDao facilityDao; - - @Resource - DepartmentDao departmentDao; - - @Resource - JobLogUtil jobLogUtil; - - private static String ROOT_FOLDER = "A0000000-0000-0000-0000-000000000000"; - private static String ROOT_SHOW = "00000000-0000-0000-0000-000000000000"; - private static String JOB_NAME = "pipe-dev.cue-testuser_shell_v1"; - - @Before - public void testMode() { - jobLauncher.testMode = true; - } - - public JobDetail buildJobDetail() { - JobSpec spec = jobLauncher.parse(new File("src/test/resources/conf/jobspec/jobspec.xml")); - return spec.getJobs().get(0).detail; - } - - public JobDetail insertJob() { - JobDetail job = this.buildJobDetail(); - job.groupId = ROOT_FOLDER; - job.showId = ROOT_SHOW; - job.logDir = jobLogUtil.getJobLogPath(job); - job.deptId = departmentDao.getDefaultDepartment().getId(); - job.facilityId = facilityDao.getDefaultFacility().getId(); - job.state = JobState.PENDING; - jobDao.insertJob(job, jobLogUtil); - return job; - } - - - public JobDetail launchJob() { - jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec.xml")); - return jobManager.findJobDetail("pipe-dev.cue-testuser_shell_v1"); - } - - @Test - @Transactional - @Rollback(true) - public void testGetDispatchJob() { - JobDetail job = insertJob(); - DispatchJob djob = jobDao.getDispatchJob(job.id); - assertEquals(djob.id, job.id); - } - - @Test - @Transactional - @Rollback(true) - public void testIsJobComplete() { - JobDetail job = insertJob(); - // returns true because there are no dispatchable frames - assertEquals(true,jobDao.isJobComplete(job)); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertJob() { - JobDetail job = this.buildJobDetail(); - job.groupId = ROOT_FOLDER; - job.showId = ROOT_SHOW; - job.logDir = jobLogUtil.getJobLogPath(job); - job.deptId = departmentDao.getDefaultDepartment().getId(); - job.facilityId= facilityDao.getDefaultFacility().getId(); - jobDao.insertJob(job, jobLogUtil); - assertNotNull(job.id); - } - - @Test - @Transactional - @Rollback(true) - public void testFindJob() { - JobDetail job = insertJob(); - JobInterface j1 = jobDao.findJob(job.name); - JobDetail j2 = jobDao.findJobDetail(job.name); - assertEquals(job.name, j1.getName()); - assertEquals(job.name, j2.getName()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetJob() { - JobDetail job = insertJob(); - jobDao.getJobDetail(job.id); - jobDao.getJob(job.id); - } - - @Test - @Transactional - @Rollback(true) - public void testGetJobsByTask() { - - PointInterface p = pointDao.getPointConfigDetail( - showDao.findShowDetail("pipe"), - departmentDao.getDefaultDepartment()); - - TaskEntity t = new TaskEntity(p, "dev.cue"); - taskDao.insertTask(t); - jobDao.getJobs(t); - } - - @Test - @Transactional - @Rollback(true) - public void testJobExists() { - assertFalse(jobDao.exists(JOB_NAME)); - JobDetail job = insertJob(); - jdbcTemplate.update("UPDATE job SET str_state='PENDING' WHERE pk_job=?", - job.id); - assertTrue(jobDao.exists(JOB_NAME)); - } - - @Test - @Transactional - @Rollback(true) - public void testDeleteJob() { - jobDao.deleteJob(insertJob()); - } - - @Test - @Transactional - @Rollback(true) - public void testActivateJob() { - jobDao.activateJob(insertJob(), JobState.PENDING); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateJobState() { - JobDetail job = insertJob(); - assertEquals(JobState.PENDING, job.state); - jobDao.updateState(job, JobState.FINISHED); - assertEquals(JobState.FINISHED.toString(), - jdbcTemplate.queryForObject( - "SELECT str_state FROM job WHERE pk_job=?", - String.class, job.getJobId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateJobFinished() { - jobDao.updateJobFinished(insertJob()); - } - - @Test - @Transactional - @Rollback(true) - public void testIsJobOverMinProc() { - JobDetail job = insertJob(); - assertFalse(jobDao.isOverMinCores(job)); - } - - @Test - @Transactional - @Rollback(true) - public void testHasPendingFrames() { - assertFalse(jobDao.hasPendingFrames(insertJob())); - } - - @Test - @Transactional - @Rollback(true) - public void testIsJobOverMaxProc() { - JobDetail job = insertJob(); - assertFalse(jobDao.isOverMaxCores(job)); - } - - @Test - @Transactional - @Rollback(true) - public void testIsJobAtMaxCores() { - JobDetail job = insertJob(); - assertFalse(jobDao.isAtMaxCores(job)); - - jdbcTemplate.update( - "UPDATE job_resource SET int_cores = int_max_cores WHERE pk_job=?", - job.getJobId()); - - assertTrue(jobDao.isAtMaxCores(job)); - - } - - @Test - @Transactional - @Rollback(true) - public void testIsOverMaxCores() { - JobDetail job = insertJob(); - jobDao.updateMaxCores(job, 500); - jdbcTemplate.update( - "UPDATE job_resource SET int_cores = 450 WHERE pk_job=?", - job.getJobId()); - - assertFalse(jobDao.isOverMaxCores(job)); - assertFalse(jobDao.isOverMaxCores(job, 50)); - assertTrue(jobDao.isOverMaxCores(job, 100)); - - jdbcTemplate.update( - "UPDATE job_resource SET int_max_cores = 200 WHERE pk_job=?", - job.getJobId()); - assertTrue(jobDao.isOverMaxCores(job)); - } - - @Test(expected=org.springframework.jdbc.UncategorizedSQLException.class) - @Transactional - @Rollback(true) - public void testMaxCoreTrigger() { - JobDetail job = insertJob(); - int maxCores = jdbcTemplate.queryForObject( - "SELECT int_max_cores FROM job_resource WHERE pk_job=?", - Integer.class, job.getJobId()); - - jdbcTemplate.update( - "UPDATE job_resource SET int_cores = ? WHERE pk_job=?", - maxCores + 1, job.getJobId()); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateJobPriority() { - JobDetail job = insertJob(); - jobDao.updatePriority(job, 199); - assertEquals(Integer.valueOf(199), jdbcTemplate.queryForObject( - "SELECT int_priority FROM job_resource WHERE pk_job=?", - Integer.class, job.getJobId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateJobMinCores() { - JobDetail job = insertJob(); - jobDao.updateMinCores(job, 100); - assertEquals(Integer.valueOf(100), jdbcTemplate.queryForObject( - "SELECT int_min_cores FROM job_resource WHERE pk_job=?", - Integer.class, job.getJobId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateJobMaxCores() { - JobDetail job = insertJob(); - jobDao.updateMaxCores(job, 100); - assertEquals(Integer.valueOf(100), jdbcTemplate.queryForObject( - "SELECT int_max_cores FROM job_resource WHERE pk_job=?", - Integer.class, job.getJobId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateJobMinCoresByGroup() { - JobDetail job = insertJob(); - GroupInterface g = groupDao.getGroup(job.groupId); - jobDao.updateMinCores(g, 100); - assertEquals(Integer.valueOf(100), jdbcTemplate.queryForObject( - "SELECT int_min_cores FROM job_resource WHERE pk_job=?", - Integer.class, job.getJobId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateJobMaxCoresByGroup() { - JobDetail job = insertJob(); - GroupInterface g = groupDao.getGroup(job.groupId); - jobDao.updateMaxCores(g, 100); - assertEquals(Integer.valueOf(100), jdbcTemplate.queryForObject( - "SELECT int_max_cores FROM job_resource WHERE pk_job=?", - Integer.class, job.getJobId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateJobPriorityByGroup() { - JobDetail job = insertJob(); - GroupInterface g = groupDao.getGroup(job.groupId); - jobDao.updatePriority(g, 100); - assertEquals(Integer.valueOf(100), jdbcTemplate.queryForObject( - "SELECT int_priority FROM job_resource WHERE pk_job=?", - Integer.class, job.getJobId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateJobMaxRss() { - long maxRss = 100000; - JobDetail job = insertJob(); - jobDao.updateMaxRSS(job, maxRss); - assertEquals(Long.valueOf(maxRss), jdbcTemplate.queryForObject( - "SELECT int_max_rss FROM job_mem WHERE pk_job=?", - Long.class, job.getJobId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateJobPaused() { - JobDetail job = insertJob(); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT b_paused FROM job WHERE pk_job=?", - Integer.class, job.getJobId())); - - jobDao.updatePaused(job, false); - - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT b_paused FROM job WHERE pk_job=?", - Integer.class, job.getJobId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateJobAutoEat() { - JobDetail job = insertJob(); - - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT b_autoeat FROM job WHERE pk_job=?", - Integer.class, job.getJobId())); - - jobDao.updateAutoEat(job, true); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT b_autoeat FROM job WHERE pk_job=?", - Integer.class, job.getJobId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateJobMaxRetries() { - JobDetail job = insertJob(); - jobDao.updateMaxFrameRetries(job,10); - assertEquals(Integer.valueOf(10), jdbcTemplate.queryForObject( - "SELECT int_max_retries FROM job WHERE pk_job=?", - Integer.class, job.getJobId())); - } - - @Test(expected=IllegalArgumentException.class) - @Transactional - @Rollback(true) - public void testUpdateJobMaxRetriesTooLow() { - JobDetail job = insertJob(); - jobDao.updateMaxFrameRetries(job,-1); - } - - @Test(expected=IllegalArgumentException.class) - @Transactional - @Rollback(true) - public void testUpdateJobMaxRetriesTooHigh() { - JobDetail job = insertJob(); - jobDao.updateMaxFrameRetries(job,100000); - } - - @Test - @Transactional - @Rollback(true) - public void testGetFrameStateTotals() { - JobSpec spec = jobLauncher.parse(new File("src/test/resources/conf/jobspec/jobspec.xml")); - jobLauncher.launch(spec); - jobDao.getFrameStateTotals(spec.getJobs().get(0).detail); - } - - @Test - @Transactional - @Rollback(true) - public void testGetExecutionSummary() { - JobDetail job = launchJob(); - ExecutionSummary summary = jobDao.getExecutionSummary(job); - } - - @Test - @Transactional - @Rollback(true) - public void testGetJobEnvironment() { - JobDetail job = launchJob(); - Map map = jobDao.getEnvironment(job); - for (Map.Entry e : map.entrySet()) { - assertEquals("VNP_VCR_SESSION", e.getKey()); - assertEquals( "9000", e.getValue()); - } - } - - @Test - @Transactional - @Rollback(true) - public void testInsertJobEnvironment() { - JobDetail job = launchJob(); - jobDao.insertEnvironment(job, "CHAMBERS","123"); - Map map = jobDao.getEnvironment(job); - assertEquals(2,map.size()); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertJobEnvironmentMap() { - JobDetail job = launchJob(); - Map map = new HashMap(); - map.put("CHAMBERS","123"); - map.put("OVER9000","123"); - - jobDao.insertEnvironment(job, map); - Map env = jobDao.getEnvironment(job); - assertEquals(3,env.size()); - } - - @Test - @Transactional - @Rollback(true) - public void testFindLastJob() { - JobSpec spec = jobLauncher.parse(new File("src/test/resources/conf/jobspec/jobspec.xml")); - jobLauncher.launch(spec); - - JobInterface job = spec.getJobs().get(0).detail; - jobDao.getFrameStateTotals(job); - jobManager.shutdownJob(job); - // this might fail - JobDetail oldJob = jobDao.findLastJob(job.getName()); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateJobLogPath() { - JobDetail job = launchJob(); - String newLogDir = "/path/to/nowhere"; - jobDao.updateLogPath(job,newLogDir); - assertEquals(newLogDir,jdbcTemplate.queryForObject( - "SELECT str_log_dir FROM job WHERE pk_job=?",String.class, job.id)); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateJobParent() { - JobDetail job = launchJob(); - - // Make a new test group. - GroupDetail root = groupDao.getRootGroupDetail(job); - - GroupDetail testGroup = new GroupDetail(); - testGroup.name = "testGroup"; - testGroup.deptId = departmentDao.getDefaultDepartment().getId(); - testGroup.showId = root.getShowId(); - - groupDao.insertGroup(testGroup, root); - - jdbcTemplate.update( - "UPDATE folder SET int_job_max_cores=-1, int_job_min_cores=-1, int_job_priority=-1 WHERE pk_folder=?", - testGroup.getId()); - - GroupDetail group = groupDao.getGroupDetail(testGroup.getId()); - jobDao.updateParent(job, group); - - assertEquals(-1,group.jobMaxCores); - assertEquals(-1,group.jobMinCores); - assertEquals(-1,group.jobPriority); - - assertEquals(group.getGroupId(),jdbcTemplate.queryForObject( - "SELECT pk_folder FROM job WHERE pk_job=?",String.class, job.id)); - - assertEquals(group.getDepartmentId(),jdbcTemplate.queryForObject( - "SELECT pk_dept FROM job WHERE pk_job=?",String.class, job.id)); - - group.jobMaxCores = 100; - group.jobMinCores = 100; - group.jobPriority = 100; - - jobDao.updateParent(job, group); - - assertEquals(Integer.valueOf(group.jobMaxCores) ,jdbcTemplate.queryForObject( - "SELECT int_max_cores FROM job_resource WHERE pk_job=?", - Integer.class, job.id)); - - assertEquals(Integer.valueOf(group.jobMinCores) ,jdbcTemplate.queryForObject( - "SELECT int_min_cores FROM job_resource WHERE pk_job=?", - Integer.class, job.id)); - - assertEquals(Integer.valueOf(group.jobPriority) ,jdbcTemplate.queryForObject( - "SELECT int_priority FROM job_resource WHERE pk_job=?", - Integer.class, job.id)); - } - - @Test - @Transactional - @Rollback(true) - public void testCueHasPendingJobs() { - jobDao.cueHasPendingJobs(new FacilityEntity("0")); - - } - - @Test - @Transactional - @Rollback(true) - public void mapPostJob() { - JobSpec spec = jobLauncher.parse( - new File("src/test/resources/conf/jobspec/jobspec_postframes.xml")); - jobLauncher.launch(spec); - - final String pk_job = spec.getJobs().get(0).detail.id; - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM job_post WHERE pk_job=?", - Integer.class, pk_job)); - } - - @Test - @Transactional - @Rollback(true) - public void activatePostJob() { - JobSpec spec = jobLauncher.parse( - new File("src/test/resources/conf/jobspec/jobspec_postframes.xml")); - jobLauncher.launch(spec); - - jobDao.activatePostJob(spec.getJobs().get(0).detail); - - assertEquals(JobState.PENDING.toString(),jdbcTemplate.queryForObject( - "SELECT str_state FROM job WHERE pk_job=?", String.class, - spec.getJobs().get(0).getPostJob().detail.id)); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateUsage() { - - JobSpec spec = jobLauncher.parse( - new File("src/test/resources/conf/jobspec/jobspec.xml")); - jobLauncher.launch(spec); - - JobInterface job = jobDao.findJob(spec.getJobs().get(0).detail.name); - - /** 60 seconds of 100 core units **/ - ResourceUsage usage = new ResourceUsage(60, 33); - - assertTrue(usage.getClockTimeSeconds() > 0); - assertTrue(usage.getCoreTimeSeconds() > 0); - - /** - * Successful frame - */ - jobDao.updateUsage(job, usage, 0); - assertEquals(Long.valueOf(usage.getClockTimeSeconds()), jdbcTemplate.queryForObject( - "SELECT int_clock_time_success FROM job_usage WHERE pk_job=?", - Long.class, job.getId())); - - assertEquals(Long.valueOf(usage.getCoreTimeSeconds()), jdbcTemplate.queryForObject( - "SELECT int_core_time_success FROM job_usage WHERE pk_job=?", - Long.class, job.getId())); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT int_frame_success_count FROM job_usage WHERE pk_job=?", - Integer.class, job.getId())); - - /** - * Failed frame - */ - jobDao.updateUsage(job, usage, 1); - assertEquals(Long.valueOf(usage.getClockTimeSeconds()), jdbcTemplate.queryForObject( - "SELECT int_clock_time_fail FROM job_usage WHERE pk_job=?", - Long.class, job.getId())); - - assertEquals(Long.valueOf(usage.getCoreTimeSeconds()), jdbcTemplate.queryForObject( - "SELECT int_core_time_fail FROM job_usage WHERE pk_job=?", - Long.class, job.getId())); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT int_frame_fail_count FROM job_usage WHERE pk_job=?", - Integer.class, job.getId())); - } -} - - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/LayerDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/LayerDaoTests.java deleted file mode 100644 index 61ee6e864..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/LayerDaoTests.java +++ /dev/null @@ -1,742 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import java.io.File; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import javax.annotation.Resource; - -import org.apache.commons.lang.StringUtils; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.BuildableLayer; -import com.imageworks.spcue.JobDetail; -import com.imageworks.spcue.LayerDetail; -import com.imageworks.spcue.LayerInterface; -import com.imageworks.spcue.LimitEntity; -import com.imageworks.spcue.LimitInterface; -import com.imageworks.spcue.ResourceUsage; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.DepartmentDao; -import com.imageworks.spcue.dao.FacilityDao; -import com.imageworks.spcue.dao.JobDao; -import com.imageworks.spcue.dao.LayerDao; -import com.imageworks.spcue.dao.LimitDao; -import com.imageworks.spcue.dispatcher.Dispatcher; -import com.imageworks.spcue.grpc.job.JobState; -import com.imageworks.spcue.grpc.job.LayerType; -import com.imageworks.spcue.service.JobLauncher; -import com.imageworks.spcue.service.JobManager; -import com.imageworks.spcue.service.JobSpec; -import com.imageworks.spcue.test.AssumingOracleEngine; -import com.imageworks.spcue.util.CueUtil; -import com.imageworks.spcue.util.FrameSet; -import com.imageworks.spcue.util.JobLogUtil; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class LayerDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - JobDao jobDao; - - @Resource - LayerDao layerDao; - - @Resource - LimitDao limitDao; - - @Resource - JobManager jobManager; - - @Resource - JobLauncher jobLauncher; - - @Resource - DepartmentDao departmentDao; - - @Resource - FacilityDao facilityDao; - - @Resource - JobLogUtil jobLogUtil; - - private static String ROOT_FOLDER = "A0000000-0000-0000-0000-000000000000"; - private static String ROOT_SHOW = "00000000-0000-0000-0000-000000000000"; - private static String LAYER_NAME = "pass_1"; - private static String JOB_NAME = "pipe-dev.cue-testuser_shell_v1"; - private static String LIMIT_NAME = "test-limit"; - private static String LIMIT_TEST_A = "testlimita"; - private static String LIMIT_TEST_B = "testlimitb"; - private static String LIMIT_TEST_C = "testlimitc"; - private static int LIMIT_MAX_VALUE = 32; - - @Before - public void testMode() { - jobLauncher.testMode = true; - } - - public LayerDetail getLayer() { - - JobSpec spec = jobLauncher.parse(new File("src/test/resources/conf/jobspec/jobspec.xml")); - JobDetail job = spec.getJobs().get(0).detail; - job.groupId = ROOT_FOLDER; - job.showId = ROOT_SHOW; - job.logDir = jobLogUtil.getJobLogPath(job); - job.deptId = departmentDao.getDefaultDepartment().getId(); - job.facilityId = facilityDao.getDefaultFacility().getId(); - jobDao.insertJob(job, jobLogUtil); - - LayerDetail lastLayer= null; - String limitId = limitDao.createLimit(LIMIT_NAME, LIMIT_MAX_VALUE); - limitDao.createLimit(LIMIT_TEST_A, 1); - limitDao.createLimit(LIMIT_TEST_B, 2); - limitDao.createLimit(LIMIT_TEST_C, 3); - - for (BuildableLayer buildableLayer: spec.getJobs().get(0).getBuildableLayers()) { - - LayerDetail layer = buildableLayer.layerDetail; - FrameSet frameSet = new FrameSet(layer.range); - int num_frames = frameSet.size(); - int chunk_size = layer.chunkSize; - - layer.jobId = job.id; - layer.showId = ROOT_SHOW; - layer.totalFrameCount = num_frames / chunk_size; - if (num_frames % chunk_size > 0) { layer.totalFrameCount++; } - - layerDao.insertLayerDetail(layer); - layerDao.insertLayerEnvironment(layer, buildableLayer.env); - layerDao.addLimit(layer, limitId); - lastLayer = layer; - } - - return lastLayer; - } - - public JobDetail getJob() { - return jobDao.findJobDetail(JOB_NAME); - } - - public String getTestLimitId(String name) { - return limitDao.findLimit(name).getLimitId(); - } - - @Test - @Transactional - @Rollback(true) - public void testIsLayerComplete() { - layerDao.isLayerComplete(getLayer()); - } - - @Test - @Transactional - @Rollback(true) - public void testIsLayerDispatchable() { - layerDao.isLayerDispatchable(getLayer()); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertLayerDetail() { - LayerDetail layer = getLayer(); - assertEquals(LAYER_NAME, layer.name); - assertEquals(layer.chunkSize, 1); - assertEquals(layer.dispatchOrder,2); - assertNotNull(layer.id); - assertNotNull(layer.jobId); - assertEquals(layer.showId,ROOT_SHOW); - } - - @Test - @Transactional - @Rollback(true) - public void testGetLayerDetail() { - LayerDetail layer = getLayer(); - assertEquals(LAYER_NAME, layer.name); - assertEquals(layer.chunkSize, 1); - assertEquals(layer.dispatchOrder,2); - assertNotNull(layer.id); - assertNotNull(layer.jobId); - assertEquals(layer.showId,ROOT_SHOW); - - LayerDetail l2 = layerDao.getLayerDetail(layer); - LayerDetail l3 = layerDao.getLayerDetail(layer.id); - assertEquals(l2, l3); - } - - @Test - @Transactional - @Rollback(true) - public void testGetLayerDetails() { - LayerDetail layer = getLayer(); - List ld = layerDao.getLayerDetails(getJob()); - assertEquals(ld.get(0).name, LAYER_NAME); - } - - @Test - @Transactional - @Rollback(true) - public void testFindLayerDetail() { - LayerDetail layer = getLayer(); - layerDao.findLayer(getJob(), "pass_1"); - } - - @Test - @Transactional - @Rollback(true) - public void testGetLayer() { - LayerDetail layer = getLayer(); - layerDao.getLayer(layer.id); - layerDao.getLayerDetail(layer); - layerDao.getLayerDetail(layer.id); - } - - @Test - @Transactional - @Rollback(true) - public void testFindLayer() { - LayerDetail layer = getLayer(); - layerDao.findLayer(getJob(), "pass_1"); - layerDao.findLayerDetail(getJob(), "pass_1"); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateLayerMinCores() { - LayerDetail layer = getLayer(); - layerDao.updateLayerMinCores(layer, 200); - LayerDetail l2 = layerDao.findLayerDetail(getJob(), "pass_1"); - assertEquals(l2.minimumCores,200); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateLayerThreadable() { - LayerDetail layer = getLayer(); - layerDao.updateThreadable(layer, false); - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT b_threadable FROM layer WHERE pk_layer=?", - Integer.class, layer.getLayerId())); - } - - - @Test - @Transactional - @Rollback(true) - public void testUpdateLayerMinMemory() { - LayerDetail layer = getLayer(); - - /* - * Check to ensure going below Dispatcher.MEM_RESERVED_MIN is - * not allowed. - */ - layerDao.updateLayerMinMemory(layer, 8096); - LayerDetail l2 = layerDao.findLayerDetail(getJob(), "pass_1"); - assertEquals(l2.minimumMemory, Dispatcher.MEM_RESERVED_MIN); - - /* - * Check regular operation. - */ - layerDao.updateLayerMinMemory(layer, CueUtil.GB); - LayerDetail l3 = layerDao.findLayerDetail(getJob(), "pass_1"); - assertEquals(l3.minimumMemory, CueUtil.GB); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateLayerTags() { - LayerDetail layer = getLayer(); - - HashSet tags = new HashSet(); - tags.add("frickjack"); - tags.add("pancake"); - - layerDao.updateLayerTags(layer, tags); - LayerDetail l2 = layerDao.findLayerDetail(getJob(), "pass_1"); - assertEquals(StringUtils.join(l2.tags," | "), "frickjack | pancake"); - - tags.clear(); - tags.add("frickjack"); - - layerDao.updateLayerTags(layer, tags); - l2 = layerDao.findLayerDetail(getJob(), "pass_1"); - assertEquals(StringUtils.join(l2.tags," | "), "frickjack"); - } - - @Test - @Transactional - @Rollback(true) - public void testGetFrameStateTotals() { - LayerDetail layer = getLayer(); - layerDao.getFrameStateTotals(layer); - jobDao.getFrameStateTotals(layer); - } - - @Test - @Transactional - @Rollback(true) - public void testGetExecutionSummary() { - LayerDetail layer = getLayer(); - layerDao.getExecutionSummary(layer); - } - - @Test - @Transactional - @Rollback(true) - public void testGetLayerEnvironment() { - LayerDetail layer = getLayer(); - Map map = layerDao.getLayerEnvironment(layer); - for (Map.Entry e : map.entrySet()) { - - } - } - - @Test - @Transactional - @Rollback(true) - public void testInsertLayerEnvironment() { - LayerDetail layer = getLayer(); - layerDao.insertLayerEnvironment(layer, "CHAMBERS","123"); - Map env = layerDao.getLayerEnvironment(layer); - assertEquals(2,env.size()); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertLayerEnvironmentMap() { - LayerDetail layer = getLayer(); - Map map = new HashMap(); - map.put("CHAMBERS","123"); - map.put("OVER9000","123"); - - layerDao.insertLayerEnvironment(layer, map); - Map env = layerDao.getLayerEnvironment(layer); - assertEquals(3,env.size()); - } - - - @Test - @Transactional - @Rollback(true) - public void testFindPastSameNameMaxRSS() { - getLayer(); - jobDao.updateState(getJob(), JobState.FINISHED); - assertEquals(JobState.FINISHED, getJob().state); - - JobDetail lastJob = null; - lastJob = jobDao.findLastJob("pipe-dev.cue-testuser_shell_v1"); - long maxRss = layerDao.findPastMaxRSS(lastJob, "pass_1"); - } - - @Test - @Transactional - @Rollback(true) - public void testFindPastTimeStampMaxRSS() { - getLayer(); - jobDao.updateState(getJob(), JobState.FINISHED); - assertEquals(JobState.FINISHED, getJob().state); - - JobDetail lastJob = null; - lastJob = jobDao.findLastJob("pipe-dev.cue-testuser_shell_v1_2011_05_03_16_03"); - long maxRss = layerDao.findPastMaxRSS(lastJob, "pass_1"); - } - - @Test - @Transactional - @Rollback(true) - public void testFindPastNewVersionMaxRSS() { - getLayer(); - jobDao.updateState(getJob(), JobState.FINISHED); - assertEquals(JobState.FINISHED, getJob().state); - - JobDetail lastJob = null; - lastJob = jobDao.findLastJob("pipe-dev.cue-testuser_shell_v2"); - long maxRss = layerDao.findPastMaxRSS(lastJob, "pass_1"); - } - - @Test - @Transactional - @Rollback(true) - public void testFindPastNewVersionTimeStampMaxRSS() { - getLayer(); - jobDao.updateState(getJob(), JobState.FINISHED); - assertEquals(JobState.FINISHED, getJob().state); - - JobDetail lastJob = null; - lastJob = jobDao.findLastJob("pipe-dev.cue-testuser_shell_v2_2011_05_03_16_03"); - long maxRss = layerDao.findPastMaxRSS(lastJob, "pass_1"); - } - - @Test(expected=org.springframework.dao.EmptyResultDataAccessException.class) - @Transactional - @Rollback(true) - public void testFindPastNewVersionFailMaxRSS() { - getLayer(); - jobDao.updateState(getJob(), JobState.FINISHED); - assertEquals(JobState.FINISHED, getJob().state); - - JobDetail lastJob = null; - lastJob = jobDao.findLastJob("pipe-dev.cue-testuser_shell_vfail_v2"); - long maxRss = layerDao.findPastMaxRSS(lastJob, "pass_1"); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateLayerMaxRSS() { - LayerDetail layer = getLayer(); - - layerDao.updateLayerMaxRSS(layer, 1000, true); - assertEquals(Long.valueOf(1000), jdbcTemplate.queryForObject( - "SELECT int_max_rss FROM layer_mem WHERE pk_layer=?", - Long.class, layer.getId())); - - layerDao.updateLayerMaxRSS(layer, 999, true); - assertEquals(Long.valueOf(999), jdbcTemplate.queryForObject( - "SELECT int_max_rss FROM layer_mem WHERE pk_layer=?", - Long.class, layer.getId())); - - layerDao.updateLayerMaxRSS(layer, 900, false); - assertEquals(Long.valueOf(999), jdbcTemplate.queryForObject( - "SELECT int_max_rss FROM layer_mem WHERE pk_layer=?", - Long.class, layer.getId())); - } - - @Test - @Transactional - @Rollback(true) - public void updateTags() { - String tag = "dillweed"; - LayerDetail layer = getLayer(); - layerDao.updateTags(layer, tag, LayerType.RENDER); - assertEquals(tag,jdbcTemplate.queryForObject( - "SELECT str_tags FROM layer WHERE pk_layer=?", String.class, layer.getLayerId())); - } - - @Test - @Transactional - @Rollback(true) - public void updateMinMemory() { - long mem = CueUtil.GB; - LayerDetail layer = getLayer(); - layerDao.updateMinMemory(layer, mem, LayerType.RENDER); - assertEquals(Long.valueOf(mem), jdbcTemplate.queryForObject( - "SELECT int_mem_min FROM layer WHERE pk_layer=?", - Long.class, layer.getLayerId())); - } - - @Test - @Transactional - @Rollback(true) - public void updateMinGpu() { - long gpu = CueUtil.GB; - LayerDetail layer = getLayer(); - layerDao.updateMinGpu(layer, gpu, LayerType.RENDER); - assertEquals(Long.valueOf(gpu),jdbcTemplate.queryForObject( - "SELECT int_gpu_min FROM layer WHERE pk_layer=?", - Long.class, layer.getLayerId())); - } - - @Test - @Transactional - @Rollback(true) - public void updateMinCores() { - int cores = CueUtil.ONE_CORE * 2; - LayerDetail layer = getLayer(); - layerDao.updateMinCores(layer, cores, LayerType.RENDER); - assertEquals(Integer.valueOf(cores), jdbcTemplate.queryForObject( - "SELECT int_cores_min FROM layer WHERE pk_layer=?", - Integer.class, layer.getLayerId())); - } - - @Test - @Transactional - @Rollback(true) - public void updateMaxCores() { - int cores = CueUtil.ONE_CORE * 2; - LayerDetail layer = getLayer(); - layerDao.updateLayerMaxCores(layer, cores); - assertEquals(Integer.valueOf(cores), jdbcTemplate.queryForObject( - "SELECT int_cores_max FROM layer WHERE pk_layer=?", - Integer.class, layer.getLayerId())); - } - - @Test - @Transactional - @Rollback(true) - public void isOptimizable() { - LayerDetail layer = getLayer(); - - assertFalse(layerDao.isOptimizable(layer, 5, 3600)); - - /* - * The succeeded count is good but the frames are too long - * Assert False - */ - jdbcTemplate.update("UPDATE layer_stat SET int_succeeded_count = 5 WHERE pk_layer=?", - layer.getLayerId()); - - jdbcTemplate.update( - "UPDATE layer_usage SET layer_usage.int_core_time_success = 3600 * 6" + - "WHERE pk_layer=?", layer.getLayerId()); - - assertFalse(layerDao.isOptimizable(layer, 5, 3600)); - - /* - * Set the frame times lower, so now we meet the criteria - * Assert True - */ - jdbcTemplate.update( - "UPDATE layer_usage SET layer_usage.int_core_time_success = 3500 * 5" + - "WHERE pk_layer=?", layer.getLayerId()); - - assertTrue(layerDao.isOptimizable(layer, 5, 3600)); - - /* - * Take the general tag away. If a layer is not a general layer - * it cannot be optmiized. - * Assert False - */ - jdbcTemplate.update( - "UPDATE layer SET str_tags=? WHERE pk_layer=?", - "desktop",layer.getLayerId()); - - assertFalse(layerDao.isOptimizable(layer, 5, 3600)); - - /* - * Layers that are already tagged util should return - * false as well. - * - * Assert False - */ - jdbcTemplate.update( - "UPDATE layer SET str_tags=? WHERE pk_layer=?", - "general | util",layer.getLayerId()); - - assertFalse(layerDao.isOptimizable(layer, 5, 3600)); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateUsage() { - LayerDetail layer = getLayer(); - - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT int_clock_time_success FROM layer_usage WHERE pk_layer=?", - Integer.class, layer.getId())); - - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT int_core_time_success FROM layer_usage WHERE pk_layer=?", - Integer.class, layer.getId())); - - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT int_frame_success_count FROM layer_usage WHERE pk_layer=?", - Integer.class, layer.getId())); - - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT int_clock_time_fail FROM layer_usage WHERE pk_layer=?", - Integer.class, layer.getId())); - - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT int_core_time_fail FROM layer_usage WHERE pk_layer=?", - Integer.class, layer.getId())); - - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT int_frame_fail_count FROM layer_usage WHERE pk_layer=?", - Integer.class, layer.getId())); - - /** 60 seconds of 100 core units **/ - ResourceUsage usage = new ResourceUsage(60, 33); - - assertTrue(usage.getClockTimeSeconds() > 0); - assertTrue(usage.getCoreTimeSeconds() > 0); - - /** - * Successful frame - */ - layerDao.updateUsage(layer, usage, 0); - assertEquals(Long.valueOf(usage.getClockTimeSeconds()), jdbcTemplate.queryForObject( - "SELECT int_clock_time_success FROM layer_usage WHERE pk_layer=?", - Long.class, layer.getId())); - - assertEquals(Long.valueOf(usage.getCoreTimeSeconds()), jdbcTemplate.queryForObject( - "SELECT int_core_time_success FROM layer_usage WHERE pk_layer=?", - Long.class, layer.getId())); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT int_frame_success_count FROM layer_usage WHERE pk_layer=?", - Integer.class, layer.getId())); - - /** - * Failed frame - */ - layerDao.updateUsage(layer, usage, 1); - assertEquals(Long.valueOf(usage.getClockTimeSeconds()), jdbcTemplate.queryForObject( - "SELECT int_clock_time_fail FROM layer_usage WHERE pk_layer=?", - Long.class, layer.getId())); - - assertEquals(Long.valueOf(usage.getCoreTimeSeconds()), jdbcTemplate.queryForObject( - "SELECT int_core_time_fail FROM layer_usage WHERE pk_layer=?", - Long.class, layer.getId())); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT int_frame_fail_count FROM layer_usage WHERE pk_layer=?", - Integer.class, layer.getId())); - } - - @Test - @Transactional - @Rollback(true) - public void isLayerThreadable() { - LayerDetail layer = getLayer(); - jdbcTemplate.update( - "UPDATE layer set b_threadable = 0 WHERE pk_layer=?", - layer.getId()); - - assertFalse(layerDao.isThreadable(layer)); - - jdbcTemplate.update( - "UPDATE layer set b_threadable = 1 WHERE pk_layer=?", - layer.getId()); - - assertTrue(layerDao.isThreadable(layer)); - } - - @Test - @Transactional - @Rollback(true) - public void enableMemoryOptimizer() { - LayerDetail layer = getLayer(); - layerDao.enableMemoryOptimizer(layer, false); - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT b_optimize FROM layer WHERE pk_layer=?", - Integer.class, layer.getLayerId())); - - layerDao.enableMemoryOptimizer(layer, true); - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT b_optimize FROM layer WHERE pk_layer=?", - Integer.class, layer.getLayerId())); - } - - @Test - @Transactional - @Rollback(true) - public void testBalanceMemory() { - LayerDetail layer = getLayer(); - assertTrue(layerDao.balanceLayerMinMemory(layer, CueUtil.GB)); - jdbcTemplate.update("UPDATE layer_mem SET int_max_rss=? WHERE pk_layer=?", - CueUtil.GB8, layer.getId()); - assertFalse(layerDao.balanceLayerMinMemory(layer, CueUtil.MB512)); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertLayerOutput() { - LayerDetail layer = getLayer(); - layerDao.insertLayerOutput(layer, "filespec1"); - layerDao.insertLayerOutput(layer, "filespec2"); - layerDao.insertLayerOutput(layer, "filespec3"); - assertEquals(3, layerDao.getLayerOutputs(layer).size()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetLimits() { - LayerDetail layer = getLayer(); - List limits = layerDao.getLimits(layer); - assertEquals(limits.size(), 1); - assertEquals(limits.get(0).id, getTestLimitId(LIMIT_NAME)); - } - - @Test - @Transactional - @Rollback(true) - public void testGetLimitNames() { - LayerDetail layer = getLayer(); - List limits = layerDao.getLimitNames(layer); - assertEquals(limits.size(), 1); - assertEquals(limits.get(0), LIMIT_NAME); - } - - @Test - @Transactional - @Rollback(true) - public void testAddLimit() { - LayerDetail layer = getLayer(); - layerDao.addLimit(layer, getTestLimitId(LIMIT_TEST_A)); - layerDao.addLimit(layer, getTestLimitId(LIMIT_TEST_B)); - layerDao.addLimit(layer, getTestLimitId(LIMIT_TEST_C)); - LayerInterface layerResult = layerDao.getLayer(layer.getLayerId()); - List limits = layerDao.getLimits(layerResult); - assertEquals(limits.size(), 4); - assertEquals(limits.get(0).id, getTestLimitId(LIMIT_NAME)); - assertEquals(limits.get(1).id, getTestLimitId(LIMIT_TEST_A)); - assertEquals(limits.get(2).id, getTestLimitId(LIMIT_TEST_B)); - assertEquals(limits.get(3).id, getTestLimitId(LIMIT_TEST_C)); - } - - @Test - @Transactional - @Rollback(true) - public void testDropLimit() { - LayerDetail layer = getLayer(); - layerDao.addLimit(layer, getTestLimitId(LIMIT_TEST_A)); - layerDao.dropLimit(layer, getTestLimitId(LIMIT_NAME)); - LayerInterface layerResult = layerDao.getLayer(layer.getLayerId()); - List limits = layerDao.getLimits(layerResult); - assertEquals(limits.size(), 1); - assertEquals(limits.get(0).id, getTestLimitId(LIMIT_TEST_A)); - layerDao.dropLimit(layer, getTestLimitId(LIMIT_TEST_A)); - LayerInterface layerResultB = layerDao.getLayer(layer.getLayerId()); - List limitsB = layerDao.getLimits(layerResultB); - assertEquals(limitsB.size(), 0); - } -} - - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/LimitDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/LimitDaoTests.java deleted file mode 100644 index ab3ce2a9a..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/LimitDaoTests.java +++ /dev/null @@ -1,137 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import javax.annotation.Resource; - -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.LimitEntity; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.LimitDao; -import com.imageworks.spcue.test.AssumingPostgresEngine; - -import static org.junit.Assert.assertEquals; - - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class LimitDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingPostgresEngine assumingPostgresEngine; - - @Resource - LimitDao limitDao; - - private static String LIMIT_NAME = "test-limit"; - private static int LIMIT_MAX_VALUE = 32; - - @Test - @Transactional - @Rollback(true) - public void testCreateLimit() { - String limitId = limitDao.createLimit(LIMIT_NAME, LIMIT_MAX_VALUE); - LimitEntity limit = limitDao.getLimit(limitId); - assertEquals(limit.id, limitId); - assertEquals(limit.name, LIMIT_NAME); - assertEquals(limit.maxValue, LIMIT_MAX_VALUE); - } - - @Test - @Transactional - @Rollback(true) - public void testDeleteLimit() { - String limitId = limitDao.createLimit(LIMIT_NAME, LIMIT_MAX_VALUE); - LimitEntity limit = limitDao.getLimit(limitId); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM limit_record WHERE pk_limit_record=?", - Integer.class, limitId)); - - limitDao.deleteLimit(limit); - - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM limit_record WHERE pk_limit_record=?", - Integer.class, limitId)); - } - - @Test - @Transactional - @Rollback(true) - public void testFindLimit() { - String limitId = limitDao.createLimit(LIMIT_NAME, LIMIT_MAX_VALUE); - - LimitEntity limit = limitDao.findLimit(LIMIT_NAME); - assertEquals(limit.name, LIMIT_NAME); - assertEquals(limit.maxValue, LIMIT_MAX_VALUE); - } - - @Test - @Transactional - @Rollback(true) - public void testGetLimit() { - String limitId = limitDao.createLimit(LIMIT_NAME, LIMIT_MAX_VALUE); - - LimitEntity limit = limitDao.getLimit(limitId); - assertEquals(limit.name, LIMIT_NAME); - assertEquals(limit.maxValue, LIMIT_MAX_VALUE); - } - - @Test - @Transactional - @Rollback(true) - public void testSetLimitName() { - String limitId = limitDao.createLimit(LIMIT_NAME, LIMIT_MAX_VALUE); - LimitEntity limit = limitDao.getLimit(limitId); - String newName = "heyIChanged"; - - limitDao.setLimitName(limit, newName); - - limit = limitDao.getLimit(limitId); - assertEquals(limit.id, limitId); - assertEquals(limit.name, newName); - assertEquals(limit.maxValue, LIMIT_MAX_VALUE); - } - - @Test - @Transactional - @Rollback(true) - public void testSetMaxValue() { - String limitId = limitDao.createLimit(LIMIT_NAME, LIMIT_MAX_VALUE); - LimitEntity limit = limitDao.getLimit(limitId); - int newValue = 600; - - limitDao.setMaxValue(limit, newValue); - - limit = limitDao.getLimit(limitId); - assertEquals(limit.id, limitId); - assertEquals(limit.name, LIMIT_NAME); - assertEquals(limit.maxValue, newValue); - } -} diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/MaintenanceDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/MaintenanceDaoTests.java deleted file mode 100644 index 4f52e10f0..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/MaintenanceDaoTests.java +++ /dev/null @@ -1,76 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import javax.annotation.Resource; - -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.MaintenanceTask; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.MaintenanceDao; -import com.imageworks.spcue.test.AssumingOracleEngine; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class MaintenanceDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - MaintenanceDao maintenanceDao; - - @Test - @Transactional - @Rollback(true) - public void testSetUpHostsToDown() { - maintenanceDao.setUpHostsToDown(); - } - - @Test - @Transactional - @Rollback(true) - public void testLockHistoricalTask() { - assertTrue(maintenanceDao.lockTask(MaintenanceTask.LOCK_HISTORICAL_TRANSFER)); - assertFalse(maintenanceDao.lockTask(MaintenanceTask.LOCK_HISTORICAL_TRANSFER)); - } - - @Test - @Transactional - @Rollback(true) - public void testUnlockHistoricalTask() { - assertTrue(maintenanceDao.lockTask(MaintenanceTask.LOCK_HISTORICAL_TRANSFER)); - maintenanceDao.unlockTask(MaintenanceTask.LOCK_HISTORICAL_TRANSFER); - } -} - - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/MatcherDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/MatcherDaoTests.java deleted file mode 100644 index e65ac8f2c..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/MatcherDaoTests.java +++ /dev/null @@ -1,140 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import javax.annotation.Resource; - -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.FilterEntity; -import com.imageworks.spcue.MatcherEntity; -import com.imageworks.spcue.ShowInterface; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.FilterDao; -import com.imageworks.spcue.dao.GroupDao; -import com.imageworks.spcue.dao.MatcherDao; -import com.imageworks.spcue.dao.ShowDao; -import com.imageworks.spcue.grpc.filter.FilterType; -import com.imageworks.spcue.grpc.filter.MatchSubject; -import com.imageworks.spcue.grpc.filter.MatchType; -import com.imageworks.spcue.test.AssumingOracleEngine; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class MatcherDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - MatcherDao matcherDao; - - @Resource - FilterDao filterDao; - - @Resource - ShowDao showDao; - - @Resource - GroupDao groupDao; - - private static String FILTER_NAME = "test_filter"; - - public ShowInterface getShow() { - return showDao.getShowDetail("00000000-0000-0000-0000-000000000000"); - } - - public MatcherEntity createMatcher() { - FilterEntity filter = createFilter(); - MatcherEntity matcher = new MatcherEntity(); - matcher.filterId = filter.id; - matcher.name = null; - matcher.showId = getShow().getId(); - matcher.subject = MatchSubject.JOB_NAME; - matcher.type = MatchType.CONTAINS; - matcher.value = "testuser"; - matcherDao.insertMatcher(matcher); - return matcher; - } - - public FilterEntity createFilter() { - FilterEntity filter = new FilterEntity(); - filter.name = FILTER_NAME; - filter.showId = "00000000-0000-0000-0000-000000000000"; - filter.type = FilterType.MATCH_ANY; - filter.enabled = true; - filterDao.insertFilter(filter); - return filter; - } - - @Test - @Transactional - @Rollback(true) - public void testInsertMatcher() { - createMatcher(); - } - - @Test - @Transactional - @Rollback(true) - public void testDeleteMatcher() { - MatcherEntity matcher = createMatcher(); - matcherDao.deleteMatcher(matcher); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateMatcher() { - MatcherEntity matcher = createMatcher(); - matcher.subject = MatchSubject.USER; - matcher.value = "testuser"; - matcher.type = MatchType.IS; - matcherDao.updateMatcher(matcher); - } - - @Test - @Transactional - @Rollback(true) - public void testGetMatcher() { - MatcherEntity matcher = createMatcher(); - matcherDao.getMatcher(matcher); - matcherDao.getMatcher(matcher.getMatcherId()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetMatchers() { - MatcherEntity matcher = createMatcher(); - matcherDao.getMatchers(matcher); - } - -} - - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/NestedWhiteboardDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/NestedWhiteboardDaoTests.java deleted file mode 100644 index f5686e602..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/NestedWhiteboardDaoTests.java +++ /dev/null @@ -1,68 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import javax.annotation.Resource; - -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.ShowEntity; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.NestedWhiteboardDao; -import com.imageworks.spcue.dao.ShowDao; -import com.imageworks.spcue.test.AssumingOracleEngine; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class NestedWhiteboardDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - NestedWhiteboardDao nestedWhiteboardDao; - - @Resource - ShowDao showDao; - - public ShowEntity getShow() { - return showDao.findShowDetail("pipe"); - } - - @Test - @Transactional - @Rollback(true) - public void testGetNestedJobWhiteboard() { - nestedWhiteboardDao.getJobWhiteboard(getShow()); - nestedWhiteboardDao.getJobWhiteboard(getShow()); - nestedWhiteboardDao.getJobWhiteboard(getShow()); - nestedWhiteboardDao.getJobWhiteboard(getShow()); - nestedWhiteboardDao.getJobWhiteboard(getShow()); - } -} - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/OwnerDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/OwnerDaoTests.java deleted file mode 100644 index 7d1951a29..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/OwnerDaoTests.java +++ /dev/null @@ -1,131 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import javax.annotation.Resource; - -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.OwnerEntity; -import com.imageworks.spcue.ShowInterface; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.OwnerDao; -import com.imageworks.spcue.service.AdminManager; -import com.imageworks.spcue.service.HostManager; -import com.imageworks.spcue.test.AssumingOracleEngine; - -import static org.junit.Assert.assertEquals; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class OwnerDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - OwnerDao ownerDao; - - @Resource - AdminManager adminManager; - - @Resource - HostManager hostManager; - - @Test - @Transactional - @Rollback(true) - public void testInsertOwner() { - ShowInterface show = adminManager.findShowEntity("pipe"); - OwnerEntity o = new OwnerEntity(); - o.name = "spongebob"; - ownerDao.insertOwner(o, show); - } - - @Test - @Transactional - @Rollback(true) - public void testIsOwner() { - ShowInterface show = adminManager.findShowEntity("pipe"); - OwnerEntity o = new OwnerEntity(); - o.name = "spongebob"; - ownerDao.insertOwner(o, show); - } - - @Test - @Transactional - @Rollback(true) - public void testGetOwner() { - ShowInterface show = adminManager.findShowEntity("pipe"); - OwnerEntity o = new OwnerEntity(); - o.name = "spongebob"; - ownerDao.insertOwner(o, show); - - assertEquals(o, ownerDao.findOwner("spongebob")); - assertEquals(o, ownerDao.getOwner(o.id)); - } - - @Test - @Transactional - @Rollback(true) - public void testDeleteOwner() { - ShowInterface show = adminManager.findShowEntity("pipe"); - OwnerEntity o = new OwnerEntity(); - o.name = "spongebob"; - ownerDao.insertOwner(o, show); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM owner WHERE pk_owner=?", - Integer.class, o.id)); - - ownerDao.deleteOwner(o); - - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM owner WHERE pk_owner=?", - Integer.class, o.id)); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateShow() { - ShowInterface show = adminManager.findShowEntity("pipe"); - OwnerEntity o = new OwnerEntity(); - o.name = "spongebob"; - ownerDao.insertOwner(o, show); - - ShowInterface newShow = adminManager.findShowEntity("edu"); - - ownerDao.updateShow(o, newShow); - - assertEquals(newShow.getShowId(), jdbcTemplate.queryForObject( - "SELECT pk_show FROM owner WHERE pk_owner=?", - String.class, o.id)); - } -} - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/PointDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/PointDaoTests.java deleted file mode 100644 index abaa34b23..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/PointDaoTests.java +++ /dev/null @@ -1,171 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import java.io.File; -import javax.annotation.Resource; - -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.DepartmentInterface; -import com.imageworks.spcue.JobDetail; -import com.imageworks.spcue.PointInterface; -import com.imageworks.spcue.ShowEntity; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.DepartmentDao; -import com.imageworks.spcue.dao.PointDao; -import com.imageworks.spcue.service.AdminManager; -import com.imageworks.spcue.service.JobLauncher; -import com.imageworks.spcue.service.JobManager; -import com.imageworks.spcue.test.AssumingOracleEngine; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class PointDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - DepartmentDao departmentDao; - - @Resource - AdminManager adminManager; - - @Resource - JobManager jobManager; - - @Resource - JobLauncher jobLauncher; - - @Resource - PointDao pointDao; - - public JobDetail launchJob() { - jobLauncher.testMode = true; - jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec.xml")); - return jobManager.findJobDetail("pipe-dev.cue-testuser_shell_v1"); - } - - @Test - @Transactional - @Rollback(true) - public void insertDepartmentConfig() { - ShowEntity show = new ShowEntity(); - show.name = "testtest"; - adminManager.createShow(show); - DepartmentInterface dept = departmentDao.findDepartment("Lighting"); - PointInterface d = pointDao.insertPointConf(show, dept); - - assertEquals(show.id, jdbcTemplate.queryForObject( - "SELECT pk_show FROM point WHERE pk_point=?", - String.class, d.getPointId())); - - assertEquals(dept.getDepartmentId(), jdbcTemplate.queryForObject( - "SELECT pk_dept FROM point WHERE pk_point=?", - String.class, d.getPointId())); - } - - @Test - @Transactional - @Rollback(true) - public void departmentConfigExists() { - ShowEntity show = new ShowEntity(); - show.name = "testtest"; - adminManager.createShow(show); - - assertTrue(pointDao.pointConfExists(show, - departmentDao.getDefaultDepartment())); - - assertFalse(pointDao.pointConfExists(show, - departmentDao.findDepartment("Lighting"))); - } - - @Test - @Transactional - @Rollback(true) - public void updateEnableTiManaged() { - ShowEntity show = new ShowEntity(); - show.name = "testtest"; - adminManager.createShow(show); - - PointInterface config = pointDao.getPointConfigDetail(show, - departmentDao.getDefaultDepartment()); - - //pointDao.updateEnableManaged(config, "Lighting", 10); - } - - @Test - @Transactional - @Rollback(true) - public void getDepartmentConfig() { - ShowEntity show = new ShowEntity(); - show.name = "testtest"; - adminManager.createShow(show); - - /* Tests both overlodaed methods */ - PointInterface configA = pointDao.getPointConfigDetail(show, - departmentDao.getDefaultDepartment()); - - PointInterface configB = pointDao.getPointConfDetail( - configA.getPointId()); - - assertEquals(configA.getPointId(), configB.getPointId()); - assertEquals(configA.getDepartmentId(), configB.getDepartmentId()); - assertEquals(configA.getShowId(), configB.getShowId()); - } - - - @Test - @Transactional - @Rollback(true) - public void testIsOverMinCores() { - - JobDetail job = launchJob(); - - PointInterface pointConfig = pointDao.getPointConfigDetail(job, - departmentDao.getDepartment(job.getDepartmentId())); - - assertFalse(pointDao.isOverMinCores(job)); - - // Now update some values so it returns true. - jdbcTemplate.update("UPDATE point SET int_cores = int_min_cores + 2000000 WHERE pk_point=?", - pointConfig.getId()); - - logger.info(jdbcTemplate.queryForObject("SELECT int_min_cores from point where pk_point=?", - Integer.class, pointConfig.getId())); - - assertTrue(pointDao.isOverMinCores(job)); - } - -} - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/ProcDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/ProcDaoTests.java deleted file mode 100644 index a069234f1..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/ProcDaoTests.java +++ /dev/null @@ -1,846 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; -import javax.annotation.Resource; - -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.DispatchFrame; -import com.imageworks.spcue.DispatchHost; -import com.imageworks.spcue.FrameDetail; -import com.imageworks.spcue.JobDetail; -import com.imageworks.spcue.LayerInterface; -import com.imageworks.spcue.VirtualProc; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.DispatcherDao; -import com.imageworks.spcue.dao.FrameDao; -import com.imageworks.spcue.dao.HostDao; -import com.imageworks.spcue.dao.LayerDao; -import com.imageworks.spcue.dao.ProcDao; -import com.imageworks.spcue.dao.criteria.Direction; -import com.imageworks.spcue.dao.criteria.FrameSearchFactory; -import com.imageworks.spcue.dao.criteria.ProcSearchFactory; -import com.imageworks.spcue.dao.criteria.ProcSearchInterface; -import com.imageworks.spcue.dao.criteria.Sort; -import com.imageworks.spcue.dispatcher.Dispatcher; -import com.imageworks.spcue.dispatcher.ResourceReservationFailureException; -import com.imageworks.spcue.grpc.host.HardwareState; -import com.imageworks.spcue.grpc.host.ProcSearchCriteria; -import com.imageworks.spcue.grpc.report.RenderHost; -import com.imageworks.spcue.service.AdminManager; -import com.imageworks.spcue.service.HostManager; -import com.imageworks.spcue.service.JobLauncher; -import com.imageworks.spcue.service.JobManager; -import com.imageworks.spcue.test.AssumingOracleEngine; -import com.imageworks.spcue.util.CueUtil; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class ProcDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - ProcDao procDao; - - @Resource - HostDao hostDao; - - @Resource - JobManager jobManager; - - @Resource - JobLauncher jobLauncher; - - @Resource - FrameDao frameDao; - - @Resource - LayerDao layerDao; - - @Resource - DispatcherDao dispatcherDao; - - @Resource - HostManager hostManager; - - @Resource - AdminManager adminManager; - - @Resource - Dispatcher dispatcher; - - @Resource - FrameSearchFactory frameSearchFactory; - - @Resource - ProcSearchFactory procSearchFactory; - - private static String PK_ALLOC = "00000000-0000-0000-0000-000000000000"; - - public DispatchHost createHost() { - - RenderHost host = RenderHost.newBuilder() - .setName("beta") - .setBootTime(1192369572) - .setFreeMcp(76020) - .setFreeMem(53500) - .setFreeSwap(20760) - .setLoad(1) - .setTotalMcp(195430) - .setTotalMem((int) CueUtil.GB32) - .setTotalSwap(20960) - .setNimbyEnabled(false) - .setNumProcs(8) - .setCoresPerProc(100) - .setState(HardwareState.UP) - .setFacility("spi") - .build(); - - DispatchHost dh = hostManager.createHost(host); - hostManager.setAllocation(dh, - adminManager.findAllocationDetail("spi", "general")); - - return hostDao.findDispatchHost("beta"); - } - - public JobDetail launchJob() { - jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec.xml")); - return jobManager.findJobDetail("pipe-dev.cue-testuser_shell_v1"); - } - - @Before - public void setDispatcherTestMode() { - dispatcher.setTestMode(true); - jobLauncher.testMode = true; - } - - @Test - @Transactional - @Rollback(true) - public void testDontVerifyRunningProc() { - DispatchHost host = createHost(); - JobDetail job = launchJob(); - FrameDetail fd = frameDao.findFrameDetail(job, "0001-pass_1_preprocess"); - DispatchFrame frame = frameDao.getDispatchFrame(fd.getId()); - VirtualProc proc = VirtualProc.build(host, frame); - dispatcher.dispatch(frame, proc); - - // Confirm was have a running frame. - assertEquals("RUNNING", jdbcTemplate.queryForObject( - "SELECT str_state FROM frame WHERE pk_frame=?", - String.class, frame.id)); - - assertTrue(procDao.verifyRunningProc(proc.getId(), frame.getId())); - jobManager.shutdownJob(job); - - int result = jdbcTemplate.update( - "UPDATE job SET ts_stopped = " + - "systimestamp - interval '10' minute " + - "WHERE pk_job=?", job.id); - - assertEquals(1, result); - assertFalse(procDao.verifyRunningProc(proc.getId(), frame.getId())); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertVirtualProc() { - - DispatchHost host = createHost(); - JobDetail job = launchJob(); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1"); - - VirtualProc proc = new VirtualProc(); - proc.allocationId = PK_ALLOC; - proc.coresReserved = 100; - proc.hostId = host.id; - proc.hostName = host.name; - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - - procDao.insertVirtualProc(proc); - procDao.verifyRunningProc(proc.getId(), frame.getId()); - } - - @Test - @Transactional - @Rollback(true) - public void testDeleteVirtualProc() { - - DispatchHost host = createHost(); - JobDetail job = launchJob(); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1"); - - VirtualProc proc = new VirtualProc(); - proc.allocationId = PK_ALLOC; - proc.coresReserved = 100; - proc.hostId = host.id; - proc.hostName = host.name; - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - - procDao.insertVirtualProc(proc); - procDao.verifyRunningProc(proc.getId(), frame.getId()); - procDao.deleteVirtualProc(proc); - } - - @Test - @Transactional - @Rollback(true) - public void testClearVirtualProcAssignment() { - - DispatchHost host = createHost(); - JobDetail job = launchJob(); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1"); - - VirtualProc proc = new VirtualProc(); - proc.allocationId = PK_ALLOC; - proc.coresReserved = 100; - proc.hostId = host.id; - proc.hostName = host.name; - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - - procDao.insertVirtualProc(proc); - procDao.verifyRunningProc(proc.getId(), frame.getId()); - procDao.clearVirtualProcAssignment(proc); - } - - @Test - @Transactional - @Rollback(true) - public void testClearVirtualProcAssignmentByFrame() { - - DispatchHost host = createHost(); - JobDetail job = launchJob(); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1"); - - VirtualProc proc = new VirtualProc(); - proc.allocationId = PK_ALLOC; - proc.coresReserved = 100; - proc.hostId = host.id; - proc.hostName = host.name; - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - - procDao.insertVirtualProc(proc); - procDao.verifyRunningProc(proc.getId(), frame.getId()); - assertTrue(procDao.clearVirtualProcAssignment(frame)); - } - - - @Test - @Transactional - @Rollback(true) - public void testUpdateVirtualProcAssignment() { - - DispatchHost host = createHost(); - - JobDetail job = launchJob(); - FrameDetail frame1 = frameDao.findFrameDetail(job, "0001-pass_1"); - FrameDetail frame2 = frameDao.findFrameDetail(job, "0002-pass_1"); - - VirtualProc proc = new VirtualProc(); - proc.allocationId = PK_ALLOC; - proc.coresReserved = 100; - proc.hostId = host.id; - proc.hostName = host.name; - proc.jobId = job.id; - proc.frameId = frame1.id; - proc.layerId = frame1.layerId; - proc.showId = frame1.showId; - - procDao.insertVirtualProc(proc); - procDao.verifyRunningProc(proc.getId(), frame1.getId()); - - proc.frameId = frame2.id; - - procDao.updateVirtualProcAssignment(proc); - procDao.verifyRunningProc(proc.getId(), frame2.getId()); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateProcMemoryUsage() { - - DispatchHost host = createHost(); - JobDetail job = launchJob(); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1"); - - VirtualProc proc = new VirtualProc(); - proc.allocationId = PK_ALLOC; - proc.coresReserved = 100; - proc.hostId = host.id; - proc.hostName = host.name; - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - - procDao.insertVirtualProc(proc); - procDao.verifyRunningProc(proc.getId(), frame.getId()); - - procDao.updateProcMemoryUsage(frame, 100, 100, 1000, 1000); - - } - - @Test - @Transactional - @Rollback(true) - public void testGetVirtualProc() { - DispatchHost host = createHost(); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM host WHERE pk_host=?", - Integer.class, host.id)); - - JobDetail job = launchJob(); - FrameDetail fd = frameDao.findFrameDetail(job, "0001-pass_1_preprocess"); - - DispatchFrame frame = frameDao.getDispatchFrame(fd.getId()); - VirtualProc proc = VirtualProc.build(host, frame); - dispatcher.dispatch(frame, proc); - - assertTrue(procDao.verifyRunningProc(proc.getId(), frame.getId())); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM proc WHERE pk_proc=?", - Integer.class, proc.id)); - - - VirtualProc verifyProc = procDao.getVirtualProc(proc.getId()); - assertEquals(host.allocationId, verifyProc.allocationId); - assertEquals(proc.coresReserved, verifyProc.coresReserved); - assertEquals(proc.frameId, verifyProc.frameId); - assertEquals(proc.hostId, verifyProc.hostId); - assertEquals(proc.id, verifyProc.id); - assertEquals(proc.jobId, verifyProc.jobId); - assertEquals(proc.layerId, verifyProc.layerId); - assertEquals(proc.showId, verifyProc.showId); - } - - @Test - @Transactional - @Rollback(true) - public void testFindVirtualProc() { - - DispatchHost host = createHost(); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM host WHERE pk_host=?", - Integer.class, host.id)); - - JobDetail job = launchJob(); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1"); - - VirtualProc proc = new VirtualProc(); - proc.allocationId = PK_ALLOC; - proc.coresReserved = 100; - proc.hostId = host.id; - proc.hostName = host.name; - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - procDao.insertVirtualProc(proc); - - procDao.findVirtualProc(frame); - } - - @Test - @Transactional - @Rollback(true) - public void testFindVirtualProcs() { - - DispatchHost host = createHost(); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM host WHERE pk_host=?", - Integer.class, host.id)); - - JobDetail job = launchJob(); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1"); - - VirtualProc proc = new VirtualProc(); - proc.allocationId = PK_ALLOC; - proc.coresReserved = 100; - proc.hostId = host.id; - proc.hostName = host.name; - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - procDao.insertVirtualProc(proc); - - assertEquals(1, procDao.findVirtualProcs(HardwareState.UP).size()); - assertEquals(1, procDao.findVirtualProcs(host).size()); - assertEquals(1, procDao.findVirtualProcs(job).size()); - assertEquals(1, procDao.findVirtualProcs(frame).size()); - assertEquals(1, procDao.findVirtualProcs(frameSearchFactory.create(job)).size()); - assertEquals(1, procDao.findVirtualProcs(frameSearchFactory.create((LayerInterface) frame)).size()); - } - - @Test - @Transactional - @Rollback(true) - public void testFindOrphanedVirtualProcs() { - DispatchHost host = createHost(); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM host WHERE pk_host=?", - Integer.class, host.id)); - - JobDetail job = launchJob(); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1"); - - VirtualProc proc = new VirtualProc(); - proc.allocationId = PK_ALLOC; - proc.coresReserved = 100; - proc.hostId = host.id; - proc.hostName = host.name; - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - procDao.insertVirtualProc(proc); - - assertEquals(0, procDao.findOrphanedVirtualProcs().size()); - - /** - * This is destructive to running jobs - */ - jdbcTemplate.update( - "UPDATE proc SET ts_ping = (systimestamp - interval '30' day)"); - - assertEquals(1, procDao.findOrphanedVirtualProcs().size()); - assertTrue(procDao.isOrphan(proc)); - } - - @Test - @Transactional - @Rollback(true) - public void testUnbookProc() { - - DispatchHost host = createHost(); - - JobDetail job = launchJob(); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1"); - - VirtualProc proc = new VirtualProc(); - proc.allocationId = PK_ALLOC; - proc.coresReserved = 100; - proc.hostId = host.id; - proc.hostName = host.name; - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - procDao.insertVirtualProc(proc); - - procDao.unbookProc(proc); - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT b_unbooked FROM proc WHERE pk_proc=?", - Integer.class, proc.id)); - } - - @Test - @Transactional - @Rollback(true) - public void testUnbookVirtualProcs() { - - DispatchHost host = createHost(); - - JobDetail job = launchJob(); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1"); - - VirtualProc proc = new VirtualProc(); - proc.allocationId = PK_ALLOC; - proc.coresReserved = 100; - proc.hostId = host.id; - proc.hostName = host.name; - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - procDao.insertVirtualProc(proc); - - - List procs = new ArrayList(); - procs.add(proc); - - procDao.unbookVirtualProcs(procs); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT b_unbooked FROM proc WHERE pk_proc=?", - Integer.class, proc.id)); - } - - - @Test(expected=ResourceReservationFailureException.class) - @Transactional - @Rollback(true) - public void testIncreaseReservedMemoryFail() { - - DispatchHost host = createHost(); - JobDetail job = launchJob(); - - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1"); - - VirtualProc proc = new VirtualProc(); - proc.allocationId = PK_ALLOC; - proc.coresReserved = 100; - proc.hostId = host.id; - proc.hostName = host.name; - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - procDao.insertVirtualProc(proc); - - procDao.increaseReservedMemory(proc, 8173264l * 8); - } - - @Test - @Transactional - @Rollback(true) - public void testIncreaseReservedMemory() { - - DispatchHost host = createHost(); - JobDetail job = launchJob(); - - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1"); - - VirtualProc proc = new VirtualProc(); - proc.allocationId = PK_ALLOC; - proc.coresReserved = 100; - proc.hostId = host.id; - proc.hostName = host.name; - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - procDao.insertVirtualProc(proc); - - procDao.increaseReservedMemory(proc, 3145728); - } - - @Test - @Transactional - @Rollback(true) - public void testFindReservedMemoryOffender() { - DispatchHost host = createHost(); - - - jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec_dispatch_test.xml")); - JobDetail job = jobManager.findJobDetail("pipe-dev.cue-testuser_shell_dispatch_test_v1"); - jobManager.setJobPaused(job, false); - - int i = 1; - List frames = dispatcherDao.findNextDispatchFrames(job, host, 6); - assertEquals(6, frames.size()); - - for (DispatchFrame frame: frames) { - - VirtualProc proc = VirtualProc.build(host, frame); - frame.minMemory = Dispatcher.MEM_RESERVED_DEFAULT; - dispatcher.dispatch(frame, proc); - - // Increase the memory usage as frames are added - procDao.updateProcMemoryUsage(frame, - 1000*i, 1000*i, 1000*i, 1000*i); - i++; - } - - // Now compare the last frame which has the highest memory - // usage to the what is returned by getWorstMemoryOffender - VirtualProc offender = procDao.getWorstMemoryOffender(host); - - FrameDetail f = frameDao.getFrameDetail(frames.get(5)); - FrameDetail o = frameDao.getFrameDetail(offender); - - assertEquals(f.getName(), o.getName()); - assertEquals(f.id, o.getFrameId()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetReservedMemory() { - DispatchHost host = createHost(); - JobDetail job = launchJob(); - - FrameDetail frameDetail = frameDao.findFrameDetail(job, "0001-pass_1"); - DispatchFrame frame = frameDao.getDispatchFrame(frameDetail.id); - - VirtualProc proc = VirtualProc.build(host, frame); - proc.frameId = frame.id; - procDao.insertVirtualProc(proc); - - VirtualProc _proc = procDao.findVirtualProc(frame); - assertEquals(Long.valueOf(Dispatcher.MEM_RESERVED_DEFAULT), jdbcTemplate.queryForObject( - "SELECT int_mem_reserved FROM proc WHERE pk_proc=?", - Long.class, _proc.id)); - assertEquals(Dispatcher.MEM_RESERVED_DEFAULT, - procDao.getReservedMemory(_proc)); - } - - @Test - @Transactional - @Rollback(true) - public void testGetReservedGpu() { - DispatchHost host = createHost(); - JobDetail job = launchJob(); - - FrameDetail frameDetail = frameDao.findFrameDetail(job, "0001-pass_1"); - DispatchFrame frame = frameDao.getDispatchFrame(frameDetail.id); - - VirtualProc proc = VirtualProc.build(host, frame); - proc.frameId = frame.id; - procDao.insertVirtualProc(proc); - - VirtualProc _proc = procDao.findVirtualProc(frame); - assertEquals(Long.valueOf(Dispatcher.GPU_RESERVED_DEFAULT), jdbcTemplate.queryForObject( - "SELECT int_gpu_reserved FROM proc WHERE pk_proc=?", - Long.class, _proc.id)); - assertEquals(Dispatcher.GPU_RESERVED_DEFAULT, - procDao.getReservedGpu(_proc)); - } - - @Test - @Transactional - @Rollback(true) - public void testBalanceUnderUtilizedProcs() { - DispatchHost host = createHost(); - JobDetail job = launchJob(); - - FrameDetail frameDetail1 = frameDao.findFrameDetail(job, "0001-pass_1"); - DispatchFrame frame1 = frameDao.getDispatchFrame(frameDetail1.id); - - VirtualProc proc1 = VirtualProc.build(host, frame1); - proc1.frameId = frame1.id; - procDao.insertVirtualProc(proc1); - - procDao.updateProcMemoryUsage(frame1, 250000, 250000, 250000, 250000); - layerDao.updateLayerMaxRSS(frame1, 250000, true); - - FrameDetail frameDetail2 = frameDao.findFrameDetail(job, "0002-pass_1"); - DispatchFrame frame2 = frameDao.getDispatchFrame(frameDetail2.id); - - VirtualProc proc2 = VirtualProc.build(host, frame2); - proc2.frameId = frame2.id; - procDao.insertVirtualProc(proc2); - - procDao.updateProcMemoryUsage(frame2, 255000, 255000,255000, 255000); - layerDao.updateLayerMaxRSS(frame2, 255000, true); - - FrameDetail frameDetail3 = frameDao.findFrameDetail(job, "0003-pass_1"); - DispatchFrame frame3 = frameDao.getDispatchFrame(frameDetail3.id); - - VirtualProc proc3 = VirtualProc.build(host, frame3); - proc3.frameId = frame3.id; - procDao.insertVirtualProc(proc3); - - procDao.updateProcMemoryUsage(frame3, 3145728, 3145728,3145728, 3145728); - layerDao.updateLayerMaxRSS(frame3,300000, true); - - procDao.balanceUnderUtilizedProcs(proc3, 100000); - procDao.increaseReservedMemory(proc3, Dispatcher.MEM_RESERVED_DEFAULT + 100000); - - // Check the target proc - VirtualProc targetProc = procDao.getVirtualProc(proc3.getId()); - assertEquals( Dispatcher.MEM_RESERVED_DEFAULT+ 100000, targetProc.memoryReserved); - - // Check other procs - VirtualProc firstProc = procDao.getVirtualProc(proc1.getId()); - assertEquals( Dispatcher.MEM_RESERVED_DEFAULT - 50000 -1 , firstProc.memoryReserved); - - VirtualProc secondProc = procDao.getVirtualProc(proc2.getId()); - assertEquals(Dispatcher.MEM_RESERVED_DEFAULT - 50000 -1, secondProc.memoryReserved); - - } - - @Test - @Transactional - @Rollback(true) - public void testGetCurrentShowId() { - - DispatchHost host = createHost(); - JobDetail job = launchJob(); - - FrameDetail frameDetail = frameDao.findFrameDetail(job, "0001-pass_1_preprocess"); - DispatchFrame frame = frameDao.getDispatchFrame(frameDetail.id); - - VirtualProc proc = VirtualProc.build(host, frame); - proc.frameId = frame.id; - procDao.insertVirtualProc(proc); - - assertEquals(job.getShowId(), procDao.getCurrentShowId(proc)); - } - - @Test - @Transactional - @Rollback(true) - public void testGetCurrentJobId() { - DispatchHost host = createHost(); - JobDetail job = launchJob(); - - FrameDetail frameDetail = frameDao.findFrameDetail(job, "0001-pass_1_preprocess"); - DispatchFrame frame = frameDao.getDispatchFrame(frameDetail.id); - - VirtualProc proc = VirtualProc.build(host, frame); - proc.frameId = frame.id; - procDao.insertVirtualProc(proc); - - assertEquals(job.getJobId(), procDao.getCurrentJobId(proc)); - } - - @Test - @Transactional - @Rollback(true) - public void testGetCurrentLayerId() { - DispatchHost host = createHost(); - JobDetail job = launchJob(); - - FrameDetail frameDetail = frameDao.findFrameDetail(job, "0001-pass_1_preprocess"); - DispatchFrame frame = frameDao.getDispatchFrame(frameDetail.id); - - VirtualProc proc = VirtualProc.build(host, frame); - proc.frameId = frame.id; - procDao.insertVirtualProc(proc); - - assertEquals(frame.getLayerId(), procDao.getCurrentLayerId(proc)); - } - - @Test - @Transactional - @Rollback(true) - public void testGetCurrentFrameId() { - DispatchHost host = createHost(); - JobDetail job = launchJob(); - - FrameDetail frameDetail = frameDao.findFrameDetail(job, "0001-pass_1_preprocess"); - DispatchFrame frame = frameDao.getDispatchFrame(frameDetail.id); - - VirtualProc proc = VirtualProc.build(host, frame); - proc.frameId = frame.id; - procDao.insertVirtualProc(proc); - - assertEquals(frame.getFrameId(), procDao.getCurrentFrameId(proc)); - } - - @Test - @Transactional - @Rollback(true) - public void getProcsBySearch() { - DispatchHost host = createHost(); - - jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec_dispatch_test.xml")); - JobDetail job = jobManager.findJobDetail("pipe-dev.cue-testuser_shell_dispatch_test_v1"); - - /* - * Book 5 procs. - */ - for (int i=1; i<6; i++) { - FrameDetail f = frameDao.findFrameDetail(job, String.format("%04d-pass_1",i)); - VirtualProc proc = new VirtualProc(); - proc.allocationId = null; - proc.coresReserved = 100; - proc.hostId = host.id; - proc.hostName = host.name; - proc.jobId = job.id; - proc.frameId = f.id; - proc.layerId = f.layerId; - proc.showId = f.showId; - procDao.insertVirtualProc(proc); - } - - ProcSearchInterface r; - - /* - * Search for all 5 running procs - */ - r = procSearchFactory.create(); - r.addSort(new Sort("proc.ts_booked",Direction.ASC)); - ProcSearchCriteria criteriaA = r.getCriteria(); - r.setCriteria(criteriaA.toBuilder().addShows("pipe").build()); - assertEquals(5, procDao.findVirtualProcs(r).size()); - - /* - * Limit the result to 1 result. - */ - r = procSearchFactory.create(); - ProcSearchCriteria criteriaB = r.getCriteria(); - r.setCriteria(criteriaB.toBuilder().addShows("pipe").addMaxResults(1).build()); - assertEquals(1, procDao.findVirtualProcs(r).size()); - - /* - * Change the first result to 1, which should limt - * the result to 4. - */ - r = procSearchFactory.create(); - ProcSearchCriteria criteriaC = r.getCriteria(); - r.setCriteria(criteriaC.toBuilder().addShows("pipe").setFirstResult(2).build()); - r.addSort(new Sort("proc.ts_booked",Direction.ASC)); - assertEquals(4, procDao.findVirtualProcs(r).size()); - - /* - * Now try to do the eqivalent of a limit/offset - */ - r = procSearchFactory.create(); - ProcSearchCriteria criteriaD = r.getCriteria(); - r.setCriteria(criteriaD.toBuilder() - .addShows("pipe") - .setFirstResult(3) - .addMaxResults(2) - .build()); - assertEquals(2, procDao.findVirtualProcs(r).size()); - } -} - - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/ServiceDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/ServiceDaoTests.java deleted file mode 100644 index 5a90c256a..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/ServiceDaoTests.java +++ /dev/null @@ -1,210 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import javax.annotation.Resource; - -import com.google.common.collect.Sets; -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.ServiceEntity; -import com.imageworks.spcue.ServiceOverrideEntity; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.ServiceDao; -import com.imageworks.spcue.test.AssumingOracleEngine; -import com.imageworks.spcue.util.CueUtil; - -import static org.junit.Assert.assertEquals; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class ServiceDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - ServiceDao serviceDao; - - @Test - @Transactional - @Rollback(true) - public void testGetService() { - ServiceEntity s1 = serviceDao.get("default"); - ServiceEntity s2 = serviceDao.get("AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA0"); - assertEquals(s1, s2); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertService() { - ServiceEntity s = new ServiceEntity(); - s.name = "dillweed"; - s.minCores = 100; - s.minMemory = CueUtil.GB4; - s.minGpu = CueUtil.GB; - s.threadable = false; - s.tags.addAll(Sets.newHashSet(new String[] { "general"})); - - serviceDao.insert(s); - assertEquals(s, serviceDao.get("dillweed")); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateService() { - ServiceEntity s = new ServiceEntity(); - s.name = "dillweed"; - s.minCores = 100; - s.minMemory = CueUtil.GB4; - s.minGpu = CueUtil.GB; - s.threadable = false; - s.tags.addAll(Sets.newHashSet(new String[] { "general"})); - - serviceDao.insert(s); - assertEquals(s, serviceDao.get("dillweed")); - - s.name = "smacktest"; - s.minCores = 200; - s.minMemory = CueUtil.GB8; - s.minGpu = CueUtil.GB2; - s.threadable = true; - s.tags = Sets.newLinkedHashSet(); - s.tags.add("linux"); - - serviceDao.update(s); - ServiceEntity s1 = serviceDao.get(s.getId()); - - assertEquals(s.name, s1.name); - assertEquals(s.minCores, s1.minCores); - assertEquals(s.minMemory, s1.minMemory); - assertEquals(s.threadable, s1.threadable); - assertEquals(s.tags.toArray()[0], s1.tags.toArray()[0]); - } - - @Test - @Transactional - @Rollback(true) - public void testDeleteService() { - ServiceEntity s = new ServiceEntity(); - s.name = "dillweed"; - s.minCores = 100; - s.minMemory = CueUtil.GB4; - s.minGpu = CueUtil.GB; - s.threadable = false; - s.tags.addAll(Sets.newHashSet(new String[] { "general"})); - - serviceDao.insert(s); - assertEquals(s, serviceDao.get("dillweed")); - - serviceDao.delete(s); - - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT COUNT(1) FROM service WHERE pk_service=?", - Integer.class, s.getId())); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertServiceOverride() { - ServiceOverrideEntity s = new ServiceOverrideEntity(); - s.name = "dillweed"; - s.minCores = 100; - s.minMemory = CueUtil.GB4; - s.minGpu = CueUtil.GB; - s.threadable = false; - s.tags.addAll(Sets.newHashSet(new String[] { "general"})); - s.showId = "00000000-0000-0000-0000-000000000000"; - - serviceDao.insert(s); - assertEquals(s, serviceDao.getOverride("dillweed")); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateServiceOverride() { - ServiceOverrideEntity s = new ServiceOverrideEntity(); - s.name = "dillweed"; - s.minCores = 100; - s.minMemory = CueUtil.GB4; - s.minGpu = CueUtil.GB2; - s.threadable = false; - s.tags.addAll(Sets.newHashSet(new String[] { "general"})); - s.showId = "00000000-0000-0000-0000-000000000000"; - - serviceDao.insert(s); - assertEquals(s, serviceDao.getOverride("dillweed")); - assertEquals(s, serviceDao.getOverride("dillweed", s.showId)); - - s.name = "smacktest"; - s.minCores = 200; - s.minMemory = CueUtil.GB8; - s.minGpu = CueUtil.GB4; - s.threadable = true; - s.tags = Sets.newLinkedHashSet(); - s.tags.add("linux"); - - serviceDao.update(s); - ServiceEntity s1 = serviceDao.getOverride(s.getId()); - - assertEquals(s.name, s1.name); - assertEquals(s.minCores, s1.minCores); - assertEquals(s.minMemory, s1.minMemory); - assertEquals(s.minGpu, s1.minGpu); - assertEquals(s.threadable, s1.threadable); - assertEquals(s.tags.toArray()[0], s1.tags.toArray()[0]); - } - - @Test - @Transactional - @Rollback(true) - public void testDeleteServiceOverride() { - ServiceOverrideEntity s = new ServiceOverrideEntity(); - s.name = "dillweed"; - s.minCores = 100; - s.minMemory = CueUtil.GB4; - s.minGpu = CueUtil.GB; - s.threadable = false; - s.tags.addAll(Sets.newHashSet(new String[] { "general"})); - s.showId = "00000000-0000-0000-0000-000000000000"; - - serviceDao.insert(s); - assertEquals(s, serviceDao.getOverride("dillweed")); - assertEquals(s, serviceDao.getOverride("dillweed", s.showId)); - serviceDao.delete(s); - - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT COUNT(1) FROM show_service WHERE pk_show_service=?", - Integer.class, s.getId())); - } -} - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/ShowDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/ShowDaoTests.java deleted file mode 100644 index c05c8baab..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/ShowDaoTests.java +++ /dev/null @@ -1,241 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import javax.annotation.Resource; - -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.dao.EmptyResultDataAccessException; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.DispatchHost; -import com.imageworks.spcue.ShowEntity; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.ShowDao; -import com.imageworks.spcue.grpc.host.HardwareState; -import com.imageworks.spcue.grpc.report.RenderHost; -import com.imageworks.spcue.service.AdminManager; -import com.imageworks.spcue.service.HostManager; -import com.imageworks.spcue.test.AssumingOracleEngine; -import com.imageworks.spcue.util.CueUtil; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class ShowDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - ShowDao showDao; - - @Resource - HostManager hostManager; - - @Resource - AdminManager adminManager; - - private static String SHOW_ID = "00000000-0000-0000-0000-000000000000"; - private static String SHOW_NAME= "pipe"; - - public DispatchHost createHost() { - - RenderHost host = RenderHost.newBuilder() - .setName("test_host") - .setBootTime(1192369572) - .setFreeMcp(76020) - .setFreeMem(53500) - .setFreeSwap(20760) - .setLoad(1) - .setTotalMcp(195430) - .setTotalMem((int) CueUtil.GB16) - .setTotalSwap((int) CueUtil.GB16) - .setNimbyEnabled(false) - .setNumProcs(2) - .setCoresPerProc(100) - .addTags("general") - .setState(HardwareState.UP) - .setFacility("spi") - .putAttributes("freeGpu", String.format("%d", CueUtil.MB512)) - .putAttributes("totalGpu", String.format("%d", CueUtil.MB512)) - .build(); - - DispatchHost dh = hostManager.createHost(host); - hostManager.setAllocation(dh, - adminManager.findAllocationDetail("spi", "general")); - - return dh; - } - - @Test - @Transactional - @Rollback(true) - public void testFindShowDetail() { - ShowEntity show = showDao.findShowDetail(SHOW_NAME); - assertEquals(SHOW_ID, show.id); - assertEquals(SHOW_NAME,show.name); - assertFalse(show.paused); - } - - @Test(expected=EmptyResultDataAccessException.class) - @Transactional - @Rollback(true) - public void testFindShowDetailByHost() { - // TODO: Add code to setup a host and make the sow - // prefer the host, then check result again. - showDao.getShowDetail(createHost()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetShowDetail() { - ShowEntity show = showDao.getShowDetail(SHOW_ID); - assertEquals(SHOW_ID, show.id); - assertEquals(SHOW_NAME,show.name); - assertFalse(show.paused); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertShow() { - ShowEntity show = new ShowEntity(); - show.name = "uber"; - showDao.insertShow(show); - - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT count(*) FROM show where str_name=?", - Integer.class, show.name)); - - ShowEntity newShow = showDao.findShowDetail(show.name); - assertEquals(newShow.id, show.id); - assertEquals(newShow.name,show.name); - assertFalse(show.paused); - } - - @Test - @Transactional - @Rollback(true) - public void testShowExists() { - assertFalse(showDao.showExists("uber")); - assertTrue(showDao.showExists("pipe")); - assertTrue(showDao.showExists("fx")); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateShowDefaultMinCores() { - ShowEntity show = showDao.findShowDetail(SHOW_NAME); - showDao.updateShowDefaultMinCores(show, 100); - assertTrue(jdbcTemplate.queryForObject( - "SELECT int_default_min_cores FROM show WHERE pk_show=?", - Integer.class, show.id) == 100); - - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateShowDefaultMaxCores() { - ShowEntity show = showDao.findShowDetail(SHOW_NAME); - showDao.updateShowDefaultMaxCores(show, 1000); - assertTrue(jdbcTemplate.queryForObject( - "SELECT int_default_max_cores FROM show WHERE pk_show=?", - Integer.class, show.id) == 1000); - - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateShowCommentEmail() { - ShowEntity show = showDao.findShowDetail(SHOW_NAME); - showDao.updateShowCommentEmail(show, new String[] {"test@imageworks.com"}); - String email = jdbcTemplate.queryForObject( - "SELECT str_comment_email FROM show WHERE pk_show=?", - String.class, show.id); - assertEquals("test@imageworks.com", email); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateBookingEnabled() { - ShowEntity show = showDao.findShowDetail(SHOW_NAME); - showDao.updateBookingEnabled(show,false); - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT b_booking_enabled FROM show WHERE pk_show=?", - Integer.class, show.id)); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateActive() { - ShowEntity show = showDao.findShowDetail(SHOW_NAME); - showDao.updateActive(show, false); - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT b_active FROM show WHERE pk_show=?", - Integer.class, show.id)); - showDao.updateActive(show, true); - assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT b_active FROM show WHERE pk_show=?", - Integer.class, show.id)); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateFrameCounters() { - ShowEntity show = showDao.findShowDetail(SHOW_NAME); - int frameSuccess = jdbcTemplate.queryForObject( - "SELECT int_frame_success_count FROM show WHERE pk_show=?", - Integer.class, show.id); - showDao.updateFrameCounters(show, 0); - int frameSucces2 = jdbcTemplate.queryForObject( - "SELECT int_frame_success_count FROM show WHERE pk_show=?", - Integer.class, show.id); - assertEquals(frameSuccess + 1,frameSucces2); - - int frameFail= jdbcTemplate.queryForObject( - "SELECT int_frame_fail_count FROM show WHERE pk_show=?", - Integer.class, show.id); - showDao.updateFrameCounters(show, 1); - int frameFail2 = jdbcTemplate.queryForObject( - "SELECT int_frame_fail_count FROM show WHERE pk_show=?", - Integer.class, show.id); - assertEquals(frameFail+ 1,frameFail2); - } -} - - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/SubscriptionDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/SubscriptionDaoTests.java deleted file mode 100644 index 5f801c475..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/SubscriptionDaoTests.java +++ /dev/null @@ -1,262 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import javax.annotation.Resource; - -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.AllocationEntity; -import com.imageworks.spcue.AllocationInterface; -import com.imageworks.spcue.FacilityInterface; -import com.imageworks.spcue.ShowInterface; -import com.imageworks.spcue.SubscriptionEntity; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.AllocationDao; -import com.imageworks.spcue.dao.FacilityDao; -import com.imageworks.spcue.dao.ShowDao; -import com.imageworks.spcue.dao.SubscriptionDao; -import com.imageworks.spcue.test.AssumingOracleEngine; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class SubscriptionDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - AllocationDao allocDao; - - @Resource - SubscriptionDao subscriptionDao; - - @Resource - AllocationDao allocationDao; - - @Resource - ShowDao showDao; - - @Resource - FacilityDao facilityDao; - - public static final String SUB_NAME = "test.pipe"; - public static final String ALLOC_NAME = "test"; - - private AllocationEntity alloc; - - public ShowInterface getShow() { - return showDao.getShowDetail("00000000-0000-0000-0000-000000000000"); - } - - public SubscriptionEntity buildSubscription(ShowInterface t, AllocationInterface a) { - SubscriptionEntity s = new SubscriptionEntity(); - s.allocationId = a.getId(); - s.showId = t.getId(); - s.burst = 500; - s.size = 100; - return s; - } - - public AllocationEntity buildAllocation() { - AllocationEntity a = new AllocationEntity(); - a.tag = "test"; - a.name = ALLOC_NAME; - a.facilityId = facilityDao.getDefaultFacility().getFacilityId(); - return a; - } - - @Before - public void before() { - alloc = new AllocationEntity(); - alloc.name = ALLOC_NAME; - alloc.tag = "test"; - allocationDao.insertAllocation( - facilityDao.getDefaultFacility(), alloc); - } - - @Test - @Transactional - @Rollback(true) - public void testHasRunningProcs() { - SubscriptionEntity s = buildSubscription(getShow(), alloc); - subscriptionDao.insertSubscription(s); - assertFalse(subscriptionDao.hasRunningProcs(s)); - } - - @Test - @Transactional - @Rollback(true) - public void testIsShowOverSize() { - - SubscriptionEntity sub = buildSubscription(getShow(), alloc); - subscriptionDao.insertSubscription(sub); - - assertFalse(this.subscriptionDao.isShowOverSize(getShow(), alloc)); - - jdbcTemplate.update( - "UPDATE subscription SET int_cores = ? WHERE pk_subscription = ?", - 100, sub.getSubscriptionId()); - - assertFalse(subscriptionDao.isShowOverSize(getShow(), alloc)); - - jdbcTemplate.update( - "UPDATE subscription SET int_cores = ? WHERE pk_subscription = ?", - 101, sub.getSubscriptionId()); - - assertEquals(true, subscriptionDao.isShowOverSize(getShow(), alloc)); - } - - @Test - @Transactional - @Rollback(true) - public void testIsShowAtOrOverSize() { - - SubscriptionEntity sub = buildSubscription(getShow(), alloc); - subscriptionDao.insertSubscription(sub); - assertFalse(this.subscriptionDao.isShowAtOrOverSize(getShow(), alloc)); - - jdbcTemplate.update( - "UPDATE subscription SET int_cores = ? WHERE pk_subscription = ?", - 100, sub.getSubscriptionId()); - - assertTrue(subscriptionDao.isShowAtOrOverSize(getShow(), alloc)); - - jdbcTemplate.update( - "UPDATE subscription SET int_cores = ? WHERE pk_subscription = ?", - 200, sub.getSubscriptionId()); - - assertTrue(subscriptionDao.isShowAtOrOverSize(getShow(), alloc)); - } - - @Test - @Transactional - @Rollback(true) - public void testIsShowOverBurst() { - subscriptionDao.insertSubscription(buildSubscription(getShow(), alloc)); - - // Burst is 500 so 600 would be over burst. - assertTrue(subscriptionDao.isShowOverBurst(getShow(), alloc, 600)); - - // Burst is 500 so 300 would be under burst. - assertFalse(subscriptionDao.isShowOverBurst(getShow(), alloc, 300)); - } - - @Test(expected=org.springframework.jdbc.UncategorizedSQLException.class) - @Transactional - @Rollback(true) - public void testIsShowAtOrOverBurst() { - - SubscriptionEntity sub = buildSubscription(getShow(), alloc); - subscriptionDao.insertSubscription(sub); - assertFalse(subscriptionDao.isShowAtOrOverBurst(getShow(), alloc)); - - jdbcTemplate.update( - "UPDATE subscription SET int_cores = ? WHERE pk_subscription = ?", - 500, sub.getSubscriptionId()); - - assertTrue(subscriptionDao.isShowAtOrOverBurst(getShow(), alloc)); - - jdbcTemplate.update( - "UPDATE subscription SET int_cores = ? WHERE pk_subscription = ?", - 501, sub.getSubscriptionId()); - - assertTrue(subscriptionDao.isShowAtOrOverBurst(getShow(), alloc)); - } - - @Test - @Transactional - @Rollback(true) - public void testGetSubscriptionDetail() { - - FacilityInterface f = facilityDao.getDefaultFacility(); - - SubscriptionEntity s = buildSubscription(getShow(), alloc); - subscriptionDao.insertSubscription(s); - assertNotNull(s.id); - assertNotNull(s.getId()); - - SubscriptionEntity s1 = subscriptionDao.getSubscriptionDetail( - s.getSubscriptionId()); - - assertEquals(alloc.getName() + ".pipe", s1.name); - assertEquals(s.burst, s1.burst); - assertEquals(s.id, s1.id); - assertEquals(s.size, s1.size); - assertEquals(s.allocationId, s1.allocationId); - } - - @Test - @Transactional - @Rollback(true) - public void testInsertSubscription() { - SubscriptionEntity s = buildSubscription(getShow(), alloc); - subscriptionDao.insertSubscription(s); - } - - @Test - @Transactional - @Rollback(true) - public void testDeleteSubscription() { - SubscriptionEntity s = buildSubscription(getShow(), alloc); - subscriptionDao.insertSubscription(s); - subscriptionDao.deleteSubscription(s); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateSubscriptionSize() { - SubscriptionEntity s = buildSubscription(getShow(), alloc); - subscriptionDao.insertSubscription(s); - subscriptionDao.updateSubscriptionSize(s, 100); - assertEquals(Integer.valueOf(100), jdbcTemplate.queryForObject( - "SELECT int_size FROM subscription WHERE pk_subscription=?", - Integer.class, s.getId())); - } - - @Test - @Transactional - @Rollback(true) - public void testUpdateSubscriptionBurst() { - SubscriptionEntity s = buildSubscription(getShow(), alloc); - subscriptionDao.insertSubscription(s); - subscriptionDao.updateSubscriptionBurst(s, 100); - assertEquals(Integer.valueOf(100), jdbcTemplate.queryForObject( - "SELECT int_burst FROM subscription WHERE pk_subscription=?", - Integer.class, s.getId())); - } -} - - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/TaskDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/TaskDaoTests.java deleted file mode 100644 index aac333ad7..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/TaskDaoTests.java +++ /dev/null @@ -1,288 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import java.io.File; -import javax.annotation.Resource; - -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.JobDetail; -import com.imageworks.spcue.PointInterface; -import com.imageworks.spcue.TaskEntity; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.DepartmentDao; -import com.imageworks.spcue.dao.PointDao; -import com.imageworks.spcue.dao.ShowDao; -import com.imageworks.spcue.dao.TaskDao; -import com.imageworks.spcue.service.JobLauncher; -import com.imageworks.spcue.service.JobManager; -import com.imageworks.spcue.test.AssumingOracleEngine; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class TaskDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - ShowDao showDao; - - @Resource - DepartmentDao departmentDao; - - @Resource - TaskDao taskDao; - - @Resource - PointDao pointDao; - - @Resource - JobManager jobManager; - - @Resource - JobLauncher jobLauncher; - - @Before - public void testMode() { - jobLauncher.testMode = true; - } - - @Test - @Transactional - @Rollback(true) - public void insertTask() { - jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec.xml")); - JobDetail job = jobManager.findJobDetail("pipe-dev.cue-testuser_shell_v1"); - - String dept = jdbcTemplate.queryForObject( - "SELECT pk_dept FROM job WHERE pk_job=?", String.class, job.getJobId()); - - // Add in a new task, the job should switch to using this task. - PointInterface p = pointDao.getPointConfigDetail( - showDao.findShowDetail("pipe"), - departmentDao.getDepartment(dept)); - - TaskEntity t = new TaskEntity(p, "dev.foo", 100); - taskDao.insertTask(t); - - t = taskDao.getTaskDetail(t.id); - taskDao.deleteTask(t); - } - - @Test - @Transactional - @Rollback(true) - public void deleteTask() { - PointInterface p = pointDao.getPointConfigDetail( - showDao.findShowDetail("pipe"), - departmentDao.getDefaultDepartment()); - TaskEntity t = new TaskEntity(p, "dev.cue", 100); - taskDao.insertTask(t); - taskDao.deleteTask(t); - } - - @Test - @Transactional - @Rollback(true) - public void deleteTasksByShowAndDepartment() { - - PointInterface p = pointDao.getPointConfigDetail( - showDao.findShowDetail("pipe"), - departmentDao.getDefaultDepartment()); - - int task_count = jdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM task WHERE pk_point=?", - Integer.class, p.getPointId()); - - TaskEntity t = new TaskEntity(p, "dev.cue"); - taskDao.insertTask(t); - - assertEquals(Integer.valueOf(task_count + 1), jdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM task WHERE pk_point=?", - Integer.class, p.getPointId())); - - taskDao.deleteTasks(p); - - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM task WHERE pk_point=?", - Integer.class, p.getPointId())); - } - - @Test - @Transactional - @Rollback(true) - public void deleteTasksByDepartmentConfig() { - - PointInterface p = pointDao.getPointConfigDetail( - showDao.findShowDetail("pipe"), - departmentDao.getDefaultDepartment()); - - TaskEntity t = new TaskEntity(p, - "dev.cue"); - t.minCoreUnits = 100; - taskDao.insertTask(t); - - taskDao.deleteTasks(p); - - /** - * This is always - */ - assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM task WHERE pk_point=?", - Integer.class, p.getPointId())); - } - - @Test - @Transactional - @Rollback(true) - public void getTaskDetail() { - - PointInterface p = pointDao.getPointConfigDetail( - showDao.findShowDetail("pipe"), - departmentDao.getDefaultDepartment()); - - TaskEntity t = new TaskEntity(p, "dev.cue"); - - taskDao.insertTask(t); - TaskEntity newTask = taskDao.getTaskDetail(t.getTaskId()); - assertEquals(newTask.id,t.id); - } - - @Test - @Transactional - @Rollback(true) - public void getTaskDetailByDept() { - PointInterface p = pointDao.getPointConfigDetail( - showDao.findShowDetail("pipe"), - departmentDao.getDefaultDepartment()); - - TaskEntity t = new TaskEntity(p, "dev.cue"); - - taskDao.insertTask(t); - TaskEntity newTask = taskDao.getTaskDetail(departmentDao.getDefaultDepartment(), "dev.cue"); - assertEquals(newTask.id,t.id); - } - - @Test - @Transactional - @Rollback(true) - public void updateTaskMinProcs() { - - PointInterface p = pointDao.getPointConfigDetail( - showDao.findShowDetail("pipe"), - departmentDao.getDefaultDepartment()); - - TaskEntity t = new TaskEntity(p, "dev.cue"); - t.minCoreUnits = 100; - taskDao.insertTask(t); - TaskEntity newTask = taskDao.getTaskDetail(t.getTaskId()); - taskDao.updateTaskMinCores(newTask, 100); - assertEquals(Integer.valueOf(100), jdbcTemplate.queryForObject( - "SELECT int_min_cores FROM task WHERE pk_task=?", - Integer.class, newTask.getTaskId())); - } - - @Test - @Transactional - @Rollback(true) - public void adjustTaskMinProcs() { - - PointInterface p = pointDao.getPointConfigDetail( - showDao.findShowDetail("pipe"), - departmentDao.getDefaultDepartment()); - - TaskEntity t = new TaskEntity(p,"dev.cue"); - t.minCoreUnits = 10; - taskDao.insertTask(t); - TaskEntity newTask = taskDao.getTaskDetail(t.getTaskId()); - taskDao.updateTaskMinCores(newTask, 100); - assertEquals(Integer.valueOf(100), jdbcTemplate.queryForObject( - "SELECT int_min_cores FROM task WHERE pk_task=?", - Integer.class, newTask.getTaskId())); - - taskDao.adjustTaskMinCores(t, 105); - - assertEquals(Integer.valueOf(100), jdbcTemplate.queryForObject( - "SELECT int_min_cores FROM task WHERE pk_task=?", - Integer.class, newTask.getTaskId())); - assertEquals(Integer.valueOf(5), jdbcTemplate.queryForObject( - "SELECT int_adjust_cores FROM task WHERE pk_task=?", - Integer.class, newTask.getTaskId())); - - taskDao.adjustTaskMinCores(t, 50); - - assertEquals(Integer.valueOf(100), jdbcTemplate.queryForObject( - "SELECT int_min_cores FROM task WHERE pk_task=?", - Integer.class, newTask.getTaskId())); - assertEquals(Integer.valueOf(-50), jdbcTemplate.queryForObject( - "SELECT int_adjust_cores FROM task WHERE pk_task=?", - Integer.class, newTask.getTaskId())); - } - - - @Test - @Transactional - @Rollback(true) - public void mergeTask() { - - PointInterface p = pointDao.getPointConfigDetail( - showDao.findShowDetail("pipe"), - departmentDao.getDefaultDepartment()); - - TaskEntity t = new TaskEntity(p, "dev.cue"); - taskDao.insertTask(t); - - assertEquals(Integer.valueOf(100), jdbcTemplate.queryForObject( - "SELECT int_min_cores FROM task WHERE pk_task=?", - Integer.class, t.getTaskId())); - - TaskEntity newTask = taskDao.getTaskDetail(t.getTaskId()); - newTask.minCoreUnits = 200; - taskDao.mergeTask(newTask); - - assertEquals(Integer.valueOf(200), jdbcTemplate.queryForObject( - "SELECT int_min_cores FROM task WHERE pk_task=?", - Integer.class, newTask.getTaskId())); - } - - @Test - @Transactional - @Rollback(true) - public void isJobManaged() { - jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec.xml")); - JobDetail job = jobManager.findJobDetail("pipe-dev.cue-testuser_shell_v1"); - assertFalse(taskDao.isManaged(job)); - } -} - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/TrackitDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/TrackitDaoTests.java deleted file mode 100644 index 9fcb60e0a..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/TrackitDaoTests.java +++ /dev/null @@ -1,60 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import java.util.List; -import javax.annotation.Resource; - -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; - -import com.imageworks.spcue.TrackitTaskDetail; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.TrackitDao; -import com.imageworks.spcue.test.AssumingOracleEngine; -import com.imageworks.spcue.test.AssumingTrackitEnabled; - -import static org.junit.Assert.assertTrue; - -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class TrackitDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Autowired - @Rule - public AssumingTrackitEnabled assumingTrackitEnabled; - - @Resource - TrackitDao trackitDao; - - @Test - public void testGetTasks() { - List result = trackitDao.getTasks("clo","Lighting"); - assertTrue(result.size() > 0); - } -} - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/WhiteboardDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/WhiteboardDaoTests.java deleted file mode 100644 index 294d22718..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/WhiteboardDaoTests.java +++ /dev/null @@ -1,1279 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dao.oracle; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; -import javax.annotation.Resource; - -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; -import org.springframework.transaction.annotation.Transactional; - -import com.imageworks.spcue.ActionEntity; -import com.imageworks.spcue.AllocationEntity; -import com.imageworks.spcue.CommentDetail; -import com.imageworks.spcue.DeedEntity; -import com.imageworks.spcue.DepartmentInterface; -import com.imageworks.spcue.DispatchFrame; -import com.imageworks.spcue.DispatchHost; -import com.imageworks.spcue.FilterEntity; -import com.imageworks.spcue.FrameDetail; -import com.imageworks.spcue.FrameInterface; -import com.imageworks.spcue.HostEntity; -import com.imageworks.spcue.JobDetail; -import com.imageworks.spcue.JobInterface; -import com.imageworks.spcue.LayerInterface; -import com.imageworks.spcue.LightweightDependency; -import com.imageworks.spcue.LocalHostAssignment; -import com.imageworks.spcue.MatcherEntity; -import com.imageworks.spcue.OwnerEntity; -import com.imageworks.spcue.PointInterface; -import com.imageworks.spcue.ServiceOverrideEntity; -import com.imageworks.spcue.ShowEntity; -import com.imageworks.spcue.ShowInterface; -import com.imageworks.spcue.Source; -import com.imageworks.spcue.TaskEntity; -import com.imageworks.spcue.VirtualProc; -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dao.ActionDao; -import com.imageworks.spcue.dao.AllocationDao; -import com.imageworks.spcue.dao.DepartmentDao; -import com.imageworks.spcue.dao.FilterDao; -import com.imageworks.spcue.dao.FrameDao; -import com.imageworks.spcue.dao.GroupDao; -import com.imageworks.spcue.dao.HostDao; -import com.imageworks.spcue.dao.LayerDao; -import com.imageworks.spcue.dao.LimitDao; -import com.imageworks.spcue.dao.MatcherDao; -import com.imageworks.spcue.dao.PointDao; -import com.imageworks.spcue.dao.ProcDao; -import com.imageworks.spcue.dao.ShowDao; -import com.imageworks.spcue.dao.WhiteboardDao; -import com.imageworks.spcue.dao.criteria.FrameSearchFactory; -import com.imageworks.spcue.dao.criteria.FrameSearchInterface; -import com.imageworks.spcue.dao.criteria.HostSearchFactory; -import com.imageworks.spcue.dao.criteria.HostSearchInterface; -import com.imageworks.spcue.dao.criteria.JobSearchFactory; -import com.imageworks.spcue.dao.criteria.JobSearchInterface; -import com.imageworks.spcue.dao.criteria.ProcSearchFactory; -import com.imageworks.spcue.dao.criteria.ProcSearchInterface; -import com.imageworks.spcue.dispatcher.DispatchSupport; -import com.imageworks.spcue.dispatcher.Dispatcher; -import com.imageworks.spcue.grpc.department.Department; -import com.imageworks.spcue.grpc.filter.ActionType; -import com.imageworks.spcue.grpc.filter.ActionValueType; -import com.imageworks.spcue.grpc.filter.FilterType; -import com.imageworks.spcue.grpc.filter.MatchSubject; -import com.imageworks.spcue.grpc.filter.MatchType; -import com.imageworks.spcue.grpc.host.HardwareState; -import com.imageworks.spcue.grpc.host.Host; -import com.imageworks.spcue.grpc.host.HostSearchCriteria; -import com.imageworks.spcue.grpc.host.LockState; -import com.imageworks.spcue.grpc.host.Owner; -import com.imageworks.spcue.grpc.host.ProcSearchCriteria; -import com.imageworks.spcue.grpc.job.Frame; -import com.imageworks.spcue.grpc.job.FrameSearchCriteria; -import com.imageworks.spcue.grpc.job.FrameState; -import com.imageworks.spcue.grpc.job.Job; -import com.imageworks.spcue.grpc.job.JobSearchCriteria; -import com.imageworks.spcue.grpc.job.Layer; -import com.imageworks.spcue.grpc.limit.Limit; -import com.imageworks.spcue.grpc.report.RenderHost; -import com.imageworks.spcue.service.BookingManager; -import com.imageworks.spcue.service.CommentManager; -import com.imageworks.spcue.service.DepartmentManager; -import com.imageworks.spcue.service.DependManager; -import com.imageworks.spcue.service.HostManager; -import com.imageworks.spcue.service.JobLauncher; -import com.imageworks.spcue.service.JobManager; -import com.imageworks.spcue.service.OwnerManager; -import com.imageworks.spcue.service.ServiceManager; -import com.imageworks.spcue.test.AssumingOracleEngine; -import com.imageworks.spcue.util.CueUtil; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - - -@Transactional -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class WhiteboardDaoTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Autowired - @Rule - public AssumingOracleEngine assumingOracleEngine; - - @Resource - AllocationDao allocationDao; - - @Resource - HostDao hostDao; - - @Resource - WhiteboardDao whiteboardDao; - - @Resource - ShowDao showDao; - - @Resource - FilterDao filterDao; - - @Resource - ProcDao procDao; - - @Resource - MatcherDao matcherDao; - - @Resource - ActionDao actionDao; - - @Resource - JobManager jobManager; - - @Resource - JobLauncher jobLauncher; - - @Resource - GroupDao groupDao; - - @Resource - LayerDao layerDao; - - @Resource - LimitDao limitDao; - - @Resource - DepartmentDao departmentDao; - - @Resource - DependManager dependManager; - - @Resource - FrameDao frameDao; - - @Resource - PointDao pointDao; - - @Resource - HostManager hostManager; - - @Resource - CommentManager commentManager; - - @Resource - DepartmentManager departmentManager; - - @Resource - Dispatcher dispatcher; - - @Resource - DispatchSupport dispatchSupport; - - @Resource - OwnerManager ownerManager; - - @Resource - BookingManager bookingManager; - - @Resource - ServiceManager serviceManager; - - @Resource - FrameSearchFactory frameSearchFactory; - - @Resource - HostSearchFactory hostSearchFactory; - - @Resource - JobSearchFactory jobSearchFactory; - - @Resource - ProcSearchFactory procSearchFactory; - - private static final String HOST = "testest"; - private static final String SHOW = "pipe"; - - @Before - public void testMode() { - jobLauncher.testMode = true; - } - - public ShowEntity getShow() { - return showDao.findShowDetail(SHOW); - } - - public FilterEntity createFilter() { - FilterEntity filter = new FilterEntity(); - filter.name = "Default"; - filter.showId = getShow().id; - filter.type = FilterType.MATCH_ANY; - filter.enabled = true; - filterDao.insertFilter(filter); - return filter; - } - - public MatcherEntity createMatcher(FilterEntity f) { - MatcherEntity matcher = new MatcherEntity(); - matcher.filterId = f.id; - matcher.name = null; - matcher.showId = getShow().getId(); - matcher.subject = MatchSubject.JOB_NAME; - matcher.type = MatchType.CONTAINS; - matcher.value = "testuser"; - matcherDao.insertMatcher(matcher); - return matcher; - } - - public ActionEntity createAction(FilterEntity f) { - ActionEntity a1 = new ActionEntity(); - a1.type = ActionType.PAUSE_JOB; - a1.filterId = f.getFilterId(); - a1.booleanValue = true; - a1.name = null; - a1.valueType = ActionValueType.BOOLEAN_TYPE; - actionDao.createAction(a1); - return a1; - } - - public RenderHost getRenderHost() { - - RenderHost host = RenderHost.newBuilder() - .setName(HOST) - .setBootTime(1192369572) - .setFreeMcp(7602) - .setFreeMem((int) Dispatcher.MEM_RESERVED_MIN * 4) - .setFreeSwap(2076) - .setLoad(1) - .setTotalMcp(19543) - .setTotalMem((int) Dispatcher.MEM_RESERVED_MIN * 4) - .setTotalSwap(2096) - .setNimbyEnabled(true) - .setNumProcs(2) - .setCoresPerProc(400) - .setState(HardwareState.DOWN) - .setFacility("spi") - .putAttributes("freeGpu", String.format("%d", CueUtil.MB512)) - .putAttributes("totalGpu", String.format("%d", CueUtil.MB512)) - .build(); - return host; - } - - public JobDetail launchJob() { - jobLauncher.testMode = true; - jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec.xml")); - return jobManager.findJobDetail("pipe-dev.cue-testuser_shell_v1"); - } - - public JobDetail launchLimitJob() { - jobLauncher.testMode = true; - jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec_limit.xml")); - return jobManager.findJobDetail("pipe-dev.cue-testuser_shell_v1"); - } - - private void createTestLimits() { - limitDao.createLimit("util", 15); - limitDao.createLimit("arnold", 20); - } - - @Test - @Transactional - @Rollback(true) - public void getService() { - whiteboardDao.getService("arnold"); - } - - @Test - @Transactional - @Rollback(true) - public void getServices() { - whiteboardDao.getDefaultServices(); - } - - @Test - @Transactional - @Rollback(true) - public void getServiceOverride() { - - ShowInterface show = getShow(); - ServiceOverrideEntity s = new ServiceOverrideEntity(); - s.name = "test"; - s.minCores = 100; - s.minMemory = 320000; - s.tags.add("general"); - s.threadable = false; - s.showId = show.getId(); - - serviceManager.createService(s); - whiteboardDao.getServiceOverride(getShow(), "test"); - } - - @Test - @Transactional - @Rollback(true) - public void getServiceOverrides() { - whiteboardDao.getServiceOverrides(getShow()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetDepend() { - - List depends = dependManager.getWhatDependsOn(launchJob()); - for (LightweightDependency depend: depends) { - whiteboardDao.getDepend(depend); - } - } - - @Test - @Transactional - @Rollback(true) - public void testGetDependById() { - - List depends = dependManager.getWhatDependsOn(launchJob()); - for (LightweightDependency depend: depends) { - whiteboardDao.getDepend(depend); - whiteboardDao.getDepend(depend.id); - } - } - - - @Test - @Transactional - @Rollback(true) - public void testGetWhatDependsOnThis() { - JobDetail job = launchJob(); - assertEquals(1,whiteboardDao.getWhatDependsOnThis(job).getDependsCount()); - - LayerInterface layer1 = layerDao.findLayer(job, "pass_1"); - assertEquals(0, whiteboardDao.getWhatDependsOnThis(layer1).getDependsCount()); - - LayerInterface layer2 = layerDao.findLayer(job, "pass_1_preprocess"); - assertEquals(1, whiteboardDao.getWhatDependsOnThis(layer2).getDependsCount()); - - FrameInterface frame = frameDao.findFrame(job, "0001-pass_1"); - assertEquals(0, whiteboardDao.getWhatDependsOnThis(frame).getDependsCount()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetWhatThisDependsOn() { - JobDetail job = launchJob(); - assertEquals(0, whiteboardDao.getWhatThisDependsOn(job).getDependsCount()); - - LayerInterface layer1 = layerDao.findLayer(job, "pass_1"); - assertEquals(1, whiteboardDao.getWhatThisDependsOn(layer1).getDependsCount()); - - LayerInterface layer2 = layerDao.findLayer(job, "pass_1_preprocess"); - assertEquals(0, whiteboardDao.getWhatThisDependsOn(layer2).getDependsCount()); - - FrameInterface frame = frameDao.findFrame(job, "0001-pass_1"); - assertEquals(1, whiteboardDao.getWhatThisDependsOn(frame).getDependsCount()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetDepends() { - JobDetail job = launchJob(); - assertEquals(1,whiteboardDao.getDepends(job).getDependsCount()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetCommentsOnJob() { - JobDetail job = launchJob(); - assertEquals(0,whiteboardDao.getComments(job).getCommentsCount()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetCommentsOnHost() { - - RenderHost host = getRenderHost(); - DispatchHost hd = hostManager.createHost(host); - hostDao.updateHostLock(hd, LockState.LOCKED, new Source("TEST")); - - CommentDetail c = new CommentDetail(); - c.message = "you suck"; - c.subject = "a useful message"; - c.user = "testuser"; - c.timestamp = null; - - commentManager.addComment(hd, c); - assertEquals(1,whiteboardDao.getComments(hd).getCommentsCount()); - } - - @Test - @Transactional - @Rollback(true) - public void testFindFilter() { - createFilter(); - whiteboardDao.findFilter(getShow(), "Default"); - } - - @Test - @Transactional - @Rollback(true) - public void testGetFilter() { - whiteboardDao.getFilter(createFilter()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetMatchers() { - FilterEntity f = createFilter(); - createMatcher(f); - whiteboardDao.getMatchers(f); - } - - @Test - @Transactional - @Rollback(true) - public void testGetMatcher() { - FilterEntity f = createFilter(); - MatcherEntity m = createMatcher(f); - whiteboardDao.getMatcher(m); - } - - @Test - @Transactional - @Rollback(true) - public void testGetActions() { - FilterEntity f = createFilter(); - createAction(f); - whiteboardDao.getActions(f); - } - - @Test - @Transactional - @Rollback(true) - public void testGetAction() { - FilterEntity f = createFilter(); - whiteboardDao.getAction(createAction(f)); - } - - @Test - @Transactional - @Rollback(true) - public void testGetFilters() { - createFilter(); - whiteboardDao.getFilters(getShow()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetFramesByFrameSearch() { - JobInterface job = launchJob(); - FrameSearchInterface r = frameSearchFactory.create(job); - FrameSearchCriteria criteria = r.getCriteria(); - r.setCriteria(criteria.toBuilder() - .setPage(1) - .setLimit(5) - .addLayers("pass_1") - .build()); - assertEquals(5, whiteboardDao.getFrames(r).getFramesCount()); - for (Frame f: whiteboardDao.getFrames(r).getFramesList()) { - assertEquals(f.getLayerName(), "pass_1"); - } - } - - @Test - @Transactional - @Rollback(true) - public void testGetLayers() { - JobDetail job = launchJob(); - whiteboardDao.getLayers(job); - - RenderHost host = getRenderHost(); - DispatchHost hd = hostManager.createHost(host); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1_preprocess"); - - - VirtualProc proc = new VirtualProc(); - proc.allocationId = null; - proc.coresReserved = 100; - proc.hostId = hd.id; - proc.hostName = host.getName(); - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - - DispatchFrame dframe = frameDao.getDispatchFrame(frame.getId()); - dispatcher.setTestMode(true); - dispatcher.dispatch(dframe, proc); - - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - dframe = frameDao.getDispatchFrame(frame.getId()); - - assertTrue(dispatchSupport.stopFrame(dframe, FrameState.SUCCEEDED, 0)); - dispatchSupport.updateUsageCounters(frame, 0); - whiteboardDao.getLayers(job); - } - - @Test - @Transactional - @Rollback(true) - public void testGetLimits() { - createTestLimits(); - List limits = whiteboardDao.getLimits(); - assertEquals(limits.size(), 2); - } - - @Test - @Transactional - @Rollback(true) - public void testGetLayerLimits() { - createTestLimits(); - JobDetail job = launchLimitJob(); - LayerInterface layer = layerDao.findLayer(job, "pass_1"); - List limits = whiteboardDao.getLimits(layer); - assertEquals(limits.size(), 1); - assertEquals(limits.get(0).getName(), "arnold"); - } - - @Test - @Transactional - @Rollback(true) - public void testGetLimit() { - String limitName = "testing"; - int limitMaxValue = 20; - String limitId = limitDao.createLimit(limitName, limitMaxValue); - Limit limit = whiteboardDao.getLimit(limitId); - assertEquals(limit.getName(), limitName); - assertEquals(limit.getMaxValue(), limitMaxValue); - } - - @Test - @Transactional - @Rollback(true) - public void testFindLimit() { - String limitName = "testing"; - int limitMaxValue = 20; - String limitId = limitDao.createLimit(limitName, limitMaxValue); - Limit limit = whiteboardDao.findLimit(limitName); - assertEquals(limit.getName(), limitName); - assertEquals(limit.getMaxValue(), limitMaxValue); - assertEquals(limit.getId(), limitId); - } - - @Test - @Transactional - @Rollback(true) - public void testStopFrameUpdatesLayerMaxRSS() { - long max_rss = 123456L; - - JobDetail job = launchJob(); - whiteboardDao.getLayers(job); - - RenderHost host = getRenderHost(); - DispatchHost hd = hostManager.createHost(host); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1_preprocess"); - - - VirtualProc proc = new VirtualProc(); - proc.allocationId = null; - proc.coresReserved = 100; - proc.hostId = hd.id; - proc.hostName = host.getName(); - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - - DispatchFrame dframe = frameDao.getDispatchFrame(frame.getId()); - dispatcher.setTestMode(true); - dispatcher.dispatch(dframe, proc); - - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - dframe = frameDao.getDispatchFrame(frame.getId()); - - // Note use of 4-arg stopFrame here to update max rss. - assertTrue(dispatchSupport.stopFrame(dframe, FrameState.SUCCEEDED, 0, max_rss)); - dispatchSupport.updateUsageCounters(frame, 0); - Layer layer = whiteboardDao.getLayer(frame.layerId); - assertEquals(max_rss, layer.getLayerStats().getMaxRss()); - } - - @Test - @Transactional - @Rollback(true) - public void testStopFrameUpdatesJobMaxRSS() { - long max_rss = 123456L; - - JobDetail job = launchJob(); - whiteboardDao.getLayers(job); - - RenderHost host = getRenderHost(); - DispatchHost hd = hostManager.createHost(host); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1_preprocess"); - - - VirtualProc proc = new VirtualProc(); - proc.allocationId = null; - proc.coresReserved = 100; - proc.hostId = hd.id; - proc.hostName = host.getName(); - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - - DispatchFrame dframe = frameDao.getDispatchFrame(frame.getId()); - dispatcher.setTestMode(true); - dispatcher.dispatch(dframe, proc); - - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - dframe = frameDao.getDispatchFrame(frame.getId()); - - // Note use of 4-arg stopFrame here to update max rss. - assertTrue(dispatchSupport.stopFrame(dframe, FrameState.SUCCEEDED, 0, max_rss)); - dispatchSupport.updateUsageCounters(frame, 0); - Job grpc_job = whiteboardDao.getJob(job.id); - assertEquals(max_rss, grpc_job.getJobStats().getMaxRss()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetJobs() { - launchJob(); - JobSearchCriteria r = JobSearchInterface.criteriaFactory(); - r = r.toBuilder().addShows("pipe").build(); - whiteboardDao.getJobs(jobSearchFactory.create(r)); - } - - @Test - @Transactional - @Rollback(true) - public void testGetJobNames() { - launchJob(); - JobSearchCriteria r = JobSearchInterface.criteriaFactory(); - r = r.toBuilder().addShows("pipe").build(); - whiteboardDao.getJobNames(jobSearchFactory.create(r)); - } - - @Test - @Transactional - @Rollback(true) - public void testGetUpdatedFrames() { - final JobDetail job = launchJob(); - List jobs = new ArrayList(); - - jobs.add(new JobInterface() { - public String getJobId() { return job.getId(); } - public String getShowId() { return null; } - public String getId() { return job.getId(); } - public String getName() { return null; } - public String getFacilityId() { throw new RuntimeException("not implemented"); } - }); - - whiteboardDao.getUpdatedFrames(job, new ArrayList(), - (int) (System.currentTimeMillis() / 1000)); - - } - - @Test(expected=IllegalArgumentException.class) - @Transactional - @Rollback(true) - public void testGetUpdatedFramesFailure() { - final JobDetail job = launchJob(); - List jobs = new ArrayList(); - - jobs.add(new JobInterface() { - public String getJobId() { return job.getId(); } - public String getShowId() { return null; } - public String getId() { return job.getId(); } - public String getName() { return null; } - public String getFacilityId() { throw new RuntimeException("not implemented"); } - }); - - // this one should fail - whiteboardDao.getUpdatedFrames(job, new ArrayList(), - (int) (System.currentTimeMillis() / 1000 - 1000000)); - } - - @Test - @Transactional - @Rollback(true) - public void testFindJob() { - JobDetail job = launchJob(); - whiteboardDao.findJob(job.name); - } - - @Test - @Transactional - @Rollback(true) - public void testGetJob() { - JobDetail job = launchJob(); - whiteboardDao.getJob(job.id); - } - - @Test - @Transactional - @Rollback(true) - public void testGetSubscriptionByID() { - whiteboardDao.getSubscription("00000000-0000-0000-0000-000000000001"); - } - - @Test - @Transactional - @Rollback(true) - public void findFindSubscription() { - whiteboardDao.findSubscription("pipe", "spi.general"); - } - - @Test - @Transactional - @Rollback(true) - public void testGetSubscriptions() { - whiteboardDao.getSubscriptions(getShow()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetSubscriptionsByAllocation() { - whiteboardDao.getSubscriptions( - allocationDao.findAllocationEntity("spi", "general")); - } - - @Test - @Transactional - @Rollback(true) - public void testGetShow() { - whiteboardDao.getShow(getShow().id); - } - - @Test - @Transactional - @Rollback(true) - public void testFindShow() { - whiteboardDao.findShow(getShow().name); - } - - @Test - @Transactional - @Rollback(true) - public void testGetShows() { - whiteboardDao.getShows(); - } - - @Test - @Transactional - @Rollback(true) - public void testGetActiveShows() { - whiteboardDao.getActiveShows(); - } - - @Test - @Transactional - @Rollback(true) - public void testFindHost() { - - try { - HostEntity h = hostManager.findHostDetail(HOST); - hostManager.deleteHost(h); - } catch (Exception e) { } - - RenderHost host = getRenderHost(); - DispatchHost hd = hostManager.createHost(host); - hostDao.updateHostLock(hd, LockState.LOCKED, new Source("TEST")); - Host h = whiteboardDao.findHost(host.getName()); - assertEquals(host.getName(), h.getName()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetHosts() { - RenderHost host = getRenderHost(); - DispatchHost hd = hostManager.createHost(host); - - HostSearchCriteria h = HostSearchInterface.criteriaFactory(); - h = h.toBuilder().addHosts(HOST).build(); - assertEquals(1, whiteboardDao.getHosts(hostSearchFactory.create(h)).getHostsCount()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetHostsByAllocation() { - RenderHost host = getRenderHost(); - AllocationEntity alloc = allocationDao.getAllocationEntity("00000000-0000-0000-0000-000000000006"); - DispatchHost hd = hostManager.createHost(host, alloc); - - HostSearchCriteria h = HostSearchInterface.criteriaFactory(); - h = h.toBuilder().addAllocs(alloc.getName()).build(); - assertEquals(1, whiteboardDao.getHosts(hostSearchFactory.create(h)).getHostsCount()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetAllocation() { - whiteboardDao.getAllocation("00000000-0000-0000-0000-000000000000"); - } - - @Test - @Transactional - @Rollback(true) - public void testFindAllocation() { - whiteboardDao.findAllocation("spi.general"); - } - - @Test - @Transactional - @Rollback(true) - public void testGetAllocations() { - whiteboardDao.getAllocations(); - } - - @Test - @Transactional - @Rollback(true) - public void testGetRootGroup() { - whiteboardDao.getRootGroup(getShow()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetGroup() { - whiteboardDao.getGroup("A0000000-0000-0000-0000-000000000000"); - } - - @Test - @Transactional - @Rollback(true) - public void testGetGroups() { - whiteboardDao.getGroups(getShow()); - whiteboardDao.getGroup(groupDao.getRootGroupId(getShow())); - whiteboardDao.getGroups(groupDao.getRootGroupDetail(getShow())); - } - - @Test - @Transactional - @Rollback(true) - public void testFindGroup() { - whiteboardDao.findGroup("pipe", "pipe"); - } - - @Test - @Transactional - @Rollback(true) - public void testFindFrame() { - JobDetail job = launchJob(); - whiteboardDao.findFrame(job.name, "pass_1", 1); - } - - @Test - @Transactional - @Rollback(true) - public void testFindFilterByName() { - createFilter(); - whiteboardDao.findFilter("pipe", "Default"); - } - - @Test - @Transactional - @Rollback(true) - public void testFindLayer() { - JobDetail job = launchJob(); - whiteboardDao.findLayer(job.name, "pass_1"); - } - - @Test - @Transactional - @Rollback(true) - public void testGetDepartment() { - ShowInterface show = showDao.findShowDetail("pipe"); - DepartmentInterface dept = departmentDao.getDefaultDepartment(); - - Department d = whiteboardDao.getDepartment(show, dept.getName()); - - assertEquals("pipe.Unknown", d.getName()); - assertEquals("Unknown", d.getDept()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetDepartments() { - ShowInterface show = showDao.findShowDetail("pipe"); - whiteboardDao.getDepartments(show); - } - - - @Test - @Transactional - @Rollback(true) - public void testGetDepartmentNames() { - assertTrue(whiteboardDao.getDepartmentNames().size() > 0); - } - - @Test - @Transactional - @Rollback(true) - public void testGetTasks() { - whiteboardDao.getTasks(showDao.findShowDetail("pipe"), - departmentDao.getDefaultDepartment()); - } - - @Test - @Transactional - @Rollback(true) - public void testGetTask() { - PointInterface p = pointDao.getPointConfigDetail( - showDao.findShowDetail("pipe"), - departmentDao.getDefaultDepartment()); - - TaskEntity t = new TaskEntity(p,"dev.cue"); - departmentManager.createTask(t); - - whiteboardDao.getTask(showDao.findShowDetail("pipe"), - departmentDao.getDefaultDepartment(), "dev.cue"); - } - - @Test - @Transactional - @Rollback(true) - public void getFrame() { - JobDetail job = launchJob(); - FrameInterface frame = frameDao.findFrame(job, "0001-pass_1_preprocess"); - assertEquals(1, whiteboardDao.getFrame(frame.getFrameId()).getNumber()); - } - - @Test - @Transactional - @Rollback(true) - public void getLayer() { - JobDetail job = launchJob(); - LayerInterface layer = layerDao.findLayer(job, "pass_1"); - assertEquals(layer.getName(),whiteboardDao.getLayer(layer.getLayerId()).getName()); - } - - @Test - @Transactional - @Rollback(true) - public void getHost() { - - RenderHost host = getRenderHost(); - DispatchHost hd = hostManager.createHost(host); - - JobDetail job = launchJob(); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1_preprocess"); - - VirtualProc proc = new VirtualProc(); - proc.allocationId = null; - proc.coresReserved = 100; - proc.hostId = hd.id; - proc.hostName = host.getName(); - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - - procDao.insertVirtualProc(proc); - assertEquals(hd.getName(), whiteboardDao.getHost(proc.getHostId()).getName()); - } - - @Test - @Transactional - @Rollback(true) - public void getProcs() { - RenderHost host = getRenderHost(); - DispatchHost hd = hostManager.createHost(host); - JobDetail job = launchJob(); - FrameDetail frame = frameDao.findFrameDetail(job, "0001-pass_1_preprocess"); - - VirtualProc proc = new VirtualProc(); - proc.allocationId = null; - proc.coresReserved = 100; - proc.hostId = hd.id; - proc.hostName = host.getName(); - proc.jobId = job.id; - proc.frameId = frame.id; - proc.layerId = frame.layerId; - proc.showId = frame.showId; - - procDao.insertVirtualProc(proc); - assertEquals(1,whiteboardDao.getProcs(proc).getProcsCount()); - } - - @Test - @Transactional - @Rollback(true) - public void getProcsBySearch() { - RenderHost host = getRenderHost(); - DispatchHost hd = hostManager.createHost(host); - - jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec_dispatch_test.xml")); - JobDetail job = jobManager.findJobDetail("pipe-dev.cue-testuser_shell_dispatch_test_v1"); - - /* - * Book 5 procs. - */ - for (int i=1; i<6; i++) { - FrameDetail f = frameDao.findFrameDetail(job, String.format("%04d-pass_1",i)); - VirtualProc proc = new VirtualProc(); - proc.allocationId = null; - proc.coresReserved = 100; - proc.hostId = hd.id; - proc.hostName = host.getName(); - proc.jobId = job.id; - proc.frameId = f.id; - proc.layerId = f.layerId; - proc.showId = f.showId; - procDao.insertVirtualProc(proc); - } - - ProcSearchInterface r; - - /* - * Search for all 5 running procs - */ - r = procSearchFactory.create(); - ProcSearchCriteria criteria = r.getCriteria(); - r.setCriteria(criteria.toBuilder().addShows("pipe").build()); - assertEquals(5, whiteboardDao.getProcs(r).getProcsCount()); - - /* - * Limit the result to 1 result. - */ - r = procSearchFactory.create(); - ProcSearchCriteria criteriaA = r.getCriteria(); - r.setCriteria(criteriaA.toBuilder() - .addShows("pipe") - .addMaxResults(1) - .build()); - assertEquals(1, whiteboardDao.getProcs(r).getProcsCount()); - - /* - * Change the first result to 1, which should limit - * the result to 4. - */ - r = procSearchFactory.create(); - ProcSearchCriteria criteriaB = r.getCriteria(); - r.setCriteria(criteriaB.toBuilder() - .addShows("pipe") - .setFirstResult(2) - .build()); - assertEquals(4, whiteboardDao.getProcs(r).getProcsCount()); - - /* - * Now try to do the equivalent of a limit/offset - */ - r = procSearchFactory.create(); - ProcSearchCriteria criteriaC = r.getCriteria(); - r.setCriteria(criteriaC.toBuilder() - .addShows("pipe") - .setFirstResult(3) - .addMaxResults(2) - .build()); - assertEquals(2, whiteboardDao.getProcs(r).getProcsCount()); - } - - @Test - @Transactional - @Rollback(true) - public void getOwner() { - ownerManager.createOwner("spongebob", - showDao.findShowDetail("pipe")); - whiteboardDao.getOwner("spongebob"); - } - - @Test - @Transactional - @Rollback(true) - public void getOwnersByShow() { - RenderHost host = getRenderHost(); - DispatchHost hd = hostManager.createHost(host); - - OwnerEntity owner = ownerManager.createOwner("spongebob", - showDao.findShowDetail("pipe")); - - ownerManager.takeOwnership(owner, hd); - - assertTrue(whiteboardDao.getOwners( - showDao.findShowDetail("pipe")).size() != 0); - } - - @Test - @Transactional - @Rollback(true) - public void getDeedsByShow() { - - RenderHost host = getRenderHost(); - DispatchHost hd = hostManager.createHost(host); - - OwnerEntity owner = ownerManager.createOwner("spongebob", - showDao.findShowDetail("pipe")); - - ownerManager.takeOwnership(owner, hd); - assertTrue(whiteboardDao.getDeeds( - showDao.findShowDetail("pipe")).getDeedsCount() != 0); - - assertTrue(whiteboardDao.getDeeds( - showDao.findShowDetail("pipe")).getDeedsCount() != 0); - } - - @Test - @Transactional - @Rollback(true) - public void getDeedsByOwner() { - - RenderHost host = getRenderHost(); - DispatchHost hd = hostManager.createHost(host); - - OwnerEntity owner = ownerManager.createOwner("spongebob", - showDao.findShowDetail("pipe")); - - ownerManager.takeOwnership(owner, hd); - assertTrue(whiteboardDao.getDeeds( - owner).getDeedsCount() != 0); - } - - @Test - @Transactional - @Rollback(true) - public void getHostsByOwner() { - - RenderHost host = getRenderHost(); - DispatchHost hd = hostManager.createHost(host); - - OwnerEntity owner = ownerManager.createOwner("spongebob", - showDao.findShowDetail("pipe")); - ownerManager.takeOwnership(owner, hd); - - assertEquals(1, whiteboardDao.getHosts(owner).getHostsCount()); - } - - @Test - @Transactional - @Rollback(true) - public void getOwnerFromDeed() { - - RenderHost host = getRenderHost(); - DispatchHost hd = hostManager.createHost(host); - - OwnerEntity owner = ownerManager.createOwner("spongebob", - showDao.findShowDetail("pipe")); - DeedEntity deed = ownerManager.takeOwnership(owner, hd); - - Owner o2 = whiteboardDao.getOwner(deed); - - assertEquals(owner.getName(), o2.getName()); - assertEquals(1, o2.getHostCount()); - } - - @Test - @Transactional - @Rollback(true) - public void getOwnerFromHost() { - - RenderHost host = getRenderHost(); - DispatchHost hd = hostManager.createHost(host); - - OwnerEntity owner = ownerManager.createOwner("spongebob", - showDao.findShowDetail("pipe")); - ownerManager.takeOwnership(owner, hd); - - Owner o2 = whiteboardDao.getOwner(hd); - - assertEquals(owner.getName(), o2.getName()); - assertEquals(1, o2.getHostCount()); - } - - - @Test - @Transactional - @Rollback(true) - public void getRenderPartition() { - - RenderHost host = getRenderHost(); - DispatchHost hd = hostManager.createHost(host); - - jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec_dispatch_test.xml")); - JobDetail job = jobManager.findJobDetail("pipe-dev.cue-testuser_shell_dispatch_test_v1"); - - LocalHostAssignment lba = new LocalHostAssignment(800, 8, CueUtil.GB8, 1); - bookingManager.createLocalHostAssignment(hd, job, lba); - - whiteboardDao.getRenderPartition(lba); - } - - @Test - @Transactional - @Rollback(true) - public void getRenderPartitionsByHost() { - - RenderHost host = getRenderHost(); - DispatchHost hd = hostManager.createHost(host); - - jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec_dispatch_test.xml")); - JobDetail job = jobManager.findJobDetail("pipe-dev.cue-testuser_shell_dispatch_test_v1"); - - LocalHostAssignment lba = new LocalHostAssignment(800, 8, CueUtil.GB8, 1); - bookingManager.createLocalHostAssignment(hd, job, lba); - - assertEquals(1, whiteboardDao.getRenderPartitions(hd).getRenderPartitionsCount()); - - } - - @Test - @Transactional - @Rollback(true) - public void getFacility() { - whiteboardDao.getFacilities(); - whiteboardDao.getFacility("spi"); - } -} - - diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/DepartmentManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/DepartmentManagerTests.java index b9bd5b5ac..4dd2bd13f 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/DepartmentManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/DepartmentManagerTests.java @@ -38,7 +38,6 @@ import com.imageworks.spcue.dao.ShowDao; import com.imageworks.spcue.service.AdminManager; import com.imageworks.spcue.service.DepartmentManager; -import com.imageworks.spcue.test.AssumingTrackitEnabled; import static org.junit.Assert.assertTrue; @@ -47,10 +46,6 @@ @ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) public class DepartmentManagerTests extends AbstractTransactionalJUnit4SpringContextTests { - @Autowired - @Rule - public AssumingTrackitEnabled assumingTrackitEnabled; - @Resource DepartmentManager departmentManager; @@ -78,13 +73,6 @@ public void enableTiManaged() { departmentManager.disableTiManaged(rp); departmentManager.enableTiManaged(rp, TEST_TI_TASK_NAME, 1000); - - // TODO(bcipriano) Once this test is enabled this assert should be updated to use - // DAO objects instead of querying the db directly. - assertTrue(0 < jdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM task,point WHERE point.pk_point = task.pk_point AND " + - "point.pk_dept=? AND point.pk_show=?", - Integer.class, dept.getDepartmentId(), show.getShowId())); } @Test @@ -105,21 +93,8 @@ public void updateTiManagedTasks() { departmentManager.disableTiManaged(rp); departmentManager.enableTiManaged(rp, TEST_TI_TASK_NAME, 1000); - // TODO(bcipriano) Once this test is enabled these asserts should be updated to use - // DAO objects instead of querying the db directly. - - assertTrue(0 < jdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM task,point WHERE point.pk_point = task.pk_point AND " + - "point.pk_dept=? AND point.pk_show=?", - Integer.class, dept.getDepartmentId(), show.getShowId())); - departmentManager.updateManagedTasks(rp); - assertTrue(0 < jdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM task,point WHERE point.pk_point = task.pk_point AND " + - "point.pk_dept=? AND point.pk_show=?", - Integer.class, dept.getDepartmentId(), show.getShowId())); - departmentManager.disableTiManaged(rp); } diff --git a/cuebot/src/test/resources/conf/ddl/oracle/test_data.sql b/cuebot/src/test/resources/conf/ddl/oracle/test_data.sql deleted file mode 100644 index bdc0e4c8f..000000000 --- a/cuebot/src/test/resources/conf/ddl/oracle/test_data.sql +++ /dev/null @@ -1,118 +0,0 @@ -Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000000','pipe',20000,100,0,0,0,0,1,1,1) --- SPLIT HERE! -Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000001','edu',20000,100,0,0,0,0,1,1,1) --- SPLIT HERE! - -Insert into SHOW_ALIAS (PK_SHOW_ALIAS,PK_SHOW,STR_NAME) values ('00000000-0000-0000-0000-000000000001','00000000-0000-0000-0000-000000000000','fx') --- SPLIT HERE! - -Insert into DEPT (PK_DEPT,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA0','Lighting',0) --- SPLIT HERE! -Insert into DEPT (PK_DEPT,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA1','Animation',0) --- SPLIT HERE! -Insert into DEPT (PK_DEPT,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA2','Hair',0) --- SPLIT HERE! -Insert into DEPT (PK_DEPT,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA3','Cloth',0) --- SPLIT HERE! -Insert into DEPT (PK_DEPT,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA4','Layout',0) --- SPLIT HERE! -Insert into DEPT (PK_DEPT,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA5','FX',0) --- SPLIT HERE! -Insert into DEPT (PK_DEPT,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA6','Pipeline',0) --- SPLIT HERE! -Insert into DEPT (PK_DEPT,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA7','S3D',0) --- SPLIT HERE! -Insert into DEPT (PK_DEPT,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA8','Unknown',1) --- SPLIT HERE! - -Insert into FACILITY (PK_FACILITY,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA1','lax',1) --- SPLIT HERE! -Insert into FACILITY (PK_FACILITY,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA0','spi',0) --- SPLIT HERE! -Insert into FACILITY (PK_FACILITY,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA2','maa',0) --- SPLIT HERE! -Insert into FACILITY (PK_FACILITY,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA3','abq',0) --- SPLIT HERE! -Insert into FACILITY (PK_FACILITY,STR_NAME,B_DEFAULT) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA4','brs',0) --- SPLIT HERE! - -Insert into FOLDER (PK_FOLDER,PK_PARENT_FOLDER,PK_SHOW,STR_NAME,B_DEFAULT,PK_DEPT,INT_JOB_MIN_CORES,INT_JOB_MAX_CORES,INT_JOB_PRIORITY,F_ORDER,B_EXCLUDE_MANAGED) values ('A0000000-0000-0000-0000-000000000000',null,'00000000-0000-0000-0000-000000000000','pipe',1,'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA8',-1,-1,-1,1,0) --- SPLIT HERE! -Insert into FOLDER (PK_FOLDER,PK_PARENT_FOLDER,PK_SHOW,STR_NAME,B_DEFAULT,PK_DEPT,INT_JOB_MIN_CORES,INT_JOB_MAX_CORES,INT_JOB_PRIORITY,F_ORDER,B_EXCLUDE_MANAGED) values ('B0000000-0000-0000-0000-000000000000',null,'00000000-0000-0000-0000-000000000001','edu',1,'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA8',-1,-1,-1,1,0) --- SPLIT HERE! - -Insert into POINT (PK_POINT,PK_DEPT,PK_SHOW,STR_TI_TASK,INT_CORES,B_MANAGED,INT_MIN_CORES,FLOAT_TIER) values ('FFEEDDCC-AAAA-AAAA-AAAA-AAAAAAAAAAA0','AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA8','00000000-0000-0000-0000-000000000000',null,0,0,0,0) --- SPLIT HERE! -Insert into POINT (PK_POINT,PK_DEPT,PK_SHOW,STR_TI_TASK,INT_CORES,B_MANAGED,INT_MIN_CORES,FLOAT_TIER) values ('FFEEDDCC-AAAA-AAAA-AAAA-AAAAAAAAAAA1','AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA8','00000000-0000-0000-0000-000000000001',null,0,0,0,0) --- SPLIT HERE! - -Insert into ALLOC (PK_ALLOC,STR_NAME,B_ALLOW_EDIT,B_DEFAULT,STR_TAG,PK_FACILITY,B_BILLABLE,B_ENABLED) values ('00000000-0000-0000-0000-000000000000','lax.general',0,0,'general','AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA1',0,1) --- SPLIT HERE! -Insert into ALLOC (PK_ALLOC,STR_NAME,B_ALLOW_EDIT,B_DEFAULT,STR_TAG,PK_FACILITY,B_BILLABLE,B_ENABLED) values ('00000000-0000-0000-0000-000000000001','lax.desktop',0,0,'desktop','AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA1',0,1) --- SPLIT HERE! -Insert into ALLOC (PK_ALLOC,STR_NAME,B_ALLOW_EDIT,B_DEFAULT,STR_TAG,PK_FACILITY,B_BILLABLE,B_ENABLED) values ('00000000-0000-0000-0000-000000000002','lax.unassigned',0,1,'unassigned','AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA1',0,1) --- SPLIT HERE! -Insert into ALLOC (PK_ALLOC,STR_NAME,B_ALLOW_EDIT,B_DEFAULT,STR_TAG,PK_FACILITY,B_BILLABLE,B_ENABLED) values ('00000000-0000-0000-0000-000000000003','maa.general',0,0,'general','AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA2',0,1) --- SPLIT HERE! -Insert into ALLOC (PK_ALLOC,STR_NAME,B_ALLOW_EDIT,B_DEFAULT,STR_TAG,PK_FACILITY,B_BILLABLE,B_ENABLED) values ('00000000-0000-0000-0000-000000000004','maa.desktop',0,0,'desktop','AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA2',0,1) --- SPLIT HERE! -Insert into ALLOC (PK_ALLOC,STR_NAME,B_ALLOW_EDIT,B_DEFAULT,STR_TAG,PK_FACILITY,B_BILLABLE,B_ENABLED) values ('00000000-0000-0000-0000-000000000005','maa.unassigned',0,0,'unassigned','AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA2',0,1) --- SPLIT HERE! -Insert into ALLOC (PK_ALLOC,STR_NAME,B_ALLOW_EDIT,B_DEFAULT,STR_TAG,PK_FACILITY,B_BILLABLE,B_ENABLED) values ('00000000-0000-0000-0000-000000000006','spi.general',1,0,'general','AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA0',0,1) --- SPLIT HERE! -Insert into ALLOC (PK_ALLOC,STR_NAME,B_ALLOW_EDIT,B_DEFAULT,STR_TAG,PK_FACILITY,B_BILLABLE,B_ENABLED) values ('00000000-0000-0000-0000-000000000007','spi.desktop',1,0,'desktop','AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA0',0,1) --- SPLIT HERE! - -Insert into SUBSCRIPTION (PK_SUBSCRIPTION,PK_ALLOC,PK_SHOW,INT_SIZE,INT_BURST,INT_CORES,FLOAT_TIER) values ('00000000-0000-0000-0000-000000000001','00000000-0000-0000-0000-000000000000','00000000-0000-0000-0000-000000000000',1000,1000,0,0) --- SPLIT HERE! -Insert into SUBSCRIPTION (PK_SUBSCRIPTION,PK_ALLOC,PK_SHOW,INT_SIZE,INT_BURST,INT_CORES,FLOAT_TIER) values ('00000000-0000-0000-0000-000000000002','00000000-0000-0000-0000-000000000001','00000000-0000-0000-0000-000000000000',1000,1000,0,0) --- SPLIT HERE! -Insert into SUBSCRIPTION (PK_SUBSCRIPTION,PK_ALLOC,PK_SHOW,INT_SIZE,INT_BURST,INT_CORES,FLOAT_TIER) values ('00000000-0000-0000-0000-000000000003','00000000-0000-0000-0000-000000000007','00000000-0000-0000-0000-000000000000',1000,1000,0,0) --- SPLIT HERE! -Insert into SUBSCRIPTION (PK_SUBSCRIPTION,PK_ALLOC,PK_SHOW,INT_SIZE,INT_BURST,INT_CORES,FLOAT_TIER) values ('00000000-0000-0000-0000-000000000004','00000000-0000-0000-0000-000000000006','00000000-0000-0000-0000-000000000000',1000,1000,0,0) --- SPLIT HERE! - -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA0','default',0,100,3355443,'general | desktop') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA1','prman',0,100,3355443,'general | desktop') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA2','arnold',1,100,3355443,'general | desktop') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA3','shell',0,100,3355443,'general | util') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA4','maya',0,100,2097152,'general | desktop') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA5','houdini',0,100,3355443,'general | desktop') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA6','svea',1,100,3355443,'general | desktop') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA7','katana',1,100,2097152,'general | desktop | util') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA8','shake',0,100,2097152,'general | desktop') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAA9','nuke',0,100,2097152,'general | desktop') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAA10','ginsu',0,50,524288,'general | desktop | util') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAA11','preprocess',0,10,393216,'util') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAA12','postprocess',0,10,524288,'util') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAA14','refcollect',0,50,1048576,'general | util') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAA15','makemovie',0,50,1048576,'util') --- SPLIT HERE! -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS,INT_CORES_MAX,INT_GPU_MIN) values ('488c75f0-eae4-4dd0-83e0-29b982adbbff','cuda',1,100,3354624,'cuda',0,262144) --- SPLIT HERE! - --- SPLIT HERE! -Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000005','MAX_FRAME_RETRIES',16,0,null,0) - --- SPLIT HERE! -Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000002','LOCK_HARDWARE_STATE_CHECK',0,30) --- SPLIT HERE! -Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000001','LOCK_HISTORICAL_TRANSFER',0,3600) --- SPLIT HERE! -Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000003','LOCK_ORPHANED_PROC_CHECK',0,30) --- SPLIT HERE! -Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000005','LOCK_TASK_UPDATE',1240618998852,3600) diff --git a/cuebot/src/test/resources/conf/spring/applicationContext-assumptions.xml b/cuebot/src/test/resources/conf/spring/applicationContext-assumptions.xml index bd4b161af..3cf18976f 100644 --- a/cuebot/src/test/resources/conf/spring/applicationContext-assumptions.xml +++ b/cuebot/src/test/resources/conf/spring/applicationContext-assumptions.xml @@ -21,14 +21,8 @@ xsi:schemaLocation=" http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd"> - - - - - - diff --git a/cuebot/src/test/resources/conf/spring/applicationContext-oracle-datasource.xml b/cuebot/src/test/resources/conf/spring/applicationContext-oracle-datasource.xml deleted file mode 100644 index 67dddc41b..000000000 --- a/cuebot/src/test/resources/conf/spring/applicationContext-oracle-datasource.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/cuebot/src/test/resources/log4j.properties b/cuebot/src/test/resources/log4j.properties index 712cdc4b3..71a45df96 100644 --- a/cuebot/src/test/resources/log4j.properties +++ b/cuebot/src/test/resources/log4j.properties @@ -29,7 +29,6 @@ log4j.logger.com.imageworks.spcue=DEBUG log4j.logger.com.imageworks.spcue.dispatcher.RqdReportManagerService=DEBUG log4j.logger.com.imageworks.spcue.service.HostManagerService=TRACE log4j.logger.com.imageworks.spcue.dispatcher=TRACE -log4j.logger.com.imageworks.spcue.dao.oracle.DispatcherDaoJdbc=DEBUG #log4j.logger.org.springframework=DEBUG diff --git a/cuebot/src/test/resources/opencue.properties b/cuebot/src/test/resources/opencue.properties index 0a59548ce..38dc15ea9 100644 --- a/cuebot/src/test/resources/opencue.properties +++ b/cuebot/src/test/resources/opencue.properties @@ -1,16 +1,6 @@ cue.proxy = tcp -h cuetest01-vm -p 9019 -t 10000:tcp -h cuetest02-vm -p 9019 -t 10000:tcp -h cuetest03-vm -p 9019 -t 10000 spring.velocity.checkTemplateLocation=false -cue.trackit.enabled=false -# If using Oracle trackit, ensure the drivers are installed and use the following config value: -# datasource.trackit-data-source.driver-class-name=oracle.jdbc.OracleDriver -datasource.trackit-data-source.jdbc-url=jdbc:oracle:oci:@dbname -datasource.trackit-data-source.username=element_ro -datasource.trackit-data-source.password=password -# Discard connections after 6 hours, this allows for gradual -# connection rebalancing. -datasource.trackit-data-source.max-age=21600000 - grpc.cue_port=8453 grpc.rqd_server_port=${CUEBOT_GRPC_RQD_SERVER_PORT:50051} grpc.max_message_bytes=104857600 diff --git a/cuegui/cuegui/MenuActions.py b/cuegui/cuegui/MenuActions.py index 5c9b63139..75f1bbc4a 100644 --- a/cuegui/cuegui/MenuActions.py +++ b/cuegui/cuegui/MenuActions.py @@ -1341,7 +1341,6 @@ def delete(self, rpcObjects=None): body, [host.data.name for host in hosts]): for host in hosts: - # Delete current render partitions to avoid oracle exception for rp in host.getRenderPartitions(): rp.delete() From bb2c25290c907455e1943db6544a1bfcfe018ac2 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Sun, 25 Apr 2021 13:34:25 -0400 Subject: [PATCH 086/277] Added unit tests for cuegui.UnbookDialog. (#929) --- ci/pylintrc_test | 1 + cuegui/cuegui/UnbookDialog.py | 34 +-- cuegui/tests/UnbookDialog_tests.py | 288 +++++++++++++++++++++++++ pycue/opencue/api.py | 2 +- pycue/opencue/search.py | 5 + pycue/opencue/wrappers/proc.py | 24 +++ pycue/opencue/wrappers/subscription.py | 6 + 7 files changed, 344 insertions(+), 16 deletions(-) create mode 100644 cuegui/tests/UnbookDialog_tests.py diff --git a/ci/pylintrc_test b/ci/pylintrc_test index e4f09c50d..4116f7e5f 100644 --- a/ci/pylintrc_test +++ b/ci/pylintrc_test @@ -70,6 +70,7 @@ disable=arguments-differ, no-self-use, protected-access, raise-missing-from, + too-many-arguments, too-many-lines, too-many-locals, too-many-public-methods, diff --git a/cuegui/cuegui/UnbookDialog.py b/cuegui/cuegui/UnbookDialog.py index e36d3e6ac..b392e3d4a 100644 --- a/cuegui/cuegui/UnbookDialog.py +++ b/cuegui/cuegui/UnbookDialog.py @@ -27,6 +27,7 @@ from PySide2 import QtCore from PySide2 import QtWidgets +import six import opencue @@ -163,22 +164,23 @@ def _convert(val): return int(convert(float(val))) if isinstance(mixed, (float, int)): - result = opencue.api.criterion_pb2.GreaterThanIntegerSearchCriterion(_convert(mixed)) - elif isinstance(mixed, str): + result = opencue.api.criterion_pb2.GreaterThanIntegerSearchCriterion( + value=_convert(mixed)) + elif isinstance(mixed, six.string_types): if mixed.startswith("gt"): result = opencue.api.criterion_pb2.GreaterThanIntegerSearchCriterion( - _convert(mixed[2:])) + value=_convert(mixed[2:])) elif mixed.startswith("lt"): result = opencue.api.criterion_pb2.LessThanIntegerSearchCriterion( - _convert(mixed[2:])) + value=_convert(mixed[2:])) elif mixed.find("-") > -1: min_frame, max_frame = mixed.split("-", 1) result = opencue.api.criterion_pb2.InRangeIntegerSearchCriterion( - _convert(min_frame), _convert(max_frame)) + min=_convert(min_frame), max=_convert(max_frame)) else: try: result = opencue.api.criterion_pb2.GreaterThanIntegerSearchCriterion( - _convert(mixed)) + value=_convert(mixed)) except ValueError: raise Exception("invalid int search input value: " + str(mixed)) elif issubclass(mixed.__class__, opencue.api.criterion_pb2.EqualsIntegerSearchCriterion): @@ -195,15 +197,17 @@ def accept(self): self.close() procSearch = opencue.search.ProcSearch() - procSearch.maxResults = [int(self.__amount.value())] - procSearch.jobs = self.__jobs - procSearch.allocs = [str(checkedBox.text()) for checkedBox in self.__matrix.checkedBoxes()] + procSearch.options['maxResults'] = [int(self.__amount.value())] + procSearch.options['jobs'] = self.__jobs + procSearch.options['allocs'] = [ + str(checkedBox.text()) for checkedBox in self.__matrix.checkedBoxes()] memoryRange = self.__memoryRangeBox.result() if memoryRange: - procSearch.memoryRange = self.handleIntCriterion(memoryRange, lambda mb: (mb*1024)) + procSearch.options['memoryRange'] = self.handleIntCriterion( + memoryRange, lambda mb: (mb*1024)) runtimeRange = self.__runtimeRangeBox.result() if runtimeRange: - procSearch.durationRange = self.handleIntCriterion( + procSearch.options['durationRange'] = self.handleIntCriterion( runtimeRange, lambda rangeMin: (rangeMin*60)) if self.__redirect.isChecked(): @@ -253,7 +257,7 @@ def accept(self): group = groups[str(group)] if job or group: - procs = opencue.api.getProcs(procSearch) + procs = opencue.api.getProcs(**procSearch.options) kill = self.__kill.isChecked() amount = 0 @@ -277,7 +281,7 @@ def accept(self): if dialog.result(): self.close() else: - procs = opencue.api.getProcs(procSearch) + procs = opencue.api.getProcs(**procSearch.options) amount = 0 for proc in procs: try: @@ -400,7 +404,7 @@ def __init__(self, procSearch, parent=None): self.setWindowTitle("Unbook and kill frames?") self.__procSearch = procSearch - self.__procs = opencue.api.getProcs(procSearch) + self.__procs = opencue.api.getProcs(**procSearch.options) self.__amount = len(self.__procs) if self.__amount == 1: @@ -413,7 +417,7 @@ def __init__(self, procSearch, parent=None): self.__jobList = QtWidgets.QTextEdit(self) self.__jobList.setText( "\n".join( - ["%s %s" % (proc.data.jobName, proc.data.frameName) for proc in self.__procs])) + ["%s %s" % (proc.data.job_name, proc.data.frame_name) for proc in self.__procs])) self.__jobList.setReadOnly(True) self.__jobList.setSizePolicy( QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Maximum)) diff --git a/cuegui/tests/UnbookDialog_tests.py b/cuegui/tests/UnbookDialog_tests.py new file mode 100644 index 000000000..2482aa201 --- /dev/null +++ b/cuegui/tests/UnbookDialog_tests.py @@ -0,0 +1,288 @@ +# Copyright (c) OpenCue Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Tests for cuegui.UnbookDialog.""" + + +import unittest + +import mock + +import PySide2.QtCore +import PySide2.QtGui + +import opencue.compiled_proto.criterion_pb2 +import opencue.compiled_proto.host_pb2 +import opencue.compiled_proto.job_pb2 +import opencue.compiled_proto.show_pb2 +import opencue.compiled_proto.subscription_pb2 +import opencue.wrappers.group +import opencue.wrappers.job +import opencue.wrappers.proc +import opencue.wrappers.show +import opencue.wrappers.subscription + +import cuegui.Style +import cuegui.UnbookDialog + +from . import test_utils + + +@mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) +class UnbookDialogTests(unittest.TestCase): + + @mock.patch('opencue.api.findShow') + @mock.patch('opencue.cuebot.Cuebot.getStub') + def setUp(self, get_stub_mock, find_show_mock): + test_utils.createApplication() + PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + cuegui.Style.init() + + show_name = 'showname' + self.job_names = [ + '%s-shotname-username_job1' % show_name, '%s-shotname-username_job2' % show_name] + self.jobs = [ + opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(name=self.job_names[0])), + opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(name=self.job_names[1]))] + show = opencue.wrappers.show.Show(opencue.compiled_proto.show_pb2.Show(name=show_name)) + self.tag_names = ['general', 'desktop'] + subscriptions = [ + opencue.wrappers.subscription.Subscription( + opencue.compiled_proto.subscription_pb2.Subscription( + name='local.%s.%s' % (self.tag_names[0], show_name))), + opencue.wrappers.subscription.Subscription( + opencue.compiled_proto.subscription_pb2.Subscription( + name='local.%s.%s' % (self.tag_names[1], show_name))), + ] + show.getSubscriptions = mock.Mock() + show.getSubscriptions.return_value = subscriptions + find_show_mock.return_value = show + + self.dialog = cuegui.UnbookDialog.UnbookDialog(self.jobs) + + def test__should_show_all_jobs_and_subscriptions(self): + self.dialog.open() + + jobs_shown = self.dialog._UnbookDialog__jobList.toPlainText().split() + self.assertEqual(self.job_names, jobs_shown) + subscription_matrix = self.dialog._UnbookDialog__matrix + subscriptions_shown = [ + checkbox.text() + for checkbox in subscription_matrix._CheckBoxSelectionMatrix__checkBoxes] + subscriptions_checked = subscription_matrix.checkedOptions() + self.assertEqual(self.tag_names, subscriptions_shown) + self.assertEqual(self.tag_names, subscriptions_checked) + + @mock.patch('PySide2.QtWidgets.QMessageBox', new=mock.Mock()) + @mock.patch('opencue.api.getProcs') + def test__should_unbook_procs(self, get_procs_mock): + num_procs = 17 + min_mem = 56 + max_mem = 143 + expected_proc_search = opencue.search.ProcSearch( + allocs=self.tag_names, jobs=self.job_names, maxResults=[num_procs], + memoryRange=[opencue.compiled_proto.criterion_pb2.InRangeIntegerSearchCriterion( + min=min_mem*1024, max=max_mem*1024)]) + returned_proc1 = opencue.wrappers.proc.Proc() + returned_proc1.unbook = mock.Mock() + returned_proc2 = opencue.wrappers.proc.Proc() + returned_proc2.unbook = mock.Mock() + get_procs_mock.return_value = [returned_proc1, returned_proc2] + + self.dialog.open() + self.dialog._UnbookDialog__amount.setValue(num_procs) + self.dialog._UnbookDialog__memoryRangeBox._RangeBox__group.setChecked(True) + self.dialog._UnbookDialog__memoryRangeBox._RangeBox__range.setChecked(True) + self.dialog._UnbookDialog__memoryRangeBox._RangeBox__min.setValue(min_mem) + self.dialog._UnbookDialog__memoryRangeBox._RangeBox__max.setValue(max_mem) + self.dialog.accept() + + get_procs_mock.assert_called_with(**expected_proc_search.options) + returned_proc1.unbook.assert_called() + returned_proc2.unbook.assert_called() + + @mock.patch('cuegui.UnbookDialog.KillConfirmationDialog') + def test__should_show_kill_confirmation_dialog(self, kill_dialog_mock): + num_procs = 2 + min_runtime = 90 + max_runtime = 105 + expected_proc_search = opencue.search.ProcSearch( + allocs=self.tag_names, jobs=self.job_names, maxResults=[num_procs], + durationRange=[opencue.compiled_proto.criterion_pb2.InRangeIntegerSearchCriterion( + min=min_runtime*60, max=max_runtime*60)]) + kill_dialog_mock.return_value.result.return_value = True + + self.dialog.open() + self.dialog._UnbookDialog__amount.setValue(num_procs) + self.dialog._UnbookDialog__kill.setChecked(True) + self.dialog._UnbookDialog__runtimeRangeBox._RangeBox__group.setChecked(True) + self.dialog._UnbookDialog__runtimeRangeBox._RangeBox__range.setChecked(True) + self.dialog._UnbookDialog__runtimeRangeBox._RangeBox__min.setValue(min_runtime) + self.dialog._UnbookDialog__runtimeRangeBox._RangeBox__max.setValue(max_runtime) + self.dialog.accept() + + kill_dialog_mock.assert_called_with(expected_proc_search, mock.ANY) + + @mock.patch('PySide2.QtWidgets.QMessageBox', new=mock.Mock()) + @mock.patch('opencue.api.getProcs') + @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('opencue.api.getActiveShows') + def test__should_redirect_proc_to_group( + self, get_active_shows_mock, get_item_mock, get_procs_mock): + num_procs = 50 + other_show_name = 'some-other-show' + group_name = 'group-to-redirect-to' + show = opencue.wrappers.show.Show( + opencue.compiled_proto.show_pb2.Show(name=other_show_name)) + group = opencue.wrappers.group.Group(opencue.compiled_proto.job_pb2.Group(name=group_name)) + show.getGroups = mock.Mock() + show.getGroups.return_value = [group] + get_active_shows_mock.return_value = [show] + get_item_mock.side_effect = [(other_show_name, True), ('Group', True), (group_name, True)] + expected_proc_search = opencue.search.ProcSearch( + allocs=self.tag_names, jobs=self.job_names, maxResults=[num_procs]) + proc_to_redirect = opencue.wrappers.proc.Proc() + proc_to_redirect.redirectToGroup = mock.Mock() + get_procs_mock.return_value = [proc_to_redirect] + + self.dialog.open() + self.dialog._UnbookDialog__amount.setValue(num_procs) + self.dialog._UnbookDialog__redirect.setChecked(True) + self.dialog.accept() + + get_procs_mock.assert_called_with(**expected_proc_search.options) + proc_to_redirect.redirectToGroup.assert_called_with(group, False) + + @mock.patch('PySide2.QtWidgets.QMessageBox', new=mock.Mock()) + @mock.patch('opencue.api.getProcs') + @mock.patch('cuegui.UnbookDialog.SelectItemsWithSearchDialog') + @mock.patch('opencue.api.getJobs') + @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('opencue.api.getActiveShows') + def test__should_redirect_proc_to_job( + self, get_active_shows_mock, get_item_mock, get_jobs_mock, select_job_mock, + get_procs_mock): + num_procs = 50 + other_show_name = 'some-other-show' + job_name = 'job-to-redirect-to' + show = opencue.wrappers.show.Show( + opencue.compiled_proto.show_pb2.Show(name=other_show_name)) + job = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(name=job_name)) + get_active_shows_mock.return_value = [show] + get_item_mock.side_effect = [(other_show_name, True), ('Job', True)] + get_jobs_mock.return_value = [job] + select_job_mock.return_value.selected.return_value = [job_name] + expected_proc_search = opencue.search.ProcSearch( + allocs=self.tag_names, jobs=self.job_names, maxResults=[num_procs]) + proc_to_redirect = opencue.wrappers.proc.Proc() + proc_to_redirect.redirectToJob = mock.Mock() + get_procs_mock.return_value = [proc_to_redirect] + + self.dialog.open() + self.dialog._UnbookDialog__amount.setValue(num_procs) + self.dialog._UnbookDialog__redirect.setChecked(True) + self.dialog.accept() + + get_procs_mock.assert_called_with(**expected_proc_search.options) + proc_to_redirect.redirectToJob.assert_called_with(job, False) + + +class SelectItemsWithSearchDialogTests(unittest.TestCase): + + def setUp(self): + test_utils.createApplication() + PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + cuegui.Style.init() + + def test__should_display_all_items(self): + items_to_be_shown = ['item1', 'item2', 'item3'] + + dialog = cuegui.UnbookDialog.SelectItemsWithSearchDialog(None, 'header', items_to_be_shown) + dialog.open() + + item_list = dialog._SelectItemsWithSearchDialog__widget._SelectItemsWithSearchWidget__list + all_items_shown = [item_list.item(x).text() for x in range(item_list.count())] + self.assertEqual(items_to_be_shown, all_items_shown) + + def test__should_filter_items(self): + initial_item_list = ['item1', 'itemSubstr2', 'item3', 'itemsubstr4'] + + dialog = cuegui.UnbookDialog.SelectItemsWithSearchDialog(None, 'header', initial_item_list) + dialog.open() + dialog._SelectItemsWithSearchDialog__widget._SelectItemsWithSearchWidget__filter.setText( + 'substr') + + item_list = dialog._SelectItemsWithSearchDialog__widget._SelectItemsWithSearchWidget__list + items_shown = [item_list.item(x).text() for x in range(item_list.count())] + self.assertEqual(['itemSubstr2', 'itemsubstr4'], items_shown) + + def test__should_return_selected_items(self): + initial_item_list = ['item1', 'item2', 'item3', 'item4'] + + dialog = cuegui.UnbookDialog.SelectItemsWithSearchDialog(None, 'header', initial_item_list) + dialog.open() + item_list = dialog._SelectItemsWithSearchDialog__widget._SelectItemsWithSearchWidget__list + item_list.item(1).setSelected(True) + item_list.item(2).setSelected(True) + dialog.accept() + + self.assertEqual(['item2', 'item3'], dialog.selected()) + + +@mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) +class KillConfirmationDialogTests(unittest.TestCase): + + def setUp(self): + test_utils.createApplication() + PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + cuegui.Style.init() + + @mock.patch('PySide2.QtWidgets.QMessageBox.information', new=mock.Mock()) + @mock.patch('opencue.api.getProcs') + def test__should_kill_procs(self, get_procs_mock): + proc_search = opencue.search.ProcSearch( + allocs=['tag1', 'tag2'], jobs=['someJob', 'anotherJob'], maxResults=[57]) + proc1 = opencue.wrappers.proc.Proc( + opencue.compiled_proto.host_pb2.Proc(job_name='someJob', frame_name='0002')) + proc1.kill = mock.Mock() + proc2 = opencue.wrappers.proc.Proc( + opencue.compiled_proto.host_pb2.Proc(job_name='anotherJob', frame_name='2847')) + proc2.kill = mock.Mock() + get_procs_mock.return_value = [proc1, proc2] + + dialog = cuegui.UnbookDialog.KillConfirmationDialog(proc_search) + dialog.accept() + + proc1.kill.assert_called() + proc2.kill.assert_called() + + @mock.patch('PySide2.QtWidgets.QMessageBox.information', new=mock.Mock()) + @mock.patch('opencue.api.getProcs') + def test__should_cancel_kill(self, get_procs_mock): + proc_search = opencue.search.ProcSearch( + allocs=['tag1', 'tag2'], jobs=['someJob', 'anotherJob'], maxResults=[57]) + proc1 = opencue.wrappers.proc.Proc( + opencue.compiled_proto.host_pb2.Proc(job_name='someJob', frame_name='0002')) + proc1.kill = mock.Mock() + proc2 = opencue.wrappers.proc.Proc( + opencue.compiled_proto.host_pb2.Proc(job_name='anotherJob', frame_name='2847')) + proc2.kill = mock.Mock() + get_procs_mock.return_value = [proc1, proc2] + + dialog = cuegui.UnbookDialog.KillConfirmationDialog(proc_search) + dialog.reject() + + proc1.kill.assert_not_called() + proc2.kill.assert_not_called() diff --git a/pycue/opencue/api.py b/pycue/opencue/api.py index 10dbed4cc..26d5ff119 100644 --- a/pycue/opencue/api.py +++ b/pycue/opencue/api.py @@ -742,7 +742,7 @@ def getProcs(**options): - "lt5" is less than 5 hours - "5-10" is range of 5 to 10 hours - :rtype: list + :rtype: list[opencue.wrapper.proc.Proc] :return: a list of Proc objects""" procSeq = search.ProcSearch.byOptions(**options).procs return [Proc(p) for p in procSeq.procs] diff --git a/pycue/opencue/search.py b/pycue/opencue/search.py index 995cf6e51..1a42f696a 100644 --- a/pycue/opencue/search.py +++ b/pycue/opencue/search.py @@ -77,6 +77,11 @@ class BaseSearch(object): def __init__(self, **options): self.options = options + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return self.options == other.options + def search(self): """Executes the search using the options provided at initiation.""" return self.byOptions(**self.options) diff --git a/pycue/opencue/wrappers/proc.py b/pycue/opencue/wrappers/proc.py index da8ab2fae..f0ab1bc96 100644 --- a/pycue/opencue/wrappers/proc.py +++ b/pycue/opencue/wrappers/proc.py @@ -59,6 +59,30 @@ def unbook(self, kill=False): host_pb2.ProcUnbookRequest(proc=self.data, kill=kill), timeout=Cuebot.Timeout) return response + def redirectToJob(self, job, kill=False): + """Unbooks the current frame from this proc and redirects the proc to a specific job. + + :type job: opencue.wrappers.job.Job + :param job: job which the proc should be booked to + :type kill: bool + :param kill: if true, the frame will be immediately killed + """ + self.stub.RedirectToJob( + host_pb2.ProcRedirectToJobRequest(proc=self.data, job_id=job.data.id, kill=kill), + timeout=Cuebot.Timeout) + + def redirectToGroup(self, group, kill=False): + """Unbooks the current frame from this proc and redirects the proc to another group. + + :type group: opencue.wrappers.group.Group + :param group: group which the proc should be booked to + :type kill: bool + :param kill: if true, the frame will be immediately killed + """ + self.stub.RedirectToGroup( + host_pb2.ProcRedirectToGroupRequest(proc=self.data, group_id=group.data.id, kill=kill), + timeout=Cuebot.Timeout) + def getHost(self): """Returns the host this proc is allocated from. diff --git a/pycue/opencue/wrappers/subscription.py b/pycue/opencue/wrappers/subscription.py index 057067e3c..5334ae413 100644 --- a/pycue/opencue/wrappers/subscription.py +++ b/pycue/opencue/wrappers/subscription.py @@ -94,6 +94,12 @@ def id(self): def name(self): """Returns the name of the subscription. + Subscription names follow the format `.`, which can also be + read as `..` due to the way allocations are named. For + example the subscription name `local.general.testing` would indicate the show `testing` + has a subscription to the `local.general` allocation. The `local.general` allocation + indicates all hosts in the facility `local` containing the `general` tag. + :rtype: str :return: name of the subscription """ From 847ee50c1732b268d55cd645d1d7c2d9e9e039da Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sun, 25 Apr 2021 10:39:52 -0700 Subject: [PATCH 087/277] Add GetDefault and SetDefault to AllcationInterface (#939) --- VERSION.in | 2 +- .../spcue/servant/ManageAllocation.java | 26 +++++++++++++++++ .../spcue/service/AdminManager.java | 2 ++ .../spcue/service/AdminManagerService.java | 11 ++++++++ .../spcue/test/service/AdminManagerTests.java | 28 +++++++++++++++++++ proto/facility.proto | 22 ++++++++++++++- pycue/opencue/api.py | 22 +++++++++++++++ pycue/tests/api_test.py | 26 +++++++++++++++++ 8 files changed, 137 insertions(+), 2 deletions(-) diff --git a/VERSION.in b/VERSION.in index 51176c7c8..c43e1055f 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.11 +0.12 diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageAllocation.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageAllocation.java index 1857cc653..665b7d79e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageAllocation.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageAllocation.java @@ -24,6 +24,8 @@ import com.imageworks.spcue.grpc.facility.AllocFindResponse; import com.imageworks.spcue.grpc.facility.AllocGetAllRequest; import com.imageworks.spcue.grpc.facility.AllocGetAllResponse; +import com.imageworks.spcue.grpc.facility.AllocGetDefaultRequest; +import com.imageworks.spcue.grpc.facility.AllocGetDefaultResponse; import com.imageworks.spcue.grpc.facility.AllocGetHostsRequest; import com.imageworks.spcue.grpc.facility.AllocGetHostsResponse; import com.imageworks.spcue.grpc.facility.AllocGetRequest; @@ -34,6 +36,8 @@ import com.imageworks.spcue.grpc.facility.AllocReparentHostsResponse; import com.imageworks.spcue.grpc.facility.AllocSetBillableRequest; import com.imageworks.spcue.grpc.facility.AllocSetBillableResponse; +import com.imageworks.spcue.grpc.facility.AllocSetDefaultRequest; +import com.imageworks.spcue.grpc.facility.AllocSetDefaultResponse; import com.imageworks.spcue.grpc.facility.AllocSetNameRequest; import com.imageworks.spcue.grpc.facility.AllocSetNameResponse; import com.imageworks.spcue.grpc.facility.AllocSetTagRequest; @@ -221,6 +225,28 @@ public void setTag( responseObserver.onCompleted(); } + @Override + public void getDefault( + AllocGetDefaultRequest request, + StreamObserver responseObserver) { + AllocationEntity alloc = adminManager.getDefaultAllocation(); + responseObserver.onNext(AllocGetDefaultResponse.newBuilder() + .setAllocation(whiteboard.getAllocation(alloc.id)) + .build()); + responseObserver.onCompleted(); + } + + @Override + public void setDefault( + AllocSetDefaultRequest request, + StreamObserver responseObserver) { + AllocationEntity alloc = adminManager.findAllocationDetail( + request.getAllocation().getFacility(), request.getAllocation().getName()); + adminManager.setDefaultAllocation(alloc); + responseObserver.onNext(AllocSetDefaultResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + public AdminManager getAdminManager() { return adminManager; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/AdminManager.java b/cuebot/src/main/java/com/imageworks/spcue/service/AdminManager.java index dd7b914cd..f6d2220df 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/AdminManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/AdminManager.java @@ -56,6 +56,8 @@ public interface AdminManager { void deleteAllocation(AllocationInterface alloc); void setAllocationName(AllocationInterface a, String name); void setAllocationTag(AllocationInterface a, String tag); + AllocationEntity getDefaultAllocation(); + void setDefaultAllocation(AllocationInterface a); AllocationEntity findAllocationDetail(String facility, String name); AllocationEntity getAllocationDetail(String id); void setAllocationBillable(AllocationInterface alloc, boolean value); diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java index 193de98a5..f99107270 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java @@ -106,6 +106,17 @@ public void setAllocationTag(AllocationInterface a, String tag) { allocationDao.updateAllocationTag(a, tag); } + @Override + @Transactional(propagation = Propagation.REQUIRED, readOnly=true) + public AllocationEntity getDefaultAllocation() { + return allocationDao.getDefaultAllocationEntity(); + } + + @Override + public void setDefaultAllocation(AllocationInterface a) { + allocationDao.setDefaultAllocation(a); + } + @Override @Transactional(propagation = Propagation.REQUIRED, readOnly=true) public ShowEntity findShowEntity(String name) { diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/AdminManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/AdminManagerTests.java index 8f1d24ffe..c4538f79f 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/AdminManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/AdminManagerTests.java @@ -65,6 +65,34 @@ public void createAllocation() { adminManager.createAllocation(facilityDao.getDefaultFacility(), a); } + @Test + @Transactional + @Rollback(true) + public void deleteAllocation() { + AllocationEntity a = new AllocationEntity(); + a.name = facilityDao.getDefaultFacility().getName() + "." + TEST_ALLOC_NAME; + a.tag = "general"; + adminManager.createAllocation(facilityDao.getDefaultFacility(), a); + adminManager.deleteAllocation(a); + } + + @Test + @Transactional + @Rollback(true) + public void setDefaultAllocation() { + AllocationEntity a = adminManager.getDefaultAllocation(); + assertEquals(a.name, facilityDao.getDefaultFacility().getName() + ".unassigned"); + + a = new AllocationEntity(); + a.name = TEST_ALLOC_NAME; + a.tag = "general"; + adminManager.createAllocation(facilityDao.getDefaultFacility(), a); + adminManager.setDefaultAllocation(a); + + a = adminManager.getDefaultAllocation(); + assertEquals(a.name, facilityDao.getDefaultFacility().getName() + "." + TEST_ALLOC_NAME); + } + @Test @Transactional @Rollback(true) diff --git a/proto/facility.proto b/proto/facility.proto index 1b427c6ca..49c95537c 100644 --- a/proto/facility.proto +++ b/proto/facility.proto @@ -62,6 +62,12 @@ service AllocationInterface { // Set the allocation tag. Setting this will re-tag all the hosts in this allocation. rpc SetTag(AllocSetTagRequest) returns (AllocSetTagResponse); + + // Return the default allocation. + rpc GetDefault(AllocGetDefaultRequest) returns (AllocGetDefaultResponse); + + // Set the default allocation. + rpc SetDefault(AllocSetDefaultRequest) returns (AllocSetDefaultResponse); } @@ -245,4 +251,18 @@ message AllocSetTagRequest { string tag = 2; } -message AllocSetTagResponse {} // +message AllocSetTagResponse {} // Empty + +// GetDefault +message AllocGetDefaultRequest {} // Empty + +message AllocGetDefaultResponse { + Allocation allocation = 1; +} + +// SetDefault +message AllocSetDefaultRequest { + Allocation allocation = 1; +} + +message AllocSetDefaultResponse {} // Empty diff --git a/pycue/opencue/api.py b/pycue/opencue/api.py index 26d5ff119..df101733c 100644 --- a/pycue/opencue/api.py +++ b/pycue/opencue/api.py @@ -644,6 +644,28 @@ def deleteAllocation(alloc): facility_pb2.AllocDeleteRequest(allocation=alloc), timeout=Cuebot.Timeout) +@util.grpcExceptionParser +def getDefaultAllocation(): + """Get the default allocation. + + :rtype: Allocation + :return: an Allocation object""" + return Allocation(Cuebot.getStub('allocation').GetDefault( + facility_pb2.AllocGetDefaultRequest(), timeout=Cuebot.Timeout).allocation) + + +@util.grpcExceptionParser +def setDefaultAllocation(alloc): + """Set the default allocation. + + :type alloc: facility_pb2.Allocation + :param alloc: allocation to set default + :rtype: facility_pb2.AllocSetDefaultResponse + :return: empty response""" + return Cuebot.getStub('allocation').SetDefault( + facility_pb2.AllocSetDefaultRequest(allocation=alloc), timeout=Cuebot.Timeout) + + @util.grpcExceptionParser def allocSetBillable(alloc, is_billable): """Sets an allocation billable or not. diff --git a/pycue/tests/api_test.py b/pycue/tests/api_test.py index 0d05246c4..cbd957869 100644 --- a/pycue/tests/api_test.py +++ b/pycue/tests/api_test.py @@ -719,6 +719,32 @@ def testDeleteAlloc(self, getStubMock): stubMock.Delete.assert_called_with( facility_pb2.AllocDeleteRequest(allocation=allocToDelete), timeout=mock.ANY) + @mock.patch('opencue.cuebot.Cuebot.getStub') + def testGetDefaultAlloc(self, getStubMock): + arbitraryId = '00000000-0000-0000-0000-012345678980' + stubMock = mock.Mock() + stubMock.GetDefault.return_value = facility_pb2.AllocGetDefaultResponse( + allocation=facility_pb2.Allocation(id=arbitraryId)) + getStubMock.return_value = stubMock + + alloc = opencue.api.getDefaultAllocation() + + stubMock.GetDefault.assert_called_with( + facility_pb2.AllocGetDefaultRequest(), timeout=mock.ANY) + self.assertEqual(arbitraryId, alloc.id()) + + @mock.patch('opencue.cuebot.Cuebot.getStub') + def testSetDefaultAlloc(self, getStubMock): + alloc = facility_pb2.Allocation(name=TEST_ALLOC_NAME) + stubMock = mock.Mock() + stubMock.SetDefault.return_value = facility_pb2.AllocSetDefaultResponse() + getStubMock.return_value = stubMock + + opencue.api.setDefaultAllocation(alloc) + + stubMock.SetDefault.assert_called_with( + facility_pb2.AllocSetDefaultRequest(allocation=alloc), timeout=mock.ANY) + @mock.patch('opencue.cuebot.Cuebot.getStub') def testAllocSetBillable(self, getStubMock): alloc = facility_pb2.Allocation(name=TEST_ALLOC_NAME) From 06d0b9a68f71e3cece6c66b4a5ea786464eef1f6 Mon Sep 17 00:00:00 2001 From: "Marcelo F. Bortolini" Date: Sun, 25 Apr 2021 19:42:52 +0200 Subject: [PATCH 088/277] CueGui - adjust minimum GPU memory value range. (#961) --- cuegui/cuegui/ServiceDialog.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cuegui/cuegui/ServiceDialog.py b/cuegui/cuegui/ServiceDialog.py index 7949f8f6b..0288168f5 100644 --- a/cuegui/cuegui/ServiceDialog.py +++ b/cuegui/cuegui/ServiceDialog.py @@ -42,7 +42,9 @@ def __init__(self, parent=None): QtWidgets.QWidget.__init__(self, parent) self.__service = None - self.gpu_max_mb = 2 * 1024 + # NOTE: As min_gpu value will be passed on later in KB, its max value in + # *KiloBytes* should not be higher than Int32(2147483647). + self.gpu_max_mb = int(2147483647 / 1024) self.gpu_min_mb = 0 self.gpu_tick_mb = 256 @@ -61,7 +63,7 @@ def __init__(self, parent=None): self.min_memory.setValue(3276) self.min_gpu = QtWidgets.QSpinBox(self) self.min_gpu.setRange(self.gpu_min_mb, self.gpu_max_mb) - self.min_gpu.setValue(0) + self.min_gpu.setValue(self.gpu_min_mb) self.min_gpu.setSingleStep(self.gpu_tick_mb) self.min_gpu.setSuffix(" MB") self.timeout = QtWidgets.QSpinBox(self) @@ -139,7 +141,7 @@ def new(self): self.min_cores.setValue(100) self.max_cores.setValue(100) self.min_memory.setValue(3276) - self.min_gpu.setValue(0) + self.min_gpu.setValue(self.gpu_min_mb) self.timeout.setValue(0) self.timeout_llu.setValue(0) self._tags_w.set_tags(['general']) From 92eabc2c533066e1ed0ab47a5f5ef92ca4d42448 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 26 Apr 2021 04:25:38 -0400 Subject: [PATCH 089/277] Skip tests needing xvfb-run in Python 2. (#965) * Skip tests needing xvfb-run in Python 2. * Whitespace. * Fix python command. --- ci/run_python_tests.sh | 9 ++++++++- cuegui/Dockerfile | 11 +---------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index db3f68fd0..14316b244 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -2,6 +2,9 @@ set -e +python_version=$(python -V) +echo "Will run tests using ${python_version}" + pip install --user -r requirements.txt -r requirements_gui.txt # Protos need to have their Python code generated in order for tests to pass. @@ -16,10 +19,14 @@ python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc python pycue/setup.py test PYTHONPATH=pycue python pyoutline/setup.py test PYTHONPATH=pycue python cueadmin/setup.py test -PYTHONPATH=pycue xvfb-run -d python cuegui/setup.py test PYTHONPATH=pycue:pyoutline python cuesubmit/setup.py test python rqd/setup.py test +# Xvfb no longer supports Python 2. +if [[ "$python_version" =~ "Python 3" ]]; then + PYTHONPATH=pycue xvfb-run -d python cuegui/setup.py test +fi + # Some environments don't have pylint available, for ones that do they should pass this flag. if [[ "$1" == "--lint" ]]; then cd pycue && python -m pylint --rcfile=../ci/pylintrc_main FileSequence && cd .. diff --git a/cuegui/Dockerfile b/cuegui/Dockerfile index 4804478f6..4a7e3728c 100644 --- a/cuegui/Dockerfile +++ b/cuegui/Dockerfile @@ -26,10 +26,8 @@ RUN yum -y install \ python36-devel \ python36-pip -RUN python -m pip install --upgrade 'pip<21' RUN python3.6 -m pip install --upgrade pip -RUN python -m pip install --upgrade 'setuptools<45' RUN python3.6 -m pip install --upgrade setuptools RUN dbus-uuidgen > /etc/machine-id @@ -38,7 +36,6 @@ COPY LICENSE ./ COPY requirements.txt ./ COPY requirements_gui.txt ./ -RUN python -m pip install -r requirements.txt -r requirements_gui.txt RUN python3.6 -m pip install -r requirements.txt -r requirements_gui.txt COPY proto/ ./proto @@ -47,7 +44,7 @@ COPY pycue/setup.py ./pycue/ COPY pycue/FileSequence ./pycue/FileSequence COPY pycue/opencue ./pycue/opencue -RUN python -m grpc_tools.protoc \ +RUN python3.6 -m grpc_tools.protoc \ -I=./proto \ --python_out=./pycue/opencue/compiled_proto \ --grpc_python_out=./pycue/opencue/compiled_proto \ @@ -65,14 +62,8 @@ COPY cuegui/cuegui ./cuegui/cuegui COPY VERSION.in VERSIO[N] ./ RUN test -e VERSION || echo "$(cat VERSION.in)-custom" | tee VERSION -RUN cd pycue && python setup.py install - RUN cd pycue && python3.6 setup.py install -# TODO(bcipriano) Lint the code here. (Issue #78) - -RUN cd cuegui && xvfb-run -d python setup.py test - RUN cd cuegui && xvfb-run -d python3.6 setup.py test RUN cp LICENSE requirements.txt requirements_gui.txt VERSION cuegui/ From 3611b765bc952f5c1141921f17332e7c8cec62af Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 26 Apr 2021 11:42:00 -0400 Subject: [PATCH 090/277] Notes from Apr 14 TSC meeting. (#962) --- tsc/meetings/2021-04-14.md | 96 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 tsc/meetings/2021-04-14.md diff --git a/tsc/meetings/2021-04-14.md b/tsc/meetings/2021-04-14.md new file mode 100644 index 000000000..48fa86ac5 --- /dev/null +++ b/tsc/meetings/2021-04-14.md @@ -0,0 +1,96 @@ +# OpenCue TSC Meeting Notes 14 Apr 2021 + +Secretary: Brian Cipriano + +TSC Attendees: + +* [ ] Matt Chambers +* [x] Brian Cipriano +* [x] Greg Denton +* [x] Ben Dines +* [ ] Dave Fellows +* [x] Daniel Neilson +* [ ] Alex Schworer +* [ ] Erik Strauss +* [x] Lars van der Bijl + +Committer Attendees: + +* [ ] Christian Smith +* [x] Diego Tavares +* [x] Idris Miles + +Agenda/Notes: + +* Goals for 2021 + * ~~User survey~~ + * ASWF Graduation + * Waiting on TAC to discuss. OpenCue deep-dive session scheduled for next week's meeting, + was bumped from last week due to guest speakers. + * New user UX + * Rename demo_data.sql + * Main code change done. + * Docs updated. + * Still todo: updating release pipeline to publish seed_data artifact. + * Publish PyPI packages + * Design + doc: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/ + * Reviewed design doc, no issues, let's proceed. + * Fix cuebot:latest tag for sandbox setup + * Proposal: publish Docker images for every new patch version (i.e. every new commit to + master) + * Proposal accepted. + * No progress implementing this yet. + * Related: for PyPI design we may want to also publish packages for every new patch + version. + * Docs refresh + * User guide refresh + * Split into "install from latest release" and "install from master" + * Needed to assist transition to publish Docker images on all commits + * No progress. + * API reference + * We have the ability to generate HTML from any commit, but we need a new CI pipeline to + build and publish this on opencue.io for new releases. + * Proposal: for now, highlight existing docs for building reference locally + * Proposal accepted. + * Brian: two PRs out to clean up this process, then we can circulate the doc page on + this: https://www.opencue.io/contributing/opencue/build-docs/ + * pyoutline examples + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/177 + * No progress yet. + * User guide for CueGUI + * No progress. + * Configuration guide for Cuebot + * No progress. + * Configuration guide for RQD + * No progress. + * PyPI design doc may cover this, design includes a review of all Python components' + config systems and requires documentation of each of them before publishing to PyPI. + * Drop Oracle support + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/840 + * PR in review: https://github.com/AcademySoftwareFoundation/OpenCue/pull/895 + * Ready to merge, needs rebase. + * GPU support + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/459 + * Starting PR in review: https://github.com/AcademySoftwareFoundation/OpenCue/issues/459 + * In review, troubleshooting test failures. + * ~~GUI to add new shows~~ + * ~~Done.~~ + * Expand DCC plugins + * No progress. + * Logging solution + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/880 + * PR approved and merged. + * Proposal: write blog post on this topic + * Proposal accepted + * No progress on this yet + * CSP Terraform docs + * No progress yet. + * Auto-close Github issues + * No progress. +* Other current work updates? + * SPI grpc fork bug + * Seeing errors + like `Warn: Fork support is only compatible with the epoll1 and poll polling strategies` + * Fix: Set `os.environ["GRPC_POLL_STRATEGY"] = "epoll1"` + * Lars working on some docker compose updates. From 46b7b09472b155df9b60461eb2e7e3dbcf27ac0c Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 26 Apr 2021 11:42:14 -0400 Subject: [PATCH 091/277] Ignore compiled protobuf code in Cuebot docker build. (#963) --- .dockerignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index 567708f05..ff43d450d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,7 @@ ci +cuebot/src/compiled_protobuf images samples sandbox/db-data tsc -venv \ No newline at end of file +venv From 895254a22fcc69bf714a48e4b80a2cdc88f10c0b Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Wed, 28 Apr 2021 07:15:37 -0700 Subject: [PATCH 092/277] Use findAllocationDetail for setDefault (#966) --- .../spcue/servant/ManageAllocation.java | 2 +- .../test/servant/ManageAllocationTests.java | 27 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageAllocation.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageAllocation.java index 665b7d79e..46edc2025 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageAllocation.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageAllocation.java @@ -240,7 +240,7 @@ public void getDefault( public void setDefault( AllocSetDefaultRequest request, StreamObserver responseObserver) { - AllocationEntity alloc = adminManager.findAllocationDetail( + AllocationEntity alloc = findAllocationDetail( request.getAllocation().getFacility(), request.getAllocation().getName()); adminManager.setDefaultAllocation(alloc); responseObserver.onNext(AllocSetDefaultResponse.newBuilder().build()); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/servant/ManageAllocationTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/servant/ManageAllocationTests.java index c63f292c6..ed2a83cd6 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/servant/ManageAllocationTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/servant/ManageAllocationTests.java @@ -30,6 +30,7 @@ import org.springframework.test.context.support.AnnotationConfigContextLoader; import org.springframework.transaction.annotation.Transactional; +import com.imageworks.spcue.AllocationEntity; import com.imageworks.spcue.config.TestAppConfig; import com.imageworks.spcue.dao.AllocationDao; import com.imageworks.spcue.dao.FacilityDao; @@ -37,6 +38,8 @@ import com.imageworks.spcue.grpc.facility.AllocCreateResponse; import com.imageworks.spcue.grpc.facility.AllocDeleteRequest; import com.imageworks.spcue.grpc.facility.AllocDeleteResponse; +import com.imageworks.spcue.grpc.facility.AllocSetDefaultRequest; +import com.imageworks.spcue.grpc.facility.AllocSetDefaultResponse; import com.imageworks.spcue.grpc.facility.Allocation; import com.imageworks.spcue.grpc.facility.Facility; import com.imageworks.spcue.servant.ManageAllocation; @@ -122,6 +125,30 @@ public void testDelete() { "Incorrect result size: expected 1, actual 0"); } } + + @Test + @Transactional + @Rollback(true) + public void testSetDefault() { + AllocationEntity alloc = allocationDao.getDefaultAllocationEntity(); + assertEquals(alloc.getName(), "lax.unassigned"); + + Allocation allocation = Allocation.newBuilder() + .setName("spi.general") + .setTag("general") + .setFacility("spi") + .build(); + AllocSetDefaultRequest request = AllocSetDefaultRequest.newBuilder() + .setAllocation(allocation) + .build(); + + FakeStreamObserver observer = + new FakeStreamObserver(); + manageAllocation.setDefault(request, observer); + + alloc = allocationDao.getDefaultAllocationEntity(); + assertEquals(alloc.getName(), "spi.general"); + } } From 9037dcb573d8047608c063b41fc34b5bc3824d07 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sun, 2 May 2021 12:22:28 -0700 Subject: [PATCH 093/277] Remove unused checkRetries API method. (#970) --- .../java/com/imageworks/spcue/dao/FrameDao.java | 9 --------- .../spcue/dao/postgres/FrameDaoJdbc.java | 15 --------------- .../spcue/test/dao/postgres/FrameDaoTests.java | 9 --------- 3 files changed, 33 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java index e576a0f59..6f9fc8016 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java @@ -71,15 +71,6 @@ public interface FrameDao { */ public FrameDetail findLongestFrame(JobInterface job); - /** - * Checks to see how many retries a frame has. If that number - * is greater than or equal to the jobs max retries, the frame - * is marked as dead. - * - * @param frame - */ - void checkRetries(FrameInterface frame); - /** * Batch inserts a frameSet of frames. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java index 5c043e995..d7aeef3b5 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java @@ -647,21 +647,6 @@ public FrameInterface findFrame(JobInterface job, String name) { FRAME_MAPPER, name, job.getJobId()); } - @Override - public void checkRetries(FrameInterface frame) { - int max_retries = getJdbcTemplate().queryForObject( - "SELECT int_max_retries FROM job WHERE pk_job=?", Integer.class, - frame.getJobId()); - - if (getJdbcTemplate().queryForObject( - "SELECT int_retries FROM frame WHERE pk_frame=?", Integer.class, - frame.getFrameId()) >= max_retries) { - getJdbcTemplate().update( - "UPDATE frame SET str_state=? WHERE pk_frame=?", - FrameState.DEAD.toString(), frame.getFrameId()); - } - } - private static final String UPDATE_FRAME_STATE = "UPDATE " + "frame "+ diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/FrameDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/FrameDaoTests.java index 602b68b96..c18c78a28 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/FrameDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/FrameDaoTests.java @@ -145,15 +145,6 @@ public JobDetail launchJob() { return jobManager.findJobDetail("pipe-dev.cue-testuser_shell_v1"); } - @Test - @Transactional - @Rollback(true) - public void testCheckRetries() { - JobDetail job = launchJob(); - frameDao.checkRetries(frameDao.findFrame(job,"0001-pass_1")); - // TODO: check to see if it actually works - } - @Test @Transactional @Rollback(true) From 362e85fdca736bc05e5ed3703c49ea2635036861 Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Fri, 21 May 2021 09:39:11 -0700 Subject: [PATCH 094/277] Enable editing existing comments. (#980) --- cuegui/cuegui/Comments.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cuegui/cuegui/Comments.py b/cuegui/cuegui/Comments.py index dc493433f..9d8452a5b 100644 --- a/cuegui/cuegui/Comments.py +++ b/cuegui/cuegui/Comments.py @@ -109,6 +109,8 @@ def __init__(self, source, parent=None): def __textEdited(self, text=None): """Called when the text boxes are modified, enables the save button""" del text + self.__textSubject.setReadOnly(False) + self.__textMessage.setReadOnly(False) self.__btnSave.setEnabled(True) def __close(self): From ec40fe0e907d1d242e0643ee544a0d4df4ca586a Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Fri, 21 May 2021 09:39:31 -0700 Subject: [PATCH 095/277] Shellout check_call uses string input. (#978) --- cuegui/cuegui/Utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuegui/cuegui/Utils.py b/cuegui/cuegui/Utils.py index 98ef4a6fb..e931ab98b 100644 --- a/cuegui/cuegui/Utils.py +++ b/cuegui/cuegui/Utils.py @@ -285,7 +285,7 @@ def checkShellOut(cmdList, lockGui=False): if not lockGui and platform.system() != "Windows": cmdList.append('&') try: - subprocess.check_call(cmdList) + subprocess.check_call(" ".join(cmdList), shell=True) except subprocess.CalledProcessError as e: text = 'Command {cmd} failed with returncode {code}. {msg}.\n' \ 'Please check your EDITOR environment variable and the ' \ From e4b6d651628972ea66fc16567f079d487756eac0 Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Fri, 21 May 2021 09:40:02 -0700 Subject: [PATCH 096/277] [CueGUI] Encode email MIMEText with plain utf-8. (#977) --- cuegui/cuegui/EmailDialog.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cuegui/cuegui/EmailDialog.py b/cuegui/cuegui/EmailDialog.py index 4775553d9..671aeff28 100644 --- a/cuegui/cuegui/EmailDialog.py +++ b/cuegui/cuegui/EmailDialog.py @@ -281,7 +281,7 @@ def email_subject(self): def email_body(self): """Get the email body text.""" - return "%s" % self.__email_body.toPlainText().toAscii() + return "%s" % self.__email_body.toPlainText() def appendToBody(self, txt): """Appends text to the email body.""" @@ -295,8 +295,8 @@ def sendEmail(self): """Sends the email.""" self.send.emit() - msg = MIMEText(self.email_body()) - msg["Subject"] = Header(self.email_subject(), continuation_ws=' ') + msg = MIMEText(self.email_body(), 'plain', 'utf-8') + msg["Subject"] = Header(self.email_subject(), 'utf-8', continuation_ws=' ') msg["To"] = self.email_to() msg["From"] = self.email_from() msg["Cc"] = self.email_cc() From 9f3fc74c211d5dfa8837ff8b9c81233ee8ccc54b Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Fri, 21 May 2021 09:40:42 -0700 Subject: [PATCH 097/277] [CueGUI] Log file red text unhighlights once search is cleared. (#981) --- cuegui/cuegui/plugins/LogViewPlugin.py | 27 ++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/cuegui/cuegui/plugins/LogViewPlugin.py b/cuegui/cuegui/plugins/LogViewPlugin.py index fb40734e4..9fc5b21d7 100644 --- a/cuegui/cuegui/plugins/LogViewPlugin.py +++ b/cuegui/cuegui/plugins/LogViewPlugin.py @@ -699,7 +699,6 @@ def _highlight_matches(self): QtGui.QTextCursor.KeepAnchor, match[-1]) self._highlight_cursor.setCharFormat(self._format) - self._matches_to_highlight.discard(match) def _clear_search_text(self): """ @@ -721,19 +720,27 @@ def _clear_search_data(self): are also removed. """ + if not self._log_file: + return + + # find matched text to "unhighlight" red by resetting the char format + highlight = self._matches[max(self._current_match - 300, 0): + min(self._current_match + 300, len(self._matches))] + matches = list(set(highlight).intersection(self._matches_to_highlight)) + + for match in matches: + self._highlight_cursor.setPosition(match[0]) + self._highlight_cursor.movePosition(QtGui.QTextCursor.Right, + QtGui.QTextCursor.KeepAnchor, + match[-1]) + self._highlight_cursor.setCharFormat(QtGui.QTextCharFormat()) + self._highlight_cursor.clearSelection() + + # reset text matches self._matches = [] self._matches_to_highlight = set() self._search_timestamp = 0 self._matches_label.setText('') - if not self._log_file: - return - - charFormat = QtGui.QTextCharFormat() - self._highlight_cursor.setPosition(QtGui.QTextCursor.Start) - self._highlight_cursor.movePosition( - QtGui.QTextCursor.End, mode=QtGui.QTextCursor.KeepAnchor) - self._highlight_cursor.setCharFormat(charFormat) - self._highlight_cursor.clearSelection() def _set_scrollbar_value(self, val): """ From 5545b72c71e9ba9c6c4bd5811e74d1844dbc5cd8 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Sat, 5 Jun 2021 12:30:32 -0400 Subject: [PATCH 098/277] Notes from Apr 28 TSC meeting. (#974) --- tsc/meetings/2021-04-28.md | 104 +++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 tsc/meetings/2021-04-28.md diff --git a/tsc/meetings/2021-04-28.md b/tsc/meetings/2021-04-28.md new file mode 100644 index 000000000..552c4151c --- /dev/null +++ b/tsc/meetings/2021-04-28.md @@ -0,0 +1,104 @@ +# OpenCue TSC Meeting Notes 28 Apr 2021 + +Secretary: Brian Cipriano + +TSC Attendees: + +* [ ] Matt Chambers +* [x] Brian Cipriano +* [x] Greg Denton +* [x] Ben Dines +* [ ] Dave Fellows +* [x] Daniel Neilson +* [ ] Alex Schworer +* [ ] Erik Strauss +* [x] Lars van der Bijl + +Committer Attendees: + +* [ ] Christian Smith +* [x] Diego Tavares +* [x] Idris Miles + +Agenda/Notes: + +* Goals for 2021 + * ~~User survey~~ + * ASWF Graduation + * Approved by TAC, approved by board. Done! + * New user UX + * Rename demo_data.sql + * Main code change done. + * Docs updated. + * Still todo: updating release pipeline to publish seed_data artifact. + * Publish PyPI packages + * Design + doc: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/ + * Proposal for config file + standardization: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/edit#heading=h.v85uo9klwqft + * Fix cuebot:latest tag for sandbox setup + * Proposal: publish Docker images for every new patch version (i.e. every new commit to + master) + * Proposal accepted. + * No progress implementing this yet. + * Related: for PyPI design we may want to also publish packages for every new patch + version. + * Brian: this happened again, sent update to Quickstarts to instruct user to build local + Cuebot to work around this for now. + * Docs refresh + * User guide refresh + * Split into "install from latest release" and "install from master" + * Needed to assist transition to publish Docker images on all commits + * No progress. + * API reference + * We have the ability to generate HTML from any commit, but we need a new CI pipeline to + build and publish this on opencue.io for new releases. + * Proposal: for now, highlight existing docs for building reference locally + * Proposal accepted. + * Brian: cleaned up local build process, sent email with doc + link: https://www.opencue.io/contributing/opencue/build-docs/ + * pyoutline examples + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/177 + * No progress yet. + * User guide for CueGUI + * No progress. + * Configuration guide for Cuebot + * In progress, will be done as part of PyPI work. + * Configuration guide for RQD + * In progress, will be done as part of PyPI work. + * ~~Drop Oracle support~~ + * GPU support + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/459 + * Starting PR in review: https://github.com/AcademySoftwareFoundation/OpenCue/issues/459 + * In review, troubleshooting test failures. + * ~~GUI to add new shows~~ + * Expand DCC plugins + * No progress. + * Logging solution + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/880 + * PR approved and merged. + * Proposal: write blog post on this topic + * Proposal accepted + * No progress on this yet + * CSP Terraform docs + * No progress yet. + * Auto-close Github issues + * No progress. + * Improve booking logic + * Wasting resources while hostreport and booking threadpools fight for resources, work needs + to be rebooked + * Let's dig up the work that Matt Chambers was working on a while ago + * RQD needs to handle dynamic pool of cuebot hosts better + * Let's file an issue and continue discussion there +* Other current work updates? + * Brian + * Xvfb no longer supports Python 2 + * Issue: https://github.com/AcademySoftwareFoundation/OpenCue/issues/964 + * PR to disable CueGUI Python 2 + tests: https://github.com/AcademySoftwareFoundation/OpenCue/pull/965 + * Any other way we can work around this? Or do we just have to accept this is going to + happen gradually? Consensus: seems like the latter. + * Remove localbook? + * Still a useful feature, though it needs some fixing up. + * Let's keep it around for now, improve if needed. + From b1741721f2c478329102ede35724e821e66f8fb9 Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Sat, 5 Jun 2021 11:50:43 -0700 Subject: [PATCH 099/277] Copy frame name to middle click clipboard. (#982) --- cuegui/cuegui/FrameMonitorTree.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cuegui/cuegui/FrameMonitorTree.py b/cuegui/cuegui/FrameMonitorTree.py index 33ccccad9..be3362095 100644 --- a/cuegui/cuegui/FrameMonitorTree.py +++ b/cuegui/cuegui/FrameMonitorTree.py @@ -299,7 +299,8 @@ def __itemSingleClickedCopy(self, item, col): selected = [ frame.data.name for frame in self.selectedObjects() if cuegui.Utils.isFrame(frame)] if selected: - QtWidgets.QApplication.clipboard().setText(" ".join(selected)) + QtWidgets.QApplication.clipboard().setText(" ".join(selected), + QtGui.QClipboard.Selection) def __itemSingleClickedViewLog(self, item, col): """Called when an item is clicked on. Views the log file contents From 32b7b53ebed14f477bb493aec694bd21788ca4df Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Sat, 5 Jun 2021 11:51:27 -0700 Subject: [PATCH 100/277] Add GRPC_POLL_STRATEGY to avoid spamming users. (#985) --- pycue/opencue/cuebot.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pycue/opencue/cuebot.py b/pycue/opencue/cuebot.py index 1211c9fe4..4589806c3 100644 --- a/pycue/opencue/cuebot.py +++ b/pycue/opencue/cuebot.py @@ -77,6 +77,9 @@ DEFAULT_MAX_MESSAGE_BYTES = 1024 ** 2 * 10 DEFAULT_GRPC_PORT = 8443 +# Avoid spamming users with epoll fork warning messages +os.environ["GRPC_POLL_STRATEGY"] = "epoll1" + class Cuebot(object): """Used to manage the connection to the Cuebot. Normally the connection to the Cuebot is made automatically as needed so you don't have to explicitly From 4e22b6570fb86025732581a776a35b24a9beda8d Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Fri, 11 Jun 2021 16:35:29 -0400 Subject: [PATCH 101/277] Notes from May 12 TSC meeting. (#990) --- tsc/meetings/2021-05-12.md | 94 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 tsc/meetings/2021-05-12.md diff --git a/tsc/meetings/2021-05-12.md b/tsc/meetings/2021-05-12.md new file mode 100644 index 000000000..71c4c3035 --- /dev/null +++ b/tsc/meetings/2021-05-12.md @@ -0,0 +1,94 @@ +# OpenCue TSC Meeting Notes 12 May 2021 + +Secretary: Brian Cipriano + +TSC Attendees: + +* [ ] Matt Chambers +* [x] Brian Cipriano +* [x] Greg Denton +* [x] Ben Dines +* [ ] Dave Fellows +* [x] Daniel Neilson +* [ ] Alex Schworer +* [ ] Erik Strauss +* [ ] Lars van der Bijl + +Committer Attendees: + +* [ ] Christian Smith +* [x] Diego Tavares +* [ ] Idris Miles + +Agenda/Notes: + +* Goals for 2021 + * User survey + * ASWF Graduation + * New user UX + * Rename demo_data.sql + * Main code change done. + * Docs updated. + * Still todo: updating release pipeline to publish seed_data artifact. + * Publish PyPI packages + * Design + doc: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/ + * Proposal for config file + standardization: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/edit#heading=h.v85uo9klwqft + * Brian has started work on config cleanup. + * Fix cuebot:latest tag for sandbox setup + * Proposal: publish Docker images for every new patch version (i.e. every new commit to + master) + * Proposal accepted. + * No progress implementing this yet. + * Related: for PyPI design we may want to also publish packages for every new patch + version. + * Docs refresh + * User guide refresh + * Split into "install from latest release" and "install from master" + * Needed to assist transition to publish Docker images on all commits + * No progress. + * API reference + * We have the ability to generate HTML from any commit, but we need a new CI pipeline to + build and publish this on opencue.io for new releases. + * Proposal: for now, highlight existing docs for building reference locally + * Proposal accepted. + * Brian: cleaned up local build process, sent email with doc + link: https://www.opencue.io/contributing/opencue/build-docs/ + * pyoutline examples + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/177 + * No progress yet. + * User guide for CueGUI + * No progress. + * Configuration guide for client-side python + * In progress, will be done as part of PyPI work. + * Configuration guide for Cuebot + * In progress, will be done as part of PyPI work. + * Configuration guide for RQD + * In progress, will be done as part of PyPI work. + * Improve scheduler logic + * Brian: couldn't locate old docs, email out to Matt to see if he has any materials around. + * Need an owner for this work. Diego has volunteered. + * Drop Oracle support + * Done. + * GPU support + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/459 + * Starting PR in review: https://github.com/AcademySoftwareFoundation/OpenCue/pull/924 + * Test failures resolved, needs review. + * GUI to add new shows + * Done. + * Expand DCC plugins + * No progress. + * Logging solution + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/880 + * PR approved and merged. + * Proposal: write blog post on this topic + * Proposal accepted + * No progress on this yet + * CSP Terraform docs + * No progress yet. + * Auto-close Github issues + * No progress. +* Other current work updates + * SPI: shutdown of old cue3 system is complete. + From b10cd0d998ede37a995d7fbeeda8e83902f6dd6e Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sun, 20 Jun 2021 09:00:40 -0700 Subject: [PATCH 102/277] Fix JobDetail mapper. (#971) --- .../spcue/dao/postgres/JobDaoJdbc.java | 2 +- .../spcue/test/dao/postgres/JobDaoTests.java | 44 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java index a3dab8f4e..3705324be 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java @@ -110,7 +110,7 @@ public JobDetail mapRow(ResultSet rs, int rowNum) throws SQLException { JobDetail job = new JobDetail(); job.id = rs.getString("pk_job"); job.showId = rs.getString("pk_show"); - job.facilityId = rs.getString("pk_show"); + job.facilityId = rs.getString("pk_facility"); job.deptId = rs.getString("pk_dept"); job.groupId = rs.getString("pk_folder"); job.logDir = rs.getString("str_log_dir"); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/JobDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/JobDaoTests.java index 3c51f34c1..07a32f184 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/JobDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/JobDaoTests.java @@ -120,10 +120,14 @@ public JobDetail insertJob() { JobDetail job = this.buildJobDetail(); job.groupId = ROOT_FOLDER; job.showId = ROOT_SHOW; + job.showName = "pipe"; job.logDir = jobLogUtil.getJobLogPath(job); job.deptId = departmentDao.getDefaultDepartment().getId(); + job.deptName = departmentDao.getDefaultDepartment().getName(); job.facilityId = facilityDao.getDefaultFacility().getId(); + job.facilityName = facilityDao.getDefaultFacility().getName(); job.state = JobState.PENDING; + job.maxCoreUnits = 10000; jobDao.insertJob(job, jobLogUtil); return job; } @@ -186,6 +190,46 @@ public void testGetJob() { jobDao.getJob(job.id); } + @Test + @Transactional + @Rollback(true) + public void testGetJobDetail() { + JobDetail src = insertJob(); + JobDetail job = jobDao.getJobDetail(src.id); + assertEquals(job.id, src.id); + assertEquals(job.name, src.name); + assertEquals(job.showId, src.showId); + assertEquals(job.facilityId, src.facilityId); + assertEquals(job.groupId, src.groupId); + assertEquals(job.deptId, src.deptId); + assertEquals(job.state, src.state); + assertEquals(job.shot, src.shot); + assertEquals(job.user, src.user); + assertEquals(job.email, src.email); + assertEquals(job.uid, src.uid); + assertEquals(job.logDir, src.logDir); + assertEquals(job.isPaused, src.isPaused); + assertEquals(job.isAutoEat, src.isAutoEat); + assertEquals(job.totalFrames, src.totalFrames); + assertEquals(job.totalLayers, src.totalLayers); + assertEquals(job.startTime, src.startTime); + assertEquals(job.stopTime, src.stopTime); + assertEquals(job.maxRetries, src.maxRetries); + assertEquals(job.os, src.os); + assertEquals(job.facilityName, src.facilityName); + assertEquals(job.deptName, src.deptName); + assertEquals(job.showName, src.showName); + assertEquals(job.priority, src.priority); + assertEquals(job.minCoreUnits, src.minCoreUnits); + assertEquals(job.maxCoreUnits, src.maxCoreUnits); + assertEquals(job.isLocal, src.isLocal); + assertEquals(job.localHostName, src.localHostName); + assertEquals(job.localMaxCores, src.localMaxCores); + assertEquals(job.localMaxMemory, src.localMaxMemory); + assertEquals(job.localThreadNumber, src.localThreadNumber); + assertEquals(job.localMaxGpu, src.localMaxGpu); + } + @Test @Transactional @Rollback(true) From c22fe12721bed24b814184ea78c5e447b3b6a477 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sun, 20 Jun 2021 09:21:19 -0700 Subject: [PATCH 103/277] Add multiple GPU support #760 (#924) --- VERSION.in | 2 +- .../com/imageworks/spcue/DispatchFrame.java | 4 +- .../com/imageworks/spcue/DispatchHost.java | 39 +- .../imageworks/spcue/ExecutionSummary.java | 27 + .../com/imageworks/spcue/GroupDetail.java | 5 + .../java/com/imageworks/spcue/HostEntity.java | 20 +- .../java/com/imageworks/spcue/Inherit.java | 2 + .../java/com/imageworks/spcue/JobDetail.java | 7 +- .../com/imageworks/spcue/LayerDetail.java | 20 +- .../imageworks/spcue/LocalHostAssignment.java | 55 +- .../com/imageworks/spcue/ResourceUsage.java | 10 +- .../com/imageworks/spcue/ServiceEntity.java | 13 +- .../java/com/imageworks/spcue/ShowEntity.java | 2 + .../com/imageworks/spcue/StrandedGpus.java | 44 + .../com/imageworks/spcue/VirtualProc.java | 13 +- .../com/imageworks/spcue/dao/BookingDao.java | 39 +- .../com/imageworks/spcue/dao/GroupDao.java | 42 + .../com/imageworks/spcue/dao/HostDao.java | 16 +- .../java/com/imageworks/spcue/dao/JobDao.java | 67 + .../com/imageworks/spcue/dao/LayerDao.java | 39 +- .../com/imageworks/spcue/dao/ProcDao.java | 2 +- .../com/imageworks/spcue/dao/ShowDao.java | 15 + .../spcue/dao/postgres/BookingDaoJdbc.java | 120 +- .../spcue/dao/postgres/DispatchQuery.java | 114 +- .../spcue/dao/postgres/DispatcherDaoJdbc.java | 33 +- .../spcue/dao/postgres/FrameDaoJdbc.java | 34 +- .../spcue/dao/postgres/GroupDaoJdbc.java | 81 +- .../spcue/dao/postgres/HostDaoJdbc.java | 103 +- .../spcue/dao/postgres/JobDaoJdbc.java | 121 +- .../spcue/dao/postgres/LayerDaoJdbc.java | 65 +- .../dao/postgres/NestedWhiteboardDaoJdbc.java | 33 +- .../spcue/dao/postgres/ProcDaoJdbc.java | 111 +- .../spcue/dao/postgres/ServiceDaoJdbc.java | 45 +- .../spcue/dao/postgres/ShowDaoJdbc.java | 20 + .../spcue/dao/postgres/WhiteboardDaoJdbc.java | 154 ++- .../spcue/dispatcher/AbstractDispatcher.java | 7 +- .../spcue/dispatcher/CoreUnitDispatcher.java | 23 +- .../spcue/dispatcher/DispatchSupport.java | 51 +- .../dispatcher/DispatchSupportService.java | 50 +- .../spcue/dispatcher/Dispatcher.java | 10 +- .../dispatcher/FrameCompleteHandler.java | 2 +- .../spcue/dispatcher/HostReportHandler.java | 14 +- .../spcue/dispatcher/LocalDispatcher.java | 22 +- .../spcue/dispatcher/RedirectManager.java | 2 + .../spcue/dispatcher/ResourceContainer.java | 10 +- .../dispatcher/commands/DispatchBookHost.java | 6 +- .../imageworks/spcue/servant/ManageFrame.java | 2 +- .../imageworks/spcue/servant/ManageGroup.java | 44 + .../imageworks/spcue/servant/ManageJob.java | 37 +- .../imageworks/spcue/servant/ManageLayer.java | 34 +- .../spcue/servant/ManageRenderPartition.java | 2 +- .../spcue/servant/ManageService.java | 8 +- .../spcue/servant/ManageServiceOverride.java | 4 +- .../imageworks/spcue/servant/ManageShow.java | 26 +- .../spcue/service/BookingManager.java | 5 +- .../spcue/service/BookingManagerService.java | 10 +- .../spcue/service/GroupManager.java | 4 + .../spcue/service/GroupManagerService.java | 26 + .../imageworks/spcue/service/HostManager.java | 11 +- .../spcue/service/HostManagerService.java | 10 +- .../imageworks/spcue/service/JobLauncher.java | 3 +- .../imageworks/spcue/service/JobManager.java | 24 + .../spcue/service/JobManagerService.java | 16 + .../com/imageworks/spcue/service/JobSpec.java | 66 +- .../com/imageworks/spcue/util/CueUtil.java | 10 +- .../migrations/V11__Support_multiple_GPU.sql | 1078 +++++++++++++++++ .../main/resources/public/dtd/cjsl-1.12.dtd | 97 ++ .../test/dao/postgres/BookingDaoTests.java | 48 +- .../test/dao/postgres/CommentDaoTests.java | 4 +- .../spcue/test/dao/postgres/DeedDaoTests.java | 4 +- .../test/dao/postgres/DispatcherDaoTests.java | 2 +- .../test/dao/postgres/FrameDaoTests.java | 5 +- .../spcue/test/dao/postgres/HostDaoTests.java | 4 +- .../spcue/test/dao/postgres/JobDaoTests.java | 2 +- .../test/dao/postgres/LayerDaoTests.java | 12 +- .../spcue/test/dao/postgres/ProcDaoTests.java | 10 +- .../test/dao/postgres/ServiceDaoTests.java | 18 +- .../spcue/test/dao/postgres/ShowDaoTests.java | 4 +- .../test/dao/postgres/WhiteboardDaoTests.java | 8 +- .../CoreUnitDispatcherGpuJobTests.java | 6 +- .../CoreUnitDispatcherGpuTests.java | 17 +- .../CoreUnitDispatcherGpusJobTests.java | 277 +++++ .../test/dispatcher/DispatchSupportTests.java | 4 +- .../dispatcher/FrameCompleteHandlerTests.java | 236 ++++ .../dispatcher/HostReportHandlerGpuTests.java | 124 ++ .../dispatcher/HostReportHandlerTests.java | 4 +- .../test/dispatcher/LocalDispatcherTests.java | 20 +- .../test/dispatcher/TestBookingQueue.java | 4 +- .../test/service/BookingManagerTests.java | 16 +- .../spcue/test/service/HostManagerTests.java | 4 +- .../spcue/test/service/JobManagerTests.java | 2 +- .../spcue/test/service/JobSpecTests.java | 19 + .../spcue/test/service/OwnerManagerTests.java | 4 +- .../test/service/ServiceManagerTests.java | 12 +- .../spcue/test/util/CueUtilTester.java | 6 +- .../resources/conf/ddl/postgres/test_data.sql | 2 +- .../src/test/resources/conf/dtd/cjsl-1.12.dtd | 97 ++ .../resources/conf/jobspec/jobspec_1_12.xml | 49 + .../jobspec/jobspec_dispatch_gpus_test.xml | 133 ++ .../conf/jobspec/jobspec_gpus_test.xml | 76 ++ cuegui/cuegui/CueJobMonitorTree.py | 68 +- cuegui/cuegui/FrameMonitorTree.py | 55 +- cuegui/cuegui/GroupDialog.py | 56 +- cuegui/cuegui/HostMonitorTree.py | 42 +- cuegui/cuegui/LayerDialog.py | 99 +- cuegui/cuegui/LayerMonitorTree.py | 37 +- cuegui/cuegui/MenuActions.py | 50 + cuegui/cuegui/ProcMonitorTree.py | 2 +- cuegui/cuegui/ServiceDialog.py | 18 +- cuegui/cuegui/config/cue_resources.yaml | 3 + cuegui/tests/FrameMonitorTree_tests.py | 2 +- cuegui/tests/LayerDialog_tests.py | 39 +- proto/facility.proto | 5 + proto/host.proto | 28 +- proto/job.proto | 151 ++- proto/renderPartition.proto | 8 +- proto/report.proto | 18 +- proto/rqd.proto | 1 + proto/service.proto | 6 +- proto/show.proto | 25 + proto/subscription.proto | 1 + pycue/opencue/wrappers/group.py | 38 + pycue/opencue/wrappers/job.py | 51 +- pycue/opencue/wrappers/layer.py | 38 +- pycue/opencue/wrappers/show.py | 26 + pycue/tests/wrappers/layer_test.py | 10 +- pyoutline/etc/outline.cfg | 2 +- pyoutline/outline/backend/cue.py | 24 + pyoutline/tests/specver_test.py | 21 + rqd/rqd/rqcore.py | 22 +- rqd/rqd/rqmachine.py | 109 +- rqd/rqd/rqnetwork.py | 9 +- rqd/tests/rqconstants_tests.py | 3 +- rqd/tests/rqmachine_tests.py | 52 +- 134 files changed, 4872 insertions(+), 711 deletions(-) create mode 100644 cuebot/src/main/java/com/imageworks/spcue/StrandedGpus.java create mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V11__Support_multiple_GPU.sql create mode 100644 cuebot/src/main/resources/public/dtd/cjsl-1.12.dtd create mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpusJobTests.java create mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java create mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java create mode 100644 cuebot/src/test/resources/conf/dtd/cjsl-1.12.dtd create mode 100644 cuebot/src/test/resources/conf/jobspec/jobspec_1_12.xml create mode 100644 cuebot/src/test/resources/conf/jobspec/jobspec_dispatch_gpus_test.xml create mode 100644 cuebot/src/test/resources/conf/jobspec/jobspec_gpus_test.xml diff --git a/VERSION.in b/VERSION.in index c43e1055f..f3040840f 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.12 +0.13 diff --git a/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java b/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java index db946b90c..781401165 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java +++ b/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java @@ -43,7 +43,9 @@ public class DispatchFrame extends FrameEntity implements FrameInterface { public int maxCores; public boolean threadable; public long minMemory; - public long minGpu; + public int minGpus; + public int maxGpus; + public long minGpuMemory; public String services; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/DispatchHost.java b/cuebot/src/main/java/com/imageworks/spcue/DispatchHost.java index e1b3cc8f2..495d0a9b1 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/DispatchHost.java +++ b/cuebot/src/main/java/com/imageworks/spcue/DispatchHost.java @@ -35,13 +35,16 @@ public class DispatchHost extends Entity public int cores; public int idleCores; + public int gpus; + public int idleGpus; + // Basically an 0 = auto, 1 = all. public int threadMode; public long memory; public long idleMemory; - public long gpu; - public long idleGpu; + public long gpuMemory; + public long idleGpuMemory; public String tags; public String os; @@ -53,11 +56,13 @@ public class DispatchHost extends Entity * booked to this host. */ public int strandedCores = 0; + public int strandedGpus = 0; // To reserve resources for future gpu job long idleMemoryOrig = 0; int idleCoresOrig = 0; - long idleGpuOrig = 0; + long idleGpuMemoryOrig = 0; + int idleGpusOrig = 0; public String getHostId() { return id; @@ -72,7 +77,7 @@ public String getFacilityId() { } @Override - public boolean hasAdditionalResources(int minCores, long minMemory, long minGpu) { + public boolean hasAdditionalResources(int minCores, long minMemory, int minGpus, long minGpuMemory) { if (idleCores < minCores) { return false; @@ -80,7 +85,10 @@ public boolean hasAdditionalResources(int minCores, long minMemory, long minGpu) else if (idleMemory < minMemory) { return false; } - else if (idleGpu < minGpu) { + else if (idleGpus < minGpus) { + return false; + } + else if (idleGpuMemory < minGpuMemory) { return false; } @@ -88,10 +96,11 @@ else if (idleGpu < minGpu) { } @Override - public void useResources(int coreUnits, long memory, long gpu) { + public void useResources(int coreUnits, long memory, int gpuUnits, long gpuMemory) { idleCores = idleCores - coreUnits; idleMemory = idleMemory - memory; - idleGpu = idleGpu - gpu; + idleGpus = idleGpus - gpuUnits; + idleGpuMemory = idleGpuMemory - gpuMemory; } /** @@ -99,14 +108,16 @@ public void useResources(int coreUnits, long memory, long gpu) { * */ public void removeGpu() { - if (idleGpu > 0 && idleGpuOrig == 0) { + if (idleGpuMemory > 0 && idleGpuMemoryOrig == 0) { idleMemoryOrig = idleMemory; idleCoresOrig = idleCores; - idleGpuOrig = idleGpu; + idleGpuMemoryOrig = idleGpuMemory; + idleGpusOrig = idleGpus; idleMemory = idleMemory - Math.min(CueUtil.GB4, idleMemory); idleCores = idleCores - Math.min(100, idleCores); - idleGpu = 0; + idleGpuMemory = idleGpuMemory - Math.min(CueUtil.GB4, idleGpuMemory); + idleGpus = idleGpus - Math.min(1, idleGpus); } } @@ -115,14 +126,16 @@ public void removeGpu() { * */ public void restoreGpu() { - if (idleGpuOrig > 0) { + if (idleGpuMemoryOrig > 0) { idleMemory = idleMemoryOrig; idleCores = idleCoresOrig; - idleGpu = idleGpuOrig; + idleGpuMemory = idleGpuMemoryOrig; + idleGpus = idleGpusOrig; idleMemoryOrig = 0; idleCoresOrig = 0; - idleGpuOrig = 0; + idleGpuMemoryOrig = 0; + idleGpusOrig = 0; } } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/ExecutionSummary.java b/cuebot/src/main/java/com/imageworks/spcue/ExecutionSummary.java index a13529ad8..afe85121a 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/ExecutionSummary.java +++ b/cuebot/src/main/java/com/imageworks/spcue/ExecutionSummary.java @@ -28,6 +28,9 @@ public class ExecutionSummary { public long coreTime; public long coreTimeSuccess; public long coreTimeFail; + public long gpuTime; + public long gpuTimeSuccess; + public long gpuTimeFail; public long highMemoryKb; public long getHighMemoryKb() { @@ -69,5 +72,29 @@ public long getCoreTimeFail() { public void setCoreTimeFail(long coreTimeFail) { this.coreTimeFail = coreTimeFail; } + + public long getGpuTime() { + return gpuTime; + } + + public void setGpuTime(long gpuTime) { + this.gpuTime = gpuTime; + } + + public long getGpuTimeSuccess() { + return gpuTimeSuccess; + } + + public void setGpuTimeSuccess(long gpuTimeSuccess) { + this.gpuTimeSuccess = gpuTimeSuccess; + } + + public long getGpuTimeFail() { + return gpuTimeFail; + } + + public void setGpuTimeFail(long gpuTimeFail) { + this.gpuTimeFail = gpuTimeFail; + } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/GroupDetail.java b/cuebot/src/main/java/com/imageworks/spcue/GroupDetail.java index b67a53b77..cd9f8a998 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/GroupDetail.java +++ b/cuebot/src/main/java/com/imageworks/spcue/GroupDetail.java @@ -23,11 +23,16 @@ public class GroupDetail extends Entity implements GroupInterface, DepartmentInt public int jobMinCores = -1; public int jobMaxCores = -1; + public int jobMinGpus = -1; + public int jobMaxGpus = -1; public int jobPriority = -1; public int minCores = -1; public int maxCores = -1; + public int minGpus = -1; + public int maxGpus = -1; + public String parentId = null; public String showId; public String deptId; diff --git a/cuebot/src/main/java/com/imageworks/spcue/HostEntity.java b/cuebot/src/main/java/com/imageworks/spcue/HostEntity.java index 96defaf61..5a019e8f1 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/HostEntity.java +++ b/cuebot/src/main/java/com/imageworks/spcue/HostEntity.java @@ -36,10 +36,12 @@ public class HostEntity extends Entity implements HostInterface { public int procs; public int cores; public int idleCores; - public int memory; - public int idleMemory; - public int gpu; - public int idleGpu; + public long memory; + public long idleMemory; + public int gpus; + public int idleGpus; + public long gpuMemory; + public long idleGpuMemory; public boolean unlockAtBoot; @@ -57,10 +59,12 @@ public HostEntity(Host grpcHost) { this.nimbyEnabled = grpcHost.getNimbyEnabled(); this.cores = (int) grpcHost.getCores(); this.idleCores = (int) grpcHost.getIdleCores(); - this.memory = (int) grpcHost.getMemory(); - this.idleMemory = (int) grpcHost.getIdleMemory(); - this.gpu = (int) grpcHost.getGpu(); - this.idleGpu = (int) grpcHost.getIdleGpu(); + this.memory = grpcHost.getMemory(); + this.idleMemory = grpcHost.getIdleMemory(); + this.gpus = (int) grpcHost.getGpus(); + this.idleGpus = (int) grpcHost.getIdleGpus(); + this.gpuMemory = grpcHost.getGpuMemory(); + this.idleGpuMemory = grpcHost.getIdleGpuMemory(); } public String getHostId() { diff --git a/cuebot/src/main/java/com/imageworks/spcue/Inherit.java b/cuebot/src/main/java/com/imageworks/spcue/Inherit.java index 73651c33d..1fdb23336 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/Inherit.java +++ b/cuebot/src/main/java/com/imageworks/spcue/Inherit.java @@ -28,6 +28,8 @@ public enum Inherit { Priority, MinCores, MaxCores, + MinGpus, + MaxGpus, All } diff --git a/cuebot/src/main/java/com/imageworks/spcue/JobDetail.java b/cuebot/src/main/java/com/imageworks/spcue/JobDetail.java index 29286ffe3..dad6f8a6d 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/JobDetail.java +++ b/cuebot/src/main/java/com/imageworks/spcue/JobDetail.java @@ -46,12 +46,15 @@ public class JobDetail extends JobEntity implements JobInterface, DepartmentInte public int priority = 1; public int minCoreUnits = 100; public int maxCoreUnits = 200000; + public int minGpuUnits = 0; + public int maxGpuUnits = 1000; public boolean isLocal = false; public String localHostName; public int localMaxCores; - public int localMaxMemory; + public long localMaxMemory; public int localThreadNumber; - public int localMaxGpu; + public int localMaxGpus; + public long localMaxGpuMemory; public String getDepartmentId() { return deptId; diff --git a/cuebot/src/main/java/com/imageworks/spcue/LayerDetail.java b/cuebot/src/main/java/com/imageworks/spcue/LayerDetail.java index 3b473f8c1..565995d9d 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/LayerDetail.java +++ b/cuebot/src/main/java/com/imageworks/spcue/LayerDetail.java @@ -32,9 +32,11 @@ public class LayerDetail extends LayerEntity implements LayerInterface { public LayerType type; public int minimumCores; public int maximumCores; + public int minimumGpus; + public int maximumGpus; public boolean isThreadable; public long minimumMemory; - public long minimumGpu; + public long minimumGpuMemory; public int chunkSize; public int timeout; public int timeout_llu; @@ -116,12 +118,20 @@ public void setMinimumMemory(long minimumMemory) { this.minimumMemory = minimumMemory; } - public long getMinimumGpu() { - return minimumGpu; + public int getMinimumGpus() { + return minimumGpus; } - public void setMinimumGpu(long minimumGpu) { - this.minimumGpu = minimumGpu; + public void setMinimumGpus(int minimumGpus) { + this.minimumGpus = minimumGpus; + } + + public long getMinimumGpuMemory() { + return minimumGpuMemory; + } + + public void setMinimumGpuMemory(long minimumGpuMemory) { + this.minimumGpuMemory = minimumGpuMemory; } public int getChunkSize() { diff --git a/cuebot/src/main/java/com/imageworks/spcue/LocalHostAssignment.java b/cuebot/src/main/java/com/imageworks/spcue/LocalHostAssignment.java index cc6287253..3e073fa73 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/LocalHostAssignment.java +++ b/cuebot/src/main/java/com/imageworks/spcue/LocalHostAssignment.java @@ -35,11 +35,13 @@ public class LocalHostAssignment extends Entity private int idleCoreUnits; private long idleMemory; - private long idleGpu; + private int idleGpuUnits; + private long idleGpuMemory; private long maxMemory; - private long maxGpu; + private long maxGpuMemory; private int maxCoreUnits; + private int maxGpuUnits; private int threads; @@ -52,15 +54,16 @@ public class LocalHostAssignment extends Entity public LocalHostAssignment() { } - public LocalHostAssignment(int maxCores, int threads, long maxMemory, long maxGpu) { + public LocalHostAssignment(int maxCores, int threads, long maxMemory, int maxGpus, long maxGpuMemory) { this.maxCoreUnits = maxCores; this.threads = threads; this.maxMemory = maxMemory; - this.maxGpu = maxGpu; + this.maxGpuUnits = maxGpus; + this.maxGpuMemory = maxGpuMemory; } @Override - public boolean hasAdditionalResources(int minCores, long minMemory, long minGpu) { + public boolean hasAdditionalResources(int minCores, long minMemory, int minGpus, long minGpuMemory) { if (idleCoreUnits < minCores) { return false; @@ -68,7 +71,10 @@ public boolean hasAdditionalResources(int minCores, long minMemory, long minGpu) else if (idleMemory < minMemory) { return false; } - else if (idleGpu < minGpu) { + else if (idleGpuUnits < minGpus) { + return false; + } + else if (idleGpuMemory < minGpuMemory) { return false; } @@ -76,10 +82,11 @@ else if (idleGpu < minGpu) { } @Override - public void useResources(int coreUnits, long memory, long gpu) { + public void useResources(int coreUnits, long memory, int gpuUnits, long gpuMemory) { idleCoreUnits = idleCoreUnits - coreUnits; idleMemory = idleMemory - memory; - idleGpu = idleGpu - gpu; + idleGpuUnits = idleGpuUnits - gpuUnits; + idleGpuMemory = idleGpuMemory - gpuMemory; } public int getThreads() { @@ -110,16 +117,24 @@ public long getIdleMemory() { return this.idleMemory; } - public long getMaxGpu() { - return maxGpu; + public int getMaxGpuUnits() { + return maxGpuUnits; + } + + public void setMaxGpuUnits(int maxGpuUnits) { + this.maxGpuUnits = maxGpuUnits; + } + + public long getMaxGpuMemory() { + return maxGpuMemory; } - public void setMaxGpu(long maxGpu) { - this.maxGpu = maxGpu; + public void setMaxGpuMemory(long maxGpuMemory) { + this.maxGpuMemory = maxGpuMemory; } - public long getIdleGpu() { - return this.idleGpu; + public long getIdleGpuMemory() { + return this.idleGpuMemory; } public int getIdleCoreUnits() { @@ -134,8 +149,16 @@ public void setIdleMemory(long idleMemory) { this.idleMemory = idleMemory; } - public void setIdleGpu(long idleGpu) { - this.idleGpu = idleGpu; + public int getIdleGpuUnits() { + return this.idleGpuUnits; + } + + public void setIdleGpuUnits(int idleGpuUnits) { + this.idleGpuUnits = idleGpuUnits; + } + + public void setIdleGpuMemory(long idleGpuMemory) { + this.idleGpuMemory = idleGpuMemory; } public String getHostId() { diff --git a/cuebot/src/main/java/com/imageworks/spcue/ResourceUsage.java b/cuebot/src/main/java/com/imageworks/spcue/ResourceUsage.java index aae8921e4..b45af0838 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/ResourceUsage.java +++ b/cuebot/src/main/java/com/imageworks/spcue/ResourceUsage.java @@ -25,9 +25,10 @@ public class ResourceUsage { private final long coreTimeSeconds; + private final long gpuTimeSeconds; private final long clockTimeSeconds; - public ResourceUsage(long clockTime, int corePoints) { + public ResourceUsage(long clockTime, int corePoints, int gpuPoints) { if (clockTime < 1) { clockTime = 1; @@ -38,14 +39,21 @@ public ResourceUsage(long clockTime, int corePoints) { coreTime = 1; } + long gpuTime = clockTime * gpuPoints; + clockTimeSeconds = clockTime; coreTimeSeconds = coreTime; + gpuTimeSeconds = gpuTime; } public long getCoreTimeSeconds() { return coreTimeSeconds; } + public long getGpuTimeSeconds() { + return gpuTimeSeconds; + } + public long getClockTimeSeconds() { return clockTimeSeconds; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/ServiceEntity.java b/cuebot/src/main/java/com/imageworks/spcue/ServiceEntity.java index 373877e69..16d03c5c5 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/ServiceEntity.java +++ b/cuebot/src/main/java/com/imageworks/spcue/ServiceEntity.java @@ -40,6 +40,17 @@ public class ServiceEntity extends Entity { */ public int maxCores = 0; + /** + * Determines the default minimum gpus per frame. + */ + public int minGpus = 0; + + /** + * Determines the default minimum gpus per frame. 0 indicates + * the feature is disabled. + */ + public int maxGpus = 0; + /** * Determines the default minimum memory per frame. */ @@ -48,7 +59,7 @@ public class ServiceEntity extends Entity { /** * Determines the default minimum gpu per frame. */ - public long minGpu = Dispatcher.GPU_RESERVED_DEFAULT; + public long minGpuMemory = Dispatcher.MEM_GPU_RESERVED_DEFAULT; /** * Determines the default tags. diff --git a/cuebot/src/main/java/com/imageworks/spcue/ShowEntity.java b/cuebot/src/main/java/com/imageworks/spcue/ShowEntity.java index 8a4d768af..1d2f675e1 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/ShowEntity.java +++ b/cuebot/src/main/java/com/imageworks/spcue/ShowEntity.java @@ -25,6 +25,8 @@ public class ShowEntity extends Entity implements ShowInterface { public boolean paused; public int defaultMinCores; public int defaultMaxCores; + public int defaultMinGpus; + public int defaultMaxGpus; public String[] commentMail; public String getShowId() { diff --git a/cuebot/src/main/java/com/imageworks/spcue/StrandedGpus.java b/cuebot/src/main/java/com/imageworks/spcue/StrandedGpus.java new file mode 100644 index 000000000..91b9ad76a --- /dev/null +++ b/cuebot/src/main/java/com/imageworks/spcue/StrandedGpus.java @@ -0,0 +1,44 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.imageworks.spcue; + +public final class StrandedGpus { + + /** + * The maximum time this object should be valid. + */ + private static final long MAX_AGE_MILLIS = 5000l; + + private final int gpus; + private final long expireTime = System.currentTimeMillis() + MAX_AGE_MILLIS; + + public StrandedGpus(int gpus) { + this.gpus = gpus; + } + + public int getGpus() { + return this.gpus; + } + + public boolean isExpired() { + return System.currentTimeMillis() > expireTime; + } +} + diff --git a/cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java b/cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java index 4316b708d..28b54799d 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java @@ -36,7 +36,11 @@ public class VirtualProc extends FrameEntity implements ProcInterface { public long memoryMax; public long virtualMemoryUsed; public long virtualMemoryMax; - public long gpuReserved; + + public int gpusReserved; + public long gpuMemoryReserved; + public long gpuMemoryUsed; + public long gpuMemoryMax; public boolean unbooked; public boolean usageRecorded = false; @@ -91,8 +95,8 @@ public static final VirtualProc build(DispatchHost host, DispatchFrame frame) { proc.coresReserved = frame.minCores; proc.memoryReserved = frame.minMemory; - // This reserves all the gpu memory on a host for one frame - proc.gpuReserved = (frame.minGpu > 0) ? host.idleGpu : 0; + proc.gpusReserved = frame.minGpus; + proc.gpuMemoryReserved = frame.minGpuMemory; /* * Frames that are announcing cores less than 100 are not multi-threaded @@ -208,7 +212,8 @@ public static final VirtualProc build(DispatchHost host, proc.coresReserved = lja.getThreads() * 100; proc.memoryReserved = frame.minMemory; - proc.gpuReserved = frame.minGpu; + proc.gpusReserved = frame.minGpus; + proc.gpuMemoryReserved = frame.minGpuMemory; int wholeCores = (int) (Math.floor(host.idleCores / 100.0)); if (wholeCores == 0) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/BookingDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/BookingDao.java index 9fe9703cb..f3bb09915 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/BookingDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/BookingDao.java @@ -38,6 +38,15 @@ public interface BookingDao { */ boolean updateMaxCores(LocalHostAssignment l, int maxCoreUnits); + /** + * Updates the maximum number of gpus the given local + * host assignment should use. + * + * @param l + * @return + */ + boolean updateMaxGpus(LocalHostAssignment l, int gpus); + /** * Updates the maximum amount of memory a given local host * assignment should use. @@ -54,7 +63,7 @@ public interface BookingDao { * @param l * @return */ - boolean updateMaxGpu(LocalHostAssignment l, long maxGpu); + boolean updateMaxGpuMemory(LocalHostAssignment l, long maxGpuMemory); /** * Create a new LocalHostAssignment attached to the given job. @@ -150,6 +159,16 @@ void insertLocalHostAssignment(HostInterface host, FrameInterface frame, */ int getCoreUsageDifference(LocalHostAssignment l, int coreUnits); + /** + * Return the difference between the number of assigned gpus and + * the given gpuUnits. + * + * @param l + * @param gpuUnits + * @return + */ + int getGpuUsageDifference(LocalHostAssignment l, int gpuUnits); + /** * Allocate additional cores from the given host. * @@ -168,6 +187,24 @@ void insertLocalHostAssignment(HostInterface host, FrameInterface frame, */ boolean deallocateCoresFromHost(HostInterface h, int cores); + /** + * Allocate additional gpus from the given host. + * + * @param h + * @param gpus + * @return + */ + boolean allocateGpusFromHost(HostInterface h, int gpus); + + /** + * Deallocate gpu from the given host, returning them to its pool. + * + * @param h + * @param gpus + * @return + */ + boolean deallocateGpusFromHost(HostInterface h, int gpus); + /** * Return true if the Host has a resource deficit. A * deficit can occur if there are more resources in use than the diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/GroupDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/GroupDao.java index 181062df6..dfb49dd9c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/GroupDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/GroupDao.java @@ -137,6 +137,40 @@ public interface GroupDao { */ public void updateMinCores(GroupInterface group, int value); + + /** + * Sets the maximum number of gpus the group should be running. + * + * @param group + * @param value + */ + void updateDefaultJobMaxGpus(GroupInterface group, int value); + + /** + * Sets the minimum number of gpus the group should be running. + * + * @param group + * @param value + */ + void updateDefaultJobMinGpus(GroupInterface group, int value); + + /** + * Sets the maximum number of gpus for this group + * + * @param group + * @param value + */ + public void updateMaxGpus(GroupInterface group, int value); + + /** + * Set the minimum number of gpus for this group + * + * @param group + * @param value + */ + + public void updateMinGpus(GroupInterface group, int value); + /** * Renames the group * @@ -186,6 +220,14 @@ public interface GroupDao { */ boolean isOverMinCores(JobInterface job); + /** + * Returns true if the group of the specified job is at or over its min gpus + * + * @param job + * @return + */ + boolean isOverMinGpus(JobInterface job); + /** * Returns true if the group is managed. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java index 04cd49f46..768bcdbd2 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java @@ -263,8 +263,8 @@ public interface HostDao { * @param freeSwap long * @param totalMcp long * @param freeMcp long - * @param totalGpu long - * @param freeGpu long + * @param totalGpuMemory long + * @param freeGpuMemory long * @param load int * @param os String */ @@ -272,7 +272,7 @@ void updateHostStats(HostInterface host, long totalMemory, long freeMemory, long totalSwap, long freeSwap, long totalMcp, long freeMcp, - long totalGpu, long freeGpu, + long totalGpuMemory, long freeGpuMemory, int load, Timestamp bootTime, String os); /** @@ -293,6 +293,16 @@ void updateHostStats(HostInterface host, */ int getStrandedCoreUnits(HostInterface h); + /** + * Return the number of whole stranded gpus on this host. The must have + * less than Dispacher.MEM_STRANDED_THRESHHOLD for the gpus to be + * considered stranded. + * + * @param h HostInterface + * @return int + */ + int getStrandedGpus(HostInterface h); + /** * Return true if the host is preferring a particular show. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/JobDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/JobDao.java index 4ffaf2f43..3882f95a7 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/JobDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/JobDao.java @@ -59,6 +59,24 @@ public interface JobDao { */ public void updateMinCores(GroupInterface g, int cores); + /** + * Updates all jobs in the speficed group to the + * max gpu value. + * + * @param g + * @param gpu + */ + public void updateMaxGpus(GroupInterface g, int gpus); + + /** + * Updates all jobs in the specifid group to the + * min gpu value. + * + * @param g + * @param gpu + */ + public void updateMinGpus(GroupInterface g, int gpus); + /** * Updates all jobs in the specified group to the * set priority. @@ -255,6 +273,39 @@ public interface JobDao { */ boolean isOverMaxCores(JobInterface job, int coreUnits); + /** + * reteurns true if job is over its minimum gpus + * + * @param job + * @return boolean + */ + boolean isOverMinGpus(JobInterface job); + + /** + * returns true if job is over max gpus + * + * @param job + * @return + */ + boolean isOverMaxGpus(JobInterface job); + + /** + * returns true if job is at its max gpus + * + * @param job + * @return + */ + boolean isAtMaxGpus(JobInterface job); + + /** + * Return true if adding given gpus to the job + * will set the job over its max gpus value. + * + * @param job + * @param gpus + * @return + */ + boolean isOverMaxGpus(JobInterface job, int gpus); /** * sets the jobs new priority value @@ -280,6 +331,22 @@ public interface JobDao { */ void updateMaxCores(JobInterface j, int v); + /** + * sets the jobs new min gpu value + * + * @param j + * @param v + */ + void updateMinGpus(JobInterface j, int v); + + /** + * sets the jobs new max gpu value + * + * @param j + * @param v + */ + void updateMaxGpus(JobInterface j, int v); + /** * Update a job's paused state * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/LayerDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/LayerDao.java index 243cbbce9..ba8295462 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/LayerDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/LayerDao.java @@ -131,6 +131,15 @@ public interface LayerDao { */ void updateLayerMinCores(LayerInterface layer, int val); + + /** + * update the number of gpus the layer requires + * + * @param layer + * @param val + */ + void updateLayerMinGpus(LayerInterface layer, int val); + /** * update the amount of memory required by all subsequent * running frames in the specified layer. @@ -147,7 +156,7 @@ public interface LayerDao { * @param layer * @param val */ - void updateLayerMinGpu(LayerInterface layer, long gpu); + void updateLayerMinGpuMemory(LayerInterface layer, long val); /** * Update a layer with new host tags. @@ -207,9 +216,9 @@ public interface LayerDao { * value is larger than the current value * * @param layer - * @param gpu + * @param val */ - void increaseLayerMinGpu(LayerInterface layer, long gpu); + void increaseLayerMinGpuMemory(LayerInterface layer, long val); /** * Tries to find a max RSS value for layer in the specified job. The @@ -256,10 +265,10 @@ public interface LayerDao { * job with the new gpu requirement. * * @param job - * @param gpu + * @param mem * @param type */ - void updateMinGpu(JobInterface job, long gpu, LayerType type); + void updateMinGpuMemory(JobInterface job, long mem, LayerType type); /** * Update all layers of the set type in the specified job @@ -271,6 +280,16 @@ public interface LayerDao { */ void updateMinCores(JobInterface job, int cores, LayerType type); + /** + * Update all layers of the set type in the specified job + * with the new min cores requirement. + * + * @param job + * @param gpus + * @param type + */ + void updateMinGpus(JobInterface job, int gpus, LayerType type); + /** * Update a layer's max cores value, which limits how * much threading can go on. @@ -395,6 +414,16 @@ public interface LayerDao { */ void updateLayerMaxCores(LayerInterface layer, int val); + /** + * Set the layer's max gpus value to the given int. The + * max gpu value will not allow the dispatcher to + * book over the given number of gpu. + * + * @param layer + * @param val + */ + void updateLayerMaxGpus(LayerInterface layer, int val); + /** * Add a limit to the given layer. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/ProcDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/ProcDao.java index 5efdd01d2..31e49a208 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/ProcDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/ProcDao.java @@ -54,7 +54,7 @@ public interface ProcDao { * @return */ - long getReservedGpu(ProcInterface proc); + long getReservedGpuMemory(ProcInterface proc); /** * Return the proc that has exceeded its reserved memory by the largest factor. diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/ShowDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/ShowDao.java index 1853662a5..f0cdcbba7 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/ShowDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/ShowDao.java @@ -81,6 +81,21 @@ public interface ShowDao { */ void updateShowDefaultMaxCores(ShowInterface s, int val); + /** + * + * @param s + * @param val + */ + void updateShowDefaultMinGpus(ShowInterface s, int val); + + /** + * + * @param s + * @param val + */ + void updateShowDefaultMaxGpus(ShowInterface s, int val); + + /** * Disabling this would stop new proc assignement. The show would get no new * procs, but any procs already assigned to a job would continue to diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/BookingDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/BookingDaoJdbc.java index 550cddc17..08e1634aa 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/BookingDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/BookingDaoJdbc.java @@ -55,12 +55,14 @@ public class BookingDaoJdbc extends "int_mem_idle,"+ "int_cores_max,"+ "int_cores_idle,"+ - "int_gpu_idle,"+ - "int_gpu_max,"+ + "int_gpu_mem_idle,"+ + "int_gpu_mem_max,"+ + "int_gpus_max,"+ + "int_gpus_idle,"+ "int_threads "+ ") " + "VALUES " + - "(?,?,?,?,?,?,?,?,?,?,?,?,?)"; + "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; @Override public void insertLocalHostAssignment(HostInterface h, JobInterface job, LocalHostAssignment l) { @@ -71,7 +73,8 @@ public void insertLocalHostAssignment(HostInterface h, JobInterface job, LocalHo l.setType(RenderPartitionType.JOB_PARTITION); l.setIdleCoreUnits(l.getMaxCoreUnits()); l.setIdleMemory(l.getMaxMemory()); - l.setIdleGpu(l.getMaxGpu()); + l.setIdleGpuUnits(l.getMaxGpuUnits()); + l.setIdleGpuMemory(l.getMaxGpuMemory()); getJdbcTemplate().update( INSERT_LOCAL_JOB_ASSIGNMENT, @@ -85,8 +88,10 @@ public void insertLocalHostAssignment(HostInterface h, JobInterface job, LocalHo l.getMaxMemory(), l.getMaxCoreUnits(), l.getMaxCoreUnits(), - l.getMaxGpu(), - l.getMaxGpu(), + l.getMaxGpuMemory(), + l.getMaxGpuMemory(), + l.getMaxGpuUnits(), + l.getMaxGpuUnits(), l.getThreads()); } @@ -100,7 +105,8 @@ public void insertLocalHostAssignment(HostInterface h, LayerInterface layer, Loc l.setType(RenderPartitionType.LAYER_PARTITION); l.setIdleCoreUnits(l.getMaxCoreUnits()); l.setIdleMemory(l.getMaxMemory()); - l.setIdleGpu(l.getMaxGpu()); + l.setIdleGpuUnits(l.getMaxGpuUnits()); + l.setIdleGpuMemory(l.getMaxGpuMemory()); getJdbcTemplate().update( INSERT_LOCAL_JOB_ASSIGNMENT, @@ -114,8 +120,10 @@ public void insertLocalHostAssignment(HostInterface h, LayerInterface layer, Loc l.getMaxMemory(), l.getMaxCoreUnits(), l.getMaxCoreUnits(), - l.getMaxGpu(), - l.getMaxGpu(), + l.getMaxGpuMemory(), + l.getMaxGpuMemory(), + l.getMaxGpuUnits(), + l.getMaxGpuUnits(), l.getThreads()); } @@ -130,7 +138,8 @@ public void insertLocalHostAssignment(HostInterface h, FrameInterface frame, Loc l.setType(RenderPartitionType.FRAME_PARTITION); l.setIdleCoreUnits(l.getMaxCoreUnits()); l.setIdleMemory(l.getMaxMemory()); - l.setIdleGpu(l.getMaxGpu()); + l.setIdleGpuUnits(l.getMaxGpuUnits()); + l.setIdleGpuMemory(l.getMaxGpuMemory()); getJdbcTemplate().update( INSERT_LOCAL_JOB_ASSIGNMENT, @@ -144,8 +153,10 @@ public void insertLocalHostAssignment(HostInterface h, FrameInterface frame, Loc l.getMaxMemory(), l.getMaxCoreUnits(), l.getMaxCoreUnits(), - l.getMaxGpu(), - l.getMaxGpu(), + l.getMaxGpuMemory(), + l.getMaxGpuMemory(), + l.getMaxGpuUnits(), + l.getMaxGpuUnits(), l.getThreads()); } public static final RowMapper LJA_MAPPER = @@ -155,11 +166,13 @@ public LocalHostAssignment mapRow(final ResultSet rs, int rowNum) throws SQLExce l.id = rs.getString("pk_host_local"); l.setMaxCoreUnits(rs.getInt("int_cores_max")); l.setMaxMemory(rs.getLong("int_mem_max")); - l.setMaxGpu(rs.getLong("int_gpu_max")); + l.setMaxGpuUnits(rs.getInt("int_gpus_max")); + l.setMaxGpuMemory(rs.getLong("int_gpu_mem_max")); l.setThreads(rs.getInt("int_threads")); l.setIdleCoreUnits(rs.getInt("int_cores_idle")); l.setIdleMemory(rs.getLong("int_mem_idle")); - l.setIdleGpu(rs.getLong("int_gpu_idle")); + l.setIdleGpuUnits(rs.getInt("int_gpus_idle")); + l.setIdleGpuMemory(rs.getLong("int_gpu_mem_idle")); l.setJobId(rs.getString("pk_job")); l.setLayerId(rs.getString("pk_layer")); l.setFrameId(rs.getString("pk_frame")); @@ -180,8 +193,10 @@ public LocalHostAssignment mapRow(final ResultSet rs, int rowNum) throws SQLExce "int_mem_max,"+ "int_cores_idle,"+ "int_cores_max,"+ - "int_gpu_idle,"+ - "int_gpu_max,"+ + "int_gpu_mem_idle,"+ + "int_gpu_mem_max,"+ + "int_gpus_idle,"+ + "int_gpus_max,"+ "int_threads, "+ "str_type " + "FROM " + @@ -257,6 +272,13 @@ public int getCoreUsageDifference(LocalHostAssignment l, int coreUnits) { Integer.class, coreUnits, l.getId()); } + @Override + public int getGpuUsageDifference(LocalHostAssignment l, int gpuUnits) { + return getJdbcTemplate().queryForObject( + "SELECT ? - int_gpus_max FROM host_local WHERE pk_host_local=?", + Integer.class, gpuUnits, l.getId()); + } + private static final String UPDATE_MAX_CORES = "UPDATE " + "host_local " + @@ -272,6 +294,21 @@ public boolean updateMaxCores(LocalHostAssignment l, int coreUnits) { coreUnits, coreUnits, l.getId()) > 0; } + private static final String UPDATE_MAX_GPUS = + "UPDATE " + + "host_local " + + "SET " + + "int_gpus_idle = int_gpus_idle + (? - int_gpus_max), " + + "int_gpus_max = ? "+ + "WHERE " + + "pk_host_local = ? "; + + @Override + public boolean updateMaxGpus(LocalHostAssignment l, int gpuUnits) { + return getJdbcTemplate().update(UPDATE_MAX_GPUS, + gpuUnits, gpuUnits, l.getId()) > 0; + } + private static final String UPDATE_MAX_MEMORY = "UPDATE " + "host_local " + @@ -287,19 +324,19 @@ public boolean updateMaxMemory(LocalHostAssignment l, long maxMemory) { UPDATE_MAX_MEMORY, maxMemory, maxMemory, l.getId()) > 0; } - private static final String UPDATE_MAX_GPU = + private static final String UPDATE_MAX_GPU_MEMORY = "UPDATE " + "host_local " + "SET " + - "int_gpu_idle = int_gpu_idle + (? - int_gpu_max), " + - "int_gpu_max = ? "+ + "int_gpu_mem_idle = int_gpu_mem_idle + (? - int_gpu_mem_max), " + + "int_gpu_mem_max = ? "+ "WHERE " + "pk_host_local = ? "; @Override - public boolean updateMaxGpu(LocalHostAssignment l, long maxGpu) { + public boolean updateMaxGpuMemory(LocalHostAssignment l, long maxGpuMemory) { return getJdbcTemplate().update( - UPDATE_MAX_GPU, maxGpu, maxGpu, l.getId()) > 0; + UPDATE_MAX_GPU_MEMORY, maxGpuMemory, maxGpuMemory, l.getId()) > 0; } @Override @@ -331,6 +368,26 @@ public boolean allocateCoresFromHost(HostInterface h, int cores) { } + /** + * + * @param h HostInterface + * @param gpus int + * @return boolean + */ + @Override + public boolean allocateGpusFromHost(HostInterface h, int gpus) { + + try { + return getJdbcTemplate().update( + "UPDATE host SET int_gpus_idle = int_gpus_idle - ? " + + "WHERE pk_host = ?", + gpus, h.getHostId()) > 0; + } catch (DataAccessException e) { + throw new ResourceReservationFailureException("Failed to allocate " + + gpus + " GPU from host, " + e); + } + } + /** * * @param h HostInterface @@ -349,12 +406,31 @@ public boolean deallocateCoresFromHost(HostInterface h, int cores) { } } + /** + * + * @param h HostInterface + * @param gpus int + * @return boolean + */ + @Override + public boolean deallocateGpusFromHost(HostInterface h, int gpus) { + try { + return getJdbcTemplate().update( + "UPDATE host SET int_gpus_idle = int_gpus_idle + ? WHERE pk_host = ?", + gpus, h.getHostId()) > 0; + } catch (DataAccessException e) { + throw new ResourceReservationFailureException("Failed to de-allocate " + + gpus + " GPU from host, " + e); + } + } + @Override public boolean hasResourceDeficit(HostInterface host) { return getJdbcTemplate().queryForObject( "SELECT COUNT(1) FROM host_local WHERE " + "(int_cores_max < int_cores_max - int_cores_idle OR " + - "int_gpu_max < int_gpu_max - int_gpu_idle OR " + + "int_gpus_max < int_gpus_max - int_gpus_idle OR " + + "int_gpu_mem_max < int_gpu_mem_max - int_gpu_mem_idle OR " + "int_mem_max < int_mem_max - int_mem_idle) AND " + "host_local.pk_host= ?", Integer.class, host.getHostId()) > 0; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java index 0443f691a..fb267ddbd 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java @@ -52,6 +52,12 @@ public class DispatchQuery { "OR " + "folder_resource.int_cores < folder_resource.int_max_cores " + ") " + + "AND " + + "(" + + "folder_resource.int_max_gpus = -1 " + + "OR " + + "folder_resource.int_gpus < folder_resource.int_max_gpus " + + ") " + "AND job.str_state = 'PENDING' " + "AND job.b_paused = false " + "AND job.pk_show = ? " + @@ -66,8 +72,10 @@ public class DispatchQuery { "AND layer.int_cores_min <= ? " + "AND layer.int_mem_min <= ? " + "AND (CASE WHEN layer.b_threadable = true THEN 1 ELSE 0 END) >= ? " + - "AND layer.int_gpu_min BETWEEN ? AND ? " + + "AND layer.int_gpus_min <= ? " + + "AND layer.int_gpu_mem_min BETWEEN ? AND ? " + "AND job_resource.int_cores + layer.int_cores_min < job_resource.int_max_cores " + + "AND job_resource.int_gpus + layer.int_gpus_min < job_resource.int_max_gpus " + "AND host.str_tags ~* ('(?x)' || layer.str_tags) " + "AND host.str_name = ? " + "AND layer.pk_layer IN (" + @@ -165,7 +173,7 @@ public class DispatchQuery { "AND " + "l.int_mem_min <= host_local.int_mem_idle " + "AND " + - "l.int_gpu_min <= host_local.int_gpu_idle " + + "l.int_gpu_mem_min <= host_local.int_gpu_mem_idle " + "AND " + "l.pk_layer IN (" + "SELECT " + @@ -219,6 +227,8 @@ public class DispatchQuery { "folder.pk_folder = folder_resource.pk_folder " + "AND " + "(folder_resource.int_max_cores = -1 OR folder_resource.int_cores < folder_resource.int_max_cores) " + + "AND " + + "(folder_resource.int_max_gpus = -1 OR folder_resource.int_gpus < folder_resource.int_max_gpus) " + "AND " + "job_resource.float_tier < 1.00 " + "AND " + @@ -263,7 +273,9 @@ public class DispatchQuery { "AND " + "l.int_mem_min <= ? " + "AND " + - "l.int_gpu_min = ? " + + "l.int_gpus_min <= ? " + + "AND " + + "l.int_gpu_mem_min = ? " + "AND " + "h.str_tags ~* ('(?x)' || l.str_tags) " + "AND " + @@ -320,10 +332,14 @@ public class DispatchQuery { "folder.pk_folder = folder_resource.pk_folder " + "AND " + "(folder_resource.int_max_cores = -1 OR folder_resource.int_cores < folder_resource.int_max_cores) " + + "AND " + + "(folder_resource.int_max_gpus = -1 OR folder_resource.int_gpus < folder_resource.int_max_gpus) " + "AND " + "job_resource.int_priority > ?" + "AND " + "job_resource.int_cores < job_resource.int_max_cores " + + "AND " + + "job_resource.int_gpus < job_resource.int_max_gpus " + "AND " + "job.str_state = 'PENDING' " + "AND " + @@ -360,7 +376,9 @@ public class DispatchQuery { "AND " + "l.int_mem_min <= ? " + "AND " + - "l.int_gpu_min = ? " + + "l.int_gpus_min <= ? " + + "AND " + + "l.int_gpu_mem_min = ? " + "AND " + "h.str_tags ~* ('(?x)' || l.str_tags) " + "AND " + @@ -417,7 +435,9 @@ public class DispatchQuery { "int_cores_min, " + "int_cores_max, " + "int_mem_min, " + - "int_gpu_min, " + + "int_gpus_min, " + + "int_gpus_max, " + + "int_gpu_mem_min, " + "str_cmd, " + "str_range, " + "int_chunk_size, " + @@ -450,7 +470,9 @@ public class DispatchQuery { "layer.int_cores_min, " + "layer.int_cores_max, " + "layer.int_mem_min, " + - "layer.int_gpu_min, " + + "layer.int_gpus_min, " + + "layer.int_gpus_max, " + + "layer.int_gpu_mem_min, " + "layer.str_cmd, " + "layer.str_range, " + "layer.int_chunk_size, " + @@ -468,7 +490,9 @@ public class DispatchQuery { "AND " + "layer.int_mem_min <= ? " + "AND " + - "layer.int_gpu_min BETWEEN ? AND ? " + + "layer.int_gpus_min <= ? " + + "AND " + + "layer.int_gpu_mem_min BETWEEN ? AND ? " + "AND " + "frame.str_state='WAITING' " + "AND " + @@ -524,9 +548,11 @@ public class DispatchQuery { "layer_type, " + "int_cores_min, " + "int_cores_max, " + + "int_gpus_min, " + + "int_gpus_max, " + "b_threadable, " + "int_mem_min, " + - "int_gpu_min, " + + "int_gpu_mem_min, " + "str_cmd, " + "str_range, " + "int_chunk_size, " + @@ -557,9 +583,11 @@ public class DispatchQuery { "layer.str_type AS layer_type, " + "layer.int_cores_min, " + "layer.int_cores_max, " + + "layer.int_gpus_min, " + + "layer.int_gpus_max, " + "layer.b_threadable, " + "layer.int_mem_min, " + - "layer.int_gpu_min, " + + "layer.int_gpu_mem_min, " + "layer.str_cmd, " + "layer.str_range, " + "layer.int_chunk_size, " + @@ -579,7 +607,9 @@ public class DispatchQuery { "AND " + "(CASE WHEN layer.b_threadable = true THEN 1 ELSE 0 END) >= ? " + "AND " + - "layer.int_gpu_min BETWEEN ? AND ? " + + "layer.int_gpus_min <= ? " + + "AND " + + "layer.int_gpu_mem_min BETWEEN ? AND ? " + "AND " + "frame.str_state='WAITING' " + "AND " + @@ -636,7 +666,9 @@ public class DispatchQuery { "int_cores_min, " + "int_cores_max, " + "int_mem_min, " + - "int_gpu_min, " + + "int_gpus_min, " + + "int_gpus_max, " + + "int_gpu_mem_min, " + "str_cmd, " + "str_range, " + "int_chunk_size, " + @@ -669,7 +701,9 @@ public class DispatchQuery { "layer.int_cores_min, " + "layer.int_cores_max, " + "layer.int_mem_min, " + - "layer.int_gpu_min, " + + "layer.int_gpus_min, " + + "layer.int_gpus_max, " + + "layer.int_gpu_mem_min, " + "layer.str_cmd, " + "layer.str_range, " + "layer.int_chunk_size, " + @@ -685,7 +719,7 @@ public class DispatchQuery { "AND " + "layer.int_mem_min <= ? " + "AND " + - "layer.int_gpu_min <= ? " + + "layer.int_gpu_mem_min <= ? " + "AND " + "frame.str_state='WAITING' " + "AND " + @@ -739,9 +773,11 @@ public class DispatchQuery { "layer_type, " + "int_cores_min, " + "int_cores_max, " + + "int_gpus_min, " + + "int_gpus_max, " + "b_threadable, " + "int_mem_min, " + - "int_gpu_min, " + + "int_gpu_mem_min, " + "str_cmd, " + "str_range, " + "int_chunk_size, " + @@ -774,7 +810,9 @@ public class DispatchQuery { "layer.int_cores_max, " + "layer.b_threadable, " + "layer.int_mem_min, " + - "layer.int_gpu_min, " + + "layer.int_gpus_min, " + + "layer.int_gpus_max, " + + "layer.int_gpu_mem_min, " + "layer.str_cmd, " + "layer.str_range, " + "layer.int_chunk_size, " + @@ -790,7 +828,7 @@ public class DispatchQuery { "AND " + "layer.int_mem_min <= ? " + "AND " + - "layer.int_gpu_min <= ? " + + "layer.int_gpu_mem_min <= ? " + "AND " + "frame.str_state='WAITING' " + "AND " + @@ -849,7 +887,9 @@ public class DispatchQuery { "int_cores_min, " + "int_cores_max, " + "int_mem_min, " + - "int_gpu_min, " + + "int_gpus_min, " + + "int_gpus_max, " + + "int_gpu_mem_min, " + "str_cmd, " + "str_range, " + "int_chunk_size, " + @@ -882,7 +922,9 @@ public class DispatchQuery { "layer.int_cores_min, " + "layer.int_cores_max, " + "layer.int_mem_min, " + - "layer.int_gpu_min, " + + "layer.int_gpus_min, " + + "layer.int_gpus_max, " + + "layer.int_gpu_mem_min, " + "layer.str_cmd, " + "layer.str_range, " + "layer.int_chunk_size, " + @@ -900,7 +942,9 @@ public class DispatchQuery { "AND " + "layer.int_mem_min <= ? " + "AND " + - "layer.int_gpu_min = ? " + + "layer.int_gpus_min <= ? " + + "AND " + + "layer.int_gpu_mem_min <= ? " + "AND " + "frame.str_state='WAITING' " + "AND " + @@ -958,7 +1002,9 @@ public class DispatchQuery { "int_cores_max, " + "b_threadable, " + "int_mem_min, " + - "int_gpu_min, " + + "int_gpus_min, " + + "int_gpus_max, " + + "int_gpu_mem_min, " + "str_cmd, " + "str_range, " + "int_chunk_size, " + @@ -991,7 +1037,9 @@ public class DispatchQuery { "layer.int_cores_max, " + "layer.b_threadable, " + "layer.int_mem_min, " + - "layer.int_gpu_min, " + + "layer.int_gpus_min, " + + "layer.int_gpus_max, " + + "layer.int_gpu_mem_min, " + "layer.str_cmd, " + "layer.str_range, " + "layer.int_chunk_size, " + @@ -1011,7 +1059,9 @@ public class DispatchQuery { "AND " + "(CASE WHEN layer.b_threadable = true THEN 1 ELSE 0 END) >= ? " + "AND " + - "layer.int_gpu_min <= ? " + + "layer.int_gpus_min <= ? " + + "AND " + + "layer.int_gpu_mem_min <= ? " + "AND " + "frame.str_state='WAITING' " + "AND " + @@ -1068,7 +1118,9 @@ public class DispatchQuery { "int_cores_min, " + "int_cores_max, " + "int_mem_min, " + - "int_gpu_min, " + + "int_gpus_min, " + + "int_gpus_max, " + + "int_gpu_mem_min, " + "str_cmd, " + "str_range, " + "int_chunk_size, " + @@ -1100,7 +1152,9 @@ public class DispatchQuery { "layer.b_threadable, " + "layer.int_cores_min, " + "layer.int_mem_min, " + - "layer.int_gpu_min, " + + "layer.int_gpus_min, " + + "layer.int_gpus_max, " + + "layer.int_gpu_mem_min, " + "layer.int_cores_max, " + "layer.str_cmd, " + "layer.str_range, " + @@ -1117,7 +1171,7 @@ public class DispatchQuery { "AND " + "layer.int_mem_min <= ? " + "AND " + - "layer.int_gpu_min <= ? " + + "layer.int_gpu_mem_min <= ? " + "AND " + "frame.str_state='WAITING' " + "AND " + @@ -1173,7 +1227,9 @@ public class DispatchQuery { "int_cores_max, " + "b_threadable, " + "int_mem_min, " + - "int_gpu_min, " + + "int_gpus_min, " + + "int_gpus_max, " + + "int_gpu_mem_min, " + "str_cmd, " + "str_range, " + "int_chunk_size, " + @@ -1206,7 +1262,9 @@ public class DispatchQuery { "layer.int_cores_max, " + "layer.b_threadable, " + "layer.int_mem_min, " + - "layer.int_gpu_min, " + + "layer.int_gpus_min, " + + "layer.int_gpus_max, " + + "layer.int_gpu_mem_min, " + "layer.str_cmd, " + "layer.str_range, " + "layer.int_chunk_size, " + @@ -1222,7 +1280,7 @@ public class DispatchQuery { "AND " + "layer.int_mem_min <= ? " + "AND " + - "layer.int_gpu_min <= ? " + + "layer.int_gpu_mem_min <= ? " + "AND " + "frame.str_state='WAITING' " + "AND " + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java index d3e50525d..10f506e9b 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java @@ -190,11 +190,12 @@ private Set findDispatchJobs(DispatchHost host, int numJobs, boolean shu s.getShowId(), host.getFacilityId(), host.os, host.idleCores, host.idleMemory, threadMode(host.threadMode), - (host.idleGpu > 0) ? 1: 0, host.idleGpu, + host.idleGpus, + (host.idleGpuMemory > 0) ? 1 : 0, host.idleGpuMemory, host.getName(), numJobs * 10)); if (result.size() < 1) { - if (host.gpu == 0) { + if (host.gpuMemory == 0) { s.skip(host.tags, host.idleCores, host.idleMemory); } } @@ -225,7 +226,8 @@ public Set findDispatchJobs(DispatchHost host, GroupInterface g) { g.getGroupId(),host.getFacilityId(), host.os, host.idleCores, host.idleMemory, threadMode(host.threadMode), - (host.idleGpu > 0) ? 1: 0, host.idleGpu, + host.idleGpus, + (host.idleGpuMemory > 0) ? 1 : 0, host.idleGpuMemory, host.getName(), 50)); return result; @@ -240,7 +242,7 @@ public List findNextDispatchFrames(JobInterface job, FIND_LOCAL_DISPATCH_FRAME_BY_JOB_AND_PROC, FrameDaoJdbc.DISPATCH_FRAME_MAPPER, proc.memoryReserved, - proc.gpuReserved, + proc.gpuMemoryReserved, job.getJobId(), limit); } @@ -250,7 +252,8 @@ public List findNextDispatchFrames(JobInterface job, FrameDaoJdbc.DISPATCH_FRAME_MAPPER, proc.coresReserved, proc.memoryReserved, - (proc.gpuReserved > 0) ? 1: 0, proc.gpuReserved, + proc.gpusReserved, + (proc.gpuMemoryReserved > 0) ? 1 : 0, proc.gpuMemoryReserved, job.getJobId(), proc.hostName, job.getJobId(), limit); } @@ -264,7 +267,7 @@ public List findNextDispatchFrames(JobInterface job, return getJdbcTemplate().query( FIND_LOCAL_DISPATCH_FRAME_BY_JOB_AND_HOST, FrameDaoJdbc.DISPATCH_FRAME_MAPPER, - host.idleMemory, host.idleGpu, job.getJobId(), + host.idleMemory, host.idleGpuMemory, job.getJobId(), limit); } else { @@ -273,7 +276,8 @@ public List findNextDispatchFrames(JobInterface job, FrameDaoJdbc.DISPATCH_FRAME_MAPPER, host.idleCores, host.idleMemory, threadMode(host.threadMode), - (host.idleGpu > 0) ? 1: 0, host.idleGpu, + host.idleGpus, + (host.idleGpuMemory > 0) ? 1 : 0, host.idleGpuMemory, job.getJobId(), host.getName(), job.getJobId(), limit); } @@ -288,7 +292,7 @@ public List findNextDispatchFrames(LayerInterface layer, return getJdbcTemplate().query( FIND_LOCAL_DISPATCH_FRAME_BY_LAYER_AND_PROC, FrameDaoJdbc.DISPATCH_FRAME_MAPPER, - proc.memoryReserved, proc.gpuReserved, + proc.memoryReserved, proc.gpuMemoryReserved, layer.getLayerId(), limit); } @@ -297,7 +301,7 @@ public List findNextDispatchFrames(LayerInterface layer, FIND_DISPATCH_FRAME_BY_LAYER_AND_PROC, FrameDaoJdbc.DISPATCH_FRAME_MAPPER, proc.coresReserved, proc.memoryReserved, - proc.gpuReserved, + proc.gpusReserved, proc.gpuMemoryReserved, layer.getLayerId(), layer.getLayerId(), proc.hostName, limit); } @@ -311,7 +315,7 @@ public List findNextDispatchFrames(LayerInterface layer, return getJdbcTemplate().query( FIND_LOCAL_DISPATCH_FRAME_BY_LAYER_AND_HOST, FrameDaoJdbc.DISPATCH_FRAME_MAPPER, - host.idleMemory, host.idleGpu, layer.getLayerId(), + host.idleMemory, host.idleGpuMemory, layer.getLayerId(), limit); } else { @@ -320,7 +324,7 @@ public List findNextDispatchFrames(LayerInterface layer, FrameDaoJdbc.DISPATCH_FRAME_MAPPER, host.idleCores, host.idleMemory, threadMode(host.threadMode), - host.idleGpu, layer.getLayerId(), layer.getLayerId(), + host.idleGpus, host.idleGpuMemory, layer.getLayerId(), layer.getLayerId(), host.getName(), limit); } } @@ -345,7 +349,7 @@ public boolean findUnderProcedJob(JobInterface excludeJob, VirtualProc proc) { Integer.class, excludeJob.getShowId(), proc.getFacilityId(), proc.os, excludeJob.getShowId(), proc.getFacilityId(), proc.os, - proc.coresReserved, proc.memoryReserved, proc.gpuReserved, + proc.coresReserved, proc.memoryReserved, proc.gpusReserved, proc.gpuMemoryReserved, proc.hostName) > 0; } catch (org.springframework.dao.EmptyResultDataAccessException e) { return false; @@ -363,7 +367,7 @@ public boolean higherPriorityJobExists(JobDetail baseJob, VirtualProc proc) { HIGHER_PRIORITY_JOB_BY_FACILITY_EXISTS, Boolean.class, baseJob.priority, proc.getFacilityId(), proc.os, proc.getFacilityId(), proc.os, - proc.coresReserved, proc.memoryReserved, proc.gpuReserved, + proc.coresReserved, proc.memoryReserved, proc.gpusReserved, proc.gpuMemoryReserved, proc.hostName); } catch (org.springframework.dao.EmptyResultDataAccessException e) { return false; @@ -384,7 +388,8 @@ public Set findDispatchJobs(DispatchHost host, show.getShowId(), host.getFacilityId(), host.os, host.idleCores, host.idleMemory, threadMode(host.threadMode), - (host.idleGpu > 0) ? 1: 0, host.idleGpu, + host.idleGpus, + (host.idleGpuMemory > 0) ? 1 : 0, host.idleGpuMemory, host.getName(), numJobs * 10)); return result; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java index d7aeef3b5..e905d8e35 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java @@ -66,7 +66,9 @@ public class FrameDaoJdbc extends JdbcDaoSupport implements FrameDao { "ts_updated = current_timestamp, " + "int_version = int_version + 1, " + "int_total_past_core_time = int_total_past_core_time + " + - "round(INTERVAL_TO_SECONDS(current_timestamp - ts_started) * int_cores / 100) " + + "round(INTERVAL_TO_SECONDS(current_timestamp - ts_started) * int_cores / 100)," + + "int_total_past_gpu_time = int_total_past_gpu_time + " + + "round(INTERVAL_TO_SECONDS(current_timestamp - ts_started) * int_gpus) " + "WHERE " + "frame.pk_frame = ? " + "AND " + @@ -93,7 +95,9 @@ public boolean updateFrameStopped(FrameInterface frame, FrameState state, "int_mem_max_used = ?, " + "int_version = int_version + 1, " + "int_total_past_core_time = int_total_past_core_time + " + - "round(INTERVAL_TO_SECONDS(current_timestamp + interval '1' second - ts_started) * int_cores / 100) " + + "round(INTERVAL_TO_SECONDS(current_timestamp + interval '1' second - ts_started) * int_cores / 100), " + + "int_total_past_gpu_time = int_total_past_gpu_time + " + + "round(INTERVAL_TO_SECONDS(current_timestamp + interval '1' second - ts_started) * int_gpus) " + "WHERE " + "frame.pk_frame = ? " + "AND " + @@ -149,7 +153,8 @@ public boolean updateFrameCleared(FrameInterface frame) { "str_host = ?, " + "int_cores = ?, " + "int_mem_reserved = ?, " + - "int_gpu_reserved = ?, " + + "int_gpus = ?, " + + "int_gpu_mem_reserved = ?, " + "ts_updated = current_timestamp, " + "ts_started = current_timestamp, " + "ts_stopped = null, " + @@ -200,7 +205,7 @@ public void updateFrameStarted(VirtualProc proc, FrameInterface frame) { int result = getJdbcTemplate().update(UPDATE_FRAME_STARTED, FrameState.RUNNING.toString(), proc.hostName, proc.coresReserved, - proc.memoryReserved, proc.gpuReserved, frame.getFrameId(), + proc.memoryReserved, proc.gpusReserved, proc.gpuMemoryReserved, frame.getFrameId(), FrameState.WAITING.toString(), frame.getVersion()); if (result == 0) { @@ -226,7 +231,8 @@ public void updateFrameStarted(VirtualProc proc, FrameInterface frame) { "str_host=?, " + "int_cores=?, "+ "int_mem_reserved = ?, " + - "int_gpu_reserved = ?, " + + "int_gpus = ?, " + + "int_gpu_mem_reserved = ?, " + "ts_updated = current_timestamp, " + "ts_started = current_timestamp, " + "ts_stopped = null, "+ @@ -240,7 +246,7 @@ public void updateFrameStarted(VirtualProc proc, FrameInterface frame) { public boolean updateFrameFixed(VirtualProc proc, FrameInterface frame) { return getJdbcTemplate().update(UPDATE_FRAME_FIXED, FrameState.RUNNING.toString(), proc.hostName, proc.coresReserved, - proc.memoryReserved, proc.gpuReserved, frame.getFrameId()) == 1; + proc.memoryReserved, proc.gpusReserved, proc.gpuMemoryReserved, frame.getFrameId()) == 1; } @Override @@ -276,7 +282,9 @@ public DispatchFrame mapRow(ResultSet rs, int rowNum) throws SQLException { frame.maxCores = rs.getInt("int_cores_max"); frame.threadable = rs.getBoolean("b_threadable"); frame.minMemory = rs.getLong("int_mem_min"); - frame.minGpu = rs.getLong("int_gpu_min"); + frame.minGpus = rs.getInt("int_gpus_min"); + frame.maxGpus = rs.getInt("int_gpus_max"); + frame.minGpuMemory = rs.getLong("int_gpu_mem_min"); frame.version = rs.getInt("int_version"); frame.services = rs.getString("str_services"); return frame; @@ -308,7 +316,9 @@ public DispatchFrame mapRow(ResultSet rs, int rowNum) throws SQLException { "layer.int_cores_max,"+ "layer.b_threadable,"+ "layer.int_mem_min, "+ - "layer.int_gpu_min, "+ + "layer.int_gpus_min,"+ + "layer.int_gpus_max,"+ + "layer.int_gpu_mem_min, "+ "layer.str_range, "+ "layer.int_chunk_size, " + "layer.str_services " + @@ -402,7 +412,7 @@ public FrameDetail mapRow(ResultSet rs, int rowNum) throws SQLException { frame.version = rs.getInt("int_version"); if (rs.getString("str_host") != null) { - frame.lastResource = String.format("%s/%d",rs.getString("str_host"),rs.getInt("int_cores")); + frame.lastResource = String.format("%s/%d/%d",rs.getString("str_host"),rs.getInt("int_cores"),rs.getInt("int_gpus")); } else { frame.lastResource = ""; @@ -931,7 +941,8 @@ public ResourceUsage mapRow(ResultSet rs, int rowNum) throws SQLException { return new ResourceUsage( rs.getLong("int_clock_time"), - rs.getInt("int_cores")); + rs.getInt("int_cores"), + rs.getInt("int_gpus")); } }; @@ -947,7 +958,8 @@ public ResourceUsage getResourceUsage(FrameInterface f) { "SELECT " + "COALESCE(interval_to_seconds(current_timestamp - ts_started), 1) " + "AS int_clock_time, " + - "COALESCE(int_cores, 100) AS int_cores " + + "COALESCE(int_cores, 100) AS int_cores," + + "int_gpus " + "FROM " + "frame " + "WHERE " + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java index 9051131ea..b502bb680 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java @@ -232,6 +232,73 @@ public boolean isOverMinCores(JobInterface job) { Integer.class, job.getJobId()) > 0; } + @Override + public void updateDefaultJobMaxGpus(GroupInterface group, int value) { + if (value <= 0) { value = CueUtil.FEATURE_DISABLED; } + if (value < CueUtil.ONE_CORE && value != CueUtil.FEATURE_DISABLED) { + String msg = "The default max cores for a job must " + + "be greater than a single core"; + throw new IllegalArgumentException(msg); + } + getJdbcTemplate().update( + "UPDATE folder SET int_job_max_gpus=? WHERE pk_folder=?", + value, group.getId()); + } + + @Override + public void updateDefaultJobMinGpus(GroupInterface group, int value) { + if (value <= 0) { value = CueUtil.FEATURE_DISABLED; } + if (value < CueUtil.ONE_CORE && value != CueUtil.FEATURE_DISABLED) { + String msg = "The default min cores for a job must " + + "be greater than a single core"; + throw new IllegalArgumentException(msg); + } + getJdbcTemplate().update( + "UPDATE folder SET int_job_min_gpu=? WHERE pk_folder=?", + value, group.getId()); + } + + @Override + public void updateMaxGpus(GroupInterface group, int value) { + if (value < 0) { value = CueUtil.FEATURE_DISABLED; } + if (value < CueUtil.ONE_CORE && value != CueUtil.FEATURE_DISABLED) { + String msg = "The group max cores feature must " + + "be a whole core or greater, pass in: " + value; + throw new IllegalArgumentException(msg); + } + + getJdbcTemplate().update( + "UPDATE folder_resource SET int_max_gpu=? WHERE pk_folder=?", + value, group.getId()); + } + + @Override + public void updateMinGpus(GroupInterface group, int value) { + if (value < 0) { value = 0; } + getJdbcTemplate().update( + "UPDATE folder_resource SET int_min_gpus=? WHERE pk_folder=?", + value, group.getId()); + } + + private static final String IS_OVER_MIN_GPUS = + "SELECT " + + "COUNT(1) " + + "FROM " + + "job,"+ + "folder_resource fr "+ + "WHERE " + + "job.pk_folder = fr.pk_folder " + + "AND " + + "fr.int_gpus > fr.int_min_gpus " + + "AND "+ + "job.pk_job = ?"; + + @Override + public boolean isOverMinGpus(JobInterface job) { + return getJdbcTemplate().queryForObject(IS_OVER_MIN_GPUS, + Integer.class, job.getJobId()) > 0; + } + @Override public void updateDefaultJobPriority(GroupInterface group, int value) { if (value < 0) { value = CueUtil.FEATURE_DISABLED; } @@ -251,6 +318,8 @@ public void updateDefaultJobPriority(GroupInterface group, int value) { "folder.pk_folder, " + "folder.int_job_max_cores,"+ "folder.int_job_min_cores,"+ + "folder.int_job_max_gpus,"+ + "folder.int_job_min_gpus,"+ "folder.int_job_priority,"+ "folder.str_name,"+ "folder.pk_parent_folder,"+ @@ -258,7 +327,9 @@ public void updateDefaultJobPriority(GroupInterface group, int value) { "folder.pk_dept,"+ "folder_level.int_level, " + "folder_resource.int_min_cores,"+ - "folder_resource.int_max_cores " + + "folder_resource.int_max_cores," + + "folder_resource.int_min_gpus,"+ + "folder_resource.int_max_gpus " + "FROM " + "folder, "+ "folder_level, " + @@ -273,6 +344,8 @@ public void updateDefaultJobPriority(GroupInterface group, int value) { "folder.pk_folder, " + "folder.int_job_max_cores,"+ "folder.int_job_min_cores,"+ + "folder.int_job_max_gpus,"+ + "folder.int_job_min_gpus,"+ "folder.int_job_priority,"+ "folder.str_name,"+ "folder.pk_parent_folder,"+ @@ -280,7 +353,9 @@ public void updateDefaultJobPriority(GroupInterface group, int value) { "folder.pk_dept,"+ "folder_level.int_level, " + "folder_resource.int_min_cores,"+ - "folder_resource.int_max_cores " + + "folder_resource.int_max_cores," + + "folder_resource.int_min_gpus,"+ + "folder_resource.int_max_gpus " + "FROM " + "folder, "+ "folder_level, " + @@ -393,6 +468,8 @@ public GroupDetail mapRow(ResultSet rs, int rowNum) throws SQLException { group.id = rs.getString("pk_folder"); group.jobMaxCores = rs.getInt("int_job_max_cores"); group.jobMinCores = rs.getInt("int_job_min_cores"); + group.jobMaxGpus = rs.getInt("int_job_max_gpus"); + group.jobMinGpus = rs.getInt("int_job_min_gpus"); group.jobPriority = rs.getInt("int_job_priority"); group.name = rs.getString("str_name"); group.parentId = rs.getString("pk_parent_folder"); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java index 1efd6b597..5c106335c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java @@ -71,10 +71,12 @@ public HostEntity mapRow(ResultSet rs, int rowNum) throws SQLException { host.unlockAtBoot = rs.getBoolean("b_unlock_boot"); host.cores = rs.getInt("int_cores"); host.idleCores = rs.getInt("int_cores_idle"); - host.memory = rs.getInt("int_mem"); - host.idleMemory = rs.getInt("int_mem_idle"); - host.gpu = rs.getInt("int_gpu"); - host.idleGpu = rs.getInt("int_gpu_idle"); + host.memory = rs.getLong("int_mem"); + host.idleMemory = rs.getLong("int_mem_idle"); + host.gpus = rs.getInt("int_gpus"); + host.idleGpus = rs.getInt("int_gpus_idle"); + host.gpuMemory = rs.getLong("int_gpu_mem"); + host.idleGpuMemory = rs.getLong("int_gpu_mem_idle"); host.dateBooted = rs.getDate("ts_booted"); host.dateCreated = rs.getDate("ts_created"); host.datePinged = rs.getDate("ts_ping"); @@ -110,8 +112,10 @@ public HostInterface mapRow(final ResultSet rs, int rowNum) throws SQLException "host.int_cores_idle,"+ "host.int_mem,"+ "host.int_mem_idle,"+ - "host.int_gpu,"+ - "host.int_gpu_idle,"+ + "host.int_gpus,"+ + "host.int_gpus_idle,"+ + "host.int_gpu_mem,"+ + "host.int_gpu_mem_idle,"+ "host.ts_created,"+ "host.str_name, " + "host_stat.str_state,"+ @@ -199,12 +203,14 @@ public DispatchHost mapRow(ResultSet rs, int rowNum) throws SQLException { host.facilityId = rs.getString("pk_facility"); host.name = rs.getString("str_name"); host.lockState = LockState.valueOf(rs.getString("str_lock_state")); - host.memory = rs.getInt("int_mem"); + host.memory = rs.getLong("int_mem"); host.cores = rs.getInt("int_cores"); - host.gpu= rs.getInt("int_gpu"); - host.idleMemory= rs.getInt("int_mem_idle"); + host.gpus = rs.getInt("int_gpus"); + host.gpuMemory = rs.getLong("int_gpu_mem"); + host.idleMemory= rs.getLong("int_mem_idle"); host.idleCores = rs.getInt("int_cores_idle"); - host.idleGpu= rs.getInt("int_gpu_idle"); + host.idleGpuMemory = rs.getLong("int_gpu_mem_idle"); + host.idleGpus = rs.getInt("int_gpus_idle"); host.isNimby = rs.getBoolean("b_nimby"); host.threadMode = rs.getInt("int_thread_mode"); host.tags = rs.getString("str_tags"); @@ -225,8 +231,10 @@ public DispatchHost mapRow(ResultSet rs, int rowNum) throws SQLException { "host.int_cores_idle, " + "host.int_mem,"+ "host.int_mem_idle, "+ - "host.int_gpu,"+ - "host.int_gpu_idle, "+ + "host.int_gpus, "+ + "host.int_gpus_idle, " + + "host.int_gpu_mem,"+ + "host.int_gpu_mem_idle, "+ "host.b_nimby, "+ "host.int_thread_mode, "+ "host.str_tags, " + @@ -276,12 +284,14 @@ public DispatchHost getDispatchHost(String id) { "int_cores_idle, " + "int_mem,"+ "int_mem_idle,"+ - "int_gpu,"+ - "int_gpu_idle,"+ + "int_gpus, " + + "int_gpus_idle, " + + "int_gpu_mem,"+ + "int_gpu_mem_idle,"+ "str_fqdn, " + "int_thread_mode "+ ") " + - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", "INSERT INTO " + "host_stat " + @@ -290,8 +300,8 @@ public DispatchHost getDispatchHost(String id) { "pk_host,"+ "int_mem_total, " + "int_mem_free,"+ - "int_gpu_total, " + - "int_gpu_free,"+ + "int_gpu_mem_total, " + + "int_gpu_mem_free,"+ "int_swap_total, " + "int_swap_free,"+ "int_mcp_total, " + @@ -347,28 +357,17 @@ public void insertRenderHost(RenderHost host, AllocationInterface a, boolean use os = Dispatcher.OS_DEFAULT; } - long totalGpu; - if (host.getAttributesMap().containsKey("totalGpu")) - totalGpu = Integer.parseInt(host.getAttributesMap().get("totalGpu")); - else - totalGpu = 0; - - long freeGpu; - if (host.getAttributesMap().containsKey("freeGpu")) - freeGpu = Integer.parseInt(host.getAttributesMap().get("freeGpu")); - else - freeGpu = 0; - - getJdbcTemplate().update(INSERT_HOST_DETAIL[0], hid, a.getAllocationId(), name, host.getNimbyEnabled(), LockState.OPEN.toString(), host.getNumProcs(), coreUnits, coreUnits, - memUnits, memUnits, totalGpu, totalGpu, + memUnits, memUnits, + host.getNumGpus(), host.getNumGpus(), + host.getTotalGpuMem(), host.getTotalGpuMem(), fqdn, threadMode.getNumber()); getJdbcTemplate().update(INSERT_HOST_DETAIL[1], hid, hid, host.getTotalMem(), host.getFreeMem(), - totalGpu, freeGpu, + host.getTotalGpuMem(), host.getFreeGpuMem(), host.getTotalSwap(), host.getFreeSwap(), host.getTotalMcp(), host.getFreeMcp(), host.getLoad(), new Timestamp(host.getBootTime() * 1000l), @@ -396,8 +395,8 @@ public CallableStatement createCallableStatement(Connection con) throws SQLExcep "int_swap_free = ?, "+ "int_mcp_total = ?, " + "int_mcp_free = ?, " + - "int_gpu_total = ?, " + - "int_gpu_free = ?, " + + "int_gpu_mem_total = ?, " + + "int_gpu_mem_free = ?, " + "int_load = ?," + "ts_booted = ?, " + "ts_ping = current_timestamp, "+ @@ -410,7 +409,7 @@ public void updateHostStats(HostInterface host, long totalMemory, long freeMemory, long totalSwap, long freeSwap, long totalMcp, long freeMcp, - long totalGpu, long freeGpu, + long totalGpuMemory, long freeGpuMemory, int load, Timestamp bootTime, String os) { @@ -420,7 +419,7 @@ public void updateHostStats(HostInterface host, getJdbcTemplate().update(UPDATE_RENDER_HOST, totalMemory, freeMemory, totalSwap, - freeSwap, totalMcp, freeMcp, totalGpu, freeGpu, load, + freeSwap, totalMcp, freeMcp, totalGpuMemory, freeGpuMemory, load, bootTime, os, host.getHostId()); } @@ -440,12 +439,8 @@ public void updateHostResources(HostInterface host, HostReport report) { long memory = convertMemoryUnits(report.getHost()); int cores = report.getHost().getNumProcs() * report.getHost().getCoresPerProc(); - - long totalGpu; - if (report.getHost().getAttributesMap().containsKey("totalGpu")) - totalGpu = Integer.parseInt(report.getHost().getAttributesMap().get("totalGpu")); - else - totalGpu = 0; + long gpu_memory = report.getHost().getTotalGpuMem(); + int gpus = report.getHost().getNumGpus(); getJdbcTemplate().update( "UPDATE " + @@ -456,16 +451,20 @@ public void updateHostResources(HostInterface host, HostReport report) { "int_cores_idle=?," + "int_mem=?," + "int_mem_idle=?, " + - "int_gpu=?," + - "int_gpu_idle=? " + + "int_gpus=?," + + "int_gpus_idle=?," + + "int_gpu_mem=?," + + "int_gpu_mem_idle=? " + "WHERE " + "pk_host=? "+ "AND " + "int_cores = int_cores_idle " + "AND " + - "int_mem = int_mem_idle", + "int_mem = int_mem_idle " + + "AND " + + "int_gpus = int_gpus_idle", report.getHost().getNimbyEnabled(), cores, cores, - memory, memory, totalGpu, totalGpu, host.getId()); + memory, memory, gpus, gpus, gpu_memory, gpu_memory, host.getId()); } @Override @@ -628,6 +627,18 @@ public int getStrandedCoreUnits(HostInterface h) { } } + @Override + public int getStrandedGpus(HostInterface h) { + try { + int idle_gpus = getJdbcTemplate().queryForObject( + "SELECT int_gpus_idle FROM host WHERE pk_host = ?", + Integer.class, h.getHostId()); + return idle_gpus; + } catch (EmptyResultDataAccessException e) { + return 0; + } + } + private static final String IS_HOST_UP = "SELECT " + "COUNT(1) " + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java index 3705324be..8009b247c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java @@ -116,6 +116,8 @@ public JobDetail mapRow(ResultSet rs, int rowNum) throws SQLException { job.logDir = rs.getString("str_log_dir"); job.maxCoreUnits = rs.getInt("int_max_cores"); job.minCoreUnits = rs.getInt("int_min_cores"); + job.maxGpuUnits = rs.getInt("int_max_gpus"); + job.minGpuUnits = rs.getInt("int_min_gpus"); job.name = rs.getString("str_name"); job.priority = rs.getInt("int_priority"); job.shot = rs.getString("str_shot"); @@ -218,6 +220,8 @@ public boolean isJobComplete(JobInterface job) { "job.int_max_retries,"+ "job_resource.int_max_cores,"+ "job_resource.int_min_cores,"+ + "job_resource.int_max_gpus,"+ + "job_resource.int_min_gpus,"+ "job_resource.int_priority,"+ "show.str_name AS show_name, " + "dept.str_name AS dept_name, "+ @@ -364,6 +368,32 @@ public void updateMaxCores(JobInterface j, int v) { v, j.getJobId()); } + @Override + public void updateMinGpus(GroupInterface g, int v) { + getJdbcTemplate().update("UPDATE job_resource SET int_min_gpus=? WHERE " + + "pk_job IN (SELECT pk_job FROM job WHERE pk_folder=?)", + v, g.getGroupId()); + } + + @Override + public void updateMaxGpus(GroupInterface g, int v) { + getJdbcTemplate().update("UPDATE job_resource SET int_max_gpus=? WHERE " + + "pk_job IN (SELECT pk_job FROM job WHERE pk_folder=?)", + v, g.getGroupId()); + } + + @Override + public void updateMinGpus(JobInterface j, int v) { + getJdbcTemplate().update("UPDATE job_resource SET int_min_gpus=? WHERE pk_job=?", + v, j.getJobId()); + } + + @Override + public void updateMaxGpus(JobInterface j, int v) { + getJdbcTemplate().update("UPDATE job_resource SET int_max_gpus=? WHERE pk_job=?", + v, j.getJobId()); + } + @Override public void updatePaused(JobInterface j, boolean b) { getJdbcTemplate().update("UPDATE job SET b_paused=? WHERE pk_job=?", @@ -632,6 +662,60 @@ public boolean isAtMaxCores(JobInterface job) { Integer.class, job.getJobId()) > 0; } + private static final String IS_JOB_OVER_MIN_GPUS = + "SELECT " + + "COUNT(1) " + + "FROM " + + "job_resource " + + "WHERE " + + "job_resource.pk_job = ? " + + "AND " + + "job_resource.int_gpus > job_resource.int_min_gpus"; + + @Override + public boolean isOverMinGpus(JobInterface job) { + return getJdbcTemplate().queryForObject(IS_JOB_OVER_MIN_GPUS, + Integer.class, job.getJobId()) > 0; + } + + private static final String IS_JOB_OVER_MAX_GPUS = + "SELECT " + + "COUNT(1) " + + "FROM " + + "job_resource " + + "WHERE " + + "job_resource.pk_job = ? " + + "AND " + + "job_resource.int_gpus + ? > job_resource.int_max_gpus"; + + @Override + public boolean isOverMaxGpus(JobInterface job) { + return getJdbcTemplate().queryForObject(IS_JOB_OVER_MAX_GPUS, + Integer.class, job.getJobId(), 0) > 0; + } + + @Override + public boolean isOverMaxGpus(JobInterface job, int gpu) { + return getJdbcTemplate().queryForObject(IS_JOB_OVER_MAX_GPUS, + Integer.class, job.getJobId(), gpu) > 0; + } + + private static final String IS_JOB_AT_MAX_GPUS = + "SELECT " + + "COUNT(1) " + + "FROM " + + "job_resource " + + "WHERE " + + "job_resource.pk_job = ? " + + "AND " + + "job_resource.int_gpus >= job_resource.int_max_gpus "; + + @Override + public boolean isAtMaxGpus(JobInterface job) { + return getJdbcTemplate().queryForObject(IS_JOB_AT_MAX_GPUS, + Integer.class, job.getJobId()) > 0; + } + @Override public void updateMaxFrameRetries(JobInterface j, int max_retries) { if (max_retries < 0) { @@ -685,8 +769,10 @@ public FrameStateTotals mapRow(ResultSet rs, int rowNum) throws SQLException { private static final String GET_EXECUTION_SUMMARY = "SELECT " + "job_usage.int_core_time_success,"+ - "job_usage.int_core_time_fail," + - "job_mem.int_max_rss " + + "job_usage.int_core_time_fail,"+ + "job_usage.int_gpu_time_success,"+ + "job_usage.int_gpu_time_fail,"+ + "job_mem.int_max_rss " + "FROM " + "job," + "job_usage, "+ @@ -707,6 +793,9 @@ public ExecutionSummary mapRow(ResultSet rs, int rowNum) throws SQLException { e.coreTimeSuccess = rs.getLong("int_core_time_success"); e.coreTimeFail = rs.getLong("int_core_time_fail"); e.coreTime = e.coreTimeSuccess + e.coreTimeFail; + e.gpuTimeSuccess = rs.getLong("int_gpu_time_success"); + e.gpuTimeFail = rs.getLong("int_gpu_time_fail"); + e.gpuTime = e.gpuTimeSuccess + e.gpuTimeFail; e.highMemoryKb = rs.getLong("int_max_rss"); return e; @@ -795,6 +884,20 @@ public void updateParent(JobInterface job, GroupDetail dest, Inherit[] inherits) } break; + case MinGpus: + if (dest.jobMinGpus != CueUtil.FEATURE_DISABLED) { + query.append("int_min_gpus=?,"); + values.add(dest.jobMinGpus); + } + break; + + case MaxGpus: + if (dest.jobMaxGpus != CueUtil.FEATURE_DISABLED) { + query.append("int_max_gpus=?,"); + values.add(dest.jobMaxGpus); + } + break; + case All: if (dest.jobPriority != CueUtil.FEATURE_DISABLED) { query.append("int_priority=?,"); @@ -810,6 +913,16 @@ public void updateParent(JobInterface job, GroupDetail dest, Inherit[] inherits) query.append("int_max_cores=?,"); values.add(dest.jobMaxCores); } + + if (dest.jobMinGpus != CueUtil.FEATURE_DISABLED) { + query.append("int_min_gpus=?,"); + values.add(dest.jobMinGpus); + } + + if (dest.jobMaxGpus != CueUtil.FEATURE_DISABLED) { + query.append("int_max_gpus=?,"); + values.add(dest.jobMaxGpus); + } break; } } @@ -851,6 +964,8 @@ public void updateParent(JobInterface job, GroupDetail dest, Inherit[] inherits) "job_stat.int_waiting_count != 0" + "AND " + "job_resource.int_cores < job_resource.int_max_cores " + + "AND " + + "job_resource.int_gpus < job_resource.int_max_gpus " + "AND " + "job.pk_facility = ? " + "LIMIT 1"; @@ -922,11 +1037,13 @@ public void updateUsage(JobInterface job, ResourceUsage usage, int exitStatus) { "job_usage " + "SET " + "int_core_time_success = int_core_time_success + ?," + + "int_gpu_time_success = int_gpu_time_success + ?," + "int_clock_time_success = int_clock_time_success + ?,"+ "int_frame_success_count = int_frame_success_count + 1 " + "WHERE " + "pk_job = ? ", usage.getCoreTimeSeconds(), + usage.getGpuTimeSeconds(), usage.getClockTimeSeconds(), job.getJobId()); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/LayerDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/LayerDaoJdbc.java index 26654f392..212963519 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/LayerDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/LayerDaoJdbc.java @@ -205,7 +205,8 @@ public LayerDetail mapRow(ResultSet rs, int rowNum) throws SQLException { layer.range = rs.getString("str_range"); layer.minimumCores = rs.getInt("int_cores_min"); layer.minimumMemory = rs.getLong("int_mem_min"); - layer.minimumGpu = rs.getLong("int_gpu_min"); + layer.minimumGpus = rs.getInt("int_gpus_min"); + layer.minimumGpuMemory = rs.getLong("int_gpu_mem_min"); layer.type = LayerType.valueOf(rs.getString("str_type")); layer.tags = Sets.newHashSet( rs.getString("str_tags").replaceAll(" ", "").split("\\|")); @@ -311,12 +312,14 @@ public LayerInterface getLayer(String id) { "int_cores_max, "+ "b_threadable, " + "int_mem_min, " + - "int_gpu_min, " + + "int_gpus_min, "+ + "int_gpus_max, "+ + "int_gpu_mem_min, " + "str_services, " + "int_timeout," + "int_timeout_llu " + ") " + - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; @Override public void insertLayerDetail(LayerDetail l) { @@ -326,7 +329,7 @@ public void insertLayerDetail(LayerDetail l) { l.range, l.chunkSize, l.dispatchOrder, StringUtils.join(l.tags," | "), l.type.toString(), l.minimumCores, l.maximumCores, l.isThreadable, - l.minimumMemory, l.minimumGpu, StringUtils.join(l.services,","), + l.minimumMemory, l.minimumGpus, l.maximumGpus, l.minimumGpuMemory, StringUtils.join(l.services,","), l.timeout, l.timeout_llu); } @@ -340,9 +343,9 @@ public void updateLayerMinMemory(LayerInterface layer, long val) { } @Override - public void updateLayerMinGpu(LayerInterface layer, long gpu) { - getJdbcTemplate().update("UPDATE layer SET int_gpu_min=? WHERE pk_layer=?", - gpu, layer.getLayerId()); + public void updateLayerMinGpuMemory(LayerInterface layer, long kb) { + getJdbcTemplate().update("UPDATE layer SET int_gpu_mem_min=? WHERE pk_layer=?", + kb, layer.getLayerId()); } private static final String BALANCE_MEM = @@ -392,9 +395,9 @@ public void increaseLayerMinMemory(LayerInterface layer, long val) { } @Override - public void increaseLayerMinGpu(LayerInterface layer, long gpu) { - getJdbcTemplate().update("UPDATE layer SET int_gpu_min=? WHERE pk_layer=? AND int_gpu_min < ?", - gpu, layer.getLayerId(), gpu); + public void increaseLayerMinGpuMemory(LayerInterface layer, long kb) { + getJdbcTemplate().update("UPDATE layer SET int_gpu_mem_min=? WHERE pk_layer=? AND int_gpu_mem_min < ?", + kb, layer.getLayerId(), kb); } @Override @@ -412,6 +415,18 @@ public void updateLayerMaxCores(LayerInterface layer, int val) { val, layer.getLayerId()); } + @Override + public void updateLayerMinGpus(LayerInterface layer, int val) { + getJdbcTemplate().update("UPDATE layer SET int_gpus_min=? WHERE pk_layer=?", + val, layer.getLayerId()); + } + + @Override + public void updateLayerMaxGpus(LayerInterface layer, int val) { + getJdbcTemplate().update("UPDATE layer SET int_gpus_max=? WHERE pk_layer=?", + val, layer.getLayerId()); + } + private static final String UPDATE_LAYER_MAX_RSS = "UPDATE " + "layer_mem " + @@ -489,6 +504,8 @@ public FrameStateTotals mapRow(ResultSet rs, int rowNum) throws SQLException { "SELECT " + "layer_usage.int_core_time_success,"+ "layer_usage.int_core_time_fail," + + "layer_usage.int_gpu_time_success,"+ + "layer_usage.int_gpu_time_fail," + "layer_usage.int_clock_time_success," + "layer_mem.int_max_rss " + "FROM " + @@ -512,6 +529,9 @@ public ExecutionSummary mapRow(ResultSet rs, int rowNum) throws SQLException { e.coreTimeSuccess = rs.getLong("int_core_time_success"); e.coreTimeFail = rs.getLong("int_core_time_fail"); e.coreTime = e.coreTimeSuccess + e.coreTimeFail; + e.gpuTimeSuccess = rs.getLong("int_gpu_time_success"); + e.gpuTimeFail = rs.getLong("int_gpu_time_fail"); + e.gpuTime = e.gpuTimeSuccess + e.gpuTimeFail; e.highMemoryKb = rs.getLong("int_max_rss"); return e; } @@ -608,10 +628,10 @@ public void updateMinMemory(JobInterface job, long mem, LayerType type) { } @Override - public void updateMinGpu(JobInterface job, long gpu, LayerType type) { + public void updateMinGpuMemory(JobInterface job, long kb, LayerType type) { getJdbcTemplate().update( - "UPDATE layer SET int_gpu_min=? WHERE pk_job=? AND str_type=?", - gpu, job.getJobId(), type.toString()); + "UPDATE layer SET int_gpu_mem_min=? WHERE pk_job=? AND str_type=?", + kb, job.getJobId(), type.toString()); } @Override @@ -621,6 +641,13 @@ public void updateMinCores(JobInterface job, int cores, LayerType type) { cores, job.getJobId(), type.toString()); } + @Override + public void updateMinGpus(JobInterface job, int gpus, LayerType type) { + getJdbcTemplate().update( + "UPDATE layer SET int_gpus_min=? WHERE pk_job=? AND str_type=?", + gpus, job.getJobId(), type.toString()); + } + @Override public void updateThreadable(LayerInterface layer, boolean threadable) { getJdbcTemplate().update( @@ -664,6 +691,8 @@ public void enableMemoryOptimizer(LayerInterface layer, boolean value) { "layer.pk_layer = ? " + "AND " + "layer.int_cores_min = 100 " + + "AND " + + "layer.int_gpus_min = 0 " + "AND " + "str_tags LIKE '%general%' " + "AND " + @@ -686,7 +715,8 @@ public boolean isOptimizable(LayerInterface l, int succeeded, float avg) { private static final String THREAD_STATS = "SELECT " + "avg(interval_to_seconds(ts_stopped - ts_started)) AS avg, " + - "int_cores " + + "int_cores, " + + "int_gpus " + "FROM " + "frame " + "WHERE " + @@ -695,8 +725,11 @@ public boolean isOptimizable(LayerInterface l, int succeeded, float avg) { "frame.int_checkpoint_count = 0 " + "AND " + "int_cores > 0 " + + "AND " + + "int_gpus > 0 " + "GROUP BY " + - "int_cores " + + "int_cores, " + + "int_gpus " + "ORDER BY " + "int_cores DESC "; @@ -724,11 +757,13 @@ public void updateUsage(LayerInterface layer, ResourceUsage usage, int exitStatu "layer_usage " + "SET " + "int_core_time_success = int_core_time_success + ?," + + "int_gpu_time_success = int_gpu_time_success + ?," + "int_clock_time_success = int_clock_time_success + ?,"+ "int_frame_success_count = int_frame_success_count + 1 " + "WHERE " + "pk_layer = ? ", usage.getCoreTimeSeconds(), + usage.getGpuTimeSeconds(), usage.getClockTimeSeconds(), layer.getLayerId()); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/NestedWhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/NestedWhiteboardDaoJdbc.java index f2cff28a5..924a65a96 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/NestedWhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/NestedWhiteboardDaoJdbc.java @@ -73,8 +73,12 @@ public CachedJobWhiteboardMapper(NestedJobWhiteboardMapper result) { "folder.int_job_priority as int_def_job_priority, " + "folder.int_job_min_cores as int_def_job_min_cores, " + "folder.int_job_max_cores as int_def_job_max_cores, " + + "folder.int_job_min_gpus as int_def_job_min_gpus, " + + "folder.int_job_max_gpus as int_def_job_max_gpus, " + "folder_resource.int_min_cores AS folder_min_cores, " + "folder_resource.int_max_cores AS folder_max_cores, " + + "folder_resource.int_min_gpus AS folder_min_gpus, " + + "folder_resource.int_max_gpus AS folder_max_gpus, " + "folder_level.int_level, " + "job.pk_job, " + "job.str_name, " + @@ -101,13 +105,18 @@ public CachedJobWhiteboardMapper(NestedJobWhiteboardMapper result) { "job_stat.int_succeeded_count, " + "job_usage.int_core_time_success, " + "job_usage.int_core_time_fail, " + + "job_usage.int_gpu_time_success, " + + "job_usage.int_gpu_time_fail, " + "job_usage.int_frame_success_count, " + "job_usage.int_frame_fail_count, " + "job_usage.int_clock_time_high, " + "job_usage.int_clock_time_success, " + "(job_resource.int_cores + job_resource.int_local_cores) AS int_cores, " + + "(job_resource.int_gpus + job_resource.int_local_gpus) AS int_gpus, " + "job_resource.int_min_cores, " + + "job_resource.int_min_gpus, " + "job_resource.int_max_cores, " + + "job_resource.int_max_gpus, " + "job_mem.int_max_rss " + "FROM " + "show, " + @@ -165,8 +174,12 @@ public NestedGroup mapRow(ResultSet rs, int rowNum) throws SQLException { .setDefaultJobPriority(rs.getInt("int_def_job_priority")) .setDefaultJobMinCores(Convert.coreUnitsToCores(rs.getInt("int_def_job_min_cores"))) .setDefaultJobMaxCores(Convert.coreUnitsToCores(rs.getInt("int_def_job_max_cores"))) + .setDefaultJobMinGpus(rs.getInt("int_def_job_min_gpus")) + .setDefaultJobMaxGpus(rs.getInt("int_def_job_max_gpus")) .setMaxCores(Convert.coreUnitsToCores(rs.getInt("folder_max_cores"))) .setMinCores(Convert.coreUnitsToCores(rs.getInt("folder_min_cores"))) + .setMaxGpus(rs.getInt("folder_max_gpus")) + .setMinGpus(rs.getInt("folder_min_gpus")) .setLevel(rs.getInt("int_level")) .setDepartment(rs.getString("dept_name")) .build(); @@ -254,6 +267,8 @@ private static final NestedJob mapResultSetToJob(ResultSet rs) throws SQLExcepti .setLogDir(rs.getString("str_log_dir")) .setMaxCores(Convert.coreUnitsToCores(rs.getInt("int_max_cores"))) .setMinCores(Convert.coreUnitsToCores(rs.getInt("int_min_cores"))) + .setMaxGpus(rs.getInt("int_max_cores")) + .setMinGpus(rs.getInt("int_min_cores")) .setName(rs.getString("str_name")) .setPriority(rs.getInt("int_priority")) .setShot(rs.getString("str_shot")) @@ -295,8 +310,10 @@ private static final NestedJob mapResultSetToJob(ResultSet rs) throws SQLExcepti "host_stat.ts_ping, " + "host.int_cores, " + "host.int_cores_idle, " + - "host.int_gpu, " + - "host.int_gpu_idle, " + + "host.int_gpus, " + + "host.int_gpus_idle, " + + "host.int_gpu_mem, " + + "host.int_gpu_mem_idle, " + "host.int_mem, " + "host.int_mem_idle, " + "host.str_lock_state, " + @@ -310,15 +327,16 @@ private static final NestedJob mapResultSetToJob(ResultSet rs) throws SQLExcepti "host_stat.int_swap_free, " + "host_stat.int_mcp_total, " + "host_stat.int_mcp_free, " + - "host_stat.int_gpu_total, " + - "host_stat.int_gpu_free, " + + "host_stat.int_gpu_mem_total, " + + "host_stat.int_gpu_mem_free, " + "host_stat.int_load, " + "proc.pk_proc, " + "proc.int_cores_reserved AS proc_cores, " + + "proc.int_gpus_reserved AS proc_gpus, " + "proc.int_mem_reserved AS proc_memory, " + "proc.int_mem_used AS used_memory, " + "proc.int_mem_max_used AS max_memory, " + - "proc.int_gpu_reserved AS proc_gpu, " + + "proc.int_gpu_mem_reserved AS proc_gpu_memory, " + "proc.ts_ping, " + "proc.ts_booked, " + "proc.ts_dispatched, " + @@ -445,10 +463,13 @@ public NestedHost mapRow(ResultSet rs, int row) throws SQLException { proc = NestedProc.newBuilder() .setId(pid) .setName(CueUtil.buildProcName(host.getName(), - rs.getInt("proc_cores"))) + rs.getInt("proc_cores"), + rs.getInt("proc_gpus"))) .setReservedCores(Convert.coreUnitsToCores( rs.getInt("proc_cores"))) + .setReservedGpus(rs.getInt("proc_gpus")) .setReservedMemory(rs.getLong("proc_memory")) + .setReservedGpuMemory(rs.getLong("proc_gpu_memory")) .setUsedMemory(rs.getLong("used_memory")) .setFrameName(rs.getString("frame_name")) .setJobName(rs.getString("job_name")) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java index be8643b7e..ba9f33c1f 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java @@ -108,9 +108,12 @@ public boolean deleteVirtualProc(VirtualProc proc) { "int_mem_reserved, " + "int_mem_pre_reserved, " + "int_mem_used, "+ - "int_gpu_reserved, " + + "int_gpus_reserved, " + + "int_gpu_mem_reserved, " + + "int_gpu_mem_pre_reserved, " + + "int_gpu_mem_used, " + "b_local " + - ") VALUES (?,?,?,?,?,?,?,?,?,?,?,?) "; + ") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) "; public void insertVirtualProc(VirtualProc proc) { proc.id = SqlUtil.genKeyRandom(); @@ -121,7 +124,9 @@ public void insertVirtualProc(VirtualProc proc) { proc.getLayerId(), proc.getJobId(), proc.getFrameId(), proc.coresReserved, proc.memoryReserved, proc.memoryReserved, Dispatcher.MEM_RESERVED_MIN, - proc.gpuReserved, proc.isLocalDispatch); + proc.gpusReserved, proc.gpuMemoryReserved, + proc.gpuMemoryReserved, Dispatcher.MEM_GPU_RESERVED_MIN, + proc.isLocalDispatch); // Update all of the resource counts procCreated(proc); @@ -278,7 +283,9 @@ public VirtualProc mapRow(ResultSet rs, int rowNum) throws SQLException { proc.coresReserved =rs.getInt("int_cores_reserved"); proc.memoryReserved = rs.getLong("int_mem_reserved"); proc.memoryMax = rs.getLong("int_mem_max_used"); - proc.gpuReserved = rs.getLong("int_gpu_reserved"); + proc.gpusReserved = rs.getInt("int_gpus_reserved"); + proc.gpuMemoryReserved = rs.getLong("int_gpu_mem_reserved"); + proc.gpuMemoryMax = rs.getLong("int_gpu_mem_max_used"); proc.virtualMemoryMax = rs.getLong("int_virt_max_used"); proc.virtualMemoryUsed = rs.getLong("int_virt_used"); proc.memoryUsed = rs.getLong("int_mem_used"); @@ -305,7 +312,10 @@ public VirtualProc mapRow(ResultSet rs, int rowNum) throws SQLException { "proc.int_mem_reserved,"+ "proc.int_mem_max_used,"+ "proc.int_mem_used,"+ - "proc.int_gpu_reserved,"+ + "proc.int_gpus_reserved,"+ + "proc.int_gpu_mem_reserved,"+ + "proc.int_gpu_mem_max_used,"+ + "proc.int_gpu_mem_used,"+ "proc.int_virt_max_used,"+ "proc.int_virt_used,"+ "host.str_name AS host_name, " + @@ -551,7 +561,10 @@ public boolean increaseReservedMemory(ProcInterface p, long value) { "int_mem_reserved," + "int_mem_max_used,"+ "int_mem_used,"+ - "int_gpu_reserved," + + "int_gpus_reserved," + + "int_gpu_mem_reserved," + + "int_gpu_mem_max_used," + + "int_gpu_mem_used," + "int_virt_max_used,"+ "int_virt_used,"+ "host_name, " + @@ -578,9 +591,9 @@ public long getReservedMemory(ProcInterface proc) { Long.class, proc.getProcId()); } - public long getReservedGpu(ProcInterface proc) { + public long getReservedGpuMemory(ProcInterface proc) { return getJdbcTemplate().queryForObject( - "SELECT int_gpu_reserved FROM proc WHERE pk_proc=?", + "SELECT int_gpu_mem_reserved FROM proc WHERE pk_proc=?", Long.class, proc.getProcId()); } @@ -694,22 +707,24 @@ private void procDestroyed(VirtualProc proc) { "SET " + "int_cores_idle = int_cores_idle + ?," + "int_mem_idle = int_mem_idle + ?, " + - "int_gpu_idle = int_gpu_idle + ? " + + "int_gpus_idle = int_gpus_idle + ?," + + "int_gpu_mem_idle = int_gpu_mem_idle + ? " + "WHERE " + "pk_host = ?", - proc.coresReserved, proc.memoryReserved, proc.gpuReserved, proc.getHostId()); + proc.coresReserved, proc.memoryReserved, proc.gpusReserved, proc.gpuMemoryReserved, proc.getHostId()); if (!proc.isLocalDispatch) { getJdbcTemplate().update( "UPDATE " + "subscription " + "SET " + - "int_cores = int_cores - ? " + + "int_cores = int_cores - ?," + + "int_gpus = int_gpus - ? " + "WHERE " + "pk_show = ? " + "AND " + "pk_alloc = ?", - proc.coresReserved, proc.getShowId(), + proc.coresReserved, proc.gpusReserved, proc.getShowId(), proc.getAllocationId()); } @@ -717,10 +732,11 @@ private void procDestroyed(VirtualProc proc) { "UPDATE " + "layer_resource " + "SET " + - "int_cores = int_cores - ? " + + "int_cores = int_cores - ?," + + "int_gpus = int_gpus - ? " + "WHERE " + "pk_layer = ?", - proc.coresReserved, proc.getLayerId()); + proc.coresReserved, proc.gpusReserved, proc.getLayerId()); if (!proc.isLocalDispatch) { @@ -728,33 +744,36 @@ private void procDestroyed(VirtualProc proc) { "UPDATE " + "job_resource " + "SET " + - "int_cores = int_cores - ? " + + "int_cores = int_cores - ?," + + "int_gpus = int_gpus - ? " + "WHERE " + "pk_job = ?", - proc.coresReserved, proc.getJobId()); + proc.coresReserved, proc.gpusReserved, proc.getJobId()); getJdbcTemplate().update( "UPDATE " + "folder_resource " + "SET " + - "int_cores = int_cores - ? " + + "int_cores = int_cores - ?," + + "int_gpus = int_gpus - ? " + "WHERE " + "pk_folder = " + "(SELECT pk_folder FROM job WHERE pk_job=?)", - proc.coresReserved, proc.getJobId()); + proc.coresReserved, proc.gpusReserved, proc.getJobId()); getJdbcTemplate().update( "UPDATE " + "point " + "SET " + - "int_cores = int_cores - ? " + + "int_cores = int_cores - ?, " + + "int_gpus = int_gpus - ? " + "WHERE " + "pk_dept = " + "(SELECT pk_dept FROM job WHERE pk_job=?) " + "AND " + "pk_show = " + "(SELECT pk_show FROM job WHERE pk_job=?) ", - proc.coresReserved, proc.getJobId(), proc.getJobId()); + proc.coresReserved, proc.gpusReserved, proc.getJobId(), proc.getJobId()); } if (proc.isLocalDispatch) { @@ -763,10 +782,11 @@ private void procDestroyed(VirtualProc proc) { "UPDATE " + "job_resource " + "SET " + - "int_local_cores = int_local_cores - ? " + + "int_local_cores = int_local_cores - ?, " + + "int_local_gpus = int_local_gpus - ? " + "WHERE " + "pk_job = ?", - proc.coresReserved, proc.getJobId()); + proc.coresReserved, proc.gpusReserved, proc.getJobId()); getJdbcTemplate().update( "UPDATE " + @@ -774,14 +794,16 @@ private void procDestroyed(VirtualProc proc) { "SET " + "int_cores_idle = int_cores_idle + ?, " + "int_mem_idle = int_mem_idle + ?, " + - "int_gpu_idle = int_gpu_idle + ? " + + "int_gpus_idle = int_gpus_idle + ?, " + + "int_gpu_mem_idle = int_gpu_mem_idle + ? " + "WHERE " + "pk_job = ? " + "AND " + "pk_host = ? ", proc.coresReserved, proc.memoryReserved, - proc.gpuReserved, + proc.gpusReserved, + proc.gpuMemoryReserved, proc.getJobId(), proc.getHostId()); } @@ -802,10 +824,11 @@ private void procCreated(VirtualProc proc) { "SET " + "int_cores_idle = int_cores_idle - ?," + "int_mem_idle = int_mem_idle - ?, " + - "int_gpu_idle = int_gpu_idle - ? " + + "int_gpus_idle = int_gpus_idle - ?," + + "int_gpu_mem_idle = int_gpu_mem_idle - ? " + "WHERE " + "pk_host = ?", - proc.coresReserved, proc.memoryReserved, proc.gpuReserved, proc.getHostId()); + proc.coresReserved, proc.memoryReserved, proc.gpusReserved, proc.gpuMemoryReserved, proc.getHostId()); /** @@ -817,12 +840,13 @@ private void procCreated(VirtualProc proc) { "UPDATE " + "subscription " + "SET " + - "int_cores = int_cores + ? " + + "int_cores = int_cores + ?," + + "int_gpus = int_gpus + ? " + "WHERE " + "pk_show = ? " + "AND " + "pk_alloc = ?", - proc.coresReserved, proc.getShowId(), + proc.coresReserved, proc.gpusReserved, proc.getShowId(), proc.getAllocationId()); } @@ -830,10 +854,11 @@ private void procCreated(VirtualProc proc) { "UPDATE " + "layer_resource " + "SET " + - "int_cores = int_cores + ? " + + "int_cores = int_cores + ?," + + "int_gpus = int_gpus + ? " + "WHERE " + "pk_layer = ?", - proc.coresReserved, proc.getLayerId()); + proc.coresReserved, proc.gpusReserved, proc.getLayerId()); if (!proc.isLocalDispatch) { @@ -841,33 +866,36 @@ private void procCreated(VirtualProc proc) { "UPDATE " + "job_resource " + "SET " + - "int_cores = int_cores + ? " + + "int_cores = int_cores + ?," + + "int_gpus = int_gpus + ? " + "WHERE " + "pk_job = ?", - proc.coresReserved, proc.getJobId()); + proc.coresReserved, proc.gpusReserved, proc.getJobId()); getJdbcTemplate().update( "UPDATE " + "folder_resource " + "SET " + - "int_cores = int_cores + ? " + + "int_cores = int_cores + ?," + + "int_gpus = int_gpus + ? " + "WHERE " + "pk_folder = " + "(SELECT pk_folder FROM job WHERE pk_job=?)", - proc.coresReserved, proc.getJobId()); + proc.coresReserved, proc.gpusReserved, proc.getJobId()); getJdbcTemplate().update( "UPDATE " + "point " + "SET " + - "int_cores = int_cores + ? " + + "int_cores = int_cores + ?," + + "int_gpus = int_gpus + ? " + "WHERE " + "pk_dept = " + "(SELECT pk_dept FROM job WHERE pk_job=?) " + "AND " + "pk_show = " + "(SELECT pk_show FROM job WHERE pk_job=?) ", - proc.coresReserved, proc.getJobId(), proc.getJobId()); + proc.coresReserved, proc.gpusReserved, proc.getJobId(), proc.getJobId()); } if (proc.isLocalDispatch) { @@ -876,23 +904,28 @@ private void procCreated(VirtualProc proc) { "UPDATE " + "job_resource " + "SET " + - "int_local_cores = int_local_cores + ? " + + "int_local_cores = int_local_cores + ?," + + "int_local_gpus = int_local_gpus + ? " + "WHERE " + "pk_job = ?", - proc.coresReserved, proc.getJobId()); + proc.coresReserved, proc.gpusReserved, proc.getJobId()); getJdbcTemplate().update( "UPDATE " + "host_local " + "SET " + "int_cores_idle = int_cores_idle - ?, " + - "int_mem_idle = int_mem_idle - ? " + + "int_mem_idle = int_mem_idle - ?," + + "int_gpus_idle = int_gpus_idle - ?, " + + "int_gpu_mem_idle = int_gpu_mem_idle - ? " + "WHERE " + "pk_job = ? " + "AND " + "pk_host = ?", proc.coresReserved, proc.memoryReserved, + proc.gpusReserved, + proc.gpuMemoryReserved, proc.getJobId(), proc.getHostId()); } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ServiceDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ServiceDaoJdbc.java index b31d9ade0..6330cc8cb 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ServiceDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ServiceDaoJdbc.java @@ -60,7 +60,9 @@ public ServiceEntity mapRow(ResultSet rs, int rowNum) throws SQLException { s.minCores = rs.getInt("int_cores_min"); s.maxCores = rs.getInt("int_cores_max"); s.minMemory = rs.getLong("int_mem_min"); - s.minGpu = rs.getLong("int_gpu_min"); + s.minGpus = rs.getInt("int_gpus_min"); + s.maxGpus = rs.getInt("int_gpus_max"); + s.minGpuMemory = rs.getLong("int_gpu_mem_min"); s.threadable = rs.getBoolean("b_threadable"); s.tags = splitTags(rs.getString("str_tags")); s.timeout = rs.getInt("int_timeout"); @@ -79,7 +81,9 @@ public ServiceOverrideEntity mapRow(ResultSet rs, int rowNum) s.minCores = rs.getInt("int_cores_min"); s.maxCores = rs.getInt("int_cores_max"); s.minMemory = rs.getLong("int_mem_min"); - s.minGpu = rs.getLong("int_gpu_min"); + s.minGpus = rs.getInt("int_gpus_min"); + s.maxGpus = rs.getInt("int_gpus_max"); + s.minGpuMemory = rs.getLong("int_gpu_mem_min"); s.threadable = rs.getBoolean("b_threadable"); s.tags = splitTags(rs.getString("str_tags")); s.showId = rs.getString("pk_show"); @@ -97,7 +101,9 @@ public ServiceOverrideEntity mapRow(ResultSet rs, int rowNum) "service.int_cores_min," + "service.int_cores_max," + "service.int_mem_min," + - "service.int_gpu_min," + + "service.int_gpus_min," + + "service.int_gpus_max," + + "service.int_gpu_mem_min," + "service.str_tags, " + "service.int_timeout, " + "service.int_timeout_llu " + @@ -119,7 +125,9 @@ public ServiceEntity get(String id) { "show_service.int_cores_min," + "show_service.int_cores_max, "+ "show_service.int_mem_min," + - "show_service.int_gpu_min," + + "show_service.int_gpus_min," + + "show_service.int_gpus_max, "+ + "show_service.int_gpu_mem_min," + "show_service.str_tags," + "show_service.int_timeout," + "show_service.int_timeout_llu," + @@ -167,18 +175,21 @@ public boolean isOverridden(String service, String show) { "int_cores_min," + "int_cores_max, "+ "int_mem_min," + - "int_gpu_min," + + "int_gpus_min," + + "int_gpus_max, "+ + "int_gpu_mem_min," + "str_tags," + "int_timeout," + "int_timeout_llu " + - ") VALUES (?,?,?,?,?,?,?,?,?,?)"; + ") VALUES (?,?,?,?,?,?,?,?,?,?,?,?)"; @Override public void insert(ServiceEntity service) { service.id = SqlUtil.genKeyRandom(); getJdbcTemplate().update(INSERT_SERVICE, service.id, service.name, service.threadable, service.minCores, - service.maxCores, service.minMemory, service.minGpu, + service.maxCores, service.minMemory, + service.minGpus, service.maxGpus, service.minGpuMemory, StringUtils.join(service.tags.toArray(), " | "), service.timeout, service.timeout_llu); } @@ -194,11 +205,13 @@ public void insert(ServiceEntity service) { "int_cores_min," + "int_cores_max," + "int_mem_min," + - "int_gpu_min," + + "int_gpus_min," + + "int_gpus_max," + + "int_gpu_mem_min," + "str_tags," + "int_timeout," + "int_timeout_llu " + - ") VALUES (?,?,?,?,?,?,?,?,?,?,?)"; + ") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)"; @Override public void insert(ServiceOverrideEntity service) { @@ -206,7 +219,7 @@ public void insert(ServiceOverrideEntity service) { getJdbcTemplate().update(INSERT_SERVICE_WITH_SHOW, service.id, service.showId, service.name, service.threadable, service.minCores, service.maxCores, service.minMemory, - service.minGpu, joinTags(service.tags), + service.minGpus, service.maxGpus, service.minGpuMemory, joinTags(service.tags), service.timeout, service.timeout_llu); } @@ -219,7 +232,9 @@ service.minGpu, joinTags(service.tags), "int_cores_min=?," + "int_cores_max=?,"+ "int_mem_min=?," + - "int_gpu_min=?," + + "int_gpus_min=?," + + "int_gpus_max=?," + + "int_gpu_mem_min=?," + "str_tags=?," + "int_timeout=?," + "int_timeout_llu=? " + @@ -230,7 +245,7 @@ service.minGpu, joinTags(service.tags), public void update(ServiceEntity service) { getJdbcTemplate().update(UPDATE_SERVICE, service.name, service.threadable, service.minCores, service.maxCores, - service.minMemory, service.minGpu, joinTags(service.tags), + service.minMemory, service.minGpus, service.maxGpus, service.minGpuMemory, joinTags(service.tags), service.timeout, service.timeout_llu, service.getId()); } @@ -243,7 +258,9 @@ service.minMemory, service.minGpu, joinTags(service.tags), "int_cores_min=?," + "int_cores_max=?," + "int_mem_min=?," + - "int_gpu_min=?," + + "int_gpus_min=?," + + "int_gpus_max=?," + + "int_gpu_mem_min=?," + "str_tags=?," + "int_timeout=?," + "int_timeout_llu=? " + @@ -254,7 +271,7 @@ service.minMemory, service.minGpu, joinTags(service.tags), public void update(ServiceOverrideEntity service) { getJdbcTemplate().update(UPDATE_SERVICE_WITH_SHOW, service.name, service.threadable, service.minCores, service.maxCores, - service.minMemory, service.minGpu, joinTags(service.tags), + service.minMemory, service.minGpus, service.maxGpus, service.minGpuMemory, joinTags(service.tags), service.timeout, service.timeout_llu, service.getId()); } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java index 893455be3..add49a178 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java @@ -44,6 +44,8 @@ public ShowEntity mapRow(ResultSet rs, int rowNum) throws SQLException { show.id = rs.getString("pk_show"); show.defaultMaxCores = rs.getInt("int_default_max_cores"); show.defaultMinCores = rs.getInt("int_default_min_cores"); + show.defaultMaxGpus = rs.getInt("int_default_max_gpus"); + show.defaultMinGpus = rs.getInt("int_default_min_gpus"); show.active = rs.getBoolean("b_active"); if (rs.getString("str_comment_email") != null) { @@ -61,6 +63,8 @@ public ShowEntity mapRow(ResultSet rs, int rowNum) throws SQLException { "show.pk_show, " + "show.int_default_max_cores, " + "show.int_default_min_cores, " + + "show.int_default_max_gpus, " + + "show.int_default_min_gpus, " + "show.str_name, " + "show.b_active, " + "show.str_comment_email " + @@ -72,6 +76,8 @@ public ShowEntity mapRow(ResultSet rs, int rowNum) throws SQLException { "show.pk_show, " + "show.int_default_max_cores, " + "show.int_default_min_cores, " + + "show.int_default_max_gpus, " + + "show.int_default_min_gpus, " + "show_alias.str_name, " + "show.b_active, " + "show.str_comment_email " + @@ -101,6 +107,8 @@ public ShowEntity getShowDetail(String id) { "show.pk_show, " + "show.int_default_max_cores, " + "show.int_default_min_cores, " + + "show.int_default_max_gpus, " + + "show.int_default_min_gpus, " + "show.str_name, " + "show.b_active, " + "show.str_comment_email " + @@ -180,6 +188,18 @@ public void updateShowDefaultMaxCores(ShowInterface s, int val) { val, s.getShowId()); } + public void updateShowDefaultMinGpus(ShowInterface s, int val) { + getJdbcTemplate().update( + "UPDATE show SET int_default_min_gpus=? WHERE pk_show=?", + val, s.getShowId()); + } + + public void updateShowDefaultMaxGpus(ShowInterface s, int val) { + getJdbcTemplate().update( + "UPDATE show SET int_default_max_gpus=? WHERE pk_show=?", + val, s.getShowId()); + } + @Override public void updateBookingEnabled(ShowInterface s, boolean enabled) { getJdbcTemplate().update( diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index 95712d605..cb3aede1f 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -895,7 +895,10 @@ public RenderPartition mapRow(ResultSet rs, int rowNum) throws SQLException { .setThreads(rs.getInt("int_threads")) .setMaxMemory(rs.getLong("int_mem_max")) .setMemory( rs.getLong("int_mem_max") - rs.getLong("int_mem_idle")) - .setMaxGpu(rs.getLong("int_gpu_max")) + .setGpus(rs.getInt("int_gpus_max") - rs.getInt("int_gpus_idle")) + .setMaxGpus(rs.getInt("int_gpus_max")) + .setGpuMemory(rs.getLong("int_gpu_mem_max") - rs.getLong("int_gpu_mem_idle")) + .setMaxGpuMemory(rs.getLong("int_gpu_mem_max")) .setHost(SqlUtil.getString(rs,"str_host_name")) .setJob(SqlUtil.getString(rs,"str_job_name")) .setRenderPartType(RenderPartitionType.valueOf(SqlUtil.getString(rs,"str_type"))) @@ -947,11 +950,13 @@ public Proc mapRow(ResultSet rs, int row) throws SQLException { return Proc.newBuilder() .setId(SqlUtil.getString(rs,"pk_proc")) .setName(CueUtil.buildProcName(SqlUtil.getString(rs,"host_name"), - rs.getInt("int_cores_reserved"))) + rs.getInt("int_cores_reserved"), rs.getInt("int_gpus_reserved"))) .setReservedCores(Convert.coreUnitsToCores(rs.getInt("int_cores_reserved"))) .setReservedMemory(rs.getLong("int_mem_reserved")) - .setReservedGpu(rs.getLong("int_gpu_reserved")) + .setReservedGpus(rs.getInt("int_gpus_reserved")) + .setReservedGpuMemory(rs.getLong("int_gpu_mem_reserved")) .setUsedMemory(rs.getLong("int_mem_used")) + .setUsedGpuMemory(rs.getLong("int_gpu_mem_used")) .setFrameName(SqlUtil.getString(rs, "frame_name")) .setJobName(SqlUtil.getString(rs,"job_name")) .setGroupName(SqlUtil.getString(rs,"folder_name")) @@ -1006,20 +1011,22 @@ public static NestedHost.Builder mapNestedHostBuilder(ResultSet rs) throws SQLEx .setFreeMcp(rs.getLong("int_mcp_free")) .setFreeMemory(rs.getLong("int_mem_free")) .setFreeSwap(rs.getLong("int_swap_free")) - .setFreeGpu(rs.getLong("int_gpu_free")) + .setFreeGpuMemory(rs.getLong("int_gpu_mem_free")) .setLoad(rs.getInt("int_load")) .setNimbyEnabled(rs.getBoolean("b_nimby")) .setCores(Convert.coreUnitsToCores(rs.getInt("int_cores"))) .setIdleCores(Convert.coreUnitsToCores(rs.getInt("int_cores_idle"))) .setMemory(rs.getLong("int_mem")) .setIdleMemory(rs.getLong("int_mem_idle")) - .setGpu(rs.getLong("int_gpu")) - .setIdleGpu(rs.getLong("int_gpu_idle")) + .setGpus(rs.getInt("int_gpus")) + .setIdleGpus(rs.getInt("int_gpus_idle")) + .setGpuMemory(rs.getLong("int_gpu_mem")) + .setIdleGpuMemory(rs.getLong("int_gpu_mem_idle")) .setState(HardwareState.valueOf(SqlUtil.getString(rs,"host_state"))) .setTotalMcp(rs.getLong("int_mcp_total")) .setTotalMemory(rs.getLong("int_mem_total")) .setTotalSwap(rs.getLong("int_swap_total")) - .setTotalGpu(rs.getLong("int_gpu_total")) + .setTotalGpuMemory(rs.getLong("int_gpu_mem_total")) .setPingTime((int) (rs.getTimestamp("ts_ping").getTime() / 1000)) .setLockState(LockState.valueOf(SqlUtil.getString(rs,"str_lock_state"))) .setHasComment(rs.getBoolean("b_comment")) @@ -1041,20 +1048,22 @@ public static Host.Builder mapHostBuilder(ResultSet rs) throws SQLException { builder.setFreeMcp(rs.getLong("int_mcp_free")); builder.setFreeMemory(rs.getLong("int_mem_free")); builder.setFreeSwap(rs.getLong("int_swap_free")); - builder.setFreeGpu(rs.getLong("int_gpu_free")); + builder.setFreeGpuMemory(rs.getLong("int_gpu_mem_free")); builder.setLoad(rs.getInt("int_load")); builder.setNimbyEnabled(rs.getBoolean("b_nimby")); builder.setCores(Convert.coreUnitsToCores(rs.getInt("int_cores"))); builder.setIdleCores(Convert.coreUnitsToCores(rs.getInt("int_cores_idle"))); builder.setMemory(rs.getLong("int_mem")); builder.setIdleMemory(rs.getLong("int_mem_idle")); - builder.setGpu(rs.getLong("int_gpu")); - builder.setIdleGpu(rs.getLong("int_gpu_idle")); + builder.setGpus(rs.getInt("int_gpus")); + builder.setIdleGpus(rs.getInt("int_gpus_idle")); + builder.setGpuMemory(rs.getLong("int_gpu_mem")); + builder.setIdleGpuMemory(rs.getLong("int_gpu_mem_idle")); builder.setState(HardwareState.valueOf(SqlUtil.getString(rs,"host_state"))); builder.setTotalMcp(rs.getLong("int_mcp_total")); builder.setTotalMemory(rs.getLong("int_mem_total")); builder.setTotalSwap(rs.getLong("int_swap_total")); - builder.setTotalGpu(rs.getLong("int_gpu_total")); + builder.setTotalGpuMemory(rs.getLong("int_gpu_mem_total")); builder.setPingTime((int) (rs.getTimestamp("ts_ping").getTime() / 1000)); builder.setLockState(LockState.valueOf(SqlUtil.getString(rs,"str_lock_state"))); builder.setHasComment(rs.getBoolean("b_comment")); @@ -1109,6 +1118,11 @@ public Allocation mapRow(ResultSet rs, int rowNum) throws SQLException { .setIdleCores(Convert.coreUnitsToCores(rs.getInt("int_idle_cores"))) .setRunningCores(Convert.coreUnitsToCores(rs.getInt("int_running_cores"))) .setLockedCores(Convert.coreUnitsToCores(rs.getInt("int_locked_cores"))) + .setGpus(rs.getInt("int_gpus")) + .setAvailableGpus(rs.getInt("int_available_gpus")) + .setIdleGpus(rs.getInt("int_idle_gpus")) + .setRunningGpus(rs.getInt("int_running_gpus")) + .setLockedGpus(rs.getInt("int_locked_gpus")) .setHosts(rs.getInt("int_hosts")) .setDownHosts(rs.getInt("int_down_hosts")) .setLockedHosts(rs.getInt("int_locked_hosts")) @@ -1128,6 +1142,7 @@ public Group mapRow(ResultSet rs, int rowNum) throws SQLException { .setDependFrames(rs.getInt("int_depend_count")) .setPendingJobs(rs.getInt("int_job_count")) .setReservedCores(Convert.coreUnitsToCores(rs.getInt("int_cores"))) + .setReservedGpus(rs.getInt("int_gpus")) .build(); return Group.newBuilder() .setId(SqlUtil.getString(rs,"pk_folder")) @@ -1136,8 +1151,12 @@ public Group mapRow(ResultSet rs, int rowNum) throws SQLException { .setDefaultJobPriority(rs.getInt("int_job_priority")) .setDefaultJobMinCores(Convert.coreUnitsToCores(rs.getInt("int_job_min_cores"))) .setDefaultJobMaxCores(Convert.coreUnitsToCores(rs.getInt("int_job_max_cores"))) + .setDefaultJobMinGpus(rs.getInt("int_job_min_gpus")) + .setDefaultJobMaxGpus(rs.getInt("int_job_max_gpus")) .setMaxCores(Convert.coreUnitsToCores(rs.getInt("int_max_cores"))) .setMinCores(Convert.coreUnitsToCores(rs.getInt("int_min_cores"))) + .setMaxGpus(rs.getInt("int_max_gpus")) + .setMinGpus(rs.getInt("int_min_gpus")) .setLevel(rs.getInt("int_level")) .setParentId(SqlUtil.getString(rs, "pk_parent_folder")) .setGroupStats(stats) @@ -1153,6 +1172,8 @@ public Job mapRow(ResultSet rs, int rowNum) throws SQLException { .setLogDir(SqlUtil.getString(rs, "str_log_dir")) .setMaxCores(Convert.coreUnitsToCores(rs.getInt("int_max_cores"))) .setMinCores(Convert.coreUnitsToCores(rs.getInt("int_min_cores"))) + .setMaxGpus(rs.getInt("int_max_gpus")) + .setMinGpus(rs.getInt("int_min_gpus")) .setName(SqlUtil.getString(rs,"str_name")) .setPriority(rs.getInt("int_priority")) .setShot(SqlUtil.getString(rs,"str_shot")) @@ -1189,6 +1210,7 @@ public static JobStats mapJobStats(ResultSet rs) throws SQLException { JobStats.Builder statsBuilder = JobStats.newBuilder() .setReservedCores(Convert.coreUnitsToCores(rs.getInt("int_cores"))) + .setReservedGpus(rs.getInt("int_gpus")) .setMaxRss(rs.getLong("int_max_rss")) .setTotalFrames(rs.getInt("int_frame_count")) .setTotalLayers(rs.getInt("int_layer_count")) @@ -1202,6 +1224,9 @@ public static JobStats mapJobStats(ResultSet rs) throws SQLException { .setFailedCoreSec(rs.getLong("int_core_time_fail")) .setRenderedCoreSec(rs.getLong("int_core_time_success")) .setTotalCoreSec( rs.getLong("int_core_time_fail") + rs.getLong("int_core_time_success")) + .setFailedGpuSec(rs.getLong("int_gpu_time_fail")) + .setRenderedGpuSec(rs.getLong("int_gpu_time_success")) + .setTotalGpuSec(rs.getLong("int_gpu_time_fail") + rs.getLong("int_gpu_time_success")) .setRenderedFrameCount( rs.getLong("int_frame_success_count")) .setFailedFrameCount(rs.getLong("int_frame_fail_count")) .setHighFrameSec(rs.getInt("int_clock_time_high")); @@ -1236,7 +1261,9 @@ public Layer mapRow(ResultSet rs, int rowNum) throws SQLException { .setMaxCores(Convert.coreUnitsToCores(rs.getInt("int_cores_max"))) .setIsThreadable(rs.getBoolean("b_threadable")) .setMinMemory(rs.getLong("int_mem_min")) - .setMinGpu(rs.getLong("int_gpu_min")) + .setMinGpus(rs.getInt("int_gpus_min")) + .setMaxGpus(rs.getInt("int_gpus_max")) + .setMinGpuMemory(rs.getLong("int_gpu_mem_min")) .setType(LayerType.valueOf(SqlUtil.getString(rs,"str_type"))) .addAllTags(Sets.newHashSet( SqlUtil.getString(rs,"str_tags"). @@ -1249,6 +1276,7 @@ public Layer mapRow(ResultSet rs, int rowNum) throws SQLException { LayerStats.Builder statsBuilder = LayerStats.newBuilder() .setReservedCores(Convert.coreUnitsToCores(rs.getInt("int_cores"))) + .setReservedGpus(rs.getInt("int_gpus")) .setMaxRss(rs.getLong("int_max_rss")) .setTotalFrames(rs.getInt("int_total_count")) .setWaitingFrames(rs.getInt("int_waiting_count")) @@ -1263,6 +1291,9 @@ public Layer mapRow(ResultSet rs, int rowNum) throws SQLException { .setRenderedCoreSec(rs.getLong("int_core_time_success")) .setTotalCoreSec( rs.getLong("int_core_time_fail") + rs.getLong("int_core_time_success")) + .setFailedGpuSec(rs.getLong("int_gpu_time_fail")) + .setRenderedGpuSec(rs.getLong("int_gpu_time_success")) + .setTotalGpuSec(rs.getLong("int_gpu_time_fail") + rs.getLong("int_gpu_time_success")) .setRenderedFrameCount( rs.getLong("int_frame_success_count")) .setFailedFrameCount(rs.getLong("int_frame_fail_count")) .setHighFrameSec(rs.getInt("int_clock_time_high")) @@ -1301,6 +1332,7 @@ public Subscription mapRow(ResultSet rs, int rowNum) throws SQLException { .setBurst(rs.getInt("int_burst")) .setName(rs.getString("name")) .setReservedCores(rs.getInt("int_cores")) + .setReservedGpus(rs.getInt("int_gpus")) .setSize(rs.getInt("int_size")) .setAllocationName(rs.getString("alloc_name")) .setShowName(rs.getString("show_name")) @@ -1321,9 +1353,10 @@ public UpdatedFrame mapRow(ResultSet rs, int rowNum) throws SQLException { .setUsedMemory(rs.getInt("int_mem_used")); if (SqlUtil.getString(rs, "str_host") != null) { - builder.setLastResource(String.format(Locale.ROOT, "%s/%2.2f", + builder.setLastResource(String.format(Locale.ROOT, "%s/%2.2f/%d", SqlUtil.getString(rs, "str_host"), - Convert.coreUnitsToCores(rs.getInt("int_cores")))); + Convert.coreUnitsToCores(rs.getInt("int_cores")), + rs.getInt("int_gpus"))); } else { builder.setLastResource(""); } @@ -1360,14 +1393,14 @@ public Frame mapRow(ResultSet rs, int rowNum) throws SQLException { .setLayerName(SqlUtil.getString(rs,"layer_name")) .setUsedMemory(rs.getLong("int_mem_used")) .setReservedMemory(rs.getLong("int_mem_reserved")) - .setReservedGpu(rs.getLong("int_gpu_reserved")) + .setReservedGpuMemory(rs.getLong("int_gpu_mem_reserved")) .setCheckpointState(CheckpointState.valueOf( SqlUtil.getString(rs,"str_checkpoint_state"))) .setCheckpointCount(rs.getInt("int_checkpoint_count")); if (SqlUtil.getString(rs,"str_host") != null) { builder.setLastResource(CueUtil.buildProcName(SqlUtil.getString(rs,"str_host"), - rs.getInt("int_cores"))); + rs.getInt("int_cores"), rs.getInt("int_gpus"))); } else { builder.setLastResource(""); } @@ -1388,9 +1421,12 @@ public Frame mapRow(ResultSet rs, int rowNum) throws SQLException { } builder.setTotalCoreTime(rs.getInt("int_total_past_core_time")); + builder.setTotalGpuTime(rs.getInt("int_total_past_gpu_time")); if (builder.getState() == FrameState.RUNNING) { builder.setTotalCoreTime(builder.getTotalCoreTime() + (int)(System.currentTimeMillis() / 1000 - builder.getStartTime()) * rs.getInt("int_cores") / 100); + builder.setTotalGpuTime(builder.getTotalGpuTime() + + (int)(System.currentTimeMillis() / 1000 - builder.getStartTime()) * rs.getInt("int_gpus")); } return builder.build(); } @@ -1406,7 +1442,9 @@ public Service mapRow(ResultSet rs, int rowNum) throws SQLException { .setMinCores(rs.getInt("int_cores_min")) .setMaxCores(rs.getInt("int_cores_max")) .setMinMemory(rs.getInt("int_mem_min")) - .setMinGpu(rs.getInt("int_gpu_min")) + .setMinGpus(rs.getInt("int_gpus_min")) + .setMaxGpus(rs.getInt("int_gpus_max")) + .setMinGpuMemory(rs.getInt("int_gpu_mem_min")) .addAllTags(Lists.newArrayList(ServiceDaoJdbc.splitTags( SqlUtil.getString(rs,"str_tags")))) .setTimeout(rs.getInt("int_timeout")) @@ -1425,7 +1463,9 @@ public ServiceOverride mapRow(ResultSet rs, int rowNum) throws SQLException { .setMinCores(rs.getInt("int_cores_min")) .setMaxCores(rs.getInt("int_cores_max")) .setMinMemory(rs.getInt("int_mem_min")) - .setMinGpu(rs.getInt("int_gpu_min")) + .setMinGpus(rs.getInt("int_gpus_min")) + .setMaxGpus(rs.getInt("int_gpus_max")) + .setMinGpuMemory(rs.getInt("int_gpu_mem_min")) .addAllTags(Lists.newArrayList(ServiceDaoJdbc.splitTags( SqlUtil.getString(rs,"str_tags")))) .setTimeout(rs.getInt("int_timeout")) @@ -1450,6 +1490,7 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { .setRenderedFrameCount(rs.getLong("int_frame_success_count")) .setFailedFrameCount(rs.getLong("int_frame_fail_count")) .setReservedCores(Convert.coreUnitsToCores(rs.getInt("int_cores"))) + .setReservedGpus(rs.getInt("int_gpus")) .setPendingJobs(rs.getInt("int_job_count")) .build(); return Show.newBuilder() @@ -1458,6 +1499,8 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { .setActive(rs.getBoolean("b_active")) .setDefaultMaxCores(Convert.coreUnitsToCores(rs.getInt("int_default_max_cores"))) .setDefaultMinCores(Convert.coreUnitsToCores(rs.getInt("int_default_min_cores"))) + .setDefaultMaxGpus(rs.getInt("int_default_max_gpus")) + .setDefaultMinGpus(rs.getInt("int_default_min_gpus")) .setBookingEnabled(rs.getBoolean("b_booking_enabled")) .setDispatchEnabled(rs.getBoolean("b_dispatch_enabled")) .setCommentEmail(SqlUtil.getString(rs,"str_comment_email")) @@ -1513,13 +1556,15 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "frame.str_state,"+ "frame.str_host,"+ "frame.int_cores,"+ + "frame.int_gpus,"+ "frame.int_mem_max_used," + "frame.int_mem_used, " + "frame.int_mem_reserved, " + - "frame.int_gpu_reserved, " + + "frame.int_gpu_mem_reserved, " + "frame.str_checkpoint_state,"+ "frame.int_checkpoint_count,"+ "frame.int_total_past_core_time,"+ + "frame.int_total_past_gpu_time,"+ "layer.str_name AS layer_name," + "job.str_name AS job_name "+ "FROM "+ @@ -1556,7 +1601,10 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "proc.int_mem_reserved, " + "proc.int_mem_used, " + "proc.int_mem_max_used, " + - "proc.int_gpu_reserved, " + + "proc.int_gpus_reserved, " + + "proc.int_gpu_mem_reserved, " + + "proc.int_gpu_mem_used, " + + "proc.int_gpu_mem_max_used, " + "proc.ts_ping, " + "proc.ts_booked, " + "proc.ts_dispatched, " + @@ -1593,6 +1641,7 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "frame.str_state,"+ "frame.str_host,"+ "frame.int_cores,"+ + "frame.int_gpus,"+ "frame.ts_llu,"+ "COALESCE(proc.int_mem_max_used, frame.int_mem_max_used) AS int_mem_max_used," + "COALESCE(proc.int_mem_used, frame.int_mem_used) AS int_mem_used " + @@ -1617,6 +1666,11 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "vs_alloc_usage.int_running_cores,"+ "vs_alloc_usage.int_available_cores,"+ "vs_alloc_usage.int_locked_cores,"+ + "vs_alloc_usage.int_gpus,"+ + "vs_alloc_usage.int_idle_gpus,"+ + "vs_alloc_usage.int_running_gpus,"+ + "vs_alloc_usage.int_available_gpus,"+ + "vs_alloc_usage.int_locked_gpus,"+ "vs_alloc_usage.int_hosts,"+ "vs_alloc_usage.int_locked_hosts,"+ "vs_alloc_usage.int_down_hosts "+ @@ -1650,6 +1704,8 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "str_ti_task,"+ "int_cores,"+ "int_min_cores,"+ + "int_gpus,"+ + "int_min_gpus,"+ "b_managed " + "FROM " + "point," + @@ -1672,6 +1728,8 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "str_ti_task,"+ "int_cores,"+ "int_min_cores,"+ + "int_gpus,"+ + "int_min_gpus,"+ "b_managed " + "FROM " + "point," + @@ -1702,11 +1760,13 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "host_local.pk_host_local,"+ "host_local.int_cores_idle,"+ "host_local.int_cores_max,"+ + "host_local.int_gpus_idle,"+ + "host_local.int_gpus_max,"+ "host_local.int_threads,"+ "host_local.int_mem_idle,"+ "host_local.int_mem_max,"+ - "host_local.int_gpu_idle,"+ - "host_local.int_gpu_max,"+ + "host_local.int_gpu_mem_idle,"+ + "host_local.int_gpu_mem_max,"+ "host_local.str_type,"+ "(SELECT str_name FROM host WHERE host.pk_host = host_local.pk_host) " + "AS str_host_name,"+ @@ -1775,6 +1835,10 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "folder.int_job_max_cores," + "folder_resource.int_min_cores,"+ "folder_resource.int_max_cores,"+ + "folder.int_job_min_gpus," + + "folder.int_job_max_gpus," + + "folder_resource.int_min_gpus,"+ + "folder_resource.int_max_gpus,"+ "folder.b_default, " + "folder_level.int_level, " + "c.int_waiting_count, " + @@ -1782,7 +1846,8 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "c.int_running_count,"+ "c.int_dead_count,"+ "c.int_job_count,"+ - "c.int_cores " + + "c.int_cores," + + "c.int_gpus " + "FROM " + "folder, " + "folder_level," + @@ -1817,6 +1882,8 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "job.str_log_dir," + "job_resource.int_max_cores," + "job_resource.int_min_cores," + + "job_resource.int_max_gpus," + + "job_resource.int_min_gpus," + "job.str_name," + "job.str_shot,"+ "job.str_state,"+ @@ -1843,12 +1910,15 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "job_stat.int_succeeded_count, "+ "job_usage.int_core_time_success, "+ "job_usage.int_core_time_fail, " + + "job_usage.int_gpu_time_success, "+ + "job_usage.int_gpu_time_fail, " + "job_usage.int_frame_success_count, "+ "job_usage.int_frame_fail_count, "+ "job_usage.int_clock_time_high,"+ "job_usage.int_clock_time_success,"+ "job_mem.int_max_rss,"+ - "(job_resource.int_cores + job_resource.int_local_cores) AS int_cores " + + "(job_resource.int_cores + job_resource.int_local_cores) AS int_cores," + + "(job_resource.int_gpus + job_resource.int_local_gpus) AS int_gpus " + "FROM " + "job,"+ "folder,"+ @@ -1885,6 +1955,8 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "layer_stat.int_succeeded_count," + "layer_usage.int_core_time_success," + "layer_usage.int_core_time_fail, "+ + "layer_usage.int_gpu_time_success," + + "layer_usage.int_gpu_time_fail, "+ "layer_usage.int_frame_success_count, "+ "layer_usage.int_frame_fail_count, "+ "layer_usage.int_clock_time_low, "+ @@ -1892,7 +1964,8 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "layer_usage.int_clock_time_success," + "layer_usage.int_clock_time_fail," + "layer_mem.int_max_rss,"+ - "layer_resource.int_cores " + + "layer_resource.int_cores," + + "layer_resource.int_gpus " + "FROM " + "layer, " + "job," + @@ -1923,6 +1996,8 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "layer_stat.int_succeeded_count, " + "layer_usage.int_core_time_success, " + "layer_usage.int_core_time_fail, " + + "layer_usage.int_gpu_time_success, " + + "layer_usage.int_gpu_time_fail, " + "layer_usage.int_frame_success_count, " + "layer_usage.int_frame_fail_count, " + "layer_usage.int_clock_time_low, " + @@ -1931,6 +2006,7 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "layer_usage.int_clock_time_fail, " + "layer_mem.int_max_rss, " + "layer_resource.int_cores, " + + "layer_resource.int_gpus, " + "limit_names.str_limit_names " + "FROM " + "layer " + @@ -1976,6 +2052,7 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "COALESCE(vs_show_stat.int_running_count,0) AS int_running_count," + "COALESCE(vs_show_stat.int_dead_count,0) AS int_dead_count," + "COALESCE(vs_show_resource.int_cores,0) AS int_cores, " + + "COALESCE(vs_show_resource.int_gpus,0) AS int_gpus, " + "COALESCE(vs_show_stat.int_job_count,0) AS int_job_count " + "FROM " + "show " + @@ -1992,7 +2069,9 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "service.int_cores_min," + "service.int_cores_max," + "service.int_mem_min," + - "service.int_gpu_min," + + "service.int_gpus_min," + + "service.int_gpus_max," + + "service.int_gpu_mem_min," + "service.str_tags," + "service.int_timeout," + "service.int_timeout_llu " + @@ -2007,7 +2086,9 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "show_service.int_cores_min," + "show_service.int_cores_max," + "show_service.int_mem_min," + - "show_service.int_gpu_min," + + "show_service.int_gpus_min," + + "show_service.int_gpus_max," + + "show_service.int_gpu_mem_min," + "show_service.str_tags," + "show_service.int_timeout," + "show_service.int_timeout_llu " + @@ -2023,6 +2104,8 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "task.str_shot,"+ "task.int_min_cores + task.int_adjust_cores AS int_min_cores, "+ "task.int_adjust_cores, " + + "task.int_min_gpus + task.int_adjust_gpus AS int_min_gpus, "+ + "task.int_adjust_gpus, " + "dept.str_name AS str_dept "+ "FROM " + "task,"+ @@ -2045,8 +2128,10 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "host.int_cores_idle,"+ "host.int_mem,"+ "host.int_mem_idle,"+ - "host.int_gpu,"+ - "host.int_gpu_idle,"+ + "host.int_gpus,"+ + "host.int_gpus_idle,"+ + "host.int_gpu_mem,"+ + "host.int_gpu_mem_idle,"+ "host.str_tags,"+ "host.str_lock_state,"+ "host.b_comment,"+ @@ -2058,8 +2143,8 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "host_stat.int_swap_free,"+ "host_stat.int_mcp_total,"+ "host_stat.int_mcp_free,"+ - "host_stat.int_gpu_total,"+ - "host_stat.int_gpu_free,"+ + "host_stat.int_gpu_mem_total,"+ + "host_stat.int_gpu_mem_free,"+ "host_stat.int_load, " + "alloc.str_name AS alloc_name " + "FROM " + @@ -2097,6 +2182,7 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "subscription.int_burst, " + "subscription.int_size, " + "subscription.int_cores, " + + "subscription.int_gpus, " + "show.str_name AS show_name, " + "alloc.str_name AS alloc_name, " + "facility.str_name AS facility_name " + @@ -2135,10 +2221,14 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "frame.int_mem_max_used," + "frame.int_mem_used, " + "frame.int_mem_reserved, " + - "frame.int_gpu_reserved, " + + "frame.int_gpus,"+ + "frame.int_gpu_mem_max_used, " + + "frame.int_gpu_mem_used, " + + "frame.int_gpu_mem_reserved, " + "frame.str_checkpoint_state,"+ "frame.int_checkpoint_count,"+ "frame.int_total_past_core_time,"+ + "frame.int_total_past_gpu_time,"+ "layer.str_name AS layer_name," + "job.str_name AS job_name, "+ "ROW_NUMBER() OVER " + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/AbstractDispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/AbstractDispatcher.java index ddf3b2a2b..73f5aef73 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/AbstractDispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/AbstractDispatcher.java @@ -129,6 +129,7 @@ public boolean dispatchHost(DispatchFrame frame, VirtualProc proc) { dispatchSummary(proc, frame, "Booking"); DispatchSupport.bookedProcs.getAndIncrement(); DispatchSupport.bookedCores.addAndGet(proc.coresReserved); + DispatchSupport.bookedGpus.addAndGet(proc.gpusReserved); return true; } catch (FrameReservationException fre) { /* @@ -222,8 +223,10 @@ private static void dispatchSummary(VirtualProc p, DispatchFrame f, String type) " cores / " + CueUtil.KbToMb(p.memoryReserved) + " memory / " + - p.gpuReserved + - " gpu on " + + p.gpusReserved + + " gpus / " + + CueUtil.KbToMb(p.gpuMemoryReserved) + + " gpu memory " + p.getName() + " to " + f.show + "/" + f.shot; logger.info(msg); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java index d57caf3e9..beacefd97 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java @@ -135,7 +135,8 @@ private List dispatchJobs(DispatchHost host, Set jobs) { if (!host.hasAdditionalResources( Dispatcher.CORE_POINTS_RESERVED_MIN, Dispatcher.MEM_RESERVED_MIN, - Dispatcher.GPU_RESERVED_MIN)) { + Dispatcher.GPU_UNITS_RESERVED_MIN, + Dispatcher.MEM_GPU_RESERVED_MIN)) { return procs; } @@ -179,7 +180,8 @@ private Set getGpuJobs(DispatchHost host, ShowInterface show) { if (host.hasAdditionalResources( Dispatcher.CORE_POINTS_RESERVED_DEFAULT, Dispatcher.MEM_RESERVED_MIN, - 1)) { + Dispatcher.GPU_UNITS_RESERVED_DEFAULT, + Dispatcher.MEM_GPU_RESERVED_DEFAULT)) { if (show == null) jobs = dispatchSupport.findDispatchJobs(host, getIntProperty("dispatcher.job_query_max")); @@ -262,11 +264,12 @@ public List dispatchHost(DispatchHost host, JobInterface job) { if (host.idleCores < frame.minCores || host.idleMemory < frame.minMemory || - host.idleGpu < frame.minGpu) { + host.idleGpus < frame.minGpus || + host.idleGpuMemory < frame.minGpuMemory) { break; } - if (!dispatchSupport.isJobBookable(job, proc.coresReserved)) { + if (!dispatchSupport.isJobBookable(job, proc.coresReserved, proc.gpusReserved)) { break; } @@ -289,17 +292,19 @@ public void wrapDispatchFrame() { DispatchSupport.bookedProcs.getAndIncrement(); DispatchSupport.bookedCores.addAndGet(proc.coresReserved); + DispatchSupport.bookedGpus.addAndGet(proc.gpusReserved); if (host.strandedCores > 0) { dispatchSupport.pickupStrandedCores(host); break; } - host.useResources(proc.coresReserved, proc.memoryReserved, proc.gpuReserved); + host.useResources(proc.coresReserved, proc.memoryReserved, proc.gpusReserved, proc.gpuMemoryReserved); if (!host.hasAdditionalResources( Dispatcher.CORE_POINTS_RESERVED_MIN, Dispatcher.MEM_RESERVED_MIN, - Dispatcher.GPU_RESERVED_MIN)) { + Dispatcher.GPU_UNITS_RESERVED_MIN, + Dispatcher.MEM_GPU_RESERVED_MIN)) { break; } else if (procs.size() >= getIntProperty("dispatcher.job_frame_dispatch_max")) { @@ -398,8 +403,10 @@ private void dispatchSummary(VirtualProc p, DispatchFrame f, String type) { " cores / " + CueUtil.KbToMb(p.memoryReserved) + " memory / " + - p.gpuReserved + - " gpu on " + + p.gpusReserved + + " gpus / " + + CueUtil.KbToMb(p.gpuMemoryReserved) + + " gpu memory " + p.getName() + " to " + f.show + "/" + f.shot; logger.trace(msg); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java index ebdd5082d..47dac264a 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java @@ -77,6 +77,11 @@ public interface DispatchSupport { */ static final AtomicLong bookedCores = new AtomicLong(0); + /** + * Long for counting how many gpus have been booked + */ + static final AtomicLong bookedGpus = new AtomicLong(0); + /** * Long for counting how many procs have been booked */ @@ -122,6 +127,16 @@ public interface DispatchSupport { */ static final AtomicLong strandedCoresCount = new AtomicLong(0); + /** + * Count number of picked up gpus. + */ + static final AtomicLong pickedUpGpusCount = new AtomicLong(0); + + /** + * Count number of stranded gpus. + */ + static final AtomicLong strandedGpusCount = new AtomicLong(0); + /** * Set the proc's frame assignment to null; * @@ -456,7 +471,7 @@ void updateProcMemoryUsage(FrameInterface frame, long rss, long maxRss, long vsi * @param job * @return */ - boolean isJobBookable(JobInterface job, int coreUnits); + boolean isJobBookable(JobInterface job, int coreUnits, int gpuUnits); /** * Return true if the specified show is at or over its @@ -511,6 +526,40 @@ void updateProcMemoryUsage(FrameInterface frame, long rss, long maxRss, long vsi */ void determineIdleCores(DispatchHost host, int load); + /** + * Pickup any gpus that were stranded on the given host. + * + * @param host + */ + void pickupStrandedGpus(DispatchHost host); + + /** + * Return true if the host has stranded gpus. + * + * @param host + * @return + */ + boolean hasStrandedGpus(HostInterface host); + + /** + * Add stranded gpus for the given host. Stranded + * gpus will automatically be added to the next frame dispatched + * from the host to make up for gpus stranded with no memory. + * + * @param host + * @param gpus + */ + void strandGpus(DispatchHost host, int gpus); + + /** + * Lowers the perceived idle gpus on a machine if + * the load is over certain threshold. + * + * @param host + * @param load + */ + void determineIdleGpus(DispatchHost host, int load); + /** * Return a set of job IDs that can take the given host. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java index 3e3d82b2f..ad1d8196c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java @@ -42,6 +42,7 @@ import com.imageworks.spcue.ResourceUsage; import com.imageworks.spcue.ShowInterface; import com.imageworks.spcue.StrandedCores; +import com.imageworks.spcue.StrandedGpus; import com.imageworks.spcue.VirtualProc; import com.imageworks.spcue.dao.BookingDao; import com.imageworks.spcue.dao.DispatcherDao; @@ -82,6 +83,9 @@ public class DispatchSupportService implements DispatchSupport { private ConcurrentHashMap strandedCores = new ConcurrentHashMap(); + private ConcurrentHashMap strandedGpus = + new ConcurrentHashMap(); + @Override public void pickupStrandedCores(DispatchHost host) { logger.info(host + "picked up stranded cores"); @@ -113,6 +117,35 @@ public void strandCores(DispatchHost host, int cores) { strandedCoresCount.getAndIncrement(); } + @Override + public void pickupStrandedGpus(DispatchHost host) { + logger.info(host + "picked up stranded gpu"); + pickedUpGpusCount.getAndIncrement(); + strandedGpus.remove(host.getHostId()); + } + + @Override + public boolean hasStrandedGpus(HostInterface host) { + StrandedGpus stranded = strandedGpus.get(host.getHostId()); + if (stranded == null) { + return false; + } + if (stranded.isExpired()) { + return false; + } + + return true; + } + + @Override + public void strandGpus(DispatchHost host, int gpus) { + logger.info(host + " found " + gpus + ", stranded gpu"); + host.strandedGpus = gpus; + strandedGpus.putIfAbsent(host.getHostId(), new StrandedGpus(gpus)); + strandedGpusCount.getAndIncrement(); + } + + @Transactional(readOnly = true) public List findNextDispatchFrames(JobInterface job, VirtualProc proc, int limit) { return dispatcherDao.findNextDispatchFrames(job, proc, limit); @@ -245,7 +278,7 @@ public boolean isJobBookable(JobInterface job) { @Override @Transactional(propagation = Propagation.REQUIRED, readOnly=true) - public boolean isJobBookable(JobInterface job, int coreUnits) { + public boolean isJobBookable(JobInterface job, int coreUnits, int gpuUnits) { if (!jobDao.hasPendingFrames(job)) { return false; @@ -255,6 +288,10 @@ public boolean isJobBookable(JobInterface job, int coreUnits) { return false; } + if (jobDao.isOverMaxGpus(job, gpuUnits)) { + return false; + } + return true; } @@ -363,6 +400,7 @@ public RunFrame prepareRqdRunFrame(VirtualProc proc, DispatchFrame frame) { .setLayerId(frame.getLayerId()) .setResourceId(proc.getProcId()) .setNumCores(proc.coresReserved) + .setNumGpus(proc.gpusReserved) .setStartTime(System.currentTimeMillis()) .setIgnoreNimby(proc.isLocalDispatch) .putAllEnvironment(jobDao.getEnvironment(frame)) @@ -370,6 +408,8 @@ public RunFrame prepareRqdRunFrame(VirtualProc proc, DispatchFrame frame) { .putEnvironment("CUE3", "1") .putEnvironment("CUE_THREADS", String.valueOf(threads)) .putEnvironment("CUE_MEMORY", String.valueOf(proc.memoryReserved)) + .putEnvironment("CUE_GPUS", String.valueOf(proc.gpusReserved)) + .putEnvironment("CUE_GPU_MEMORY", String.valueOf(proc.gpuMemoryReserved)) .putEnvironment("CUE_LOG_PATH", frame.logDir) .putEnvironment("CUE_RANGE", frame.range) .putEnvironment("CUE_CHUNK", String.valueOf(frame.chunkSize)) @@ -575,6 +615,14 @@ public void determineIdleCores(DispatchHost host, int load) { } } + @Override + public void determineIdleGpus(DispatchHost host, int load) { + int idleGpu = host.gpus - load; + if (idleGpu < host.idleGpus) { + host.idleGpus = idleGpu; + } + } + public DispatcherDao getDispatcherDao() { return dispatcherDao; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java index d29c51f9c..3440fb595 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java @@ -44,6 +44,10 @@ public interface Dispatcher { // The minimum amount of core points you can assign to a frame. public static final int CORE_POINTS_RESERVED_MIN = 10; + // The minimum amount of gpu points you can assign to a frame. + public static final int GPU_UNITS_RESERVED_DEFAULT = 0; + public static final int GPU_UNITS_RESERVED_MIN = 0; + // Amount of load per core a host can have before the perceived // number of idle cores is modified to reflect load conditions // on the host. @@ -69,13 +73,13 @@ public interface Dispatcher { // The default amount of gpu memory reserved for a frame if no gpu memory // reservation settings are specified - public static final long GPU_RESERVED_DEFAULT = 0; + public static final long MEM_GPU_RESERVED_DEFAULT = 0; // The minimum amount of gpu memory that can be assigned to a frame. - public static final long GPU_RESERVED_MIN = 0; + public static final long MEM_GPU_RESERVED_MIN = 0; // The maximum amount of gpu memory that can be assigned to a frame. - public static final long GPU_RESERVED_MAX = CueUtil.GB4; + public static final long MEM_GPU_RESERVED_MAX = CueUtil.GB * 1024; // Return value for cleared frame public static final int EXIT_STATUS_FRAME_CLEARED = 299; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java index fe2482720..20afd374c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java @@ -401,7 +401,7 @@ else if (report.getHost().getNimbyLocked()) { // Then check for higher priority jobs // If not, rebook this job if (job.autoUnbook && proc.coresReserved >= 100) { - if (jobManager.isOverMinCores(job)) { + if (jobManager.isOverMinCores(job) && jobManager.isOverMinGpus(job)) { try { boolean unbook = diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java index 69b602430..34e02021e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java @@ -135,18 +135,6 @@ public void handleHostReport(HostReport report, boolean isBoot) { long startTime = System.currentTimeMillis(); try { - long totalGpu; - if (report.getHost().getAttributesMap().containsKey("totalGpu")) - totalGpu = Integer.parseInt(report.getHost().getAttributesMap().get("totalGpu")); - else - totalGpu = 0; - - long freeGpu; - if (report.getHost().getAttributesMap().containsKey("freeGpu")) - freeGpu = Integer.parseInt(report.getHost().getAttributesMap().get("freeGpu")); - else - freeGpu = 0; - long swapOut = 0; if (report.getHost().getAttributesMap().containsKey("swapout")) { swapOut = Integer.parseInt(report.getHost().getAttributesMap().get("swapout")); @@ -163,7 +151,7 @@ public void handleHostReport(HostReport report, boolean isBoot) { rhost.getTotalMem(), rhost.getFreeMem(), rhost.getTotalSwap(), rhost.getFreeSwap(), rhost.getTotalMcp(), rhost.getFreeMcp(), - totalGpu, freeGpu, + rhost.getTotalGpuMem(), rhost.getFreeGpuMem(), rhost.getLoad(), new Timestamp(rhost.getBootTime() * 1000l), rhost.getAttributesMap().get("SP_OS")); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LocalDispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LocalDispatcher.java index 55497b83e..23bf6f73a 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LocalDispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LocalDispatcher.java @@ -111,7 +111,8 @@ private List dispatchHost(DispatchHost host, JobInterface job, */ if (!lha.hasAdditionalResources(lha.getThreads() * 100, frame.minMemory, - frame.minGpu)) { + frame.minGpus, + frame.minGpuMemory)) { continue; } @@ -141,10 +142,11 @@ private List dispatchHost(DispatchHost host, JobInterface job, * This should stay here and not go into VirtualProc * or else the count will be off if you fail to book. */ - lha.useResources(proc.coresReserved, proc.memoryReserved, proc.gpuReserved); + lha.useResources(proc.coresReserved, proc.memoryReserved, proc.gpusReserved, proc.gpuMemoryReserved); if (!lha.hasAdditionalResources(lha.getThreads() * 100, Dispatcher.MEM_RESERVED_MIN, - Dispatcher.GPU_RESERVED_MIN)) { + Dispatcher.GPU_UNITS_RESERVED_MIN, + Dispatcher.MEM_GPU_RESERVED_MIN)) { break; } @@ -196,7 +198,8 @@ private List dispatchHost(DispatchHost host, LayerInterface layer, */ if (!lha.hasAdditionalResources(lha.getThreads() * 100, frame.minMemory, - frame.minGpu)) { + frame.minGpus, + frame.minGpuMemory)) { continue; } @@ -226,10 +229,11 @@ private List dispatchHost(DispatchHost host, LayerInterface layer, * This should stay here and not go into VirtualProc * or else the count will be off if you fail to book. */ - lha.useResources(proc.coresReserved, proc.memoryReserved, proc.gpuReserved); + lha.useResources(proc.coresReserved, proc.memoryReserved, proc.gpusReserved, proc.gpuMemoryReserved); if (!lha.hasAdditionalResources(100, Dispatcher.MEM_RESERVED_MIN, - Dispatcher.GPU_RESERVED_MIN)) { + Dispatcher.GPU_UNITS_RESERVED_MIN, + Dispatcher.MEM_GPU_RESERVED_MIN)) { break; } @@ -272,7 +276,8 @@ private List dispatchHost(DispatchHost host, FrameInterface frame, DispatchFrame dframe = jobManager.getDispatchFrame(frame.getId()); if (!lha.hasAdditionalResources(lha.getMaxCoreUnits(), dframe.minMemory, - dframe.minGpu)) { + lha.getMaxGpuUnits(), + dframe.minGpuMemory)) { return procs; } @@ -382,7 +387,8 @@ private void prepHost(DispatchHost host, LocalHostAssignment lha) { host.isLocalDispatch = true; host.idleCores = lha.getIdleCoreUnits(); host.idleMemory = lha.getIdleMemory(); - host.idleGpu = lha.getIdleGpu(); + host.idleGpus = lha.getIdleGpuUnits(); + host.idleGpuMemory = lha.getIdleGpuMemory(); } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/RedirectManager.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/RedirectManager.java index a3519f10e..24b1681e9 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/RedirectManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/RedirectManager.java @@ -258,6 +258,8 @@ public boolean addRedirect(VirtualProc proc, GroupInterface group, DispatchHost host = hostManager.getDispatchHost(proc.getHostId()); host.idleCores = proc.coresReserved; host.idleMemory = proc.memoryReserved; + host.idleGpus = proc.gpusReserved; + host.idleGpuMemory = proc.gpuMemoryReserved; if (dispatchSupport.findDispatchJobs(host, group).size() < 1) { logger.info("Failed to find a pending job in group: " + group.getName()); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/ResourceContainer.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/ResourceContainer.java index c829eb390..0d1141bc1 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/ResourceContainer.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/ResourceContainer.java @@ -27,19 +27,21 @@ public interface ResourceContainer { * * @param minCores * @param minMemory - * @param minGpu + * @param minGpus + * @param minGpuMemory * @return */ - public boolean hasAdditionalResources(int minCores, long minMemory, long minGpu); + public boolean hasAdditionalResources(int minCores, long minMemory, int minGpus, long minGpuMemory); /** * Subtract the given resources from the grand totals. * * @param coreUnits * @param memory - * @param gpu + * @param gpuUnits + * @param gpuMemory */ - public void useResources(int coreUnits, long memory, long gpu); + public void useResources(int coreUnits, long memory, int gpuUnits, long gpuMemory); } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java index c8971a0f4..594444573 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java @@ -92,14 +92,16 @@ else if (job != null) { if (host.hasAdditionalResources( Dispatcher.CORE_POINTS_RESERVED_MIN, Dispatcher.MEM_RESERVED_MIN, - Dispatcher.GPU_RESERVED_MIN)) { + Dispatcher.GPU_UNITS_RESERVED_MIN, + Dispatcher.MEM_GPU_RESERVED_MIN)) { dispatcher.dispatchHost(host); } if (host.hasAdditionalResources( Dispatcher.CORE_POINTS_RESERVED_MIN, Dispatcher.MEM_RESERVED_MIN, - Dispatcher.GPU_RESERVED_MIN)) { + Dispatcher.GPU_UNITS_RESERVED_MIN, + Dispatcher.MEM_GPU_RESERVED_MIN)) { dispatcher.dispatchHostToAllShows(host); } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageFrame.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageFrame.java index 7cc811cfb..27511fb94 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageFrame.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageFrame.java @@ -269,7 +269,7 @@ public void addRenderPartition(FrameAddRenderPartitionRequest request, lha.setThreads(request.getThreads()); lha.setMaxCoreUnits(request.getMaxCores() * 100); lha.setMaxMemory(request.getMaxMemory()); - lha.setMaxGpu(request.getMaxGpu()); + lha.setMaxGpuMemory(request.getMaxGpuMemory()); lha.setType(RenderPartitionType.FRAME_PARTITION); if (localBookingSupport.bookLocal(frame, request.getHost(), request.getUsername(), lha)) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageGroup.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageGroup.java index 8fd7e10ce..b8f3cd43e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageGroup.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageGroup.java @@ -55,6 +55,10 @@ import com.imageworks.spcue.grpc.job.GroupSetDefJobMaxCoresResponse; import com.imageworks.spcue.grpc.job.GroupSetDefJobMinCoresRequest; import com.imageworks.spcue.grpc.job.GroupSetDefJobMinCoresResponse; +import com.imageworks.spcue.grpc.job.GroupSetDefJobMaxGpusRequest; +import com.imageworks.spcue.grpc.job.GroupSetDefJobMaxGpusResponse; +import com.imageworks.spcue.grpc.job.GroupSetDefJobMinGpusRequest; +import com.imageworks.spcue.grpc.job.GroupSetDefJobMinGpusResponse; import com.imageworks.spcue.grpc.job.GroupSetDefJobPriorityRequest; import com.imageworks.spcue.grpc.job.GroupSetDefJobPriorityResponse; import com.imageworks.spcue.grpc.job.GroupSetDeptRequest; @@ -65,6 +69,10 @@ import com.imageworks.spcue.grpc.job.GroupSetMaxCoresResponse; import com.imageworks.spcue.grpc.job.GroupSetMinCoresRequest; import com.imageworks.spcue.grpc.job.GroupSetMinCoresResponse; +import com.imageworks.spcue.grpc.job.GroupSetMaxGpusRequest; +import com.imageworks.spcue.grpc.job.GroupSetMaxGpusResponse; +import com.imageworks.spcue.grpc.job.GroupSetMinGpusRequest; +import com.imageworks.spcue.grpc.job.GroupSetMinGpusResponse; import com.imageworks.spcue.grpc.job.GroupSetNameRequest; import com.imageworks.spcue.grpc.job.GroupSetNameResponse; import com.imageworks.spcue.grpc.job.Job; @@ -189,6 +197,24 @@ public void setDefaultJobMinCores(GroupSetDefJobMinCoresRequest request, StreamO responseObserver.onCompleted(); } + @Override + public void setDefaultJobMaxGpus(GroupSetDefJobMaxGpusRequest request, + StreamObserver responseObserver) { + GroupInterface group = getGroupInterface(request.getGroup()); + groupManager.setGroupDefaultJobMaxGpus(group, request.getMaxGpus()); + responseObserver.onNext(GroupSetDefJobMaxGpusResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + + @Override + public void setDefaultJobMinGpus(GroupSetDefJobMinGpusRequest request, + StreamObserver responseObserver) { + GroupInterface group = getGroupInterface(request.getGroup()); + groupManager.setGroupDefaultJobMinGpus(group, request.getMinGpus()); + responseObserver.onNext(GroupSetDefJobMinGpusResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + @Override public void setName(GroupSetNameRequest request, StreamObserver responseObserver) { GroupInterface group = getGroupInterface(request.getGroup()); @@ -262,6 +288,24 @@ public void setMinCores(GroupSetMinCoresRequest request, responseObserver.onCompleted(); } + @Override + public void setMaxGpus(GroupSetMaxGpusRequest request, + StreamObserver responseObserver) { + GroupInterface group = getGroupInterface(request.getGroup()); + groupManager.setGroupMaxGpus(group, request.getMaxGpus()); + responseObserver.onNext(GroupSetMaxGpusResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + + @Override + public void setMinGpus(GroupSetMinGpusRequest request, + StreamObserver responseObserver) { + GroupInterface group = getGroupInterface(request.getGroup()); + groupManager.setGroupMinGpus(group, request.getMinGpus()); + responseObserver.onNext(GroupSetMinGpusResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + public GroupDao getGroupDao() { return groupDao; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java index dd00476cd..c13177a74 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java @@ -124,10 +124,14 @@ import com.imageworks.spcue.grpc.job.JobSetGroupResponse; import com.imageworks.spcue.grpc.job.JobSetMaxCoresRequest; import com.imageworks.spcue.grpc.job.JobSetMaxCoresResponse; +import com.imageworks.spcue.grpc.job.JobSetMaxGpusRequest; +import com.imageworks.spcue.grpc.job.JobSetMaxGpusResponse; import com.imageworks.spcue.grpc.job.JobSetMaxRetriesRequest; import com.imageworks.spcue.grpc.job.JobSetMaxRetriesResponse; import com.imageworks.spcue.grpc.job.JobSetMinCoresRequest; import com.imageworks.spcue.grpc.job.JobSetMinCoresResponse; +import com.imageworks.spcue.grpc.job.JobSetMinGpusRequest; +import com.imageworks.spcue.grpc.job.JobSetMinGpusResponse; import com.imageworks.spcue.grpc.job.JobSetPriorityRequest; import com.imageworks.spcue.grpc.job.JobSetPriorityResponse; import com.imageworks.spcue.grpc.job.JobStaggerFramesRequest; @@ -376,6 +380,36 @@ public void setMinCores(JobSetMinCoresRequest request, StreamObserver responseObserver) { + try{ + setupJobData(request.getJob()); + jobDao.updateMaxGpus(job, request.getVal()); + responseObserver.onNext(JobSetMaxGpusResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } + } + + @Override + public void setMinGpus(JobSetMinGpusRequest request, StreamObserver responseObserver) { + try{ + setupJobData(request.getJob()); + jobDao.updateMinGpus(job, request.getVal()); + responseObserver.onNext(JobSetMinGpusResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } + } + @Override public void setPriority(JobSetPriorityRequest request, StreamObserver responseObserver) { try{ @@ -772,7 +806,8 @@ public void addRenderPartition(JobAddRenderPartRequest request, StreamObserver responseObserver) { + updateLayer(request.getLayer()); + jobManager.setLayerMinGpus(layer, request.getMinGpus()); + responseObserver.onNext(LayerSetMinGpusResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + @Override public void setMinMemory(LayerSetMinMemoryRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); @@ -234,10 +246,11 @@ public void setMinMemory(LayerSetMinMemoryRequest request, StreamObserver responseObserver) { + public void setMinGpuMemory(LayerSetMinGpuMemoryRequest request, + StreamObserver responseObserver) { updateLayer(request.getLayer()); - layerDao.updateLayerMinGpu(layer, request.getGpu()); - responseObserver.onNext(LayerSetMinGpuResponse.newBuilder().build()); + layerDao.updateLayerMinGpuMemory(layer, request.getGpuMemory()); + responseObserver.onNext(LayerSetMinGpuMemoryResponse.newBuilder().build()); responseObserver.onCompleted(); } @@ -388,7 +401,8 @@ public void addRenderPartition(LayerAddRenderPartitionRequest request, lha.setThreads(request.getThreads()); lha.setMaxCoreUnits(request.getMaxCores() * 100); lha.setMaxMemory(request.getMaxMemory()); - lha.setMaxGpu(request.getMaxGpu()); + lha.setMaxGpuUnits(request.getMaxGpus()); + lha.setMaxGpuMemory(request.getMaxGpuMemory()); lha.setType(RenderPartitionType.LAYER_PARTITION); if (localBookingSupport.bookLocal(layer, request.getHost(), request.getUsername(), lha)) { RenderPartition partition = whiteboard.getRenderPartition(lha); @@ -449,6 +463,14 @@ public void setMaxCores(LayerSetMaxCoresRequest request, StreamObserver responseObserver) { + updateLayer(request.getLayer()); + jobManager.setLayerMaxGpus(layer, request.getMaxGpus()); + responseObserver.onNext(LayerSetMaxGpusResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + public DependManager getDependManager() { return dependManager; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageRenderPartition.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageRenderPartition.java index 88e53ee95..413f1982c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageRenderPartition.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageRenderPartition.java @@ -45,7 +45,7 @@ public void delete(RenderPartDeleteRequest request, StreamObserver responseObserver) { LocalHostAssignment localJobAssign = getLocalHostAssignment(request.getRenderPartition()); - bookingManager.setMaxResources(localJobAssign, request.getCores(), request.getMemory(), request.getGpu()); + bookingManager.setMaxResources(localJobAssign, request.getCores(), request.getMemory(), request.getGpus(), request.getGpuMemory()); responseObserver.onNext(RenderPartSetMaxResourcesResponse.newBuilder().build()); responseObserver.onCompleted(); } diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageService.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageService.java index eae767006..70a15f3bf 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageService.java @@ -55,7 +55,9 @@ public void createService(ServiceCreateServiceRequest request, service.minCores = request.getData().getMinCores(); service.maxCores = request.getData().getMaxCores(); service.minMemory = request.getData().getMinMemory(); - service.minGpu = request.getData().getMinGpu(); + service.minGpus = request.getData().getMinGpus(); + service.maxGpus = request.getData().getMaxGpus(); + service.minGpuMemory = request.getData().getMinGpuMemory(); service.tags = Sets.newLinkedHashSet(request.getData().getTagsList()); service.threadable = request.getData().getThreadable(); service.timeout = request.getData().getTimeout(); @@ -129,7 +131,9 @@ private ServiceEntity toServiceEntity(Service service) { entity.minCores = service.getMinCores(); entity.maxCores = service.getMaxCores(); entity.minMemory = service.getMinMemory(); - entity.minGpu = service.getMinGpu(); + entity.minGpus = service.getMinGpus(); + entity.maxGpus = service.getMaxGpus(); + entity.minGpuMemory = service.getMinGpuMemory(); entity.tags = new LinkedHashSet<> (service.getTagsList()); entity.threadable = service.getThreadable(); entity.timeout = service.getTimeout(); diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageServiceOverride.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageServiceOverride.java index bd90575b5..ed3d46107 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageServiceOverride.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageServiceOverride.java @@ -67,7 +67,9 @@ private ServiceEntity toServiceEntity(Service service) { entity.minCores = service.getMinCores(); entity.maxCores = service.getMaxCores(); entity.minMemory = service.getMinMemory(); - entity.minGpu = service.getMinGpu(); + entity.minGpus = service.getMinGpus(); + entity.maxGpus = service.getMaxGpus(); + entity.minGpuMemory = service.getMinGpuMemory(); entity.tags = new LinkedHashSet<>(service.getTagsList()); entity.threadable = service.getThreadable(); entity.timeout = service.getTimeout(); diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageShow.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageShow.java index 6dd94a5ff..6e5fbcbe8 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageShow.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageShow.java @@ -93,6 +93,10 @@ import com.imageworks.spcue.grpc.show.ShowSetDefaultMaxCoresResponse; import com.imageworks.spcue.grpc.show.ShowSetDefaultMinCoresRequest; import com.imageworks.spcue.grpc.show.ShowSetDefaultMinCoresResponse; +import com.imageworks.spcue.grpc.show.ShowSetDefaultMaxGpusRequest; +import com.imageworks.spcue.grpc.show.ShowSetDefaultMaxGpusResponse; +import com.imageworks.spcue.grpc.show.ShowSetDefaultMinGpusRequest; +import com.imageworks.spcue.grpc.show.ShowSetDefaultMinGpusResponse; import com.imageworks.spcue.grpc.subscription.Subscription; import com.imageworks.spcue.grpc.subscription.SubscriptionSeq; import com.imageworks.spcue.service.AdminManager; @@ -257,6 +261,24 @@ public void setDefaultMinCores(ShowSetDefaultMinCoresRequest request, responseObserver.onCompleted(); } + @Override + public void setDefaultMaxGpus(ShowSetDefaultMaxGpusRequest request, + StreamObserver responseObserver) { + ShowEntity show = getShowEntity(request.getShow()); + showDao.updateShowDefaultMaxGpus(show, request.getMaxGpus()); + responseObserver.onNext(ShowSetDefaultMaxGpusResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + + @Override + public void setDefaultMinGpus(ShowSetDefaultMinGpusRequest request, + StreamObserver responseObserver) { + ShowEntity show = getShowEntity(request.getShow()); + showDao.updateShowDefaultMinGpus(show, request.getMinGpus()); + responseObserver.onNext(ShowSetDefaultMinGpusResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + @Override public void findFilter(ShowFindFilterRequest request, StreamObserver responseObserver) { @@ -361,7 +383,9 @@ public void createServiceOverride(ShowCreateServiceOverrideRequest request, service.minCores = requestService.getMinCores(); service.maxCores = requestService.getMaxCores(); service.minMemory = requestService.getMinMemory(); - service.minGpu = requestService.getMinGpu(); + service.minGpus = requestService.getMinGpus(); + service.maxGpus = requestService.getMaxGpus(); + service.minGpuMemory = requestService.getMinGpuMemory(); service.tags = Sets.newLinkedHashSet(requestService.getTagsList()); service.threadable = requestService.getThreadable(); serviceManager.createService(service); diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/BookingManager.java b/cuebot/src/main/java/com/imageworks/spcue/service/BookingManager.java index 6c348eeb4..b5cfb8455 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/BookingManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/BookingManager.java @@ -122,9 +122,10 @@ public void createLocalHostAssignment(DispatchHost host, * @param l * @param maxCoreUnits * @param maxMemory - * @param maxGpu + * @param maxGpuUnits + * @param maxGpuMemory */ - void setMaxResources(LocalHostAssignment l, int maxCoreUnits, long maxMemory, long maxGpu); + void setMaxResources(LocalHostAssignment l, int maxCoreUnits, long maxMemory, int maxGpuUnits, long maxGpuMemory); /** * Remove a LocalHostAssignment if there are no procs assigned to it. diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/BookingManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/BookingManagerService.java index 02b2fe948..1a2b6cee2 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/BookingManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/BookingManagerService.java @@ -72,7 +72,7 @@ public boolean hasActiveLocalFrames(HostInterface host) { @Override public void setMaxResources(LocalHostAssignment l, int maxCoreUnits, - long maxMemory, long maxGpu) { + long maxMemory, int maxGpuUnits, long maxGpuMemory) { HostInterface host = hostDao.getHost(l.getHostId()); @@ -84,8 +84,12 @@ public void setMaxResources(LocalHostAssignment l, int maxCoreUnits, bookingDao.updateMaxMemory(l, maxMemory); } - if (maxGpu > 0) { - bookingDao.updateMaxGpu(l, maxGpu); + if (maxGpuUnits > 0) { + bookingDao.updateMaxGpus(l, maxGpuUnits); + } + + if (maxGpuMemory > 0) { + bookingDao.updateMaxGpuMemory(l, maxGpuMemory); } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/GroupManager.java b/cuebot/src/main/java/com/imageworks/spcue/service/GroupManager.java index 9017304f8..2e3cf70be 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/GroupManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/GroupManager.java @@ -34,6 +34,10 @@ public interface GroupManager { void setGroupMinCores(GroupInterface g, int coreUnits); void setGroupDefaultJobMinCores(GroupInterface g, int coreUnits); void setGroupDefaultJobMaxCores(GroupInterface g, int coreUnits); + void setGroupMaxGpus(GroupInterface g, int gpuUnits); + void setGroupMinGpus(GroupInterface g, int gpuUnits); + void setGroupDefaultJobMinGpus(GroupInterface g, int gpuUnits); + void setGroupDefaultJobMaxGpus(GroupInterface g, int gpuUnits); void setGroupDefaultJobPriority(GroupInterface g, int priority); /** diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/GroupManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/GroupManagerService.java index 7e785c0ea..89fc25193 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/GroupManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/GroupManagerService.java @@ -78,6 +78,32 @@ public void setGroupMinCores(GroupInterface g, int coreUnits) { groupDao.updateMinCores(g,coreUnits); } + @Override + public void setGroupDefaultJobMaxGpus(GroupInterface g, int gpuUnits) { + groupDao.updateDefaultJobMaxGpus(g,gpuUnits); + if (gpuUnits != CueUtil.FEATURE_DISABLED && !groupDao.isManaged(g)) { + jobDao.updateMaxGpus(g, gpuUnits); + } + } + + @Override + public void setGroupDefaultJobMinGpus(GroupInterface g, int gpuUnits) { + groupDao.updateDefaultJobMinGpus(g,gpuUnits); + if (gpuUnits != CueUtil.FEATURE_DISABLED && !groupDao.isManaged(g)) { + jobDao.updateMinGpus(g, gpuUnits); + } + } + + @Override + public void setGroupMaxGpus(GroupInterface g, int gpuUnits) { + groupDao.updateMaxGpus(g, gpuUnits); + } + + @Override + public void setGroupMinGpus(GroupInterface g, int gpuUnits) { + groupDao.updateMinGpus(g, gpuUnits); + } + @Override public void setGroupParent(GroupInterface group, GroupInterface newParent) { groupDao.updateGroupParent(group, newParent); diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java b/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java index 19704e65e..8b176c77e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java @@ -112,8 +112,8 @@ public interface HostManager { * @param freeSwap * @param totalMcp * @param freeMcp - * @param totalGpu - * @param freeGpu + * @param totalGpuMemory + * @param freeGpuMemory * @param load * @param bootTime * @param os @@ -122,7 +122,7 @@ void setHostStatistics(HostInterface host, long totalMemory, long freeMemory, long totalSwap, long freeSwap, long totalMcp, long freeMcp, - long totalGpu, long freeGpu, + long totalGpuMemory, long freeGpuMemory, int load, Timestamp bootTime, String os); @@ -212,6 +212,11 @@ void setHostStatistics(HostInterface host, */ int getStrandedCoreUnits(HostInterface h); + /** + * Return the number of stranded cores on the host. + */ + int getStrandedGpuUnits(HostInterface h); + /** * Return true of the host prefers a particular show. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java index 9bbaaa6e4..ccd355889 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java @@ -124,7 +124,7 @@ public void setHostStatistics(HostInterface host, long totalMemory, long freeMemory, long totalSwap, long freeSwap, long totalMcp, long freeMcp, - long totalGpu, long freeGpu, + long totalGpuMemory, long freeGpuMemory, int load, Timestamp bootTime, String os) { @@ -132,7 +132,7 @@ public void setHostStatistics(HostInterface host, totalMemory, freeMemory, totalSwap, freeSwap, totalMcp, freeMcp, - totalGpu, freeGpu, + totalGpuMemory, freeGpuMemory, load, bootTime, os); } @@ -268,6 +268,12 @@ public int getStrandedCoreUnits(HostInterface h) { return hostDao.getStrandedCoreUnits(h); } + @Override + @Transactional(propagation = Propagation.REQUIRED, readOnly=true) + public int getStrandedGpuUnits(HostInterface h) { + return hostDao.getStrandedGpus(h); + } + @Override @Transactional(propagation = Propagation.REQUIRED, readOnly=true) public boolean verifyRunningProc(String procId, String frameId) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobLauncher.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobLauncher.java index e84bebb16..14f2a5741 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobLauncher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobLauncher.java @@ -109,7 +109,8 @@ public void launch(final JobSpec spec) { lha.setThreads(d.localThreadNumber); lha.setMaxCoreUnits(d.localMaxCores * 100); lha.setMaxMemory(d.localMaxMemory); - lha.setMaxGpu(d.localMaxGpu); + lha.setMaxGpuUnits(d.localMaxGpus); + lha.setMaxGpuMemory(d.localMaxGpuMemory); lha.setType(RenderPartitionType.JOB_PARTITION); try { diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobManager.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobManager.java index 4ab1b2120..6ab4bb38e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobManager.java @@ -292,6 +292,14 @@ public interface JobManager { */ boolean isOverMinCores(JobInterface job); + /** + * Return true if the given job is booked greater than min gpus. + * + * @param job + * @return + */ + boolean isOverMinGpus(JobInterface job); + /** * Increase the layer memory requirement to given KB value. * @@ -453,6 +461,22 @@ public interface JobManager { */ void setLayerMinCores(LayerInterface layer, int coreUnits); + /** + * Update the max gpu value for the given layer. + * + * @param layer + * @param gpuUnits + */ + void setLayerMaxGpus(LayerInterface layer, int gpuUnits); + + /** + * Update the min gpu value for the given layer. + * + * @param layer + * @param gpuUnits + */ + void setLayerMinGpus(LayerInterface layer, int gpuUnits); + /** * Add a limit to the given layer. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java index 68821ed64..a4f6f1ebb 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java @@ -135,6 +135,12 @@ public boolean isOverMinCores(JobInterface job) { return jobDao.isOverMinCores(job); } + @Override + @Transactional(propagation = Propagation.REQUIRED, readOnly=true) + public boolean isOverMinGpus(JobInterface job) { + return jobDao.isOverMinGpus(job); + } + @Transactional(propagation = Propagation.REQUIRED, readOnly=true) public DispatchJob getDispatchJob(String id) { return jobDao.getDispatchJob(id); @@ -450,6 +456,16 @@ public void setLayerMaxCores(LayerInterface layer, int coreUnits) { layerDao.updateLayerMaxCores(layer, coreUnits); } + @Override + public void setLayerMinGpus(LayerInterface layer, int gpu) { + layerDao.updateLayerMinGpus(layer, gpu); + } + + @Override + public void setLayerMaxGpus(LayerInterface layer, int gpu) { + layerDao.updateLayerMaxGpus(layer, gpu); + } + @Override public void addLayerLimit(LayerInterface layer, String limitId) { layerDao.addLimit(layer, limitId); diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java index 1e498eaf4..30bf7cfd3 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java @@ -286,11 +286,17 @@ private BuildableJob handleJobTag(Element jobTag) { if (local.getAttributeValue("cores") != null) job.localMaxCores = Integer.parseInt(local.getAttributeValue("cores")); if (local.getAttributeValue("memory") != null) - job.localMaxMemory = Integer.parseInt(local.getAttributeValue("memory")); + job.localMaxMemory = Long.parseLong(local.getAttributeValue("memory")); if (local.getAttributeValue("threads") != null) job.localThreadNumber = Integer.parseInt(local.getAttributeValue("threads")); - if (local.getAttributeValue("gpu") != null) - job.localMaxGpu = Integer.parseInt(local.getAttributeValue("gpu")); + if (local.getAttributeValue("gpus") != null) + job.localMaxGpus = Integer.parseInt(local.getAttributeValue("gpus")); + if (local.getAttributeValue("gpu") != null) { + logger.warn(job.name + " localbook has the deprecated gpu. Use gpu_memory."); + job.localMaxGpuMemory = Long.parseLong(local.getAttributeValue("gpu")); + } + if (local.getAttributeValue("gpu_memory") != null) + job.localMaxGpuMemory = Long.parseLong(local.getAttributeValue("gpu_memory")); } job.maxCoreUnits = 20000; @@ -423,11 +429,12 @@ private void handleLayerTags(BuildableJob buildableJob, Element jobTag) { determineResourceDefaults(layerTag, buildableJob, layer); determineChunkSize(layerTag, layer); determineMinimumCores(layerTag, layer); + determineMinimumGpus(layerTag, layer); determineThreadable(layerTag, layer); determineTags(buildableJob, layer, layerTag); determineMinimumMemory(buildableJob, layerTag, layer, buildableLayer); - determineMinimumGpu(buildableJob, layerTag, layer); + determineMinimumGpuMemory(buildableJob, layerTag, layer); // set a timeout value on the layer if (layerTag.getChildTextTrim("timeout") != null) { @@ -521,44 +528,53 @@ else if (minMemory < Dispatcher.MEM_RESERVED_MIN) { } /** - * If the gpu option is set, set minimumGpu to that supplied value + * If the gpu_memory option is set, set minimumGpuMemory to that supplied value * * @param layerTag * @param layer */ - private void determineMinimumGpu(BuildableJob buildableJob, Element layerTag, + private void determineMinimumGpuMemory(BuildableJob buildableJob, Element layerTag, LayerDetail layer) { - if (layerTag.getChildTextTrim("gpu") == null) { + String gpu = layerTag.getChildTextTrim("gpu"); + String gpuMemory = layerTag.getChildTextTrim("gpu_memory"); + if (gpu == null && gpuMemory == null) { return; } - long minGpu; - String memory = layerTag.getChildTextTrim("gpu").toLowerCase(); + String memory = null; + if (gpu != null) { + logger.warn(buildableJob.detail.name + "/" + layer.name + + " has the deprecated gpu. Use gpu_memory."); + memory = gpu.toLowerCase(); + } + if (gpuMemory != null) + memory = gpuMemory.toLowerCase(); + long minGpuMemory; try { - minGpu = convertMemoryInput(memory); + minGpuMemory = convertMemoryInput(memory); // Some quick sanity checks to make sure gpu memory hasn't gone // over or under reasonable defaults. - if (minGpu> Dispatcher.GPU_RESERVED_MAX) { + if (minGpuMemory > Dispatcher.MEM_GPU_RESERVED_MAX) { throw new SpecBuilderException("Gpu memory requirements exceed " + "maximum. Are you specifying the correct units?"); } - else if (minGpu < Dispatcher.GPU_RESERVED_MIN) { + else if (minGpuMemory < Dispatcher.MEM_GPU_RESERVED_MIN) { logger.warn(buildableJob.detail.name + "/" + layer.name + "Specified too little gpu memory, defaulting to: " + - Dispatcher.GPU_RESERVED_MIN); - minGpu = Dispatcher.GPU_RESERVED_MIN; + Dispatcher.MEM_GPU_RESERVED_MIN); + minGpuMemory = Dispatcher.MEM_GPU_RESERVED_MIN; } - layer.minimumGpu = minGpu; + layer.minimumGpuMemory = minGpuMemory; } catch (Exception e) { logger.info("Error setting gpu memory for " + buildableJob.detail.name + "/" + layer.name + " failed, reason: " + e + ". Using default."); - layer.minimumGpu = Dispatcher.GPU_RESERVED_DEFAULT; + layer.minimumGpuMemory = Dispatcher.MEM_GPU_RESERVED_DEFAULT; } } @@ -598,6 +614,20 @@ private void determineMinimumCores(Element layerTag, LayerDetail layer) { layer.minimumCores = corePoints; } + /** + * Gpu is a int. + * + * If no gpu value is specified, we default to the value of + * Dispatcher.GPU_RESERVED_DEFAULT + */ + private void determineMinimumGpus(Element layerTag, LayerDetail layer) { + + String gpus = layerTag.getChildTextTrim("gpus"); + if (gpus != null) { + layer.minimumGpus = Integer.valueOf(gpus); + } + } + private void determineChunkSize(Element layerTag, LayerDetail layer) { layer.chunkSize = Integer.parseInt(layerTag.getChildTextTrim("chunk")); } @@ -702,7 +732,9 @@ private void determineResourceDefaults(Element layerTag, layer.maximumCores = primaryService.maxCores; layer.minimumCores = primaryService.minCores; layer.minimumMemory = primaryService.minMemory; - layer.minimumGpu = primaryService.minGpu; + layer.maximumGpus = primaryService.maxGpus; + layer.minimumGpus = primaryService.minGpus; + layer.minimumGpuMemory = primaryService.minGpuMemory; layer.tags.addAll(primaryService.tags); layer.services.addAll(services); layer.limits.addAll(limits); diff --git a/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java b/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java index a7d89e7ee..f8b028cfa 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java +++ b/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java @@ -207,11 +207,11 @@ public static final String KbToMb(long kb) { return String.format("%dMB", kb / 1024); } - public static final long convertKbToFakeKb64bit(int Kb) { + public static final long convertKbToFakeKb64bit(long Kb) { return (long) (Math.ceil((Kb * 0.0009765625) * 0.0009765625) * 1048576) - Dispatcher.MEM_RESERVED_SYSTEM; } - public static final long convertKbToFakeKb32bit(int Kb) { + public static final long convertKbToFakeKb32bit(long Kb) { return (long) (Math.floor((Kb * 0.0009765625) * 0.0009765625) * 1048576) - Dispatcher.MEM_RESERVED_SYSTEM; } @@ -235,10 +235,10 @@ public final static String buildFrameName(LayerInterface layer, int num) { return String.format("%04d-%s", num, layer.getName()); } - public final static String buildProcName(String host, int cores) { - return String.format(Locale.ROOT, "%s/%4.2f", host, Convert.coreUnitsToCores(cores)); - + public final static String buildProcName(String host, int cores, int gpus) { + return String.format(Locale.ROOT, "%s/%4.2f/%d", host, Convert.coreUnitsToCores(cores), gpus); } + /** * for logging how long an operation took * diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V11__Support_multiple_GPU.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V11__Support_multiple_GPU.sql new file mode 100644 index 000000000..acb292586 --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V11__Support_multiple_GPU.sql @@ -0,0 +1,1078 @@ +-- Support multiple GPU + +-- frame_history + +ALTER TABLE frame_history ADD COLUMN int_gpus INT DEFAULT 0 NOT NULL; +ALTER TABLE frame_history ADD COLUMN int_gpu_mem_reserved BIGINT DEFAULT 0 NOT NULL; +ALTER TABLE frame_history ADD COLUMN int_gpu_mem_max_used BIGINT DEFAULT 0 NOT NULL; + + +-- show_service + +ALTER TABLE show_service RENAME COLUMN int_gpu_min TO int_gpu_mem_min; +ALTER TABLE show_service ALTER COLUMN int_gpu_mem_min TYPE BIGINT; +ALTER TABLE show_service ADD COLUMN int_gpus_min INT DEFAULT 0 NOT NULL; +ALTER TABLE show_service ADD COLUMN int_gpus_max INT DEFAULT 0 NOT NULL; + +ALTER INDEX i_show_service_int_gpu_min RENAME TO i_show_service_int_gpu_mem_min;; +CREATE INDEX i_show_service_int_gpus_min ON show_service (int_gpus_min); + + +-- host_local + +DROP TRIGGER verify_host_local ON host_local; +ALTER TABLE host_local ALTER COLUMN int_mem_max TYPE BIGINT; +ALTER TABLE host_local ALTER COLUMN int_mem_idle TYPE BIGINT; +ALTER TABLE host_local RENAME COLUMN int_gpu_idle TO int_gpu_mem_idle; +ALTER TABLE host_local ALTER COLUMN int_gpu_mem_idle TYPE BIGINT; +ALTER TABLE host_local RENAME COLUMN int_gpu_max TO int_gpu_mem_max; +ALTER TABLE host_local ALTER COLUMN int_gpu_mem_max TYPE BIGINT; +ALTER TABLE host_local ADD COLUMN int_gpus_idle INT DEFAULT 0 NOT NULL; +ALTER TABLE host_local ADD COLUMN int_gpus_max INT DEFAULT 0 NOT NULL; + +CREATE INDEX i_host_local_int_gpus_idle ON host_local (int_gpus_idle); +CREATE INDEX i_host_local_int_gpus_max ON host_local (int_gpus_max); + + +-- service + +ALTER TABLE service RENAME COLUMN int_gpu_min TO int_gpu_mem_min; +ALTER TABLE service ALTER COLUMN int_gpu_mem_min TYPE BIGINT; +ALTER TABLE service ADD COLUMN int_gpus_min INT DEFAULT 0 NOT NULL; +ALTER TABLE service ADD COLUMN int_gpus_max INT DEFAULT 0 NOT NULL; + +ALTER INDEX i_service_int_gpu_min RENAME TO i_service_int_gpu_mem_min; +CREATE INDEX i_service_int_gpus_min ON service (int_gpus_min); + + +-- job_local + +ALTER TABLE job_local ADD COLUMN int_gpus INT DEFAULT 0 NOT NULL; +ALTER TABLE job_local ADD COLUMN int_max_gpus INT DEFAULT 0 NOT NULL; + + +-- task + +ALTER TABLE task ADD COLUMN int_min_gpus INT DEFAULT 0 NOT NULL; +ALTER TABLE task ADD COLUMN int_adjust_gpus INT DEFAULT 0 NOT NULL; + + +-- point + +ALTER TABLE point ADD COLUMN int_gpus INT DEFAULT 0 NOT NULL; +ALTER TABLE point ADD COLUMN int_min_gpus INT DEFAULT 0 NOT NULL; + + +-- folder_resource + +ALTER TABLE folder_resource ADD COLUMN int_gpus INT DEFAULT 0 NOT NULL; +ALTER TABLE folder_resource ADD COLUMN int_max_gpus INT DEFAULT -1 NOT NULL; +ALTER TABLE folder_resource ADD COLUMN int_min_gpus INT DEFAULT 0 NOT NULL; + +CREATE INDEX i_folder_res_int_max_gpus ON folder_resource (int_max_gpus); + + +-- layer_history + +ALTER TABLE layer_history ADD COLUMN int_gpus_min INT DEFAULT 0 NOT NULL; +ALTER TABLE layer_history ADD COLUMN int_gpu_time_success BIGINT DEFAULT 0 NOT NULL; +ALTER TABLE layer_history ADD COLUMN int_gpu_time_fail BIGINT DEFAULT 0 NOT NULL; +ALTER TABLE layer_history ADD COLUMN int_gpu_mem_min BIGINT DEFAULT 0 NOT NULL; +ALTER TABLE layer_history ADD COLUMN int_gpu_mem_max BIGINT DEFAULT 0 NOT NULL; + + +-- job_history + +ALTER TABLE job_history ADD COLUMN int_gpu_time_success BIGINT DEFAULT 0 NOT NULL; +ALTER TABLE job_history ADD COLUMN int_gpu_time_fail BIGINT DEFAULT 0 NOT NULL; +ALTER TABLE job_history ADD COLUMN int_gpu_mem_max BIGINT DEFAULT 0 NOT NULL; + + +-- job_usage + +ALTER TABLE job_usage ADD COLUMN int_gpu_time_success BIGINT DEFAULT 0 NOT NULL; +ALTER TABLE job_usage ADD COLUMN int_gpu_time_fail BIGINT DEFAULT 0 NOT NULL; + + +-- job_resource + +ALTER TABLE job_resource ALTER COLUMN int_max_rss TYPE BIGINT; +ALTER TABLE job_resource ALTER COLUMN int_max_vss TYPE BIGINT; +ALTER TABLE job_resource ADD COLUMN int_gpus INT DEFAULT 0 NOT NULL; +ALTER TABLE job_resource ADD COLUMN int_min_gpus INT DEFAULT 0 NOT NULL; +ALTER TABLE job_resource ADD COLUMN int_max_gpus INT DEFAULT 100 NOT NULL; +ALTER TABLE job_resource ADD COLUMN int_local_gpus INT DEFAULT 0 NOT NULL; +ALTER TABLE job_resource ADD COLUMN int_gpu_mem_max BIGINT DEFAULT 0 NOT NULL; + +CREATE INDEX i_job_resource_gpus_min_max ON job_resource (int_min_gpus, int_max_gpus); +CREATE INDEX i_job_resource_gpus ON job_resource (int_gpus); +CREATE INDEX i_job_resource_max_gpus ON job_resource (int_max_gpus); + + +-- subscription + +ALTER TABLE subscription ADD COLUMN int_gpus INT DEFAULT 0 NOT NULL; + + +-- show + +ALTER TABLE show ADD COLUMN int_default_min_gpus INT DEFAULT 100 NOT NULL; +ALTER TABLE show ADD COLUMN int_default_max_gpus INT DEFAULT 100000 NOT NULL; + + +-- proc + +ALTER TABLE proc RENAME COLUMN int_gpu_reserved TO int_gpu_mem_reserved; +ALTER TABLE proc ALTER COLUMN int_gpu_mem_reserved TYPE BIGINT; +ALTER TABLE proc ADD COLUMN int_gpus_reserved INT DEFAULT 0 NOT NULL; +ALTER TABLE proc ADD COLUMN int_gpu_mem_used BIGINT DEFAULT 0 NOT NULL; +ALTER TABLE proc ADD COLUMN int_gpu_mem_max_used BIGINT DEFAULT 0 NOT NULL; +ALTER TABLE proc ADD COLUMN int_gpu_mem_pre_reserved BIGINT DEFAULT 0 NOT NULL; + +ALTER INDEX i_proc_int_gpu_reserved RENAME TO i_proc_int_gpu_mem_reserved; + + +-- layer_usage + +ALTER TABLE layer_usage ADD COLUMN int_gpu_time_success BIGINT DEFAULT 0 NOT NULL; +ALTER TABLE layer_usage ADD COLUMN int_gpu_time_fail BIGINT DEFAULT 0 NOT NULL; + + +-- layer_mem + +ALTER TABLE layer_mem ALTER COLUMN int_max_rss TYPE BIGINT; +ALTER TABLE layer_mem ALTER COLUMN int_max_vss TYPE BIGINT; +ALTER TABLE layer_mem ADD COLUMN int_gpu_mem_max BIGINT DEFAULT 0 NOT NULL; + + +-- layer_resource + +ALTER TABLE layer_resource ALTER COLUMN int_max_rss TYPE BIGINT; +ALTER TABLE layer_resource ALTER COLUMN int_max_vss TYPE BIGINT; +ALTER TABLE layer_resource ADD COLUMN int_gpus INT DEFAULT 0 NOT NULL; +ALTER TABLE layer_resource ADD COLUMN int_gpu_mem_max BIGINT DEFAULT 0 NOT NULL; + + +-- layer + +ALTER TABLE layer RENAME COLUMN int_gpu_min TO int_gpu_mem_min; +ALTER TABLE layer ALTER COLUMN int_gpu_mem_min TYPE BIGINT; +ALTER TABLE layer ADD COLUMN int_gpus_min BIGINT DEFAULT 0 NOT NULL; +ALTER TABLE layer ADD COLUMN int_gpus_max BIGINT DEFAULT 0 NOT NULL; + +ALTER INDEX i_layer_int_gpu_min RENAME TO i_layer_int_gpu_mem_min; +CREATE INDEX i_layer_cores_gpus_mem ON layer (int_cores_min, int_gpus_min, int_mem_min, int_gpu_mem_min); +CREATE INDEX i_layer_cores_gpus_mem_thread ON layer (int_cores_min, int_gpus_min, int_mem_min, int_gpu_mem_min, b_threadable); + + +-- job_mem + +ALTER TABLE job_mem ALTER COLUMN int_max_rss TYPE BIGINT; +ALTER TABLE job_mem ALTER COLUMN int_max_vss TYPE BIGINT; +ALTER TABLE job_mem ADD COLUMN int_gpu_mem_max BIGINT DEFAULT 0 NOT NULL; + + +-- job + +ALTER TABLE job ADD COLUMN int_min_gpus INT DEFAULT 0 NOT NULL; +ALTER TABLE job ADD COLUMN int_max_gpus INT DEFAULT 100000 NOT NULL; + + +-- host_stat + +ALTER TABLE host_stat RENAME COLUMN int_gpu_total TO int_gpu_mem_total; +ALTER TABLE host_stat ALTER COLUMN int_gpu_mem_total TYPE BIGINT; +ALTER TABLE host_stat RENAME COLUMN int_gpu_free TO int_gpu_mem_free; +ALTER TABLE host_stat ALTER COLUMN int_gpu_mem_free TYPE BIGINT; + +ALTER INDEX i_host_stat_int_gpu_total RENAME TO i_host_stat_int_gpu_mem_total; +ALTER INDEX i_host_stat_int_gpu_free RENAME TO i_host_stat_int_gpu_mem_free; + + +-- host + +ALTER TABLE host RENAME COLUMN int_gpu TO int_gpu_mem; +ALTER TABLE host ALTER COLUMN int_gpu_mem TYPE BIGINT; +ALTER TABLE host RENAME COLUMN int_gpu_idle TO int_gpu_mem_idle; +ALTER TABLE host ALTER COLUMN int_gpu_mem_idle TYPE BIGINT; +ALTER TABLE host ADD COLUMN int_gpus BIGINT DEFAULT 0 NOT NULL; +ALTER TABLE host ADD COLUMN int_gpus_idle BIGINT DEFAULT 0 NOT NULL; + +CREATE INDEX i_host_int_gpu_mem ON host (int_gpu_mem); +CREATE INDEX i_host_int_gpu_mem_idle ON host (int_gpu_mem_idle); +CREATE INDEX i_host_int_gpus ON host (int_gpus); +CREATE INDEX i_host_int_gpus_idle ON host (int_gpus_idle); + + +-- frame + +ALTER TABLE frame RENAME COLUMN int_gpu_reserved TO int_gpu_mem_reserved; +ALTER TABLE frame ALTER COLUMN int_gpu_mem_reserved TYPE BIGINT; +ALTER TABLE frame ADD COLUMN int_gpu_mem_used BIGINT DEFAULT 0 NOT NULL; +ALTER TABLE frame ADD COLUMN int_gpu_mem_max_used BIGINT DEFAULT 0 NOT NULL; +ALTER TABLE frame ADD COLUMN int_gpus INT DEFAULT 0 NOT NULL; +ALTER TABLE frame ADD COLUMN int_total_past_gpu_time INT DEFAULT 0 NOT NULL; + +ALTER INDEX i_frame_int_gpu_reserved RENAME TO i_frame_int_gpu_mem_reserved; + + +-- folder + +ALTER TABLE folder ADD COLUMN int_job_min_gpus INT DEFAULT -1 NOT NULL; +ALTER TABLE folder ADD COLUMN int_job_max_gpus INT DEFAULT -1 NOT NULL; +ALTER TABLE folder ADD COLUMN int_min_gpus INT DEFAULT 0 NOT NULL; +ALTER TABLE folder ADD COLUMN int_max_gpus INT DEFAULT -1 NOT NULL; + + +-- Views + +DROP VIEW vs_show_resource; +CREATE VIEW vs_show_resource (pk_show, int_cores, int_gpus) AS + SELECT + job.pk_show, + SUM(int_cores) AS int_cores, SUM(int_gpus) AS int_gpus + FROM + job, + job_resource + WHERE + job.pk_job = job_resource.pk_job + AND + job.str_state='PENDING' + GROUP BY + job.pk_show; + + +DROP VIEW vs_job_resource; +CREATE VIEW vs_job_resource (pk_job, int_procs, int_cores, int_gpus, int_mem_reserved) AS + SELECT + job.pk_job, + COUNT(proc.pk_proc) AS int_procs, + COALESCE(SUM(int_cores_reserved),0) AS int_cores, + COALESCE(SUM(int_gpus_reserved),0) AS int_gpus, + COALESCE(SUM(int_mem_reserved),0) AS int_mem_reserved + FROM + job LEFT JOIN proc ON (proc.pk_job = job.pk_job) + GROUP BY + job.pk_job; + + +DROP VIEW vs_alloc_usage; +CREATE VIEW vs_alloc_usage (pk_alloc, int_cores, int_idle_cores, int_running_cores, int_locked_cores, int_available_cores, int_gpus, int_idle_gpus, int_running_gpus, int_locked_gpus, int_available_gpus, int_hosts, int_locked_hosts, int_down_hosts) AS + SELECT + alloc.pk_alloc, + COALESCE(SUM(host.int_cores),0) AS int_cores, + COALESCE(SUM(host.int_cores_idle),0) AS int_idle_cores, + COALESCE(SUM(host.int_cores - host.int_cores_idle),0) as int_running_cores, + COALESCE((SELECT SUM(int_cores) FROM host WHERE host.pk_alloc=alloc.pk_alloc AND (str_lock_state='NIMBY_LOCKED' OR str_lock_state='LOCKED')),0) AS int_locked_cores, + COALESCE((SELECT SUM(int_cores_idle) FROM host h,host_stat hs WHERE h.pk_host = hs.pk_host AND h.pk_alloc=alloc.pk_alloc AND h.str_lock_state='OPEN' AND hs.str_state ='UP'),0) AS int_available_cores, + COALESCE(SUM(host.int_gpus),0) AS int_gpus, + COALESCE(SUM(host.int_gpus_idle),0) AS int_idle_gpus, + COALESCE(SUM(host.int_gpus - host.int_gpus_idle),0) as int_running_gpus, + COALESCE((SELECT SUM(int_gpus) FROM host WHERE host.pk_alloc=alloc.pk_alloc AND (str_lock_state='NIMBY_LOCKED' OR str_lock_state='LOCKED')),0) AS int_locked_gpus, + COALESCE((SELECT SUM(int_gpus_idle) FROM host h,host_stat hs WHERE h.pk_host = hs.pk_host AND h.pk_alloc=alloc.pk_alloc AND h.str_lock_state='OPEN' AND hs.str_state ='UP'),0) AS int_available_gpus, + COUNT(host.pk_host) AS int_hosts, + (SELECT COUNT(*) FROM host WHERE host.pk_alloc=alloc.pk_alloc AND str_lock_state='LOCKED') AS int_locked_hosts, + (SELECT COUNT(*) FROM host h,host_stat hs WHERE h.pk_host = hs.pk_host AND h.pk_alloc=alloc.pk_alloc AND hs.str_state='DOWN') AS int_down_hosts + FROM + alloc LEFT JOIN host ON (alloc.pk_alloc = host.pk_alloc) + GROUP BY + alloc.pk_alloc; + + +DROP VIEW vs_folder_counts; +CREATE VIEW vs_folder_counts (pk_folder, int_depend_count, int_waiting_count, int_running_count, int_dead_count, int_cores, int_gpus, int_job_count) AS + SELECT + folder.pk_folder, + COALESCE(SUM(int_depend_count),0) AS int_depend_count, + COALESCE(SUM(int_waiting_count),0) AS int_waiting_count, + COALESCE(SUM(int_running_count),0) AS int_running_count, + COALESCE(SUM(int_dead_count),0) AS int_dead_count, + COALESCE(SUM(int_cores),0) AS int_cores, + COALESCE(SUM(int_gpus),0) AS int_gpus, + COALESCE(COUNT(job.pk_job),0) AS int_job_count +FROM + folder + LEFT JOIN + job ON (folder.pk_folder = job.pk_folder AND job.str_state='PENDING') + LEFT JOIN + job_stat ON (job.pk_job = job_stat.pk_job) + LEFT JOIN + job_resource ON (job.pk_job = job_resource.pk_job) + GROUP BY + folder.pk_folder; + + +DROP VIEW v_history_frame; +CREATE VIEW v_history_frame (pk_frame_history, pk_frame, pk_layer, pk_job, str_name, str_state, + int_mem_reserved, int_mem_max_used, int_cores, int_gpu_mem_reserved, int_gpu_mem_max_used, int_gpus, + str_host, int_exit_status, str_alloc_name, + b_alloc_billable, str_facility_name, int_ts_started, int_ts_stopped, int_checkpoint_count, + str_show_name, dt_last_modified) AS + SELECT + fh.PK_FRAME_HISTORY, + fh.PK_FRAME, + fh.PK_LAYER, + fh.PK_JOB, + fh.STR_NAME, + fh.STR_STATE, + fh.INT_MEM_RESERVED, + fh.INT_MEM_MAX_USED, + fh.INT_CORES, + fh.INT_GPU_MEM_RESERVED, + fh.INT_GPU_MEM_MAX_USED, + fh.INT_GPUS, + fh.STR_HOST, + fh.INT_EXIT_STATUS, + a.STR_NAME STR_ALLOC_NAME, + a.B_BILLABLE B_ALLOC_BILLABLE, + f.STR_NAME STR_FACILITY_NAME, + fh.INT_TS_STARTED, + fh.INT_TS_STOPPED, + fh.INT_CHECKPOINT_COUNT, + null str_show_name, + fh.dt_last_modified + FROM frame_history fh + JOIN job_history jh + ON fh.pk_job = jh.pk_job + LEFT OUTER JOIN alloc a + ON fh.pk_alloc = a.pk_alloc + LEFT OUTER JOIN facility f + ON a.pk_facility = f.pk_facility + WHERE fh.dt_last_modified >= (SELECT dt_begin FROM history_period) + AND fh.dt_last_modified < (SELECT dt_end FROM history_period); + + +DROP VIEW v_history_job; +CREATE VIEW v_history_job (pk_job, str_name, str_shot, str_user, int_core_time_success, int_core_time_fail, int_gpu_time_success, int_gpu_time_fail, int_frame_count, int_layer_count, int_waiting_count, int_dead_count, int_depend_count, int_eaten_count, int_succeeded_count, int_running_count, int_max_rss, int_gpu_mem_max, b_archived, str_facility_name, str_dept_name, int_ts_started, int_ts_stopped, str_show_name, dt_last_modified) AS + select +jh.PK_JOB, +jh.STR_NAME, +jh.STR_SHOT, +jh.STR_USER, +jh.INT_CORE_TIME_SUCCESS, +jh.INT_CORE_TIME_FAIL, +jh.INT_GPU_TIME_SUCCESS, +jh.INT_GPU_TIME_FAIL, +jh.INT_FRAME_COUNT, +jh.INT_LAYER_COUNT, +jh.INT_WAITING_COUNT, +jh.INT_DEAD_COUNT, +jh.INT_DEPEND_COUNT, +jh.INT_EATEN_COUNT, +jh.INT_SUCCEEDED_COUNT, +jh.INT_RUNNING_COUNT, +jh.INT_MAX_RSS, +jh.INT_GPU_MEM_MAX, +jh.B_ARCHIVED, +f.str_name STR_FACILITY_NAME, +d.str_name str_dept_name, +jh.INT_TS_STARTED, +jh.INT_TS_STOPPED, +s.str_name str_show_name, +jh.dt_last_modified +from job_history jh, show s, facility f, dept d +where jh.pk_show = s.pk_show +and jh.pk_facility = f.pk_facility +and jh.pk_dept = d.pk_dept +and ( + jh.dt_last_modified >= ( + select dt_begin + from history_period + ) + or + jh.int_ts_stopped = 0 +); + + +DROP VIEW v_history_layer; +CREATE VIEW v_history_layer (pk_layer, pk_job, str_name, str_type, int_cores_min, + int_mem_min, int_gpus_min, int_gpu_mem_min, int_core_time_success, int_core_time_fail, + int_gpu_time_success, int_gpu_time_fail, int_frame_count, int_layer_count, + int_waiting_count, int_dead_count, int_depend_count, int_eaten_count, int_succeeded_count, + int_running_count, int_max_rss, int_gpu_mem_max, b_archived, str_services, str_show_name, dt_last_modified) AS + SELECT +lh.PK_LAYER, +lh.PK_JOB, +lh.STR_NAME, +lh.STR_TYPE, +lh.INT_CORES_MIN, +lh.INT_MEM_MIN, +lh.INT_GPUS_MIN, +lh.INT_GPU_MEM_MIN, +lh.INT_CORE_TIME_SUCCESS, +lh.INT_CORE_TIME_FAIL, +lh.INT_GPU_TIME_SUCCESS, +lh.INT_GPU_TIME_FAIL, +lh.INT_FRAME_COUNT, +lh.INT_LAYER_COUNT, +lh.INT_WAITING_COUNT, +lh.INT_DEAD_COUNT, +lh.INT_DEPEND_COUNT, +lh.INT_EATEN_COUNT, +lh.INT_SUCCEEDED_COUNT, +lh.INT_RUNNING_COUNT, +lh.INT_MAX_RSS, +lh.INT_GPU_MEM_MAX, +lh.B_ARCHIVED, +lh.STR_SERVICES, +s.str_name str_show_name, +lh.dt_last_modified +from layer_history lh, job_history jh, show s +where lh.pk_job = jh.pk_job +and jh.pk_show = s.pk_show +and jh.dt_last_modified >= ( + select dt_begin + from history_period +) +and jh.dt_last_modified < ( + select dt_end + from history_period +); + + +-- Types + +ALTER TYPE JobStatType ADD ATTRIBUTE int_gpu_time_success BIGINT; +ALTER TYPE JobStatType ADD ATTRIBUTE int_gpu_time_fail BIGINT; +ALTER TYPE JobStatType ADD ATTRIBUTE int_gpu_mem_max BIGINT; + +ALTER TYPE LayerStatType ADD ATTRIBUTE int_gpu_time_success BIGINT; +ALTER TYPE LayerStatType ADD ATTRIBUTE int_gpu_time_fail BIGINT; +ALTER TYPE LayerStatType ADD ATTRIBUTE int_gpu_mem_max BIGINT; + + +-- Functions + +CREATE OR REPLACE FUNCTION recalculate_subs() +RETURNS VOID AS $body$ +DECLARE + r RECORD; +BEGIN + -- + -- concatenates all tags in host_tag and sets host.str_tags + -- + UPDATE subscription SET int_cores = 0; + UPDATE subscription SET int_gpus = 0; + FOR r IN + SELECT proc.pk_show, alloc.pk_alloc, sum(proc.int_cores_reserved) as c, sum(proc.int_gpus_reserved) as d + FROM proc, host, alloc + WHERE proc.pk_host = host.pk_host AND host.pk_alloc = alloc.pk_alloc + GROUP BY proc.pk_show, alloc.pk_alloc + LOOP + UPDATE subscription SET int_cores = r.c, int_gpus = r.d WHERE pk_alloc=r.pk_alloc AND pk_show=r.pk_show; + + END LOOP; +END; +$body$ +LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION tmp_populate_folder() +RETURNS VOID AS $body$ +DECLARE + t RECORD; +BEGIN + FOR t IN + SELECT pk_folder, pk_show, sum(int_cores) AS c, sum(int_gpus) AS d + FROM job, job_resource + WHERE job.pk_job = job_resource.pk_job + GROUP by pk_folder, pk_show + LOOP + UPDATE folder_resource SET int_cores = t.c, int_gpus = t.d WHERE pk_folder = t.pk_folder; + COMMIT; + END LOOP; +END; +$body$ +LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION tmp_populate_point() +RETURNS VOID AS $body$ +DECLARE + t RECORD; +BEGIN + FOR t IN + SELECT pk_dept, pk_show, sum(int_cores) AS c, sum(int_gpus) AS d + FROM job, job_resource + WHERE job.pk_job = job_resource.pk_job + GROUP BY pk_dept, pk_show + LOOP + UPDATE point SET int_cores = t.c , int_gpus = t.d WHERE pk_show = t.pk_show AND pk_dept = t.pk_dept; + END LOOP; +END; +$body$ +LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION tmp_populate_sub() +RETURNS VOID AS $body$ +DECLARE + t RECORD; +BEGIN + FOR t IN + SELECT proc.pk_show, host.pk_alloc, sum(int_cores_reserved) AS c, sum(int_gpus_reserved) AS d + FROM proc, host + WHERE proc.pk_host = host.pk_host + GROUP BY proc.pk_show, host.pk_alloc + LOOP + UPDATE subscription SET int_cores = t.c, int_gpus = t.d WHERE pk_show = t.pk_show AND pk_alloc = t.pk_alloc; + END LOOP; +END; +$body$ +LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION trigger__after_job_moved() +RETURNS TRIGGER AS $body$ +DECLARE + int_core_count INT; + int_gpu_count INT; +BEGIN + SELECT int_cores, int_gpus INTO int_core_count, int_gpu_count + FROM job_resource WHERE pk_job = NEW.pk_job; + + IF int_core_count > 0 THEN + UPDATE folder_resource SET int_cores = int_cores + int_core_count + WHERE pk_folder = NEW.pk_folder; + + UPDATE folder_resource SET int_cores = int_cores - int_core_count + WHERE pk_folder = OLD.pk_folder; + END IF; + + IF int_gpu_count > 0 THEN + UPDATE folder_resource SET int_gpus = int_gpus + int_gpu_count + WHERE pk_folder = NEW.pk_folder; + + UPDATE folder_resource SET int_gpus = int_gpus - int_gpu_count + WHERE pk_folder = OLD.pk_folder; + END IF; + RETURN NULL; +END +$body$ +LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION trigger__before_delete_job() +RETURNS TRIGGER AS $body$ +DECLARE + js JobStatType; +BEGIN + SELECT + job_usage.int_core_time_success, + job_usage.int_core_time_fail, + job_usage.int_gpu_time_success, + job_usage.int_gpu_time_fail, + job_stat.int_waiting_count, + job_stat.int_dead_count, + job_stat.int_depend_count, + job_stat.int_eaten_count, + job_stat.int_succeeded_count, + job_stat.int_running_count, + job_mem.int_max_rss, + job_mem.int_gpu_mem_max + INTO + js + FROM + job_mem, + job_usage, + job_stat + WHERE + job_usage.pk_job = job_mem.pk_job + AND + job_stat.pk_job = job_mem.pk_job + AND + job_mem.pk_job = OLD.pk_job; + + UPDATE + job_history + SET + pk_dept = OLD.pk_dept, + int_core_time_success = js.int_core_time_success, + int_core_time_fail = js.int_core_time_fail, + int_gpu_time_success = js.int_gpu_time_success, + int_gpu_time_fail = js.int_gpu_time_fail, + int_frame_count = OLD.int_frame_count, + int_layer_count = OLD.int_layer_count, + int_waiting_count = js.int_waiting_count, + int_dead_count = js.int_dead_count, + int_depend_count = js.int_depend_count, + int_eaten_count = js.int_eaten_count, + int_succeeded_count = js.int_succeeded_count, + int_running_count = js.int_running_count, + int_max_rss = js.int_max_rss, + int_gpu_mem_max = js.int_gpu_mem_max, + b_archived = true, + int_ts_stopped = COALESCE(epoch(OLD.ts_stopped), epoch(current_timestamp)) + WHERE + pk_job = OLD.pk_job; + + DELETE FROM depend WHERE pk_job_depend_on=OLD.pk_job OR pk_job_depend_er=OLD.pk_job; + DELETE FROM frame WHERE pk_job=OLD.pk_job; + DELETE FROM layer WHERE pk_job=OLD.pk_job; + DELETE FROM job_env WHERE pk_job=OLD.pk_job; + DELETE FROM job_stat WHERE pk_job=OLD.pk_job; + DELETE FROM job_resource WHERE pk_job=OLD.pk_job; + DELETE FROM job_usage WHERE pk_job=OLD.pk_job; + DELETE FROM job_mem WHERE pk_job=OLD.pk_job; + DELETE FROM comments WHERE pk_job=OLD.pk_job; + + RETURN OLD; +END +$body$ +LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION trigger__after_job_finished() +RETURNS TRIGGER AS $body$ +DECLARE + ts INT := cast(epoch(current_timestamp) as integer); + js JobStatType; + ls LayerStatType; + one_layer RECORD; +BEGIN + SELECT + job_usage.int_core_time_success, + job_usage.int_core_time_fail, + job_usage.int_gpu_time_success, + job_usage.int_gpu_time_fail, + job_stat.int_waiting_count, + job_stat.int_dead_count, + job_stat.int_depend_count, + job_stat.int_eaten_count, + job_stat.int_succeeded_count, + job_stat.int_running_count, + job_mem.int_max_rss, + job_mem.int_gpu_mem_max + INTO + js + FROM + job_mem, + job_usage, + job_stat + WHERE + job_usage.pk_job = job_mem.pk_job + AND + job_stat.pk_job = job_mem.pk_job + AND + job_mem.pk_job = NEW.pk_job; + + UPDATE + job_history + SET + pk_dept = NEW.pk_dept, + int_core_time_success = js.int_core_time_success, + int_core_time_fail = js.int_core_time_fail, + int_gpu_time_success = js.int_gpu_time_success, + int_gpu_time_fail = js.int_gpu_time_fail, + int_frame_count = NEW.int_frame_count, + int_layer_count = NEW.int_layer_count, + int_waiting_count = js.int_waiting_count, + int_dead_count = js.int_dead_count, + int_depend_count = js.int_depend_count, + int_eaten_count = js.int_eaten_count, + int_succeeded_count = js.int_succeeded_count, + int_running_count = js.int_running_count, + int_max_rss = js.int_max_rss, + int_gpu_mem_max = js.int_gpu_mem_max, + int_ts_stopped = ts + WHERE + pk_job = NEW.pk_job; + + FOR one_layer IN (SELECT pk_layer from layer where pk_job = NEW.pk_job) + LOOP + SELECT + layer_usage.int_core_time_success, + layer_usage.int_core_time_fail, + layer_usage.int_gpu_time_success, + layer_usage.int_gpu_time_fail, + layer_stat.int_total_count, + layer_stat.int_waiting_count, + layer_stat.int_dead_count, + layer_stat.int_depend_count, + layer_stat.int_eaten_count, + layer_stat.int_succeeded_count, + layer_stat.int_running_count, + layer_mem.int_max_rss, + layer_mem.int_gpu_mem_max + INTO + ls + FROM + layer_mem, + layer_usage, + layer_stat + WHERE + layer_usage.pk_layer = layer_mem.pk_layer + AND + layer_stat.pk_layer = layer_mem.pk_layer + AND + layer_mem.pk_layer = one_layer.pk_layer; + + UPDATE + layer_history + SET + int_core_time_success = ls.int_core_time_success, + int_core_time_fail = ls.int_core_time_fail, + int_gpu_time_success = ls.int_gpu_time_success, + int_gpu_time_fail = ls.int_gpu_time_fail, + int_frame_count = ls.int_total_count, + int_waiting_count = ls.int_waiting_count, + int_dead_count = ls.int_dead_count, + int_depend_count = ls.int_depend_count, + int_eaten_count = ls.int_eaten_count, + int_succeeded_count = ls.int_succeeded_count, + int_running_count = ls.int_running_count, + int_max_rss = ls.int_max_rss, + int_gpu_mem_max = ls.int_gpu_mem_max + WHERE + pk_layer = one_layer.pk_layer; + END LOOP; + + /** + * Delete any local core assignments from this job. + **/ + DELETE FROM job_local WHERE pk_job=NEW.pk_job; + + RETURN NEW; +END; +$body$ +LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION trigger__after_job_dept_update() +RETURNS TRIGGER AS $body$ +DECLARE + int_running_cores INT; + int_running_gpus INT; +BEGIN + /** + * Handles the accounting for moving a job between departments. + **/ + SELECT int_cores, int_gpus INTO int_running_cores, int_running_gpus + FROM job_resource WHERE pk_job = NEW.pk_job; + + IF int_running_cores > 0 THEN + UPDATE point SET int_cores = int_cores + int_running_cores + WHERE pk_dept = NEW.pk_dept AND pk_show = NEW.pk_show; + + UPDATE point SET int_cores = int_cores - int_running_cores + WHERE pk_dept = OLD.pk_dept AND pk_show = OLD.pk_show; + END IF; + + IF int_running_gpus > 0 THEN + UPDATE point SET int_gpus = int_gpus + int_running_gpus + WHERE pk_dept = NEW.pk_dept AND pk_show = NEW.pk_show; + + UPDATE point SET int_gpus = int_gpus - int_running_gpus + WHERE pk_dept = OLD.pk_dept AND pk_show = OLD.pk_show; + END IF; + + RETURN NULL; +END; +$body$ +LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION trigger__verify_host_local() +RETURNS TRIGGER AS $body$ +BEGIN + /** + * Check to see if the new cores exceeds max cores. This check is only + * done if NEW.int_max_cores is equal to OLD.int_max_cores and + * NEW.int_cores > OLD.int_cores, otherwise this error will be thrown + * when people lower the max. + **/ + IF NEW.int_cores_idle < 0 THEN + RAISE EXCEPTION 'host local doesnt have enough idle cores.'; + END IF; + + IF NEW.int_mem_idle < 0 THEN + RAISE EXCEPTION 'host local doesnt have enough idle memory'; + END IF; + + IF NEW.int_gpus_idle < 0 THEN + RAISE EXCEPTION 'host local doesnt have enough GPU idle cores.'; + END IF; + + IF NEW.int_gpu_mem_idle < 0 THEN + RAISE EXCEPTION 'host local doesnt have enough GPU idle memory.'; + END IF; + + RETURN NEW; +END; +$body$ +LANGUAGE PLPGSQL; + +CREATE TRIGGER verify_host_local BEFORE UPDATE ON host_local +FOR EACH ROW + WHEN ((NEW.int_cores_max = OLD.int_cores_max AND NEW.int_mem_max = OLD.int_mem_max) AND + (NEW.int_cores_idle != OLD.int_cores_idle OR NEW.int_mem_idle != OLD.int_mem_idle) AND + (NEW.int_gpus_max = OLD.int_gpus_max AND NEW.int_gpu_mem_max = OLD.int_gpu_mem_max) AND + (NEW.int_gpus_idle != OLD.int_gpus_idle OR NEW.int_gpu_mem_idle != OLD.int_gpu_mem_idle)) + EXECUTE PROCEDURE trigger__verify_host_local(); + + +CREATE OR REPLACE FUNCTION trigger__after_insert_layer() +RETURNS TRIGGER AS $body$ +BEGIN + INSERT INTO layer_stat (pk_layer_stat, pk_layer, pk_job) VALUES (NEW.pk_layer, NEW.pk_layer, NEW.pk_job); + INSERT INTO layer_resource (pk_layer_resource, pk_layer, pk_job) VALUES (NEW.pk_layer, NEW.pk_layer, NEW.pk_job); + INSERT INTO layer_usage (pk_layer_usage, pk_layer, pk_job) VALUES (NEW.pk_layer, NEW.pk_layer, NEW.pk_job); + INSERT INTO layer_mem (pk_layer_mem, pk_layer, pk_job) VALUES (NEW.pk_layer, NEW.pk_layer, NEW.pk_job); + + INSERT INTO layer_history + (pk_layer, pk_job, str_name, str_type, int_cores_min, int_mem_min, int_gpus_min, int_gpu_mem_min, b_archived,str_services) + VALUES + (NEW.pk_layer, NEW.pk_job, NEW.str_name, NEW.str_type, NEW.int_cores_min, NEW.int_mem_min, NEW.int_gpus_min, NEW.int_gpu_mem_min, false, NEW.str_services); + + RETURN NEW; +END; +$body$ +LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION trigger__before_delete_layer() +RETURNS TRIGGER AS $body$ +DECLARE + js LayerStatType; +BEGIN + SELECT + layer_usage.int_core_time_success, + layer_usage.int_core_time_fail, + layer_usage.int_gpu_time_success, + layer_usage.int_gpu_time_fail, + layer_stat.int_total_count, + layer_stat.int_waiting_count, + layer_stat.int_dead_count, + layer_stat.int_depend_count, + layer_stat.int_eaten_count, + layer_stat.int_succeeded_count, + layer_stat.int_running_count, + layer_mem.int_max_rss, + layer_mem.int_gpu_mem_max + INTO + js + FROM + layer_mem, + layer_usage, + layer_stat + WHERE + layer_usage.pk_layer = layer_mem.pk_layer + AND + layer_stat.pk_layer = layer_mem.pk_layer + AND + layer_mem.pk_layer = OLD.pk_layer; + + UPDATE + layer_history + SET + int_core_time_success = js.int_core_time_success, + int_core_time_fail = js.int_core_time_fail, + int_gpu_time_success = js.int_gpu_time_success, + int_gpu_time_fail = js.int_gpu_time_fail, + int_frame_count = js.int_total_count, + int_waiting_count = js.int_waiting_count, + int_dead_count = js.int_dead_count, + int_depend_count = js.int_depend_count, + int_eaten_count = js.int_eaten_count, + int_succeeded_count = js.int_succeeded_count, + int_running_count = js.int_running_count, + int_max_rss = js.int_max_rss, + int_gpu_mem_max = js.int_gpu_mem_max, + b_archived = true + WHERE + pk_layer = OLD.pk_layer; + + DELETE FROM layer_resource where pk_layer=OLD.pk_layer; + DELETE FROM layer_stat where pk_layer=OLD.pk_layer; + DELETE FROM layer_usage where pk_layer=OLD.pk_layer; + DELETE FROM layer_env where pk_layer=OLD.pk_layer; + DELETE FROM layer_mem where pk_layer=OLD.pk_layer; + DELETE FROM layer_output where pk_layer=OLD.pk_layer; + + RETURN OLD; +END; +$body$ +LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION trigger__verify_host_resources() +RETURNS TRIGGER AS $body$ +BEGIN + IF NEW.int_cores_idle < 0 THEN + RAISE EXCEPTION 'unable to allocate additional core units'; + END IF; + + If NEW.int_mem_idle < 0 THEN + RAISE EXCEPTION 'unable to allocate additional memory'; + END IF; + + If NEW.int_gpus_idle < 0 THEN + RAISE EXCEPTION 'unable to allocate additional GPU units'; + END IF; + + If NEW.int_gpu_mem_idle < 0 THEN + RAISE EXCEPTION 'unable to allocate additional GPU memory'; + END IF; + RETURN NEW; +END; +$body$ +LANGUAGE PLPGSQL; + +DROP TRIGGER verify_host_resources ON host; +CREATE TRIGGER verify_host_resources BEFORE UPDATE ON host +FOR EACH ROW + WHEN (NEW.int_cores_idle != OLD.int_cores_idle + OR NEW.int_mem_idle != OLD.int_mem_idle + OR NEW.int_gpus_idle != OLD.int_gpus_idle + OR NEW.int_gpu_mem_idle != OLD.int_gpu_mem_idle) + EXECUTE PROCEDURE trigger__verify_host_resources(); + + +CREATE OR REPLACE FUNCTION trigger__verify_job_resources() +RETURNS TRIGGER AS $body$ +BEGIN + /** + * Check to see if the new cores exceeds max cores. This check is only + * done if NEW.int_max_cores is equal to OLD.int_max_cores and + * NEW.int_cores > OLD.int_cores, otherwise this error will be thrown + * at the wrong time. + **/ + IF NEW.int_cores > NEW.int_max_cores THEN + RAISE EXCEPTION 'job has exceeded max cores'; + END IF; + IF NEW.int_gpus > NEW.int_max_gpus THEN + RAISE EXCEPTION 'job has exceeded max GPU units'; + END IF; + RETURN NEW; +END; +$body$ +LANGUAGE PLPGSQL; + +DROP TRIGGER verify_job_resources ON job_resource; +CREATE TRIGGER verify_job_resources BEFORE UPDATE ON job_resource +FOR EACH ROW + WHEN (NEW.int_max_cores = OLD.int_max_cores AND NEW.int_cores > OLD.int_cores OR + NEW.int_max_gpus = OLD.int_max_gpus AND NEW.int_gpus > OLD.int_gpus) + EXECUTE PROCEDURE trigger__verify_job_resources(); + + +CREATE OR REPLACE FUNCTION trigger__update_proc_update_layer() +RETURNS TRIGGER AS $body$ +DECLARE + lr RECORD; +BEGIN + FOR lr IN ( + SELECT + pk_layer + FROM + layer_stat + WHERE + pk_layer IN (OLD.pk_layer, NEW.pk_layer) + ORDER BY layer_stat.pk_layer DESC + ) LOOP + + IF lr.pk_layer = OLD.pk_layer THEN + + UPDATE layer_resource SET + int_cores = int_cores - OLD.int_cores_reserved, + int_gpus = int_gpus - OLD.int_gpus_reserved + WHERE + pk_layer = OLD.pk_layer; + + ELSE + + UPDATE layer_resource SET + int_cores = int_cores + NEW.int_cores_reserved, + int_gpus = int_gpus + NEW.int_gpus_reserved + WHERE + pk_layer = NEW.pk_layer; + END IF; + + END LOOP; + RETURN NULL; +END; +$body$ +LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION trigger__frame_history_open() +RETURNS TRIGGER AS $body$ +DECLARE + str_pk_alloc VARCHAR(36) := null; + int_checkpoint INT := 0; +BEGIN + + IF OLD.str_state = 'RUNNING' THEN + + IF NEW.int_exit_status = 299 THEN + + EXECUTE 'DELETE FROM frame_history WHERE int_ts_stopped = 0 AND pk_frame=$1' USING + NEW.pk_frame; + + ELSE + If NEW.str_state = 'CHECKPOINT' THEN + int_checkpoint := 1; + END IF; + + EXECUTE + 'UPDATE + frame_history + SET + int_mem_max_used=$1, + int_gpu_mem_max_used=$2, + int_ts_stopped=$3, + int_exit_status=$4, + int_checkpoint_count=$5 + WHERE + int_ts_stopped = 0 AND pk_frame=$6' + USING + NEW.int_mem_max_used, + NEW.int_gpu_mem_max_used, + epoch(current_timestamp), + NEW.int_exit_status, + int_checkpoint, + NEW.pk_frame; + END IF; + END IF; + + IF NEW.str_state = 'RUNNING' THEN + + SELECT pk_alloc INTO str_pk_alloc FROM host WHERE str_name=NEW.str_host; + + EXECUTE + 'INSERT INTO + frame_history + ( + pk_frame, + pk_layer, + pk_job, + str_name, + str_state, + int_cores, + int_mem_reserved, + int_gpus, + int_gpu_mem_reserved, + str_host, + int_ts_started, + pk_alloc + ) + VALUES + ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)' + USING NEW.pk_frame, + NEW.pk_layer, + NEW.pk_job, + NEW.str_name, + 'RUNNING', + NEW.int_cores, + NEW.int_mem_reserved, + NEW.int_gpus, + NEW.int_gpu_mem_reserved, + NEW.str_host, + epoch(current_timestamp), + str_pk_alloc; + END IF; + RETURN NULL; + +END; +$body$ +LANGUAGE PLPGSQL; diff --git a/cuebot/src/main/resources/public/dtd/cjsl-1.12.dtd b/cuebot/src/main/resources/public/dtd/cjsl-1.12.dtd new file mode 100644 index 000000000..222e04cfc --- /dev/null +++ b/cuebot/src/main/resources/public/dtd/cjsl-1.12.dtd @@ -0,0 +1,97 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/BookingDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/BookingDaoTests.java index 8c09dfde4..c6c03d604 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/BookingDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/BookingDaoTests.java @@ -109,8 +109,8 @@ public DispatchHost createHost() { .setState(HardwareState.UP) .setFacility("spi") .addTags("general") - .putAttributes("freeGpu", String.format("%d", CueUtil.MB512)) - .putAttributes("totalGpu", String.format("%d", CueUtil.MB512)) + .setFreeGpuMem((int) CueUtil.MB512) + .setTotalGpuMem((int) CueUtil.MB512) .build(); DispatchHost dh = hostManager.createHost(host); hostManager.setAllocation(dh, @@ -138,7 +138,7 @@ public void insertLocalJobAssignment() { LocalHostAssignment lja = new LocalHostAssignment(); lja.setMaxCoreUnits(200); lja.setMaxMemory(CueUtil.GB4); - lja.setMaxGpu(1); + lja.setMaxGpuMemory(1); lja.setThreads(2); bookingDao.insertLocalHostAssignment(h, j, lja); @@ -149,7 +149,7 @@ public void insertLocalJobAssignment() { Integer.class, j.getJobId())); assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT int_gpu_max FROM host_local WHERE pk_job=?", + "SELECT int_gpu_mem_max FROM host_local WHERE pk_job=?", Integer.class, j.getJobId())); assertEquals(Integer.valueOf(200), jdbcTemplate.queryForObject( @@ -161,7 +161,7 @@ public void insertLocalJobAssignment() { Long.class, j.getJobId())); assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT int_gpu_max FROM host_local WHERE pk_job=?", + "SELECT int_gpu_mem_max FROM host_local WHERE pk_job=?", Integer.class, j.getJobId())); assertEquals(Integer.valueOf(200), jdbcTemplate.queryForObject( @@ -185,7 +185,7 @@ public void insertLocalLayerAssignment() { LocalHostAssignment lja = new LocalHostAssignment(); lja.setMaxCoreUnits(200); lja.setMaxMemory(CueUtil.GB4); - lja.setMaxGpu(1); + lja.setMaxGpuMemory(1); lja.setThreads(2); bookingDao.insertLocalHostAssignment(h, layer, lja); @@ -212,7 +212,7 @@ public void insertLocalLayerAssignment() { Long.class, j.getJobId())); assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT int_gpu_max FROM host_local WHERE pk_job=?", + "SELECT int_gpu_mem_max FROM host_local WHERE pk_job=?", Integer.class, j.getJobId())); assertEquals(Integer.valueOf(200), jdbcTemplate.queryForObject( @@ -237,7 +237,7 @@ public void insertLocalFrameAssignment() { LocalHostAssignment lja = new LocalHostAssignment(); lja.setMaxCoreUnits(200); lja.setMaxMemory(CueUtil.GB4); - lja.setMaxGpu(1); + lja.setMaxGpuMemory(1); lja.setThreads(2); bookingDao.insertLocalHostAssignment(h, frame, lja); @@ -264,7 +264,7 @@ public void insertLocalFrameAssignment() { Long.class, j.getJobId())); assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( - "SELECT int_gpu_max FROM host_local WHERE pk_job=?", + "SELECT int_gpu_mem_max FROM host_local WHERE pk_job=?", Integer.class, j.getJobId())); assertEquals(Integer.valueOf(200), jdbcTemplate.queryForObject( @@ -288,7 +288,7 @@ public void testGetLocalJobAssignment() { lja.setMaxCoreUnits(200); lja.setMaxMemory(CueUtil.GB4); lja.setThreads(2); - lja.setMaxGpu(1); + lja.setMaxGpuMemory(1); bookingDao.insertLocalHostAssignment(h, j, lja); @@ -297,7 +297,7 @@ public void testGetLocalJobAssignment() { assertEquals(lja.getMaxCoreUnits(), lja2.getMaxCoreUnits()); assertEquals(lja.getMaxMemory(), lja2.getMaxMemory()); - assertEquals(lja.getMaxGpu(), lja2.getMaxGpu()); + assertEquals(lja.getMaxGpuMemory(), lja2.getMaxGpuMemory()); assertEquals(lja.getThreads(), lja2.getThreads()); } @@ -314,7 +314,7 @@ public void testGetRenderPartition() { lja.setMaxCoreUnits(200); lja.setMaxMemory(CueUtil.GB4); lja.setThreads(2); - lja.setMaxGpu(1); + lja.setMaxGpuMemory(1); bookingDao.insertLocalHostAssignment(h, j, lja); @@ -324,7 +324,7 @@ public void testGetRenderPartition() { assertEquals(lja.getMaxCoreUnits(), lja2.getMaxCoreUnits()); assertEquals(lja.getMaxMemory(), lja2.getMaxMemory()); assertEquals(lja.getThreads(), lja2.getThreads()); - assertEquals(lja.getMaxGpu(), lja2.getMaxGpu()); + assertEquals(lja.getMaxGpuMemory(), lja2.getMaxGpuMemory()); RenderPartition rp = whiteboard.getRenderPartition(lja2); @@ -332,9 +332,9 @@ public void testGetRenderPartition() { assertEquals(lja2.getMaxMemory(), rp.getMaxMemory()); assertEquals(lja2.getThreads(), rp.getThreads()); logger.info("--------------------"); - logger.info(lja2.getMaxGpu()); - logger.info(rp.getMaxGpu()); - assertEquals(lja2.getMaxGpu(), rp.getMaxGpu()); + logger.info(lja2.getMaxGpuMemory()); + logger.info(rp.getMaxGpuMemory()); + assertEquals(lja2.getMaxGpuMemory(), rp.getMaxGpuMemory()); assertEquals(h.getName(), rp.getHost()); assertEquals(j.getName(), rp.getJob()); } @@ -351,7 +351,7 @@ public void testGetProcs() { lja.setMaxCoreUnits(200); lja.setMaxMemory(CueUtil.GB4); lja.setThreads(2); - lja.setMaxGpu(1); + lja.setMaxGpuMemory(1); bookingDao.insertLocalHostAssignment(h, j, lja); @@ -370,7 +370,7 @@ public void updateMaxCores() { lja.setMaxCoreUnits(200); lja.setMaxMemory(CueUtil.GB4); lja.setThreads(2); - lja.setMaxGpu(1); + lja.setMaxGpuMemory(1); bookingDao.insertLocalHostAssignment(h, j, lja); assertTrue(bookingDao.updateMaxCores(lja, 100)); @@ -403,7 +403,7 @@ public void updateMaxMemory() { lja.setMaxCoreUnits(200); lja.setMaxMemory(CueUtil.GB4); lja.setThreads(2); - lja.setMaxGpu(1); + lja.setMaxGpuMemory(1); bookingDao.insertLocalHostAssignment(h, j, lja); bookingDao.updateMaxMemory(lja, CueUtil.GB2); @@ -424,7 +424,7 @@ public void updateMaxMemory() { @Test @Transactional @Rollback(true) - public void updateMaxGpu() { + public void updateMaxGpuMemory() { DispatchHost h = createHost(); JobDetail j = launchJob(); @@ -433,7 +433,7 @@ public void updateMaxGpu() { lja.setMaxCoreUnits(200); lja.setMaxMemory(CueUtil.GB4); lja.setThreads(2); - lja.setMaxGpu(1); + lja.setMaxGpuMemory(1); bookingDao.insertLocalHostAssignment(h, j, lja); bookingDao.updateMaxMemory(lja, CueUtil.GB2); @@ -442,15 +442,15 @@ public void updateMaxGpu() { assertEquals(CueUtil.GB2, lj2.getIdleMemory()); assertEquals(CueUtil.GB2, lj2.getMaxMemory()); - assertEquals(1, lj2.getMaxGpu()); + assertEquals(1, lj2.getMaxGpuMemory()); - bookingDao.updateMaxGpu(lja, 2); + bookingDao.updateMaxGpuMemory(lja, 2); lj2 = bookingDao.getLocalJobAssignment(lja.id); assertEquals(CueUtil.GB2, lj2.getIdleMemory()); assertEquals(CueUtil.GB2, lj2.getMaxMemory()); - assertEquals(2, lj2.getMaxGpu()); + assertEquals(2, lj2.getMaxGpuMemory()); } } diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/CommentDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/CommentDaoTests.java index 886400823..668e666e9 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/CommentDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/CommentDaoTests.java @@ -153,8 +153,8 @@ public void testInsertCommentOnHost() { .addTags("linux") .setState(HardwareState.UP) .setFacility("spi") - .putAttributes("freeGpu", String.format("%d", CueUtil.MB512)) - .putAttributes("totalGpu", String.format("%d", CueUtil.MB512)) + .setFreeGpuMem((int) CueUtil.MB512) + .setTotalGpuMem((int) CueUtil.MB512) .build(); CommentDetail d = new CommentDetail(); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DeedDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DeedDaoTests.java index 23a93a2c1..a04e7e5e6 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DeedDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DeedDaoTests.java @@ -86,8 +86,8 @@ public DispatchHost createHost() { .addTags("general") .setState(HardwareState.UP) .setFacility("spi") - .putAttributes("freeGpu", String.format("%d", CueUtil.MB512)) - .putAttributes("totalGpu", String.format("%d", CueUtil.MB512)) + .setFreeGpuMem((int) CueUtil.MB512) + .setTotalGpuMem((int) CueUtil.MB512) .build(); DispatchHost dh = hostManager.createHost(host); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java index abf9e34a7..a400bb26a 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java @@ -374,7 +374,7 @@ public void testFindDispatchJobsByLocal() { lja.setThreads(1); lja.setMaxMemory(CueUtil.GB16); lja.setMaxCoreUnits(200); - lja.setMaxGpu(1); + lja.setMaxGpuMemory(1); bookingDao.insertLocalHostAssignment(host, job, lja); jobs = dispatcherDao.findLocalDispatchJobs(host); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/FrameDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/FrameDaoTests.java index c18c78a28..8d64d918e 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/FrameDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/FrameDaoTests.java @@ -60,6 +60,7 @@ import com.imageworks.spcue.service.JobLauncher; import com.imageworks.spcue.service.JobManager; import com.imageworks.spcue.test.AssumingPostgresEngine; +import com.imageworks.spcue.util.CueUtil; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -126,8 +127,8 @@ public void create() { .addAllTags(ImmutableList.of("mcore", "4core", "8g")) .setState(HardwareState.UP) .setFacility("spi") - .putAttributes("freeGpu", "512") - .putAttributes("totalGpu", "512") + .setFreeGpuMem((int) CueUtil.MB512) + .setTotalGpuMem((int) CueUtil.MB512) .build(); hostManager.createHost(host); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java index b53e1078a..df965893b 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java @@ -100,8 +100,8 @@ public static RenderHost buildRenderHost(String name) { .addAllTags(ImmutableList.of("linux", "64bit")) .setState(HardwareState.UP) .setFacility("spi") - .putAttributes("freeGpu", String.format("%d", CueUtil.MB512)) - .putAttributes("totalGpu", String.format("%d", CueUtil.MB512)) + .setFreeGpuMem((int) CueUtil.MB512) + .setTotalGpuMem((int) CueUtil.MB512) .build(); return host; diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/JobDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/JobDaoTests.java index 07a32f184..597489592 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/JobDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/JobDaoTests.java @@ -683,7 +683,7 @@ public void testUpdateUsage() { JobInterface job = jobDao.findJob(spec.getJobs().get(0).detail.name); /** 60 seconds of 100 core units **/ - ResourceUsage usage = new ResourceUsage(60, 33); + ResourceUsage usage = new ResourceUsage(60, 33, 0); assertTrue(usage.getClockTimeSeconds() > 0); assertTrue(usage.getCoreTimeSeconds() > 0); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java index 938107f36..06864a9bc 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java @@ -471,12 +471,12 @@ public void updateMinMemory() { @Test @Transactional @Rollback(true) - public void updateMinGpu() { - long gpu = CueUtil.GB; + public void updateMinGpuMemory() { + long mem = CueUtil.GB; LayerDetail layer = getLayer(); - layerDao.updateMinGpu(layer, gpu, LayerType.RENDER); - assertEquals(Long.valueOf(gpu),jdbcTemplate.queryForObject( - "SELECT int_gpu_min FROM layer WHERE pk_layer=?", + layerDao.updateMinGpuMemory(layer, mem, LayerType.RENDER); + assertEquals(Long.valueOf(mem),jdbcTemplate.queryForObject( + "SELECT int_gpu_mem_min FROM layer WHERE pk_layer=?", Long.class, layer.getLayerId())); } @@ -590,7 +590,7 @@ public void testUpdateUsage() { Integer.class, layer.getId())); /** 60 seconds of 100 core units **/ - ResourceUsage usage = new ResourceUsage(60, 33); + ResourceUsage usage = new ResourceUsage(60, 33, 0); assertTrue(usage.getClockTimeSeconds() > 0); assertTrue(usage.getCoreTimeSeconds() > 0); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java index c43d98180..6c9efc3e5 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java @@ -633,7 +633,7 @@ public void testGetReservedMemory() { @Test @Transactional @Rollback(true) - public void testGetReservedGpu() { + public void testGetReservedGpuMemory() { DispatchHost host = createHost(); JobDetail job = launchJob(); @@ -645,11 +645,11 @@ public void testGetReservedGpu() { procDao.insertVirtualProc(proc); VirtualProc _proc = procDao.findVirtualProc(frame); - assertEquals(Long.valueOf(Dispatcher.GPU_RESERVED_DEFAULT), jdbcTemplate.queryForObject( - "SELECT int_gpu_reserved FROM proc WHERE pk_proc=?", + assertEquals(Long.valueOf(Dispatcher.MEM_GPU_RESERVED_DEFAULT), jdbcTemplate.queryForObject( + "SELECT int_gpu_mem_reserved FROM proc WHERE pk_proc=?", Long.class, _proc.id)); - assertEquals(Dispatcher.GPU_RESERVED_DEFAULT, - procDao.getReservedGpu(_proc)); + assertEquals(Dispatcher.MEM_GPU_RESERVED_DEFAULT, + procDao.getReservedGpuMemory(_proc)); } @Test diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ServiceDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ServiceDaoTests.java index fee824fc1..16168f245 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ServiceDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ServiceDaoTests.java @@ -70,7 +70,7 @@ public void testInsertService() { s.timeout = 0; s.timeout_llu = 0; s.minMemory = CueUtil.GB4; - s.minGpu = CueUtil.GB; + s.minGpuMemory = CueUtil.GB; s.threadable = false; s.tags.addAll(Sets.newHashSet(new String[] { "general"})); @@ -88,7 +88,7 @@ public void testUpdateService() { s.timeout = 0; s.timeout_llu = 0; s.minMemory = CueUtil.GB4; - s.minGpu = CueUtil.GB; + s.minGpuMemory = CueUtil.GB; s.threadable = false; s.tags.addAll(Sets.newHashSet(new String[] { "general"})); @@ -100,7 +100,7 @@ public void testUpdateService() { s.timeout = 0; s.timeout_llu = 0; s.minMemory = CueUtil.GB8; - s.minGpu = CueUtil.GB2; + s.minGpuMemory = CueUtil.GB2; s.threadable = true; s.tags = Sets.newLinkedHashSet(); s.tags.add("linux"); @@ -125,7 +125,7 @@ public void testDeleteService() { s.timeout = 0; s.timeout_llu = 0; s.minMemory = CueUtil.GB4; - s.minGpu = CueUtil.GB; + s.minGpuMemory = CueUtil.GB; s.threadable = false; s.tags.addAll(Sets.newHashSet(new String[] { "general"})); @@ -149,7 +149,7 @@ public void testInsertServiceOverride() { s.timeout = 0; s.timeout_llu = 0; s.minMemory = CueUtil.GB4; - s.minGpu = CueUtil.GB; + s.minGpuMemory = CueUtil.GB; s.threadable = false; s.tags.addAll(Sets.newHashSet(new String[] { "general"})); s.showId = "00000000-0000-0000-0000-000000000000"; @@ -168,7 +168,7 @@ public void testUpdateServiceOverride() { s.timeout = 0; s.timeout_llu = 0; s.minMemory = CueUtil.GB4; - s.minGpu = CueUtil.GB2; + s.minGpuMemory = CueUtil.GB2; s.threadable = false; s.tags.addAll(Sets.newHashSet(new String[] { "general"})); s.showId = "00000000-0000-0000-0000-000000000000"; @@ -182,7 +182,7 @@ public void testUpdateServiceOverride() { s.timeout = 10; s.timeout_llu = 10; s.minMemory = CueUtil.GB8; - s.minGpu = CueUtil.GB4; + s.minGpuMemory = CueUtil.GB4; s.threadable = true; s.tags = Sets.newLinkedHashSet(); s.tags.add("linux"); @@ -195,7 +195,7 @@ public void testUpdateServiceOverride() { assertEquals(s.timeout, s1.timeout); assertEquals(s.timeout_llu, s1.timeout_llu); assertEquals(s.minMemory, s1.minMemory); - assertEquals(s.minGpu, s1.minGpu); + assertEquals(s.minGpuMemory, s1.minGpuMemory); assertEquals(s.threadable, s1.threadable); assertEquals(s.tags.toArray()[0], s1.tags.toArray()[0]); } @@ -210,7 +210,7 @@ public void testDeleteServiceOverride() { s.timeout = 0; s.timeout_llu = 0; s.minMemory = CueUtil.GB4; - s.minGpu = CueUtil.GB; + s.minGpuMemory = CueUtil.GB; s.threadable = false; s.tags.addAll(Sets.newHashSet(new String[] { "general"})); s.showId = "00000000-0000-0000-0000-000000000000"; diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java index b27114554..d430ab3b0 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java @@ -84,8 +84,8 @@ public DispatchHost createHost() { .addTags("general") .setState(HardwareState.UP) .setFacility("spi") - .putAttributes("freeGpu", String.format("%d", CueUtil.MB512)) - .putAttributes("totalGpu", String.format("%d", CueUtil.MB512)) + .setFreeGpuMem((int) CueUtil.MB512) + .setTotalGpuMem((int) CueUtil.MB512) .build(); DispatchHost dh = hostManager.createHost(host); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java index 99449337b..8807514d4 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java @@ -278,8 +278,8 @@ public RenderHost getRenderHost() { .setCoresPerProc(400) .setState(HardwareState.DOWN) .setFacility("spi") - .putAttributes("freeGpu", String.format("%d", CueUtil.MB512)) - .putAttributes("totalGpu", String.format("%d", CueUtil.MB512)) + .setFreeGpuMem((int) CueUtil.MB512) + .setTotalGpuMem((int) CueUtil.MB512) .build(); return host; } @@ -1246,7 +1246,7 @@ public void getRenderPartition() { jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec_dispatch_test.xml")); JobDetail job = jobManager.findJobDetail("pipe-dev.cue-testuser_shell_dispatch_test_v1"); - LocalHostAssignment lba = new LocalHostAssignment(800, 8, CueUtil.GB8, 1); + LocalHostAssignment lba = new LocalHostAssignment(800, 8, CueUtil.GB8, 1, 1); bookingManager.createLocalHostAssignment(hd, job, lba); whiteboardDao.getRenderPartition(lba); @@ -1263,7 +1263,7 @@ public void getRenderPartitionsByHost() { jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec_dispatch_test.xml")); JobDetail job = jobManager.findJobDetail("pipe-dev.cue-testuser_shell_dispatch_test_v1"); - LocalHostAssignment lba = new LocalHostAssignment(800, 8, CueUtil.GB8, 1); + LocalHostAssignment lba = new LocalHostAssignment(800, 8, CueUtil.GB8, 1, 1); bookingManager.createLocalHostAssignment(hd, job, lba); assertEquals(1, whiteboardDao.getRenderPartitions(hd).getRenderPartitionsCount()); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuJobTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuJobTests.java index d99041b65..4cc1c1f03 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuJobTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuJobTests.java @@ -114,8 +114,8 @@ public void createHost() { .setState(HardwareState.UP) .setFacility("spi") .putAttributes("SP_OS", "Linux") - .putAttributes("freeGpu", String.format("%d", CueUtil.MB512)) - .putAttributes("totalGpu", String.format("%d", CueUtil.MB512)) + .setFreeGpuMem((int) CueUtil.MB512) + .setTotalGpuMem((int) CueUtil.MB512) .build(); hostManager.createHost(host, @@ -153,7 +153,7 @@ public void testDispatchGpuRemovedHostToNonGpuJob() { host.idleMemory = host.idleMemory - Math.min(CueUtil.GB4, host.idleMemory); host.idleCores = host.idleCores - Math.min(100, host.idleCores); - host.idleGpu = 0; + host.idleGpuMemory = 0; List procs = dispatcher.dispatchHost(host, job); assertEquals(0, procs.size()); } diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuTests.java index 9318258ad..0a4f6b74a 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuTests.java @@ -114,8 +114,8 @@ public void createHost() { .setState(HardwareState.UP) .setFacility("spi") .putAttributes("SP_OS", "Linux") - .putAttributes("freeGpu", String.format("%d", CueUtil.MB512)) - .putAttributes("totalGpu", String.format("%d", CueUtil.MB512)) + .setFreeGpuMem((int) CueUtil.MB512) + .setTotalGpuMem((int) CueUtil.MB512) .build(); hostManager.createHost(host, @@ -153,7 +153,7 @@ public void testDispatchGpuRemovedHostToNonGpuJob() { host.idleMemory = host.idleMemory - Math.min(CueUtil.GB4, host.idleMemory); host.idleCores = host.idleCores - Math.min(100, host.idleCores); - host.idleGpu = 0; + host.idleGpuMemory = 0; List procs = dispatcher.dispatchHost(host, job); assertEquals(1, procs.size()); } @@ -202,17 +202,20 @@ public void testDispatchHostRemoveRestoreGpu() { long idleMemoryOrig = host.idleMemory; int idleCoresOrig = host.idleCores; - long idleGpuOrig = host.idleGpu; + long idleGpuMemoryOrig = host.idleGpuMemory; + int idleGpusOrig = host.idleGpus; host.removeGpu(); - assertEquals(0, host.idleGpu); + assertEquals(0, host.idleGpuMemory); + assertEquals(0, host.idleGpus); assertEquals(idleMemoryOrig - CueUtil.GB4, host.idleMemory); assertEquals(idleCoresOrig - 100, host.idleCores); host.restoreGpu(); assertEquals(idleMemoryOrig, host.idleMemory); assertEquals(idleCoresOrig, host.idleCores); - assertEquals(idleGpuOrig, host.idleGpu); + assertEquals(idleGpuMemoryOrig, host.idleGpuMemory); + assertEquals(idleGpusOrig, host.idleGpus); } @Test @@ -222,7 +225,7 @@ public void dispatchProcToJob() { DispatchHost host = getHost(); JobDetail job = getJob(); - host.idleGpu = 0; + host.idleGpuMemory = 0; List procs = dispatcher.dispatchHost(host, job); VirtualProc proc = procs.get(0); dispatcher.dispatchProcToJob(proc, job); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpusJobTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpusJobTests.java new file mode 100644 index 000000000..4972b8f9b --- /dev/null +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpusJobTests.java @@ -0,0 +1,277 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.imageworks.spcue.test.dispatcher; + +import java.io.File; +import java.util.List; +import javax.annotation.Resource; + +import org.junit.Before; +import org.junit.Test; +import org.springframework.test.annotation.Rollback; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.transaction.annotation.Transactional; + +import com.imageworks.spcue.DispatchFrame; +import com.imageworks.spcue.DispatchHost; +import com.imageworks.spcue.LayerDetail; +import com.imageworks.spcue.JobDetail; +import com.imageworks.spcue.VirtualProc; +import com.imageworks.spcue.dao.criteria.FrameSearchFactory; +import com.imageworks.spcue.dao.FrameDao; +import com.imageworks.spcue.dao.LayerDao; +import com.imageworks.spcue.depend.LayerOnLayer; +import com.imageworks.spcue.dispatcher.DispatchSupport; +import com.imageworks.spcue.dispatcher.Dispatcher; +import com.imageworks.spcue.grpc.host.HardwareState; +import com.imageworks.spcue.grpc.job.FrameState; +import com.imageworks.spcue.grpc.report.RenderHost; +import com.imageworks.spcue.service.AdminManager; +import com.imageworks.spcue.service.DependManager; +import com.imageworks.spcue.service.HostManager; +import com.imageworks.spcue.service.JobLauncher; +import com.imageworks.spcue.service.JobManager; +import com.imageworks.spcue.test.TransactionalTest; +import com.imageworks.spcue.util.CueUtil; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@ContextConfiguration +public class CoreUnitDispatcherGpusJobTests extends TransactionalTest { + + @Resource + JobManager jobManager; + + @Resource + JobLauncher jobLauncher; + + @Resource + HostManager hostManager; + + @Resource + AdminManager adminManager; + + @Resource + Dispatcher dispatcher; + + @Resource + DispatchSupport dispatchSupport; + + @Resource + LayerDao layerDao; + + @Resource + FrameDao frameDao; + + @Resource + FrameSearchFactory frameSearchFactory; + + @Resource + DependManager dependManager; + + private static final String HOSTNAME = "beta"; + + private static final String CPU_JOB = "pipe-default-testuser_test_cpu"; + + private static final String GPU_JOB = "pipe-default-testuser_test_gpu"; + + private static final String GPU_OVERBOOK_JOB = "pipe-default-testuser_test_gpu_overbook"; + + @Before + public void launchJob() { + jobLauncher.testMode = true; + jobLauncher.launch( + new File("src/test/resources/conf/jobspec/jobspec_dispatch_gpus_test.xml")); + } + + @Before + public void setTestMode() { + dispatcher.setTestMode(true); + } + + @Before + public void createHost() { + RenderHost host = RenderHost.newBuilder() + .setName(HOSTNAME) + .setBootTime(1192369572) + .setFreeMcp(76020) + .setFreeMem((int) CueUtil.GB8) + .setFreeSwap(20760) + .setLoad(0) + .setTotalMcp(195430) + .setTotalMem(CueUtil.GB8) + .setTotalSwap(CueUtil.GB2) + .setNimbyEnabled(false) + .setNumProcs(40) + .setCoresPerProc(100) + .addTags("test") + .setState(HardwareState.UP) + .setFacility("spi") + .putAttributes("SP_OS", "Linux") + .setNumGpus(8) + .setFreeGpuMem(CueUtil.GB32) + .setTotalGpuMem(CueUtil.GB32) + .build(); + + hostManager.createHost(host, + adminManager.findAllocationDetail("spi", "general")); + } + + public DispatchHost getHost() { + return hostManager.findDispatchHost(HOSTNAME); + } + + @Test + @Transactional + @Rollback(true) + public void testDispatchHost() { + DispatchHost host = getHost(); + + List procs = dispatcher.dispatchHost(host); + // All jobs are paused. procs should be empty. + assertTrue(procs.isEmpty()); + } + + @Test + @Transactional + @Rollback(true) + public void testDispatchCpuJob() { + JobDetail job = jobManager.findJobDetail(CPU_JOB); + jobManager.setJobPaused(job, false); + + DispatchHost host = getHost(); + List procs = dispatcher.dispatchHost(host, job); + // Cuebot doesn't dispatch non-GPU job to GPU host. procs should be empty. + assertTrue(procs.isEmpty()); + } + + @Test + @Transactional + @Rollback(true) + public void testDispatchGpuJob() { + JobDetail job = jobManager.findJobDetail(GPU_JOB); + jobManager.setJobPaused(job, false); + + DispatchHost host = getHost(); + List procs = dispatcher.dispatchHost(host, job); + + /* + * The job contains 4 layers. + * - test_gpus_0_layer gpus=0 gpu_memory=1 + * - test_gpu_memory_0_layer gpus=1 gpu_memory=0 + * - test_gpus_1_layer gpus=1 gpu_memory=1 + * - test_gpus_4_kayer gpus=4 gpu_memory=7g + * + * Cuebot doesn't dispatch test_gpu_memory_0_layer because gpu_memory is 0. + * Also job_frame_dispatch_max is 2, + * the procs should be test_gpus_0_layer and test_gpus_1_layer. + */ + assertEquals(2, procs.size()); + + VirtualProc proc0 = procs.get(0); + LayerDetail layer0 = layerDao.findLayerDetail(job, "test_gpus_0_layer"); + assertEquals(layer0.id, proc0.layerId); + assertEquals(100, proc0.coresReserved); + assertEquals(3355443, proc0.memoryReserved); + assertEquals(0, proc0.gpusReserved); + assertEquals(1048576, proc0.gpuMemoryReserved); + + VirtualProc proc1 = procs.get(1); + LayerDetail layer1 = layerDao.findLayerDetail(job, "test_gpus_1_layer"); + assertEquals(layer1.id, proc1.layerId); + assertEquals(100, proc1.coresReserved); + assertEquals(3355443, proc1.memoryReserved); + assertEquals(1, proc1.gpusReserved); + assertEquals(1048576, proc0.gpuMemoryReserved); + } + + @Test + @Transactional + @Rollback(true) + public void testDispatchGpuJobWithDependency() { + JobDetail job = jobManager.findJobDetail(GPU_JOB); + LayerDetail dl0 = layerDao.findLayerDetail(job, "test_gpus_0_layer"); + LayerDetail dl1 = layerDao.findLayerDetail(job, "test_gpu_memory_0_layer"); + LayerOnLayer depend = new LayerOnLayer(dl0, dl1); + dependManager.createDepend(depend); + jobManager.setJobPaused(job, false); + + DispatchHost host = getHost(); + List procs = dispatcher.dispatchHost(host, job); + + /* + * The job contains 4 layers. + * - test_gpus_0_layer gpus=0 gpu_memory=1 + * - test_gpu_memory_0_layer gpus=1 gpu_memory=0 + * - test_gpus_1_layer gpus=1 gpu_memory=1 + * - test_gpus_4_kayer gpus=4 gpu_memory=7g + * + * Cuebot doesn't dispatch test_gpu_memory_0_layer because gpu_memory is 0. + * And test_gpus_0_layer depends on test_gpu_memory_0_layer. + * So the procs should be test_gpus_1_layer and test_gpus_4_layer. + */ + assertEquals(2, procs.size()); + + VirtualProc proc0 = procs.get(0); + LayerDetail layer0 = layerDao.findLayerDetail(job, "test_gpus_1_layer"); + assertEquals(layer0.id, proc0.layerId); + assertEquals(100, proc0.coresReserved); + assertEquals(3355443, proc0.memoryReserved); + assertEquals(1, proc0.gpusReserved); + assertEquals(1048576, proc0.gpuMemoryReserved); + + VirtualProc proc1 = procs.get(1); + LayerDetail layer1 = layerDao.findLayerDetail(job, "test_gpus_4_layer"); + assertEquals(layer1.id, proc1.layerId); + assertEquals(100, proc1.coresReserved); + assertEquals(3355443, proc1.memoryReserved); + assertEquals(4, proc1.gpusReserved); + assertEquals(7340032, proc1.gpuMemoryReserved); + } + + @Test + @Transactional + @Rollback(true) + public void testDispatchGpuOverbookJob() { + JobDetail job = jobManager.findJobDetail(GPU_OVERBOOK_JOB); + jobManager.setJobPaused(job, false); + + DispatchHost host = getHost(); + List procs = dispatcher.dispatchHost(host, job); + + /* + * The job contains 2 layers. + * - test_gpus_6_layer gpus=6 gpu_memory=1 + * - test_gpus_3_layer gpus=3 gpu_memory=1 + * the procs should be only test_gpus_6_layer since host only has 8 GPUs. + */ + assertEquals(1, procs.size()); + + VirtualProc proc0 = procs.get(0); + LayerDetail layer0 = layerDao.findLayerDetail(job, "test_gpus_6_layer"); + assertEquals(layer0.id, proc0.layerId); + assertEquals(100, proc0.coresReserved); + assertEquals(3355443, proc0.memoryReserved); + assertEquals(6, proc0.gpusReserved); + assertEquals(1048576, proc0.gpuMemoryReserved); + } +} + diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/DispatchSupportTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/DispatchSupportTests.java index baa2353fb..98c60fd9c 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/DispatchSupportTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/DispatchSupportTests.java @@ -110,8 +110,8 @@ public void createHost() { .setState(HardwareState.UP) .setFacility("spi") .putAttributes("SP_OS", "Linux") - .putAttributes("freeGpu", String.format("%d", CueUtil.MB512)) - .putAttributes("totalGpu", String.format("%d", CueUtil.MB512)) + .setFreeGpuMem((int) CueUtil.MB512) + .setTotalGpuMem((int) CueUtil.MB512) .build(); hostManager.createHost(host, diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java new file mode 100644 index 000000000..8888e0453 --- /dev/null +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java @@ -0,0 +1,236 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.imageworks.spcue.test.dispatcher; + +import java.io.File; +import java.util.List; +import javax.annotation.Resource; + +import org.junit.Before; +import org.junit.Test; +import org.springframework.test.annotation.Rollback; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.transaction.annotation.Transactional; + +import com.imageworks.spcue.DispatchHost; +import com.imageworks.spcue.FrameInterface; +import com.imageworks.spcue.JobDetail; +import com.imageworks.spcue.LayerDetail; +import com.imageworks.spcue.VirtualProc; +import com.imageworks.spcue.dao.LayerDao; +import com.imageworks.spcue.dispatcher.Dispatcher; +import com.imageworks.spcue.dispatcher.FrameCompleteHandler; +import com.imageworks.spcue.grpc.host.HardwareState; +import com.imageworks.spcue.grpc.report.FrameCompleteReport; +import com.imageworks.spcue.grpc.report.RenderHost; +import com.imageworks.spcue.grpc.report.RunningFrameInfo; +import com.imageworks.spcue.service.AdminManager; +import com.imageworks.spcue.service.HostManager; +import com.imageworks.spcue.service.JobLauncher; +import com.imageworks.spcue.service.JobManager; +import com.imageworks.spcue.test.TransactionalTest; +import com.imageworks.spcue.util.CueUtil; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +@ContextConfiguration +public class FrameCompleteHandlerTests extends TransactionalTest { + + @Resource + AdminManager adminManager; + + @Resource + FrameCompleteHandler frameCompleteHandler; + + @Resource + HostManager hostManager; + + @Resource + JobLauncher jobLauncher; + + @Resource + JobManager jobManager; + + @Resource + LayerDao layerDao; + + @Resource + Dispatcher dispatcher; + + private static final String HOSTNAME = "beta"; + + @Before + public void setTestMode() { + dispatcher.setTestMode(true); + } + + @Before + public void launchJob() { + jobLauncher.testMode = true; + jobLauncher.launch( + new File("src/test/resources/conf/jobspec/jobspec_gpus_test.xml")); + } + + @Before + public void createHost() { + RenderHost host = RenderHost.newBuilder() + .setName(HOSTNAME) + .setBootTime(1192369572) + .setFreeMcp(76020) + .setFreeMem((int) CueUtil.GB8) + .setFreeSwap(20760) + .setLoad(0) + .setTotalMcp(195430) + .setTotalMem(CueUtil.GB8) + .setTotalSwap(CueUtil.GB2) + .setNimbyEnabled(false) + .setNumProcs(40) + .setCoresPerProc(100) + .setState(HardwareState.UP) + .setFacility("spi") + .putAttributes("SP_OS", "Linux") + .setNumGpus(8) + .setFreeGpuMem(CueUtil.GB16 * 8) + .setTotalGpuMem(CueUtil.GB16 * 8) + .build(); + + hostManager.createHost(host, + adminManager.findAllocationDetail("spi", "general")); + } + + public DispatchHost getHost() { + return hostManager.findDispatchHost(HOSTNAME); + } + + @Test + @Transactional + @Rollback(true) + public void testGpuReport() { + JobDetail job = jobManager.findJobDetail("pipe-default-testuser_test0"); + LayerDetail layer = layerDao.findLayerDetail(job, "layer0"); + jobManager.setJobPaused(job, false); + + DispatchHost host = getHost(); + List procs = dispatcher.dispatchHost(host); + assertEquals(1, procs.size()); + VirtualProc proc = procs.get(0); + + assertEquals(7, host.idleGpus); + assertEquals(CueUtil.GB16 * 8 - CueUtil.GB, host.idleGpuMemory); + + RunningFrameInfo info = RunningFrameInfo.newBuilder() + .setJobId(proc.getJobId()) + .setLayerId(proc.getLayerId()) + .setFrameId(proc.getFrameId()) + .setResourceId(proc.getProcId()) + .build(); + FrameCompleteReport report = FrameCompleteReport.newBuilder() + .setFrame(info) + .setExitStatus(0) + .build(); + frameCompleteHandler.handleFrameCompleteReport(report); + + assertTrue(jobManager.isLayerComplete(layer)); + assertTrue(jobManager.isJobComplete(job)); + } + + @Test + @Transactional + @Rollback(true) + public void testGpuReportMultiple() { + JobDetail job0 = jobManager.findJobDetail("pipe-default-testuser_test0"); + LayerDetail layer0_0 = layerDao.findLayerDetail(job0, "layer0"); + jobManager.setJobPaused(job0, false); + + JobDetail job1 = jobManager.findJobDetail("pipe-default-testuser_test1"); + LayerDetail layer1_0 = layerDao.findLayerDetail(job1, "layer0"); + jobManager.setJobPaused(job1, false); + + DispatchHost host = getHost(); + List procs = dispatcher.dispatchHost(host); + assertEquals(2, procs.size()); + + assertEquals(4, host.idleGpus); + assertEquals(CueUtil.GB16 * 8 - CueUtil.GB2, host.idleGpuMemory); + + for (VirtualProc proc : procs) { + RunningFrameInfo info = RunningFrameInfo.newBuilder() + .setJobId(proc.getJobId()) + .setLayerId(proc.getLayerId()) + .setFrameId(proc.getFrameId()) + .setResourceId(proc.getProcId()) + .build(); + FrameCompleteReport report = FrameCompleteReport.newBuilder() + .setFrame(info) + .setExitStatus(0) + .build(); + frameCompleteHandler.handleFrameCompleteReport(report); + } + + assertTrue(jobManager.isLayerComplete(layer0_0)); + assertTrue(jobManager.isJobComplete(job0)); + assertTrue(jobManager.isLayerComplete(layer1_0)); + assertTrue(jobManager.isJobComplete(job1)); + } + + @Test + @Transactional + @Rollback(true) + public void testGpuReportOver() { + JobDetail job1 = jobManager.findJobDetail("pipe-default-testuser_test1"); + LayerDetail layer1_0 = layerDao.findLayerDetail(job1, "layer0"); + jobManager.setJobPaused(job1, false); + + JobDetail job2 = jobManager.findJobDetail("pipe-default-testuser_test2"); + LayerDetail layer2_0 = layerDao.findLayerDetail(job2, "layer0"); + jobManager.setJobPaused(job2, false); + + DispatchHost host = getHost(); + List procs = dispatcher.dispatchHost(host); + assertEquals(1, procs.size()); + + assertTrue(host.idleGpus == 5 || host.idleGpus == 2); + assertEquals(CueUtil.GB16 * 8 - CueUtil.GB, host.idleGpuMemory); + + for (VirtualProc proc : procs) { + RunningFrameInfo info = RunningFrameInfo.newBuilder() + .setJobId(proc.getJobId()) + .setLayerId(proc.getLayerId()) + .setFrameId(proc.getFrameId()) + .setResourceId(proc.getProcId()) + .build(); + FrameCompleteReport report = FrameCompleteReport.newBuilder() + .setFrame(info) + .setExitStatus(0) + .build(); + frameCompleteHandler.handleFrameCompleteReport(report); + } + + assertEquals(1, + (jobManager.isLayerComplete(layer1_0) ? 1 : 0) + + (jobManager.isLayerComplete(layer2_0) ? 1 : 0)); + assertEquals(1, + (jobManager.isJobComplete(job1) ? 1 : 0) + + (jobManager.isJobComplete(job2) ? 1 : 0)); + } +} + diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java new file mode 100644 index 000000000..dee9d0792 --- /dev/null +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java @@ -0,0 +1,124 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.imageworks.spcue.test.dispatcher; + +import javax.annotation.Resource; + +import org.junit.Before; +import org.junit.Test; +import org.springframework.test.annotation.Rollback; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.transaction.annotation.Transactional; + +import com.imageworks.spcue.DispatchHost; +import com.imageworks.spcue.dispatcher.Dispatcher; +import com.imageworks.spcue.dispatcher.HostReportHandler; +import com.imageworks.spcue.grpc.host.HardwareState; +import com.imageworks.spcue.grpc.host.LockState; +import com.imageworks.spcue.grpc.report.CoreDetail; +import com.imageworks.spcue.grpc.report.HostReport; +import com.imageworks.spcue.grpc.report.RenderHost; +import com.imageworks.spcue.service.AdminManager; +import com.imageworks.spcue.service.HostManager; +import com.imageworks.spcue.test.TransactionalTest; +import com.imageworks.spcue.util.CueUtil; + +import static org.junit.Assert.assertEquals; + +@ContextConfiguration +public class HostReportHandlerGpuTests extends TransactionalTest { + + @Resource + AdminManager adminManager; + + @Resource + HostManager hostManager; + + @Resource + HostReportHandler hostReportHandler; + + @Resource + Dispatcher dispatcher; + + private static final String HOSTNAME = "beta"; + + @Before + public void setTestMode() { + dispatcher.setTestMode(true); + } + + private static CoreDetail getCoreDetail(int total, int idle, int booked, int locked) { + return CoreDetail.newBuilder() + .setTotalCores(total) + .setIdleCores(idle) + .setBookedCores(booked) + .setLockedCores(locked) + .build(); + } + + private DispatchHost getHost() { + return hostManager.findDispatchHost(HOSTNAME); + } + + private static RenderHost getRenderHost() { + return RenderHost.newBuilder() + .setName(HOSTNAME) + .setBootTime(1192369572) + .setFreeMcp(76020) + .setFreeMem(53500) + .setFreeSwap(20760) + .setLoad(0) + .setTotalMcp(195430) + .setTotalMem(1048576L * 4096) + .setTotalSwap(20960) + .setNimbyEnabled(false) + .setNumProcs(2) + .setCoresPerProc(100) + .addTags("test") + .setState(HardwareState.UP) + .setFacility("spi") + .putAttributes("SP_OS", "Linux") + .setNumGpus(64) + .setFreeGpuMem(1048576L * 2000) + .setTotalGpuMem(1048576L * 2048) + .build(); + } + + @Test + @Transactional + @Rollback(true) + public void testHandleHostReport() { + CoreDetail cores = getCoreDetail(200, 200, 0, 0); + HostReport report = HostReport.newBuilder() + .setHost(getRenderHost()) + .setCoreInfo(cores) + .build(); + + hostReportHandler.handleHostReport(report, true); + DispatchHost host = getHost(); + assertEquals(host.lockState, LockState.OPEN); + assertEquals(host.memory, 4294443008L); + assertEquals(host.gpus, 64); + assertEquals(host.idleGpus, 64); + assertEquals(host.gpuMemory, 1048576L * 2048); + assertEquals(host.idleGpuMemory, 2147483648L); + } +} + diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java index f24375cce..f4d706cdb 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java @@ -104,8 +104,8 @@ private static RenderHost getRenderHost() { .setState(HardwareState.UP) .setFacility("spi") .putAttributes("SP_OS", "Linux") - .putAttributes("freeGpu", String.format("%d", CueUtil.MB512)) - .putAttributes("totalGpu", String.format("%d", CueUtil.MB512)) + .setFreeGpuMem((int) CueUtil.MB512) + .setTotalGpuMem((int) CueUtil.MB512) .build(); } diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/LocalDispatcherTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/LocalDispatcherTests.java index 88f195111..97a270085 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/LocalDispatcherTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/LocalDispatcherTests.java @@ -109,8 +109,8 @@ public void createHost() { .setFacility("spi") .addTags("test") .putAttributes("SP_OS", "Linux") - .putAttributes("freeGpu", String.format("%d", CueUtil.MB512)) - .putAttributes("totalGpu", String.format("%d", CueUtil.MB512)) + .setFreeGpuMem((int) CueUtil.MB512) + .setTotalGpuMem((int) CueUtil.MB512) .build(); hostManager.createHost(host, @@ -171,7 +171,7 @@ public void testDispatchHostAutoDetectLayer() { JobDetail job = getJob(); LayerInterface layer = jobManager.getLayers(job).get(0); - LocalHostAssignment lba = new LocalHostAssignment(300, 1, CueUtil.GB8, 1); + LocalHostAssignment lba = new LocalHostAssignment(300, 1, CueUtil.GB8, 0, 0); bookingManager.createLocalHostAssignment(host, layer, lba); List procs = localDispatcher.dispatchHost(host); @@ -205,7 +205,7 @@ public void testDispatchHostAutoDetectFrame() { LayerInterface layer = jobManager.getLayers(job).get(0); FrameInterface frame = jobManager.findFrame(layer, 5); - LocalHostAssignment lba = new LocalHostAssignment(200, 1, CueUtil.GB8, 1); + LocalHostAssignment lba = new LocalHostAssignment(200, 1, CueUtil.GB8, 0, 0); bookingManager.createLocalHostAssignment(host, frame, lba); List procs = localDispatcher.dispatchHost(host); @@ -228,7 +228,7 @@ public void testDispatchHostToLocalJob() { DispatchHost host = getHost(); JobDetail job = getJob(); - LocalHostAssignment lba = new LocalHostAssignment(200, 1, CueUtil.GB8, 1); + LocalHostAssignment lba = new LocalHostAssignment(200, 1, CueUtil.GB8, 0, 0); bookingManager.createLocalHostAssignment(host, job, lba); List procs = localDispatcher.dispatchHost(host, job); @@ -258,7 +258,7 @@ public void testDispatchHostToLocalLayer() { JobDetail job = getJob(); LayerInterface layer = jobManager.getLayers(job).get(0); - LocalHostAssignment lba = new LocalHostAssignment(300, 1, CueUtil.GB8, 1); + LocalHostAssignment lba = new LocalHostAssignment(300, 1, CueUtil.GB8, 0, 0); bookingManager.createLocalHostAssignment(host, layer, lba); List procs = localDispatcher.dispatchHost(host, layer); @@ -292,7 +292,7 @@ public void testDispatchHostToLocalFrame() { LayerInterface layer = jobManager.getLayers(job).get(0); FrameInterface frame = jobManager.findFrame(layer, 5); - LocalHostAssignment lba = new LocalHostAssignment(200, 1, CueUtil.GB8, 1); + LocalHostAssignment lba = new LocalHostAssignment(200, 1, CueUtil.GB8, 0, 0); bookingManager.createLocalHostAssignment(host, frame, lba); List procs = localDispatcher.dispatchHost(host, frame); @@ -317,7 +317,7 @@ public void testDispatchHostToLocalFrameTwice() { LayerInterface layer = jobManager.getLayers(job).get(0); FrameInterface frame = jobManager.findFrame(layer, 5); - LocalHostAssignment lba = new LocalHostAssignment(200, 1, CueUtil.GB8, 1); + LocalHostAssignment lba = new LocalHostAssignment(200, 1, CueUtil.GB8, 0, 0); bookingManager.createLocalHostAssignment(host, frame, lba); List procs = localDispatcher.dispatchHost(host, frame); @@ -345,7 +345,7 @@ public void testDispatchHostToLocalJobDeficit() { DispatchHost host = getHost(); JobDetail job = getJob(); - LocalHostAssignment lba = new LocalHostAssignment(800, 8, CueUtil.GB8, 1); + LocalHostAssignment lba = new LocalHostAssignment(800, 8, CueUtil.GB8, 0, 0); bookingManager.createLocalHostAssignment(host, job, lba); List procs = localDispatcher.dispatchHost(host, job); @@ -365,7 +365,7 @@ public void testDispatchHostToLocalJobDeficit() { * Now, lower our min cores to create a deficit. */ assertFalse(bookingManager.hasResourceDeficit(host)); - bookingManager.setMaxResources(lba, 700, 0, 1); + bookingManager.setMaxResources(lba, 700, 0, 0, 0); assertTrue(bookingManager.hasResourceDeficit(host)); } } diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java index a34cc1d3e..7502e0687 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java @@ -74,8 +74,8 @@ public void create() { .setState(HardwareState.UP) .setFacility("spi") .addAllTags(ImmutableList.of("mcore", "4core", "8g")) - .putAttributes("freeGpu", String.format("%d", CueUtil.MB512)) - .putAttributes("totalGpu", String.format("%d", CueUtil.MB512)) + .setFreeGpuMem((int) CueUtil.MB512) + .setTotalGpuMem((int) CueUtil.MB512) .build(); hostManager.createHost(host); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/BookingManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/BookingManagerTests.java index e94705898..9b6813c33 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/BookingManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/BookingManagerTests.java @@ -125,8 +125,8 @@ public DispatchHost createHost() { .setState(HardwareState.UP) .setFacility("spi") .addTags("general") - .putAttributes("freeGpu", String.format("%d", CueUtil.MB512)) - .putAttributes("totalGpu", String.format("%d", CueUtil.MB512)) + .setFreeGpuMem((int) CueUtil.MB512) + .setTotalGpuMem((int) CueUtil.MB512) .build(); DispatchHost dh = hostManager.createHost(host); @@ -319,23 +319,23 @@ public void setMaxResources() { /* * Lower the cores. */ - bookingManager.setMaxResources(lja, 100, CueUtil.GB2, CueUtil.MB256); + bookingManager.setMaxResources(lja, 100, CueUtil.GB2, 1, CueUtil.MB256); LocalHostAssignment l2 = bookingManager.getLocalHostAssignment(lja.id); assertEquals(100, l2.getMaxCoreUnits()); assertEquals(CueUtil.GB2, l2.getMaxMemory()); - assertEquals(CueUtil.MB256, l2.getMaxGpu()); + assertEquals(CueUtil.MB256, l2.getMaxGpuMemory()); /* * Raise the values. */ - bookingManager.setMaxResources(lja, 200, CueUtil.GB4, CueUtil.MB512); + bookingManager.setMaxResources(lja, 200, CueUtil.GB4, 1, CueUtil.MB512); l2 = bookingManager.getLocalHostAssignment(lja.id); assertEquals(200, l2.getMaxCoreUnits()); assertEquals(CueUtil.GB4, l2.getMaxMemory()); - assertEquals(CueUtil.MB512, l2.getMaxGpu()); + assertEquals(CueUtil.MB512, l2.getMaxGpuMemory()); } @Test @@ -351,7 +351,7 @@ public void setIllegalMaxResources() { LocalHostAssignment lja = new LocalHostAssignment(); lja.setMaxCoreUnits(200); lja.setMaxMemory(CueUtil.GB4); - lja.setMaxGpu(CueUtil.MB512); + lja.setMaxGpuMemory(CueUtil.MB512); lja.setThreads(2); bookingManager.createLocalHostAssignment(h, j, lja); @@ -359,7 +359,7 @@ public void setIllegalMaxResources() { /* * Raise the cores too high */ - bookingManager.setMaxResources(lja, 800, CueUtil.GB2, 0); + bookingManager.setMaxResources(lja, 800, CueUtil.GB2, 0, 0); } @Test diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/HostManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/HostManagerTests.java index ed89219da..cf86e5362 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/HostManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/HostManagerTests.java @@ -114,8 +114,8 @@ public DispatchHost createHost() { .setState(HardwareState.UP) .setFacility("spi") .addAllTags(ImmutableList.of("linux", "64bit")) - .putAttributes("freeGpu", "512") - .putAttributes("totalGpu", "512") + .setFreeGpuMem((int) CueUtil.MB512) + .setTotalGpuMem((int) CueUtil.MB512) .build(); hostDao.insertRenderHost(host, diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/JobManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/JobManagerTests.java index b2446fe20..3be56bf06 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/JobManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/JobManagerTests.java @@ -463,7 +463,7 @@ public void optimizeLayer() { .stream() .limit(5) .forEach(frame -> frameDao.updateFrameState(frame, FrameState.SUCCEEDED)); - layerDao.updateUsage(layer, new ResourceUsage(100, 3500 * 5), 0); + layerDao.updateUsage(layer, new ResourceUsage(100, 3500 * 5, 0), 0); // Test to make sure our optimization jobManager.optimizeLayer(layer, 100, CueUtil.MB512, 120); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/JobSpecTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/JobSpecTests.java index e68daa551..d3f94dce7 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/JobSpecTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/JobSpecTests.java @@ -29,6 +29,8 @@ import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; import org.springframework.test.context.support.AnnotationConfigContextLoader; +import com.imageworks.spcue.BuildableJob; +import com.imageworks.spcue.LayerDetail; import com.imageworks.spcue.SpecBuilderException; import com.imageworks.spcue.config.TestAppConfig; import com.imageworks.spcue.service.JobLauncher; @@ -95,4 +97,21 @@ public void testParseInvalidShot() { "Shot names must be alpha numeric, no dashes or punctuation."); } } + + @Test + public void testParseGpuSuccess() { + String xml = readJobSpec("jobspec_1_12.xml"); + JobSpec spec = jobLauncher.parse(xml); + assertEquals(spec.getDoc().getDocType().getPublicID(), + "SPI Cue Specification Language"); + assertEquals(spec.getDoc().getDocType().getSystemID(), + "http://localhost:8080/spcue/dtd/cjsl-1.12.dtd"); + assertEquals(spec.getJobs().size(), 1); + BuildableJob job = spec.getJobs().get(0); + assertEquals(job.detail.name, "testing-default-testuser_test"); + LayerDetail layer = job.getBuildableLayers().get(0).layerDetail; + assertEquals(layer.getMinimumGpus(), 1); + assertEquals(layer.getMinimumGpuMemory(), 1048576); + } + } diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/OwnerManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/OwnerManagerTests.java index e11bab099..224dcac75 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/OwnerManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/OwnerManagerTests.java @@ -82,8 +82,8 @@ public DispatchHost createHost() { .setState(HardwareState.UP) .setFacility("spi") .addTags("general") - .putAttributes("freeGpu", String.format("%d", CueUtil.MB512)) - .putAttributes("totalGpu", String.format("%d", CueUtil.MB512)) + .setFreeGpuMem((int) CueUtil.MB512) + .setTotalGpuMem((int) CueUtil.MB512) .build(); DispatchHost dh = hostManager.createHost(host); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/ServiceManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/ServiceManagerTests.java index 3573cbe59..5354d763e 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/ServiceManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/ServiceManagerTests.java @@ -84,7 +84,7 @@ public void testCreateService() { s.name = "dillweed"; s.minCores = 100; s.minMemory = CueUtil.GB4; - s.minGpu = CueUtil.GB2; + s.minGpuMemory = CueUtil.GB2; s.threadable = false; s.timeout = 0; s.timeout_llu = 0; @@ -105,7 +105,7 @@ public void testOverrideExistingService() { s.timeout = 10; s.timeout_llu = 10; s.minMemory = CueUtil.GB8; - s.minGpu = CueUtil.GB2; + s.minGpuMemory = CueUtil.GB2; s.threadable = false; s.tags.addAll(Sets.newHashSet("general")); s.showId = "00000000-0000-0000-0000-000000000000"; @@ -118,7 +118,7 @@ public void testOverrideExistingService() { assertEquals(10, newService.timeout); assertEquals(10, newService.timeout_llu); assertEquals(CueUtil.GB8, newService.minMemory); - assertEquals(CueUtil.GB2, newService.minGpu); + assertEquals(CueUtil.GB2, newService.minGpuMemory); assertFalse(newService.threadable); assertTrue(s.tags.contains("general")); @@ -127,7 +127,7 @@ public void testOverrideExistingService() { // now check the original is back. newService = serviceManager.getService("arnold", s.showId); assertEquals(100, newService.minCores); - assertEquals(0, newService.minGpu); + assertEquals(0, newService.minGpuMemory); } @Test @@ -151,7 +151,7 @@ public void testJobLaunch() { assertEquals(shell.minCores, shellLayer.minimumCores); assertEquals(shell.minMemory, shellLayer.minimumMemory); - assertEquals(shell.minGpu, shellLayer.minimumGpu); + assertEquals(shell.minGpuMemory, shellLayer.minimumGpuMemory); assertFalse(shellLayer.isThreadable); assertEquals(shell.tags, shellLayer.tags); assertThat(shellLayer.services, contains("shell", "katana", "unknown")); @@ -164,7 +164,7 @@ public void testJobLaunch() { assertEquals(cuda.minCores, cudaLayer.minimumCores); assertEquals(cuda.minMemory, cudaLayer.minimumMemory); - assertEquals(cuda.minGpu, cudaLayer.minimumGpu); + assertEquals(cuda.minGpuMemory, cudaLayer.minimumGpuMemory); assertFalse(cudaLayer.isThreadable); assertEquals(cuda.tags, cudaLayer.tags); assertThat(cudaLayer.services, contains("cuda")); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/util/CueUtilTester.java b/cuebot/src/test/java/com/imageworks/spcue/test/util/CueUtilTester.java index 9bfc19e41..d3a4abe76 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/util/CueUtilTester.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/util/CueUtilTester.java @@ -154,9 +154,9 @@ public void testCoreUnitsToCoresWithScale() { @Test public void testBuildProcName() { - assertEquals("drack100/1.00", CueUtil.buildProcName("drack100",100)); - assertEquals("drack100/1.40", CueUtil.buildProcName("drack100",140)); - assertEquals("drack100/2.01", CueUtil.buildProcName("drack100",201)); + assertEquals("drack100/1.00/1", CueUtil.buildProcName("drack100",100,1)); + assertEquals("drack100/1.40/0", CueUtil.buildProcName("drack100",140,0)); + assertEquals("drack100/2.01/2", CueUtil.buildProcName("drack100",201,2)); } @Test diff --git a/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql b/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql index b5596b91d..14c56afcc 100644 --- a/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql +++ b/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql @@ -102,7 +102,7 @@ Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN, Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS) values ('AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAA15','makemovie',false,50,1048576,'util') -Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS,INT_CORES_MAX,INT_GPU_MIN) values ('488c75f0-eae4-4dd0-83e0-29b982adbbff','cuda',true,100,3354624,'cuda',0,262144) +Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS,INT_CORES_MAX,INT_GPU_MEM_MIN) values ('488c75f0-eae4-4dd0-83e0-29b982adbbff','cuda',true,100,3354624,'cuda',0,262144) Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000005','MAX_FRAME_RETRIES',16,0,null,false) diff --git a/cuebot/src/test/resources/conf/dtd/cjsl-1.12.dtd b/cuebot/src/test/resources/conf/dtd/cjsl-1.12.dtd new file mode 100644 index 000000000..222e04cfc --- /dev/null +++ b/cuebot/src/test/resources/conf/dtd/cjsl-1.12.dtd @@ -0,0 +1,97 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cuebot/src/test/resources/conf/jobspec/jobspec_1_12.xml b/cuebot/src/test/resources/conf/jobspec/jobspec_1_12.xml new file mode 100644 index 000000000..65036370b --- /dev/null +++ b/cuebot/src/test/resources/conf/jobspec/jobspec_1_12.xml @@ -0,0 +1,49 @@ + + + + + + + + + local + testing + default + testuser + 9860 + + + False + 2 + False + + + + echo $CUE_GPU_CORES + 1-10 + 1 + 1 + 1 + + + shell + + + + + + diff --git a/cuebot/src/test/resources/conf/jobspec/jobspec_dispatch_gpus_test.xml b/cuebot/src/test/resources/conf/jobspec/jobspec_dispatch_gpus_test.xml new file mode 100644 index 000000000..dd3ce459b --- /dev/null +++ b/cuebot/src/test/resources/conf/jobspec/jobspec_dispatch_gpus_test.xml @@ -0,0 +1,133 @@ + + + + + + + + + spi + pipe + default + testuser + 9860 + + + True + 2 + False + + + + echo $CUE_GPU_CORES + 1-10 + 1 + 1 + + + shell + + + + + echo $CUE_GPU_CORES + 1-10 + 1 + 1 + + + shell + + + + + echo $CUE_GPU_CORES + 1-10 + 1 + 1 + 1 + + + shell + + + + + echo $CUE_GPU_CORES + 1-10 + 1 + 4 + 7g + + + shell + + + + + + + True + 2 + False + + + + echo $CUE_GPU_CORES + 1-10 + 1 + 6 + 1 + + + shell + + + + + echo $CUE_GPU_CORES + 1-10 + 1 + 3 + 7g + + + shell + + + + + + + True + 2 + False + + + + echo CPU + 1-10 + 1 + + + shell + + + + + + diff --git a/cuebot/src/test/resources/conf/jobspec/jobspec_gpus_test.xml b/cuebot/src/test/resources/conf/jobspec/jobspec_gpus_test.xml new file mode 100644 index 000000000..3b6b00ae7 --- /dev/null +++ b/cuebot/src/test/resources/conf/jobspec/jobspec_gpus_test.xml @@ -0,0 +1,76 @@ + + + + + + + + + spi + pipe + default + testuser + 9860 + + + True + + + true + 0 + 1 + 1 + 1 + + shell + + + + + + + True + + + true + 0 + 1 + 3 + 1 + + shell + + + + + + + True + + + true + 0 + 1 + 6 + 1 + + shell + + + + + diff --git a/cuegui/cuegui/CueJobMonitorTree.py b/cuegui/cuegui/CueJobMonitorTree.py index 8d1307771..3c8fe9633 100644 --- a/cuegui/cuegui/CueJobMonitorTree.py +++ b/cuegui/cuegui/CueJobMonitorTree.py @@ -98,53 +98,71 @@ def __init__(self, parent): data=lambda job: "%.02f" % job.data.job_stats.reserved_cores, sort=lambda job: job.data.job_stats.reserved_cores, tip="The number of reserved cores.") - self.addColumn("Wait", 45, id=6, + self.addColumn("Gpus", 55, id=6, + data=lambda job: "%d" % job.data.job_stats.reserved_gpus, + sort=lambda job: job.data.job_stats.reserved_gpus, + tip="The number of reserved gpus.") + self.addColumn("Wait", 45, id=7, data=lambda job: job.data.job_stats.waiting_frames, sort=lambda job: job.data.job_stats.waiting_frames, tip="The number of waiting frames.") - self.addColumn("Depend", 55, id=7, + self.addColumn("Depend", 55, id=8, data=lambda job: job.data.job_stats.depend_frames, sort=lambda job: job.data.job_stats.depend_frames, tip="The number of dependent frames.") - self.addColumn("Total", 50, id=8, + self.addColumn("Total", 50, id=9, data=lambda job: job.data.job_stats.total_frames, sort=lambda job: job.data.job_stats.total_frames, tip="The total number of frames.") - self.addColumn("_Booking Bar", 150, id=9, + self.addColumn("_Booking Bar", 150, id=10, delegate=cuegui.ItemDelegate.JobBookingBarDelegate) - self.addColumn("Min", 38, id=10, + self.addColumn("Min", 38, id=11, data=lambda job: "%.0f" % job.data.min_cores, sort=lambda job: job.data.min_cores, tip="The minimum number of running cores that the cuebot\n" "will try to maintain.") - self.addColumn("Max", 38, id=11, + self.addColumn("Max", 38, id=12, data=lambda job: "%.0f" % job.data.max_cores, sort=lambda job: job.data.max_cores, tip="The maximum number of running cores that the cuebot\n" "will allow.") + self.addColumn("Min Gpus", 38, id=13, + data=lambda job: "%d" % job.data.min_gpus, + sort=lambda job: job.data.min_gpus, + tip="The minimum number of running gpus that the cuebot\n" + "will try to maintain.") + self.addColumn("Max Gpus", 38, id=14, + data=lambda job: "%d" % job.data.max_gpus, + sort=lambda job: job.data.max_gpus, + tip="The maximum number of running gpus that the cuebot\n" + "will allow.") self.addColumn( - "Age", 50, id=12, + "Age", 50, id=15, data=lambda job: cuegui.Utils.secondsToHHHMM(self.currtime - job.data.start_time), sort=lambda job: self.currtime - job.data.start_time, tip="The HOURS:MINUTES since the job was launched.") - self.addColumn("Pri", 30, id=13, + self.addColumn("Pri", 30, id=16, data=lambda job: job.data.priority, sort=lambda job: job.data.priority, tip="The job priority. The cuebot uses this as a suggestion\n" "to determine what job needs the next available matching\n" "resource.") - self.addColumn("ETA", 65, id=14, + self.addColumn("ETA", 65, id=17, data=lambda job: "", tip="(Inacurate and disabled until a better solution exists)\n" "A very rough estimate of the number of HOURS:MINUTES\n" "it will be before the entire job is done.") - self.addColumn("MaxRss", 60, id=15, + self.addColumn("MaxRss", 60, id=18, data=lambda job: cuegui.Utils.memoryToString(job.data.job_stats.max_rss), sort=lambda job: job.data.job_stats.max_rss, tip="The most memory used at one time by any single frame.") - self.addColumn("_Blank", 20, id=16, + self.addColumn("MaxGpuMem", 60, id=19, + data=lambda job: cuegui.Utils.memoryToString(job.data.job_stats.max_gpu_mem), + sort=lambda job: job.data.job_stats.max_gpu_mem, + tip="The most gpu memory used at one time by any single frame.") + self.addColumn("_Blank", 20, id=20, tip="Spacer") - self.addColumn("Progress", 0, id=17, + self.addColumn("Progress", 0, id=21, delegate=cuegui.ItemDelegate.JobThinProgressBarDelegate, tip="A visual overview of the job progress.\n" "Green \t is succeeded\n" @@ -164,23 +182,31 @@ def __init__(self, parent): self.addColumn("", 0, id=5, data=lambda group: "%.2f" % group.data.stats.reserved_cores) self.addColumn("", 0, id=6, + data=lambda group: "%d" % group.data.stats.reserved_gpus) + self.addColumn("", 0, id=7, data=lambda group: group.data.stats.waiting_frames) - self.addColumn("", 0, id=7) self.addColumn("", 0, id=8) - self.addColumn("", 0, id=9, - data=lambda group: (group.data.min_cores or "")) + self.addColumn("", 0, id=9) self.addColumn("", 0, id=10, + data=lambda group: (group.data.min_cores or "")) + self.addColumn("", 0, id=11, data=lambda group: ( group.data.max_cores > 0 and group.data.max_cores or "")) - self.addColumn("", 0, id=11) - self.addColumn("", 0, id=12) - self.addColumn("", 0, id=13) + self.addColumn("", 0, id=12, + data=lambda group: (group.data.min_gpus or "")) + self.addColumn("", 0, id=13, + data=lambda group: ( + group.data.max_gpus > 0 and group.data.max_gpus or "")) self.addColumn("", 0, id=14) self.addColumn("", 0, id=15) - self.addColumn("", 0, id=16, + self.addColumn("", 0, id=16) + self.addColumn("", 0, id=17) + self.addColumn("", 0, id=18) + self.addColumn("", 0, id=19) + self.addColumn("", 0, id=20, data=lambda group: (group.data.department != "Unknown" and group.data.department or "")) - self.addColumn("", 0, id=17) + self.addColumn("", 0, id=21) cuegui.AbstractTreeWidget.AbstractTreeWidget.__init__(self, parent) @@ -528,6 +554,8 @@ def contextMenuEvent(self, e): menu.addSeparator() self.__menuActions.jobs().addAction(menu, "setMinCores") self.__menuActions.jobs().addAction(menu, "setMaxCores") + self.__menuActions.jobs().addAction(menu, "setMinGpu") + self.__menuActions.jobs().addAction(menu, "setMaxGpu") self.__menuActions.jobs().addAction(menu, "setPriority") self.__menuActions.jobs().addAction(menu, "setMaxRetries") if counts["job"] == 1: diff --git a/cuegui/cuegui/FrameMonitorTree.py b/cuegui/cuegui/FrameMonitorTree.py index be3362095..41df65209 100644 --- a/cuegui/cuegui/FrameMonitorTree.py +++ b/cuegui/cuegui/FrameMonitorTree.py @@ -100,25 +100,29 @@ def __init__(self, parent): data=lambda job, frame: (self.getCores(frame, format_as_string=True) or ""), sort=lambda job, frame: (self.getCores(frame)), tip="The number of cores a frame is using") - self.addColumn("Host", 120, id=6, + self.addColumn("GPUs", 55, id=6, + data=lambda job, frame: (self.getGpus(frame, format_as_string=True) or ""), + sort=lambda job, frame: (self.getGpus(frame)), + tip="The number of gpus a frame is using") + self.addColumn("Host", 120, id=7, data=lambda job, frame: frame.data.last_resource, sort=lambda job, frame: frame.data.last_resource, tip="The last or current resource that the frame used or is using.") - self.addColumn("Retries", 55, id=7, + self.addColumn("Retries", 55, id=8, data=lambda job, frame: frame.data.retry_count, sort=lambda job, frame: frame.data.retry_count, tip="The number of times that each frame has had to retry.") - self.addColumn("_CheckpointEnabled", 20, id=8, + self.addColumn("_CheckpointEnabled", 20, id=9, data=lambda job, frame: "", sort=lambda job, frame: ( frame.data.checkpoint_state == opencue.api.job_pb2.ENABLED), tip="A green check mark here indicates the frame has written out at least " "1 checkpoint segment.") - self.addColumn("CheckP", 55, id=9, + self.addColumn("CheckP", 55, id=10, data=lambda job, frame: frame.data.checkpoint_count, sort=lambda job, frame: frame.data.checkpoint_count, tip="The number of times a frame has been checkpointed.") - self.addColumn("Runtime", 70, id=10, + self.addColumn("Runtime", 70, id=11, data=lambda job, frame: (cuegui.Utils.secondsToHMMSS( frame.data.start_time and frame.data.stop_time and @@ -138,7 +142,7 @@ def __init__(self, parent): tip="The amount of HOURS:MINUTES:SECONDS that the frame\n" "has run for or last ran for.\n") - self.addColumn("LLU", 70, id=11, + self.addColumn("LLU", 70, id=12, data=lambda job, frame: (frame.data.state == opencue.api.job_pb2.RUNNING and self.frameLogDataBuffer.getLastLineData( job, frame)[FrameLogDataBuffer.LLU] or ""), @@ -150,7 +154,7 @@ def __init__(self, parent): "time without an update is an indication of a stuck\n" "frame for most types of jobs") - self.addColumn("Memory", 60, id=12, + self.addColumn("Memory", 60, id=13, data=lambda job, frame: ( frame.data.state == opencue.api.job_pb2.RUNNING and cuegui.Utils.memoryToString(frame.data.used_memory) or @@ -162,7 +166,20 @@ def __init__(self, parent): "If a frame is not running:\n" "\t The most memory this frame has used at one time.") - self.addColumn("Remain", 70, id=13, + self.addColumn("GPU Memory", 60, id=14, + data=lambda job, frame: ( + frame.data.state == opencue.api.job_pb2.RUNNING and + cuegui.Utils.memoryToString(frame.data.used_gpu_memory) or + cuegui.Utils.memoryToString(frame.data.max_gpu_memory)), + sort=lambda job, frame: (frame.data.state == opencue.api.job_pb2.RUNNING and + frame.data.used_gpu_memory or + frame.data.max_gpu_memory), + tip="If a frame is running:\n" + "\t The amount of GPU memory currently used by the frame.\n" + "If a frame is not running:\n" + "\t The most GPU memory this frame has used at one time.") + + self.addColumn("Remain", 70, id=15, data=lambda job, frame: (frame.data.state == opencue.api.job_pb2.RUNNING and self.frameEtaDataBuffer.getEtaFormatted(job, frame) or ""), @@ -170,16 +187,16 @@ def __init__(self, parent): self.frameEtaDataBuffer.getEta(job, frame) or -1), tip="Hours:Minutes:Seconds remaining.") - self.addColumn("Start Time", 100, id=14, + self.addColumn("Start Time", 100, id=16, data=lambda job, frame: (self.getTimeString(frame.data.start_time) or ""), sort=lambda job, frame: (self.getTimeString(frame.data.start_time) or ""), tip="The time the frame was started or retried.") - self.addColumn("Stop Time", 100, id=15, + self.addColumn("Stop Time", 100, id=17, data=lambda job, frame: (self.getTimeString(frame.data.stop_time) or ""), sort=lambda job, frame: (self.getTimeString(frame.data.stop_time) or ""), tip="The time that the frame finished or died.") - self.addColumn("Last Line", 0, id=16, + self.addColumn("Last Line", 0, id=18, data=lambda job, frame: (frame.data.state == opencue.api.job_pb2.RUNNING and self.frameLogDataBuffer.getLastLineData( job, frame)[FrameLogDataBuffer.LASTLINE] or ""), @@ -240,7 +257,7 @@ def getCores(frame, format_as_string=False): """Gets the number of cores a frame is using.""" cores = None - m = re.search(r".*\/(\d+\.?\d*)", frame.data.last_resource) + m = re.search(r".*\/(\d+\.?\d*)\/.*", frame.data.last_resource) if m: cores = float(m.group(1)) @@ -249,6 +266,20 @@ def getCores(frame, format_as_string=False): return cores + @staticmethod + def getGpus(frame, format_as_string=False): + """Gets the number of gpus a frame is using.""" + gpus = None + + m = re.search(r".*\/.*\/(\d+)", frame.data.last_resource) + if m: + gpus = m.group(1) + + if not format_as_string: + gpus = int(gpus) + + return gpus + @staticmethod def getTimeString(timestamp): """Gets a timestamp formatted as a string.""" diff --git a/cuegui/cuegui/GroupDialog.py b/cuegui/cuegui/GroupDialog.py index 2c59f405c..2a38d906a 100644 --- a/cuegui/cuegui/GroupDialog.py +++ b/cuegui/cuegui/GroupDialog.py @@ -56,6 +56,11 @@ def __init__(self, parentGroup, modifyGroup, defaults, parent): __minCores = defaults["minCores"] __maxCores = defaults["maxCores"] + __defaultJobMinGpus = defaults["defaultJobMinGpus"] + __defaultJobMaxGpus = defaults["defaultJobMaxGpus"] + __minGpus = defaults["minGpus"] + __maxGpus = defaults["maxGpus"] + self.setWindowTitle(__title) layout.addWidget(QtWidgets.QLabel(__message, self), 0, 1, 1, 3) @@ -90,8 +95,25 @@ def __init__(self, parentGroup, modifyGroup, defaults, parent): __modify and __maxCores != -1.0, __maxCores, 1) + (self._defaultJobMinGpusCheck, self._defaultJobMinGpusValue) = \ + self.__createToggleSpinBox("Job Default Minimum Gpus", 8, + __modify and __defaultJobMinGpus != -1, + __defaultJobMinGpus, 1) + (self._defaultJobMaxGpusCheck, self._defaultJobMaxGpusValue) = \ + self.__createToggleSpinBox("Job Default Maximum Gpus", 9, + __modify and __defaultJobMaxGpus != -1, + __defaultJobMaxGpus, 1) + (self._minGpusCheck, self._minGpusValue) = \ + self.__createToggleSpinBox("Group Minimum Gpus", 10, + __modify and __minGpus != 0, + __minGpus) + (self._maxGpusCheck, self._maxGpusValue) = \ + self.__createToggleSpinBox("Group Maximum Gpus", 11, + __modify and __maxGpus != -1, + __maxGpus, 1) + self.__createButtons( - QtWidgets.QDialogButtonBox.Save | QtWidgets.QDialogButtonBox.Cancel, 8, 3) + QtWidgets.QDialogButtonBox.Save | QtWidgets.QDialogButtonBox.Cancel, 12, 3) def __createToggleDoubleSpinBox( self, text, row, startEnabled = False, currentValue = 0, minValue = 0): @@ -169,6 +191,26 @@ def accept(self): float(self._maxCoresValue.value()), __group.data.max_cores, float(-1)) + self.__setValue(self._defaultJobMinGpusCheck, + __group.setDefaultJobMinGpus, + float(self._defaultJobMinGpusValue.value()), + __group.data.default_job_min_gpus, -1) + + self.__setValue(self._defaultJobMaxGpusCheck, + __group.setDefaultJobMaxGpus, + float(self._defaultJobMaxGpusValue.value()), + __group.data.default_job_max_gpus, -1) + + self.__setValue(self._minGpusCheck, + __group.setMinGpus, + float(self._minGpusValue.value()), + __group.data.min_gpus, 0) + + self.__setValue(self._maxGpusCheck, + __group.setMaxGpus, + float(self._maxGpusValue.value()), + __group.data.max_gpus, -1) + self.close() @staticmethod @@ -195,7 +237,11 @@ def __init__(self, modifyGroup, parent=None): "defaultJobMinCores": modifyGroup.data.default_job_min_cores, "defaultJobMaxCores": modifyGroup.data.default_job_max_cores, "minCores": modifyGroup.data.min_cores, - "maxCores": modifyGroup.data.max_cores} + "maxCores": modifyGroup.data.max_cores, + "defaultJobMinGpus": modifyGroup.data.default_job_min_gpus, + "defaultJobMaxGpus": modifyGroup.data.default_job_max_gpus, + "minGpus": modifyGroup.data.min_gpus, + "maxGpus": modifyGroup.data.max_gpus} GroupDialog.__init__(self, None, modifyGroup, defaults, parent) @@ -212,5 +258,9 @@ def __init__(self, parentGroup, parent=None): "defaultJobMinCores": 1.0, "defaultJobMaxCores": 1.0, "minCores": 0.0, - "maxCores": 1.0} + "maxCores": 1.0, + "defaultJobMinGpus": 0, + "defaultJobMaxGpus": 0, + "minGpus": 0, + "maxGpus": 0} GroupDialog.__init__(self, parentGroup, None, defaults, parent) diff --git a/cuegui/cuegui/HostMonitorTree.py b/cuegui/cuegui/HostMonitorTree.py index f9e5a7c90..3574b4c71 100644 --- a/cuegui/cuegui/HostMonitorTree.py +++ b/cuegui/cuegui/HostMonitorTree.py @@ -78,9 +78,9 @@ def __init__(self, parent): data=lambda host: cuegui.Utils.memoryToString(host.data.free_memory), sort=lambda host: host.data.free_memory, tip="The amount of used memory (red) vs available gpu memory (green)") - self.addColumn("GPU", 60, id=6, - data=lambda host: cuegui.Utils.memoryToString(host.data.free_gpu), - sort=lambda host: host.data.free_gpu, + self.addColumn("GPU Memory", 60, id=6, + data=lambda host: cuegui.Utils.memoryToString(host.data.free_gpu_memory), + sort=lambda host: host.data.free_gpu_memory, delegate=cuegui.ItemDelegate.HostGpuBarDelegate, tip="The amount of used gpu memory (red) vs available gpu memory (green)") self.addColumn("freeMcp", 60, id=7, @@ -105,27 +105,36 @@ def __init__(self, parent): data=lambda host: cuegui.Utils.memoryToString(host.data.idle_memory), sort=lambda host: host.data.idle_memory, tip="The amount of unreserved memory.") - self.addColumn("GPU", 50, id=12, - data=lambda host: cuegui.Utils.memoryToString(host.data.gpu), - sort=lambda host: host.data.gpu, + self.addColumn("GPUs", 50, id=12, + data=lambda host: "%d" % host.data.gpus, + sort=lambda host: host.data.gpus, + tip="The total number of gpus.\n\n" + "On a frame it is the number of gpus reserved.") + self.addColumn("Idle GPUs", 40, id=13, + data=lambda host: "%d" % host.data.idle_gpus, + sort=lambda host: host.data.idle_gpus, + tip="The number of gpus that are not reserved.") + self.addColumn("GPU Mem", 50, id=14, + data=lambda host: cuegui.Utils.memoryToString(host.data.gpu_memory), + sort=lambda host: host.data.gpu_memory, tip="The total amount of reservable gpu memory.\n\n" "On a frame it is the amount of gpu memory reserved.") - self.addColumn("Idle", 50, id=13, - data=lambda host: cuegui.Utils.memoryToString(host.data.idle_gpu), - sort=lambda host: host.data.idle_gpu, + self.addColumn("Gpu Mem Idle", 50, id=15, + data=lambda host: cuegui.Utils.memoryToString(host.data.idle_gpu_memory), + sort=lambda host: host.data.idle_gpu_memory, tip="The amount of unreserved gpu memory.") - self.addColumn("Ping", 50, id=14, + self.addColumn("Ping", 50, id=16, data=lambda host: int(time.time() - host.data.ping_time), sort=lambda host: host.data.ping_time, tip="The number of seconds since the cuebot last received\n" "a report from the host. A host is configured to report\n" "in every 60 seconds so a number larger than this\n" "indicates a problem") - self.addColumn("Hardware", 70, id=15, + self.addColumn("Hardware", 70, id=17, data=lambda host: HardwareState.Name(host.data.state), tip="The state of the hardware as Up or Down.\n\n" "On a frame it is the amount of memory used.") - self.addColumn("Locked", 90, id=16, + self.addColumn("Locked", 90, id=18, data=lambda host: LockState.Name(host.data.lock_state), tip="A host can be:\n" "Locked \t\t It was manually locked to prevent booking\n" @@ -133,12 +142,12 @@ def __init__(self, parent): "NimbyLocked \t It is a desktop machine and there is\n" "\t\t someone actively using it or not enough \n" "\t\t resources are available on a desktop.") - self.addColumn("ThreadMode", 80, id=17, + self.addColumn("ThreadMode", 80, id=19, data=lambda host: ThreadMode.Name(host.data.thread_mode), tip="A frame that runs on this host will:\n" "All: Use all cores.\n" "Auto: Use the number of cores as decided by the cuebot.\n") - self.addColumn("Tags/Job", 50, id=18, + self.addColumn("Tags/Job", 50, id=20, data=lambda host: ",".join(host.data.tags), tip="The tags applied to the host.\n\n" "On a frame it is the name of the job.") @@ -340,7 +349,8 @@ def data(self, col, role): self.rpcObject.data.total_memory] if role == QtCore.Qt.UserRole + 3: - return [self.rpcObject.data.total_gpu - self.rpcObject.data.free_gpu, - self.rpcObject.data.total_gpu] + return [self.rpcObject.data.total_gpu_memory - + self.rpcObject.data.free_gpu_memory, + self.rpcObject.data.total_gpu_memory] return cuegui.Constants.QVARIANT_NULL diff --git a/cuegui/cuegui/LayerDialog.py b/cuegui/cuegui/LayerDialog.py index c337dda80..4c79b805f 100644 --- a/cuegui/cuegui/LayerDialog.py +++ b/cuegui/cuegui/LayerDialog.py @@ -117,12 +117,12 @@ def __init__(self, layers, parent=None): self.mem_max_kb = int(self.mem_max_gb * 1024 * 1024) self.mem_min_kb = int(self.mem_min_gb * 1024 * 1024) - self.gpu_max_kb = 2 * 1024 * 1024 - self.gpu_min_kb = 0 - self.gpu_tick_kb = 256 * 1024 - self.gpu_max_gb = 2.0 - self.gpu_min_gb = 0.0 - self.gpu_tick_gb = .25 + self.gpu_mem_max_kb = 256 * 1024 * 1024 + self.gpu_mem_min_kb = 0 + self.gpu_mem_tick_kb = 256 * 1024 + self.gpu_mem_max_gb = 256.0 + self.gpu_mem_min_gb = 0.0 + self.gpu_mem_tick_gb = .25 self.__group = QtWidgets.QGroupBox("Resource Options", self) @@ -180,16 +180,28 @@ def __init__(self, layers, parent=None): # Limits self.__limits = LayerLimitsWidget(self.__layers, self) + # Min gpus + self.__min_gpus = QtWidgets.QSpinBox(self) + self.__min_gpus.setValue(0) + self.__min_gpus.setRange(0, int(self._cfg().get('max_gpus', 16))) + self.__min_gpus.setSingleStep(1) + + # Max gpus + self.__max_gpus = QtWidgets.QSpinBox(self) + self.__max_gpus.setRange(0, int(self._cfg().get('max_gpus', 16))) + self.__max_gpus.setSingleStep(1) + # GPU Memory - self.__gpu = SlideSpinner(self) - self.__gpu.slider.setMinimumWidth(200) - self.__gpu.slider.setRange(self.gpu_min_kb, self.gpu_max_kb // self.gpu_tick_kb) - self.__gpu.slider.setTickInterval(1) - self.__gpu.slider.setSingleStep(1) - self.__gpu.slider.setPageStep(1) - self.__gpu.spinner.setSuffix(' GB') - self.__gpu.spinner.setRange(self.gpu_min_gb, self.gpu_max_gb) - self.__gpu.spinner.setSingleStep(self.gpu_tick_gb) + self.__gpu_mem = SlideSpinner(self) + self.__gpu_mem.slider.setMinimumWidth(200) + self.__gpu_mem.slider.setRange(self.gpu_mem_min_kb, + self.gpu_mem_max_kb // self.gpu_mem_tick_kb) + self.__gpu_mem.slider.setTickInterval(1) + self.__gpu_mem.slider.setSingleStep(1) + self.__gpu_mem.slider.setPageStep(1) + self.__gpu_mem.spinner.setSuffix(' GB') + self.__gpu_mem.spinner.setRange(self.gpu_mem_min_gb, self.gpu_mem_max_gb) + self.__gpu_mem.spinner.setSingleStep(self.gpu_mem_tick_gb) # Our dialog buttons. self.__buttons = QtWidgets.QDialogButtonBox(QtWidgets.QDialogButtonBox.Save | @@ -200,16 +212,18 @@ def __init__(self, layers, parent=None): # Setup signals self.__mem.slider.valueChanged.connect(self.__translateToMemSpinbox) self.__mem.spinner.valueChanged.connect(self.__translateToMemSlider) - self.__gpu.slider.valueChanged.connect(self.__translateToGpuSpinbox) - self.__gpu.spinner.valueChanged.connect(self.__translateToGpuSlider) + self.__gpu_mem.slider.valueChanged.connect(self.__translateToGpuMemSpinbox) + self.__gpu_mem.spinner.valueChanged.connect(self.__translateToGpuMemSlider) self.__buttons.accepted.connect(self.verify) self.__buttons.rejected.connect(self.reject) # Set actual values once signals are setup self.__mem.slider.setValue(self.getMaxMemory()) - self.__gpu.slider.setValue(self.getMaxGpu()) + self.__gpu_mem.slider.setValue(self.getMaxGpuMemory()) self.__core.setValue(self.getMinCores()) self.__max_cores.setValue(self.getMaxCores()) + self.__min_gpus.setValue(self.getMinGpus()) + self.__max_gpus.setValue(self.getMaxGpus()) self.__timeout.setValue(self.getTimeout()) self.__timeout_llu.setValue(self.getTimeoutLLU()) @@ -236,8 +250,16 @@ def __init__(self, layers, parent=None): self.__thread, True), multiSelect)) + layout.addWidget(EnableableItem(LayerPropertiesItem("Min GPUs:", + self.__min_gpus, + False), + multiSelect)) + layout.addWidget(EnableableItem(LayerPropertiesItem("Max GPUs:", + self.__max_gpus, + False), + multiSelect)) layout.addWidget(EnableableItem(LayerPropertiesItem("Minimum Gpu Memory:", - self.__gpu, + self.__gpu_mem, False), multiSelect)) layout.addWidget(EnableableItem(LayerPropertiesItem("Timeout:", @@ -280,8 +302,8 @@ def verify(self): if mem_value < self.mem_min_kb or mem_value > self.mem_max_kb: warning("The memory setting is too high.") return False - gpu_value = self.__gpu.slider.value() - if gpu_value < self.gpu_min_kb or gpu_value > self.gpu_max_kb: + gpu_mem_value = self.__gpu_mem.slider.value() + if gpu_mem_value < self.gpu_mem_min_kb or gpu_mem_value > self.gpu_mem_max_kb: warning("The gpu memory setting is too high.") return False @@ -302,8 +324,8 @@ def apply(self): layer.setMaxCores(self.__max_cores.value() * 100.0) if self.__thread.isEnabled(): layer.setThreadable(self.__thread.isChecked()) - if self.__gpu.isEnabled(): - layer.setMinGpu(self.__gpu.slider.value() * self.gpu_tick_kb) + if self.__gpu_mem.isEnabled(): + layer.setMinGpuMemory(self.__gpu_mem.slider.value() * self.gpu_mem_tick_kb) if self.__timeout.isEnabled(): layer.setTimeout(self.__timeout.value()) if self.__timeout_llu.isEnabled(): @@ -322,9 +344,9 @@ def getMaxMemory(self): result = layer.data.min_memory return result - def getMaxGpu(self): - """Gets the layer max GPU.""" - return max([layer.data.min_gpu // self.gpu_tick_kb for layer in self.__layers]) + def getMaxGpuMemory(self): + """Gets the layer max GPU memory.""" + return max([layer.data.min_gpu_memory // self.gpu_mem_tick_kb for layer in self.__layers]) def getMinCores(self): """Gets the layer min cores.""" @@ -342,6 +364,22 @@ def getMaxCores(self): result = layer.data.max_cores return result + def getMinGpus(self): + """Gets the layer min gpus.""" + result = 0 + for layer in self.__layers: + if layer.data.min_gpus > result: + result = layer.data.min_gpus + return result + + def getMaxGpus(self): + """Gets the layer max gpus.""" + result = 0 + for layer in self.__layers: + if layer.data.max_gpus > result: + result = layer.data.max_gpus + return result + def getThreading(self): """Gets whether the layer is threadable.""" result = False @@ -382,12 +420,11 @@ def __translateToMemSpinbox(self, value): def __translateToMemSlider(self, value): self.__mem.slider.setValue(int(value * 1048576.0)) - def __translateToGpuSpinbox(self, value): - self.__gpu.spinner.setValue(float(value * self.gpu_tick_kb) / 1024.0 / 1024.0) - - def __translateToGpuSlider(self, value): - self.__gpu.slider.setValue(int(value * 1024.0 * 1024.0) // self.gpu_tick_kb) + def __translateToGpuMemSpinbox(self, value): + self.__gpu_mem.spinner.setValue(float(value * self.gpu_mem_tick_kb) / 1024.0 / 1024.0) + def __translateToGpuMemSlider(self, value): + self.__gpu_mem.slider.setValue(int(value * 1024.0 * 1024.0) // self.gpu_mem_tick_kb) class LayerTagsWidget(QtWidgets.QWidget): """ diff --git a/cuegui/cuegui/LayerMonitorTree.py b/cuegui/cuegui/LayerMonitorTree.py index fdf0c249e..5b15450b6 100644 --- a/cuegui/cuegui/LayerMonitorTree.py +++ b/cuegui/cuegui/LayerMonitorTree.py @@ -74,65 +74,70 @@ def __init__(self, parent): "will reserve for its use. If the frame begins to use\n" "more memory than this, the cuebot will increase this\n" "number.") - self.addColumn("Gpu", 40, id=8, - data=lambda layer: cuegui.Utils.memoryToString(layer.data.min_gpu), - sort=lambda layer: layer.data.min_gpu, + self.addColumn("Gpus", 45, id=8, + data=lambda layer: "%d" % layer.data.min_gpus, + sort=lambda layer: layer.data.min_gpus, + tip="The number of gpus that the frames in this layer\n" + "will reserve as a minimum.") + self.addColumn("Gpu Memory", 40, id=9, + data=lambda layer: cuegui.Utils.memoryToString(layer.data.min_gpu_memory), + sort=lambda layer: layer.data.min_gpu_memory, tip="The amount of gpu memory each frame in this layer\n" "will reserve for its use. Note that we may not have\n" "machines as much gpu memory as you request.") self.addColumn( - "MaxRss", 60, id=9, + "MaxRss", 60, id=10, data=lambda layer: cuegui.Utils.memoryToString(layer.data.layer_stats.max_rss), sort=lambda layer: layer.data.layer_stats.max_rss, tip="Maximum amount of memory used by any frame in\n" "this layer at any time since the job was launched.") - self.addColumn("Total", 40, id=10, + self.addColumn("Total", 40, id=11, data=lambda layer: layer.data.layer_stats.total_frames, sort=lambda layer: layer.data.layer_stats.total_frames, tip="Total number of frames in this layer.") - self.addColumn("Done", 40, id=11, + self.addColumn("Done", 40, id=12, data=lambda layer: layer.data.layer_stats.succeeded_frames, sort=lambda layer: layer.data.layer_stats.succeeded_frames, tip="Total number of done frames in this layer.") - self.addColumn("Run", 40, id=12, + self.addColumn("Run", 40, id=13, data=lambda layer: layer.data.layer_stats.running_frames, sort=lambda layer: layer.data.layer_stats.running_frames, tip="Total number or running frames in this layer.") - self.addColumn("Depend", 53, id=13, + self.addColumn("Depend", 53, id=14, data=lambda layer: layer.data.layer_stats.depend_frames, sort=lambda layer: layer.data.layer_stats.depend_frames, tip="Total number of dependent frames in this layer.") - self.addColumn("Wait", 40, id=14, + self.addColumn("Wait", 40, id=15, data=lambda layer: layer.data.layer_stats.waiting_frames, sort=lambda layer: layer.data.layer_stats.waiting_frames, tip="Total number of waiting frames in this layer.") - self.addColumn("Eaten", 40, id=15, + self.addColumn("Eaten", 40, id=16, data=lambda layer: layer.data.layer_stats.eaten_frames, sort=lambda layer: layer.data.layer_stats.eaten_frames, tip="Total number of eaten frames in this layer.") - self.addColumn("Dead", 40, id=16, + self.addColumn("Dead", 40, id=17, data=lambda layer: layer.data.layer_stats.dead_frames, sort=lambda layer: layer.data.layer_stats.dead_frames, tip="Total number of dead frames in this layer.") self.addColumn( - "Avg", 65, id=17, + "Avg", 65, id=18, data=lambda layer: cuegui.Utils.secondsToHHMMSS(layer.data.layer_stats.avg_frame_sec), sort=lambda layer: layer.data.layer_stats.avg_frame_sec, tip="Average number of HOURS:MINUTES:SECONDS per frame\nin this layer.") - self.addColumn("Tags", 100, id=18, + self.addColumn("Tags", 100, id=19, data=lambda layer: " | ".join(layer.data.tags), tip="The tags define what resources may be booked on\n" "frames in this layer.") - self.addColumn("Progress", 100, id=19, + self.addColumn("Progress", 100, id=20, delegate=cuegui.ItemDelegate.ProgressDelegate, data=lambda layer: layer.percentCompleted(), sort=lambda layer: layer.percentCompleted(), tip="Progress for the Layer") - self.addColumn("Timeout", 45, id=20, + self.addColumn("Timeout", 45, id=21, data=lambda layer: cuegui.Utils.secondsToHHHMM(layer.data.timeout*60), sort=lambda layer: layer.data.timeout, tip="Timeout for the frames, Hours:Minutes") - self.addColumn("Timeout LLU", 45, id=21, + self.addColumn("Timeout LLU", 45, id=22, data=lambda layer: cuegui.Utils.secondsToHHHMM(layer.data.timeout_llu*60), sort=lambda layer: layer.data.timeout_llu, tip="Timeout for a frames\' LLU, Hours:Minutes") diff --git a/cuegui/cuegui/MenuActions.py b/cuegui/cuegui/MenuActions.py index 75f1bbc4a..68f0b6cc8 100644 --- a/cuegui/cuegui/MenuActions.py +++ b/cuegui/cuegui/MenuActions.py @@ -269,6 +269,38 @@ def setMaxCores(self, rpcObjects=None): job.setMaxCores(float(value)) self._update() + setMinGpu_info = ["Set Minimum Gpu...", "Set Job(s) Minimum Gpu", "configure"] + def setMinGpu(self, rpcObjects=None): + jobs = self._getOnlyJobObjects(rpcObjects) + if jobs: + current = max([job.data.min_cores for job in jobs]) + title = "Set Minimum Gpu" + body = "Please enter the new minimum gpu value:" + (value, choice) = QtWidgets.QInputDialog.getDouble(self._caller, + title, body, + current, + 0, 50000, 0) + if choice: + for job in jobs: + job.setMinGpu(float(value)) + self._update() + + setMaxGpu_info = ["Set Maximum Gpu...", "Set Job(s) Maximum Gpu", "configure"] + def setMaxGpu(self, rpcObjects=None): + jobs = self._getOnlyJobObjects(rpcObjects) + if jobs: + current = max([job.data.max_cores for job in jobs]) + title = "Set Maximum Gpu" + body = "Please enter the new maximum gpu value:" + (value, choice) = QtWidgets.QInputDialog.getDouble(self._caller, + title, body, + current, + 0, 50000, 0) + if choice: + for job in jobs: + job.setMaxGpu(float(value)) + self._update() + setPriority_info = ["Set Priority...", None, "configure"] def setPriority(self, rpcObjects=None): @@ -1460,6 +1492,24 @@ def clearRepair(self, rpcObjects=None): host.setHardwareState(down) self._update() + setThreadModeAuto_info = ["Thread Mode Auto", None, "configure"] + def setThreadModeAuto(self, rpcObjects=None): + for host in self._getOnlyHostObjects(rpcObjects): + host.setThreadMode("AUTO") + self._update() + + setThreadModeAll_info = ["Thread Mode All", None, "configure"] + def setThreadModeAll(self, rpcObjects=None): + for host in self._getOnlyHostObjects(rpcObjects): + host.setThreadMode("ALL") + self._update() + + setThreadModeVariable_info = ["Thread Mode Variable", None, "configure"] + def setThreadModeVariable(self, rpcObjects=None): + for host in self._getOnlyHostObjects(rpcObjects): + host.setThreadMode("VARIABLE") + self._update() + class ProcActions(AbstractActions): """Actions for procs.""" diff --git a/cuegui/cuegui/ProcMonitorTree.py b/cuegui/cuegui/ProcMonitorTree.py index efbf0c86f..9f67dabe3 100644 --- a/cuegui/cuegui/ProcMonitorTree.py +++ b/cuegui/cuegui/ProcMonitorTree.py @@ -60,7 +60,7 @@ def __init__(self, parent): tip="The amount of memory used.") self.addColumn( "GPU Used", 100, id=5, - data=lambda proc: cuegui.Utils.memoryToString(proc.data.reserved_gpu), + data=lambda proc: cuegui.Utils.memoryToString(proc.data.reserved_gpu_memory), tip="The amount of gpu memory used.") self.addColumn( "Age", 60, id=6, diff --git a/cuegui/cuegui/ServiceDialog.py b/cuegui/cuegui/ServiceDialog.py index 0288168f5..f88e568c0 100644 --- a/cuegui/cuegui/ServiceDialog.py +++ b/cuegui/cuegui/ServiceDialog.py @@ -61,11 +61,11 @@ def __init__(self, parent=None): self.min_memory = QtWidgets.QSpinBox(self) self.min_memory.setRange(512, int(self._cfg().get('max_memory', 48)) * 1024) self.min_memory.setValue(3276) - self.min_gpu = QtWidgets.QSpinBox(self) - self.min_gpu.setRange(self.gpu_min_mb, self.gpu_max_mb) - self.min_gpu.setValue(self.gpu_min_mb) - self.min_gpu.setSingleStep(self.gpu_tick_mb) - self.min_gpu.setSuffix(" MB") + self.min_gpu_memory = QtWidgets.QSpinBox(self) + self.min_gpu_memory.setRange(self.gpu_min_mb, self.gpu_max_mb) + self.min_gpu_memory.setValue(self.gpu_min_mb) + self.min_gpu_memory.setSingleStep(self.gpu_tick_mb) + self.min_gpu_memory.setSuffix(" MB") self.timeout = QtWidgets.QSpinBox(self) self.timeout.setRange(0, 4320) self.timeout.setValue(0) @@ -84,7 +84,7 @@ def __init__(self, parent=None): layout.addWidget(QtWidgets.QLabel("Min Memory MB:", self), 4, 0) layout.addWidget(self.min_memory, 4, 1) layout.addWidget(QtWidgets.QLabel("Min Gpu Memory MB:", self), 5, 0) - layout.addWidget(self.min_gpu, 5, 1) + layout.addWidget(self.min_gpu_memory, 5, 1) layout.addWidget(QtWidgets.QLabel("Timeout (in minutes):", self), 6, 0) layout.addWidget(self.timeout, 6, 1) layout.addWidget(QtWidgets.QLabel("Timeout LLU (in minutes):", self), 7, 0) @@ -124,7 +124,7 @@ def setService(self, service): self.min_cores.setValue(service.data.min_cores) self.max_cores.setValue(service.data.max_cores) self.min_memory.setValue(service.data.min_memory // 1024) - self.min_gpu.setValue(service.data.min_gpu // 1024) + self.min_gpu_memory.setValue(service.data.min_gpu_memory // 1024) self._tags_w.set_tags(service.data.tags) self.timeout.setValue(service.data.timeout) self.timeout_llu.setValue(service.data.timeout_llu) @@ -141,7 +141,7 @@ def new(self): self.min_cores.setValue(100) self.max_cores.setValue(100) self.min_memory.setValue(3276) - self.min_gpu.setValue(self.gpu_min_mb) + self.min_gpu_memory.setValue(self.gpu_min_mb) self.timeout.setValue(0) self.timeout_llu.setValue(0) self._tags_w.set_tags(['general']) @@ -168,7 +168,7 @@ def save(self): service.setMinCores(self.min_cores.value()) service.setMaxCores(self.max_cores.value()) service.setMinMemory(self.min_memory.value() * 1024) - service.setMinGpu(self.min_gpu.value() * 1024) + service.setMinGpu(self.min_gpu_memory.value() * 1024) service.setTimeout(self.timeout.value()) service.setTimeoutLLU(self.timeout_llu.value()) service.setTags(self._tags_w.get_tags()) diff --git a/cuegui/cuegui/config/cue_resources.yaml b/cuegui/cuegui/config/cue_resources.yaml index d54cfcbfc..501b6aff4 100644 --- a/cuegui/cuegui/config/cue_resources.yaml +++ b/cuegui/cuegui/config/cue_resources.yaml @@ -10,6 +10,9 @@ max_cores: 32 max_memory: 128 +max_gpus: 8 +max_gpu_memory: 128 + # Redirect Plugin maximum allowed core-hour cutoff. # Users will not be able to search for procs with frames that have been diff --git a/cuegui/tests/FrameMonitorTree_tests.py b/cuegui/tests/FrameMonitorTree_tests.py index e28e8229f..c3c2b4963 100644 --- a/cuegui/tests/FrameMonitorTree_tests.py +++ b/cuegui/tests/FrameMonitorTree_tests.py @@ -120,7 +120,7 @@ def test_tickFullUpdate(self, getFramesMock, getUpdatedFramesMock): def test_getCores(self): frame = opencue.wrappers.frame.Frame( - opencue.compiled_proto.job_pb2.Frame(last_resource='foo/125.82723')) + opencue.compiled_proto.job_pb2.Frame(last_resource='foo/125.82723/0')) self.assertEqual(125.82723, self.frameMonitorTree.getCores(frame)) self.assertEqual('125.83', self.frameMonitorTree.getCores(frame, format_as_string=True)) diff --git a/cuegui/tests/LayerDialog_tests.py b/cuegui/tests/LayerDialog_tests.py index 1f9624d59..5e515775d 100644 --- a/cuegui/tests/LayerDialog_tests.py +++ b/cuegui/tests/LayerDialog_tests.py @@ -55,13 +55,15 @@ def setUp(self, get_stub_mock, get_layer_mock, get_limits_mock): 'layer1Id': opencue.wrappers.layer.Layer( opencue.compiled_proto.job_pb2.Layer( id='layer1Id', name='layer1Name', range='1-5', tags=['tag1', 'tag2'], - min_cores=1, max_cores=3, is_threadable=False, min_memory=2097152, min_gpu=1, + min_cores=1, max_cores=3, is_threadable=False, + min_memory=2097152, min_gpu_memory=1, chunk_size=1, timeout=30, timeout_llu=1, memory_optimizer_enabled=True, limits=['limit1Name', 'limit2Name'])), 'layer2Id': opencue.wrappers.layer.Layer( opencue.compiled_proto.job_pb2.Layer( id='layer2Id', name='layer2Name', range='2-22', tags=['tag2', 'tag3'], - min_cores=2, max_cores=2, is_threadable=True, min_memory=6291456, min_gpu=2, + min_cores=2, max_cores=2, is_threadable=True, + min_memory=6291456, min_gpu_memory=2, chunk_size=5, timeout=60, timeout_llu=5, memory_optimizer_enabled=False, limits=['limit2Name', 'limit3Name'])), } @@ -124,12 +126,12 @@ def test__should_display_current_values(self): self.assertTrue(self.layer_properties_dialog._LayerPropertiesDialog__thread.isChecked()) self.assertEqual( - int(self.layer_properties_dialog.gpu_min_gb * 1024 * 1024), - self.layer_properties_dialog._LayerPropertiesDialog__gpu.slider.minimum()) + int(self.layer_properties_dialog.gpu_mem_min_gb * 1024 * 1024), + self.layer_properties_dialog._LayerPropertiesDialog__gpu_mem.slider.minimum()) self.assertEqual( - int(self.layer_properties_dialog.gpu_max_gb * 1024 * 1024) // - int(self.layer_properties_dialog.gpu_tick_gb * 1024 * 1024), - self.layer_properties_dialog._LayerPropertiesDialog__gpu.slider.maximum()) + int(self.layer_properties_dialog.gpu_mem_max_gb * 1024 * 1024) // + int(self.layer_properties_dialog.gpu_mem_tick_gb * 1024 * 1024), + self.layer_properties_dialog._LayerPropertiesDialog__gpu_mem.slider.maximum()) # Layer with the highest timeout determines the initial value. self.assertEqual(60, self.layer_properties_dialog._LayerPropertiesDialog__timeout.value()) @@ -163,13 +165,13 @@ def test__should_fail_on_memory_too_low(self): self.assertFalse(self.layer_properties_dialog.verify()) def test__should_fail_on_gpu_too_high(self): - self.layer_properties_dialog._LayerPropertiesDialog__gpu.slider.setValue( - self.layer_properties_dialog.gpu_max_kb * 2) + self.layer_properties_dialog._LayerPropertiesDialog__gpu_mem.slider.setValue( + self.layer_properties_dialog.gpu_mem_max_kb * 2) self.assertFalse(self.layer_properties_dialog.verify()) def test__should_fail_on_gpu_too_low(self): - self.layer_properties_dialog._LayerPropertiesDialog__gpu.slider.setValue( - self.layer_properties_dialog.gpu_min_kb / 3) + self.layer_properties_dialog._LayerPropertiesDialog__gpu_mem.slider.setValue( + self.layer_properties_dialog.gpu_mem_min_kb / 3) self.assertFalse(self.layer_properties_dialog.verify()) def test__should_apply_new_settings(self): @@ -206,9 +208,10 @@ def test__should_apply_new_settings(self): self.layer_properties_dialog._LayerPropertiesDialog__thread.parent().parent().enable(True) self.layer_properties_dialog._LayerPropertiesDialog__thread.setChecked(new_is_threadable) - new_min_gpu = 6 - self.layer_properties_dialog._LayerPropertiesDialog__gpu.parent().parent().enable(True) - self.layer_properties_dialog._LayerPropertiesDialog__gpu.slider.setValue(new_min_gpu) + new_min_gpu_memory = 6 + self.layer_properties_dialog._LayerPropertiesDialog__gpu_mem.parent().parent().enable(True) + self.layer_properties_dialog._LayerPropertiesDialog__gpu_mem.slider.setValue( + new_min_gpu_memory) new_timeout = 20 self.layer_properties_dialog._LayerPropertiesDialog__timeout.parent().parent().enable(True) @@ -239,10 +242,10 @@ def test__should_apply_new_settings(self): layer2_mock.setMaxCores.assert_called_with(100 * new_max_cores) layer1_mock.setThreadable.assert_called_with(new_is_threadable) layer2_mock.setThreadable.assert_called_with(new_is_threadable) - layer1_mock.setMinGpu.assert_called_with( - new_min_gpu * self.layer_properties_dialog.gpu_tick_kb) - layer2_mock.setMinGpu.assert_called_with( - new_min_gpu * self.layer_properties_dialog.gpu_tick_kb) + layer1_mock.setMinGpuMemory.assert_called_with( + new_min_gpu_memory * self.layer_properties_dialog.gpu_mem_tick_kb) + layer2_mock.setMinGpuMemory.assert_called_with( + new_min_gpu_memory * self.layer_properties_dialog.gpu_mem_tick_kb) layer1_mock.setTimeout.assert_called_with(new_timeout) layer2_mock.setTimeout.assert_called_with(new_timeout) layer1_mock.setTimeoutLLU.assert_called_with(new_timeout_llu) diff --git a/proto/facility.proto b/proto/facility.proto index 49c95537c..ae7f3d4b8 100644 --- a/proto/facility.proto +++ b/proto/facility.proto @@ -105,6 +105,11 @@ message AllocationStats { int32 hosts = 6; int32 locked_hosts = 7; int32 down_hosts = 8; + float gpus = 9; + float available_gpus = 10; + float idle_gpus = 11; + float running_gpus = 12; + float locked_gpus = 13; } diff --git a/proto/host.proto b/proto/host.proto index 90c957816..f2193bdb9 100644 --- a/proto/host.proto +++ b/proto/host.proto @@ -249,16 +249,16 @@ message Host { float idle_cores = 7; int64 memory = 8; int64 idle_memory = 9; - int64 gpu = 10; - int64 idle_gpu = 11; + int64 gpu_memory = 10; + int64 idle_gpu_memory = 11; int64 total_swap = 12; int64 total_memory = 13; - int64 total_gpu = 14; + int64 total_gpu_memory = 14; int64 total_mcp = 15; int64 free_swap = 16; int64 free_memory = 17; int64 free_mcp = 18; - int64 free_gpu = 19; + int64 free_gpu_memory = 19; int32 load = 20; int32 boot_time = 21; int32 ping_time = 22; @@ -267,6 +267,8 @@ message Host { HardwareState state = 25; LockState lock_state = 26; ThreadMode thread_mode = 27; + float gpus = 28; + float idle_gpus = 29; } message HostSearchCriteria { @@ -292,16 +294,16 @@ message NestedHost { float idle_cores = 7; int64 memory = 8; int64 idle_memory = 9; - int64 gpu = 10; - int64 idle_gpu = 11; + int64 gpu_memory = 10; + int64 idle_gpu_memory = 11; int64 total_swap = 12; int64 total_memory = 13; - int64 total_gpu = 14; + int64 total_gpu_memory = 14; int64 total_mcp = 15; int64 free_swap = 16; int64 free_memory = 17; int64 free_mcp = 18; - int64 free_gpu = 19; + int64 free_gpu_memory = 19; int32 load = 20; int32 boot_time = 21; int32 ping_time = 22; @@ -311,6 +313,8 @@ message NestedHost { LockState lock_state = 26; ThreadMode thread_mode = 27; NestedProcSeq procs = 28; + float gpus = 29; + float idle_gpus = 30; } message NestedHostSeq { @@ -328,7 +332,7 @@ message NestedProc { int32 bookedTime = 8; int32 dispatch_time = 9; int64 reserved_memory = 10; - int64 reserverd_gpu = 11; + int64 reserved_gpu_memory = 11; int64 used_memory = 12; float reserved_cores = 13; bool unbooked = 14; @@ -336,6 +340,8 @@ message NestedProc { string redirect_target = 16; repeated string services = 17; NestedHost parent = 18; + int64 used_gpu_memory = 19; + float reserved_gpus = 20; } message NestedProcSeq { @@ -360,13 +366,15 @@ message Proc { int32 bookedTime = 8; int32 dispatch_time = 9; int64 reserved_memory = 10; - int64 reserved_gpu = 11; + int64 reserved_gpu_memory = 11; int64 used_memory = 12; float reserved_cores = 13; bool unbooked = 14; string log_path = 15; string redirect_target = 16; repeated string services = 17; + int64 used_gpu_memory = 18; + float reserved_gpus = 19; } message ProcSearchCriteria { diff --git a/proto/job.proto b/proto/job.proto index 4a74f3aa8..240fba609 100644 --- a/proto/job.proto +++ b/proto/job.proto @@ -100,6 +100,12 @@ service GroupInterface { // Set the Default Job Min Core values to all in the provided group rpc SetDefaultJobMinCores(GroupSetDefJobMinCoresRequest) returns (GroupSetDefJobMinCoresResponse); + // Set the Default Job Max Gpu values to all in the provided group + rpc SetDefaultJobMaxGpus(GroupSetDefJobMaxGpusRequest) returns (GroupSetDefJobMaxGpusResponse); + + // Set the Default Job Min Gpu values to all in the provided group + rpc SetDefaultJobMinGpus(GroupSetDefJobMinGpusRequest) returns (GroupSetDefJobMinGpusResponse); + // Set the Default Job Priority values to all in the provided group rpc SetDefaultJobPriority(GroupSetDefJobPriorityRequest) returns (GroupSetDefJobPriorityResponse); @@ -115,6 +121,12 @@ service GroupInterface { // Set the groups Min Cores values rpc SetMinCores(GroupSetMinCoresRequest) returns (GroupSetMinCoresResponse); + // Set the group's Max Gpu value + rpc SetMaxGpus(GroupSetMaxGpusRequest) returns (GroupSetMaxGpusResponse); + + // Set the groups Min Gpu values + rpc SetMinGpus(GroupSetMinGpusRequest) returns (GroupSetMinGpusResponse); + // Set the groups name rpc SetName(GroupSetNameRequest) returns (GroupSetNameResponse); } @@ -248,6 +260,12 @@ service JobInterface { // Sets the minimum number of procs that can run on this job rpc SetMinCores(JobSetMinCoresRequest) returns (JobSetMinCoresResponse); + // Sets the maximum number of Gpu that can run on this job + rpc SetMaxGpus(JobSetMaxGpusRequest) returns (JobSetMaxGpusResponse); + + // Sets the minimum number of Gpu that can run on this job + rpc SetMinGpus(JobSetMinGpusRequest) returns (JobSetMinGpusResponse); + // Sets the job priority rpc SetPriority(JobSetPriorityRequest) returns (JobSetPriorityResponse); @@ -331,7 +349,16 @@ service LayerInterface { // Set the Min Cores for this layer rpc SetMinCores(LayerSetMinCoresRequest) returns (LayerSetMinCoresResponse); - // Set the Min gpu value for the layer + // The maximum number of Gpu to run on a given frame within this layer. + rpc SetMaxGpus(LayerSetMaxGpusRequest) returns (LayerSetMaxGpusResponse); + + // Set the Min Gpus for this layer + rpc SetMinGpus(LayerSetMinGpusRequest) returns (LayerSetMinGpusResponse); + + // Set the Min gpu memory value for the layer + rpc SetMinGpuMemory(LayerSetMinGpuMemoryRequest) returns (LayerSetMinGpuMemoryResponse); + + // [Deprecated] Set the Min gpu memory value for the layer rpc SetMinGpu(LayerSetMinGpuRequest) returns (LayerSetMinGpuResponse); // Set the Min Memory value for the layer @@ -450,12 +477,15 @@ message Frame { int64 max_rss = 11; int64 used_memory = 12; int64 reserved_memory = 13; - int64 reserved_gpu = 14; + int64 reserved_gpu_memory = 14; string last_resource = 15; CheckpointState checkpoint_state = 16; int32 checkpoint_count = 17; int32 total_core_time = 18; int32 llu_time = 19; + int32 total_gpu_time = 20; + int64 max_gpu_memory = 21; + int64 used_gpu_memory = 22; } // Object for frame searching @@ -499,6 +529,8 @@ message UpdatedFrame { int64 used_memory = 8; string last_resource = 9; int32 llu_time = 10; + int64 max_gpu_memory = 11; + int64 used_gpu_memory = 12; } message UpdatedFrameSeq { @@ -528,6 +560,10 @@ message Group { int32 level = 9; string parent_id = 10; GroupStats group_stats = 11; + float default_job_min_gpus = 12; + float default_job_max_gpus = 13; + float min_gpus = 14; + float max_gpus = 15; } message GroupSeq { @@ -541,6 +577,7 @@ message GroupStats { int32 waiting_frames = 4; int32 pending_jobs = 5; float reserved_cores = 6; + float reserved_gpus = 7; } // JOB ---- @@ -567,6 +604,8 @@ message Job { int32 start_time = 18; int32 stop_time = 19; JobStats job_stats = 20; + float min_gpus = 21; + float max_gpus = 22; } // Use to filter the job search. Please note that by searching for non-pending jobs, the output is limited to 200 jobs @@ -606,6 +645,11 @@ message JobStats { int64 failed_core_sec = 18; int64 max_rss = 19; float reserved_cores = 20; + int64 total_gpu_sec = 21; + int64 rendered_gpu_sec = 22; + int64 failed_gpu_sec = 23; + float reserved_gpus = 24; + int64 max_gpu_memory = 25; } // LAYER ---- @@ -618,7 +662,7 @@ message Layer { float max_cores = 6; bool is_threadable = 7; int64 min_memory = 8; - int64 min_gpu = 9; + int64 min_gpu_memory = 9; int32 chunk_size = 10; int32 dispatch_order = 11; LayerType type = 12; @@ -631,6 +675,8 @@ message Layer { repeated string limits = 17; int32 timeout = 18; int32 timeout_llu = 19; + float min_gpus = 20; + float max_gpus = 21; } message LayerSeq { @@ -658,6 +704,11 @@ message LayerStats { int64 failed_core_sec = 18; int64 max_rss = 19; float reserved_cores = 20; + int64 total_gpu_sec = 21; + int64 rendered_gpu_sec = 22; + int64 failed_gpu_sec = 23; + float reserved_gpus = 24; + int64 max_gpu_memory = 25; } // NestedGroup --- @@ -675,6 +726,10 @@ message NestedGroup { NestedGroupSeq groups = 11; repeated string jobs = 12; GroupStats stats = 13; + float default_job_min_gpus = 14; + float default_job_max_gpus = 15; + float min_gpus = 16; + float max_gpus = 17; } message NestedGroupSeq { @@ -706,6 +761,8 @@ message NestedJob { int32 stop_time = 19; NestedGroup parent = 20; JobStats stats = 21; + float min_gpus = 22; + float max_gpus = 23; } @@ -719,8 +776,9 @@ message FrameAddRenderPartitionRequest { int32 threads = 3; int32 max_cores = 4; int64 max_memory = 5; - int64 max_gpu = 6; + int64 max_gpu_memory = 6; string username = 7; + int32 max_gpus = 8; } message FrameAddRenderPartitionResponse { @@ -944,6 +1002,22 @@ message GroupSetDefJobMinCoresRequest { message GroupSetDefJobMinCoresResponse {} // Empty +// SetDefaultJobMaxGpus +message GroupSetDefJobMaxGpusRequest { + Group group = 1; + int32 max_gpus = 2; +} + +message GroupSetDefJobMaxGpusResponse {} // Empty + +// SetDefaultJobMinGpus +message GroupSetDefJobMinGpusRequest { + Group group = 1; + int32 min_gpus = 2; +} + +message GroupSetDefJobMinGpusResponse {} // Empty + // SetDefJobPriority message GroupSetDefJobPriorityRequest { Group group = 1; @@ -984,6 +1058,22 @@ message GroupSetMinCoresRequest { message GroupSetMinCoresResponse {} // Empty +// SetMaxGpus +message GroupSetMaxGpusRequest { + Group group = 1; + int32 max_gpus = 2; +} + +message GroupSetMaxGpusResponse {} // Empty + +// SetMinGpus +message GroupSetMinGpusRequest { + Group group = 1; + int32 min_gpus = 2; +} + +message GroupSetMinGpusResponse {} // Empty + // SetName message GroupSetNameRequest { Group group = 1; @@ -1008,8 +1098,9 @@ message JobAddRenderPartRequest { int32 threads = 3; int32 max_cores = 4; int64 max_memory = 5; - int64 max_gpu = 6; + int64 max_gpu_memory = 6; string username = 7; + int32 max_gpus = 8; } message JobAddRenderPartResponse { @@ -1296,6 +1387,14 @@ message JobSetMaxCoresRequest { message JobSetMaxCoresResponse {} // Empty +// SetMaxGpus +message JobSetMaxGpusRequest { + Job job = 1; + int32 val = 2; +} + +message JobSetMaxGpusResponse {} // Empty + // SetMaxRetries message JobSetMaxRetriesRequest { Job job = 1; @@ -1312,6 +1411,14 @@ message JobSetMinCoresRequest { message JobSetMinCoresResponse {} // Empty +// SetMinGpus +message JobSetMinGpusRequest { + Job job = 1; + int32 val = 2; +} + +message JobSetMinGpusResponse {} // Empty + // SetPriority message JobSetPriorityRequest { Job job = 1; @@ -1346,8 +1453,9 @@ message LayerAddRenderPartitionRequest { int32 threads = 3; int32 max_cores = 4; int64 max_memory = 5; - int64 max_gpu = 6; + int64 max_gpu_memory = 6; string username = 7; + int32 max_gpus = 8; } message LayerAddRenderPartitionResponse { @@ -1545,14 +1653,39 @@ message LayerSetMinCoresRequest { message LayerSetMinCoresResponse {} // Empty -// SetMinGpu +// [Deprecated] SetMinGpu message LayerSetMinGpuRequest { - Layer layer = 1; - int64 gpu = 2; + Layer layer = 1 [deprecated=true]; + int64 gpu = 2 [deprecated=true]; } +// [Deprecated] message LayerSetMinGpuResponse {} // Empty +// SetMaxGpus +message LayerSetMaxGpusRequest { + Layer layer = 1; + int32 max_gpus = 2; +} + +message LayerSetMaxGpusResponse {} // Empty + +// SetMinGpus +message LayerSetMinGpusRequest { + Layer layer = 1; + int32 min_gpus = 2; +} + +message LayerSetMinGpusResponse {} // Empty + +// SetMinGpuMemory +message LayerSetMinGpuMemoryRequest { + Layer layer = 1; + int64 gpu_memory = 2; +} + +message LayerSetMinGpuMemoryResponse {} // Empty + // SetMinMemory message LayerSetMinMemoryRequest { Layer layer = 1; diff --git a/proto/renderPartition.proto b/proto/renderPartition.proto index 9d7b554f3..c8553ed3c 100644 --- a/proto/renderPartition.proto +++ b/proto/renderPartition.proto @@ -40,8 +40,11 @@ message RenderPartition { int64 memory = 8; int32 max_cores = 9; int64 max_memory = 10; - int64 max_gpu = 11; + int64 max_gpu_memory = 11; int32 threads = 12; + int32 gpus = 13; + int32 max_gpus = 14; + int64 gpu_memory = 15; } message RenderPartitionSeq { @@ -61,7 +64,8 @@ message RenderPartSetMaxResourcesRequest { RenderPartition render_partition = 1; int32 cores = 2; int64 memory = 3; - int64 gpu = 4; + int64 gpu_memory = 4; + int32 gpus = 5; } message RenderPartSetMaxResourcesResponse {} // Empty diff --git a/proto/report.proto b/proto/report.proto index 7a1fffdb2..d53103663 100644 --- a/proto/report.proto +++ b/proto/report.proto @@ -59,17 +59,20 @@ message RenderHost { string facility= 4; // The name of the facility that the host is in int32 num_procs = 5; // the number of physical procs on this machine int32 cores_per_proc = 6; // the number of cores per proc - int32 total_swap = 7; // the total size of the swap in kB - int32 total_mem = 8; // the total size of the main memory pool in kB - int32 total_mcp = 9; // the total size of MCP in kB - int32 free_swap = 10; // the current amount of free swap in kB - int32 free_mem = 11; // the current amount of free memory in kB - int32 free_mcp = 12; // the current amount of free MCP in kB + int64 total_swap = 7; // the total size of the swap in kB + int64 total_mem = 8; // the total size of the main memory pool in kB + int64 total_mcp = 9; // the total size of MCP in kB + int64 free_swap = 10; // the current amount of free swap in kB + int64 free_mem = 11; // the current amount of free memory in kB + int64 free_mcp = 12; // the current amount of free MCP in kB int32 load = 13; // the current load on the proc int32 boot_time = 14; // the time the proc was booted repeated string tags = 15; // an array of default tags that are added to the host record host.HardwareState state = 16; // hardware state for the host map attributes = 17; // additional data can be provided about the host + int32 num_gpus = 18; // the number of physical GPU's + int64 free_gpu_mem = 19; // the current amount of free gpu memory in kB + int64 total_gpu_mem = 20; // the total size of gpu memory in kB }; message RunningFrameInfo { @@ -87,6 +90,9 @@ message RunningFrameInfo { int64 vsize = 12; // kB map attributes = 13; //additional data can be provided about the running frame int64 llu_time = 14; + int32 num_gpus = 15; + int64 max_used_gpu_memory = 16; // kB + int64 used_gpu_memory = 17; // kB }; diff --git a/proto/rqd.proto b/proto/rqd.proto index a4b83f0c3..f67add41b 100644 --- a/proto/rqd.proto +++ b/proto/rqd.proto @@ -110,6 +110,7 @@ message RunFrame { bool ignore_nimby = 20; map environment = 21; map attributes = 22; + int32 num_gpus = 23; } message RunFrameSeq { diff --git a/proto/service.proto b/proto/service.proto index 8b554b388..23633bc40 100644 --- a/proto/service.proto +++ b/proto/service.proto @@ -46,11 +46,13 @@ message Service { bool threadable = 3; int32 min_cores = 4; int32 max_cores = 5; - int32 min_memory = 6; - int32 min_gpu = 7; + int64 min_memory = 6; + int64 min_gpu_memory = 7; repeated string tags = 8; int32 timeout = 9; int32 timeout_llu = 10; + int32 min_gpus = 11; + int32 max_gpus = 12; } message ServiceSeq { diff --git a/proto/show.proto b/proto/show.proto index ca237cc54..f0fbf34b4 100644 --- a/proto/show.proto +++ b/proto/show.proto @@ -99,6 +99,12 @@ service ShowInterface { // sets a show's default min procs rpc SetDefaultMinCores(ShowSetDefaultMinCoresRequest) returns (ShowSetDefaultMinCoresResponse); + + // sets a show's default max Gpus + rpc SetDefaultMaxGpus(ShowSetDefaultMaxGpusRequest) returns (ShowSetDefaultMaxGpusResponse); + + // sets a show's default min Gpus + rpc SetDefaultMinGpus(ShowSetDefaultMinGpusRequest) returns (ShowSetDefaultMinGpusResponse); } @@ -114,6 +120,8 @@ message Show { bool dispatch_enabled = 7; bool active = 8; ShowStats show_stats = 9; + float default_min_gpus = 10; + float default_max_gpus = 11; } message ShowSeq { @@ -130,6 +138,7 @@ message ShowStats { int64 rendered_frame_count = 7; int64 failed_frame_count = 8; float reserved_cores = 9; + float reserved_gpus = 10; } @@ -375,3 +384,19 @@ message ShowSetDefaultMinCoresRequest { } message ShowSetDefaultMinCoresResponse {} // Empty + +// SetDefaultMaxGpus +message ShowSetDefaultMaxGpusRequest { + Show show = 1; + int32 max_gpus = 2; +} + +message ShowSetDefaultMaxGpusResponse {} // Empty + +// SetDefaultMinGpus +message ShowSetDefaultMinGpusRequest { + Show show = 1; + int32 min_gpus = 2; +} + +message ShowSetDefaultMinGpusResponse {} // Empty diff --git a/proto/subscription.proto b/proto/subscription.proto index 8c7817da4..8ac100e7e 100644 --- a/proto/subscription.proto +++ b/proto/subscription.proto @@ -41,6 +41,7 @@ message Subscription { int32 size = 6; int32 burst = 7; int32 reserved_cores = 8; + int32 reserved_gpus = 9; } message SubscriptionSeq { diff --git a/pycue/opencue/wrappers/group.py b/pycue/opencue/wrappers/group.py index 59685cc8b..1c8175e57 100644 --- a/pycue/opencue/wrappers/group.py +++ b/pycue/opencue/wrappers/group.py @@ -67,6 +67,24 @@ def setMinCores(self, value): self.stub.SetMinCores(job_pb2.GroupSetMinCoresRequest(group=self.data, min_cores=value), timeout=Cuebot.Timeout) + def setMaxGpus(self, value): + """Sets the maximum gpus of everything in the group. + + :type value: int + :param value: new maximum number of gpus + """ + self.stub.SetMaxGpus(job_pb2.GroupSetMaxGpusRequest(group=self.data, max_gpus=value), + timeout=Cuebot.Timeout) + + def setMinGpus(self, value): + """Sets the minimum gpus of everything the group. + + :type value: int + :param value: new minimum number of gpus + """ + self.stub.SetMinGpus(job_pb2.GroupSetMinGpusRequest(group=self.data, min_gpus=value), + timeout=Cuebot.Timeout) + def setDefaultJobPriority(self, value): """Sets the default job priority for everything in the group. @@ -97,6 +115,26 @@ def setDefaultJobMaxCores(self, value): job_pb2.GroupSetDefJobMaxCoresRequest(group=self.data, max_cores=value), timeout=Cuebot.Timeout) + def setDefaultJobMinGpus(self, value): + """Sets the default job minimum gpus for everything in the group. + + :type value: int + :param value: new default job minimum gpus + """ + self.stub.SetDefaultJobMinGpus( + job_pb2.GroupSetDefJobMinGpusRequest(group=self.data, min_gpus=value), + timeout=Cuebot.Timeout) + + def setDefaultJobMaxGpus(self, value): + """Sets the default job maximum gpus for everything in the group. + + :type value: int + :param value: new default job maximum gpus + """ + self.stub.SetDefaultJobMaxGpus( + job_pb2.GroupSetDefJobMaxGpusRequest(group=self.data, max_gpus=value), + timeout=Cuebot.Timeout) + def getGroups(self): """Returns child groups of this group. diff --git a/pycue/opencue/wrappers/job.py b/pycue/opencue/wrappers/job.py index eb977a280..733c91e61 100644 --- a/pycue/opencue/wrappers/job.py +++ b/pycue/opencue/wrappers/job.py @@ -126,6 +126,20 @@ def setMaxCores(self, maxCores): self.stub.SetMaxCores(job_pb2.JobSetMaxCoresRequest(job=self.data, val=maxCores), timeout=Cuebot.Timeout) + def setMinGpus(self, minGpus): + """Sets the minimum procs value + :type minGpus: int + :param minGpus: New minimum cores value""" + self.stub.SetMinGpus(job_pb2.JobSetMinGpusRequest(job=self.data, val=minGpus), + timeout=Cuebot.Timeout) + + def setMaxGpus(self, maxGpus): + """Sets the maximum procs value + :type maxGpus: int + :param maxGpus: New maximum cores value""" + self.stub.SetMaxGpus(job_pb2.JobSetMaxGpusRequest(job=self.data, val=maxGpus), + timeout=Cuebot.Timeout) + def setPriority(self, priority): """Sets the job priority. @@ -211,7 +225,7 @@ def setAutoEating(self, value): self.stub.SetAutoEat(job_pb2.JobSetAutoEatRequest(job=self.data, value=value), timeout=Cuebot.Timeout) - def addRenderPartition(self, hostname, threads, max_cores, num_mem, max_gpu): + def addRenderPartition(self, hostname, threads, max_cores, num_mem, max_gpus, max_gpu_memory): """Adds a render partition to the job. :type hostname: str @@ -222,8 +236,10 @@ def addRenderPartition(self, hostname, threads, max_cores, num_mem, max_gpu): :param max_cores: max cores enabled for the partition :type num_mem: int :param num_mem: amount of memory reserved for the partition - :type max_gpu: int - :param max_gpu: max gpu cores enabled for the partition + :type max_gpus: int + :param max_gpus: max gpu cores enabled for the partition + :type max_gpu_memory: int + :param max_gpu_memory: amount of gpu memory reserved for the partition """ self.stub.AddRenderPartition( job_pb2.JobAddRenderPartRequest(job=self.data, @@ -231,7 +247,8 @@ def addRenderPartition(self, hostname, threads, max_cores, num_mem, max_gpu): threads=threads, max_cores=max_cores, max_memory=num_mem, - max_gpu=max_gpu, + max_gpus=max_gpus, + max_gpu_memory=max_gpu_memory, username=os.getenv("USER", "unknown"))) def getWhatDependsOnThis(self): @@ -492,6 +509,20 @@ def maxCores(self): """ return self.data.max_cores + def minGpus(self): + """Returns the minimum number of gpus the job needs. + :rtype: int + :return: job's min gpus + """ + return self.data.min_gpus + + def maxGpus(self): + """Returns the maximum number of gpus the job will use. + :rtype: int + :return: job's max gpus + """ + return self.data.max_gpus + def os(self): """Returns the job's operating system. @@ -823,6 +854,18 @@ def setMaxCores(self, maxCores): """ self.asJob().setMaxCores(maxCores) + def setMinGpus(self, minGpus): + """Sets the minimum gpus value + :type minGpus: int + :param minGpus: New minimum gpus value""" + self.asJob().setMinGpus(minGpus) + + def setMaxGpus(self, maxGpus): + """Sets the maximum gpus value + :type maxGpus: int + :param maxGpus: New maximum gpus value""" + self.asJob().setMaxGpus(maxGpus) + def setPriority(self, priority): """Sets the job priority. diff --git a/pycue/opencue/wrappers/layer.py b/pycue/opencue/wrappers/layer.py index 7c33b68e7..0e34985bf 100644 --- a/pycue/opencue/wrappers/layer.py +++ b/pycue/opencue/wrappers/layer.py @@ -140,14 +140,30 @@ def setMinCores(self, cores): job_pb2.LayerSetMinCoresRequest(layer=self.data, cores=cores/100.0), timeout=Cuebot.Timeout) - def setMinGpu(self, gpu): + def setMaxGpus(self, max_gpus): + """Sets the maximum number of gpus that this layer requires. + :type max_gpus: int + :param max_gpus: gpu cores""" + return self.stub.SetMaxGpus( + job_pb2.LayerSetMaxGpusRequest(layer=self.data, max_gpus=max_gpus), + timeout=Cuebot.Timeout) + + def setMinGpus(self, min_gpus): + """Sets the minimum number of gpus that this layer requires. + :type min_gpus: int + :param min_gpus: gou cores""" + return self.stub.SetMinGpus( + job_pb2.LayerSetMinGpusRequest(layer=self.data, min_gpus=min_gpus), + timeout=Cuebot.Timeout) + + def setMinGpuMemory(self, gpu_memory): """Sets the minimum number of gpu memory that this layer requires. - :type gpu: int - :param gpu: gpu value + :type gpu_memory: int + :param gpu_memory: gpu_memory value """ - return self.stub.SetMinGpu( - job_pb2.LayerSetMinGpuRequest(layer=self.data, gpu=gpu), + return self.stub.SetMinGpuMemory( + job_pb2.LayerSetMinGpuMemoryRequest(layer=self.data, gpu_memory=gpu_memory), timeout=Cuebot.Timeout) def setMinMemory(self, memory): @@ -401,6 +417,12 @@ def coresReserved(self): """ return self.data.layer_stats.reserved_cores + def gpusReserved(self): + """Returns the number of gpus reserved on this layer + :rtype: float + :return: gpus reserved""" + return self.data.layer_stats.reserved_gpus + def minCores(self): """Returns the minimum number of cores that frames in this layer require. @@ -409,6 +431,12 @@ def minCores(self): """ return self.data.min_cores + def minGpus(self): + """Returns the minimum number of gpus that frames in this layer require + :rtype: int + :return: Minimum number of gpus required""" + return self.data.min_gpus + def minMemory(self): """Returns the minimum amount of memory that frames in this layer require. diff --git a/pycue/opencue/wrappers/show.py b/pycue/opencue/wrappers/show.py index 750d05645..9fff4b47d 100644 --- a/pycue/opencue/wrappers/show.py +++ b/pycue/opencue/wrappers/show.py @@ -167,6 +167,32 @@ def setDefaultMinCores(self, mincores): timeout=Cuebot.Timeout) return response + def setDefaultMaxGpus(self, maxgpus): + """Sets the default maximum number of gpus + that new jobs are launched with. + :type: float + :param: value to set maxGpu to + :rtype: show_pb2.ShowSetDefaultMaxGpuResponse + :return: response is empty + """ + response = self.stub.SetDefaultMaxGpus(show_pb2.ShowSetDefaultMaxGpusRequest( + show=self.data, max_gpu=maxgpus), + timeout=Cuebot.Timeout) + return response + + def setDefaultMinGpus(self, mingpus): + """Sets the default minimum number of gpus + all new jobs are launched with. + :type: float + :param: value to set minGpus to + :rtype: show_pb2.ShowSetDefaultMinGpusResponse + :return: response is empty + """ + response = self.stub.SetDefaultMinGpus(show_pb2.ShowSetDefaultMinGpusRequest( + show=self.data, min_gpu=mingpus), + timeout=Cuebot.Timeout) + return response + def findFilter(self, name): """Finds a filter by name. diff --git a/pycue/tests/wrappers/layer_test.py b/pycue/tests/wrappers/layer_test.py index cf8fb0c33..4f5578681 100644 --- a/pycue/tests/wrappers/layer_test.py +++ b/pycue/tests/wrappers/layer_test.py @@ -201,18 +201,18 @@ def testSetMaxCores(self, getStubMock): job_pb2.LayerSetMaxCoresRequest(layer=layer.data, cores=testCoresActual), timeout=mock.ANY) - def testSetMinGpu(self, getStubMock): + def testSetMinGpuMemory(self, getStubMock): stubMock = mock.Mock() - stubMock.SetMinGpu.return_value = job_pb2.LayerSetMinGpuResponse() + stubMock.SetMinGpuMemory.return_value = job_pb2.LayerSetMinGpuResponse() getStubMock.return_value = stubMock testCores = 100 layer = opencue.wrappers.layer.Layer( job_pb2.Layer(name=TEST_LAYER_NAME)) - layer.setMinGpu(testCores) + layer.setMinGpuMemory(testCores) - stubMock.SetMinGpu.assert_called_with( - job_pb2.LayerSetMinGpuRequest(layer=layer.data, gpu=testCores), + stubMock.SetMinGpuMemory.assert_called_with( + job_pb2.LayerSetMinGpuMemoryRequest(layer=layer.data, gpu_memory=testCores), timeout=mock.ANY) def testSetMinMemory(self, getStubMock): diff --git a/pyoutline/etc/outline.cfg b/pyoutline/etc/outline.cfg index 6d0f129a4..fafbfa636 100644 --- a/pyoutline/etc/outline.cfg +++ b/pyoutline/etc/outline.cfg @@ -5,7 +5,7 @@ wrapper_dir = %(home)s/wrappers user_dir = bin_dir = %(home)s/bin backend = cue -spec_version = 1.11 +spec_version = 1.12 facility = local domain = example.com maxretries = 2 diff --git a/pyoutline/outline/backend/cue.py b/pyoutline/outline/backend/cue.py index 6cd03105d..333b5ac59 100644 --- a/pyoutline/outline/backend/cue.py +++ b/pyoutline/outline/backend/cue.py @@ -323,6 +323,30 @@ def _serialize(launcher, use_pycuerun): if layer.get_arg("memory"): sub_element(spec_layer, "memory", "%s" % (layer.get_arg("memory"))) + gpus = None + if layer.get_arg("gpus"): + if spec_version >= Version("1.12"): + gpus = layer.get_arg("gpus") + else: + _warning_spec_version(spec_version, "gpus") + + gpu_memory = None + if layer.get_arg("gpu_memory"): + if spec_version >= Version("1.12"): + gpu_memory = layer.get_arg("gpu_memory") + else: + _warning_spec_version(spec_version, "gpu_memory") + + if gpus or gpu_memory: + # Cuebot expects non-zero positive value on gpus and gpu_memory + if gpus is None: + gpus = 1 + if gpu_memory is None: + gpu_memory = "1g" + + sub_element(spec_layer, "gpus", "%d" % gpus) + sub_element(spec_layer, "gpu_memory", "%s" % gpu_memory) + if layer.get_arg("timeout"): if spec_version >= Version("1.10"): sub_element(spec_layer, "timeout", "%s" % (layer.get_arg("timeout"))) diff --git a/pyoutline/tests/specver_test.py b/pyoutline/tests/specver_test.py index 3244e6807..f4c077d4b 100644 --- a/pyoutline/tests/specver_test.py +++ b/pyoutline/tests/specver_test.py @@ -63,3 +63,24 @@ def test_1_11(self): self.assertEqual(root.find("job/layers/layer/timeout").text, "420") self.assertEqual(root.find("job/layers/layer/timeout_llu").text, "4200") self.assertEqual(root.find("job/priority").text, "42") + + def _makeGpuSpec(self): + ol = outline.Outline(name="spec_version_test") + layer = outline.modules.shell.Shell("test_layer", command=["/bin/ls"]) + layer.set_arg("gpus", 4) + layer.set_arg("gpu_memory", 8 * 1024 * 1024) + ol.add_layer(layer) + l = outline.cuerun.OutlineLauncher(ol) + return Et.fromstring(l.serialize()) + + def test_gpu_1_11(self): + outline.config.set("outline", "spec_version", "1.11") + root = self._makeGpuSpec() + self.assertIsNone(root.find("job/layers/layer/gpus")) + self.assertIsNone(root.find("job/layers/layer/gpus_memory")) + + def test_gpu_1_12(self): + outline.config.set("outline", "spec_version", "1.12") + root = self._makeGpuSpec() + self.assertEqual(root.find("job/layers/layer/gpus").text, "4") + self.assertEqual(root.find("job/layers/layer/gpu_memory").text, "8388608") diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index dd58e35a6..b7dcae31a 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -90,7 +90,7 @@ def __createEnvVariables(self): self.frameEnv["maxframetime"] = "0" self.frameEnv["minspace"] = "200" self.frameEnv["CUE3"] = "True" - self.frameEnv["CUE_GPU_MEMORY"] = str(self.rqCore.machine.getGpuMemory()) + self.frameEnv["CUE_GPU_MEMORY"] = str(self.rqCore.machine.getGpuMemoryFree()) self.frameEnv["SP_NOMYCSHRC"] = "1" for key in self.runFrame.environment: @@ -103,6 +103,10 @@ def __createEnvVariables(self): len(self.runFrame.attributes['CPU_LIST'].split(',')))) self.frameEnv['CUE_HT'] = "True" + # Add GPU's to use all assigned GPU cores + if 'GPU_LIST' in self.runFrame.attributes: + self.frameEnv['CUE_GPU_CORES'] = self.runFrame.attributes['GPU_LIST'] + def _createCommandFile(self, command): """Creates a file that subprocess. Popen then executes. @type command: string @@ -187,6 +191,8 @@ def __writeFooter(self): print("%-20s%s" % ("endTime", time.ctime(self.endTime)), file=self.rqlog) print("%-20s%s" % ("maxrss", self.frameInfo.maxRss), file=self.rqlog) + print("%-20s%s" % ("maxUsedGpuMemory", + self.frameInfo.maxUsedGpuMemory), file=self.rqlog) print("%-20s%s" % ("utime", self.frameInfo.utime), file=self.rqlog) print("%-20s%s" % ("stime", self.frameInfo.stime), file=self.rqlog) print("%-20s%s" % ("renderhost", self.rqCore.machine.getHostname()), file=self.rqlog) @@ -531,7 +537,9 @@ def run(self): # Delay keeps the cuebot from spamming failing booking requests time.sleep(10) finally: - self.rqCore.releaseCores(self.runFrame.num_cores, runFrame.attributes.get('CPU_LIST')) + self.rqCore.releaseCores(self.runFrame.num_cores, runFrame.attributes.get('CPU_LIST'), + runFrame.attributes.get('GPU_LIST') + if 'GPU_LIST' in self.runFrame.attributes else None) self.rqCore.deleteFrame(self.runFrame.frame_id) @@ -733,7 +741,7 @@ def killAllFrame(self, reason): pass time.sleep(1) - def releaseCores(self, reqRelease, releaseHT=None): + def releaseCores(self, reqRelease, releaseHT=None, releaseGpus=None): """The requested number of cores are released @type reqRelease: int @param reqRelease: Number of cores to release, 100 = 1 physical core""" @@ -753,6 +761,9 @@ def releaseCores(self, reqRelease, releaseHT=None): if releaseHT: self.machine.releaseHT(releaseHT) + if releaseGpus: + self.machine.releaseGpus(releaseGpus) + finally: self.__threadLock.release() @@ -851,6 +862,11 @@ def launchFrame(self, runFrame): if reserveHT: runFrame.attributes['CPU_LIST'] = reserveHT + if runFrame.num_gpus: + reserveGpus = self.machine.reserveGpus(runFrame.num_gpus) + if reserveGpus: + runFrame.attributes['GPU_LIST'] = reserveGpus + # They must be available at this point, reserve them # pylint: disable=no-member self.cores.idle_cores -= runFrame.num_cores diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 915c13afc..3da883111 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -74,6 +74,7 @@ def __init__(self, rqCore, coreInfo): self.__rqCore = rqCore self.__coreInfo = coreInfo self.__tasksets = set() + self.__gpusets = set() if platform.system() == 'Linux': self.__vmstat = rqd.rqswap.VmStat() @@ -97,6 +98,7 @@ def __init__(self, rqCore, coreInfo): self.__pidHistory = {} self.setupHT() + self.setupGpu() def isNimbySafeToRunJobs(self): """Returns False if nimby should be triggered due to resource limits""" @@ -276,6 +278,14 @@ def rssUpdate(self, frames): frame.rss = rss frame.maxRss = max(rss, frame.maxRss) + if 'GPU_LIST' in frame.runFrame.attributes: + usedGpuMemory = 0 + for unitId in frame.runFrame.attributes.get('GPU_LIST').split(','): + usedGpuMemory += self.getGpuMemoryUsed(unitId) + + frame.usedGpuMemory = usedGpuMemory + frame.maxUsedGpuMemory = max(usedGpuMemory, frame.maxUsedGpuMemory) + if os.path.exists(frame.runFrame.log_dir_file): stat = os.stat(frame.runFrame.log_dir_file).st_mtime frame.lluTime = int(stat) @@ -315,44 +325,71 @@ def getBootTime(self): return int(line.split()[1]) return 0 + @rqd.rqutil.Memoize + def getGpuCount(self): + """Returns the total gpu's on the machine""" + return self.__getGpuValues()['count'] + @rqd.rqutil.Memoize def getGpuMemoryTotal(self): """Returns the total gpu memory in kb for CUE_GPU_MEMORY""" return self.__getGpuValues()['total'] - def getGpuMemory(self): + def getGpuMemoryFree(self): """Returns the available gpu memory in kb for CUE_GPU_MEMORY""" return self.__getGpuValues()['free'] + def getGpuMemoryUsed(self, unitId): + """Returns the available gpu memory in kb for CUE_GPU_MEMORY""" + usedMemory = self.__getGpuValues()['used'] + return usedMemory[unitId] if unitId in usedMemory else 0 + # pylint: disable=attribute-defined-outside-init + def __resetGpuResults(self): + self.gpuResults = {'count': 0, 'total': 0, 'free': 0, 'used': {}, 'updated': 0} + def __getGpuValues(self): if not hasattr(self, 'gpuNotSupported'): if not hasattr(self, 'gpuResults'): - self.gpuResults = {'total': 0, 'free': 0, 'updated': 0} + self.__resetGpuResults() if not rqd.rqconstants.ALLOW_GPU: self.gpuNotSupported = True return self.gpuResults - if self.gpuResults['updated'] > time.time() - 60: + if self.gpuResults['updated'] > int(time.time()) - 60: return self.gpuResults try: - # /shots/spi/home/bin/spinux1/cudaInfo - # /shots/spi/home/bin/rhel7/cudaInfo - cudaInfo = subprocess.getoutput('/usr/local/spi/rqd3/cudaInfo') - if 'There is no device supporting CUDA' in cudaInfo: - self.gpuNotSupported = True - else: - results = cudaInfo.splitlines()[-1].split() - # TotalMem 1023 Mb FreeMem 968 Mb - # The int(math.ceil(int(x) / 32.0) * 32) rounds up to the next multiple of 32 - self.gpuResults['total'] = ( - int(math.ceil(int(results[1]) / 32.0) * 32) * KILOBYTE) - self.gpuResults['free'] = int(results[4]) * KILOBYTE - self.gpuResults['updated'] = time.time() + nvidia_smi = subprocess.getoutput( + 'nvidia-smi --query-gpu=memory.total,memory.free,count' + ' --format=csv,noheader') + total = 0 + free = 0 + count = 0 + unitId = 0 + for line in nvidia_smi.splitlines(): + # Example "16130 MiB, 16103 MiB, 8" + # 1 MiB = 1048.576 KB + l = line.split() + unitTotal = math.ceil(int(l[0]) * 1048.576) + unitFree = math.ceil(int(l[2]) * 1048.576) + total += unitTotal + free += unitFree + count = int(l[-1]) + self.gpuResults['used'][str(unitId)] = unitTotal - unitFree + unitId += 1 + + self.gpuResults['total'] = int(total) + self.gpuResults['free'] = int(free) + self.gpuResults['count'] = count + self.gpuResults['updated'] = int(time.time()) # pylint: disable=broad-except except Exception as e: + self.gpuNotSupported = True + self.__resetGpuResults() log.warning( - 'Failed to get FreeMem from cudaInfo due to: %s at %s', + 'Failed to query nvidia-smi due to: %s at %s', e, traceback.extract_tb(sys.exc_info()[2])) + else: + self.__resetGpuResults() return self.gpuResults def __getSwapout(self): @@ -592,7 +629,10 @@ def updateMachineStats(self): self.__renderHost.free_swap = freeSwapMem self.__renderHost.free_mem = freeMem + cachedMem - self.__renderHost.attributes['freeGpu'] = str(self.getGpuMemory()) + self.__renderHost.num_gpus = self.getGpuCount() + self.__renderHost.total_gpu_mem = self.getGpuMemoryTotal() + self.__renderHost.free_gpu_mem = self.getGpuMemoryFree() + self.__renderHost.attributes['swapout'] = self.__getSwapout() elif platform.system() == 'Darwin': @@ -647,6 +687,10 @@ def setupHT(self): if self.__enabledHT(): self.__tasksets = set(range(self.__coreInfo.total_cores // 100)) + def setupGpu(self): + """ Setup rqd for Gpus """ + self.__gpusets = set(range(self.getGpuCount())) + def reserveHT(self, reservedCores): """ Reserve cores for use by taskset taskset -c 0,1,8,9 COMMAND @@ -698,3 +742,32 @@ def releaseHT(self, reservedHT): for core in reservedHT.split(','): if int(core) < self.__coreInfo.total_cores // 100: self.__tasksets.add(int(core)) + + def reserveGpus(self, reservedGpus): + """ Reserve gpus + @type reservedGpus: int + @param reservedGpus: The total gpus reserved by the frame. + @rtype: string + @return: The gpu-list. ex: '0,1,8,9' + """ + if len(self.__gpusets) < reservedGpus: + err = 'Not launching, insufficient GPUs to reserve based on reservedGpus' + log.critical(err) + raise rqd.rqexceptions.CoreReservationFailureException(err) + + gpusets = [] + for _ in range(reservedGpus): + gpu = self.__gpusets.pop() + gpusets.append(str(gpu)) + + return ','.join(gpusets) + + def releaseGpus(self, reservedGpus): + """ Release gpus + @type: string + @param: The gpu-list to release. ex: '0,1,8,9' + """ + log.debug('GPU set: Releasing gpu - %s', reservedGpus) + for gpu in reservedGpus.split(','): + if int(gpu) < self.getGpuCount(): + self.__gpusets.add(int(gpu)) diff --git a/rqd/rqd/rqnetwork.py b/rqd/rqd/rqnetwork.py index da5bee5cc..43abc8229 100644 --- a/rqd/rqd/rqnetwork.py +++ b/rqd/rqd/rqnetwork.py @@ -62,6 +62,10 @@ def __init__(self, rqCore, runFrame): self.vsize = 0 self.maxVsize = 0 + self.numGpus = 0 + self.usedGpuMemory = 0 + self.maxUsedGpuMemory = 0 + self.realtime = 0 self.utime = 0 self.stime = 0 @@ -84,7 +88,10 @@ def runningFrameInfo(self): max_vsize=self.maxVsize, vsize=self.vsize, attributes=self.runFrame.attributes, - llu_time=self.lluTime + llu_time=self.lluTime, + num_gpus=self.numGpus, + max_used_gpu_memory=self.maxUsedGpuMemory, + used_gpu_memory=self.usedGpuMemory ) return runningFrameInfo diff --git a/rqd/tests/rqconstants_tests.py b/rqd/tests/rqconstants_tests.py index 0df71790c..46da55f39 100644 --- a/rqd/tests/rqconstants_tests.py +++ b/rqd/tests/rqconstants_tests.py @@ -41,7 +41,6 @@ from .rqmachine_tests import ( CPUINFO, - CUDAINFO, LOADAVG_LOW_USAGE, MEMINFO_MODERATE_USAGE, PROC_STAT, @@ -78,7 +77,7 @@ def decorator(*args, **kwargs): return decorator -@mock.patch("subprocess.getoutput", new=mock.MagicMock(return_value=CUDAINFO)) +@mock.patch("subprocess.getoutput", new=mock.MagicMock(return_value="")) @mock.patch.object( rqd.rqutil.Memoize, "isCached", new=mock.MagicMock(return_value=False) ) diff --git a/rqd/tests/rqmachine_tests.py b/rqd/tests/rqmachine_tests.py index 7c7a650e1..b4bfe7bad 100644 --- a/rqd/tests/rqmachine_tests.py +++ b/rqd/tests/rqmachine_tests.py @@ -153,17 +153,13 @@ '16781318 0 0 0 0 17 4 0 0 0 0 0 6303248 6304296 23932928 140725890743234 ' '140725890743420 140725890743420 140725890744298 0') -CUDAINFO = ' TotalMem 1023 Mb FreeMem 968 Mb' - -@mock.patch('subprocess.getoutput', new=mock.MagicMock(return_value=CUDAINFO)) @mock.patch.object(rqd.rqutil.Memoize, 'isCached', new=mock.MagicMock(return_value=False)) @mock.patch('platform.system', new=mock.MagicMock(return_value='Linux')) @mock.patch('os.statvfs', new=mock.MagicMock()) @mock.patch('rqd.rqutil.getHostname', new=mock.MagicMock(return_value='arbitrary-hostname')) class MachineTests(pyfakefs.fake_filesystem_unittest.TestCase): - @mock.patch('subprocess.getoutput', new=mock.MagicMock(return_value=CUDAINFO)) @mock.patch('os.statvfs', new=mock.MagicMock()) @mock.patch('platform.system', new=mock.MagicMock(return_value='Linux')) def setUp(self): @@ -318,29 +314,39 @@ def test_getBootTime(self): self.assertEqual(1569882758, self.machine.getBootTime()) - @mock.patch( - 'subprocess.getoutput', - new=mock.MagicMock(return_value=' TotalMem 1023 Mb FreeMem 968 Mb')) - def test_getGpuMemoryTotal(self): + def _resetGpuStat(self): if hasattr(self.machine, 'gpuNotSupported'): delattr(self.machine, 'gpuNotSupported') if hasattr(self.machine, 'gpuResults'): delattr(self.machine, 'gpuResults') - rqd.rqconstants.ALLOW_GPU = True - - self.assertEqual(1048576, self.machine.getGpuMemoryTotal()) - @mock.patch( - 'subprocess.getoutput', - new=mock.MagicMock(return_value=' TotalMem 1023 Mb FreeMem 968 Mb')) - def test_getGpuMemory(self): - if hasattr(self.machine, 'gpuNotSupported'): - delattr(self.machine, 'gpuNotSupported') - if hasattr(self.machine, 'gpuResults'): - delattr(self.machine, 'gpuResults') - rqd.rqconstants.ALLOW_GPU = True + @mock.patch.object( + rqd.rqconstants, 'ALLOW_GPU', new=mock.MagicMock(return_value=True)) + @mock.patch('subprocess.getoutput', + new=mock.MagicMock(return_value='16130 MiB, 16119 MiB, 1')) + def test_getGpuStat(self): + self._resetGpuStat() + self.assertEqual(1, self.machine.getGpuCount()) + self.assertEqual(16913531, self.machine.getGpuMemoryTotal()) + self.assertEqual(16901997, self.machine.getGpuMemoryFree()) - self.assertEqual(991232, self.machine.getGpuMemory()) + @mock.patch.object( + rqd.rqconstants, 'ALLOW_GPU', new=mock.MagicMock(return_value=True)) + @mock.patch('subprocess.getoutput', + new=mock.MagicMock(return_value="""\ +16130 MiB, 16103 MiB, 8 +16130 MiB, 16119 MiB, 8 +16130 MiB, 16119 MiB, 8 +16130 MiB, 16119 MiB, 8 +16130 MiB, 4200 MiB, 8 +16130 MiB, 16119 MiB, 8 +16130 MiB, 16119 MiB, 8 +16130 MiB, 16119 MiB, 8""")) + def test_multipleGpus(self): + self._resetGpuStat() + self.assertEqual(8, self.machine.getGpuCount()) + self.assertEqual(135308248, self.machine.getGpuMemoryTotal()) + self.assertEqual(122701222, self.machine.getGpuMemoryFree()) def test_getPathEnv(self): self.assertEqual( @@ -365,16 +371,12 @@ def test_reboot(self, popenMock): popenMock.assert_called_with(['/usr/bin/sudo', '/sbin/reboot', '-f']) - @mock.patch( - 'subprocess.getoutput', - new=mock.MagicMock(return_value=' TotalMem 1023 Mb FreeMem 968 Mb')) def test_getHostInfo(self): # pylint: disable=no-member hostInfo = self.machine.getHostInfo() self.assertEqual(4105212, hostInfo.free_swap) self.assertEqual(25699176, hostInfo.free_mem) - self.assertEqual('991232', hostInfo.attributes['freeGpu']) self.assertEqual('0', hostInfo.attributes['swapout']) self.assertEqual(25, hostInfo.load) self.assertEqual(False, hostInfo.nimby_enabled) From 963eff2a8646b7c6f015ff45770e1569c739415f Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Sun, 20 Jun 2021 12:46:47 -0400 Subject: [PATCH 104/277] Fix JobDao unit test. (#992) --- .../com/imageworks/spcue/test/dao/postgres/JobDaoTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/JobDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/JobDaoTests.java index 597489592..d4163d7f6 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/JobDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/JobDaoTests.java @@ -227,7 +227,7 @@ public void testGetJobDetail() { assertEquals(job.localMaxCores, src.localMaxCores); assertEquals(job.localMaxMemory, src.localMaxMemory); assertEquals(job.localThreadNumber, src.localThreadNumber); - assertEquals(job.localMaxGpu, src.localMaxGpu); + assertEquals(job.localMaxGpus, src.localMaxGpus); } @Test From b7d06de91885360abe10d158b64ef3d0fe78b4eb Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Tue, 29 Jun 2021 11:04:42 -0700 Subject: [PATCH 105/277] Fix crash on macOS by GRPC_POLL_STRATEGY=epoll1 (#995) --- pycue/opencue/cuebot.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pycue/opencue/cuebot.py b/pycue/opencue/cuebot.py index 4589806c3..78e9aec8c 100644 --- a/pycue/opencue/cuebot.py +++ b/pycue/opencue/cuebot.py @@ -23,6 +23,7 @@ import atexit import logging import os +import platform import yaml import grpc @@ -77,8 +78,9 @@ DEFAULT_MAX_MESSAGE_BYTES = 1024 ** 2 * 10 DEFAULT_GRPC_PORT = 8443 -# Avoid spamming users with epoll fork warning messages -os.environ["GRPC_POLL_STRATEGY"] = "epoll1" +if platform.system() != 'Darwin': + # Avoid spamming users with epoll fork warning messages + os.environ["GRPC_POLL_STRATEGY"] = "epoll1" class Cuebot(object): """Used to manage the connection to the Cuebot. Normally the connection From 97f38b980749081c6491be7f3c0cb05d4a4faf13 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Sun, 18 Jul 2021 14:50:46 -0400 Subject: [PATCH 106/277] Split Python linting into its own CI job. (#993) --- .github/workflows/testing-pipeline.yml | 11 +++++- ci/run_python_lint.sh | 48 ++++++++++++++++++++++++++ ci/run_python_tests.sh | 23 ------------ 3 files changed, 58 insertions(+), 24 deletions(-) create mode 100755 ci/run_python_lint.sh diff --git a/.github/workflows/testing-pipeline.yml b/.github/workflows/testing-pipeline.yml index 5d8acd88a..3e63f35d1 100644 --- a/.github/workflows/testing-pipeline.yml +++ b/.github/workflows/testing-pipeline.yml @@ -35,7 +35,16 @@ jobs: steps: - uses: actions/checkout@v2 - name: Run Python Tests - run: ci/run_python_tests.sh --lint + run: ci/run_python_tests.sh + + lint_python_2020: + name: Lint Python Code (CY2020) + runs-on: ubuntu-latest + container: aswf/ci-opencue:2020 + steps: + - uses: actions/checkout@v2 + - name: Lint Python Code + run: ci/run_python_lint.sh test_cuebot_2020: name: Build Cuebot and Run Unit Tests (CY2020) diff --git a/ci/run_python_lint.sh b/ci/run_python_lint.sh new file mode 100755 index 000000000..0311854b2 --- /dev/null +++ b/ci/run_python_lint.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +set -e + +python_version=$(python -V) +echo "Will run Python lint using ${python_version}" + +pip install --user -r requirements.txt -r requirements_gui.txt + +# Protos need to have their Python code generated in order for tests to pass. +python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto +python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto + +# Fix imports to work in both Python 2 and 3. See +# for more info. +2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py + +cd pycue +python -m pylint --rcfile=../ci/pylintrc_main FileSequence +python -m pylint --rcfile=../ci/pylintrc_main opencue --ignore=opencue/compiled_proto +python -m pylint --rcfile=../ci/pylintrc_test tests +cd .. + +cd pyoutline +PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main outline +PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests +cd .. + +cd cueadmin +PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main cueadmin +PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests +cd .. + +cd cuegui +PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main cuegui --ignore=cuegui/images,cuegui/images/crystal +PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests +cd .. + +cd cuesubmit +PYTHONPATH=../pycue:../pyoutline python -m pylint --rcfile=../ci/pylintrc_main cuesubmit +PYTHONPATH=../pycue:../pyoutline python -m pylint --rcfile=../ci/pylintrc_test tests +cd .. + +cd rqd +python -m pylint --rcfile=../ci/pylintrc_main rqd --ignore=rqd/compiled_proto +python -m pylint --rcfile=../ci/pylintrc_test tests +cd .. diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index 14316b244..093ab9c62 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -26,26 +26,3 @@ python rqd/setup.py test if [[ "$python_version" =~ "Python 3" ]]; then PYTHONPATH=pycue xvfb-run -d python cuegui/setup.py test fi - -# Some environments don't have pylint available, for ones that do they should pass this flag. -if [[ "$1" == "--lint" ]]; then - cd pycue && python -m pylint --rcfile=../ci/pylintrc_main FileSequence && cd .. - cd pycue && python -m pylint --rcfile=../ci/pylintrc_main opencue --ignore=opencue/compiled_proto && cd .. - cd pycue && python -m pylint --rcfile=../ci/pylintrc_test tests && cd .. - - cd pyoutline && PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main outline && cd .. - cd pyoutline && PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests && cd .. - - cd cueadmin && PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main cueadmin && cd .. - cd cueadmin && PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests && cd .. - - cd cuegui && PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main cuegui --ignore=cuegui/images,cuegui/images/crystal && cd .. - cd cuegui && PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests && cd .. - - cd cuesubmit && PYTHONPATH=../pycue:../pyoutline python -m pylint --rcfile=../ci/pylintrc_main cuesubmit && cd .. - cd cuesubmit && PYTHONPATH=../pycue:../pyoutline python -m pylint --rcfile=../ci/pylintrc_test tests && cd .. - - cd rqd && python -m pylint --rcfile=../ci/pylintrc_main rqd --ignore=rqd/compiled_proto && cd .. - cd rqd && python -m pylint --rcfile=../ci/pylintrc_test tests && cd .. -fi - From 377e8d65238c0d343199e9778b88019263c9e3d0 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Sun, 18 Jul 2021 11:56:48 -0700 Subject: [PATCH 107/277] Fix Grpc unavailable error. (#859) --- pycue/opencue/cuebot.py | 114 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 111 insertions(+), 3 deletions(-) diff --git a/pycue/opencue/cuebot.py b/pycue/opencue/cuebot.py index 78e9aec8c..1e91251e0 100644 --- a/pycue/opencue/cuebot.py +++ b/pycue/opencue/cuebot.py @@ -20,6 +20,8 @@ from builtins import object from random import shuffle +import abc +import time import atexit import logging import os @@ -168,6 +170,18 @@ def setChannel(): hosts = list(Cuebot.Hosts) shuffle(hosts) maxMessageBytes = config.get('cuebot.max_message_bytes', DEFAULT_MAX_MESSAGE_BYTES) + + # create interceptors + interceptors = ( + RetryOnRpcErrorClientInterceptor( + max_attempts=4, + sleeping_policy=ExponentialBackoff(init_backoff_ms=100, + max_backoff_ms=1600, + multiplier=2), + status_for_retry=(grpc.StatusCode.UNAVAILABLE,), + ), + ) + for host in hosts: if ':' in host: connectStr = host @@ -176,9 +190,11 @@ def setChannel(): logger.debug('connecting to gRPC at %s', connectStr) # TODO(bcipriano) Configure gRPC TLS. (Issue #150) try: - Cuebot.RpcChannel = grpc.insecure_channel(connectStr, options=[ - ('grpc.max_send_message_length', maxMessageBytes), - ('grpc.max_receive_message_length', maxMessageBytes)]) + Cuebot.RpcChannel = grpc.intercept_channel( + grpc.insecure_channel(connectStr, options=[ + ('grpc.max_send_message_length', maxMessageBytes), + ('grpc.max_receive_message_length', maxMessageBytes)]), + *interceptors) # Test the connection Cuebot.getStub('cue').GetSystemStats( cue_pb2.CueGetSystemStatsRequest(), timeout=Cuebot.Timeout) @@ -275,3 +291,95 @@ def getStub(cls, name): def getConfig(): """Gets the Cuebot config object, originally read in from the config file on disk.""" return config + + +# Python 2/3 compatible implementation of ABC +ABC = abc.ABCMeta('ABC', (object,), {'__slots__': ()}) + + +class SleepingPolicy(ABC): + """ + Implement policy for sleeping between API retries + """ + @abc.abstractmethod + def sleep(self, attempt): + """ + How long to sleep in milliseconds. + :param attempt: the number of attempt (starting from zero) + """ + assert attempt >= 0 + + +class ExponentialBackoff(SleepingPolicy): + """ + Implement policy that will increase retry period by exponentially in every try + """ + def __init__(self, + init_backoff_ms, + max_backoff_ms, + multiplier=2): + """ + inputs in ms + """ + self._init_backoff = init_backoff_ms + self._max_backoff = max_backoff_ms + self._multiplier = multiplier + + def sleep(self, attempt): + sleep_time_ms = min( + self._init_backoff * self._multiplier ** attempt, + self._max_backoff + ) + time.sleep(sleep_time_ms / 1000.0) + + +class RetryOnRpcErrorClientInterceptor( + grpc.UnaryUnaryClientInterceptor, + grpc.StreamUnaryClientInterceptor +): + """ + Implement Client/Stream interceptors for GRPC channels to retry + calls that failed with retry-able states. This is required for + handling server interruptions that are not automatically handled + by grpc.insecure_channel + """ + def __init__(self, + max_attempts, + sleeping_policy, + status_for_retry=None): + self._max_attempts = max_attempts + self._sleeping_policy = sleeping_policy + self._retry_statuses = status_for_retry + + def _intercept_call(self, continuation, client_call_details, + request_or_iterator): + for attempt in range(self._max_attempts): + try: + return continuation(client_call_details, + request_or_iterator) + except grpc.RpcError as response: + # Return if it was last attempt + if attempt == (self._max_attempts - 1): + return response + + # If status code is not in retryable status codes + if self._retry_statuses \ + and hasattr(response, 'code') \ + and response.code() \ + not in self._retry_statuses: + return response + + self._sleeping_policy.sleep(attempt) + except Exception: + raise + + def intercept_unary_unary(self, continuation, client_call_details, + request): + return self._intercept_call(continuation, client_call_details, + request) + + def intercept_stream_unary( + self, continuation, client_call_details, request_iterator + ): + return self._intercept_call(continuation, client_call_details, + request_iterator) From a444043d5e03caeabe726d68304cf94bfb8e89f5 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Sun, 18 Jul 2021 17:52:04 -0400 Subject: [PATCH 108/277] Fix lint warnings. (#996) --- pycue/opencue/cuebot.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pycue/opencue/cuebot.py b/pycue/opencue/cuebot.py index 1e91251e0..0e143e119 100644 --- a/pycue/opencue/cuebot.py +++ b/pycue/opencue/cuebot.py @@ -363,6 +363,7 @@ def _intercept_call(self, continuation, client_call_details, return response # If status code is not in retryable status codes + # pylint: disable=no-member if self._retry_statuses \ and hasattr(response, 'code') \ and response.code() \ @@ -370,8 +371,6 @@ def _intercept_call(self, continuation, client_call_details, return response self._sleeping_policy.sleep(attempt) - except Exception: - raise def intercept_unary_unary(self, continuation, client_call_details, request): From dc75d31bb620f1246d9ed82239ff75172239e8c7 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Sun, 18 Jul 2021 18:01:46 -0400 Subject: [PATCH 109/277] Add some log messages to run_python_lint. (#997) --- ci/run_python_lint.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ci/run_python_lint.sh b/ci/run_python_lint.sh index 0311854b2..bd86c9188 100755 --- a/ci/run_python_lint.sh +++ b/ci/run_python_lint.sh @@ -16,32 +16,38 @@ python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py 2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py +echo "Running lint for pycue/..." cd pycue python -m pylint --rcfile=../ci/pylintrc_main FileSequence python -m pylint --rcfile=../ci/pylintrc_main opencue --ignore=opencue/compiled_proto python -m pylint --rcfile=../ci/pylintrc_test tests cd .. +echo "Running lint for pyoutline/..." cd pyoutline PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main outline PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests cd .. +echo "Running lint for cueadmin/..." cd cueadmin PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main cueadmin PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests cd .. +echo "Running lint for cuegui/..." cd cuegui PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main cuegui --ignore=cuegui/images,cuegui/images/crystal PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests cd .. +echo "Running lint for cuesubmit/..." cd cuesubmit PYTHONPATH=../pycue:../pyoutline python -m pylint --rcfile=../ci/pylintrc_main cuesubmit PYTHONPATH=../pycue:../pyoutline python -m pylint --rcfile=../ci/pylintrc_test tests cd .. +echo "Running lint for rqd/..." cd rqd python -m pylint --rcfile=../ci/pylintrc_main rqd --ignore=rqd/compiled_proto python -m pylint --rcfile=../ci/pylintrc_test tests From 97fd3081a1862972b26e6e20ddce5e02a44c0291 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Sun, 18 Jul 2021 18:49:04 -0400 Subject: [PATCH 110/277] Notes from June 9 and June 23 TSC meetings. (#998) --- tsc/meetings/2021-06-09.md | 115 +++++++++++++++++++++++++++++++++++ tsc/meetings/2021-06-23.md | 120 +++++++++++++++++++++++++++++++++++++ 2 files changed, 235 insertions(+) create mode 100644 tsc/meetings/2021-06-09.md create mode 100644 tsc/meetings/2021-06-23.md diff --git a/tsc/meetings/2021-06-09.md b/tsc/meetings/2021-06-09.md new file mode 100644 index 000000000..1bef06ccf --- /dev/null +++ b/tsc/meetings/2021-06-09.md @@ -0,0 +1,115 @@ +# OpenCue TSC Meeting Notes 9 June 2021 + +Secretary: Brian Cipriano + +TSC Attendees: + +* [ ] Matt Chambers +* [x] Brian Cipriano +* [x] Greg Denton +* [x] Ben Dines +* [ ] Dave Fellows +* [x] Daniel Neilson +* [ ] Alex Schworer +* [ ] Erik Strauss +* [ ] Lars van der Bijl + +Committer Attendees: + +* [ ] Christian Smith +* [x] Diego Tavares +* [x] Idris Miles + +Agenda/Notes: + +* Goals for 2021 + * ~~User survey~~ + * ~~ASWF Graduation~~ + * New user UX + * Rename demo_data.sql + * Main code change done. + * Docs updated. + * Still todo: updating release pipeline to publish seed_data artifact. + * Publish PyPI packages + * Design + doc: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/ + * Proposal for config file + standardization: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/edit#heading=h.v85uo9klwqft + * Config cleanup for pycue done and ready for review. Tested on macOS and Windows. + * Next up will do config cleanup on other components. + * Fix cuebot:latest tag for sandbox setup + * Proposal: publish Docker images for every new patch version (i.e. every new commit to + master) + * Proposal accepted. + * No progress implementing this yet. + * Related: for PyPI design we may want to also publish packages for every new patch + version. + * Docs refresh + * User guide refresh + * Split into "install from latest release" and "install from master" + * Needed to assist transition to publish Docker images on all commits + * No progress. + * API reference + * We have the ability to generate HTML from any commit, but we need a new CI pipeline to + build and publish this on opencue.io for new releases. + * Proposal: for now, highlight existing docs for building reference locally + * Proposal accepted. + * Brian: cleaned up local build process, sent email with doc + link: https://www.opencue.io/contributing/opencue/build-docs/ + * pyoutline examples + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/177 + * No progress yet. + * User guide for CueGUI + * No progress. + * Configuration guide for client-side python + * In progress, will be done as part of PyPI work. + * Configuration guide for Cuebot + * In progress, will be done as part of PyPI work. + * Mention tuning RQD connection cache size to number of expected hosts. + * Configuration guide for RQD + * In progress, will be done as part of PyPI work. + * Improve scheduler logic + * Brian: couldn't locate old docs, email out to Matt to see if he has any materials around. + * Need an owner for this work. Diego has volunteered. + * Haven't been able to reach Matt yet. + * Let's try to locate Matt's fork from a while back. + * Might speak to Weta for advice as well. + * ~~Drop Oracle support~~ + * ~~Done.~~ + * GPU support + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/459 + * Starting PR in review: https://github.com/AcademySoftwareFoundation/OpenCue/pull/924 + * PR approved, ready to merge. Could use some production testing. + * ~~GUI to add new shows~~ + * ~~Done.~~ + * Expand DCC plugins + * No progress. + * Logging solution + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/880 + * PR approved and merged. + * Proposal: write blog post on this topic + * Proposal accepted + * No progress on this yet + * CSP Terraform docs + * No progress yet. + * Auto-close Github issues + * Proposal: + * Any issues in a Project have been identified as important and will be ignored. + * Other issues: + * If no update for 60 days, will get the label "stale" and a comment notifying of + auto-close. + * If no further update for 7 days, will be closed. + * Any issue update will reset the timer — remove the "stale" tag + * Action https://github.com/marketplace/actions/close-stale-issues will be used. + * May need another action to label issues if they are in a project, the above action doesn't + appear to have options for ignoring issues based on project membership. +* Other current work updates + * Bug where services would need restart + * cuebot -> rqd module, calls not getting responses + * needed to set deadline on all grpc calls, fixed issue + * cuebot threadpools able to self-heal now + * Cache of RQD connections + * need to tune cache size to about number of expected RQD hosts + * Set concurrency cache settings + * set to number of threads expected + * Issue about RQD hardcoded timezone diff --git a/tsc/meetings/2021-06-23.md b/tsc/meetings/2021-06-23.md new file mode 100644 index 000000000..669e7cbcc --- /dev/null +++ b/tsc/meetings/2021-06-23.md @@ -0,0 +1,120 @@ +# OpenCue TSC Meeting Notes 23 June 2021 + +Secretary: Brian Cipriano + +TSC Attendees: + +* [ ] Matt Chambers +* [x] Brian Cipriano +* [x] Greg Denton +* [x] Ben Dines +* [ ] Dave Fellows +* [x] Daniel Neilson +* [ ] Alex Schworer +* [ ] Erik Strauss +* [ ] Lars van der Bijl + +Committer Attendees: + +* [ ] Christian Smith +* [x] Diego Tavares +* [ ] Idris Miles + +Agenda/Notes: + +* Completed 2021 goals + * User survey + * ASWF graduation + * Drop Oracle support + * GUI to add new shows +* Todo 2021 goals + * New user UX + * Rename demo_data.sql + * Main code change done. + * Docs updated. + * Release pipeline updated. Done! + * Publish PyPI packages + * Design + doc: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/ + * Proposal for config file + standardization: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/edit#heading=h.v85uo9klwqft + * Config cleanup for pycue done and ready for review. Tested on macOS and Windows. + * Next up will do config cleanup on other components. + * Fix cuebot:latest tag for sandbox setup + * Proposal: publish Docker images for every new patch version (i.e. every new commit to + master) + * Proposal accepted. + * No progress implementing this yet. + * Related: for PyPI design we may want to also publish packages for every new patch + version. + * Docs refresh + * User guide refresh + * Split into "install from latest release" and "install from master" + * Needed to assist transition to publish Docker images on all commits + * No progress. + * API reference + * We have the ability to generate HTML from any commit, but we need a new CI pipeline to + build and publish this on opencue.io for new releases. + * Proposal: for now, highlight existing docs for building reference locally + * Proposal accepted. + * Brian: cleaned up local build process, sent email with doc + link: https://www.opencue.io/contributing/opencue/build-docs/ + * pyoutline examples + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/177 + * No progress yet. + * User guide for CueGUI + * No progress. + * Configuration guide for client-side python + * In progress, will be done as part of PyPI work. + * Configuration guide for Cuebot + * In progress, will be done as part of PyPI work. + * Configuration guide for RQD + * In progress, will be done as part of PyPI work. + * Improve scheduler logic + * Diego has volunteered to own this. + * Email reply from Matt, looped in others. Let's continue the discussion there. + * GPU support + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/459 + * Starting PR in review: https://github.com/AcademySoftwareFoundation/OpenCue/pull/924 + * PR merged. This is done for now pending further testing, after which we should improve + Windows support. + * Expand DCC plugins + * No progress. + * Logging solution + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/880 + * PR approved and merged. + * Proposal: write blog post on this topic + * Proposal accepted + * No progress on this yet + * CSP Terraform docs + * No progress yet. + * Auto-close Github issues + * Proposal: + * Any issues in a Project have been identified as important and will be ignored. + * Other issues: + * If no update for 60 days, will get the label "stale" and a comment notifying of + auto-close. + * If no further update for 7 days, will be closed. + * Any issue update will rest the timer. + * Action https://github.com/marketplace/actions/close-stale-issues will be used. + * May need another action to label issues if they are in a project, the above action doesn't + appear to have options for ignoring issues based on project membership. +* Other current work updates + * Brian: split lint into its own CI job to avoid confusion, reduce runtime. + * Diego + * grpc connections PR + * threadpools PR + * Discussion to revive cloud work/plugin + * GSoC Cloud Plugin + * Is very simple, provides ability to manually adjust instance group size. + * Need to finalize GSoC Azure support, Greg volunteered to test this. + * Need to add AWS support. + * Beyond that, we'd like to allow autoscaling based on the amount of work in the queue. + * We need to be very careful with this as bugs could cause user money to be burned on + cloud resources. We should provide the tools for users to build this themselves but + stop short of building this fully into Cuebot. +* Open Source Days 2021 + * Need rough idea and presenters list. + * Brian will give short project update at beginning of session. + * Diego has volunteered to present some of SPI's work, Ben to assist. + * Let's start an email thread to discuss further. From 87bcf7ab6eb246afb70e7259c7d9f7946f556bf5 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Mon, 19 Jul 2021 08:31:37 -0700 Subject: [PATCH 111/277] Add deadline for rpcClient calls (#994) * Update dispatchQuery to use min_cores Sorting jobs only by priority causes a situation where low priority jobs can get starved by a constant flow of high priority jobs. The new formula adds a modifier to the sorting rank to take into account the number of cores the job is requesting and also the number of days the job is waiting on the queue. Priorities numbers over 200 will mostly override the formula and work as a priority only based scheduling. sort = priority + (100 * (1 - (job.cores/job.int_min_cores))) + (age in days) Besides that, also take layer_int_cores_min into account when filtering folder_resourse limitations to avoid allocating more cores than the folder limits. (cherry picked from commit 566411aeeddc60983a30eabe121fd03263d05525) * Revert "Update dispatchQuery to use min_cores" This reverts commit 2eb4936c * Add deadline for rpcClient calls Calls are frequently getting locked, setting a timeout is strongly recommended on the grpc documentation. Change rqd cache arguments Increasing the cache size to reduce cacheMiss (Currently at 50%) and reduce expiration to avoid too many channels idle. --- cuebot/build.gradle | 2 +- .../imageworks/spcue/rqd/RqdClientGrpc.java | 21 ++++++++++++++----- .../spring/applicationContext-service.xml | 6 ++++++ cuebot/src/main/resources/opencue.properties | 8 +++++-- cuebot/src/test/resources/opencue.properties | 4 ++++ 5 files changed, 33 insertions(+), 8 deletions(-) diff --git a/cuebot/build.gradle b/cuebot/build.gradle index d4f00e193..4f93bd229 100644 --- a/cuebot/build.gradle +++ b/cuebot/build.gradle @@ -44,7 +44,7 @@ dependencies { compile group: 'com.google.guava', name: 'guava', version: '26.0-android' compile group: 'com.sun.mail', name: 'mailapi', version: '1.5.4' compile group: 'commons-lang', name: 'commons-lang', version: '2.6' - compile group: 'io.grpc', name: 'grpc-all', version: '1.14.0' + compile group: 'io.grpc', name: 'grpc-all', version: '1.36.2' compile group: 'org.apache.activemq', name: 'activemq-pool', version: activemqVersion compile group: 'org.apache.velocity', name: 'velocity', version: '1.7' compile group: 'org.jdom', name: 'jdom', version: '1.1.3' diff --git a/cuebot/src/main/java/com/imageworks/spcue/rqd/RqdClientGrpc.java b/cuebot/src/main/java/com/imageworks/spcue/rqd/RqdClientGrpc.java index 6aa6866ea..d9ad91cfd 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/rqd/RqdClientGrpc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/rqd/RqdClientGrpc.java @@ -53,21 +53,27 @@ public final class RqdClientGrpc implements RqdClient { private final int rqdCacheSize; private final int rqdCacheExpiration; + private final int rqdCacheConcurrency; private final int rqdServerPort; + private final int rqdTaskDeadlineSeconds; private LoadingCache channelCache; private boolean testMode = false; - public RqdClientGrpc(int rqdServerPort, int rqdCacheSize, int rqdCacheExpiration) { + public RqdClientGrpc(int rqdServerPort, int rqdCacheSize, int rqdCacheExpiration, + int rqdCacheConcurrency, int rqdTaskDeadline) { this.rqdServerPort = rqdServerPort; this.rqdCacheSize = rqdCacheSize; this.rqdCacheExpiration = rqdCacheExpiration; + this.rqdCacheConcurrency = rqdCacheConcurrency; + this.rqdTaskDeadlineSeconds = rqdTaskDeadline; } private void buildChannelCache() { this.channelCache = CacheBuilder.newBuilder() .maximumSize(rqdCacheSize) + .concurrencyLevel(rqdCacheConcurrency) .expireAfterAccess(rqdCacheExpiration, TimeUnit.MINUTES) .removalListener(new RemovalListener() { @Override @@ -80,8 +86,9 @@ public void onRemoval(RemovalNotification removal){ new CacheLoader() { @Override public ManagedChannel load(String host) throws Exception { - ManagedChannelBuilder channelBuilder = ManagedChannelBuilder.forAddress( - host, rqdServerPort).usePlaintext(); + ManagedChannelBuilder channelBuilder = ManagedChannelBuilder + .forAddress(host, rqdServerPort) + .usePlaintext(); return channelBuilder.build(); } }); @@ -92,7 +99,9 @@ private RqdInterfaceGrpc.RqdInterfaceBlockingStub getStub(String host) throws Ex buildChannelCache(); } ManagedChannel channel = channelCache.get(host); - return RqdInterfaceGrpc.newBlockingStub(channel); + return RqdInterfaceGrpc + .newBlockingStub(channel) + .withDeadlineAfter(rqdTaskDeadlineSeconds, TimeUnit.SECONDS); } private RunningFrameGrpc.RunningFrameBlockingStub getRunningFrameStub(String host) throws ExecutionException { @@ -100,7 +109,9 @@ private RunningFrameGrpc.RunningFrameBlockingStub getRunningFrameStub(String hos buildChannelCache(); } ManagedChannel channel = channelCache.get(host); - return RunningFrameGrpc.newBlockingStub(channel); + return RunningFrameGrpc + .newBlockingStub(channel) + .withDeadlineAfter(rqdTaskDeadlineSeconds, TimeUnit.SECONDS); } public void setHostLock(HostInterface host, LockState lock) { diff --git a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml index 24bd94195..5f1738d69 100644 --- a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml +++ b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml @@ -46,6 +46,12 @@ ${grpc.rqd_cache_expiration} + + ${grpc.rqd_cache_concurrency} + + + ${grpc.rqd_task_deadline} + diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index 82404ee1c..4df5d601f 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -13,9 +13,13 @@ grpc.cue_port=${CUEBOT_GRPC_CUE_PORT:8443} grpc.rqd_server_port=${CUEBOT_GRPC_RQD_SERVER_PORT:8444} grpc.max_message_bytes=104857600 # Number of entries allowed in the RQD channel cache -grpc.rqd_cache_size=500 +grpc.rqd_cache_size=2000 # RQD Channel Cache Expiration in Minutes -grpc.rqd_cache_expiration=30 +grpc.rqd_cache_expiration=5 +# RQD Channel Cache expected concurrency +grpc.rqd_cache_concurrency=20 +# RQD Channel task deadline in seconds +grpc.rqd_task_deadline=10 # Whether or not to enable publishing to a messaging topic. # Set to a boolean value. See com/imageworks/spcue/services/JmsMover.java. diff --git a/cuebot/src/test/resources/opencue.properties b/cuebot/src/test/resources/opencue.properties index 38dc15ea9..5ac924592 100644 --- a/cuebot/src/test/resources/opencue.properties +++ b/cuebot/src/test/resources/opencue.properties @@ -8,6 +8,10 @@ grpc.max_message_bytes=104857600 grpc.rqd_cache_size=500 # RQD Channel Cache Expiration in Minutes grpc.rqd_cache_expiration=30 +# RQD Channel Cache expected concurrency +grpc.rqd_cache_concurrency=20 +# RQD Channel task deadline in seconds +grpc.rqd_task_deadline=10 log.frame-log-root=/arbitraryLogDirectory From 08c8d7f635c6cd9ede3649ef981def18c50f479f Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Thu, 29 Jul 2021 11:21:29 -0700 Subject: [PATCH 112/277] Keep staggerFrames interleaving start frame static. (#975) --- .../src/main/java/com/imageworks/spcue/util/FrameRange.java | 5 ++--- .../java/com/imageworks/spcue/test/util/FrameRangeTests.java | 4 ++-- .../java/com/imageworks/spcue/test/util/FrameSetTests.java | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/util/FrameRange.java b/cuebot/src/main/java/com/imageworks/spcue/util/FrameRange.java index 58713d49d..2e601f72c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/util/FrameRange.java +++ b/cuebot/src/main/java/com/imageworks/spcue/util/FrameRange.java @@ -62,7 +62,7 @@ public class FrameRange { * set of frames unique from the first set. This process is repeated until interleaveSize * reaches 1. * - * Example: 1-10:5 == 1, 6, 2, 4, 8, 10, 3, 5, 7, 9. + * Example: 1-10:5 == 1, 6, 3, 5 ,7 ,9, 2, 4, 8, 10. */ public FrameRange(String frameRange) { frameList = parseFrameRange(frameRange); @@ -166,10 +166,9 @@ private static ImmutableList getSteppedRange( private static ImmutableList getInterleavedRange(Integer start, Integer end, Integer step) { validateStepSign(start, end, step); Set interleavedFrames = new LinkedHashSet<>(); - int incrValue = step / abs(step); + while (abs(step) > 0) { interleavedFrames.addAll(getIntRange(start, end, step)); - start += incrValue; step /= 2; } return ImmutableList.copyOf(interleavedFrames); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/util/FrameRangeTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/util/FrameRangeTests.java index a49bf88c3..aa9cfcc07 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/util/FrameRangeTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/util/FrameRangeTests.java @@ -103,14 +103,14 @@ public void testNegativeInvertedStep() { public void testInterleave() { FrameRange result = new FrameRange("1-10:5"); - assertThat(result.getAll()).containsExactly(1, 6, 2, 4, 8, 10, 3, 5, 7, 9); + assertThat(result.getAll()).containsExactly(1, 6, 3, 5, 7, 9, 2, 4, 8, 10); } @Test public void testNegativeInterleave() { FrameRange result = new FrameRange("10-1:-5"); - assertThat(result.getAll()).containsExactly(10, 5, 9, 7, 3, 1, 8, 6, 4, 2); + assertThat(result.getAll()).containsExactly(10, 5, 8, 6, 4, 2, 9, 7, 3, 1); } @Test diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/util/FrameSetTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/util/FrameSetTests.java index 223813f46..404d53b0f 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/util/FrameSetTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/util/FrameSetTests.java @@ -12,7 +12,7 @@ public void shouldSplitListAndMaintainOrder() { FrameSet result = new FrameSet("57,1-3,4-2,12-15x2,76-70x-3,5-12y3,1-7:5"); assertThat(result.getAll()).containsExactly( - 57, 1, 2, 3, 4, 3, 2, 12, 14, 76, 73, 70, 6, 7, 9, 10, 12, 1, 6, 2, 4, 3, 5, 7); + 57, 1, 2, 3, 4, 3, 2, 12, 14, 76, 73, 70, 6, 7, 9, 10, 12, 1, 6, 3, 5, 7, 2, 4); } @Test From f0b64ec1d96c4195855822ca3050f726383a8c7e Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Thu, 29 Jul 2021 12:35:18 -0700 Subject: [PATCH 113/277] Sortlambda works on all rpcObject types (#986) --- cuegui/cuegui/AbstractWidgetItem.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cuegui/cuegui/AbstractWidgetItem.py b/cuegui/cuegui/AbstractWidgetItem.py index fec1dc5fe..548dde72a 100644 --- a/cuegui/cuegui/AbstractWidgetItem.py +++ b/cuegui/cuegui/AbstractWidgetItem.py @@ -27,9 +27,6 @@ from PySide2 import QtCore from PySide2 import QtWidgets -import opencue -import opencue.wrappers.job - import cuegui.Constants import cuegui.Logger import cuegui.Style @@ -92,14 +89,15 @@ def data(self, col, role): return cuegui.Constants.QVARIANT_NULL def __lt__(self, other): - """Custom sorting for columns that have a function defined for sorting""" + """Custom sorting for columns that have a function defined for sorting + (uses the sort lambda function defined in the subclasses' addColumn definition).""" sortLambda = self.column_info[self.treeWidget().sortColumn()][SORT_LAMBDA] column = self.treeWidget().sortColumn() - if sortLambda and isinstance(other.rpcObject, opencue.wrappers.job.Job): + if sortLambda: # pylint: disable=broad-except try: return sortLambda(self.rpcObject) < sortLambda(other.rpcObject) except Exception: - logger.warning("Sort failed on column %s, using text sort.", column) + logger.info("Sort failed on column %s, using text sort.", column) return str(self.text(column)) < str(other.text(column)) From 02456f56949b8d6a3f6f4923fd434a0dd7e22b21 Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Thu, 29 Jul 2021 12:36:00 -0700 Subject: [PATCH 114/277] Fix scroll bar position from jumping back to the top (#984) --- cuegui/cuegui/JobMonitorTree.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cuegui/cuegui/JobMonitorTree.py b/cuegui/cuegui/JobMonitorTree.py index 58773c0ac..2e4698e5d 100644 --- a/cuegui/cuegui/JobMonitorTree.py +++ b/cuegui/cuegui/JobMonitorTree.py @@ -411,7 +411,7 @@ def _processUpdate(self, work, rpcObjects): self._itemsLock.lockForWrite() - # include rpcObjects from self._items that are not in jobObjects + # include rpcObjects from self._items that are not in rpcObjects for proxy, item in list(self._items.items()): if not proxy in rpcObjects: rpcObjects[proxy] = item.rpcObject @@ -435,7 +435,7 @@ def _processUpdate(self, work, rpcObjects): if proxy in self.__userColors: self._items[proxy].setUserColor(self.__userColors[proxy]) - self.verticalScrollBar().setValue(scrolled) + self.verticalScrollBar().setRange(scrolled, len(rpcObjects.keys()) - scrolled) list(map(lambda key: self._items[key].setSelected(True), [key for key in selectedKeys if key in self._items])) From 6e302cb9fe5e1d7df52dd6fd1dee70820f33ab55 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Thu, 29 Jul 2021 16:21:03 -0400 Subject: [PATCH 115/277] Support new pycue configuration paths. (#972) --- pycue/opencue/config.py | 96 +++++++++++++++++++++++++ pycue/opencue/cuebot.py | 40 +++++------ pycue/tests/config_test.py | 143 +++++++++++++++++++++++++++++++++++++ pycue/tests/cuebot_test.py | 73 +++++++++++++++++++ 4 files changed, 332 insertions(+), 20 deletions(-) create mode 100644 pycue/opencue/config.py create mode 100644 pycue/tests/config_test.py create mode 100644 pycue/tests/cuebot_test.py diff --git a/pycue/opencue/config.py b/pycue/opencue/config.py new file mode 100644 index 000000000..7d1040a87 --- /dev/null +++ b/pycue/opencue/config.py @@ -0,0 +1,96 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""OpenCue configuration.""" + +import logging +import os +import platform + +import yaml + + +logger = logging.getLogger("opencue") + + +# Config file from which default settings are loaded. This file is distributed with the +# opencue Python library. +__DEFAULT_CONFIG_FILE = os.path.join(os.path.dirname(__file__), 'default.yaml') + +# Environment variables which can be used to define a custom config file. Any settings +# defined in this file will be used instead of the defaults. +__CONFIG_FILE_ENV_VARS = [ + # OPENCUE_CONFIG_FILE is the preferred setting to use. + 'OPENCUE_CONFIG_FILE', + # OPENCUE_CONF is deprecated, but kept for now for backwards compatibility. + 'OPENCUE_CONF', +] + + +def config_base_directory(): + """Returns the OpenCue config base directory. + + This platform-dependent directory, stored within your user profile, is used by + OpenCue components as the default location for various configuration files. Typically + if you store your config files in this location, there is no need to set environment + variables to indicate where your config files are located -- OpenCue should recognize + them automatically. + + NOTE: This work is ongoing. Over time more OpenCue components will start using this + base directory. See https://github.com/AcademySoftwareFoundation/OpenCue/issues/785. + + :rtype: str + :return: config file base directory + """ + if platform.system() == 'Windows': + return os.path.join(os.path.expandvars('%APPDATA%'), 'opencue') + return os.path.join(os.path.expanduser('~'), '.config', 'opencue') + + +def load_config_from_file(): + """Loads configuration settings from config file on the local system. + + Default settings are read from default.yaml which is distributed with the opencue library. + User-provided config is then read from disk, in order of preference: + - Path defined by the OPENCUE_CONFIG_FILE environment variable. + - Path defined by the OPENCUE_CONF environment variable. + - Path within the config base directory (i.e. ~/.config/opencue/opencue.yaml) + + :rtype: dict + :return: config settings + """ + with open(__DEFAULT_CONFIG_FILE) as file_object: + config = yaml.load(file_object, Loader=yaml.SafeLoader) + + user_config_file = None + + for config_file_env_var in __CONFIG_FILE_ENV_VARS: + logger.debug('Checking for opencue config file path in %s', config_file_env_var) + config_file_from_env = os.environ.get(config_file_env_var) + if config_file_from_env and os.path.exists(config_file_from_env): + user_config_file = config_file_from_env + break + + if not user_config_file: + config_from_user_profile = os.path.join(config_base_directory(), 'opencue.yaml') + logger.debug('Checking for opencue config at %s', config_from_user_profile) + if os.path.exists(config_from_user_profile): + user_config_file = config_from_user_profile + + if user_config_file: + logger.info('Loading opencue config from %s', user_config_file) + with open(user_config_file) as file_object: + config.update(yaml.load(file_object, Loader=yaml.SafeLoader)) + + return config diff --git a/pycue/opencue/cuebot.py b/pycue/opencue/cuebot.py index 0e143e119..cb36d9d64 100644 --- a/pycue/opencue/cuebot.py +++ b/pycue/opencue/cuebot.py @@ -26,10 +26,10 @@ import logging import os import platform -import yaml import grpc +import opencue.config from opencue.compiled_proto import comment_pb2 from opencue.compiled_proto import comment_pb2_grpc from opencue.compiled_proto import criterion_pb2 @@ -67,15 +67,6 @@ logger = logging.getLogger("opencue") -default_config = os.path.join(os.path.dirname(__file__), 'default.yaml') -with open(default_config) as file_object: - config = yaml.load(file_object, Loader=yaml.SafeLoader) - -# check for facility specific configurations. -fcnf = os.environ.get('OPENCUE_CONF', '') -if os.path.exists(fcnf): - with open(fcnf) as file_object: - config.update(yaml.load(file_object, Loader=yaml.SafeLoader)) DEFAULT_MAX_MESSAGE_BYTES = 1024 ** 2 * 10 DEFAULT_GRPC_PORT = 8443 @@ -96,7 +87,8 @@ class Cuebot(object): RpcChannel = None Hosts = [] Stubs = {} - Timeout = config.get('cuebot.timeout', 10000) + Config = opencue.config.load_config_from_file() + Timeout = Config.get('cuebot.timeout', 10000) PROTO_MAP = { 'action': filter_pb2, @@ -150,13 +142,20 @@ class Cuebot(object): } @staticmethod - def init(): + def init(config=None): """Main init method for setting up the Cuebot object. - Sets the communication channel and hosts.""" + Sets the communication channel and hosts. + + :type config: dict + :param config: config dictionary, this will override the config read from disk + """ + if config: + Cuebot.Config = config + Cuebot.Timeout = config.get('cuebot.timeout', Cuebot.Timeout) if os.getenv("CUEBOT_HOSTS"): Cuebot.setHosts(os.getenv("CUEBOT_HOSTS").split(",")) else: - facility_default = config.get("cuebot.facility_default") + facility_default = Cuebot.Config.get("cuebot.facility_default") Cuebot.setFacility(facility_default) if Cuebot.Hosts is None: raise CueException('Cuebot host not set. Please ensure CUEBOT_HOSTS is set ' + @@ -169,7 +168,7 @@ def setChannel(): # gRPC must specify a single host. Randomize host list to balance load across cuebots. hosts = list(Cuebot.Hosts) shuffle(hosts) - maxMessageBytes = config.get('cuebot.max_message_bytes', DEFAULT_MAX_MESSAGE_BYTES) + maxMessageBytes = Cuebot.Config.get('cuebot.max_message_bytes', DEFAULT_MAX_MESSAGE_BYTES) # create interceptors interceptors = ( @@ -186,7 +185,8 @@ def setChannel(): if ':' in host: connectStr = host else: - connectStr = '%s:%s' % (host, config.get('cuebot.grpc_port', DEFAULT_GRPC_PORT)) + connectStr = '%s:%s' % ( + host, Cuebot.Config.get('cuebot.grpc_port', DEFAULT_GRPC_PORT)) logger.debug('connecting to gRPC at %s', connectStr) # TODO(bcipriano) Configure gRPC TLS. (Issue #150) try: @@ -228,12 +228,12 @@ def setFacility(facility): :type facility: str :param facility: a facility named in the config file""" - if facility not in list(config.get("cuebot.facility").keys()): - default = config.get("cuebot.facility_default") + if facility not in list(Cuebot.Config.get("cuebot.facility").keys()): + default = Cuebot.Config.get("cuebot.facility_default") logger.warning("The facility '%s' does not exist, defaulting to %s", facility, default) facility = default logger.debug("setting facility to: %s", facility) - hosts = config.get("cuebot.facility")[facility] + hosts = Cuebot.Config.get("cuebot.facility")[facility] Cuebot.setHosts(hosts) @staticmethod @@ -290,7 +290,7 @@ def getStub(cls, name): @staticmethod def getConfig(): """Gets the Cuebot config object, originally read in from the config file on disk.""" - return config + return Cuebot.Config # Python 2/3 compatible implementation of ABC diff --git a/pycue/tests/config_test.py b/pycue/tests/config_test.py new file mode 100644 index 000000000..9045365eb --- /dev/null +++ b/pycue/tests/config_test.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python + +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for `opencue.config`.""" + +import os +import unittest + +import mock +import pyfakefs.fake_filesystem_unittest + +import opencue.config + + +EXPECTED_DEFAULT_CONFIG = { + 'logger.format': '%(levelname)-9s %(module)-10s %(message)s', + 'logger.level': 'WARNING', + 'cuebot.protocol': 'tcp', + 'cuebot.grpc_port': 8443, + 'cuebot.timeout': 10000, + 'cuebot.max_message_bytes': 104857600, + 'cuebot.exception_retries': 3, + 'cuebot.facility_default': 'local', + 'cuebot.facility': { + 'local': ['localhost:8443'], + 'dev': ['cuetest02-vm.example.com:8443'], + 'cloud': [ + 'cuebot1.example.com:8443', + 'cuebot2.example.com:8443', + 'cuebot3.example.com:8443' + ], + }, +} + +USER_CONFIG = """ +cuebot.facility_default: fake-facility-01 +cuebot.facility: + fake-facility-01: + - fake-cuebot-01:1234 + fake-facility-02: + - fake-cuebot-02:5678 + - fake-cuebot-03:9012 +""" + + +class ConfigTests(pyfakefs.fake_filesystem_unittest.TestCase): + def setUp(self): + self.setUpPyfakefs() + self.fs.add_real_file( + os.path.join(os.path.dirname(opencue.__file__), 'default.yaml'), read_only=True) + os.unsetenv('OPENCUE_CONFIG_FILE') + os.unsetenv('OPENCUE_CONF') + + @mock.patch('platform.system', new=mock.Mock(return_value='Linux')) + @mock.patch('os.path.expanduser', new=mock.Mock(return_value='/home/username')) + def test__should_return_config_dir_unix(self): + self.assertEqual('/home/username/.config/opencue', opencue.config.config_base_directory()) + + @mock.patch('platform.system', new=mock.Mock(return_value='Windows')) + @mock.patch( + 'os.path.expandvars', new=mock.Mock(return_value='C:/Users/username/AppData/Roaming')) + def test__should_return_config_dir_windows(self): + self.assertEqual( + 'C:/Users/username/AppData/Roaming/opencue', opencue.config.config_base_directory()) + + def test__should_load_default_config(self): + self.assertIsNone(os.environ.get('OPENCUE_CONFIG_FILE')) + self.assertIsNone(os.environ.get('OPENCUE_CONF')) + + config = opencue.config.load_config_from_file() + + self.assertEqual(EXPECTED_DEFAULT_CONFIG, config) + + def test__should_load_user_config(self): + config_file_path = '/path/to/config.yaml' + self.fs.create_file(config_file_path, contents=USER_CONFIG) + os.environ['OPENCUE_CONFIG_FILE'] = config_file_path + # Define some invalid config using the old setting name, this ensures the old env var + # will be ignored if the new one is set. + config_file_path_legacy = '/path/to/legacy/config.yaml' + self.fs.create_file(config_file_path_legacy, contents='invalid yaml') + os.environ['OPENCUE_CONF'] = config_file_path_legacy + + config = opencue.config.load_config_from_file() + + self.assertEqual('fake-facility-01', config['cuebot.facility_default']) + self.assertEqual(['fake-cuebot-01:1234'], config['cuebot.facility']['fake-facility-01']) + self.assertEqual( + ['fake-cuebot-02:5678', 'fake-cuebot-03:9012'], + config['cuebot.facility']['fake-facility-02']) + # Settings not defined in user config should still have default values. + self.assertEqual(10000, config['cuebot.timeout']) + self.assertEqual(3, config['cuebot.exception_retries']) + + def test__should_load_user_config_from_legacy_var(self): + config_file_path = '/path/to/config.yaml' + self.fs.create_file(config_file_path, contents=USER_CONFIG) + os.environ['OPENCUE_CONF'] = config_file_path + + config = opencue.config.load_config_from_file() + + self.assertEqual('fake-facility-01', config['cuebot.facility_default']) + self.assertEqual(['fake-cuebot-01:1234'], config['cuebot.facility']['fake-facility-01']) + self.assertEqual( + ['fake-cuebot-02:5678', 'fake-cuebot-03:9012'], + config['cuebot.facility']['fake-facility-02']) + # Settings not defined in user config should still have default values. + self.assertEqual(10000, config['cuebot.timeout']) + self.assertEqual(3, config['cuebot.exception_retries']) + + @mock.patch('platform.system', new=mock.Mock(return_value='Linux')) + @mock.patch('os.path.expanduser', new=mock.Mock(return_value='/home/username')) + def test__should_load_user_config_from_user_profile(self): + config_file_path = '/home/username/.config/opencue/opencue.yaml' + self.fs.create_file(config_file_path, contents=USER_CONFIG) + + config = opencue.config.load_config_from_file() + + self.assertEqual('fake-facility-01', config['cuebot.facility_default']) + self.assertEqual(['fake-cuebot-01:1234'], config['cuebot.facility']['fake-facility-01']) + self.assertEqual( + ['fake-cuebot-02:5678', 'fake-cuebot-03:9012'], + config['cuebot.facility']['fake-facility-02']) + # Settings not defined in user config should still have default values. + self.assertEqual(10000, config['cuebot.timeout']) + self.assertEqual(3, config['cuebot.exception_retries']) + + +if __name__ == '__main__': + unittest.main() diff --git a/pycue/tests/cuebot_test.py b/pycue/tests/cuebot_test.py new file mode 100644 index 000000000..78e20fb97 --- /dev/null +++ b/pycue/tests/cuebot_test.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for `opencue.cuebot`.""" + +import unittest + +import mock + +import opencue + + +TESTING_CONFIG = { + "cuebot.facility_default": "fake-facility-01", + "cuebot.facility": { + "fake-facility-01": [ + "fake-cuebot-01", + ], + "fake-facility-02": [ + "fake-cuebot-02", + "fake-cuebot-03", + ], + }, +} + + +class CuebotTests(unittest.TestCase): + def setUp(self): + self.cuebot = opencue.Cuebot() + # Mocking the cue service ensures the initial healthcheck request made to Cuebot + # will succeed. + self.cuebot.SERVICE_MAP['cue'] = mock.Mock() + + def test__should_set_hosts_and_channel(self): + healthcheck_mock = mock.Mock() + self.cuebot.SERVICE_MAP['cue'] = healthcheck_mock + + self.cuebot.init(config=TESTING_CONFIG) + + self.assertEqual(["fake-cuebot-01"], self.cuebot.Hosts) + self.assertIsNotNone(self.cuebot.RpcChannel) + healthcheck_mock.assert_called_with(self.cuebot.RpcChannel) + + def test__should_set_known_facility(self): + self.cuebot.init(config=TESTING_CONFIG) + + self.cuebot.setFacility('fake-facility-02') + + self.assertEqual(['fake-cuebot-02', 'fake-cuebot-03'], self.cuebot.Hosts) + + def test__should_ignore_unknown_facility(self): + self.cuebot.init(config=TESTING_CONFIG) + + self.cuebot.setFacility('unknown-facility') + + self.assertEqual(['fake-cuebot-01'], self.cuebot.Hosts) + + +if __name__ == '__main__': + unittest.main() From 409c2a7ca2f84aed092f5d713a62e4864394efa1 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Sat, 31 Jul 2021 18:58:02 -0400 Subject: [PATCH 116/277] Upgrade gRPC and PySide Python dependencies. (#1003) --- VERSION.in | 2 +- ci/pylintrc_main | 3 +- cuegui/cuegui/AbstractDialog.py | 2 ++ cuegui/cuegui/AbstractTreeWidget.py | 8 ++--- cuegui/cuegui/Action.py | 2 +- cuegui/cuegui/Comments.py | 4 +++ cuegui/cuegui/ConfirmationDialog.py | 2 ++ cuegui/cuegui/CreateShowDialog.py | 2 ++ cuegui/cuegui/CreatorDialog.py | 2 ++ cuegui/cuegui/CueJobMonitorTree.py | 2 +- cuegui/cuegui/CueStateBarWidget.py | 2 +- cuegui/cuegui/DependWizard.py | 2 +- cuegui/cuegui/EmailDialog.py | 4 +++ cuegui/cuegui/FilterDialog.py | 30 +++++++++-------- cuegui/cuegui/FrameMonitor.py | 14 ++++---- cuegui/cuegui/FrameMonitorTree.py | 4 +-- cuegui/cuegui/GarbageCollector.py | 2 +- cuegui/cuegui/GroupDialog.py | 4 +++ cuegui/cuegui/HostMonitor.py | 6 ++-- cuegui/cuegui/HostMonitorTree.py | 2 ++ cuegui/cuegui/JobMonitorTree.py | 2 ++ cuegui/cuegui/LayerDialog.py | 6 +++- cuegui/cuegui/LimitsWidget.py | 4 ++- cuegui/cuegui/LocalBooking.py | 8 +++-- cuegui/cuegui/Main.py | 17 +++++----- cuegui/cuegui/MainWindow.py | 20 ++++++------ cuegui/cuegui/MenuActions.py | 2 +- cuegui/cuegui/ProcMonitor.py | 26 ++++++--------- cuegui/cuegui/ProcMonitorTree.py | 2 +- cuegui/cuegui/ProgressDialog.py | 2 +- cuegui/cuegui/Redirect.py | 8 +++-- cuegui/cuegui/ServiceDialog.py | 4 ++- cuegui/cuegui/ShowDialog.py | 12 ++++--- cuegui/cuegui/ShowsWidget.py | 2 +- cuegui/cuegui/SubscriptionGraphWidget.py | 8 ++--- cuegui/cuegui/SubscriptionsWidget.py | 2 +- cuegui/cuegui/TagsWidget.py | 2 +- cuegui/cuegui/TasksDialog.py | 2 ++ cuegui/cuegui/TextEditDialog.py | 2 ++ cuegui/cuegui/UnbookDialog.py | 6 +++- cuegui/cuegui/images/crystal/icons_rcc.py | 2 ++ cuegui/cuegui/images/icons_rcc.py | 2 ++ cuegui/cuegui/plugins/AttributesPlugin.py | 2 +- cuegui/cuegui/plugins/LogViewPlugin.py | 32 +++++++++++-------- cuegui/cuegui/plugins/MonitorCuePlugin.py | 28 ++++++++-------- .../cuegui/plugins/MonitorJobDetailsPlugin.py | 2 +- cuegui/cuegui/plugins/MonitorJobsPlugin.py | 28 ++++++++-------- cuegui/cuegui/plugins/ShowsPlugin.py | 2 +- cuegui/tests/test_utils.py | 3 ++ cuesubmit/cuesubmit/__main__.py | 2 ++ cuesubmit/cuesubmit/ui/Command.py | 2 +- cuesubmit/cuesubmit/ui/Job.py | 2 ++ cuesubmit/cuesubmit/ui/SettingsWidgets.py | 6 ++-- cuesubmit/cuesubmit/ui/Submit.py | 4 +++ cuesubmit/cuesubmit/ui/Widgets.py | 10 ++++-- proto/README.md | 6 ++++ requirements.txt | 6 ++-- requirements_gui.txt | 2 +- 58 files changed, 227 insertions(+), 150 deletions(-) diff --git a/VERSION.in b/VERSION.in index f3040840f..948a54727 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.13 +0.14 diff --git a/ci/pylintrc_main b/ci/pylintrc_main index b4510f658..85edaf4b4 100644 --- a/ci/pylintrc_main +++ b/ci/pylintrc_main @@ -60,7 +60,8 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use "--disable=all --enable=classes # --disable=W". -disable=duplicate-code, +disable=c-extension-no-member, + duplicate-code, fixme, invalid-name, locally-disabled, diff --git a/cuegui/cuegui/AbstractDialog.py b/cuegui/cuegui/AbstractDialog.py index 613b3d83c..f6d701fe4 100644 --- a/cuegui/cuegui/AbstractDialog.py +++ b/cuegui/cuegui/AbstractDialog.py @@ -37,8 +37,10 @@ def _newCheckBoxSelectionMatrix(title, allowedOptions, checkedOptions, parent=No def _newDialogButtonBox(self, buttons, orientation=QtCore.Qt.Horizontal): buttonBox = QtWidgets.QDialogButtonBox(buttons, orientation, self) + # pylint: disable=no-member buttonBox.accepted.connect(self.accept) buttonBox.rejected.connect(self.reject) + # pylint: enable=no-member return buttonBox def _addWidgetRow(self, *widgets): diff --git a/cuegui/cuegui/AbstractTreeWidget.py b/cuegui/cuegui/AbstractTreeWidget.py index cf74590ef..96250453e 100644 --- a/cuegui/cuegui/AbstractTreeWidget.py +++ b/cuegui/cuegui/AbstractTreeWidget.py @@ -100,10 +100,10 @@ def __init__(self, parent): self.__setupColumnMenu() + # pylint: disable=no-member self.itemClicked.connect(self.__itemSingleClickedEmitToApp) self.itemDoubleClicked.connect(self.__itemDoubleClickedEmitToApp) self._timer.timeout.connect(self.updateRequest) - # pylint: disable=no-member QtGui.qApp.request_update.connect(self.updateRequest) # pylint: enable=no-member @@ -215,7 +215,7 @@ def startTicksUpdate(self, updateInterval, self.ticksLock = QtCore.QMutex() self.__ticksTimer = QtCore.QTimer(self) - self.__ticksTimer.timeout.connect(self.__tick) + self.__ticksTimer.timeout.connect(self.__tick) # pylint: disable=no-member self.__ticksTimer.start(1000) self.ticksWithoutUpdate = 999 @@ -509,7 +509,7 @@ def __setupColumnMenu(self): self.__dropdown.setFixedHeight(self.header().height() - 10) self.__dropdown.setToolTip("Click to select columns to display") self.__dropdown.setIcon(QtGui.QIcon(":column_popdown.png")) - self.__dropdown.clicked.connect(self.__displayColumnMenu) + self.__dropdown.clicked.connect(self.__displayColumnMenu) # pylint: disable=no-member layout = QtWidgets.QHBoxLayout(self.header()) layout.setContentsMargins(0, 0, 0, 0) @@ -521,7 +521,7 @@ def __displayColumnMenu(self): self.__dropdown.height())) menu = QtWidgets.QMenu(self) - menu.triggered.connect(self.__handleColumnMenu) + menu.triggered.connect(self.__handleColumnMenu) # pylint: disable=no-member for col in range(self.columnCount()): if self.columnWidth(col) or self.isColumnHidden(col): name = self.__columnInfoByType[self.__columnPrimaryType][col][COLUMN_NAME] diff --git a/cuegui/cuegui/Action.py b/cuegui/cuegui/Action.py index 853e6a087..f9a083e09 100644 --- a/cuegui/cuegui/Action.py +++ b/cuegui/cuegui/Action.py @@ -103,4 +103,4 @@ def __init__(self,callback=None, parent=None): self.setText("Refresh") self.setIcon(QtGui.QIcon(":/images/stock-refresh.png")) if callback: - self.triggered.connect(callback) + self.triggered.connect(callback) # pylint: disable=no-member diff --git a/cuegui/cuegui/Comments.py b/cuegui/cuegui/Comments.py index 9d8452a5b..c24a9802e 100644 --- a/cuegui/cuegui/Comments.py +++ b/cuegui/cuegui/Comments.py @@ -94,6 +94,7 @@ def __init__(self, source, parent=None): btnLayout.addWidget(self.__btnClose) layout.addLayout(btnLayout) + # pylint: disable=no-member self.__treeSubjects.itemSelectionChanged.connect(self.__itemChanged) self.__comboMacro.currentTextChanged.connect(self.__macroHandle) self.__btnSave.pressed.connect(self.__saveComment) @@ -102,6 +103,7 @@ def __init__(self, source, parent=None): self.__btnClose.pressed.connect(self.__close) self.__textSubject.textEdited.connect(self.__textEdited) self.__textMessage.textChanged.connect(self.__textEdited) + # pylint: enable=no-member self.refreshComments() self.__macroLoad() @@ -333,8 +335,10 @@ def __init__(self, name="", subject="", message="", parent=None): layout.addWidget(self.__textMessage) layout.addLayout(btnLayout) + # pylint: disable=no-member self.__btnSave.pressed.connect(self.__save) self.__btnCancel.pressed.connect(self.reject) + # pylint: enable=no-member def __save(self): """Validates and then exits from the dialog in success""" diff --git a/cuegui/cuegui/ConfirmationDialog.py b/cuegui/cuegui/ConfirmationDialog.py index 0c71e88e3..68f0f7acd 100644 --- a/cuegui/cuegui/ConfirmationDialog.py +++ b/cuegui/cuegui/ConfirmationDialog.py @@ -74,5 +74,7 @@ def __init__(self, title, text, items=[], parent=None): self.setMaximumSize(400,300) self.setWindowTitle(title) + # pylint: disable=no-member __btn_accept.clicked.connect(self.accept) __btn_cancel.clicked.connect(self.reject) + # pylint: enable=no-member diff --git a/cuegui/cuegui/CreateShowDialog.py b/cuegui/cuegui/CreateShowDialog.py index 8bb733151..148bd52d2 100644 --- a/cuegui/cuegui/CreateShowDialog.py +++ b/cuegui/cuegui/CreateShowDialog.py @@ -65,8 +65,10 @@ def __init__(self, parent=None): self.layout().addWidget(self.__create_btn, 5, 1) self.layout().addWidget(self.__cancel_btn, 5, 2) + # pylint: disable=no-member self.__create_btn.clicked.connect(self.__createShow) self.__cancel_btn.clicked.connect(self.__cancelDialog) + # pylint: enable=no-member self.adjustSize() def __createSubscriptionWidget(self): diff --git a/cuegui/cuegui/CreatorDialog.py b/cuegui/cuegui/CreatorDialog.py index 0e436b201..a28d2f00b 100644 --- a/cuegui/cuegui/CreatorDialog.py +++ b/cuegui/cuegui/CreatorDialog.py @@ -97,8 +97,10 @@ def __init__(self, show=None, parent=None): layout.addWidget(self.__buttons) self.resize(400, 0) + # pylint: disable=no-member self.__buttons.accepted.connect(self.create) self.__buttons.rejected.connect(self.close) + # pylint: enable=no-member def create(self): self.__creator.create() diff --git a/cuegui/cuegui/CueJobMonitorTree.py b/cuegui/cuegui/CueJobMonitorTree.py index 3c8fe9633..97d22cb39 100644 --- a/cuegui/cuegui/CueJobMonitorTree.py +++ b/cuegui/cuegui/CueJobMonitorTree.py @@ -222,9 +222,9 @@ def __init__(self, parent): # pylint: disable=no-member QtGui.qApp.facility_changed.connect(self.removeAllShows) - # pylint: enable=no-member self.itemClicked.connect(self.__itemSingleClickedCopy) self.itemClicked.connect(self.__itemSingleClickedComment) + # pylint: enable=no-member # Skip updates if the user is scrolling self._limitUpdatesDuringScrollSetup() diff --git a/cuegui/cuegui/CueStateBarWidget.py b/cuegui/cuegui/CueStateBarWidget.py index 3bf8be62d..780ea319d 100644 --- a/cuegui/cuegui/CueStateBarWidget.py +++ b/cuegui/cuegui/CueStateBarWidget.py @@ -62,7 +62,7 @@ def __init__(self, sourceTree, parent=None): self.__timer = QtCore.QTimer(self) self.__lastUpdate = 0 - self.__timer.timeout.connect(self.updateColors) + self.__timer.timeout.connect(self.updateColors) # pylint: disable=no-member self.__sourceTree.verticalScrollBar().valueChanged.connect(self.update) self.__sourceTree.verticalScrollBar().rangeChanged.connect(self.__updateColors) diff --git a/cuegui/cuegui/DependWizard.py b/cuegui/cuegui/DependWizard.py index 6910ee685..f328f76d7 100644 --- a/cuegui/cuegui/DependWizard.py +++ b/cuegui/cuegui/DependWizard.py @@ -521,7 +521,7 @@ def __init__(self, parent): self._addLabel("Depend on Job:", 0, 0) self.__jobFilterLineEdit = self._addLineEdit(2, 0, "") - self.__jobFilterLineEdit.textChanged.connect(self.filterJobs) + self.__jobFilterLineEdit.textChanged.connect(self.filterJobs) # pylint: disable=no-member self.__jobList = self._addListWidget(3, 0) diff --git a/cuegui/cuegui/EmailDialog.py b/cuegui/cuegui/EmailDialog.py index 671aeff28..f4f83387e 100644 --- a/cuegui/cuegui/EmailDialog.py +++ b/cuegui/cuegui/EmailDialog.py @@ -130,8 +130,10 @@ def __init__(self, job, frames, parent=None): ly.addWidget(self.__txt_find) ly.addWidget(self.__txt_log) + # pylint: disable=no-member self.__sel_frames.activated.connect(self.switchLogEvent) self.__txt_find.returnPressed.connect(self.findEvent) + # pylint: enable=no-member # pylint: disable=inconsistent-return-statements def __getFrame(self, name): @@ -247,8 +249,10 @@ def __init__(self, job, parent=None): hlayout.addWidget(self.__btnCancel) vlayout.addLayout(hlayout) + # pylint: disable=no-member self.__btnSend.clicked.connect(self.sendEmail) self.__btnCancel.clicked.connect(self.cancel.emit) + # pylint: enable=no-member def giveFocus(self): """Initializes widget state when the widget gains focus.""" diff --git a/cuegui/cuegui/FilterDialog.py b/cuegui/cuegui/FilterDialog.py index f130eb77e..73b4153da 100644 --- a/cuegui/cuegui/FilterDialog.py +++ b/cuegui/cuegui/FilterDialog.py @@ -110,6 +110,7 @@ def __init__(self, show, parent=None): glayout.addWidget(self.__btnAddAction, 7, 7, 1, 1) glayout.addWidget(self.__btnDone, 8, 7, 1, 1) + # pylint: disable=no-member self.__filters.itemClicked.connect(self.__itemSingleClicked) self.__btnRefresh.clicked.connect(self.__refresh) self.__btnAddFilter.clicked.connect(self.__createFilter) @@ -120,6 +121,7 @@ def __init__(self, show, parent=None): self.__btnDeleteAllActions.clicked.connect(self.__actions.deleteAllActions) self.__btnAddAction.clicked.connect(self.__actions.createAction) self.__btnDone.clicked.connect(self.accept) + # pylint: enable=no-member def __createFilter(self): """Prompts the user to create a new filter""" @@ -584,13 +586,13 @@ def updateWidgets(self): combo = QtWidgets.QCheckBox(self.parent()) combo.setFocusPolicy(QtCore.Qt.NoFocus) self.treeWidget().setItemWidget(self, 1, combo) - combo.stateChanged.connect(self.setEnabled) + combo.stateChanged.connect(self.setEnabled) # pylint: disable=no-member self.__widgets["enabled"] = combo combo = NoWheelComboBox(self.parent()) combo.addItems(FILTERTYPE) self.treeWidget().setItemWidget(self, 3, combo) - combo.currentIndexChanged.connect(self.setType) + combo.currentIndexChanged.connect(self.setType) # pylint: disable=no-member self.__widgets["type"] = combo self.__widgets["type"].setCurrentIndex(self.rpcObject.type()) @@ -653,23 +655,23 @@ def updateWidgets(self): combo = NoWheelComboBox(parent) combo.addItems(MATCHSUBJECT) treeWidget.setItemWidget(self, 0, combo) - combo.currentIndexChanged.connect(self.setSubject) + combo.currentIndexChanged.connect(self.setSubject) # pylint: disable=no-member self.__widgets["subject"] = combo combo = NoWheelComboBox(parent) combo.addItems(MATCHTYPE) treeWidget.setItemWidget(self, 1, combo) - combo.currentIndexChanged.connect(self.setType) + combo.currentIndexChanged.connect(self.setType) # pylint: disable=no-member self.__widgets["type"] = combo edit = QtWidgets.QLineEdit("", parent) treeWidget.setItemWidget(self, 2, edit) - edit.editingFinished.connect(self.setInput) + edit.editingFinished.connect(self.setInput) # pylint: disable=no-member self.__widgets["input"] = edit btn = QtWidgets.QPushButton(QtGui.QIcon(":kill.png"), "", parent) treeWidget.setItemWidget(self, 3, btn) - btn.clicked.connect(self.delete) + btn.clicked.connect(self.delete) # pylint: disable=no-member self.__widgets["delete"] = btn self.__widgets["subject"].setCurrentIndex(self.rpcObject.subject()) @@ -755,12 +757,12 @@ def updateWidgets(self): if self.rpcObject.type() in (opencue.api.filter_pb2.PAUSE_JOB,): widget = NoWheelComboBox(self.parent()) widget.addItems(PAUSETYPE) - widget.currentIndexChanged.connect(self.__setValue) + widget.currentIndexChanged.connect(self.__setValue) # pylint: disable=no-member elif self.rpcObject.type() in (opencue.api.filter_pb2.SET_JOB_PRIORITY,): widget = NoWheelSpinBox(self.parent()) widget.setMaximum(99999) - widget.editingFinished.connect(self.__setValue) + widget.editingFinished.connect(self.__setValue) # pylint: disable=no-member elif self.rpcObject.type() in (opencue.api.filter_pb2.SET_ALL_RENDER_LAYER_MEMORY, opencue.api.filter_pb2.SET_ALL_RENDER_LAYER_CORES): @@ -768,7 +770,7 @@ def updateWidgets(self): widget.setDecimals(2) widget.setSingleStep(.10) widget.setMaximum(MAX_RENDER_MEM) - widget.editingFinished.connect(self.__setValue) + widget.editingFinished.connect(self.__setValue) # pylint: disable=no-member elif self.rpcObject.type() in (opencue.api.filter_pb2.SET_JOB_MAX_CORES, opencue.api.filter_pb2.SET_JOB_MIN_CORES): @@ -776,21 +778,21 @@ def updateWidgets(self): widget.setDecimals(0) widget.setSingleStep(1) widget.setMaximum(1000) - widget.editingFinished.connect(self.__setValue) + widget.editingFinished.connect(self.__setValue) # pylint: disable=no-member elif self.rpcObject.type() in (opencue.api.filter_pb2.SET_ALL_RENDER_LAYER_TAGS,): widget = QtWidgets.QLineEdit("", self.parent()) - widget.editingFinished.connect(self.__setValue) + widget.editingFinished.connect(self.__setValue) # pylint: disable=no-member elif self.rpcObject.type() in (opencue.api.filter_pb2.MOVE_JOB_TO_GROUP,): widget = NoWheelComboBox(self.parent()) widget.addItems(list(self.treeWidget().groupNames.keys())) - widget.currentIndexChanged.connect(self.__setValue) + widget.currentIndexChanged.connect(self.__setValue) # pylint: disable=no-member elif self.rpcObject.type() in (opencue.api.filter_pb2.SET_MEMORY_OPTIMIZER,): widget = NoWheelComboBox(self.parent()) widget.addItems(MEMOPTTYPE) - widget.currentIndexChanged.connect(self.__setValue) + widget.currentIndexChanged.connect(self.__setValue) # pylint: disable=no-member if widget: self.treeWidget().setItemWidget(self, 1, widget) @@ -798,7 +800,7 @@ def updateWidgets(self): btn = QtWidgets.QPushButton(QtGui.QIcon(":kill.png"), "", self.parent()) self.treeWidget().setItemWidget(self, 2, btn) - btn.clicked.connect(self.delete) + btn.clicked.connect(self.delete) # pylint: disable=no-member self.__widgets["delete"] = btn # Update the widget with the current value diff --git a/cuegui/cuegui/FrameMonitor.py b/cuegui/cuegui/FrameMonitor.py index fc83fe1a4..caae78efd 100644 --- a/cuegui/cuegui/FrameMonitor.py +++ b/cuegui/cuegui/FrameMonitor.py @@ -168,7 +168,7 @@ def _refreshButtonSetup(self, layout): self.btn_refresh = QtWidgets.QPushButton("Refresh") self.btn_refresh.setFocusPolicy(QtCore.Qt.NoFocus) layout.addWidget(self.btn_refresh) - self.btn_refresh.clicked.connect(self.frameMonitorTree.updateRequest) + self.btn_refresh.clicked.connect(self.frameMonitorTree.updateRequest) # pylint: disable=no-member self.frameMonitorTree.updated.connect(self._refreshButtonDisableHandle) def _refreshButtonEnableHandle(self): @@ -191,7 +191,7 @@ def _clearButtonSetup(self, layout): btn.setFocusPolicy(QtCore.Qt.NoFocus) btn.setContentsMargins(0,0,0,0) layout.addWidget(btn) - btn.clicked.connect(self._clearButtonHandle) + btn.clicked.connect(self._clearButtonHandle) # pylint: disable=no-member def _clearButtonHandle(self): """Called when the clear button is clicked""" @@ -213,14 +213,14 @@ def _pageButtonSetup(self, layout): self.prev_page_btn.setFocusPolicy(QtCore.Qt.NoFocus) self.prev_page_btn.setContentsMargins(0, 0, 0, 0) layout.addWidget(self.prev_page_btn) - self.prev_page_btn.clicked.connect(lambda: self._pageButtonsHandle(-1)) + self.prev_page_btn.clicked.connect(lambda: self._pageButtonsHandle(-1)) # pylint: disable=no-member # Next page button self.next_page_btn = QtWidgets.QPushButton(">") self.next_page_btn.setFocusPolicy(QtCore.Qt.NoFocus) self.next_page_btn.setContentsMargins(0, 0, 0, 0) layout.addWidget(self.next_page_btn) - self.next_page_btn.clicked.connect(lambda: self._pageButtonsHandle(1)) + self.next_page_btn.clicked.connect(lambda: self._pageButtonsHandle(1)) # pylint: disable=no-member self.frameMonitorTree.job_changed.connect(self._updatePageButtonState) # Page number label @@ -299,7 +299,7 @@ def _selectStatusSetup(self, layout): menu = QtWidgets.QMenu(self) btn.setMenu(menu) - menu.triggered.connect(self._selectStatusHandle) + menu.triggered.connect(self._selectStatusHandle) # pylint: disable=no-member for item in ["Clear", None, "Succeeded", "Running", "Waiting", "Depend", "Dead", "Eaten"]: if item: @@ -347,7 +347,7 @@ def _filterLayersUpdate(self): else: menu = QtWidgets.QMenu(self) btn.setMenu(menu) - menu.triggered[QtWidgets.QAction].connect(self._filterLayersHandle) + menu.triggered[QtWidgets.QAction].connect(self._filterLayersHandle) # pylint: disable=unsubscriptable-object if self.frameMonitorTree.getJob(): layers = [x.data.name for x in self.frameMonitorTree.getJob().getLayers()] @@ -425,7 +425,7 @@ def _filterStatusSetup(self, layout): menu = QtWidgets.QMenu(self) btn.setMenu(menu) - menu.triggered.connect(self._filterStatusHandle) + menu.triggered.connect(self._filterStatusHandle) # pylint: disable=no-member for item in [("Clear", QtCore.Qt.ALT + QtCore.Qt.Key_QuoteLeft), None, diff --git a/cuegui/cuegui/FrameMonitorTree.py b/cuegui/cuegui/FrameMonitorTree.py index 41df65209..cfda922ec 100644 --- a/cuegui/cuegui/FrameMonitorTree.py +++ b/cuegui/cuegui/FrameMonitorTree.py @@ -216,8 +216,8 @@ def __init__(self, parent): self.ticksWithoutUpdate = 999 self.__lastUpdateTime = None - self.itemClicked.connect(self.__itemSingleClickedCopy) - self.itemClicked.connect(self.__itemSingleClickedViewLog) + self.itemClicked.connect(self.__itemSingleClickedCopy) # pylint: disable=no-member + self.itemClicked.connect(self.__itemSingleClickedViewLog) # pylint: disable=no-member self.itemDoubleClicked.connect(self.__itemDoubleClickedViewLog) self.header().sortIndicatorChanged.connect(self.__sortByColumnSave) diff --git a/cuegui/cuegui/GarbageCollector.py b/cuegui/cuegui/GarbageCollector.py index ec0587dbd..83cc06bc7 100644 --- a/cuegui/cuegui/GarbageCollector.py +++ b/cuegui/cuegui/GarbageCollector.py @@ -41,7 +41,7 @@ def __init__(self, parent, debug=False): self.debug = debug self.timer = QtCore.QTimer(self) - self.timer.timeout.connect(self.check) + self.timer.timeout.connect(self.check) # pylint: disable=no-member self.threshold = gc.get_threshold() gc.disable() diff --git a/cuegui/cuegui/GroupDialog.py b/cuegui/cuegui/GroupDialog.py index 2a38d906a..0f949b2f7 100644 --- a/cuegui/cuegui/GroupDialog.py +++ b/cuegui/cuegui/GroupDialog.py @@ -140,15 +140,19 @@ def __createToggleInput(self, text, row, inputWidget, startEnabled): self.layout().addWidget(check, row, 0) self.layout().addWidget(label, row, 1) self.layout().addWidget(inputWidget, row, 2) + # pylint: disable=no-member check.stateChanged.connect(inputWidget.setEnabled) check.stateChanged.connect(label.setEnabled) + # pylint: enable=no-member return (check, inputWidget) def __createButtons(self, buttons, row, width): self.__buttons = QtWidgets.QDialogButtonBox(buttons, QtCore.Qt.Horizontal, self) self.layout().addWidget(self.__buttons, row, 1, 1, width) + # pylint: disable=no-member self.__buttons.accepted.connect(self.accept) self.__buttons.rejected.connect(self.reject) + # pylint: enable=no-member def accept(self): __name = str(self._nameValue.text()) diff --git a/cuegui/cuegui/HostMonitor.py b/cuegui/cuegui/HostMonitor.py index 0c467d004..ad6c16a5e 100644 --- a/cuegui/cuegui/HostMonitor.py +++ b/cuegui/cuegui/HostMonitor.py @@ -108,7 +108,7 @@ def __filterByHostNameSetup(self, layout): self.__filterByHostNameLastInput = None - self.__filterByHostName.editingFinished.connect(self.__filterByHostNameHandle) + self.__filterByHostName.editingFinished.connect(self.__filterByHostNameHandle) # pylint: disable=no-member btn = QtWidgets.QPushButton("Clr") btn.setMaximumHeight(FILTER_HEIGHT) @@ -294,7 +294,7 @@ def __refreshButtonSetup(self, layout): self.btn_refresh.setMaximumHeight(FILTER_HEIGHT) self.btn_refresh.setFocusPolicy(QtCore.Qt.NoFocus) layout.addWidget(self.btn_refresh) - self.btn_refresh.clicked.connect(self.hostMonitorTree.updateRequest) + self.btn_refresh.clicked.connect(self.hostMonitorTree.updateRequest) # pylint: disable=no-member self.hostMonitorTree.updated.connect(self.__refreshButtonDisableHandle) def __refreshButtonEnableHandle(self): @@ -318,7 +318,7 @@ def __clearButtonSetup(self, layout): btn.setFocusPolicy(QtCore.Qt.NoFocus) btn.setContentsMargins(0,0,0,0) layout.addWidget(btn) - btn.clicked.connect(self.__clearButtonHandle) + btn.clicked.connect(self.__clearButtonHandle) # pylint: disable=no-member def __clearButtonHandle(self): """Called when the clear button is clicked""" diff --git a/cuegui/cuegui/HostMonitorTree.py b/cuegui/cuegui/HostMonitorTree.py index 3574b4c71..06143769d 100644 --- a/cuegui/cuegui/HostMonitorTree.py +++ b/cuegui/cuegui/HostMonitorTree.py @@ -163,8 +163,10 @@ def __init__(self, parent): self.setDropIndicatorShown(True) self.setDragEnabled(True) + # pylint: disable=no-member self.itemClicked.connect(self.__itemSingleClickedCopy) self.itemClicked.connect(self.__itemSingleClickedComment) + # pylint: enable=no-member # Don't use the standard space bar to refresh # pylint: disable=no-member diff --git a/cuegui/cuegui/JobMonitorTree.py b/cuegui/cuegui/JobMonitorTree.py index 2e4698e5d..2f2a6c63d 100644 --- a/cuegui/cuegui/JobMonitorTree.py +++ b/cuegui/cuegui/JobMonitorTree.py @@ -161,8 +161,10 @@ def __init__(self, parent): self.setDragEnabled(True) self.setDragDropMode(QtWidgets.QAbstractItemView.DragDrop) + # pylint: disable=no-member self.itemClicked.connect(self.__itemSingleClickedCopy) self.itemClicked.connect(self.__itemSingleClickedComment) + # pylint: enable=no-member self.__load = {} self.startTicksUpdate(20, False, 60) diff --git a/cuegui/cuegui/LayerDialog.py b/cuegui/cuegui/LayerDialog.py index 4c79b805f..eb49b520c 100644 --- a/cuegui/cuegui/LayerDialog.py +++ b/cuegui/cuegui/LayerDialog.py @@ -51,7 +51,7 @@ def __init__(self, widget, enable, parent=None): if enable: self.__widget.setDisabled(True) layout.addWidget(self.__checkbox) - self.__checkbox.toggled.connect(self.enable) + self.__checkbox.toggled.connect(self.enable) # pylint: disable=no-member layout.addWidget(self.__widget) def getWidget(self): @@ -210,12 +210,14 @@ def __init__(self, layers, parent=None): self) # Setup signals + # pylint: disable=no-member self.__mem.slider.valueChanged.connect(self.__translateToMemSpinbox) self.__mem.spinner.valueChanged.connect(self.__translateToMemSlider) self.__gpu_mem.slider.valueChanged.connect(self.__translateToGpuMemSpinbox) self.__gpu_mem.spinner.valueChanged.connect(self.__translateToGpuMemSlider) self.__buttons.accepted.connect(self.verify) self.__buttons.rejected.connect(self.reject) + # pylint: enable=no-member # Set actual values once signals are setup self.__mem.slider.setValue(self.getMaxMemory()) @@ -520,8 +522,10 @@ def __init__(self, layers, parent=None): QtCore.Qt.Horizontal, self) + # pylint: disable=no-member self.__buttons.accepted.connect(self.accept) self.__buttons.rejected.connect(self.reject) + # pylint: enable=no-member def accept(self): self._tags_widget.apply() diff --git a/cuegui/cuegui/LimitsWidget.py b/cuegui/cuegui/LimitsWidget.py index b7eaccd3b..ec0753f2e 100644 --- a/cuegui/cuegui/LimitsWidget.py +++ b/cuegui/cuegui/LimitsWidget.py @@ -57,8 +57,10 @@ def __init__(self, parent): layout.addWidget(self.__btnRefresh, 0, 2) layout.addWidget(self.__monitorLimits, 2, 0, 3, 4) + # pylint: disable=no-member self.__btnAddLimit.clicked.connect(self.__addLimit) self.__btnRefresh.clicked.connect(self.updateSoon) + # pylint: enable=no-member self.__menuActions = cuegui.MenuActions.MenuActions(self, self.updateSoon, list) @@ -108,8 +110,8 @@ def __init__(self, parent): self.__menuActions = cuegui.MenuActions.MenuActions( self, self.updateSoon, self.selectedObjects) - self.itemClicked.connect(self.__itemSingleClickedToDouble) # pylint: disable=no-member + self.itemClicked.connect(self.__itemSingleClickedToDouble) QtGui.qApp.facility_changed.connect(self.__facilityChanged) # pylint: enable=no-member diff --git a/cuegui/cuegui/LocalBooking.py b/cuegui/cuegui/LocalBooking.py index 475f0536f..a69270198 100644 --- a/cuegui/cuegui/LocalBooking.py +++ b/cuegui/cuegui/LocalBooking.py @@ -76,7 +76,7 @@ def __init__(self, target, parent=None): self.__msg_widget = QtWidgets.QLabel(msg, self) self.layout().addWidget(self.__msg_widget) self.layout().addWidget(self.__deed_button) - self.__deed_button.pressed.connect(self.deedLocalhost) + self.__deed_button.pressed.connect(self.deedLocalhost) # pylint: disable=no-member self.__lba_group.setDisabled(True) self.__text_target = QtWidgets.QLabel(self.__target.data.name, self) @@ -124,9 +124,8 @@ def __init__(self, target, parent=None): self.__btn_clear = QtWidgets.QPushButton("Clear", self) - # # Setup the signals. - # + # pylint: disable=no-member self.__btn_clear.pressed.connect(self.clearCurrentHost) self.__select_host.activated.connect(self.__host_changed) self.__num_mem.valueChanged.connect(self.__text_num_mem.setValue) @@ -135,6 +134,7 @@ def __init__(self, target, parent=None): self.__num_frames.valueChanged.connect(self.__calculateCores) self.__run_mem.valueChanged.connect(self.__text_run_mem.setValue) self.__text_run_mem.valueChanged.connect(self.__run_mem.setValue) + # pylint: enable=no-member self.layout().addWidget(QtWidgets.QLabel("Target Host:")) self.layout().addWidget(self.__select_host) @@ -367,9 +367,11 @@ def __init__(self, target, parent=None): self.layout().addWidget(self.__booking) self.layout().addLayout(btn_layout) + # pylint: disable=no-member self.__booking.hosts_changed.connect(self.__updateOkButtion) self.__btn_ok.pressed.connect(self.doLocalBooking) self.__btn_cancel.pressed.connect(self.close) + # pylint: enable=no-member def __updateOkButtion(self): self.__btn_ok.setDisabled(not self.__booking.hostAvailable()) diff --git a/cuegui/cuegui/Main.py b/cuegui/cuegui/Main.py index b2c540c74..b97e3d885 100644 --- a/cuegui/cuegui/Main.py +++ b/cuegui/cuegui/Main.py @@ -72,6 +72,7 @@ def startup(app_name, app_version, argv): """Starts an application window.""" app = CueGuiApplication(argv) + QtGui.qApp = app # Start splash screen splash = cuegui.SplashWindow.SplashWindow( @@ -84,10 +85,12 @@ def startup(app_name, app_version, argv): app.setWindowIcon(QtGui.QIcon('%s/windowIcon.png' % cuegui.Constants.RESOURCE_PATH)) app.setApplicationName(app_name) - app.lastWindowClosed.connect(app.quit) + app.lastWindowClosed.connect(app.quit) # pylint: disable=no-member + # pylint: disable=attribute-defined-outside-init QtGui.qApp.threadpool = cuegui.ThreadPool.ThreadPool(3, parent=app) QtGui.qApp.threads = [] + # pylint: enable=attribute-defined-outside-init config_path = "/.%s/config" % app_name.lower() settings = QtCore.QSettings(QtCore.QSettings.IniFormat, QtCore.QSettings.UserScope, config_path) @@ -96,7 +99,7 @@ def startup(app_name, app_version, argv): if settings.value('RevertLayout'): os.remove(local) - QtGui.qApp.settings = settings + QtGui.qApp.settings = settings # pylint: disable=attribute-defined-outside-init cuegui.Style.init() @@ -130,18 +133,14 @@ def startup(app_name, app_version, argv): # TODO(#609) Refactor the CueGUI classes to make this garbage collector # replacement unnecessary. - # pylint: disable=unused-variable - gc = cuegui.GarbageCollector.GarbageCollector(parent=app, debug=False) - # pylint: enable=unused-variable - app.aboutToQuit.connect(closingTime) + gc = cuegui.GarbageCollector.GarbageCollector(parent=app, debug=False) # pylint: disable=unused-variable + app.aboutToQuit.connect(closingTime) # pylint: disable=no-member app.exec_() def closingTime(): """Window close callback.""" logger.info("Closing all threads...") - # pylint: disable=no-member - threads = QtGui.qApp.threads - # pylint: enable=no-member + threads = QtGui.qApp.threads # pylint: disable=no-member for thread in threads: cuegui.Utils.shutdownThread(thread) diff --git a/cuegui/cuegui/MainWindow.py b/cuegui/cuegui/MainWindow.py index 35891e2fd..9fd5c5492 100644 --- a/cuegui/cuegui/MainWindow.py +++ b/cuegui/cuegui/MainWindow.py @@ -205,14 +205,14 @@ def __createMenus(self): # Menu Bar: File -> Close Window close = QtWidgets.QAction(QtGui.QIcon('icons/exit.png'), '&Close Window', self) close.setStatusTip('Close Window') - close.triggered.connect(self.__windowCloseWindow) + close.triggered.connect(self.__windowCloseWindow) # pylint: disable=no-member self.fileMenu.addAction(close) # Menu Bar: File -> Exit Application exitAction = QtWidgets.QAction(QtGui.QIcon('icons/exit.png'), 'E&xit Application', self) exitAction.setShortcut('Ctrl+Q') exitAction.setStatusTip('Exit application') - exitAction.triggered.connect(self.__windowCloseApplication) + exitAction.triggered.connect(self.__windowCloseApplication) # pylint: disable=no-member self.fileMenu.addAction(exitAction) self.__windowMenuSetup(self.windowMenu) @@ -223,17 +223,17 @@ def __createMenus(self): # Menu Bar: Help -> Online User Guide. action = QtWidgets.QAction('Online User Guide', self) - action.triggered.connect(self.openUserGuide) + action.triggered.connect(self.openUserGuide) # pylint: disable=no-member self.helpMenu.addAction(action) # Menu Bar: Help -> Make a Suggestion action = QtWidgets.QAction('Make a Suggestion', self) - action.triggered.connect(self.openSuggestionPage) + action.triggered.connect(self.openSuggestionPage) # pylint: disable=no-member self.helpMenu.addAction(action) # Menu Bar: Help -> Report a Bug action = QtWidgets.QAction('Report a Bug', self) - action.triggered.connect(self.openBugPage) + action.triggered.connect(self.openBugPage) # pylint: disable=no-member self.helpMenu.addAction(action) self.helpMenu.addSeparator() @@ -242,7 +242,7 @@ def __createMenus(self): about = QtWidgets.QAction(QtGui.QIcon('icons/about.png'), 'About', self) about.setShortcut('F1') about.setStatusTip('About') - about.triggered.connect(self.displayAbout) + about.triggered.connect(self.displayAbout) # pylint: disable=no-member self.helpMenu.addAction(about) ################################################################################ @@ -255,17 +255,17 @@ def __windowMenuSetup(self, menu): # Menu Bar: Window -> Change Window Title changeTitle = QtWidgets.QAction("Change Window Title", self) - changeTitle.triggered.connect(self.__windowMenuHandleChangeTitle) + changeTitle.triggered.connect(self.__windowMenuHandleChangeTitle) # pylint: disable=no-member menu.addAction(changeTitle) # Menu Bar: Window -> Save Window Settings saveWindowSettings = QtWidgets.QAction("Save Window Settings", self) - saveWindowSettings.triggered.connect(self.__saveSettings) + saveWindowSettings.triggered.connect(self.__saveSettings) # pylint: disable=no-member menu.addAction(saveWindowSettings) # Menu Bar: Window -> Revert To Default Window Layout revertWindowSettings = QtWidgets.QAction("Revert To Default Window Layout", self) - revertWindowSettings.triggered.connect(self.__revertLayout) + revertWindowSettings.triggered.connect(self.__revertLayout) # pylint: disable=no-member menu.addAction(revertWindowSettings) menu.addSeparator() @@ -405,7 +405,7 @@ def __toggleFullscreenSetup(self, menu): QtGui.QIcon('icons/fullscreen.png'), 'Toggle Full-Screen', self) fullscreen.setShortcut('Ctrl+F') fullscreen.setStatusTip('Toggle Full-Screen') - fullscreen.triggered.connect(self.__toggleFullscreen) + fullscreen.triggered.connect(self.__toggleFullscreen) # pylint: disable=no-member menu.addAction(fullscreen) def __toggleFullscreen(self): diff --git a/cuegui/cuegui/MenuActions.py b/cuegui/cuegui/MenuActions.py index 68f0b6cc8..fc615283f 100644 --- a/cuegui/cuegui/MenuActions.py +++ b/cuegui/cuegui/MenuActions.py @@ -159,7 +159,7 @@ def addAction(self, menu, actionName, callback = None): if isinstance(callback, six.string_types): callback = getattr(self, callback) - action.triggered.connect(callback) + action.triggered.connect(callback) # pylint: disable=no-member self.__actionCache[key] = action menu.addAction(self.__actionCache[key]) diff --git a/cuegui/cuegui/ProcMonitor.py b/cuegui/cuegui/ProcMonitor.py index 63035d7da..84e01fd4b 100644 --- a/cuegui/cuegui/ProcMonitor.py +++ b/cuegui/cuegui/ProcMonitor.py @@ -71,10 +71,8 @@ def __init__(self, parent): self.__viewHostsSetup() - # pylint: disable=no-member - if bool(int(QtGui.qApp.settings.value("AutoRefreshMonitorProc", 1))): + if bool(int(QtGui.qApp.settings.value("AutoRefreshMonitorProc", 1))): # pylint: disable=no-member self.updateRequest() - # pylint: enable=no-member def updateRequest(self): """Requests an update to the widget's contents.""" @@ -112,11 +110,11 @@ def __filterByHostNameSetup(self, layout): btn.setFocusPolicy(QtCore.Qt.NoFocus) btn.setFixedWidth(24) layout.addWidget(btn) - btn.clicked.connect(self.__filterByHostNameClear) + btn.clicked.connect(self.__filterByHostNameClear) # pylint: disable=no-member self.__filterByHostNameClearBtn = btn self.__filterByHostNameLastInput = None - self.__filterByHostName.editingFinished.connect(self.__filterByHostNameHandle) + self.__filterByHostName.editingFinished.connect(self.__filterByHostNameHandle) # pylint: disable=no-member def __filterByHostNameHandle(self): hosts = str(self.__filterByHostName.text()).split() @@ -138,7 +136,7 @@ def __refreshToggleCheckBoxSetup(self, layout): layout.addWidget(checkBox) if self.procMonitorTree.enableRefresh: checkBox.setCheckState(QtCore.Qt.Checked) - checkBox.stateChanged.connect(self.__refreshToggleCheckBoxHandle) + checkBox.stateChanged.connect(self.__refreshToggleCheckBoxHandle) # pylint: disable=no-member __refreshToggleCheckBoxCheckBox = checkBox def __refreshToggleCheckBoxHandle(self, state): @@ -158,7 +156,7 @@ def __refreshButtonSetup(self, layout): self.btn_refresh.setMaximumHeight(FILTER_HEIGHT) self.btn_refresh.setFocusPolicy(QtCore.Qt.NoFocus) layout.addWidget(self.btn_refresh) - self.btn_refresh.clicked.connect(self.procMonitorTree.updateRequest) + self.btn_refresh.clicked.connect(self.procMonitorTree.updateRequest) # pylint: disable=no-member self.procMonitorTree.updated.connect(self.__refreshButtonDisableHandle) def __refreshButtonEnableHandle(self): @@ -182,7 +180,7 @@ def __clearButtonSetup(self, layout): btn.setFocusPolicy(QtCore.Qt.NoFocus) btn.setContentsMargins(0,0,0,0) layout.addWidget(btn) - btn.clicked.connect(self.__clearButtonHandle) + btn.clicked.connect(self.__clearButtonHandle) # pylint: disable=no-member def __clearButtonHandle(self): """Called when the clear button is clicked""" @@ -194,9 +192,7 @@ def __clearButtonHandle(self): # Monitors and handles the view_procs signal # ============================================================================== def __viewProcsSetup(self): - # pylint: disable=no-member - QtGui.qApp.view_procs.connect(self.__viewProcsHandle) - # pylint: enable=no-member + QtGui.qApp.view_procs.connect(self.__viewProcsHandle) # pylint: disable=no-member def __viewProcsHandle(self, hosts): self.procMonitorTree.procSearch.options['host'] = hosts @@ -206,9 +202,7 @@ def __viewProcsHandle(self, hosts): # Views procs when a host is double clicked # ============================================================================== def __hostDoubleClickedSetup(self): - # pylint: disable=no-member - QtGui.qApp.view_object.connect(self.__hostDoubleClickedHandle) - # pylint: enable=no-member + QtGui.qApp.view_object.connect(self.__hostDoubleClickedHandle) # pylint: disable=no-member def __hostDoubleClickedHandle(self, rpcObject): if cuegui.Utils.isHost(rpcObject): @@ -219,9 +213,7 @@ def __hostDoubleClickedHandle(self, rpcObject): # Monitors and handles the view_hosts signal # ============================================================================== def __viewHostsSetup(self): - # pylint: disable=no-member - QtGui.qApp.view_hosts.connect(self.__viewHostsHandle) - # pylint: enable=no-member + QtGui.qApp.view_hosts.connect(self.__viewHostsHandle) # pylint: disable=no-member def __viewHostsHandle(self, hosts): if hosts: diff --git a/cuegui/cuegui/ProcMonitorTree.py b/cuegui/cuegui/ProcMonitorTree.py index 9f67dabe3..97171e08f 100644 --- a/cuegui/cuegui/ProcMonitorTree.py +++ b/cuegui/cuegui/ProcMonitorTree.py @@ -85,7 +85,7 @@ def __init__(self, parent): self.__menuActions = cuegui.MenuActions.MenuActions( self, self.updateSoon, self.selectedObjects) - self.itemClicked.connect(self.__itemSingleClickedCopy) + self.itemClicked.connect(self.__itemSingleClickedCopy) # pylint: disable=no-member self.itemDoubleClicked.connect(self.__itemDoubleClickedViewLog) # Don't use the standard space bar to refresh diff --git a/cuegui/cuegui/ProgressDialog.py b/cuegui/cuegui/ProgressDialog.py index 399a1c64a..375e2a189 100644 --- a/cuegui/cuegui/ProgressDialog.py +++ b/cuegui/cuegui/ProgressDialog.py @@ -83,7 +83,7 @@ def __init__(self, title, function, work, concurrent, cancelTitle, self.setFixedSize(300, 100) self.setWindowTitle(title) - self.__btn_cancel.clicked.connect(self.cancel) + self.__btn_cancel.clicked.connect(self.cancel) # pylint: disable=no-member self.show() diff --git a/cuegui/cuegui/Redirect.py b/cuegui/cuegui/Redirect.py index cdbaa5b35..ffdbfb534 100644 --- a/cuegui/cuegui/Redirect.py +++ b/cuegui/cuegui/Redirect.py @@ -80,7 +80,7 @@ def __init__(self, parent=None): # This is used to provide the number of allocations selected # on the button title. - self.__menu.triggered.connect(self.__afterClicked) + self.__menu.triggered.connect(self.__afterClicked) # pylint: disable=no-member def refresh(self): """Refreshes the full list of allocations.""" @@ -165,7 +165,7 @@ def __init__(self, show, name, parent=None): self.setMenu(self.__menu) - self.__menu.aboutToShow.connect(self.__populate_menu) + self.__menu.aboutToShow.connect(self.__populate_menu) # pylint: disable=no-member # pylint: disable=inconsistent-return-statements def __loadShow(self, show): @@ -281,8 +281,10 @@ def __init__(self, parent=None): layout.addWidget(self.__group) layout.addLayout(layout1) + # pylint: disable=no-member self.__job_box.textChanged.connect(self.detect) self.__show_combo.currentIndexChanged.connect(self.showChanged) + # pylint: enable=no-member def _cfg(self): ''' @@ -411,10 +413,12 @@ def __init__(self, parent=None): layout.addWidget(self.__controls) layout.addWidget(self.__tree) + # pylint: disable=no-member self.__controls.getUpdateButton().pressed.connect(self.update) self.__controls.getRedirectButton().pressed.connect(self.redirect) self.__controls.getSelectAllButton().pressed.connect(self.selectAll) self.__controls.getClearButton().pressed.connect(self.clearTarget) + # pylint: enable=no-member def __get_selected_procs_by_alloc(self, selected_items): """ diff --git a/cuegui/cuegui/ServiceDialog.py b/cuegui/cuegui/ServiceDialog.py index f88e568c0..5f78cf670 100644 --- a/cuegui/cuegui/ServiceDialog.py +++ b/cuegui/cuegui/ServiceDialog.py @@ -99,7 +99,7 @@ def __init__(self, parent=None): layout.addWidget(self.__buttons, 9, 1) - self.__buttons.accepted.connect(self.save) + self.__buttons.accepted.connect(self.save) # pylint: disable=no-member def _cfg(self): """ @@ -207,10 +207,12 @@ def __init__(self, show, parent=None): self.__btn_layout.addStretch() layout.addLayout(self.__btn_layout) + # pylint: disable=no-member self.__btn_new.clicked.connect(self.newService) self.__btn_del.clicked.connect(self.delService) self.__form.saved.connect(self.saved) self.__service_list.currentItemChanged.connect(self.selected) + # pylint: enable=no-member self.refresh() self.__service_list.setCurrentRow(0, QtCore.QItemSelectionModel.Select) diff --git a/cuegui/cuegui/ShowDialog.py b/cuegui/cuegui/ShowDialog.py index 865e75aa6..7acdfec26 100644 --- a/cuegui/cuegui/ShowDialog.py +++ b/cuegui/cuegui/ShowDialog.py @@ -56,8 +56,10 @@ def __init__(self, show, parent=None): self.layout().addWidget(self.__btnSave, 5, 1) self.layout().addWidget(self.__btnClose, 5, 2) + # pylint: disable=no-member self.__btnSave.clicked.connect(self.__saveChanges) self.__btnClose.clicked.connect(self.__closeDialog) + # pylint: enable=no-member def __createSettingsPage(self): """Settings Page""" @@ -72,7 +74,7 @@ def __createSettingsPage(self): ctrl.setValue(self.__show.data.default_max_cores) page.layout().addWidget(ctrl, 0, 0) page.layout().addWidget(label, 0, 1, 1, 4) - ctrl.valueChanged.connect(self.__valueChanged) + ctrl.valueChanged.connect(self.__valueChanged) # pylint: disable=no-member self.__defaultMaxCores = ctrl label = QtWidgets.QLabel("Default minimum cores", self) @@ -82,7 +84,7 @@ def __createSettingsPage(self): ctrl.setValue(self.__show.data.default_min_cores) page.layout().addWidget(ctrl, 1, 0) page.layout().addWidget(label, 1, 1, 1, 4) - ctrl.valueChanged.connect(self.__valueChanged) + ctrl.valueChanged.connect(self.__valueChanged) # pylint: disable=no-member self.__defaultMinCores = ctrl label = QtWidgets.QLabel("Comment Notification Email", self) @@ -90,7 +92,7 @@ def __createSettingsPage(self): text.setText(self.__show.data.comment_email) page.layout().addWidget(text, 2, 0) page.layout().addWidget(label, 2, 1, 1, 4) - text.textChanged.connect(self.__valueChanged) + text.textChanged.connect(self.__valueChanged) # pylint: disable=no-member self.__show_email = text return page @@ -105,7 +107,7 @@ def __createBookingPage(self): ctrl.setChecked(self.__show.data.booking_enabled) page.layout().addWidget(ctrl, 0, 0) page.layout().addWidget(label, 0, 1, 1, 4) - ctrl.stateChanged.connect(self.__valueChanged) + ctrl.stateChanged.connect(self.__valueChanged) # pylint: disable=no-member self.__bookingEnabled = ctrl label = QtWidgets.QLabel("Enable dispatch", self) @@ -113,7 +115,7 @@ def __createBookingPage(self): ctrl.setChecked(self.__show.data.dispatch_enabled) page.layout().addWidget(ctrl, 1, 0) page.layout().addWidget(label, 1, 1, 1, 4) - ctrl.stateChanged.connect(self.__valueChanged) + ctrl.stateChanged.connect(self.__valueChanged) # pylint: disable=no-member self.__dispatchEnabled = ctrl return page diff --git a/cuegui/cuegui/ShowsWidget.py b/cuegui/cuegui/ShowsWidget.py index b0343834f..a91685cd1 100644 --- a/cuegui/cuegui/ShowsWidget.py +++ b/cuegui/cuegui/ShowsWidget.py @@ -62,8 +62,8 @@ def __init__(self, parent): self.__menuActions = cuegui.MenuActions.MenuActions( self, self.updateSoon, self.selectedObjects) - self.itemClicked.connect(self.__itemSingleClickedToDouble) # pylint: disable=no-member + self.itemClicked.connect(self.__itemSingleClickedToDouble) QtGui.qApp.facility_changed.connect(self.__facilityChanged) # pylint: enable=no-member diff --git a/cuegui/cuegui/SubscriptionGraphWidget.py b/cuegui/cuegui/SubscriptionGraphWidget.py index 2e122328c..e129ed3eb 100644 --- a/cuegui/cuegui/SubscriptionGraphWidget.py +++ b/cuegui/cuegui/SubscriptionGraphWidget.py @@ -43,7 +43,7 @@ def __init__(self, parent): self.__showMenuActions = {} self.__subBars = [] self.__timer = QtCore.QTimer(self) - self.__timer.timeout.connect(self.update_data) + self.__timer.timeout.connect(self.update_data) # pylint: disable=no-member self.__timer.setInterval(1000 * 5) widget = QtWidgets.QWidget() @@ -57,7 +57,7 @@ def __init__(self, parent): showMenuBtn = QtWidgets.QPushButton(" Shows") showMenuBtn.setFixedWidth(100) - showMenuBtn.pressed.connect(self.__showMenuCheck) + showMenuBtn.pressed.connect(self.__showMenuCheck) # pylint: disable=no-member self.__showMenu = QtWidgets.QMenu(self) showMenuBtn.setMenu(self.__showMenu) @@ -65,7 +65,7 @@ def __init__(self, parent): showMenuBtn.setFocusPolicy(QtCore.Qt.NoFocus) self.__showMenu.setFont(cuegui.Constants.STANDARD_FONT) - self.__showMenu.triggered.connect(self.__showMenuHandle) + self.__showMenu.triggered.connect(self.__showMenuHandle) # pylint: disable=no-member layout = QtWidgets.QVBoxLayout(self) layout.addWidget(showMenuBtn) @@ -228,7 +228,7 @@ def contextMenuEvent(self, e): menu.addSeparator() if self.__show: new_action = QtWidgets.QAction('Add new subscription', self) - new_action.triggered.connect(self.createSubscription) + new_action.triggered.connect(self.createSubscription) # pylint: disable=no-member menu.addAction(new_action) menu.exec_(QtCore.QPoint(e.globalX(),e.globalY())) diff --git a/cuegui/cuegui/SubscriptionsWidget.py b/cuegui/cuegui/SubscriptionsWidget.py index 3acb97921..8be93ad00 100644 --- a/cuegui/cuegui/SubscriptionsWidget.py +++ b/cuegui/cuegui/SubscriptionsWidget.py @@ -64,10 +64,10 @@ def __init__(self, parent): layout.addWidget(self.__btnAddSubscription, 0, 3) layout.addWidget(self.__monitorSubscriptions, 2, 0, 3, 4) + # pylint: disable=no-member self.__btnShowProperties.clicked.connect(self.__showProperties) self.__btnAddSubscription.clicked.connect(self.__addSubscription) self.__comboShows.currentIndexChanged.connect(self.setShow) - # pylint: disable=no-member QtGui.qApp.view_object.connect(self.setShow) QtGui.qApp.facility_changed.connect(self.changeFacility) # pylint: enable=no-member diff --git a/cuegui/cuegui/TagsWidget.py b/cuegui/cuegui/TagsWidget.py index cde10b910..e9d6dc129 100644 --- a/cuegui/cuegui/TagsWidget.py +++ b/cuegui/cuegui/TagsWidget.py @@ -61,7 +61,7 @@ def __init__(self, allowed_tags=None, parent=None): self.__enable_custom = QtWidgets.QCheckBox('Custom Tags', self) self.__custom = QtWidgets.QLineEdit(self) self.__custom.setDisabled(True) - self.__enable_custom.toggled.connect(self.toggleCustom) + self.__enable_custom.toggled.connect(self.toggleCustom) # pylint: disable=no-member layout.addWidget(self.__enable_custom) layout.addWidget(self.__custom) layout.setContentsMargins(0, 0, 0, 0) diff --git a/cuegui/cuegui/TasksDialog.py b/cuegui/cuegui/TasksDialog.py index f624fcf6e..614b07c2d 100644 --- a/cuegui/cuegui/TasksDialog.py +++ b/cuegui/cuegui/TasksDialog.py @@ -76,12 +76,14 @@ def __init__(self, show, parent=None): glayout.addWidget(self.__btnRefresh, 4, 1) glayout.addWidget(self.__btnDone, 4, 2) + # pylint: disable=no-member self.__btnMinCores.clicked.connect(self.setMinCores) self.__checkManaged.clicked.connect(self.setManaged) self.__btnAddTask.clicked.connect(self.__tasks.createTask) self.__btnRefresh.clicked.connect(self.refresh) self.__comboDepartments.currentIndexChanged.connect(self.setDepartment) self.__btnDone.clicked.connect(self.accept) + # pylint: enable=no-member self.getDepartments() diff --git a/cuegui/cuegui/TextEditDialog.py b/cuegui/cuegui/TextEditDialog.py index d6a14526d..ed25fe45d 100644 --- a/cuegui/cuegui/TextEditDialog.py +++ b/cuegui/cuegui/TextEditDialog.py @@ -57,8 +57,10 @@ def __init__(self, title, text, default="", parent=None): self.setMaximumSize(400,300) self.setWindowTitle(title) + # pylint: disable=no-member __btn_accept.clicked.connect(self.accept) __btn_cancel.clicked.connect(self.reject) + # pylint: enable=no-member self.__textEdit.setText(default) self.__textEdit.setFocus(QtCore.Qt.OtherFocusReason) diff --git a/cuegui/cuegui/UnbookDialog.py b/cuegui/cuegui/UnbookDialog.py index b392e3d4a..e74149665 100644 --- a/cuegui/cuegui/UnbookDialog.py +++ b/cuegui/cuegui/UnbookDialog.py @@ -133,6 +133,7 @@ def __createRangeBox(self, layout, name, units, max_frame): __layout.addWidget(__maxLabel, 0, 4) # Setting the minimum should disable the right hand side of the range + # pylint: disable=no-member __lessThan.toggled.connect(__min.setDisabled) __lessThan.toggled.connect(__toLabel.setDisabled) __lessThan.toggled.connect(__minLabel.setDisabled) @@ -141,6 +142,7 @@ def __createRangeBox(self, layout, name, units, max_frame): __moreThan.toggled.connect(__max.setDisabled) __moreThan.toggled.connect(__toLabel.setDisabled) __moreThan.toggled.connect(__maxLabel.setDisabled) + # pylint: enable=no-member layout.addWidget(__group) @@ -345,7 +347,7 @@ def __init__( self.__filter = QtWidgets.QLineEdit("", self) self.layout().addWidget(self.__filter, 2, 0) - self.__filter.textChanged.connect(self.filterJobs) + self.__filter.textChanged.connect(self.filterJobs) # pylint: disable=no-member self.__list = QtWidgets.QListWidget(self) self.__list.setSelectionMode(selectionMode) @@ -431,8 +433,10 @@ def __init__(self, procSearch, parent=None): layout.addWidget(self.__jobList) layout.addWidget(self.__buttons) + # pylint: disable=no-member self.__buttons.accepted.connect(self.accept) self.__buttons.rejected.connect(self.reject) + # pylint: enable=no-member def accept(self): """Kills the procs.""" diff --git a/cuegui/cuegui/images/crystal/icons_rcc.py b/cuegui/cuegui/images/crystal/icons_rcc.py index 0ce55b24f..87d4c83d9 100644 --- a/cuegui/cuegui/images/crystal/icons_rcc.py +++ b/cuegui/cuegui/images/crystal/icons_rcc.py @@ -15,6 +15,8 @@ # -*- coding: utf-8 -*- +# pylint: disable=missing-function-docstring,missing-module-docstring + # Resource object code # # Created by: The Resource Compiler for PyQt (Qt v4.6.2) diff --git a/cuegui/cuegui/images/icons_rcc.py b/cuegui/cuegui/images/icons_rcc.py index cc9df44af..b313c2d56 100644 --- a/cuegui/cuegui/images/icons_rcc.py +++ b/cuegui/cuegui/images/icons_rcc.py @@ -14,6 +14,8 @@ # -*- coding: utf-8 -*- +# pylint: disable=missing-function-docstring,missing-module-docstring + # Resource object code # # Created: Tue Jul 15 11:26:55 2008 diff --git a/cuegui/cuegui/plugins/AttributesPlugin.py b/cuegui/cuegui/plugins/AttributesPlugin.py index da4693d06..b20825204 100644 --- a/cuegui/cuegui/plugins/AttributesPlugin.py +++ b/cuegui/cuegui/plugins/AttributesPlugin.py @@ -224,7 +224,7 @@ def addData(parent, value): self.addTopLevelItem(root) self.expandAll() - self.itemSelectionChanged.connect(self.itemSingleClickedCopy) + self.itemSelectionChanged.connect(self.itemSingleClickedCopy) # pylint: disable=no-member def itemSingleClickedCopy(self): """Event handler that copies the text of the selected line to the clipboard on click.""" diff --git a/cuegui/cuegui/plugins/LogViewPlugin.py b/cuegui/cuegui/plugins/LogViewPlugin.py index 9fc5b21d7..8e8486a1b 100644 --- a/cuegui/cuegui/plugins/LogViewPlugin.py +++ b/cuegui/cuegui/plugins/LogViewPlugin.py @@ -99,20 +99,22 @@ def __init__(self, parent): self.document().setDefaultFont(self.font) self._line_num_area = LineNumberArea(self) + # pylint: disable=no-member self.blockCountChanged.connect(self.update_line_number_area_width) self.updateRequest.connect(self.update_line_number_area) self.cursorPositionChanged.connect(self.highlight_current_line) + # pylint: enable=no-member self.update_line_number_area_width() self.setReadOnly(True) self.setMaximumBlockCount(20000) self.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) - self.customContextMenuRequested.connect(self.context_menu) + self.customContextMenuRequested.connect(self.context_menu) # pylint: disable=no-member self.copy_action = QtWidgets.QAction('Copy', self) self.copy_action.setStatusTip('Copy Selection') self.copy_action.setShortcut('Ctrl+C') - self.copy_action.triggered[bool].connect(lambda triggered: + self.copy_action.triggered[bool].connect(lambda triggered: # pylint: disable=unsubscriptable-object self.copy_selection(QtGui.QClipboard.Clipboard)) self.addAction(self.copy_action) @@ -231,9 +233,11 @@ def highlight_current_line(self): crnt_selection = QtWidgets.QTextEdit.ExtraSelection() line_color = QtGui.QColor(QtCore.Qt.red).lighter(12) + # pylint: disable=no-member crnt_selection.format.setBackground(line_color) crnt_selection.format.setProperty(QtGui.QTextFormat.FullWidthSelection, True) + # pylint: enable=no-member crnt_selection.cursor = self.textCursor() crnt_selection.cursor.clearSelection() self.setExtraSelections([crnt_selection]) @@ -314,23 +318,23 @@ def __init__(self, parent=None): path_layout = QtWidgets.QHBoxLayout(path_widget) path_layout.setContentsMargins(0, 0, 0, 0) self._first_log_button = QtWidgets.QPushButton('<<', self) - self._first_log_button.clicked.connect( + self._first_log_button.clicked.connect( # pylint: disable=no-member lambda: self._load_other_log(float('inf'))) self._first_log_button.setEnabled(False) self._first_log_button.setToolTip('Load First Log') path_layout.addWidget(self._first_log_button) self._prev_log_button = QtWidgets.QPushButton('<', self) - self._prev_log_button.clicked.connect(lambda: self._load_other_log(1)) + self._prev_log_button.clicked.connect(lambda: self._load_other_log(1)) # pylint: disable=no-member self._prev_log_button.setEnabled(False) self._prev_log_button.setToolTip('Load Previous Log') path_layout.addWidget(self._prev_log_button) self._next_log_button = QtWidgets.QPushButton('>', self) - self._next_log_button.clicked.connect(lambda: self._load_other_log(-1)) + self._next_log_button.clicked.connect(lambda: self._load_other_log(-1)) # pylint: disable=no-member self._next_log_button.setEnabled(False) self._next_log_button.setToolTip('Load Next Log') path_layout.addWidget(self._next_log_button) self._last_log_button = QtWidgets.QPushButton('>>', self) - self._last_log_button.clicked.connect( + self._last_log_button.clicked.connect( # pylint: disable=no-member lambda: self._load_other_log(-float('inf'))) self._last_log_button.setEnabled(False) self._last_log_button.setToolTip('Load Current Log') @@ -353,7 +357,7 @@ def __init__(self, parent=None): self._word_wrap_checkbox.setFont(font) path_layout.addWidget(self._word_wrap_checkbox) self._word_wrap_checkbox.setCheckState(QtCore.Qt.Checked) - self._word_wrap_checkbox.stateChanged.connect(self._set_word_wrap) + self._word_wrap_checkbox.stateChanged.connect(self._set_word_wrap) # pylint: disable=no-member # Content content_widget = QtWidgets.QWidget(self) @@ -377,30 +381,30 @@ def __init__(self, parent=None): search_layout = QtWidgets.QHBoxLayout(search_widget) self._case_stv_checkbox = QtWidgets.QCheckBox('Aa') search_layout.addWidget(self._case_stv_checkbox) - self._case_stv_checkbox.stateChanged.connect(self._move_to_search_box) + self._case_stv_checkbox.stateChanged.connect(self._move_to_search_box) # pylint: disable=no-member self._search_box = QtWidgets.QLineEdit('', self) self._search_box.setClearButtonEnabled(True) self._search_box.setPlaceholderText('Search log...') search_layout.addWidget(self._search_box) self._search_box.show() - self._search_box.editingFinished.connect(self._find_text) + self._search_box.editingFinished.connect(self._find_text) # pylint: disable=no-member self._search_button = QtWidgets.QPushButton('Find', self) search_layout.addWidget(self._search_button) self._prev_button = QtWidgets.QPushButton('Prev') - self._prev_button.clicked.connect(self._move_to_prev_match) + self._prev_button.clicked.connect(self._move_to_prev_match) # pylint: disable=no-member self._next_button = QtWidgets.QPushButton('Next') - self._next_button.clicked.connect(self._move_to_next_match) + self._next_button.clicked.connect(self._move_to_next_match) # pylint: disable=no-member search_layout.addWidget(self._next_button) search_layout.addWidget(self._prev_button) search_refresh_button = QtWidgets.QPushButton('Refresh', self) search_layout.addWidget(search_refresh_button) - search_refresh_button.clicked.connect(self._move_to_search_box) + search_refresh_button.clicked.connect(self._move_to_search_box) # pylint: disable=no-member clear_search_button = QtWidgets.QPushButton('Clr', self) search_layout.addWidget(clear_search_button) - clear_search_button.clicked.connect(self._clear_search_text) - self._search_button.clicked.connect(self._find_text) + clear_search_button.clicked.connect(self._clear_search_text) # pylint: disable=no-member + self._search_button.clicked.connect(self._find_text) # pylint: disable=no-member matches_widget = QtWidgets.QWidget(self) matches_layout = QtWidgets.QHBoxLayout(matches_widget) diff --git a/cuegui/cuegui/plugins/MonitorCuePlugin.py b/cuegui/cuegui/plugins/MonitorCuePlugin.py index 6f07b80e0..8edbda25b 100644 --- a/cuegui/cuegui/plugins/MonitorCuePlugin.py +++ b/cuegui/cuegui/plugins/MonitorCuePlugin.py @@ -112,7 +112,7 @@ def __expandAllSetup(self): btn.setIcon(QtGui.QIcon(":down.png")) btn.setFocusPolicy(QtCore.Qt.NoFocus) btn.setToolTip("Expand all groups") - btn.clicked.connect(self.__monitorCue.expandAll) + btn.clicked.connect(self.__monitorCue.expandAll) # pylint: disable=no-member def __collapseAllSetup(self): """Sets up the collapse all button""" @@ -121,7 +121,7 @@ def __collapseAllSetup(self): btn.setIcon(QtGui.QIcon(":up.png")) btn.setFocusPolicy(QtCore.Qt.NoFocus) btn.setToolTip("Collapse all groups") - btn.clicked.connect(self.__monitorCue.collapseAll) + btn.clicked.connect(self.__monitorCue.collapseAll) # pylint: disable=no-member def __buttonSetup(self, layout): btn = QtWidgets.QPushButton(QtGui.QIcon(":eat.png"), "") @@ -129,35 +129,35 @@ def __buttonSetup(self, layout): btn.setFocusPolicy(QtCore.Qt.NoFocus) btn.setFlat(True) layout.addWidget(btn) - btn.clicked.connect(self.__monitorCue.actionEatSelectedItems) + btn.clicked.connect(self.__monitorCue.actionEatSelectedItems) # pylint: disable=no-member btn = QtWidgets.QPushButton(QtGui.QIcon(":retry.png"), "") btn.setToolTip("Retries all dead frames for selected jobs") btn.setFocusPolicy(QtCore.Qt.NoFocus) btn.setFlat(True) layout.addWidget(btn) - btn.clicked.connect(self.__monitorCue.actionRetrySelectedItems) + btn.clicked.connect(self.__monitorCue.actionRetrySelectedItems) # pylint: disable=no-member btn = QtWidgets.QPushButton(QtGui.QIcon(":kill.png"), "") btn.setToolTip("Kill selected jobs") btn.setFocusPolicy(QtCore.Qt.NoFocus) btn.setFlat(True) layout.addWidget(btn) - btn.clicked.connect(self.__monitorCue.actionKillSelectedItems) + btn.clicked.connect(self.__monitorCue.actionKillSelectedItems) # pylint: disable=no-member btn = QtWidgets.QPushButton(QtGui.QIcon(":pause.png"), "") btn.setToolTip("Pause selected jobs") btn.setFocusPolicy(QtCore.Qt.NoFocus) btn.setFlat(True) layout.addWidget(btn) - btn.clicked.connect(self.__monitorCue.actionPauseSelectedItems) + btn.clicked.connect(self.__monitorCue.actionPauseSelectedItems) # pylint: disable=no-member btn = QtWidgets.QPushButton(QtGui.QIcon(":unpause.png"), "") btn.setToolTip("Unpause selected jobs") btn.setFocusPolicy(QtCore.Qt.NoFocus) btn.setFlat(True) layout.addWidget(btn) - btn.clicked.connect(self.__monitorCue.actionResumeSelectedItems) + btn.clicked.connect(self.__monitorCue.actionResumeSelectedItems) # pylint: disable=no-member ################################################################################ @@ -168,15 +168,15 @@ def __showMenuSetup(self): """Sets up the show selection menu""" self.__showMenuBtn = QtWidgets.QPushButton("Shows ",self) self.__showMenuBtn.setIcon(QtGui.QIcon(":show.png")) - self.__showMenuBtn.pressed.connect(self.__showMenuCheck) + self.__showMenuBtn.pressed.connect(self.__showMenuCheck) # pylint: disable=no-member self.__toolbar.addWidget(self.__showMenuBtn) self.__showMenu = QtWidgets.QMenu(self) self.__showMenuBtn.setMenu(self.__showMenu) self.__showMenuBtn.setFocusPolicy(QtCore.Qt.NoFocus) self.__showMenu.setFont(cuegui.Constants.STANDARD_FONT) - self.__showMenu.triggered.connect(self.__showMenuHandle) # pylint: disable=no-member + self.__showMenu.triggered.connect(self.__showMenuHandle) QtGui.qApp.facility_changed.connect(self.__showMenuUpdate) # pylint: enable=no-member @@ -248,23 +248,23 @@ def __selectJobsSetup(self): select_btn = QtWidgets.QPushButton("Select:") select_btn.setFocusPolicy(QtCore.Qt.NoFocus) self.__toolbar.addWidget(select_btn) - select_btn.clicked.connect(self.__selectJobsHandle) + select_btn.clicked.connect(self.__selectJobsHandle) # pylint: disable=no-member self._selectJobsEditBox = JobSelectEditBox(self) self.__toolbar.addWidget(self._selectJobsEditBox) - self._selectJobsEditBox.returnPressed.connect(self.__selectJobsHandle) + self._selectJobsEditBox.returnPressed.connect(self.__selectJobsHandle) # pylint: disable=no-member clear_btn = QtWidgets.QPushButton("Clr") clear_btn.setFocusPolicy(QtCore.Qt.NoFocus) clear_btn.setFixedWidth(24) self.__toolbar.addWidget(clear_btn) - clear_btn.clicked.connect(self._selectJobsEditBox.actionClear) + clear_btn.clicked.connect(self._selectJobsEditBox.actionClear) # pylint: disable=no-member mine_btn = QtWidgets.QPushButton("selectMine") mine_btn.setFocusPolicy(QtCore.Qt.NoFocus) mine_btn.setFixedWidth(70) self.__toolbar.addWidget(mine_btn) - mine_btn.clicked.connect(self.__selectJobsHandleMine) + mine_btn.clicked.connect(self.__selectJobsHandleMine) # pylint: disable=no-member def __selectJobsHandle(self, value = None): """This will select all jobs that have a name that contain the substring @@ -379,7 +379,7 @@ def actionClear(self): def _actionSelect(self): """Signals that a return was pressed""" - self.returnPressed.emit() + self.returnPressed.emit() # pylint: disable=no-member def keyPressEvent(self, event): """Let the parent handle any space key presses""" diff --git a/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py b/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py index b731736c4..809cefbfc 100644 --- a/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py @@ -70,7 +70,7 @@ def __init__(self, parent): QtGui.qApp.facility_changed.connect(self.__setJob) # pylint: enable=no-member self.__monitorLayers.handle_filter_layers_byLayer.connect(self.handleLayerFilter) - self.__splitter.splitterMoved.connect(self.__splitterMoved) + self.__splitter.splitterMoved.connect(self.__splitterMoved) # pylint: disable=no-member self.pluginRegisterSettings([("splitterSize", self.__splitter.sizes, diff --git a/cuegui/cuegui/plugins/MonitorJobsPlugin.py b/cuegui/cuegui/plugins/MonitorJobsPlugin.py index 5f9c5b20e..ee8fea472 100644 --- a/cuegui/cuegui/plugins/MonitorJobsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorJobsPlugin.py @@ -144,11 +144,11 @@ def _regexLoadJobsSetup(self, layout): btn = QtWidgets.QPushButton("Load:") btn.setFocusPolicy(QtCore.Qt.NoFocus) layout.addWidget(btn) - btn.clicked.connect(self._regexLoadJobsHandle) + btn.clicked.connect(self._regexLoadJobsHandle) # pylint: disable=no-member self.__regexLoadJobsEditBox = JobRegexLoadEditBox(self) layout.addWidget(self.__regexLoadJobsEditBox) - self.__regexLoadJobsEditBox.returnPressed.connect(self._regexLoadJobsHandle) + self.__regexLoadJobsEditBox.returnPressed.connect(self._regexLoadJobsHandle) # pylint: disable=no-member def _loadFinishedJobsSetup(self, layout): """Ensures that when querying jobs that finished jobs are included. @@ -157,7 +157,7 @@ def _loadFinishedJobsSetup(self, layout): @type layout: QLayout""" self.__loadFinishedJobsCheckBox = JobLoadFinishedCheckBox(self) layout.addWidget(self.__loadFinishedJobsCheckBox) - self.__loadFinishedJobsCheckBox.stateChanged.connect(self._regexLoadJobsHandle) + self.__loadFinishedJobsCheckBox.stateChanged.connect(self._regexLoadJobsHandle) # pylint: disable=no-member def _regexLoadJobsHandle(self): """This will select all jobs that have a name that contain the substring @@ -184,7 +184,7 @@ def _buttonSetup(self, layout): clearButton.setFocusPolicy(QtCore.Qt.NoFocus) clearButton.setFixedWidth(24) layout.addWidget(clearButton) - clearButton.clicked.connect(self.__regexLoadJobsEditBox.actionClear) + clearButton.clicked.connect(self.__regexLoadJobsEditBox.actionClear) # pylint: disable=no-member spacer = QtWidgets.QWidget() spacer.setFixedWidth(20) @@ -194,7 +194,7 @@ def _buttonSetup(self, layout): mineCheckbox.setFocusPolicy(QtCore.Qt.NoFocus) mineCheckbox.setChecked(True) layout.addWidget(mineCheckbox) - mineCheckbox.stateChanged.connect(self.jobMonitor.setLoadMine) + mineCheckbox.stateChanged.connect(self.jobMonitor.setLoadMine) # pylint: disable=no-member self._loadFinishedJobsSetup(self.__toolbar) @@ -203,56 +203,56 @@ def _buttonSetup(self, layout): finishedButton.setFocusPolicy(QtCore.Qt.NoFocus) finishedButton.setFlat(True) layout.addWidget(finishedButton) - finishedButton.clicked.connect(self.jobMonitor.removeFinishedItems) + finishedButton.clicked.connect(self.jobMonitor.removeFinishedItems) # pylint: disable=no-member allButton = QtWidgets.QPushButton(QtGui.QIcon(":eject.png"), "All") allButton.setToolTip("Unmonitor all jobs") allButton.setFocusPolicy(QtCore.Qt.NoFocus) allButton.setFlat(True) layout.addWidget(allButton) - allButton.clicked.connect(self.jobMonitor.removeAllItems) + allButton.clicked.connect(self.jobMonitor.removeAllItems) # pylint: disable=no-member removeSelectedButton = QtWidgets.QPushButton(QtGui.QIcon(":eject.png"), "") removeSelectedButton.setToolTip("Unmonitor selected jobs") removeSelectedButton.setFocusPolicy(QtCore.Qt.NoFocus) removeSelectedButton.setFlat(True) layout.addWidget(removeSelectedButton) - removeSelectedButton.clicked.connect(self.jobMonitor.actionRemoveSelectedItems) + removeSelectedButton.clicked.connect(self.jobMonitor.actionRemoveSelectedItems) # pylint: disable=no-member eatSelectedButton = QtWidgets.QPushButton(QtGui.QIcon(":eat.png"), "") eatSelectedButton.setToolTip("Eats all dead frames for selected jobs") eatSelectedButton.setFocusPolicy(QtCore.Qt.NoFocus) eatSelectedButton.setFlat(True) layout.addWidget(eatSelectedButton) - eatSelectedButton.clicked.connect(self.jobMonitor.actionEatSelectedItems) + eatSelectedButton.clicked.connect(self.jobMonitor.actionEatSelectedItems) # pylint: disable=no-member retryButton = QtWidgets.QPushButton(QtGui.QIcon(":retry.png"), "") retryButton.setToolTip("Retries all dead frames for selected jobs") retryButton.setFocusPolicy(QtCore.Qt.NoFocus) retryButton.setFlat(True) layout.addWidget(retryButton) - retryButton.clicked.connect(self.jobMonitor.actionRetrySelectedItems) + retryButton.clicked.connect(self.jobMonitor.actionRetrySelectedItems) # pylint: disable=no-member killButton = QtWidgets.QPushButton(QtGui.QIcon(":kill.png"), "") killButton.setToolTip("Kill selected jobs") killButton.setFocusPolicy(QtCore.Qt.NoFocus) killButton.setFlat(True) layout.addWidget(killButton) - killButton.clicked.connect(self.jobMonitor.actionKillSelectedItems) + killButton.clicked.connect(self.jobMonitor.actionKillSelectedItems) # pylint: disable=no-member pauseButton = QtWidgets.QPushButton(QtGui.QIcon(":pause.png"), "") pauseButton.setToolTip("Pause selected jobs") pauseButton.setFocusPolicy(QtCore.Qt.NoFocus) pauseButton.setFlat(True) layout.addWidget(pauseButton) - pauseButton.clicked.connect(self.jobMonitor.actionPauseSelectedItems) + pauseButton.clicked.connect(self.jobMonitor.actionPauseSelectedItems) # pylint: disable=no-member unpauseButton = QtWidgets.QPushButton(QtGui.QIcon(":unpause.png"), "") unpauseButton.setToolTip("Unpause selected jobs") unpauseButton.setFocusPolicy(QtCore.Qt.NoFocus) unpauseButton.setFlat(True) layout.addWidget(unpauseButton) - unpauseButton.clicked.connect(self.jobMonitor.actionResumeSelectedItems) + unpauseButton.clicked.connect(self.jobMonitor.actionResumeSelectedItems) # pylint: disable=no-member class JobLoadFinishedCheckBox(QtWidgets.QCheckBox): @@ -314,7 +314,7 @@ def actionClear(self): self.setText("") def _actionLoad(self): - self.returnPressed.emit() + self.returnPressed.emit() # pylint: disable=no-member def toggleReadOnly(self): """Toggles the textbox readonly setting.""" diff --git a/cuegui/cuegui/plugins/ShowsPlugin.py b/cuegui/cuegui/plugins/ShowsPlugin.py index 5b0b94aad..deda343f9 100644 --- a/cuegui/cuegui/plugins/ShowsPlugin.py +++ b/cuegui/cuegui/plugins/ShowsPlugin.py @@ -43,7 +43,7 @@ def __init__(self, parent): self.__showsWidget = cuegui.ShowsWidget.ShowsWidget(self) self.__createShowButton = QtWidgets.QPushButton("Create Show") self.__createShowButton.setFixedWidth(150) - self.__createShowButton.clicked.connect(self.onCreateShowClicked) + self.__createShowButton.clicked.connect(self.onCreateShowClicked) # pylint: disable=no-member self.layout().addWidget(self.__createShowButton) self.layout().addWidget(self.__showsWidget) diff --git a/cuegui/tests/test_utils.py b/cuegui/tests/test_utils.py index 81f22f6cc..dfcb446db 100644 --- a/cuegui/tests/test_utils.py +++ b/cuegui/tests/test_utils.py @@ -16,6 +16,8 @@ """Common utility functions for CueGUI test code.""" +import PySide2.QtGui + import cuegui.Main @@ -27,3 +29,4 @@ def createApplication(): global __QAPPLICATION_SINGLETON if __QAPPLICATION_SINGLETON is None: __QAPPLICATION_SINGLETON = cuegui.Main.CueGuiApplication() + PySide2.QtGui.qApp = __QAPPLICATION_SINGLETON diff --git a/cuesubmit/cuesubmit/__main__.py b/cuesubmit/cuesubmit/__main__.py index 50c111ab0..3adecc770 100644 --- a/cuesubmit/cuesubmit/__main__.py +++ b/cuesubmit/cuesubmit/__main__.py @@ -23,6 +23,7 @@ from __future__ import absolute_import import sys +from PySide2 import QtGui from PySide2 import QtWidgets from cuesubmit import Constants @@ -65,6 +66,7 @@ def __init__(self, name, *args, **kwargs): def main(): """Entrypoint for the CueSubmit application.""" app = CueSubmitApp(sys.argv) + QtGui.qApp = app app.startup() app.exec_() diff --git a/cuesubmit/cuesubmit/ui/Command.py b/cuesubmit/cuesubmit/ui/Command.py index 6377f2ce3..548e2a836 100644 --- a/cuesubmit/cuesubmit/ui/Command.py +++ b/cuesubmit/cuesubmit/ui/Command.py @@ -39,7 +39,7 @@ def __init__(self, parent=None): def setupConnections(self): """Sets up widget signals.""" - self.commandTextBox.commandBox.textChanged.connect(self.textChanged.emit) + self.commandTextBox.commandBox.textChanged.connect(self.textChanged.emit) # pylint: disable=no-member def setText(self, text): """Set the given text to the command box diff --git a/cuesubmit/cuesubmit/ui/Job.py b/cuesubmit/cuesubmit/ui/Job.py index c9ac15bca..557b972fd 100644 --- a/cuesubmit/cuesubmit/ui/Job.py +++ b/cuesubmit/cuesubmit/ui/Job.py @@ -73,10 +73,12 @@ def setupUi(self): def setupConnections(self): """Sets up widget signals.""" self.table.selectionModel().selectionChanged.connect(self.updateSelection) + # pylint: disable=no-member self.addLayerButton.clicked.connect(self.newLayer) self.deleteLayerButton.clicked.connect(self.removeRow) self.upButton.clicked.connect(self.moveUp) self.downButton.clicked.connect(self.moveDown) + # pylint: enable=no-member def setupButtons(self): """Creates buttons working with job layers.""" diff --git a/cuesubmit/cuesubmit/ui/SettingsWidgets.py b/cuesubmit/cuesubmit/ui/SettingsWidgets.py index 43b7ed7dd..01601265a 100644 --- a/cuesubmit/cuesubmit/ui/SettingsWidgets.py +++ b/cuesubmit/cuesubmit/ui/SettingsWidgets.py @@ -92,7 +92,7 @@ def setupUi(self): def setupConnections(self): """Sets up widget signals.""" - self.mayaFileInput.lineEdit.textChanged.connect(self.dataChanged.emit) + self.mayaFileInput.lineEdit.textChanged.connect(self.dataChanged.emit) # pylint: disable=no-member def setCommandData(self, commandData): self.mayaFileInput.setText(commandData.get('mayaFile', '')) @@ -149,7 +149,7 @@ def setupUi(self): def setupConnections(self): """Sets up widget signals.""" - self.fileInput.lineEdit.textChanged.connect(self.dataChanged.emit) + self.fileInput.lineEdit.textChanged.connect(self.dataChanged.emit) # pylint: disable=no-member def setCommandData(self, commandData): self.fileInput.setText(commandData.get('nukeFile', '')) @@ -214,8 +214,10 @@ def setupUi(self): def setupConnections(self): """Sets up widget signals.""" + # pylint: disable=no-member self.fileInput.lineEdit.textChanged.connect(self.dataChanged.emit) self.outputPath.lineEdit.textChanged.connect(self.dataChanged.emit) + # pylint: enable=no-member def setCommandData(self, commandData): self.fileInput.setText(commandData.get('nukeFile', '')) diff --git a/cuesubmit/cuesubmit/ui/Submit.py b/cuesubmit/cuesubmit/ui/Submit.py index fb0c6de9e..1e0246d17 100644 --- a/cuesubmit/cuesubmit/ui/Submit.py +++ b/cuesubmit/cuesubmit/ui/Submit.py @@ -58,8 +58,10 @@ def __init__(self, parent=None): def setupConnections(self): """Sets up widget signals.""" + # pylint: disable=no-member self.submitButton.pressed.connect(self.submitPressed) self.cancelButton.pressed.connect(self.cancelPressed) + # pylint: enable=no-member def submitPressed(self): """Handler for when submit button has been pressed.""" @@ -222,6 +224,7 @@ def showEvent(self, event): def setupConnections(self): """Sets up widget signals.""" + # pylint: disable=no-member self.submitButtons.cancelled.connect(self.cancel) self.submitButtons.submitted.connect(self.submit) self.jobTreeWidget.selectionChanged.connect(self.jobLayerSelectionChanged) @@ -235,6 +238,7 @@ def setupConnections(self): self.coresInput.lineEdit.textChanged.connect(self.jobDataChanged) self.chunkInput.lineEdit.textChanged.connect(self.jobDataChanged) self.dependSelector.optionsMenu.triggered.connect(self.dependencyChanged) + # pylint: enable=no-member def setupUi(self): """Creates the widget layout.""" diff --git a/cuesubmit/cuesubmit/ui/Widgets.py b/cuesubmit/cuesubmit/ui/Widgets.py index b1b6f0b54..1bcf9b951 100644 --- a/cuesubmit/cuesubmit/ui/Widgets.py +++ b/cuesubmit/cuesubmit/ui/Widgets.py @@ -69,8 +69,10 @@ def setupUi(self): def setupConnections(self): """Sets up widget signals.""" + # pylint: disable=no-member self.lineEdit.textChanged.connect(self.validateText) self.lineEdit.focusChange.connect(self.textFocusChange) + # pylint: enable=no-member def setText(self, text): """Set the text to the given value. @@ -198,7 +200,7 @@ def setupUi(self): def setupConnections(self): """Sets up widget signals.""" - self.optionsMenu.triggered.connect(self.updateLabel) + self.optionsMenu.triggered.connect(self.updateLabel) # pylint: disable=no-member def setOptions(self, options): """Add options to the menu options. @@ -303,12 +305,14 @@ def setupUi(self): def setupConnections(self): """Sets up widget signals.""" + # pylint: disable=no-member self.toggle.valueChanged.connect(self.valueChanged.emit) self.toggle.sliderPressed.connect(self.sliderPressed.emit) self.toggle.sliderMoved.connect(self.sliderMoved.emit) self.toggle.sliderReleased.connect(self.sliderReleased.emit) self.toggle.actionTriggered.connect(self.actionTriggered.emit) self.toggle.rangeChanged.connect(self.rangeChanged.emit) + # pylint: enable=no-member class CueToggle(QtWidgets.QSlider): @@ -326,8 +330,10 @@ def __init__(self, *args, **kwargs): def setupConnections(self): """Sets up widget signals.""" + # pylint: disable=no-member self.valueChanged.connect(self.change) self.sliderPressed.connect(self.toggle) + # pylint: enable=no-member def change(self): """Action when the toggle is dragged.""" @@ -377,7 +383,7 @@ def __init__(self, parent=None): def setupHelpConnections(self): """Sets up widget signal for the help button.""" - self.helpButton.clicked.connect(self.toggleHelp) + self.helpButton.clicked.connect(self.toggleHelp) # pylint: disable=no-member def setHelpText(self): """Set the help text to the widget.""" diff --git a/proto/README.md b/proto/README.md index 154a1d5d7..0b039b462 100644 --- a/proto/README.md +++ b/proto/README.md @@ -14,12 +14,15 @@ To generate: ```sh python -m grpc_tools.protoc -I=. --python_out=../rqd/rqd/compiled_proto --grpc_python_out=../rqd/rqd/compiled_proto ./*.proto +2to3 -wn -f import ../rqd/rqd/compiled_proto/*_pb2*.py ``` For Windows (Powershell): ```powershell python -m grpc_tools.protoc --proto_path=. --python_out=../rqd/rqd/compiled_proto --grpc_python_out=../rqd/rqd/compiled_proto (ls *.proto).Name +cd ..\rqd\rqd\compiled_proto\ +2to3 -wn -f import (ls *_pb2*.py).Name ``` @@ -29,12 +32,15 @@ To generate: ```sh python -m grpc_tools.protoc -I=. --python_out=../pycue/opencue/compiled_proto --grpc_python_out=../pycue/opencue/compiled_proto ./*.proto +2to3 -wn -f import ../pycue/opencue/compiled_proto/*_pb2*.py ``` For Windows (Powershell): ```powershell python -m grpc_tools.protoc --proto_path=. --python_out=../pycue/opencue/compiled_proto --grpc_python_out=../pycue/opencue/compiled_proto (ls *.proto).Name +cd ..\pycue\opencue\compiled_proto\ +2to3 -wn -f import (ls *_pb2*.py).Name ``` diff --git a/requirements.txt b/requirements.txt index f5c9a537f..f1a3a6f42 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,10 @@ enum34==1.1.6 future==0.17.1 futures==3.2.0;python_version<"3.0" -grpcio==1.16.0 -grpcio-tools==1.16.0 +grpcio==1.26.0;python_version<"3.0" +grpcio-tools==1.26.0;python_version<"3.0" +grpcio==1.39.0;python_version>="3.0" +grpcio-tools==1.39.0;python_version>="3.0" mock==2.0.0 packaging==20.9 pathlib==1.0.1;python_version<"3.4" diff --git a/requirements_gui.txt b/requirements_gui.txt index 6f35031ad..d9aa53b35 100644 --- a/requirements_gui.txt +++ b/requirements_gui.txt @@ -1 +1 @@ -PySide2==5.11.2 +PySide2==5.15.2 From 87712fcc830826613c6dd874d23f14495e044227 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Sat, 31 Jul 2021 19:58:31 -0400 Subject: [PATCH 117/277] Fix X window issues in the Docker image. (#1004) --- cuegui/Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cuegui/Dockerfile b/cuegui/Dockerfile index 4a7e3728c..f3cd788ce 100644 --- a/cuegui/Dockerfile +++ b/cuegui/Dockerfile @@ -15,10 +15,15 @@ RUN yum -y install \ gcc \ libXi \ libXrender \ + libxkbcommon-x11.x86_64 \ mesa-libGL \ python-devel \ which \ - Xvfb + Xvfb \ + xcb-util-image.x86_64 \ + xcb-util-keysyms.x86_64 \ + xcb-util-renderutil.x86_64 \ + xcb-util-wm.x86_64 RUN yum -y install \ python-pip \ From dd8dfbe65fd0d181291bd3918eff0140eac9811a Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 2 Aug 2021 15:13:17 -0400 Subject: [PATCH 118/277] Update sandbox README to point to the user guide. (#1009) --- sandbox/README.md | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/sandbox/README.md b/sandbox/README.md index 8fcc71fa8..e2cadb902 100644 --- a/sandbox/README.md +++ b/sandbox/README.md @@ -1,31 +1,16 @@ # OpenCue sandbox environment -The sandbox environment provides a way to run a test OpenCue deployment. You -can use the sandbox environment to run small tests or development work. The sandbox -environment runs OpenCue components in separate Docker containers on your local -machine. +The sandbox environment provides a way to run a test OpenCue deployment. You can use the sandbox +environment to run small tests or development work. The sandbox environment runs OpenCue components +in separate Docker containers on your local machine. To learn how to run the sandbox environment, see https://www.opencue.io/docs/quick-starts/. ## Monitoring -To get started with monitoring there is also an additional docker-compose which sets up +To get started with monitoring there is also an additional Docker compose file which sets up monitoring for key services. -This can be started from the OpenCue root directory with: -```bash -docker-compose --project-directory . -f sandbox/docker-compose.yml -f sandbox/docker-compose.monitoring.yml up -``` - -Spins up a monitoring stack - -http://localhost:3000/ - -login: admin -pass: admin - -### Loki logging - -Too use loki to store logs requires installing the docker drivers. see: -https://grafana.com/docs/loki/latest/clients/docker-driver/ \ No newline at end of file +To learn how to run the sandbox environment with monitoring, +see https://www.opencue.io/docs/other-guides/monitoring-with-prometheus-loki-and-grafana/. From 606d7d42d6dfe7f9d2049a75a1c37aa26ec43832 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 2 Aug 2021 16:13:40 -0400 Subject: [PATCH 119/277] Make setShortcut arg an explicit QKeySequence. (#1007) --- cuegui/cuegui/FrameMonitor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cuegui/cuegui/FrameMonitor.py b/cuegui/cuegui/FrameMonitor.py index caae78efd..a854d574f 100644 --- a/cuegui/cuegui/FrameMonitor.py +++ b/cuegui/cuegui/FrameMonitor.py @@ -25,6 +25,7 @@ import math from PySide2 import QtCore +from PySide2 import QtGui from PySide2 import QtWidgets import FileSequence @@ -440,7 +441,7 @@ def _filterStatusSetup(self, layout): if item[0] != "Clear": a.setCheckable(True) if item[1]: - a.setShortcut(item[1]) + a.setShortcut(QtGui.QKeySequence(item[1])) menu.addAction(a) else: menu.addSeparator() From f9b680231592bdbec1dd979b3ccd696142ef37d0 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 2 Aug 2021 16:13:55 -0400 Subject: [PATCH 120/277] Expand supported Python and VFX Reference Platform versions. (#1010) --- .github/workflows/testing-pipeline.yml | 37 ++++++++++++++++++++------ README.md | 4 +-- cueadmin/setup.py | 2 ++ cuegui/setup.py | 2 ++ cuesubmit/setup.py | 5 ++++ pycue/setup.py | 2 ++ pyoutline/setup.py | 2 ++ rqd/setup.py | 5 ++++ 8 files changed, 49 insertions(+), 10 deletions(-) diff --git a/.github/workflows/testing-pipeline.yml b/.github/workflows/testing-pipeline.yml index 3e63f35d1..2ca7464bd 100644 --- a/.github/workflows/testing-pipeline.yml +++ b/.github/workflows/testing-pipeline.yml @@ -37,20 +37,32 @@ jobs: - name: Run Python Tests run: ci/run_python_tests.sh - lint_python_2020: - name: Lint Python Code (CY2020) + test_cuebot_2020: + name: Build Cuebot and Run Unit Tests (CY2020) runs-on: ubuntu-latest - container: aswf/ci-opencue:2020 + container: + image: aswf/ci-opencue:2020 steps: - uses: actions/checkout@v2 - - name: Lint Python Code - run: ci/run_python_lint.sh + - name: Build with Gradle + run: | + chown -R aswfuser:aswfgroup . + su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser - test_cuebot_2020: - name: Build Cuebot and Run Unit Tests (CY2020) + test_python_2021: + name: Run Python Unit Tests (CY2021) + runs-on: ubuntu-latest + container: aswf/ci-opencue:2021 + steps: + - uses: actions/checkout@v2 + - name: Run Python Tests + run: ci/run_python_tests.sh + + test_cuebot_2021: + name: Build Cuebot and Run Unit Tests (CY2021) runs-on: ubuntu-latest container: - image: aswf/ci-opencue:2020 + image: aswf/ci-opencue:2021 steps: - uses: actions/checkout@v2 - name: Build with Gradle @@ -58,6 +70,15 @@ jobs: chown -R aswfuser:aswfgroup . su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser + lint_python: + name: Lint Python Code + runs-on: ubuntu-latest + container: aswf/ci-opencue:2021 + steps: + - uses: actions/checkout@v2 + - name: Lint Python Code + run: ci/run_python_lint.sh + test_sphinx: name: Test Documentation Build runs-on: ubuntu-latest diff --git a/README.md b/README.md index a198513c8..5ef264e19 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ![OpenCue](/images/opencue_logo_with_text.png) -[![Supported VFX Platform Versions](https://img.shields.io/badge/vfx%20platform-2019--2020-lightgrey.svg)](http://www.vfxplatform.com/) -![Supported Python Versions](https://img.shields.io/badge/python-2.7%2C%203.6%2C%203.7-blue.svg) +[![Supported VFX Platform Versions](https://img.shields.io/badge/vfx%20platform-2019--2021-lightgrey.svg)](http://www.vfxplatform.com/) +![Supported Python Versions](https://img.shields.io/badge/python-2.7%2C%203.6%2C%203.9-blue.svg) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/2837/badge)](https://bestpractices.coreinfrastructure.org/projects/2837) - [Introduction](#Introduction) diff --git a/cueadmin/setup.py b/cueadmin/setup.py index 6b6ec92e6..3b829a089 100644 --- a/cueadmin/setup.py +++ b/cueadmin/setup.py @@ -46,6 +46,8 @@ 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], packages=find_packages(), entry_points={ diff --git a/cuegui/setup.py b/cuegui/setup.py index eff519b23..79882eda3 100644 --- a/cuegui/setup.py +++ b/cuegui/setup.py @@ -46,6 +46,8 @@ 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], packages=find_packages(), package_data={ diff --git a/cuesubmit/setup.py b/cuesubmit/setup.py index dd4f8b462..e9b82d3b4 100644 --- a/cuesubmit/setup.py +++ b/cuesubmit/setup.py @@ -43,6 +43,11 @@ 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], packages=find_packages(), package_data={ diff --git a/pycue/setup.py b/pycue/setup.py index 0906e9351..0750ab016 100644 --- a/pycue/setup.py +++ b/pycue/setup.py @@ -46,6 +46,8 @@ 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], packages=find_packages(exclude=['tests']), package_data={ diff --git a/pyoutline/setup.py b/pyoutline/setup.py index 5663a5fa0..a18b50d4f 100644 --- a/pyoutline/setup.py +++ b/pyoutline/setup.py @@ -46,6 +46,8 @@ 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], packages=find_packages(exclude=['tests']), data_files=[ diff --git a/rqd/setup.py b/rqd/setup.py index f3151326f..c1f7b9cc4 100644 --- a/rqd/setup.py +++ b/rqd/setup.py @@ -43,6 +43,11 @@ 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], packages=find_packages(), entry_points={ From be251e2a2e2600ae8c978a9dff7489ecd08a77d1 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 2 Aug 2021 23:08:37 -0400 Subject: [PATCH 121/277] Fix python badge to list all 3.x versions. (#1011) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5ef264e19..a556d1ae5 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ![OpenCue](/images/opencue_logo_with_text.png) [![Supported VFX Platform Versions](https://img.shields.io/badge/vfx%20platform-2019--2021-lightgrey.svg)](http://www.vfxplatform.com/) -![Supported Python Versions](https://img.shields.io/badge/python-2.7%2C%203.6%2C%203.9-blue.svg) +![Supported Python Versions](https://img.shields.io/badge/python-2.7%2C%203.6%2C%203.7%2C%203.8%2C%203.9-blue.svg) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/2837/badge)](https://bestpractices.coreinfrastructure.org/projects/2837) - [Introduction](#Introduction) From 21bfbfe158fdb0e730a80db3ec2134efc15e0366 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Sun, 29 Aug 2021 17:17:21 -0400 Subject: [PATCH 122/277] Notes from July 21 TSC meeting. (#1002) --- tsc/meetings/2021-07-21.md | 116 +++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 tsc/meetings/2021-07-21.md diff --git a/tsc/meetings/2021-07-21.md b/tsc/meetings/2021-07-21.md new file mode 100644 index 000000000..da316fb91 --- /dev/null +++ b/tsc/meetings/2021-07-21.md @@ -0,0 +1,116 @@ +# OpenCue TSC Meeting Notes 21 July 2021 + +Secretary: Brian Cipriano + +TSC Attendees: + +* [ ] Matt Chambers +* [x] Brian Cipriano +* [x] Greg Denton +* [x] Ben Dines +* [ ] Dave Fellows +* [x] Daniel Neilson +* [ ] Alex Schworer +* [ ] Erik Strauss +* [ ] Lars van der Bijl + +Committer Attendees: + +* [ ] Christian Smith +* [x] Diego Tavares +* [ ] Idris Miles + +Agenda/Notes: + +* Completed 2021 Goals + * User survey + * ASWF graduation + * Drop Oracle support + * Rename demo_data.sql + * Expanded GPU support +* Todo 2021 Goals + * Open Source Days 2021 + * OpenCue session Wednesday, August 4 at 10:55 AM - 11:35 AM PT + * Slides + * Google docs + template: https://docs.google.com/presentation/d/12GJ_6xSm0fwBlZON3VVGhYr5gQKRgu82YIVOvLH3U74/edit?usp=sharing + * Let's have a single master deck to avoid switching between presenters. Can start with + separate decks and merge later. Last OSD we used "gain control" so each speaker could + control their own slides. + * Discuss SPI portion of the session. + * Shotgun integration. Can we share? + * Sanitized Grafana boards? + * Diego/Ben will look into this. + * Brian: to start email checkin thread and plan a rehearsal meeting. + * New user UX + * Publish PyPI packages + * Design + doc: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/ + * Proposal for config file + standardization: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/edit#heading=h.v85uo9klwqft + * Config cleanup for pycue done and ready for review. Tested on macOS and Windows. + * Next up will do config cleanup on other components. + * No update here yet. + * Fix cuebot:latest tag for sandbox setup + * Proposal: publish Docker images for every new patch version (i.e. every new commit to + master) + * Proposal accepted. + * No progress implementing this yet. + * Related: for PyPI design we may want to also publish packages for every new patch + version. + * Docs refresh + * User guide refresh + * Split into "install from latest release" and "install from master" + * Needed to assist transition to publish Docker images on all commits + * No progress. + * API reference + * We have the ability to generate HTML from any commit, but we need a new CI pipeline to + build and publish this on opencue.io for new releases. + * Proposal: for now, highlight existing docs for building reference locally + * Proposal accepted. + * Brian: cleaned up local build process, sent email with doc + link: https://www.opencue.io/contributing/opencue/build-docs/ + * pyoutline examples + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/177 + * No progress yet. + * User guide for CueGUI + * No progress. + * Configuration guide for client-side python + * In progress, will be done as part of PyPI work. + * Configuration guide for Cuebot + * In progress, will be done as part of PyPI work. + * Configuration guide for RQD + * In progress, will be done as part of PyPI work. + * Improve scheduler logic + * Diego has volunteered to own this. + * Email reply from Matt, looped in others. + * Let's start github issue/PR for this + * Create `rfc` tag + * Expand DCC plugins + * No progress. + * Logging solution + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/880 + * PR approved and merged. + * Proposal: write blog post on this topic + * Proposal accepted + * No progress on this yet + * CSP Terraform docs + * No progress yet. + * Auto-close Github issues + * Proposal: + * Any issues in a Project have been identified as important and will be ignored. + * Other issues: + * If no update for 60 days, will get the label "stale" and a comment notifying of + auto-close. + * If no further update for 7 days, will be closed. + * Any issue update will rest the timer. + * Action https://github.com/marketplace/actions/close-stale-issues will be used. + * May need another action to label issues if they are in a project, the above action doesn't + appear to have options for ignoring issues based on project membership. + * No progress yet. + * Expanded Cloud functionality + * Finalize GSoC plugin + * Test Azure support + * Add AWS support + * Design expanded functionality e.g. autoscaling for work in the queue. + * Greg: started to test / clean up azure support. Looking good so far. From e743be187539352ebb98120c0f4c858b7ef9e871 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sun, 12 Sep 2021 12:38:54 -0700 Subject: [PATCH 123/277] [rqd] Add automatic gRPC retries, retry grpc.StatusCode.UNAVAILABLE. (#1015) --- rqd/rqd/rqnetwork.py | 110 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 107 insertions(+), 3 deletions(-) diff --git a/rqd/rqd/rqnetwork.py b/rqd/rqd/rqnetwork.py index 43abc8229..7f6db08e1 100644 --- a/rqd/rqd/rqnetwork.py +++ b/rqd/rqd/rqnetwork.py @@ -23,6 +23,7 @@ from builtins import object from concurrent import futures from random import shuffle +import abc import atexit import logging as log import os @@ -223,11 +224,23 @@ def closeChannel(self): def __getChannel(self): # TODO(bcipriano) Add support for the facility nameserver or drop this concept? (Issue #152) if self.channel is None: + # create interceptors + interceptors = ( + RetryOnRpcErrorClientInterceptor( + max_attempts=4, + sleeping_policy=ExponentialBackoff(init_backoff_ms=100, + max_backoff_ms=1600, + multiplier=2), + status_for_retry=(grpc.StatusCode.UNAVAILABLE,), + ), + ) + cuebots = rqd.rqconstants.CUEBOT_HOSTNAME.split() shuffle(cuebots) - for cuebotHostname in cuebots: - self.channel = grpc.insecure_channel('%s:%s' % (cuebotHostname, - rqd.rqconstants.CUEBOT_GRPC_PORT)) + if len(cuebots) > 0: + self.channel = grpc.insecure_channel('%s:%s' % (cuebots[0], + rqd.rqconstants.CUEBOT_GRPC_PORT), + *interceptors) atexit.register(self.closeChannel) def __getReportStub(self): @@ -253,3 +266,94 @@ def reportRunningFrameCompletion(self, report): request = rqd.compiled_proto.report_pb2.RqdReportRunningFrameCompletionRequest( frame_complete_report=report) stub.ReportRunningFrameCompletion(request, timeout=rqd.rqconstants.RQD_TIMEOUT) + + +# Python 2/3 compatible implementation of ABC +ABC = abc.ABCMeta('ABC', (object,), {'__slots__': ()}) + + +class SleepingPolicy(ABC): + """ + Implement policy for sleeping between API retries + """ + @abc.abstractmethod + def sleep(self, attempt): + """ + How long to sleep in milliseconds. + :param attempt: the number of attempt (starting from zero) + """ + assert attempt >= 0 + + +class ExponentialBackoff(SleepingPolicy): + """ + Implement policy that will increase retry period by exponentially in every try + """ + def __init__(self, + init_backoff_ms, + max_backoff_ms, + multiplier=2): + """ + inputs in ms + """ + self._init_backoff = init_backoff_ms + self._max_backoff = max_backoff_ms + self._multiplier = multiplier + + def sleep(self, attempt): + sleep_time_ms = min( + self._init_backoff * self._multiplier ** attempt, + self._max_backoff + ) + time.sleep(sleep_time_ms / 1000.0) + + +class RetryOnRpcErrorClientInterceptor( + grpc.UnaryUnaryClientInterceptor, + grpc.StreamUnaryClientInterceptor +): + """ + Implement Client/Stream interceptors for GRPC channels to retry + calls that failed with retry-able states. This is required for + handling server interruptions that are not automatically handled + by grpc.insecure_channel + """ + def __init__(self, + max_attempts, + sleeping_policy, + status_for_retry=None): + self._max_attempts = max_attempts + self._sleeping_policy = sleeping_policy + self._retry_statuses = status_for_retry + + def _intercept_call(self, continuation, client_call_details, + request_or_iterator): + for attempt in range(self._max_attempts): + try: + return continuation(client_call_details, + request_or_iterator) + except grpc.RpcError as response: + # Return if it was last attempt + if attempt == (self._max_attempts - 1): + return response + + # If status code is not in retryable status codes + # pylint: disable=no-member + if self._retry_statuses \ + and hasattr(response, 'code') \ + and response.code() \ + not in self._retry_statuses: + return response + + self._sleeping_policy.sleep(attempt) + + def intercept_unary_unary(self, continuation, client_call_details, + request): + return self._intercept_call(continuation, client_call_details, + request) + + def intercept_stream_unary( + self, continuation, client_call_details, request_iterator + ): + return self._intercept_call(continuation, client_call_details, + request_iterator) From 341a6d1602dfe59e4f797e547ffb9edafddde037 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sun, 12 Sep 2021 12:41:19 -0700 Subject: [PATCH 124/277] Add GPU stats to allocation wrapper API. (#1016) --- pycue/opencue/wrappers/allocation.py | 40 ++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/pycue/opencue/wrappers/allocation.py b/pycue/opencue/wrappers/allocation.py index 8539dbf4e..bbbb9b2d4 100644 --- a/pycue/opencue/wrappers/allocation.py +++ b/pycue/opencue/wrappers/allocation.py @@ -167,6 +167,46 @@ def totalLockedCores(self): """ return self.data.stats.locked_cores + def totalGpus(self): + """Returns the total number of gpus in the allocation. + + :rtype: float + :return: total number of gpus in the allocation + """ + return self.data.stats.gpus + + def totalAvailableGpus(self): + """Returns the total number of gpus available for booking in the allocation. + + :rtype: float + :return: total number of gpus in the allocation + """ + return self.data.stats.available_gpus + + def totalIdleGpus(self): + """Returns the total number of idle gpus in the allocation. + + :rtype: float + :return: total number of idle gpus in the allocation + """ + return self.data.stats.idle_gpus + + def totalRunningGpus(self): + """Returns the total number of running gpus in the allocation. + + :rtype: float + :return: total number of running gpus in the allocation + """ + return self.data.stats.running_gpus + + def totalLockedGpus(self): + """Returns the total number of locked gpus in the allocation. + + :rtype: float + :return: total number of locked gpus in the allocation + """ + return self.data.stats.locked_gpus + def totalHosts(self): """Returns the total number of hosts in the allocation. From 42ffe1f499f6741bf7c8e94e84065dde52297a7b Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sun, 12 Sep 2021 12:43:43 -0700 Subject: [PATCH 125/277] Fix number of GPU units in RunningFrameInfo. (#1017) --- rqd/rqd/rqnetwork.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rqd/rqd/rqnetwork.py b/rqd/rqd/rqnetwork.py index 7f6db08e1..7e212dbee 100644 --- a/rqd/rqd/rqnetwork.py +++ b/rqd/rqd/rqnetwork.py @@ -63,7 +63,6 @@ def __init__(self, rqCore, runFrame): self.vsize = 0 self.maxVsize = 0 - self.numGpus = 0 self.usedGpuMemory = 0 self.maxUsedGpuMemory = 0 @@ -90,7 +89,7 @@ def runningFrameInfo(self): vsize=self.vsize, attributes=self.runFrame.attributes, llu_time=self.lluTime, - num_gpus=self.numGpus, + num_gpus=self.runFrame.num_gpus, max_used_gpu_memory=self.maxUsedGpuMemory, used_gpu_memory=self.usedGpuMemory ) From 053f479c4ede1933f3125f879f8561c7ce6d3750 Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Sun, 12 Sep 2021 12:46:29 -0700 Subject: [PATCH 126/277] Make sure nestedJobWhiteboard is processed in order. (#973) --- .../dao/postgres/NestedWhiteboardDaoJdbc.java | 97 ++++++++++++++----- 1 file changed, 75 insertions(+), 22 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/NestedWhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/NestedWhiteboardDaoJdbc.java index 924a65a96..8c920c5a4 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/NestedWhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/NestedWhiteboardDaoJdbc.java @@ -23,10 +23,10 @@ import java.sql.SQLException; import java.sql.Timestamp; import java.util.ArrayList; +import java.util.concurrent.ConcurrentHashMap; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import org.springframework.jdbc.core.RowMapper; import org.springframework.jdbc.core.support.JdbcDaoSupport; @@ -157,10 +157,56 @@ public CachedJobWhiteboardMapper(NestedJobWhiteboardMapper result) { "AND " + "folder.pk_dept = dept.pk_dept "; - class NestedJobWhiteboardMapper implements RowMapper { + private class ChildrenEntry { + String key; + int level; + List children; + String name; + + public ChildrenEntry(String key, int level, String name) { + this.key = key; + this.level = level; + this.children = new ArrayList<>(); + this.name = name; + } + + public List getChildren() { + return children; + } + + public void addChild(String child) { + children.add(child); + } + + public String getKey() { + return key; + } + + public String getName() { + return name; + } + + public int compareTo(ChildrenEntry o) { + // Invert order + return Integer.compare(o.level, this.level); + } - public Map groups = new HashMap(50); - public Map> childrenMap = new HashMap>(); + @Override + public String toString() { + StringBuilder out = new StringBuilder(); + String spacing = " ".repeat(Math.max(0, this.level + 1)); + out.append(spacing); + out.append(key + "(c " + name + ")"); + for (String id : children) { + out.append("\n " + spacing + id.substring(0, 4)); + } + return out.toString(); + } + } + + class NestedJobWhiteboardMapper implements RowMapper { + public Map groups = new HashMap(50); + public Map childrenMap = new HashMap(); public String rootGroupID; @Override @@ -186,12 +232,16 @@ public NestedGroup mapRow(ResultSet rs, int rowNum) throws SQLException { String parentGroupId = rs.getString("pk_parent_folder"); if (parentGroupId != null) { - List children = childrenMap.get(parentGroupId); - if (children == null) { - children = new ArrayList<>(); - childrenMap.put(parentGroupId, children); + ChildrenEntry childrenEntry = childrenMap.get(parentGroupId); + if (childrenEntry == null) { + childrenEntry = new ChildrenEntry( + parentGroupId, group.getLevel() - 1, rs.getString("group_name")); + childrenEntry.addChild(groupId); + childrenMap.put(parentGroupId, childrenEntry); + } + else { + childrenEntry.addChild(groupId); } - children.add(groupId); } else { rootGroupID = rs.getString("pk_folder"); @@ -223,19 +273,22 @@ public NestedGroup mapRow(ResultSet rs, int rowNum) throws SQLException { } private NestedJobWhiteboardMapper updateConnections(NestedJobWhiteboardMapper mapper) { - for (Map.Entry> entry : mapper.childrenMap.entrySet()) { - NestedGroup group = mapper.groups.get(entry.getKey()); - NestedGroupSeq.Builder childrenBuilder = NestedGroupSeq.newBuilder(); - for (String childId : entry.getValue()) { - NestedGroup child = mapper.groups.get(childId); - child = child.toBuilder().setParent(group).build(); - childrenBuilder.addNestedGroups(child); - mapper.groups.put(childId, child); - } - group = group.toBuilder() - .setGroups(childrenBuilder.build()) - .build(); - mapper.groups.put(entry.getKey(), group); + ArrayList orderedChildren = new ArrayList<>(mapper.childrenMap.values()); + orderedChildren.sort(ChildrenEntry::compareTo); + + for (ChildrenEntry entry : orderedChildren) { + NestedGroup group = mapper.groups.get(entry.getKey()); + NestedGroupSeq.Builder childrenBuilder = NestedGroupSeq.newBuilder(); + for (String childId : entry.getChildren()) { + NestedGroup child = mapper.groups.get(childId); + child = child.toBuilder().setParent(group).build(); + childrenBuilder.addNestedGroups(child); + mapper.groups.put(childId, child); + } + group = group.toBuilder() + .setGroups(childrenBuilder.build()) + .build(); + mapper.groups.put(entry.getKey(), group); } return mapper; } From 537168f916e2649ece327d3abf2a01fd38d91159 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sun, 12 Sep 2021 13:12:36 -0700 Subject: [PATCH 127/277] Remove unused stranded GPU code. (#1020) --- .../com/imageworks/spcue/StrandedGpus.java | 44 ------------------- .../spcue/dispatcher/DispatchSupport.java | 34 -------------- .../dispatcher/DispatchSupportService.java | 41 ----------------- 3 files changed, 119 deletions(-) delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/StrandedGpus.java diff --git a/cuebot/src/main/java/com/imageworks/spcue/StrandedGpus.java b/cuebot/src/main/java/com/imageworks/spcue/StrandedGpus.java deleted file mode 100644 index 91b9ad76a..000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/StrandedGpus.java +++ /dev/null @@ -1,44 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue; - -public final class StrandedGpus { - - /** - * The maximum time this object should be valid. - */ - private static final long MAX_AGE_MILLIS = 5000l; - - private final int gpus; - private final long expireTime = System.currentTimeMillis() + MAX_AGE_MILLIS; - - public StrandedGpus(int gpus) { - this.gpus = gpus; - } - - public int getGpus() { - return this.gpus; - } - - public boolean isExpired() { - return System.currentTimeMillis() > expireTime; - } -} - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java index 47dac264a..3aa9f3ab9 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java @@ -526,40 +526,6 @@ void updateProcMemoryUsage(FrameInterface frame, long rss, long maxRss, long vsi */ void determineIdleCores(DispatchHost host, int load); - /** - * Pickup any gpus that were stranded on the given host. - * - * @param host - */ - void pickupStrandedGpus(DispatchHost host); - - /** - * Return true if the host has stranded gpus. - * - * @param host - * @return - */ - boolean hasStrandedGpus(HostInterface host); - - /** - * Add stranded gpus for the given host. Stranded - * gpus will automatically be added to the next frame dispatched - * from the host to make up for gpus stranded with no memory. - * - * @param host - * @param gpus - */ - void strandGpus(DispatchHost host, int gpus); - - /** - * Lowers the perceived idle gpus on a machine if - * the load is over certain threshold. - * - * @param host - * @param load - */ - void determineIdleGpus(DispatchHost host, int load); - /** * Return a set of job IDs that can take the given host. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java index ad1d8196c..35fe506c4 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java @@ -42,7 +42,6 @@ import com.imageworks.spcue.ResourceUsage; import com.imageworks.spcue.ShowInterface; import com.imageworks.spcue.StrandedCores; -import com.imageworks.spcue.StrandedGpus; import com.imageworks.spcue.VirtualProc; import com.imageworks.spcue.dao.BookingDao; import com.imageworks.spcue.dao.DispatcherDao; @@ -83,9 +82,6 @@ public class DispatchSupportService implements DispatchSupport { private ConcurrentHashMap strandedCores = new ConcurrentHashMap(); - private ConcurrentHashMap strandedGpus = - new ConcurrentHashMap(); - @Override public void pickupStrandedCores(DispatchHost host) { logger.info(host + "picked up stranded cores"); @@ -117,35 +113,6 @@ public void strandCores(DispatchHost host, int cores) { strandedCoresCount.getAndIncrement(); } - @Override - public void pickupStrandedGpus(DispatchHost host) { - logger.info(host + "picked up stranded gpu"); - pickedUpGpusCount.getAndIncrement(); - strandedGpus.remove(host.getHostId()); - } - - @Override - public boolean hasStrandedGpus(HostInterface host) { - StrandedGpus stranded = strandedGpus.get(host.getHostId()); - if (stranded == null) { - return false; - } - if (stranded.isExpired()) { - return false; - } - - return true; - } - - @Override - public void strandGpus(DispatchHost host, int gpus) { - logger.info(host + " found " + gpus + ", stranded gpu"); - host.strandedGpus = gpus; - strandedGpus.putIfAbsent(host.getHostId(), new StrandedGpus(gpus)); - strandedGpusCount.getAndIncrement(); - } - - @Transactional(readOnly = true) public List findNextDispatchFrames(JobInterface job, VirtualProc proc, int limit) { return dispatcherDao.findNextDispatchFrames(job, proc, limit); @@ -615,14 +582,6 @@ public void determineIdleCores(DispatchHost host, int load) { } } - @Override - public void determineIdleGpus(DispatchHost host, int load) { - int idleGpu = host.gpus - load; - if (idleGpu < host.idleGpus) { - host.idleGpus = idleGpu; - } - } - public DispatcherDao getDispatcherDao() { return dispatcherDao; } From 01764798f6af10cf9100620f56289e00f3dd69f0 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sun, 12 Sep 2021 13:28:52 -0700 Subject: [PATCH 128/277] Support GPU for Windows. (#1024) --- rqd/rqd/rqmachine.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 3da883111..13ae37df8 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -644,6 +644,9 @@ def updateMachineStats(self): self.__renderHost.free_mcp = TEMP_DEFAULT self.__renderHost.free_swap = int(stats.ullAvailPageFile / 1024) self.__renderHost.free_mem = int(stats.ullAvailPhys / 1024) + self.__renderHost.num_gpus = self.getGpuCount() + self.__renderHost.total_gpu_mem = self.getGpuMemoryTotal() + self.__renderHost.free_gpu_mem = self.getGpuMemoryFree() # Updates dynamic information self.__renderHost.load = self.getLoadAvg() From 5a022304944ce73a4c924f5694bcc0c0a4d6a1c9 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sun, 12 Sep 2021 13:31:56 -0700 Subject: [PATCH 129/277] [cuebot] Use long internally for Max RSS. (#1027) --- cuebot/src/main/java/com/imageworks/spcue/FrameDetail.java | 2 +- .../java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/FrameDetail.java b/cuebot/src/main/java/com/imageworks/spcue/FrameDetail.java index 60c07a030..891523805 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/FrameDetail.java +++ b/cuebot/src/main/java/com/imageworks/spcue/FrameDetail.java @@ -30,7 +30,7 @@ public class FrameDetail extends FrameEntity implements FrameInterface { public int dependCount; public int retryCount; public int exitStatus; - public int maxRss; + public long maxRss; public int dispatchOrder; public String lastResource; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java index e905d8e35..79ab96728 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java @@ -400,7 +400,7 @@ public FrameDetail mapRow(ResultSet rs, int rowNum) throws SQLException { frame.jobId = rs.getString("pk_job"); frame.layerId = rs.getString("pk_layer"); frame.showId = rs.getString("pk_show"); - frame.maxRss = rs.getInt("int_mem_max_used"); + frame.maxRss = rs.getLong("int_mem_max_used"); frame.name = rs.getString("str_name"); frame.number = rs.getInt("int_number"); frame.dispatchOrder = rs.getInt("int_dispatch_order"); From 929090917dfe568c1a18a1287baff541fe6e8d56 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sun, 12 Sep 2021 13:34:37 -0700 Subject: [PATCH 130/277] [rqd] Add workaround for PowerShell Exitcode. (#1028) --- rqd/rqd/rqcore.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index b7dcae31a..a6809e85c 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -44,6 +44,9 @@ import rqd.rqutil +INT32_MAX = 2147483647 +INT32_MIN = -2147483648 + class FrameAttendantThread(threading.Thread): """Once a frame has been received and checked by RQD, this class handles the launching, waiting on, and cleanup work related to running the @@ -361,6 +364,10 @@ def runWindows(self): # Find exitStatus and exitSignal returncode = frameInfo.forkedCommand.returncode + if returncode < INT32_MIN: + returncode = 303 + if returncode > INT32_MAX: + returncode = 304 frameInfo.exitStatus = returncode frameInfo.exitSignal = returncode From 1cedc891b08bd79b8c889adf0bb82a32b5de30e7 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sun, 12 Sep 2021 13:38:49 -0700 Subject: [PATCH 131/277] [rqd] Remove extra space and extra MB from log output. (#1031) --- .../com/imageworks/spcue/dispatcher/HostReportHandler.java | 2 +- rqd/rqd/rqcore.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java index 34e02021e..d9f670838 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java @@ -430,7 +430,7 @@ private void handleMemoryReservations(final DispatchHost host, final HostReport + proc.getName() + " was OOM"); try { killQueue.execute(new DispatchRqdKillFrame(proc, "The frame required " + - CueUtil.KbToMb(f.getRss()) + "MB but the machine only has " + + CueUtil.KbToMb(f.getRss()) + " but the machine only has " + CueUtil.KbToMb(host.memory), rqdClient)); } catch (TaskRejectedException e) { logger.warn("Unable to queue RQD kill, task rejected, " + e); diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index a6809e85c..7cb08030b 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -183,7 +183,8 @@ def __writeFooter(self): self.endTime = time.time() self.frameInfo.runTime = int(self.endTime - self.startTime) try: - print("\n", "="*59, file=self.rqlog) + print("", file=self.rqlog) + print("="*59, file=self.rqlog) print("RenderQ Job Complete\n", file=self.rqlog) print("%-20s%s" % ("exitStatus", self.frameInfo.exitStatus), file=self.rqlog) print("%-20s%s" % ("exitSignal", self.frameInfo.exitSignal), file=self.rqlog) From 4deedac76f0c8c0a93e7b1b7fa05b2dc20f4ad24 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sun, 12 Sep 2021 13:41:40 -0700 Subject: [PATCH 132/277] [cuebot] Update GPU memory usage in the database from host report. (#1032) --- .../main/java/com/imageworks/spcue/dao/ProcDao.java | 3 ++- .../com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java | 7 +++++-- .../imageworks/spcue/dispatcher/DispatchSupport.java | 4 +++- .../spcue/dispatcher/DispatchSupportService.java | 6 ++++-- .../imageworks/spcue/dispatcher/HostReportHandler.java | 3 ++- .../spcue/test/dao/postgres/ProcDaoTests.java | 10 +++++----- 6 files changed, 21 insertions(+), 12 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/ProcDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/ProcDao.java index 31e49a208..c96e8e28e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/ProcDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/ProcDao.java @@ -150,7 +150,8 @@ public interface ProcDao { * @param maxKb */ void updateProcMemoryUsage(FrameInterface f, long rss, long maxRss, - long vsize, long maxVsize); + long vsize, long maxVsize, + long usedGpuMemory, long maxUsedGpuMemory); /** * get aq virual proc from its unique id diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java index ba9f33c1f..43c7b081b 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java @@ -234,13 +234,15 @@ public boolean clearVirtualProcAssignment(FrameInterface frame) { "int_mem_max_used = ?," + "int_virt_used = ?, " + "int_virt_max_used = ?, " + + "int_gpu_mem_used = ?, " + + "int_gpu_mem_max_used = ?, " + "ts_ping = current_timestamp " + "WHERE " + "pk_frame = ?"; @Override public void updateProcMemoryUsage(FrameInterface f, long rss, long maxRss, - long vss, long maxVss) { + long vss, long maxVss, long usedGpuMemory, long maxUsedGpuMemory) { /* * This method is going to repeat for a proc every 1 minute, so * if the proc is being touched by another thread, then return @@ -256,7 +258,8 @@ public void updateProcMemoryUsage(FrameInterface f, long rss, long maxRss, String.class, f.getFrameId()).equals(f.getFrameId())) { getJdbcTemplate().update(UPDATE_PROC_MEMORY_USAGE, - rss, maxRss, vss, maxVss, f.getFrameId()); + rss, maxRss, vss, maxVss, + usedGpuMemory, maxUsedGpuMemory, f.getFrameId()); } } catch (DataAccessException dae) { logger.info("The proc for frame " + f + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java index 3aa9f3ab9..0fcfbef26 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java @@ -442,9 +442,11 @@ List findNextDispatchFrames(LayerInterface layer, VirtualProc pro * @param maxRss * @param vsize * @param maxVsize + * @param usedGpuMemory + * @param maxUsedGpuMemory */ void updateProcMemoryUsage(FrameInterface frame, long rss, long maxRss, long vsize, - long maxVsize); + long maxVsize, long usedGpuMemory, long maxUsedGpuMemory); /** * Return true if adding the given core units would put the show diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java index 35fe506c4..53d882035 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java @@ -535,8 +535,10 @@ public void lostProc(VirtualProc proc, String reason, int exitStatus) { @Override @Transactional(propagation = Propagation.REQUIRED) public void updateProcMemoryUsage(FrameInterface frame, long rss, long maxRss, - long vsize, long maxVsize) { - procDao.updateProcMemoryUsage(frame, rss, maxRss, vsize, maxVsize); + long vsize, long maxVsize, + long usedGpuMemory, long maxUsedGpuMemory) { + procDao.updateProcMemoryUsage(frame, rss, maxRss, vsize, maxVsize, + usedGpuMemory, maxUsedGpuMemory); } @Override diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java index d9f670838..92f7335c6 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java @@ -572,7 +572,8 @@ private void updateMemoryUsage(List rFrames) { rf.getRss(), rf.getMaxRss()); dispatchSupport.updateProcMemoryUsage(frame, - rf.getRss(), rf.getMaxRss(), rf.getVsize(), rf.getMaxVsize()); + rf.getRss(), rf.getMaxRss(), rf.getVsize(), rf.getMaxVsize(), + rf.getUsedGpuMemory(), rf.getMaxUsedGpuMemory()); } updateJobMemoryUsage(rFrames); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java index 6c9efc3e5..dbecca54d 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java @@ -326,7 +326,7 @@ public void testUpdateProcMemoryUsage() { procDao.insertVirtualProc(proc); procDao.verifyRunningProc(proc.getId(), frame.getId()); - procDao.updateProcMemoryUsage(frame, 100, 100, 1000, 1000); + procDao.updateProcMemoryUsage(frame, 100, 100, 1000, 1000, 0, 0); } @@ -593,7 +593,7 @@ public void testFindReservedMemoryOffender() { // Increase the memory usage as frames are added procDao.updateProcMemoryUsage(frame, - 1000*i, 1000*i, 1000*i, 1000*i); + 1000*i, 1000*i, 1000*i, 1000*i, 0, 0); i++; } @@ -666,7 +666,7 @@ public void testBalanceUnderUtilizedProcs() { proc1.frameId = frame1.id; procDao.insertVirtualProc(proc1); - procDao.updateProcMemoryUsage(frame1, 250000, 250000, 250000, 250000); + procDao.updateProcMemoryUsage(frame1, 250000, 250000, 250000, 250000, 0, 0); layerDao.updateLayerMaxRSS(frame1, 250000, true); FrameDetail frameDetail2 = frameDao.findFrameDetail(job, "0002-pass_1"); @@ -676,7 +676,7 @@ public void testBalanceUnderUtilizedProcs() { proc2.frameId = frame2.id; procDao.insertVirtualProc(proc2); - procDao.updateProcMemoryUsage(frame2, 255000, 255000,255000, 255000); + procDao.updateProcMemoryUsage(frame2, 255000, 255000,255000, 255000, 0, 0); layerDao.updateLayerMaxRSS(frame2, 255000, true); FrameDetail frameDetail3 = frameDao.findFrameDetail(job, "0003-pass_1"); @@ -686,7 +686,7 @@ public void testBalanceUnderUtilizedProcs() { proc3.frameId = frame3.id; procDao.insertVirtualProc(proc3); - procDao.updateProcMemoryUsage(frame3, 3145728, 3145728,3145728, 3145728); + procDao.updateProcMemoryUsage(frame3, 3145728, 3145728,3145728, 3145728, 0, 0); layerDao.updateLayerMaxRSS(frame3,300000, true); procDao.balanceUnderUtilizedProcs(proc3, 100000); From da28ae905b81e7d1125db2073a369fdc0ae9acd4 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sun, 12 Sep 2021 13:58:09 -0700 Subject: [PATCH 133/277] [cuebot] Combine frame usage and memory usage updates. (#1006) --- .../com/imageworks/spcue/dao/FrameDao.java | 18 +----- .../spcue/dao/postgres/FrameDaoJdbc.java | 27 +++------ .../spcue/dispatcher/DispatchSupport.java | 14 ++--- .../dispatcher/DispatchSupportService.java | 23 ++------ .../spcue/dispatcher/HostReportHandler.java | 31 ++-------- .../test/dao/criteria/FrameSearchTests.java | 2 +- .../dispatcher/HostReportHandlerTests.java | 56 ++++++++++++++++++- .../resources/conf/jobspec/jobspec_simple.xml | 47 ++++++++++++++++ 8 files changed, 125 insertions(+), 93 deletions(-) create mode 100644 cuebot/src/test/resources/conf/jobspec/jobspec_simple.xml diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java index 6f9fc8016..7c22c3e07 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java @@ -307,20 +307,7 @@ boolean updateFrameStopped(FrameInterface frame, FrameState state, int exitStatu ResourceUsage getResourceUsage(FrameInterface f); /** - * Update Frame usage values for the given frame. The - * frame must be in the Running state. If the frame - * is locked by another thread, the process is aborted because - * we'll most likely get a new update one minute later. - * - * @param f - * @param lluTime - * @throws FrameReservationException if the frame is locked - * by another thread. - */ - void updateFrameUsage(FrameInterface f, long lluTime); - - /** - * Update memory usage values for the given frame. The + * Update memory usage values and LLU time for the given frame. The * frame must be in the Running state. If the frame * is locked by another thread, the process is aborted because * we'll most likely get a new update one minute later. @@ -328,10 +315,11 @@ boolean updateFrameStopped(FrameInterface frame, FrameState state, int exitStatu * @param f * @param maxRss * @param rss + * @param lluTime * @throws FrameReservationException if the frame is locked * by another thread. */ - void updateFrameMemoryUsage(FrameInterface f, long maxRss, long rss); + void updateFrameMemoryUsageAndLluTime(FrameInterface f, long maxRss, long rss, long lluTime); /** * Attempt to put a exclusive row lock on the given diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java index 79ab96728..a7e0dad47 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java @@ -966,35 +966,22 @@ public ResourceUsage getResourceUsage(FrameInterface f) { "pk_frame = ?", RESOURCE_USAGE_MAPPER, f.getFrameId()); } - private static final String UPDATE_FRAME_IO_USAGE = - "UPDATE " + - "frame " + - "SET " + - "ts_updated = current_timestamp," + - "ts_llu = ? " + - "WHERE " + - "pk_frame = ? "; - - @Override - public void updateFrameUsage(FrameInterface f, long lluTime) { - getJdbcTemplate().update(UPDATE_FRAME_IO_USAGE, - new Timestamp(lluTime * 1000l), f.getFrameId()); - } - - private static final String UPDATE_FRAME_MEMORY_USAGE = + private static final String UPDATE_FRAME_MEMORY_USAGE_AND_LLU_TIME = "UPDATE " + "frame " + "SET " + "ts_updated = current_timestamp," + "int_mem_max_used = ?," + - "int_mem_used = ? " + + "int_mem_used = ?," + + "ts_llu = ? " + "WHERE " + "pk_frame = ? "; @Override - public void updateFrameMemoryUsage(FrameInterface f, long maxRss, long rss) { - getJdbcTemplate().update(UPDATE_FRAME_MEMORY_USAGE, - maxRss, rss, f.getFrameId()); + public void updateFrameMemoryUsageAndLluTime(FrameInterface f, long maxRss, long rss, + long lluTime) { + getJdbcTemplate().update(UPDATE_FRAME_MEMORY_USAGE_AND_LLU_TIME, + maxRss, rss, new Timestamp(lluTime * 1000l), f.getFrameId()); } /** diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java index 0fcfbef26..bff3dd6af 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java @@ -416,21 +416,15 @@ List findNextDispatchFrames(LayerInterface layer, VirtualProc pro void clearFrame(DispatchFrame frame); /** - * Update usage data for the given frame. - * - * @param frame - * @param lluTime - */ - void updateFrameUsage(FrameInterface frame, long lluTime); - - /** - * Update memory usage data for the given frame. + * Update Memory usage data and LLU time for the given frame. * * @param frame * @param rss * @param maxRss + * @param lluTime */ - void updateFrameMemoryUsage(FrameInterface frame, long rss, long maxRss); + void updateFrameMemoryUsageAndLluTime(FrameInterface frame, long rss, long maxRss, + long lluTime); /** * Update memory usage data for a given frame's proc record. The diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java index 53d882035..4a871803e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java @@ -543,33 +543,18 @@ public void updateProcMemoryUsage(FrameInterface frame, long rss, long maxRss, @Override @Transactional(propagation = Propagation.REQUIRED) - public void updateFrameUsage(FrameInterface frame, long lluTime) { + public void updateFrameMemoryUsageAndLluTime(FrameInterface frame, long rss, long maxRss, + long lluTime) { try { - frameDao.updateFrameUsage(frame, lluTime); + frameDao.updateFrameMemoryUsageAndLluTime(frame, maxRss, rss, lluTime); } catch (FrameReservationException ex) { // Eat this, the frame was not in the correct state or // was locked by another thread. The only reason it would // be locked by another thread would be if the state is // changing. - logger.warn("failed to update io stats for frame: " + frame); - } - } - - @Override - @Transactional(propagation = Propagation.REQUIRED) - public void updateFrameMemoryUsage(FrameInterface frame, long rss, long maxRss) { - - try { - frameDao.updateFrameMemoryUsage(frame, maxRss, rss); - } - catch (FrameReservationException ex) { - // Eat this, the frame was not in the correct state or - // was locked by another thread. The only reason it would - // be locked by another thread would be if the state is - // changing. - logger.warn("failed to update memory stats for frame: " + frame); + logger.warn("failed to update memory usage and LLU time for frame: " + frame); } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java index 92f7335c6..33d7a3cf8 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java @@ -188,15 +188,9 @@ public void handleHostReport(HostReport report, boolean isBoot) { /* * Updates memory usage for the proc, frames, - * jobs, and layers. + * jobs, and layers. And LLU time for the frames. */ - updateMemoryUsage(report.getFramesList()); - - /* - * Updates usage for the proc, frames, - * jobs, and layers. - */ - updateFrameUsage(report.getFramesList()); + updateMemoryUsageAndLluTime(report.getFramesList()); /* * kill frames that have over run. @@ -545,31 +539,18 @@ private void killTimedOutFrames(HostReport report) { } /** - * Update IO usage for the given list of frames. - * - * @param rFrames - */ - private void updateFrameUsage(List rFrames) { - - for (RunningFrameInfo rf: rFrames) { - FrameInterface frame = jobManager.getFrame(rf.getFrameId()); - dispatchSupport.updateFrameUsage(frame, rf.getLluTime()); - } - } - - /** - * Update memory usage for the given list of frames. + * Update memory usage and LLU time for the given list of frames. * * @param rFrames */ - private void updateMemoryUsage(List rFrames) { + private void updateMemoryUsageAndLluTime(List rFrames) { for (RunningFrameInfo rf: rFrames) { FrameInterface frame = jobManager.getFrame(rf.getFrameId()); - dispatchSupport.updateFrameMemoryUsage(frame, - rf.getRss(), rf.getMaxRss()); + dispatchSupport.updateFrameMemoryUsageAndLluTime(frame, + rf.getRss(), rf.getMaxRss(), rf.getLluTime()); dispatchSupport.updateProcMemoryUsage(frame, rf.getRss(), rf.getMaxRss(), rf.getVsize(), rf.getMaxVsize(), diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/criteria/FrameSearchTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/criteria/FrameSearchTests.java index ca6d3c2c2..1e8f01669 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/criteria/FrameSearchTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/criteria/FrameSearchTests.java @@ -235,7 +235,7 @@ public void filterByMemoryRange() { i -> { FrameInterface frame = frameDao.findFrame(layer, i); frameDao.updateFrameState(frame, FrameState.RUNNING); - frameDao.updateFrameMemoryUsage(frame, CueUtil.GB * 5, CueUtil.GB); + frameDao.updateFrameMemoryUsageAndLluTime(frame, CueUtil.GB * 5, CueUtil.GB, 0); }); FrameSearchInterface frameSearch = frameSearchFactory.create(); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java index f4d706cdb..d27f76c32 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java @@ -19,6 +19,9 @@ package com.imageworks.spcue.test.dispatcher; +import java.io.File; +import java.sql.Timestamp; +import java.util.List; import javax.annotation.Resource; import org.junit.Before; @@ -32,15 +35,20 @@ import com.imageworks.spcue.dispatcher.Dispatcher; import com.imageworks.spcue.dispatcher.HostReportHandler; import com.imageworks.spcue.FacilityInterface; +import com.imageworks.spcue.FrameDetail; import com.imageworks.spcue.grpc.host.HardwareState; import com.imageworks.spcue.grpc.host.LockState; import com.imageworks.spcue.grpc.report.CoreDetail; import com.imageworks.spcue.grpc.report.HostReport; import com.imageworks.spcue.grpc.report.RenderHost; +import com.imageworks.spcue.grpc.report.RunningFrameInfo; import com.imageworks.spcue.service.AdminManager; import com.imageworks.spcue.service.HostManager; +import com.imageworks.spcue.service.JobLauncher; +import com.imageworks.spcue.service.JobManager; import com.imageworks.spcue.test.TransactionalTest; import com.imageworks.spcue.util.CueUtil; +import com.imageworks.spcue.VirtualProc; import static org.junit.Assert.assertEquals; @@ -59,6 +67,12 @@ public class HostReportHandlerTests extends TransactionalTest { @Resource Dispatcher dispatcher; + @Resource + JobLauncher jobLauncher; + + @Resource + JobManager jobManager; + private static final String HOSTNAME = "beta"; private static final String NEW_HOSTNAME = "gamma"; @@ -91,12 +105,12 @@ private static RenderHost getRenderHost() { .setName(HOSTNAME) .setBootTime(1192369572) .setFreeMcp(76020) - .setFreeMem(53500) + .setFreeMem((int) CueUtil.GB8) .setFreeSwap(20760) .setLoad(0) .setTotalMcp(195430) - .setTotalMem(8173264) - .setTotalSwap(20960) + .setTotalMem(CueUtil.GB8) + .setTotalSwap(CueUtil.GB2) .setNimbyEnabled(false) .setNumProcs(2) .setCoresPerProc(100) @@ -213,5 +227,41 @@ public void testHandleHostReportWithNonExistentTags() { DispatchHost host = hostManager.findDispatchHost(NEW_HOSTNAME); assertEquals(host.getAllocationId(), alloc.id); } + + @Test + @Transactional + @Rollback(true) + public void testMemoryAndLlu() { + jobLauncher.testMode = true; + jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec_simple.xml")); + + DispatchHost host = getHost(); + List procs = dispatcher.dispatchHost(host); + assertEquals(1, procs.size()); + VirtualProc proc = procs.get(0); + + CoreDetail cores = getCoreDetail(200, 200, 0, 0); + long now = System.currentTimeMillis(); + + RunningFrameInfo info = RunningFrameInfo.newBuilder() + .setJobId(proc.getJobId()) + .setLayerId(proc.getLayerId()) + .setFrameId(proc.getFrameId()) + .setResourceId(proc.getProcId()) + .setLluTime(now / 1000) + .setMaxRss(420000) + .build(); + HostReport report = HostReport.newBuilder() + .setHost(getRenderHost()) + .setCoreInfo(cores) + .addFrames(info) + .build(); + + hostReportHandler.handleHostReport(report, false); + + FrameDetail frame = jobManager.getFrameDetail(proc.getFrameId()); + assertEquals(frame.dateLLU, new Timestamp(now / 1000 * 1000)); + assertEquals(420000, frame.maxRss); + } } diff --git a/cuebot/src/test/resources/conf/jobspec/jobspec_simple.xml b/cuebot/src/test/resources/conf/jobspec/jobspec_simple.xml new file mode 100644 index 000000000..d717374d5 --- /dev/null +++ b/cuebot/src/test/resources/conf/jobspec/jobspec_simple.xml @@ -0,0 +1,47 @@ + + + + + + + + + spi + pipe + default + testuser + 9860 + + + False + 2 + False + + + + echo hello + 0 + 1 + + + shell + + + + + + From f5933e6f323f8b7a06ed21c009f5f7abe9abfea3 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Tue, 19 Oct 2021 11:23:38 -0700 Subject: [PATCH 134/277] Lock protobuf to version 3.17.3 for Python 2. (#1046) --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index f1a3a6f42..cac91168b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ grpcio-tools==1.39.0;python_version>="3.0" mock==2.0.0 packaging==20.9 pathlib==1.0.1;python_version<"3.4" +protobuf==3.17.3;python_version<"3.0" psutil==5.6.6 pyfakefs==3.6 pylint==2.6.0;python_version>="3.7" From ba90de8afb2ce49f589f5c20f99fec5345df189c Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Tue, 19 Oct 2021 11:25:37 -0700 Subject: [PATCH 135/277] [rqd] Add usage of gRPC intercept_channel. (#1047) --- rqd/rqd/rqnetwork.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rqd/rqd/rqnetwork.py b/rqd/rqd/rqnetwork.py index 7e212dbee..127d97100 100644 --- a/rqd/rqd/rqnetwork.py +++ b/rqd/rqd/rqnetwork.py @@ -238,8 +238,8 @@ def __getChannel(self): shuffle(cuebots) if len(cuebots) > 0: self.channel = grpc.insecure_channel('%s:%s' % (cuebots[0], - rqd.rqconstants.CUEBOT_GRPC_PORT), - *interceptors) + rqd.rqconstants.CUEBOT_GRPC_PORT)) + self.channel = grpc.intercept_channel(self.channel, *interceptors) atexit.register(self.closeChannel) def __getReportStub(self): From db938da9ce99108c60af251ee5d1014bc9dd648b Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Wed, 20 Oct 2021 11:59:59 -0700 Subject: [PATCH 136/277] [cuebot] Fix timeDiff for exception message. (#1049) --- .../com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index cb3aede1f..f632d0f73 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -577,8 +577,8 @@ public CommentSeq getComments(JobInterface j) { public UpdatedFrameCheckResult getUpdatedFrames(JobInterface job, List layers, int epochSeconds) { - if ((System.currentTimeMillis() / 1000) - epochSeconds > 60) { - long timeDiff = System.currentTimeMillis() - epochSeconds; + long timeDiff = (System.currentTimeMillis() / 1000) - epochSeconds; + if (timeDiff > 60) { throw new IllegalArgumentException("the last update timestamp cannot be over " + "a minute off the current time, difference was: " + timeDiff); } From 553f7aeabaa2c7ff415ef0e8210a1e760b933291 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Wed, 20 Oct 2021 12:01:16 -0700 Subject: [PATCH 137/277] [pycue] Improve GPU API methods. (#1050) --- cuegui/cuegui/ServiceDialog.py | 2 +- pycue/opencue/wrappers/service.py | 48 +++++++++++++++++++++++----- pycue/opencue/wrappers/show.py | 4 +-- pycue/tests/wrappers/service_test.py | 24 ++++++++++++++ pycue/tests/wrappers/show_test.py | 26 +++++++++++++++ 5 files changed, 93 insertions(+), 11 deletions(-) diff --git a/cuegui/cuegui/ServiceDialog.py b/cuegui/cuegui/ServiceDialog.py index 5f78cf670..bdeec4dd6 100644 --- a/cuegui/cuegui/ServiceDialog.py +++ b/cuegui/cuegui/ServiceDialog.py @@ -168,7 +168,7 @@ def save(self): service.setMinCores(self.min_cores.value()) service.setMaxCores(self.max_cores.value()) service.setMinMemory(self.min_memory.value() * 1024) - service.setMinGpu(self.min_gpu_memory.value() * 1024) + service.setMinGpuMemory(self.min_gpu_memory.value() * 1024) service.setTimeout(self.timeout.value()) service.setTimeoutLLU(self.timeout_llu.value()) service.setTags(self._tags_w.get_tags()) diff --git a/pycue/opencue/wrappers/service.py b/pycue/opencue/wrappers/service.py index 1a941772b..3878d9b3d 100644 --- a/pycue/opencue/wrappers/service.py +++ b/pycue/opencue/wrappers/service.py @@ -161,21 +161,53 @@ def setMinMemory(self, minMemory): """ self.data.min_memory = minMemory - def minGpu(self): - """Returns the minimum gpu of the service. + def minGpus(self): + """Returns the minimum gpus of the service. :rtype: int - :return: min gpu + :return: min gpus """ - return self.data.min_gpu + return self.data.min_gpus - def setMinGpu(self, minGpu): - """Sets the minimum gpu of the service. + def setMinGpus(self, minGpus): + """Sets the minimum gpus of the service. :type: int - :param: new min gpu + :param: new min gpus """ - self.data.min_gpu = minGpu + self.data.min_gpus = minGpus + + def maxGpus(self): + """Returns the maximum gpus of the service. + + :rtype: int + :return: max gpus + """ + return self.data.max_gpus + + def setMaxGpus(self, maxGpus): + """Sets the maximum gpus of the service. + + :type: int + :param: new max gpus + """ + self.data.max_gpus = maxGpus + + def minGpuMemory(self): + """Returns the minimum gpu memory of the service. + + :rtype: int + :return: min gpu memory + """ + return self.data.min_gpu_memory + + def setMinGpuMemory(self, minGpuMemory): + """Sets the minimum gpu memory of the service. + + :type: int + :param: new min gpu memory + """ + self.data.min_gpu_memory = minGpuMemory def tags(self): """Returns the list of tags for the service. diff --git a/pycue/opencue/wrappers/show.py b/pycue/opencue/wrappers/show.py index 9fff4b47d..90094e757 100644 --- a/pycue/opencue/wrappers/show.py +++ b/pycue/opencue/wrappers/show.py @@ -176,7 +176,7 @@ def setDefaultMaxGpus(self, maxgpus): :return: response is empty """ response = self.stub.SetDefaultMaxGpus(show_pb2.ShowSetDefaultMaxGpusRequest( - show=self.data, max_gpu=maxgpus), + show=self.data, max_gpus=maxgpus), timeout=Cuebot.Timeout) return response @@ -189,7 +189,7 @@ def setDefaultMinGpus(self, mingpus): :return: response is empty """ response = self.stub.SetDefaultMinGpus(show_pb2.ShowSetDefaultMinGpusRequest( - show=self.data, min_gpu=mingpus), + show=self.data, min_gpus=mingpus), timeout=Cuebot.Timeout) return response diff --git a/pycue/tests/wrappers/service_test.py b/pycue/tests/wrappers/service_test.py index 92b2e3b2c..e46b29bdf 100644 --- a/pycue/tests/wrappers/service_test.py +++ b/pycue/tests/wrappers/service_test.py @@ -28,6 +28,9 @@ TEST_SERVICE_NAME = 'testService' +TEST_MIN_GPUS = 2 +TEST_MAX_GPUS = 7 +TEST_MIN_GPU_MEMORY = 8 * 1024 * 1024 * 1024 @mock.patch('opencue.cuebot.Cuebot.getStub') @@ -99,6 +102,27 @@ def testUpdate(self, getStubMock): stubMock.Update.assert_called_with( service_pb2.ServiceUpdateRequest(service=wrapper.data), timeout=mock.ANY) + def testGpus(self, getStubMock): + stubMock = mock.Mock() + stubMock.GetService.return_value = service_pb2.ServiceGetServiceResponse( + service=service_pb2.Service(name=TEST_SERVICE_NAME)) + getStubMock.return_value = stubMock + + wrapper = opencue.wrappers.service.Service() + service = wrapper.getService(name=TEST_SERVICE_NAME) + self.assertEqual(service.minGpus(), 0) + self.assertEqual(service.maxGpus(), 0) + self.assertEqual(service.minGpuMemory(), 0) + service.setMinGpus(TEST_MIN_GPUS) + service.setMaxGpus(TEST_MAX_GPUS) + service.setMinGpuMemory(TEST_MIN_GPU_MEMORY) + self.assertEqual(service.minGpus(), TEST_MIN_GPUS) + self.assertEqual(service.maxGpus(), TEST_MAX_GPUS) + self.assertEqual(service.minGpuMemory(), TEST_MIN_GPU_MEMORY) + + stubMock.GetService.assert_called_with( + service_pb2.ServiceGetServiceRequest(name=TEST_SERVICE_NAME), timeout=mock.ANY) + self.assertEqual(service.name(), TEST_SERVICE_NAME) if __name__ == '__main__': unittest.main() diff --git a/pycue/tests/wrappers/show_test.py b/pycue/tests/wrappers/show_test.py index b7cc3dd9a..b8fb26285 100644 --- a/pycue/tests/wrappers/show_test.py +++ b/pycue/tests/wrappers/show_test.py @@ -44,6 +44,8 @@ TEST_SUBSCRIPTION_BURST = 1200 TEST_MIN_CORES = 42 TEST_MAX_CORES = 47 +TEST_MIN_GPUS = 2 +TEST_MAX_GPUS = 7 TEST_ENABLE_VALUE = False TEST_GROUP_NAME = 'group' TEST_GROUP_DEPT = 'lighting' @@ -197,6 +199,30 @@ def testSetDefaultMinCores(self, getStubMock): show_pb2.ShowSetDefaultMinCoresRequest(show=show.data, min_cores=TEST_MIN_CORES), timeout=mock.ANY) + def testSetDefaultMaxGpus(self, getStubMock): + stubMock = mock.Mock() + stubMock.SetDefaultMaxGpus.return_value = show_pb2.ShowSetDefaultMaxGpusResponse() + getStubMock.return_value = stubMock + + show = opencue.wrappers.show.Show(show_pb2.Show(name=TEST_SHOW_NAME)) + show.setDefaultMaxGpus(TEST_MAX_GPUS) + + stubMock.SetDefaultMaxGpus.assert_called_with( + show_pb2.ShowSetDefaultMaxGpusRequest(show=show.data, max_gpus=TEST_MAX_GPUS), + timeout=mock.ANY) + + def testSetDefaultMinGpus(self, getStubMock): + stubMock = mock.Mock() + stubMock.SetDefaultMinGpus.return_value = show_pb2.ShowSetDefaultMinGpusResponse() + getStubMock.return_value = stubMock + + show = opencue.wrappers.show.Show(show_pb2.Show(name=TEST_SHOW_NAME)) + show.setDefaultMinGpus(TEST_MIN_GPUS) + + stubMock.SetDefaultMinGpus.assert_called_with( + show_pb2.ShowSetDefaultMinGpusRequest(show=show.data, min_gpus=TEST_MIN_GPUS), + timeout=mock.ANY) + def testFindFilter(self, getStubMock): stubMock = mock.Mock() stubMock.FindFilter.return_value = show_pb2.ShowFindFilterResponse( From 9f82ae5f3d7d5b431dac314083bff7ce1b76e8ac Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Wed, 20 Oct 2021 12:01:58 -0700 Subject: [PATCH 138/277] [cuebot] Fix typos in GPU queries. (#1051) --- .../spcue/dao/postgres/GroupDaoJdbc.java | 4 +- .../test/dao/postgres/GroupDaoTests.java | 65 +++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java index b502bb680..5c9947451 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java @@ -254,7 +254,7 @@ public void updateDefaultJobMinGpus(GroupInterface group, int value) { throw new IllegalArgumentException(msg); } getJdbcTemplate().update( - "UPDATE folder SET int_job_min_gpu=? WHERE pk_folder=?", + "UPDATE folder SET int_job_min_gpus=? WHERE pk_folder=?", value, group.getId()); } @@ -268,7 +268,7 @@ public void updateMaxGpus(GroupInterface group, int value) { } getJdbcTemplate().update( - "UPDATE folder_resource SET int_max_gpu=? WHERE pk_folder=?", + "UPDATE folder_resource SET int_max_gpus=? WHERE pk_folder=?", value, group.getId()); } diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/GroupDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/GroupDaoTests.java index aa528be95..8700204d9 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/GroupDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/GroupDaoTests.java @@ -230,6 +230,39 @@ public void testUpdateDefaultJobMinCores() { Integer.class, group.getGroupId())); } + @Test + @Transactional + @Rollback(true) + public void testUpdateDefaultJobMaxGpus() { + GroupDetail group = createGroup(); + assertEquals(Integer.valueOf(-1), jdbcTemplate.queryForObject( + "SELECT int_job_max_gpus FROM folder WHERE pk_folder=?", + Integer.class, group.getGroupId())); + groupDao.updateDefaultJobMaxGpus(group, 100); + assertEquals(Integer.valueOf(100), jdbcTemplate.queryForObject( + "SELECT int_job_max_gpus FROM folder WHERE pk_folder=?", + Integer.class, group.getGroupId())); + groupDao.updateDefaultJobMaxGpus(group, -1); + assertEquals(Integer.valueOf(-1), jdbcTemplate.queryForObject( + "SELECT int_job_max_gpus FROM folder WHERE pk_folder=?", + Integer.class, group.getGroupId())); + + } + + @Test + @Transactional + @Rollback(true) + public void testUpdateDefaultJobMinGpus() { + GroupDetail group = createGroup(); + assertEquals(Integer.valueOf(-1), jdbcTemplate.queryForObject( + "SELECT int_job_min_gpus FROM folder WHERE pk_folder=?", + Integer.class, group.getGroupId())); + groupDao.updateDefaultJobMinGpus(group, 100); + assertEquals(Integer.valueOf(100), jdbcTemplate.queryForObject( + "SELECT int_job_min_gpus FROM folder WHERE pk_folder=?", + Integer.class, group.getGroupId())); + } + @Test @Transactional @Rollback(true) @@ -276,6 +309,38 @@ public void testUpdateMaxCores() { Integer.class, group.getGroupId())); } + @Test + @Transactional + @Rollback(true) + public void testUpdateMinGpus() { + GroupDetail group = createGroup(); + assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( + "SELECT int_min_gpus FROM folder_resource WHERE pk_folder=?", + Integer.class, group.getGroupId())); + groupDao.updateMinGpus(group, 10); + assertEquals(Integer.valueOf(10), jdbcTemplate.queryForObject( + "SELECT int_min_gpus FROM folder_resource WHERE pk_folder=?", + Integer.class, group.getGroupId())); + } + + @Test + @Transactional + @Rollback(true) + public void testUpdateMaxGpus() { + GroupDetail group = createGroup(); + assertEquals(Integer.valueOf(-1), jdbcTemplate.queryForObject( + "SELECT int_max_gpus FROM folder_resource WHERE pk_folder=?", + Integer.class, group.getGroupId())); + groupDao.updateMaxGpus(group, 100); + assertEquals(Integer.valueOf(100), jdbcTemplate.queryForObject( + "SELECT int_max_gpus FROM folder_resource WHERE pk_folder=?", + Integer.class, group.getGroupId())); + groupDao.updateMaxGpus(group, -5); + assertEquals(Integer.valueOf(-1), jdbcTemplate.queryForObject( + "SELECT int_max_gpus FROM folder_resource WHERE pk_folder=?", + Integer.class, group.getGroupId())); + } + @Test @Transactional @Rollback(true) From 66c0ee103209ed70daada5e3eda256bffe072213 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Wed, 20 Oct 2021 12:02:57 -0700 Subject: [PATCH 139/277] Add CY2022 to testing pipeline. (#1048) --- .github/workflows/testing-pipeline.yml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testing-pipeline.yml b/.github/workflows/testing-pipeline.yml index 2ca7464bd..d3583bd0e 100644 --- a/.github/workflows/testing-pipeline.yml +++ b/.github/workflows/testing-pipeline.yml @@ -70,10 +70,31 @@ jobs: chown -R aswfuser:aswfgroup . su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser + test_python_2022: + name: Run Python Unit Tests (CY2022) + runs-on: ubuntu-latest + container: aswf/ci-opencue:2022 + steps: + - uses: actions/checkout@v2 + - name: Run Python Tests + run: ci/run_python_tests.sh + + test_cuebot_2022: + name: Build Cuebot and Run Unit Tests (CY2022) + runs-on: ubuntu-latest + container: + image: aswf/ci-opencue:2022 + steps: + - uses: actions/checkout@v2 + - name: Build with Gradle + run: | + chown -R aswfuser:aswfgroup . + su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser + lint_python: name: Lint Python Code runs-on: ubuntu-latest - container: aswf/ci-opencue:2021 + container: aswf/ci-opencue:2022 steps: - uses: actions/checkout@v2 - name: Lint Python Code From 393a774e8f7addc01b0073829141fbfe24824817 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Wed, 20 Oct 2021 12:04:08 -0700 Subject: [PATCH 140/277] [cuebot] Fix GPU calculation in FrameCompleteHandler. (#1053) --- .../com/imageworks/spcue/dao/GroupDao.java | 8 -------- .../java/com/imageworks/spcue/dao/JobDao.java | 8 -------- .../spcue/dao/postgres/GroupDaoJdbc.java | 19 ------------------- .../spcue/dao/postgres/JobDaoJdbc.java | 16 ---------------- .../dispatcher/FrameCompleteHandler.java | 2 +- .../imageworks/spcue/service/JobManager.java | 8 -------- .../spcue/service/JobManagerService.java | 6 ------ 7 files changed, 1 insertion(+), 66 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/GroupDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/GroupDao.java index dfb49dd9c..87cd950d0 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/GroupDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/GroupDao.java @@ -220,14 +220,6 @@ public interface GroupDao { */ boolean isOverMinCores(JobInterface job); - /** - * Returns true if the group of the specified job is at or over its min gpus - * - * @param job - * @return - */ - boolean isOverMinGpus(JobInterface job); - /** * Returns true if the group is managed. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/JobDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/JobDao.java index 3882f95a7..11df75ffe 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/JobDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/JobDao.java @@ -273,14 +273,6 @@ public interface JobDao { */ boolean isOverMaxCores(JobInterface job, int coreUnits); - /** - * reteurns true if job is over its minimum gpus - * - * @param job - * @return boolean - */ - boolean isOverMinGpus(JobInterface job); - /** * returns true if job is over max gpus * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java index 5c9947451..b18a67c93 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java @@ -280,25 +280,6 @@ public void updateMinGpus(GroupInterface group, int value) { value, group.getId()); } - private static final String IS_OVER_MIN_GPUS = - "SELECT " + - "COUNT(1) " + - "FROM " + - "job,"+ - "folder_resource fr "+ - "WHERE " + - "job.pk_folder = fr.pk_folder " + - "AND " + - "fr.int_gpus > fr.int_min_gpus " + - "AND "+ - "job.pk_job = ?"; - - @Override - public boolean isOverMinGpus(JobInterface job) { - return getJdbcTemplate().queryForObject(IS_OVER_MIN_GPUS, - Integer.class, job.getJobId()) > 0; - } - @Override public void updateDefaultJobPriority(GroupInterface group, int value) { if (value < 0) { value = CueUtil.FEATURE_DISABLED; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java index 8009b247c..890f5ea81 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java @@ -662,22 +662,6 @@ public boolean isAtMaxCores(JobInterface job) { Integer.class, job.getJobId()) > 0; } - private static final String IS_JOB_OVER_MIN_GPUS = - "SELECT " + - "COUNT(1) " + - "FROM " + - "job_resource " + - "WHERE " + - "job_resource.pk_job = ? " + - "AND " + - "job_resource.int_gpus > job_resource.int_min_gpus"; - - @Override - public boolean isOverMinGpus(JobInterface job) { - return getJdbcTemplate().queryForObject(IS_JOB_OVER_MIN_GPUS, - Integer.class, job.getJobId()) > 0; - } - private static final String IS_JOB_OVER_MAX_GPUS = "SELECT " + "COUNT(1) " + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java index 20afd374c..fe2482720 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java @@ -401,7 +401,7 @@ else if (report.getHost().getNimbyLocked()) { // Then check for higher priority jobs // If not, rebook this job if (job.autoUnbook && proc.coresReserved >= 100) { - if (jobManager.isOverMinCores(job) && jobManager.isOverMinGpus(job)) { + if (jobManager.isOverMinCores(job)) { try { boolean unbook = diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobManager.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobManager.java index 6ab4bb38e..dfb4873b9 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobManager.java @@ -292,14 +292,6 @@ public interface JobManager { */ boolean isOverMinCores(JobInterface job); - /** - * Return true if the given job is booked greater than min gpus. - * - * @param job - * @return - */ - boolean isOverMinGpus(JobInterface job); - /** * Increase the layer memory requirement to given KB value. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java index a4f6f1ebb..8477916b8 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java @@ -135,12 +135,6 @@ public boolean isOverMinCores(JobInterface job) { return jobDao.isOverMinCores(job); } - @Override - @Transactional(propagation = Propagation.REQUIRED, readOnly=true) - public boolean isOverMinGpus(JobInterface job) { - return jobDao.isOverMinGpus(job); - } - @Transactional(propagation = Propagation.REQUIRED, readOnly=true) public DispatchJob getDispatchJob(String id) { return jobDao.getDispatchJob(id); From f0cc28e687738bd9ef363d42950ad511d2f6d209 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Wed, 20 Oct 2021 12:05:01 -0700 Subject: [PATCH 141/277] [cuegui] Fix GPU field access in CueJobMonitorTree. (#1054) --- cuegui/cuegui/CueJobMonitorTree.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cuegui/cuegui/CueJobMonitorTree.py b/cuegui/cuegui/CueJobMonitorTree.py index 97d22cb39..f61a416a1 100644 --- a/cuegui/cuegui/CueJobMonitorTree.py +++ b/cuegui/cuegui/CueJobMonitorTree.py @@ -157,8 +157,9 @@ def __init__(self, parent): sort=lambda job: job.data.job_stats.max_rss, tip="The most memory used at one time by any single frame.") self.addColumn("MaxGpuMem", 60, id=19, - data=lambda job: cuegui.Utils.memoryToString(job.data.job_stats.max_gpu_mem), - sort=lambda job: job.data.job_stats.max_gpu_mem, + data=lambda job: cuegui.Utils.memoryToString( + job.data.job_stats.max_gpu_memory), + sort=lambda job: job.data.job_stats.max_gpu_memory, tip="The most gpu memory used at one time by any single frame.") self.addColumn("_Blank", 20, id=20, tip="Spacer") From 2276cf2896bfdee1f24d99554934dfa170d6ce5a Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Wed, 20 Oct 2021 12:06:44 -0700 Subject: [PATCH 142/277] [cuegui] Fix GPU editing and add scroll area to Layer Properties dialog. (#1055) --- cuegui/cuegui/LayerDialog.py | 19 +++++++++++--- cuegui/cuegui/MenuActions.py | 50 ++++++++++++++++++++++++------------ 2 files changed, 49 insertions(+), 20 deletions(-) diff --git a/cuegui/cuegui/LayerDialog.py b/cuegui/cuegui/LayerDialog.py index eb49b520c..3efd36de3 100644 --- a/cuegui/cuegui/LayerDialog.py +++ b/cuegui/cuegui/LayerDialog.py @@ -229,7 +229,14 @@ def __init__(self, layers, parent=None): self.__timeout.setValue(self.getTimeout()) self.__timeout_llu.setValue(self.getTimeoutLLU()) + topLayout = QtWidgets.QVBoxLayout() + topWidget = QtWidgets.QWidget() + topWidget.setLayout(topLayout) + scrollArea = QtWidgets.QScrollArea(widgetResizable=True) + scrollArea.setWidget(topWidget) + QtWidgets.QVBoxLayout(self) + self.layout().addWidget(scrollArea) layout = QtWidgets.QVBoxLayout() layout.addWidget(EnableableItem(LayerPropertiesItem("Minimum Memory:", @@ -275,10 +282,10 @@ def __init__(self, layers, parent=None): layout.addStretch() self.__group.setLayout(layout) - self.layout().addWidget(EnableableItem(self.__tags, multiSelect)) - self.layout().addWidget(EnableableItem(self.__limits, multiSelect)) - self.layout().addWidget(self.__group) - self.layout().addWidget(self.__buttons) + topLayout.addWidget(EnableableItem(self.__tags, multiSelect)) + topLayout.addWidget(EnableableItem(self.__limits, multiSelect)) + topLayout.addWidget(self.__group) + topLayout.addWidget(self.__buttons) def _cfg(self): """ @@ -326,6 +333,10 @@ def apply(self): layer.setMaxCores(self.__max_cores.value() * 100.0) if self.__thread.isEnabled(): layer.setThreadable(self.__thread.isChecked()) + if self.__min_gpus.isEnabled(): + layer.setMinGpus(self.__min_gpus.value()) + if self.__max_gpus.isEnabled(): + layer.setMaxGpus(self.__max_cores.value()) if self.__gpu_mem.isEnabled(): layer.setMinGpuMemory(self.__gpu_mem.slider.value() * self.gpu_mem_tick_kb) if self.__timeout.isEnabled(): diff --git a/cuegui/cuegui/MenuActions.py b/cuegui/cuegui/MenuActions.py index fc615283f..b756d3f8e 100644 --- a/cuegui/cuegui/MenuActions.py +++ b/cuegui/cuegui/MenuActions.py @@ -269,36 +269,36 @@ def setMaxCores(self, rpcObjects=None): job.setMaxCores(float(value)) self._update() - setMinGpu_info = ["Set Minimum Gpu...", "Set Job(s) Minimum Gpu", "configure"] - def setMinGpu(self, rpcObjects=None): + setMinGpus_info = ["Set Minimum Gpus...", "Set Job(s) Minimum Gpus", "configure"] + def setMinGpus(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) if jobs: - current = max([job.data.min_cores for job in jobs]) - title = "Set Minimum Gpu" - body = "Please enter the new minimum gpu value:" - (value, choice) = QtWidgets.QInputDialog.getDouble(self._caller, + current = max([job.data.min_gpus for job in jobs]) + title = "Set Minimum Gpus" + body = "Please enter the new minimum gpus value:" + (value, choice) = QtWidgets.QInputDialog.getInt(self._caller, title, body, current, - 0, 50000, 0) + 0, 500, 0) if choice: for job in jobs: - job.setMinGpu(float(value)) + job.setMinGpus(int(value)) self._update() - setMaxGpu_info = ["Set Maximum Gpu...", "Set Job(s) Maximum Gpu", "configure"] - def setMaxGpu(self, rpcObjects=None): + setMaxGpus_info = ["Set Maximum Gpus...", "Set Job(s) Maximum Gpus", "configure"] + def setMaxGpus(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) if jobs: - current = max([job.data.max_cores for job in jobs]) - title = "Set Maximum Gpu" - body = "Please enter the new maximum gpu value:" - (value, choice) = QtWidgets.QInputDialog.getDouble(self._caller, + current = max([job.data.max_gpus for job in jobs]) + title = "Set Maximum Gpus" + body = "Please enter the new maximum gpus value:" + (value, choice) = QtWidgets.QInputDialog.getInt(self._caller, title, body, current, - 0, 50000, 0) + 0, 500, 0) if choice: for job in jobs: - job.setMaxGpu(float(value)) + job.setMaxGpus(int(value)) self._update() setPriority_info = ["Set Priority...", None, "configure"] @@ -645,6 +645,24 @@ def setMinMemoryKb(self, rpcObjects=None): layer.setMinMemory(int(value * 1048576)) self._update() + setMinGpuMemoryKb_info = [ + "Set Minimum Gpu Memory", + "Set the amount of Gpu memory required for this layer", "configure"] + + def setMinGpuMemoryKb(self, rpcObjects=None): + layers = self._getOnlyLayerObjects(rpcObjects) + if layers: + current = max([layer.data.min_gpu_memory / 1048576 for layer in layers]) + title = "Set minimum amount of Gpu memory required" + body = ('Please enter the new minimum amount of Gpu memory in GB that frames ' + 'in the selected layer(s) should require:') + (value, choice) = QtWidgets.QInputDialog.getDouble( + self._caller, title, body, current, 0.01, 64.0, 1) + if choice: + for layer in layers: + layer.setMinGpuMemory(int(value * 1048576)) + self._update() + useLocalCores_info = [ "Use local cores...", "Set a single layer to use the local desktop cores.", "configure"] From 4b156af647e21eb5d9aa4c8f6938348cfbba54e5 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Wed, 20 Oct 2021 12:07:55 -0700 Subject: [PATCH 143/277] [cuebot] Avoid NullPointerException in SortableShow. (#1056) --- cuebot/src/main/java/com/imageworks/spcue/SortableShow.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/SortableShow.java b/cuebot/src/main/java/com/imageworks/spcue/SortableShow.java index 855b95857..83fe7da5d 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/SortableShow.java +++ b/cuebot/src/main/java/com/imageworks/spcue/SortableShow.java @@ -77,7 +77,9 @@ public boolean isSkipped(AllocationInterface a) { } public void skip(String tags, long cores, long memory) { - failed.put(tags, new long[] { cores, memory}); + if (tags != null) { + failed.put(tags, new long[] { cores, memory}); + } } /** From fc0b509c4ec9773e2e8b3c0fdaacfa42a6d17400 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Fri, 22 Oct 2021 10:30:57 -0700 Subject: [PATCH 144/277] [rqd] Replace isAlive with is_alive. (#1052) --- rqd/rqd/rqcore.py | 6 +++--- rqd/rqd/rqnetwork.py | 4 ++-- rqd/tests/rqcore_tests.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index 7cb08030b..beeca70ae 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -303,7 +303,7 @@ def runLinux(self): frameInfo.pid = frameInfo.forkedCommand.pid - if not self.rqCore.updateRssThread.isAlive(): + if not self.rqCore.updateRssThread.is_alive(): self.rqCore.updateRssThread = threading.Timer(rqd.rqconstants.RSS_UPDATE_INTERVAL, self.rqCore.updateRss) self.rqCore.updateRssThread.start() @@ -356,7 +356,7 @@ def runWindows(self): frameInfo.pid = frameInfo.forkedCommand.pid - if not self.rqCore.updateRssThread.isAlive(): + if not self.rqCore.updateRssThread.is_alive(): self.rqCore.updateRssThread = threading.Timer(rqd.rqconstants.RSS_UPDATE_INTERVAL, self.rqCore.updateRss) self.rqCore.updateRssThread.start() @@ -404,7 +404,7 @@ def runDarwin(self): frameInfo.pid = frameInfo.forkedCommand.pid - if not self.rqCore.updateRssThread.isAlive(): + if not self.rqCore.updateRssThread.is_alive(): self.rqCore.updateRssThread = threading.Timer(rqd.rqconstants.RSS_UPDATE_INTERVAL, self.rqCore.updateRss) self.rqCore.updateRssThread.start() diff --git a/rqd/rqd/rqnetwork.py b/rqd/rqd/rqnetwork.py index 127d97100..28018d5f5 100644 --- a/rqd/rqd/rqnetwork.py +++ b/rqd/rqd/rqnetwork.py @@ -105,9 +105,9 @@ def kill(self, message=""): if self.frameAttendantThread is None: log.warning( "Kill requested before frameAttendantThread is created for: %s", self.frameId) - elif self.frameAttendantThread.isAlive() and self.pid is None: + elif self.frameAttendantThread.is_alive() and self.pid is None: log.warning("Kill requested before pid is available for: %s", self.frameId) - elif self.frameAttendantThread.isAlive(): + elif self.frameAttendantThread.is_alive(): # pylint: disable=broad-except try: if not self.killMessage and message: diff --git a/rqd/tests/rqcore_tests.py b/rqd/tests/rqcore_tests.py index 90ef4618b..bf357dddc 100644 --- a/rqd/tests/rqcore_tests.py +++ b/rqd/tests/rqcore_tests.py @@ -206,7 +206,7 @@ def test_deleteFrame(self): def test_killAllFrame(self): frameAttendantThread = mock.MagicMock() - frameAttendantThread.isAlive.return_value = False + frameAttendantThread.is_alive.return_value = False frame1Id = 'frame1' frame2Id = 'frame2' frame3Id = 'frame3' @@ -230,7 +230,7 @@ def test_killAllFrame(self): def test_killAllFrameIgnoreNimby(self): frameAttendantThread = mock.MagicMock() - frameAttendantThread.isAlive.return_value = False + frameAttendantThread.is_alive.return_value = False frame1Id = 'frame1' frame2Id = 'frame2' frame1 = rqd.rqnetwork.RunningFrame( From a05b90a5809dc8b797a41f116ae569735ff1b763 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sat, 4 Dec 2021 14:17:13 -0800 Subject: [PATCH 145/277] Add FindLimit to pycue API (#1034) --- pycue/opencue/api.py | 11 +++++++++++ pycue/tests/api_test.py | 15 +++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/pycue/opencue/api.py b/pycue/opencue/api.py index df101733c..602170692 100644 --- a/pycue/opencue/api.py +++ b/pycue/opencue/api.py @@ -794,3 +794,14 @@ def getLimits(): :return: a list of Limit objects""" return [Limit(limit) for limit in Cuebot.getStub('limit').GetAll( limit_pb2.LimitGetAllRequest(), timeout=Cuebot.Timeout).limits] + +@util.grpcExceptionParser +def findLimit(name): + """Returns the Limit object that matches the name. + + :type name: str + :param name: a string that represents a limit to return + :rtype: Limit + :return: the matching Limit object""" + return Limit(Cuebot.getStub('limit').Find( + limit_pb2.LimitFindRequest(name=name), timeout=Cuebot.Timeout).limit) diff --git a/pycue/tests/api_test.py b/pycue/tests/api_test.py index cbd957869..b20acfff9 100644 --- a/pycue/tests/api_test.py +++ b/pycue/tests/api_test.py @@ -832,6 +832,21 @@ def testGetLimits(self, getStubMock): self.assertEqual(len(limits), 1) self.assertEqual(limits[0].name(), TEST_LIMIT_NAME) + @mock.patch('opencue.cuebot.Cuebot.getStub') + def testFindLimit(self, getStubMock): + stubMock = mock.Mock() + stubMock.Find.return_value = limit_pb2.LimitFindResponse( + limit=limit_pb2.Limit(name=TEST_LIMIT_NAME, max_value=42)) + getStubMock.return_value = stubMock + + limit = opencue.api.findLimit(TEST_LIMIT_NAME) + self.assertEqual(TEST_LIMIT_NAME, limit.name()) + self.assertEqual(42, limit.maxValue()) + + stubMock.Find.assert_called_with( + limit_pb2.LimitFindRequest(name=TEST_LIMIT_NAME), + timeout=mock.ANY) + if __name__ == '__main__': unittest.main() From d4b790f6362fd2ade98e739091b52ee6e377f03e Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sat, 4 Dec 2021 14:47:05 -0800 Subject: [PATCH 146/277] [rqd] Fix use of hyperthreadingMultiplier. (#1013) --- rqd/rqd/rqmachine.py | 11 ++++++++--- rqd/tests/rqmachine_tests.py | 3 +++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 13ae37df8..753056485 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -309,7 +309,7 @@ def getLoadAvg(self): loadAvgFile = open(rqd.rqconstants.PATH_LOADAVG, "r") loadAvg = int(float(loadAvgFile.read().split()[0]) * 100) if self.__enabledHT(): - loadAvg = loadAvg // 2 + loadAvg = loadAvg // self.__getHyperthreadingMultiplier() loadAvg = loadAvg + rqd.rqconstants.LOAD_MODIFIER loadAvg = max(loadAvg, 0) return loadAvg @@ -549,7 +549,7 @@ def __initMachineStats(self, pathCpuInfo=None): self.__renderHost.num_procs = __numProcs self.__renderHost.cores_per_proc = __totalCores // __numProcs - if hyperthreadingMultiplier > 1: + if hyperthreadingMultiplier >= 1: self.__renderHost.attributes['hyperthreadingMultiplier'] = str(hyperthreadingMultiplier) def getWindowsMemory(self): @@ -684,6 +684,9 @@ def getBootReport(self): def __enabledHT(self): return 'hyperthreadingMultiplier' in self.__renderHost.attributes + def __getHyperthreadingMultiplier(self): + return int(self.__renderHost.attributes['hyperthreadingMultiplier']) + def setupHT(self): """ Setup rqd for hyper-threading """ @@ -719,11 +722,13 @@ def reserveHT(self, reservedCores): log.critical(err) raise rqd.rqexceptions.CoreReservationFailureException(err) + hyperthreadingMultiplier = self.__getHyperthreadingMultiplier() tasksets = [] for _ in range(reservedCores // 100): core = self.__tasksets.pop() tasksets.append(str(core)) - tasksets.append(str(core + self.__coreInfo.total_cores // 100)) + if hyperthreadingMultiplier > 1: + tasksets.append(str(core + self.__coreInfo.total_cores // 100)) log.debug('Taskset: Reserving cores - %s', ','.join(tasksets)) diff --git a/rqd/tests/rqmachine_tests.py b/rqd/tests/rqmachine_tests.py index b4bfe7bad..c2d4bad87 100644 --- a/rqd/tests/rqmachine_tests.py +++ b/rqd/tests/rqmachine_tests.py @@ -304,6 +304,9 @@ def test_getLoadAvg(self): @mock.patch.object( rqd.rqmachine.Machine, '_Machine__enabledHT', new=mock.MagicMock(return_value=True)) + @mock.patch.object( + rqd.rqmachine.Machine, '_Machine__getHyperthreadingMultiplier', + new=mock.MagicMock(return_value=2)) def test_getLoadAvgHT(self): self.loadavg.set_contents(LOADAVG_HIGH_USAGE) From 108a1d2b97c626392ed57e5334b11f72cc9d87c5 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Sat, 4 Dec 2021 14:58:46 -0800 Subject: [PATCH 147/277] [rqd] Fix /proc/PID/stat parsing to support executable names with spaces and brackets. (#1029) --- rqd/rqd/rqmachine.py | 2 +- rqd/tests/rqmachine_tests.py | 28 +++++++++++++++++++++------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 753056485..565fcd21a 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -198,7 +198,7 @@ def rssUpdate(self, frames): if pid.isdigit(): try: with open("/proc/%s/stat" % pid, "r") as statFile: - statFields = statFile.read().split() + statFields = [None, None] + statFile.read().rsplit(")", 1)[-1].split() # See "man proc" pids[pid] = { diff --git a/rqd/tests/rqmachine_tests.py b/rqd/tests/rqmachine_tests.py index c2d4bad87..b95467d1d 100644 --- a/rqd/tests/rqmachine_tests.py +++ b/rqd/tests/rqmachine_tests.py @@ -148,10 +148,13 @@ softirq 10802040 0 3958368 410 1972314 394501 0 1 3631586 0 844860 ''' -PROC_PID_STAT = ('105 (time) S 7 105 105 0 -1 4210688 317 0 1 0 31 13 0 0 20 0 1 0 17385159 ' - '4460544 154 18446744073709551615 4194304 4204692 140725890735264 0 0 0 0 ' - '16781318 0 0 0 0 17 4 0 0 0 0 0 6303248 6304296 23932928 140725890743234 ' - '140725890743420 140725890743420 140725890744298 0') +PROC_STAT_SUFFIX = (' S 7 105 105 0 -1 4210688 317 0 1 0 31 13 0 0 20 0 1 0 17385159 ' + '4460544 154 18446744073709551615 4194304 4204692 140725890735264 0 0 0 0 ' + '16781318 0 0 0 0 17 4 0 0 0 0 0 6303248 6304296 23932928 140725890743234 ' + '140725890743420 140725890743420 140725890744298 0') +PROC_PID_STAT = '105 (time)' + PROC_STAT_SUFFIX +PROC_PID_STAT_WITH_SPACES = '105 (test space)' + PROC_STAT_SUFFIX +PROC_PID_STAT_WITH_BRACKETS = '105 (test) (brackets)' + PROC_STAT_SUFFIX @mock.patch.object(rqd.rqutil.Memoize, 'isCached', new=mock.MagicMock(return_value=False)) @@ -274,12 +277,11 @@ def test_isUserLoggedInWithNoDisplayOrProcess(self, processIterMock): self.assertFalse(self.machine.isUserLoggedIn()) - @mock.patch('time.time', new=mock.MagicMock(return_value=1570057887.61)) - def test_rssUpdate(self): + def _test_rssUpdate(self, proc_stat): rqd.rqconstants.SYS_HERTZ = 100 pid = 105 frameId = 'unused-frame-id' - self.fs.create_file('/proc/%d/stat' % pid, contents=PROC_PID_STAT) + self.fs.create_file('/proc/%d/stat' % pid, contents=proc_stat) runningFrame = rqd.rqnetwork.RunningFrame(self.rqCore, rqd.compiled_proto.rqd_pb2.RunFrame()) runningFrame.pid = pid @@ -295,6 +297,18 @@ def test_rssUpdate(self): self.assertEqual(4356, updatedFrameInfo.vsize) self.assertAlmostEqual(0.034444696691, float(updatedFrameInfo.attributes['pcpu'])) + @mock.patch('time.time', new=mock.MagicMock(return_value=1570057887.61)) + def test_rssUpdate(self): + self._test_rssUpdate(PROC_PID_STAT) + + @mock.patch('time.time', new=mock.MagicMock(return_value=1570057887.61)) + def test_rssUpdateWithSpaces(self): + self._test_rssUpdate(PROC_PID_STAT_WITH_SPACES) + + @mock.patch('time.time', new=mock.MagicMock(return_value=1570057887.61)) + def test_rssUpdateWithBrackets(self): + self._test_rssUpdate(PROC_PID_STAT_WITH_BRACKETS) + @mock.patch.object( rqd.rqmachine.Machine, '_Machine__enabledHT', new=mock.MagicMock(return_value=False)) def test_getLoadAvg(self): From ae62cd9b5d74e457aa65a851fbaf078c386aac1b Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Sun, 5 Dec 2021 19:14:53 -0500 Subject: [PATCH 148/277] [docs] Upgrade Sphinx and theme to fix dependency issue. (#1073) --- docs/conf.py | 3 +-- docs/requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index d273b2c57..2c8322352 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -87,14 +87,13 @@ html_theme = 'sphinx_rtd_theme' # Logo (optional) html_logo = "images/opencue_logo_navbar.png" - +html_baseurl = 'https://www.opencue.io/' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # html_theme_options = { - 'canonical_url': 'https://www.opencue.io/', 'logo_only': True, 'display_version': True, 'prev_next_buttons_location': 'bottom', diff --git a/docs/requirements.txt b/docs/requirements.txt index 607db205d..9d324374d 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ -sphinx==3.0.3 -sphinx_rtd_theme==0.4.3 +sphinx==4.3.1 +sphinx-rtd-theme==1.0.0 From cc6edd8d64902d1249134287b473671d4cfb54cf Mon Sep 17 00:00:00 2001 From: oliviascarfone <58369679+oliviascarfone@users.noreply.github.com> Date: Wed, 8 Dec 2021 14:52:35 -0800 Subject: [PATCH 149/277] Use openjdk:11-jre-slim-buster (#1068) --- cuebot/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuebot/Dockerfile b/cuebot/Dockerfile index 56ae9acb1..2dd5ae960 100644 --- a/cuebot/Dockerfile +++ b/cuebot/Dockerfile @@ -49,7 +49,7 @@ RUN chmod +x create_rpm.sh && ./create_rpm.sh cuebot "$(cat VERSION)" # ----------------- # RUN # ----------------- -FROM openjdk:11-jre-slim +FROM openjdk:11-jre-slim-buster # First line after FROM should be unique to avoid --cache-from weirdness. RUN echo "Cuebot runtime stage" From 7e05c4ebcb6519a3546499d31c21852a83d8ca7b Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Tue, 14 Dec 2021 15:07:59 -0800 Subject: [PATCH 150/277] Add thread pool properties (#1008) --- .../spcue/dispatcher/BookingQueue.java | 35 +++++-- .../spcue/dispatcher/HostReportQueue.java | 30 ++++-- .../ThreadPoolTaskExecutorWrapper.java | 63 +++++++++++++ .../com/imageworks/spcue/util/CueUtil.java | 15 ++- .../spring/applicationContext-service.xml | 68 +++++++++----- cuebot/src/main/resources/opencue.properties | 42 +++++++++ .../test/dispatcher/TestBookingQueue.java | 10 +- .../test/dispatcher/ThreadPoolTests.java | 93 +++++++++++++++++++ cuebot/src/test/resources/opencue.properties | 24 +++++ 9 files changed, 339 insertions(+), 41 deletions(-) create mode 100644 cuebot/src/main/java/com/imageworks/spcue/dispatcher/ThreadPoolTaskExecutorWrapper.java create mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/ThreadPoolTests.java diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java index 0c7563dbf..7428f0bca 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java @@ -27,18 +27,19 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.imageworks.spcue.dispatcher.commands.DispatchBookHost; +import com.imageworks.spcue.util.CueUtil; + import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; public class BookingQueue extends ThreadPoolExecutor { private static final Logger logger = Logger.getLogger(BookingQueue.class); - private static final int INITIAL_QUEUE_SIZE = 1000; - - private static final int THREADS_MINIMUM = 6; - private static final int THREADS_MAXIMUM = 6; private static final int THREADS_KEEP_ALIVE_SECONDS = 10; + private int queueCapacity; private int baseSleepTimeMillis = 400; private AtomicBoolean isShutdown = new AtomicBoolean(false); @@ -50,11 +51,25 @@ public class BookingQueue extends ThreadPoolExecutor { .weakValues() .build(); - public BookingQueue(int sleepTimeMs) { - super(THREADS_MINIMUM, THREADS_MAXIMUM, THREADS_KEEP_ALIVE_SECONDS, - TimeUnit.SECONDS, new LinkedBlockingQueue(INITIAL_QUEUE_SIZE)); + private BookingQueue(int corePoolSize, int maxPoolSize, int queueCapacity, int sleepTimeMs) { + super(corePoolSize, maxPoolSize, THREADS_KEEP_ALIVE_SECONDS, + TimeUnit.SECONDS, new LinkedBlockingQueue(queueCapacity)); + this.queueCapacity = queueCapacity; this.baseSleepTimeMillis = sleepTimeMs; this.setRejectedExecutionHandler(rejectCounter); + logger.info("BookingQueue" + + " core:" + getCorePoolSize() + + " max:" + getMaximumPoolSize() + + " capacity:" + queueCapacity + + " sleepTimeMs:" + sleepTimeMs); + } + + @Autowired + public BookingQueue(Environment env, String propertyKeyPrefix, int sleepTimeMs) { + this(CueUtil.getIntProperty(env, propertyKeyPrefix, "core_pool_size"), + CueUtil.getIntProperty(env, propertyKeyPrefix, "max_pool_size"), + CueUtil.getIntProperty(env, propertyKeyPrefix, "queue_capacity"), + sleepTimeMs); } public void execute(DispatchBookHost r) { @@ -71,6 +86,10 @@ public long getRejectedTaskCount() { return rejectCounter.getRejectCount(); } + public int getQueueCapacity() { + return queueCapacity; + } + public void shutdown() { if (!isShutdown.getAndSet(true)) { logger.info("clearing out booking queue: " + this.getQueue().size()); @@ -87,7 +106,7 @@ public void shutdown() { public int sleepTime() { if (!isShutdown.get()) { int sleep = (int) (baseSleepTimeMillis - (((this.getQueue().size () / - (float) INITIAL_QUEUE_SIZE) * baseSleepTimeMillis)) * 2); + (float) queueCapacity) * baseSleepTimeMillis)) * 2); if (sleep < 0) { sleep = 0; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportQueue.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportQueue.java index 34e8d13e6..922172202 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportQueue.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportQueue.java @@ -25,25 +25,37 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; import com.imageworks.spcue.dispatcher.commands.DispatchHandleHostReport; +import com.imageworks.spcue.util.CueUtil; public class HostReportQueue extends ThreadPoolExecutor { private static final Logger logger = Logger.getLogger(HostReportQueue.class); - private static final int THREAD_POOL_SIZE_INITIAL = 6; - private static final int THREAD_POOL_SIZE_MAX = 8; - private static final int QUEUE_SIZE_INITIAL = 1000; - private QueueRejectCounter rejectCounter = new QueueRejectCounter(); private AtomicBoolean isShutdown = new AtomicBoolean(false); + private int queueCapacity; - public HostReportQueue() { - super(THREAD_POOL_SIZE_INITIAL, THREAD_POOL_SIZE_MAX, 10 , TimeUnit.SECONDS, - new LinkedBlockingQueue(QUEUE_SIZE_INITIAL)); + private HostReportQueue(String name, int corePoolSize, int maxPoolSize, int queueCapacity) { + super(corePoolSize, maxPoolSize, 10 , TimeUnit.SECONDS, + new LinkedBlockingQueue(queueCapacity)); + this.queueCapacity = queueCapacity; this.setRejectedExecutionHandler(rejectCounter); + logger.info(name + + " core:" + getCorePoolSize() + + " max:" + getMaximumPoolSize() + + " capacity:" + queueCapacity); + } + @Autowired + public HostReportQueue(Environment env, String name, String propertyKeyPrefix) { + this(name, + CueUtil.getIntProperty(env, propertyKeyPrefix, "core_pool_size"), + CueUtil.getIntProperty(env, propertyKeyPrefix, "max_pool_size"), + CueUtil.getIntProperty(env, propertyKeyPrefix, "queue_capacity")); } public void execute(DispatchHandleHostReport r) { @@ -60,6 +72,10 @@ public long getRejectedTaskCount() { return rejectCounter.getRejectCount(); } + public int getQueueCapacity() { + return queueCapacity; + } + public void shutdown() { if (!isShutdown.getAndSet(true)) { logger.info("Shutting down report pool, currently " + this.getActiveCount() + " active threads."); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/ThreadPoolTaskExecutorWrapper.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/ThreadPoolTaskExecutorWrapper.java new file mode 100644 index 000000000..8b749cdb1 --- /dev/null +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/ThreadPoolTaskExecutorWrapper.java @@ -0,0 +1,63 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.imageworks.spcue.dispatcher; + +import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import com.imageworks.spcue.util.CueUtil; + +/** + * Wrapper class of spring ThreadPoolTaskExecutor to initialize with the thread pool properties. + */ +public class ThreadPoolTaskExecutorWrapper extends ThreadPoolTaskExecutor { + + private static final Logger logger = Logger.getLogger(ThreadPoolTaskExecutorWrapper.class); + private static final long serialVersionUID = -2977068663355369141L; + + private int queueCapacity; + + private ThreadPoolTaskExecutorWrapper(String name, int corePoolSize, int maxPoolSize, + int queueCapacity) { + super(); + this.setMaxPoolSize(maxPoolSize); + this.setCorePoolSize(corePoolSize); + this.setQueueCapacity(queueCapacity); + this.queueCapacity = queueCapacity; + logger.info(name + + " core:" + getCorePoolSize() + + " max:" + getMaxPoolSize() + + " capacity:" + queueCapacity); + } + + @Autowired + public ThreadPoolTaskExecutorWrapper(Environment env, String name, String propertyKeyPrefix) { + this(name, + CueUtil.getIntProperty(env, propertyKeyPrefix, "core_pool_size"), + CueUtil.getIntProperty(env, propertyKeyPrefix, "max_pool_size"), + CueUtil.getIntProperty(env, propertyKeyPrefix, "queue_capacity")); + } + + public int getQueueCapacity() { + return queueCapacity; + } +} diff --git a/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java b/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java index f8b028cfa..990511f47 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java +++ b/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java @@ -46,6 +46,7 @@ import javax.mail.util.ByteArrayDataSource; import org.apache.log4j.Logger; +import org.springframework.core.env.Environment; import com.imageworks.spcue.LayerInterface; import com.imageworks.spcue.SpcueRuntimeException; @@ -351,5 +352,17 @@ public static List normalizeFrameRange(FrameSet frameSet, int chunkSize return Collections.unmodifiableList( new ArrayList(result)); } -} + /** + * Get "{prefix}.{key}" property int value + * + * @param env + * @param prefix Example "dispatcher.report_queue" + * @param key Example "core_pool_size" + */ + public static int getIntProperty(Environment env, String prefix, String key) + throws IllegalStateException { + Integer value = env.getRequiredProperty(prefix + "." + key, Integer.class); + return value.intValue(); + } +} diff --git a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml index 5f1738d69..521c74dd1 100644 --- a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml +++ b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml @@ -54,28 +54,34 @@ - - - - - - - - - - + + + + LaunchQueue + + + dispatcher.launch_queue + - - - - + + + + DispatchPool + + + dispatcher.dispatch_pool + - - - - + + + + ManagePool + + + dispatcher.manage_pool + @@ -92,11 +98,31 @@ - - + + + + ReportQueue + + + dispatcher.report_queue + + + + + + KillQueue + + + dispatcher.kill_queue + + - + + + dispatcher.booking_queue + + 300 diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index 4df5d601f..714d744a0 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -44,6 +44,48 @@ dispatcher.job_frame_dispatch_max=8 # Maximum number of frames to dispatch from a host at one time. dispatcher.host_frame_dispatch_max=12 +# Number of threads to keep in the pool for launching job. +dispatcher.launch_queue.core_pool_size=1 +# Maximum number of threads to allow in the pool for launching job. +dispatcher.launch_queue.max_pool_size=1 +# Queue capacity for launching job. +dispatcher.launch_queue.queue_capacity=100 + +# Number of threads to keep in the pool for various operation. +dispatcher.dispatch_pool.core_pool_size=4 +# Maximum number of threads to allow in the pool for various operation. +dispatcher.dispatch_pool.max_pool_size=4 +# Queue capacity for various operation. +dispatcher.dispatch_pool.queue_capacity=500 + +# Number of threads to keep in the pool for management operation. +dispatcher.manage_pool.core_pool_size=8 +# Maximum number of threads to allow in the pool for management operation. +dispatcher.manage_pool.max_pool_size=8 +# Queue capacity for management operation. +dispatcher.manage_pool.queue_capacity=250 + +# Number of threads to keep in the pool for handling Host Report. +dispatcher.report_queue.core_pool_size=6 +# Maximum number of threads to allow in the pool for handling Host Report. +dispatcher.report_queue.max_pool_size=8 +# Queue capacity for handling Host Report. +dispatcher.report_queue.queue_capacity=1000 + +# Number of threads to keep in the pool for kill frame operation. +dispatcher.kill_queue.core_pool_size=6 +# Maximum number of threads to allow in the pool for kill frame operation. +dispatcher.kill_queue.max_pool_size=8 +# Queue capacity for kill frame operation. +dispatcher.kill_queue.queue_capacity=1000 + +# Number of threads to keep in the pool for booking. +dispatcher.booking_queue.core_pool_size=6 +# Maximum number of threads to allow in the pool for booking. +dispatcher.booking_queue.max_pool_size=6 +# Queue capacity for booking. +dispatcher.booking_queue.queue_capacity=1000 + # Jobs will be archived to the history tables after being completed for this long. history.archive_jobs_cutoff_hours=72 diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java index 7502e0687..319c4cfe1 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java @@ -54,6 +54,9 @@ public class TestBookingQueue extends AbstractTransactionalJUnit4SpringContextTe @Resource HostManager hostManager; + @Resource + BookingQueue bookingQueue; + private static final String HOSTNAME = "beta"; @Before @@ -90,11 +93,10 @@ public void testBookingQueue() { host1.idleCores = 500; DispatchHost host2 = hostDao.findDispatchHost(HOSTNAME); DispatchHost host3 = hostDao.findDispatchHost(HOSTNAME); - BookingQueue queue = new BookingQueue(1000); - queue.execute(new DispatchBookHost(host2,dispatcher)); - queue.execute(new DispatchBookHost(host3,dispatcher)); - queue.execute(new DispatchBookHost(host1,dispatcher)); + bookingQueue.execute(new DispatchBookHost(host2,dispatcher)); + bookingQueue.execute(new DispatchBookHost(host3,dispatcher)); + bookingQueue.execute(new DispatchBookHost(host1,dispatcher)); try { Thread.sleep(10000); } catch (InterruptedException e) { diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/ThreadPoolTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/ThreadPoolTests.java new file mode 100644 index 000000000..76c9a3e0e --- /dev/null +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/ThreadPoolTests.java @@ -0,0 +1,93 @@ +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.imageworks.spcue.test.dispatcher; + +import java.util.concurrent.ThreadPoolExecutor; +import javax.annotation.Resource; + +import com.imageworks.spcue.config.TestAppConfig; +import com.imageworks.spcue.dispatcher.BookingQueue; +import com.imageworks.spcue.dispatcher.HostReportQueue; +import com.imageworks.spcue.dispatcher.ThreadPoolTaskExecutorWrapper; + +import org.junit.Test; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; +import org.springframework.test.context.support.AnnotationConfigContextLoader; + +import static org.junit.Assert.assertEquals; + +@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) +public class ThreadPoolTests extends AbstractTransactionalJUnit4SpringContextTests { + + @Resource + ThreadPoolTaskExecutor launchQueue; + + @Resource + ThreadPoolTaskExecutor dispatchPool; + + @Resource + ThreadPoolTaskExecutor managePool; + + @Resource + ThreadPoolExecutor reportQueue; + + @Resource + ThreadPoolExecutor killQueue; + + @Resource + BookingQueue bookingQueue; + + private int getQueueCapacity(ThreadPoolTaskExecutor t) { + return ((ThreadPoolTaskExecutorWrapper) t).getQueueCapacity(); + } + + private int getQueueCapacity(ThreadPoolExecutor t) { + return ((HostReportQueue) t).getQueueCapacity(); + } + + @Test + public void testPropertyValues() { + assertEquals(1, launchQueue.getCorePoolSize()); + assertEquals(1, launchQueue.getMaxPoolSize()); + assertEquals(100, getQueueCapacity(launchQueue)); + + assertEquals(4, dispatchPool.getCorePoolSize()); + assertEquals(4, dispatchPool.getMaxPoolSize()); + assertEquals(500, getQueueCapacity(dispatchPool)); + + assertEquals(8, managePool.getCorePoolSize()); + assertEquals(8, managePool.getMaxPoolSize()); + assertEquals(250, getQueueCapacity(managePool)); + + assertEquals(6, reportQueue.getCorePoolSize()); + assertEquals(8, reportQueue.getMaximumPoolSize()); + assertEquals(1000, getQueueCapacity(reportQueue)); + + assertEquals(6, killQueue.getCorePoolSize()); + assertEquals(8, killQueue.getMaximumPoolSize()); + assertEquals(1000, getQueueCapacity(killQueue)); + + assertEquals(6, bookingQueue.getCorePoolSize()); + assertEquals(6, bookingQueue.getMaximumPoolSize()); + assertEquals(1000, bookingQueue.getQueueCapacity()); + } +} + diff --git a/cuebot/src/test/resources/opencue.properties b/cuebot/src/test/resources/opencue.properties index 5ac924592..d492f0a87 100644 --- a/cuebot/src/test/resources/opencue.properties +++ b/cuebot/src/test/resources/opencue.properties @@ -21,3 +21,27 @@ dispatcher.job_lock_concurrency_level=3 dispatcher.frame_query_max=10 dispatcher.job_frame_dispatch_max=2 dispatcher.host_frame_dispatch_max=12 + +dispatcher.launch_queue.core_pool_size=1 +dispatcher.launch_queue.max_pool_size=1 +dispatcher.launch_queue.queue_capacity=100 + +dispatcher.dispatch_pool.core_pool_size=4 +dispatcher.dispatch_pool.max_pool_size=4 +dispatcher.dispatch_pool.queue_capacity=500 + +dispatcher.manage_pool.core_pool_size=8 +dispatcher.manage_pool.max_pool_size=8 +dispatcher.manage_pool.queue_capacity=250 + +dispatcher.report_queue.core_pool_size=6 +dispatcher.report_queue.max_pool_size=8 +dispatcher.report_queue.queue_capacity=1000 + +dispatcher.kill_queue.core_pool_size=6 +dispatcher.kill_queue.max_pool_size=8 +dispatcher.kill_queue.queue_capacity=1000 + +dispatcher.booking_queue.core_pool_size=6 +dispatcher.booking_queue.max_pool_size=6 +dispatcher.booking_queue.queue_capacity=1000 From 5122518dff3a815e038403ff2d8fbe4c0d0fcb82 Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Tue, 14 Dec 2021 15:09:29 -0800 Subject: [PATCH 151/277] Only restore jobIds added within last 3 days (#983) * Only restore jobIds added within last 3 days * Remove unused exception variable e * Fix pylint errors * Refactor restore jobIds code Changed constant to more descriptive name, changed multi line text to use implicit joins, fixed arg type. * Fixed MonitorJobPlugin restore constants Previous commit mixed constant job threshold's limit with days, corrected it. Added comment description for timestamp in addJob. --- cuegui/cuegui/JobMonitorTree.py | 35 ++++++++++++++---- cuegui/cuegui/plugins/MonitorJobsPlugin.py | 42 +++++++++++++++++----- 2 files changed, 61 insertions(+), 16 deletions(-) diff --git a/cuegui/cuegui/JobMonitorTree.py b/cuegui/cuegui/JobMonitorTree.py index 2f2a6c63d..7552b2859 100644 --- a/cuegui/cuegui/JobMonitorTree.py +++ b/cuegui/cuegui/JobMonitorTree.py @@ -226,23 +226,44 @@ def setLoadMine(self, value): @type value: boolean or QtCore.Qt.Checked or QtCore.Qt.Unchecked""" self.__loadMine = (value is True or value == QtCore.Qt.Checked) - def addJob(self, job): + def addJob(self, job, timestamp=None): """Adds a job to the list. With locking" @param job: Job can be None, a job object, or a job name. - @type job: job, string, None""" + @type job: job, string, None + @param timestamp: UTC time of the specific date the job was + added to be monitored + @type timestamp: float""" newJobObj = cuegui.Utils.findJob(job) self.ticksLock.lock() try: if newJobObj: - objectKey = cuegui.Utils.getObjectKey(newJobObj) - self.__load[objectKey] = newJobObj - self.__jobTimeLoaded[objectKey] = time.time() + jobKey = cuegui.Utils.getObjectKey(newJobObj) + if not self.__groupDependent: + self.__load[jobKey] = newJobObj + self.__jobTimeLoaded[jobKey] = timestamp if timestamp else time.time() finally: self.ticksLock.unlock() def getJobProxies(self): - """Gets a list of IDs of monitored jobs.""" - return list(self._items.keys()) + """Get a list of the JobProxies that are being monitored in the session + which will be saved to the config file + + Returning a sorted list based on the most recent timestamp - restoring jobs is capped + by LOAD_LIMIT, so restore the most recent jobs the user added to their session + + :return: list of tuples of the JobId and timestamp + """ + jobIdsTimeLoaded = [] + + for jobProxy, _ in self._items.items(): + try: + jobIdsTimeLoaded.append((jobProxy, self.__jobTimeLoaded[jobProxy])) + except KeyError: + # set timestamp to epoch time if timestamp not found + jobIdsTimeLoaded.append((jobProxy, 0)) + + # sort list on recent timestamps, only restoring the first n jobs (defined by LOAD_LIMIT) + return list(sorted(jobIdsTimeLoaded, key=lambda x: x[1], reverse=True)) def _removeItem(self, item): """Removes an item from the TreeWidget without locking diff --git a/cuegui/cuegui/plugins/MonitorJobsPlugin.py b/cuegui/cuegui/plugins/MonitorJobsPlugin.py index ee8fea472..6743a0717 100644 --- a/cuegui/cuegui/plugins/MonitorJobsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorJobsPlugin.py @@ -22,6 +22,7 @@ from builtins import str from builtins import map +from datetime import datetime import re import weakref @@ -46,7 +47,8 @@ PLUGIN_DESCRIPTION = "Monitors a list of jobs" PLUGIN_PROVIDES = "MonitorJobsDockWidget" REGEX_EMPTY_STRING = re.compile("^$") - +JOB_RESTORE_THRESHOLD_DAYS = 3 +JOB_RESTORE_THRESHOLD_LIMIT = 200 class MonitorJobsDockWidget(cuegui.AbstractDockWidget.AbstractDockWidget): """Plugin for listing active jobs and managing them.""" @@ -106,14 +108,36 @@ def getJobIds(self): return list(map(opencue.id, self.jobMonitor.getJobProxies())) def restoreJobIds(self, jobIds): - """Monitors a list of jobs.""" - for jobId in jobIds: - try: - self.jobMonitor.addJob(jobId) - except opencue.EntityNotFoundException: - logger.warning("Unable to load previously loaded job since " - "it was moved to the historical " - "database: %s", jobId) + """Restore monitored jobs from previous saved state + Only load jobs that have a timestamp less than or equal to the time a job lives on the farm + (jobs are moved to historical database) + + :param jobIds: monitored jobs ids and their timestamp from previous working state + (loaded from config.ini file) + ex: [("Job.f156be87-987a-48b9-b9da-774cd58674a3", 1612482716.170947),... + :type jobIds: list[tuples] + """ + today = datetime.datetime.now() + limit = JOB_RESTORE_THRESHOLD_LIMIT if len(jobIds) > \ + JOB_RESTORE_THRESHOLD_LIMIT else len(jobIds) + msg = ('Unable to load previously loaded job since it was moved ' + 'to the historical database: {0}') + + try: + for jobId, timestamp in jobIds[:limit]: + loggedTime = datetime.datetime.fromtimestamp(timestamp) + if (today - loggedTime).days <= JOB_RESTORE_THRESHOLD_DAYS: + try: + self.jobMonitor.addJob(jobId, timestamp) + except opencue.EntityNotFoundException: + logger.info(msg, jobId) + except ValueError: + # load older format + for jobId in jobIds[:limit]: + try: + self.jobMonitor.addJob(jobId) + except opencue.EntityNotFoundException: + logger.info(msg, jobId) def pluginRestoreState(self, saved_settings): """Called on plugin start with any previously saved state. From b52e6227b4a987e94863e503c64b3b73d789289a Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Tue, 14 Dec 2021 15:10:21 -0800 Subject: [PATCH 152/277] Allow override max CPU cores and GPU units via Job spec (#1000) * Allow override max CPU cores and GPU units via Job spec * Bump up version * s/1.12/1.13/ --- VERSION.in | 2 +- .../com/imageworks/spcue/BuildableJob.java | 6 ++ .../spcue/service/JobManagerService.java | 8 ++ .../com/imageworks/spcue/service/JobSpec.java | 9 ++ .../main/resources/public/dtd/cjsl-1.13.dtd | 99 +++++++++++++++++++ .../spcue/test/dao/postgres/JobDaoTests.java | 10 ++ .../spcue/test/service/JobSpecTests.java | 14 +++ .../src/test/resources/conf/dtd/cjsl-1.13.dtd | 99 +++++++++++++++++++ .../resources/conf/jobspec/jobspec_1_13.xml | 51 ++++++++++ .../conf/jobspec/override_max_cores_gpus.xml | 50 ++++++++++ pyoutline/etc/outline.cfg | 2 +- pyoutline/outline/backend/cue.py | 10 ++ pyoutline/outline/loader.py | 46 ++++++++- pyoutline/tests/specver_test.py | 19 ++++ 14 files changed, 422 insertions(+), 3 deletions(-) create mode 100644 cuebot/src/main/resources/public/dtd/cjsl-1.13.dtd create mode 100644 cuebot/src/test/resources/conf/dtd/cjsl-1.13.dtd create mode 100644 cuebot/src/test/resources/conf/jobspec/jobspec_1_13.xml create mode 100644 cuebot/src/test/resources/conf/jobspec/override_max_cores_gpus.xml diff --git a/VERSION.in b/VERSION.in index 948a54727..2856407c0 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.14 +0.15 diff --git a/cuebot/src/main/java/com/imageworks/spcue/BuildableJob.java b/cuebot/src/main/java/com/imageworks/spcue/BuildableJob.java index 8719f51d6..2c9d213a9 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/BuildableJob.java +++ b/cuebot/src/main/java/com/imageworks/spcue/BuildableJob.java @@ -37,6 +37,12 @@ public class BuildableJob { */ public JobDetail detail; + /** + * Maximum CPU cores and GPU units overrides. + */ + public Integer maxCoresOverride = null; + public Integer maxGpusOverride = null; + /** * List of layers */ diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java index 8477916b8..6da593712 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java @@ -65,6 +65,7 @@ import com.imageworks.spcue.util.CueUtil; import com.imageworks.spcue.util.FrameSet; import com.imageworks.spcue.util.JobLogUtil; +import com.imageworks.spcue.util.Convert; @Transactional public class JobManagerService implements JobManager { @@ -173,6 +174,13 @@ public void launchJobSpec(JobSpec spec) { for (BuildableJob job: spec.getJobs()) { JobDetail d = createJob(job); + if (job.maxCoresOverride != null) { + jobDao.updateMaxCores(d, + Convert.coresToWholeCoreUnits(job.maxCoresOverride.intValue())); + } + if (job.maxGpusOverride != null) { + jobDao.updateMaxGpus(d, job.maxGpusOverride.intValue()); + } if (job.getPostJob() != null) { BuildableJob postJob = job.getPostJob(); postJob.env.put("CUE_PARENT_JOB_ID", d.id); diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java index 30bf7cfd3..a8ffb7d4c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java @@ -328,6 +328,15 @@ private BuildableJob handleJobTag(Element jobTag) { } } + if (jobTag.getChildTextTrim("maxcores") != null) { + buildableJob.maxCoresOverride = Integer.valueOf(jobTag + .getChildTextTrim("maxcores")); + } + if (jobTag.getChildTextTrim("maxgpus") != null) { + buildableJob.maxGpusOverride = Integer.valueOf(jobTag + .getChildTextTrim("maxgpus")); + } + if (jobTag.getChildTextTrim("priority") != null) { job.priority = Integer.valueOf(jobTag.getChildTextTrim("priority")); } diff --git a/cuebot/src/main/resources/public/dtd/cjsl-1.13.dtd b/cuebot/src/main/resources/public/dtd/cjsl-1.13.dtd new file mode 100644 index 000000000..ff7ad71e3 --- /dev/null +++ b/cuebot/src/main/resources/public/dtd/cjsl-1.13.dtd @@ -0,0 +1,99 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/JobDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/JobDaoTests.java index d4163d7f6..3bfff75d6 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/JobDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/JobDaoTests.java @@ -720,6 +720,16 @@ public void testUpdateUsage() { "SELECT int_frame_fail_count FROM job_usage WHERE pk_job=?", Integer.class, job.getId())); } + + @Test + @Transactional + @Rollback(true) + public void testOverrideMaxCoresAndGpus() { + jobLauncher.launch(new File("src/test/resources/conf/jobspec/override_max_cores_gpus.xml")); + JobDetail job = jobManager.findJobDetail("pipe-dev.cue-testuser_test"); + assertEquals(job.maxCoreUnits, 42000); + assertEquals(job.maxGpuUnits, 42); + } } diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/JobSpecTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/JobSpecTests.java index d3f94dce7..119be6160 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/JobSpecTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/JobSpecTests.java @@ -114,4 +114,18 @@ public void testParseGpuSuccess() { assertEquals(layer.getMinimumGpuMemory(), 1048576); } + @Test + public void testParseMaxCoresAndMaxGpus() { + String xml = readJobSpec("jobspec_1_13.xml"); + JobSpec spec = jobLauncher.parse(xml); + assertEquals(spec.getDoc().getDocType().getPublicID(), + "SPI Cue Specification Language"); + assertEquals(spec.getDoc().getDocType().getSystemID(), + "http://localhost:8080/spcue/dtd/cjsl-1.13.dtd"); + assertEquals(spec.getJobs().size(), 1); + BuildableJob job = spec.getJobs().get(0); + assertEquals(job.maxCoresOverride, Integer.valueOf(420)); + assertEquals(job.maxGpusOverride, Integer.valueOf(42)); + } + } diff --git a/cuebot/src/test/resources/conf/dtd/cjsl-1.13.dtd b/cuebot/src/test/resources/conf/dtd/cjsl-1.13.dtd new file mode 100644 index 000000000..ff7ad71e3 --- /dev/null +++ b/cuebot/src/test/resources/conf/dtd/cjsl-1.13.dtd @@ -0,0 +1,99 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cuebot/src/test/resources/conf/jobspec/jobspec_1_13.xml b/cuebot/src/test/resources/conf/jobspec/jobspec_1_13.xml new file mode 100644 index 000000000..36d40c95d --- /dev/null +++ b/cuebot/src/test/resources/conf/jobspec/jobspec_1_13.xml @@ -0,0 +1,51 @@ + + + + + + + + + local + testing + default + testuser + 9860 + + + False + 2 + 420 + 42 + False + + + + echo $CUE_GPU_CORES + 1-10 + 1 + 1 + 1 + + + shell + + + + + + diff --git a/cuebot/src/test/resources/conf/jobspec/override_max_cores_gpus.xml b/cuebot/src/test/resources/conf/jobspec/override_max_cores_gpus.xml new file mode 100644 index 000000000..273fcb0ee --- /dev/null +++ b/cuebot/src/test/resources/conf/jobspec/override_max_cores_gpus.xml @@ -0,0 +1,50 @@ + + + + + + + + + pipe + dev.cue + testuser + 9860 + + + False + 2 + 420 + 42 + False + + + + echo $CUE_GPU_CORES + 1-10 + 1 + 1 + 1 + + + shell + + + + + + diff --git a/pyoutline/etc/outline.cfg b/pyoutline/etc/outline.cfg index fafbfa636..6347825f9 100644 --- a/pyoutline/etc/outline.cfg +++ b/pyoutline/etc/outline.cfg @@ -5,7 +5,7 @@ wrapper_dir = %(home)s/wrappers user_dir = bin_dir = %(home)s/bin backend = cue -spec_version = 1.12 +spec_version = 1.13 facility = local domain = example.com maxretries = 2 diff --git a/pyoutline/outline/backend/cue.py b/pyoutline/outline/backend/cue.py index 333b5ac59..8c111a727 100644 --- a/pyoutline/outline/backend/cue.py +++ b/pyoutline/outline/backend/cue.py @@ -258,6 +258,16 @@ def _serialize(launcher, use_pycuerun): elif launcher.get("priority"): _warning_spec_version(spec_version, "priority") sub_element(j, "maxretries", str(launcher.get("maxretries"))) + if spec_version >= Version("1.13"): + if ol.get_maxcores(): + sub_element(j, "maxcores", str(ol.get_maxcores())) + if ol.get_maxgpus(): + sub_element(j, "maxgpus", str(ol.get_maxgpus())) + else: + if ol.get_maxcores(): + _warning_spec_version(spec_version, "maxcores") + if ol.get_maxgpus(): + _warning_spec_version(spec_version, "maxgpus") sub_element(j, "autoeat", str(launcher.get("autoeat"))) if ol.get_arg("localbook"): diff --git a/pyoutline/outline/loader.py b/pyoutline/outline/loader.py index 265272be6..3db6b2c4f 100644 --- a/pyoutline/outline/loader.py +++ b/pyoutline/outline/loader.py @@ -126,6 +126,10 @@ def decode_layer(layer): ol.set_name(data["name"]) if "facility" in data: ol.set_facility(data["facility"]) + if "maxcores" in data: + ol.set_maxcores(data["maxcores"]) + if "maxgpus" in data: + ol.set_maxgpus(data["maxgpus"]) if "range" in data: ol.set_frame_range(data["range"]) @@ -219,7 +223,8 @@ class Outline(object): def __init__(self, name=None, frame_range=None, path=None, serialize=True, name_unique=False, current=False, - shot=None, show=None, user=None, facility=None): + shot=None, show=None, user=None, facility=None, + maxcores=None, maxgpus=None): """ :type name: string :param name: A name for the outline instance. This will become @@ -254,6 +259,11 @@ def __init__(self, name=None, frame_range=None, path=None, :param facility: The launch facility to be used. If not specified the RENDER_TO and FACILITY environment variables will be checked. + :type maxcores: int + :param maxcores: The maximum number of CPU cores for the job. + + :type maxgpus: int + :param maxgpus: The maximum number of GPU units for the job. """ object.__init__(self) @@ -321,6 +331,16 @@ def __init__(self, name=None, frame_range=None, path=None, # self.__facility = facility + # + # The maximum number of CPU cores to use, or None. + # + self.__maxcores = maxcores + + # + # The maximum number of CPU cores to use, or None. + # + self.__maxgpus = maxgpus + # # The outline session. The session is setup during the setup # phase. Every job has a unique session which maps to a @@ -630,6 +650,30 @@ def set_facility(self, facility): """ self.__facility = facility + def get_maxcores(self): + """Return the maximum number of CPU cores fot this outline.""" + return self.__maxcores + + def set_maxcores(self, maxcores): + """Set the maximum number of CPU cores for this outline instance. + + :type maxcores: int + :param maxcores: The maximum number of CPU cores to set. + """ + self.__maxcores = maxcores + + def get_maxgpus(self): + """Return the maximum number of GPU units fot this outline.""" + return self.__maxgpus + + def set_maxgpus(self, maxgpus): + """Set the maximum number of GPU units for this outline instance. + + :type maxcores: int + :param maxcores: The maximum number of GPU units to set. + """ + self.__maxgpus = maxgpus + def get_mode(self): """Return the current mode of this outline object. diff --git a/pyoutline/tests/specver_test.py b/pyoutline/tests/specver_test.py index f4c077d4b..d04d65bb4 100644 --- a/pyoutline/tests/specver_test.py +++ b/pyoutline/tests/specver_test.py @@ -84,3 +84,22 @@ def test_gpu_1_12(self): root = self._makeGpuSpec() self.assertEqual(root.find("job/layers/layer/gpus").text, "4") self.assertEqual(root.find("job/layers/layer/gpu_memory").text, "8388608") + + def _makeMaxCoresGpusSpec(self): + ol = outline.Outline(name="override_max_cores_and_gpus", maxcores=8, maxgpus=7) + layer = outline.modules.shell.Shell("test_layer", command=["/bin/ls"]) + ol.add_layer(layer) + l = outline.cuerun.OutlineLauncher(ol) + return Et.fromstring(l.serialize()) + + def test_max_cores_gpus_1_12(self): + outline.config.set("outline", "spec_version", "1.12") + root = self._makeMaxCoresGpusSpec() + self.assertIsNone(root.find("job/maxcores")) + self.assertIsNone(root.find("job/maxgpus")) + + def test_max_cores_gpus_1_13(self): + outline.config.set("outline", "spec_version", "1.13") + root = self._makeMaxCoresGpusSpec() + self.assertEqual(root.find("job/maxcores").text, "8") + self.assertEqual(root.find("job/maxgpus").text, "7") From 8551be7988e4f8d7876cfe9df018dfc9d3fe2d4a Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Tue, 14 Dec 2021 15:11:09 -0800 Subject: [PATCH 153/277] Get RSS and %CPU for Windows (#1023) * Add winps native extension for Windows RQD * Use winps * Factor out __updateGpuAndLlu * Fix bugs * Use disable=no-member instead of disable=E1101 * Set protobuf version 3.17.3 for CY2019 --- rqd/rqd/rqmachine.py | 55 ++++++++--- rqd/winps/setup.py | 14 +++ rqd/winps/winps.cpp | 228 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 281 insertions(+), 16 deletions(-) create mode 100644 rqd/winps/setup.py create mode 100644 rqd/winps/winps.cpp diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 565fcd21a..70a7dd2e6 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -45,8 +45,13 @@ # pylint: disable=import-error,wrong-import-position if platform.system() in ('Linux', 'Darwin'): import resource -elif platform.system() == "win32": - import win32api +elif platform.system() == "Windows": + winpsIsAvailable = False + try: + import winps + winpsIsAvailable = True + except ImportError: + pass # pylint: enable=import-error,wrong-import-position import psutil @@ -188,8 +193,38 @@ def isUserLoggedIn(self): return True return False + def __updateGpuAndLlu(self, frame): + if 'GPU_LIST' in frame.runFrame.attributes: + usedGpuMemory = 0 + for unitId in frame.runFrame.attributes.get('GPU_LIST').split(','): + usedGpuMemory += self.getGpuMemoryUsed(unitId) + + frame.usedGpuMemory = usedGpuMemory + frame.maxUsedGpuMemory = max(usedGpuMemory, frame.maxUsedGpuMemory) + + if os.path.exists(frame.runFrame.log_dir_file): + stat = os.stat(frame.runFrame.log_dir_file).st_mtime + frame.lluTime = int(stat) + def rssUpdate(self, frames): """Updates the rss and maxrss for all running frames""" + if platform.system() == 'Windows' and winpsIsAvailable: + values = list(frames.values()) + pids = [frame.pid for frame in list( + filter(lambda frame: frame.pid > 0, values) + )] + # pylint: disable=no-member + stats = winps.update(pids) + # pylint: enable=no-member + for frame in values: + self.__updateGpuAndLlu(frame) + if frame.pid > 0 and frame.pid in stats: + stat = stats[frame.pid] + frame.rss = stat["rss"] // 1024 + frame.maxRss = max(frame.rss, frame.maxRss) + frame.runFrame.attributes["pcpu"] = str(stat["pcpu"]) + return + if platform.system() != 'Linux': return @@ -278,23 +313,13 @@ def rssUpdate(self, frames): frame.rss = rss frame.maxRss = max(rss, frame.maxRss) - if 'GPU_LIST' in frame.runFrame.attributes: - usedGpuMemory = 0 - for unitId in frame.runFrame.attributes.get('GPU_LIST').split(','): - usedGpuMemory += self.getGpuMemoryUsed(unitId) - - frame.usedGpuMemory = usedGpuMemory - frame.maxUsedGpuMemory = max(usedGpuMemory, frame.maxUsedGpuMemory) - - if os.path.exists(frame.runFrame.log_dir_file): - stat = os.stat(frame.runFrame.log_dir_file).st_mtime - frame.lluTime = int(stat) - frame.vsize = vsize frame.maxVsize = max(vsize, frame.maxVsize) frame.runFrame.attributes["pcpu"] = str(pcpu) + self.__updateGpuAndLlu(frame) + # Store the current data for the next check self.__pidHistory = pidData @@ -423,8 +448,6 @@ def getPathEnv(self): @rqd.rqutil.Memoize def getTempPath(self): """Returns the correct mcp path for the given machine""" - if platform.system() == "win32": - return win32api.GetTempPath() if os.path.isdir("/mcp/"): return "/mcp/" return '%s/' % tempfile.gettempdir() diff --git a/rqd/winps/setup.py b/rqd/winps/setup.py new file mode 100644 index 000000000..5db5c7ad4 --- /dev/null +++ b/rqd/winps/setup.py @@ -0,0 +1,14 @@ +from distutils.core import setup, Extension + +winps = Extension('winps', + define_macros = [('MAJOR_VERSION', '1'), + ('MINOR_VERSION', '0')], + include_dirs = [], + libraries = [], + library_dirs = [], + sources = ['winps.cpp']) + +setup (name = 'winps', + version = '1.0', + description = 'Windows ps for RQD', + ext_modules = [winps]) diff --git a/rqd/winps/winps.cpp b/rqd/winps/winps.cpp new file mode 100644 index 000000000..a7c058324 --- /dev/null +++ b/rqd/winps/winps.cpp @@ -0,0 +1,228 @@ +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define PY_SSIZE_T_CLEAN +#include + +#include +// TlHelp32.h should be included after Windows.h +#include +#include + +#include +#include +#include +#include + +namespace { + +// Process stat history +struct Snapshot { + uint64_t creationTimeInFiletime; + uint64_t totalTimeInFiletime; + uint64_t wallTimeInFiletime; + double pidPcpu; +}; +std::map history; + +// FILETIME -> uint64_t, in 100-nanosecond unit +uint64_t convertFiletime(const FILETIME& ft) { + union TimeUnion { + FILETIME ft; + ULARGE_INTEGER ul; + }; + TimeUnion tu; + tu.ft = ft; + return tu.ul.QuadPart; +} + +void traverse( + const std::map>& parentChildrenMap, + DWORD pid, + uint64_t& rss, + double& pcpu) { + HANDLE hProcess = + OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, false, pid); + if (hProcess != nullptr) { + // RSS + PROCESS_MEMORY_COUNTERS pmc; + if (GetProcessMemoryInfo(hProcess, &pmc, sizeof(pmc))) { + rss += pmc.WorkingSetSize; + } + + // %CPU + FILETIME creationTime; + FILETIME exitTime; + FILETIME kernelTime; + FILETIME userTime; + if (GetProcessTimes( + hProcess, &creationTime, &exitTime, &kernelTime, &userTime)) { + // Current time in FILETIME + FILETIME now; + GetSystemTimeAsFileTime(&now); + + // Process start time + uint64_t creationTimeInFiletime = convertFiletime(creationTime); + + // Total time of kernel and user mode on this process + uint64_t totalTimeInFiletime = + convertFiletime(kernelTime) + convertFiletime(userTime); + + // Walltime of this process + uint64_t wallTimeInFiletime = + convertFiletime(now) - creationTimeInFiletime; + + if (wallTimeInFiletime > 0) { + auto it = history.find(pid); + if (it != history.end() && + it->second.creationTimeInFiletime == creationTimeInFiletime) { + // Percent cpu using decaying average, 50% from 10 seconds ago, + // 50% from last 10 seconds: + const auto& last = it->second; + double pidPcpu = static_cast( + totalTimeInFiletime - last.totalTimeInFiletime) / + static_cast(wallTimeInFiletime - last.wallTimeInFiletime); + pcpu += (last.pidPcpu + pidPcpu) / 2.0; // %cpu + history[pid] = Snapshot{ + creationTimeInFiletime, + totalTimeInFiletime, + wallTimeInFiletime, + pidPcpu}; + } else { + double pidPcpu = static_cast(totalTimeInFiletime) / + static_cast(wallTimeInFiletime); + pcpu += pidPcpu; + + history[pid] = Snapshot{ + creationTimeInFiletime, + totalTimeInFiletime, + wallTimeInFiletime, + pidPcpu}; + } + } + } + } + + const auto it = parentChildrenMap.find(pid); + if (it != parentChildrenMap.end()) { + for (const auto childPid : it->second) { + traverse(parentChildrenMap, childPid, rss, pcpu); + } + } +} + +PyObject* winpsUpdate(PyObject* self, PyObject* args) { + /* + * :param list pids: a list of pid(int) to look into + * :return: RSS and %CPU dict, or None if invalid inputs or error occurred + * :rtype: dict (key=pid, value={rss:uint64_t, pcpu:double}) + */ + PyObject* list; + if (!PyArg_ParseTuple(args, "O!", &PyList_Type, &list)) { + return nullptr; + } + + // Take a snapshot of all processes and create parent-children process map + std::map> parentChildrenMap; + HANDLE snapshotHandle = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); + PROCESSENTRY32 processEntry = {}; + processEntry.dwSize = sizeof(PROCESSENTRY32); + if (Process32First(snapshotHandle, &processEntry)) { + do { + if (processEntry.th32ProcessID != 0) { + parentChildrenMap[processEntry.th32ParentProcessID].push_back( + processEntry.th32ProcessID); + } + } while (Process32Next(snapshotHandle, &processEntry)); + } + CloseHandle(snapshotHandle); + + // output = {} + PyObject* output = PyDict_New(); + if (output == nullptr) { + return nullptr; + } + + // Iterate the pids list + Py_ssize_t listSize = PyList_Size(list); + for (Py_ssize_t i = 0; i < listSize; i++) { + PyObject* pidObject = PyList_GetItem(list, i); + if (pidObject == nullptr) { + return nullptr; + } + DWORD pid = PyLong_AsUnsignedLong(pidObject); + if (PyErr_Occurred()) { + return nullptr; + } + + // Traverse process tree to add up RSS and %CPU from the pid + uint64_t rss = 0; + double pcpu = 0; + traverse(parentChildrenMap, pid, rss, pcpu); + + // stat = {} + PyObject* stat = PyDict_New(); + if (stat == nullptr) { + return nullptr; + } + + // stat["rss"] = rss + PyObject* rssObject = PyLong_FromUnsignedLongLong(rss); + if (rssObject == nullptr) { + return nullptr; + } + if (PyDict_SetItemString(stat, "rss", rssObject) != 0) { + return nullptr; + } + + // stat["pcpu"] = pcpu + PyObject* pcpuObject = PyFloat_FromDouble(pcpu); + if (pcpuObject == nullptr) { + return nullptr; + } + if (PyDict_SetItemString(stat, "pcpu", pcpuObject) != 0) { + return nullptr; + } + + // output[pid] = stat + if (PyDict_SetItem(output, pidObject, stat) != 0) { + return nullptr; + } + } + + return output; +} + +PyMethodDef winpsMethods[] = { + {"update", + winpsUpdate, + METH_VARARGS, + "Updates internal state and returns rss and pcpu"}, + {NULL, NULL, 0, NULL}}; + +PyModuleDef winpsModule = { + PyModuleDef_HEAD_INIT, + "winps", + nullptr, + -1, + winpsMethods, +}; + +} // namespace + +PyMODINIT_FUNC PyInit_winps() { + return PyModule_Create(&winpsModule); +} From 1ed72ada783ff0cbb59386e14bbf0f2f70e238c7 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Tue, 14 Dec 2021 15:11:21 -0800 Subject: [PATCH 154/277] Delete CORE_POINTS_RESERVED_MAX check logic (#1030) --- .../main/java/com/imageworks/spcue/dispatcher/Dispatcher.java | 3 --- cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java | 3 +-- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java index 3440fb595..848220fcd 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java @@ -34,9 +34,6 @@ public interface Dispatcher { - // Maximum number of core points that can be assigned to a frame - public static final int CORE_POINTS_RESERVED_MAX = 2400; - // The default number of core points assigned to a frame, if no core // point value is specified public static final int CORE_POINTS_RESERVED_DEFAULT = 100; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java index a8ffb7d4c..db33729ee 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java @@ -615,8 +615,7 @@ private void determineMinimumCores(Element layerTag, LayerDetail layer) { corePoints = Integer.valueOf(cores); } - if (corePoints < Dispatcher.CORE_POINTS_RESERVED_MIN - || corePoints > Dispatcher.CORE_POINTS_RESERVED_MAX) { + if (corePoints < Dispatcher.CORE_POINTS_RESERVED_MIN) { corePoints = Dispatcher.CORE_POINTS_RESERVED_DEFAULT; } From 7b47548b8e92d8ab77e2dda4e548f5839b6fa7df Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Tue, 14 Dec 2021 15:13:28 -0800 Subject: [PATCH 155/277] [Cuebot][DB] Add history control (#1058) * Disable to update history tables if DISABLE_HISTORY config exists * Bump up version * Add history control unittests * Delete history for CI --- .../migrations/V13__Add_history_control.sql | 397 ++++++++++++++++++ .../test/dispatcher/HistoryControlTests.java | 198 +++++++++ 2 files changed, 595 insertions(+) create mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V13__Add_history_control.sql create mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V13__Add_history_control.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V13__Add_history_control.sql new file mode 100644 index 000000000..16896d4c5 --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V13__Add_history_control.sql @@ -0,0 +1,397 @@ +-- Add history control + +CREATE OR REPLACE FUNCTION trigger__after_insert_job() +RETURNS TRIGGER AS $body$ +BEGIN + INSERT INTO job_stat (pk_job_stat,pk_job) VALUES(NEW.pk_job,NEW.pk_job); + INSERT INTO job_resource (pk_job_resource,pk_job) VALUES(NEW.pk_job,NEW.pk_job); + INSERT INTO job_usage (pk_job_usage,pk_job) VALUES(NEW.pk_job,NEW.pk_job); + INSERT INTO job_mem (pk_job_mem,pk_job) VALUES (NEW.pk_job,NEW.pk_job); + + IF NOT EXISTS (SELECT FROM config WHERE str_key='DISABLE_HISTORY') THEN + + INSERT INTO job_history + (pk_job, pk_show, pk_facility, pk_dept, str_name, str_shot, str_user, int_ts_started) + VALUES + (NEW.pk_job, NEW.pk_show, NEW.pk_facility, NEW.pk_dept, + NEW.str_name, NEW.str_shot, NEW.str_user, epoch(current_timestamp)); + + END IF; + + RETURN NULL; +END; +$body$ +LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION trigger__before_delete_job() +RETURNS TRIGGER AS $body$ +DECLARE + js JobStatType; +BEGIN + IF NOT EXISTS (SELECT FROM config WHERE str_key='DISABLE_HISTORY') THEN + + SELECT + job_usage.int_core_time_success, + job_usage.int_core_time_fail, + job_usage.int_gpu_time_success, + job_usage.int_gpu_time_fail, + job_stat.int_waiting_count, + job_stat.int_dead_count, + job_stat.int_depend_count, + job_stat.int_eaten_count, + job_stat.int_succeeded_count, + job_stat.int_running_count, + job_mem.int_max_rss, + job_mem.int_gpu_mem_max + INTO + js + FROM + job_mem, + job_usage, + job_stat + WHERE + job_usage.pk_job = job_mem.pk_job + AND + job_stat.pk_job = job_mem.pk_job + AND + job_mem.pk_job = OLD.pk_job; + + UPDATE + job_history + SET + pk_dept = OLD.pk_dept, + int_core_time_success = js.int_core_time_success, + int_core_time_fail = js.int_core_time_fail, + int_gpu_time_success = js.int_gpu_time_success, + int_gpu_time_fail = js.int_gpu_time_fail, + int_frame_count = OLD.int_frame_count, + int_layer_count = OLD.int_layer_count, + int_waiting_count = js.int_waiting_count, + int_dead_count = js.int_dead_count, + int_depend_count = js.int_depend_count, + int_eaten_count = js.int_eaten_count, + int_succeeded_count = js.int_succeeded_count, + int_running_count = js.int_running_count, + int_max_rss = js.int_max_rss, + int_gpu_mem_max = js.int_gpu_mem_max, + b_archived = true, + int_ts_stopped = COALESCE(epoch(OLD.ts_stopped), epoch(current_timestamp)) + WHERE + pk_job = OLD.pk_job; + + END IF; + + DELETE FROM depend WHERE pk_job_depend_on=OLD.pk_job OR pk_job_depend_er=OLD.pk_job; + DELETE FROM frame WHERE pk_job=OLD.pk_job; + DELETE FROM layer WHERE pk_job=OLD.pk_job; + DELETE FROM job_env WHERE pk_job=OLD.pk_job; + DELETE FROM job_stat WHERE pk_job=OLD.pk_job; + DELETE FROM job_resource WHERE pk_job=OLD.pk_job; + DELETE FROM job_usage WHERE pk_job=OLD.pk_job; + DELETE FROM job_mem WHERE pk_job=OLD.pk_job; + DELETE FROM comments WHERE pk_job=OLD.pk_job; + + RETURN OLD; +END +$body$ +LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION trigger__after_job_finished() +RETURNS TRIGGER AS $body$ +DECLARE + ts INT := cast(epoch(current_timestamp) as integer); + js JobStatType; + ls LayerStatType; + one_layer RECORD; +BEGIN + IF NOT EXISTS (SELECT FROM config WHERE str_key='DISABLE_HISTORY') THEN + + SELECT + job_usage.int_core_time_success, + job_usage.int_core_time_fail, + job_usage.int_gpu_time_success, + job_usage.int_gpu_time_fail, + job_stat.int_waiting_count, + job_stat.int_dead_count, + job_stat.int_depend_count, + job_stat.int_eaten_count, + job_stat.int_succeeded_count, + job_stat.int_running_count, + job_mem.int_max_rss, + job_mem.int_gpu_mem_max + INTO + js + FROM + job_mem, + job_usage, + job_stat + WHERE + job_usage.pk_job = job_mem.pk_job + AND + job_stat.pk_job = job_mem.pk_job + AND + job_mem.pk_job = NEW.pk_job; + + UPDATE + job_history + SET + pk_dept = NEW.pk_dept, + int_core_time_success = js.int_core_time_success, + int_core_time_fail = js.int_core_time_fail, + int_gpu_time_success = js.int_gpu_time_success, + int_gpu_time_fail = js.int_gpu_time_fail, + int_frame_count = NEW.int_frame_count, + int_layer_count = NEW.int_layer_count, + int_waiting_count = js.int_waiting_count, + int_dead_count = js.int_dead_count, + int_depend_count = js.int_depend_count, + int_eaten_count = js.int_eaten_count, + int_succeeded_count = js.int_succeeded_count, + int_running_count = js.int_running_count, + int_max_rss = js.int_max_rss, + int_gpu_mem_max = js.int_gpu_mem_max, + int_ts_stopped = ts + WHERE + pk_job = NEW.pk_job; + + FOR one_layer IN (SELECT pk_layer from layer where pk_job = NEW.pk_job) + LOOP + SELECT + layer_usage.int_core_time_success, + layer_usage.int_core_time_fail, + layer_usage.int_gpu_time_success, + layer_usage.int_gpu_time_fail, + layer_stat.int_total_count, + layer_stat.int_waiting_count, + layer_stat.int_dead_count, + layer_stat.int_depend_count, + layer_stat.int_eaten_count, + layer_stat.int_succeeded_count, + layer_stat.int_running_count, + layer_mem.int_max_rss, + layer_mem.int_gpu_mem_max + INTO + ls + FROM + layer_mem, + layer_usage, + layer_stat + WHERE + layer_usage.pk_layer = layer_mem.pk_layer + AND + layer_stat.pk_layer = layer_mem.pk_layer + AND + layer_mem.pk_layer = one_layer.pk_layer; + + UPDATE + layer_history + SET + int_core_time_success = ls.int_core_time_success, + int_core_time_fail = ls.int_core_time_fail, + int_gpu_time_success = ls.int_gpu_time_success, + int_gpu_time_fail = ls.int_gpu_time_fail, + int_frame_count = ls.int_total_count, + int_waiting_count = ls.int_waiting_count, + int_dead_count = ls.int_dead_count, + int_depend_count = ls.int_depend_count, + int_eaten_count = ls.int_eaten_count, + int_succeeded_count = ls.int_succeeded_count, + int_running_count = ls.int_running_count, + int_max_rss = ls.int_max_rss, + int_gpu_mem_max = ls.int_gpu_mem_max + WHERE + pk_layer = one_layer.pk_layer; + END LOOP; + + END IF; + + /** + * Delete any local core assignments from this job. + **/ + DELETE FROM job_local WHERE pk_job=NEW.pk_job; + + RETURN NEW; +END; +$body$ +LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION trigger__after_insert_layer() +RETURNS TRIGGER AS $body$ +BEGIN + INSERT INTO layer_stat (pk_layer_stat, pk_layer, pk_job) VALUES (NEW.pk_layer, NEW.pk_layer, NEW.pk_job); + INSERT INTO layer_resource (pk_layer_resource, pk_layer, pk_job) VALUES (NEW.pk_layer, NEW.pk_layer, NEW.pk_job); + INSERT INTO layer_usage (pk_layer_usage, pk_layer, pk_job) VALUES (NEW.pk_layer, NEW.pk_layer, NEW.pk_job); + INSERT INTO layer_mem (pk_layer_mem, pk_layer, pk_job) VALUES (NEW.pk_layer, NEW.pk_layer, NEW.pk_job); + + IF NOT EXISTS (SELECT FROM config WHERE str_key='DISABLE_HISTORY') THEN + + INSERT INTO layer_history + (pk_layer, pk_job, str_name, str_type, int_cores_min, int_mem_min, int_gpus_min, int_gpu_mem_min, b_archived,str_services) + VALUES + (NEW.pk_layer, NEW.pk_job, NEW.str_name, NEW.str_type, NEW.int_cores_min, NEW.int_mem_min, NEW.int_gpus_min, NEW.int_gpu_mem_min, false, NEW.str_services); + + END IF; + + RETURN NEW; +END; +$body$ +LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION trigger__before_delete_layer() +RETURNS TRIGGER AS $body$ +DECLARE + js LayerStatType; +BEGIN + IF NOT EXISTS (SELECT FROM config WHERE str_key='DISABLE_HISTORY') THEN + + SELECT + layer_usage.int_core_time_success, + layer_usage.int_core_time_fail, + layer_usage.int_gpu_time_success, + layer_usage.int_gpu_time_fail, + layer_stat.int_total_count, + layer_stat.int_waiting_count, + layer_stat.int_dead_count, + layer_stat.int_depend_count, + layer_stat.int_eaten_count, + layer_stat.int_succeeded_count, + layer_stat.int_running_count, + layer_mem.int_max_rss, + layer_mem.int_gpu_mem_max + INTO + js + FROM + layer_mem, + layer_usage, + layer_stat + WHERE + layer_usage.pk_layer = layer_mem.pk_layer + AND + layer_stat.pk_layer = layer_mem.pk_layer + AND + layer_mem.pk_layer = OLD.pk_layer; + + UPDATE + layer_history + SET + int_core_time_success = js.int_core_time_success, + int_core_time_fail = js.int_core_time_fail, + int_gpu_time_success = js.int_gpu_time_success, + int_gpu_time_fail = js.int_gpu_time_fail, + int_frame_count = js.int_total_count, + int_waiting_count = js.int_waiting_count, + int_dead_count = js.int_dead_count, + int_depend_count = js.int_depend_count, + int_eaten_count = js.int_eaten_count, + int_succeeded_count = js.int_succeeded_count, + int_running_count = js.int_running_count, + int_max_rss = js.int_max_rss, + int_gpu_mem_max = js.int_gpu_mem_max, + b_archived = true + WHERE + pk_layer = OLD.pk_layer; + + END IF; + + DELETE FROM layer_resource where pk_layer=OLD.pk_layer; + DELETE FROM layer_stat where pk_layer=OLD.pk_layer; + DELETE FROM layer_usage where pk_layer=OLD.pk_layer; + DELETE FROM layer_env where pk_layer=OLD.pk_layer; + DELETE FROM layer_mem where pk_layer=OLD.pk_layer; + DELETE FROM layer_output where pk_layer=OLD.pk_layer; + + RETURN OLD; +END; +$body$ +LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION trigger__frame_history_open() +RETURNS TRIGGER AS $body$ +DECLARE + str_pk_alloc VARCHAR(36) := null; + int_checkpoint INT := 0; +BEGIN + + IF NOT EXISTS (SELECT FROM config WHERE str_key='DISABLE_HISTORY') THEN + + IF OLD.str_state = 'RUNNING' THEN + + IF NEW.int_exit_status = 299 THEN + + EXECUTE 'DELETE FROM frame_history WHERE int_ts_stopped = 0 AND pk_frame=$1' USING + NEW.pk_frame; + + ELSE + If NEW.str_state = 'CHECKPOINT' THEN + int_checkpoint := 1; + END IF; + + EXECUTE + 'UPDATE + frame_history + SET + int_mem_max_used=$1, + int_gpu_mem_max_used=$2, + int_ts_stopped=$3, + int_exit_status=$4, + int_checkpoint_count=$5 + WHERE + int_ts_stopped = 0 AND pk_frame=$6' + USING + NEW.int_mem_max_used, + NEW.int_gpu_mem_max_used, + epoch(current_timestamp), + NEW.int_exit_status, + int_checkpoint, + NEW.pk_frame; + END IF; + END IF; + + IF NEW.str_state = 'RUNNING' THEN + + SELECT pk_alloc INTO str_pk_alloc FROM host WHERE str_name=NEW.str_host; + + EXECUTE + 'INSERT INTO + frame_history + ( + pk_frame, + pk_layer, + pk_job, + str_name, + str_state, + int_cores, + int_mem_reserved, + int_gpus, + int_gpu_mem_reserved, + str_host, + int_ts_started, + pk_alloc + ) + VALUES + ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)' + USING NEW.pk_frame, + NEW.pk_layer, + NEW.pk_job, + NEW.str_name, + 'RUNNING', + NEW.int_cores, + NEW.int_mem_reserved, + NEW.int_gpus, + NEW.int_gpu_mem_reserved, + NEW.str_host, + epoch(current_timestamp), + str_pk_alloc; + END IF; + + END IF; + RETURN NULL; + +END; +$body$ +LANGUAGE PLPGSQL; diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java new file mode 100644 index 000000000..e8dc15517 --- /dev/null +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java @@ -0,0 +1,198 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.imageworks.spcue.test.dispatcher; + +import java.io.File; +import java.util.List; +import javax.annotation.Resource; + +import org.junit.Before; +import org.junit.Test; +import org.springframework.test.annotation.Rollback; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.transaction.annotation.Transactional; + +import com.imageworks.spcue.DispatchHost; +import com.imageworks.spcue.FrameInterface; +import com.imageworks.spcue.JobDetail; +import com.imageworks.spcue.LayerDetail; +import com.imageworks.spcue.VirtualProc; +import com.imageworks.spcue.dao.LayerDao; +import com.imageworks.spcue.dispatcher.Dispatcher; +import com.imageworks.spcue.dispatcher.FrameCompleteHandler; +import com.imageworks.spcue.grpc.host.HardwareState; +import com.imageworks.spcue.grpc.report.FrameCompleteReport; +import com.imageworks.spcue.grpc.report.RenderHost; +import com.imageworks.spcue.grpc.report.RunningFrameInfo; +import com.imageworks.spcue.service.AdminManager; +import com.imageworks.spcue.service.HostManager; +import com.imageworks.spcue.service.JobLauncher; +import com.imageworks.spcue.service.JobManager; +import com.imageworks.spcue.test.TransactionalTest; +import com.imageworks.spcue.util.CueUtil; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +@ContextConfiguration +public class HistoryControlTests extends TransactionalTest { + + @Resource + AdminManager adminManager; + + @Resource + FrameCompleteHandler frameCompleteHandler; + + @Resource + HostManager hostManager; + + @Resource + JobLauncher jobLauncher; + + @Resource + JobManager jobManager; + + @Resource + LayerDao layerDao; + + @Resource + Dispatcher dispatcher; + + private static final String HOSTNAME = "beta"; + private static final String DELETE_HISTORY = + "DELETE FROM frame_history; " + + "DELETE FROM job_history; "; + private static final String DISABLE_HISTORY = + "INSERT INTO " + + "config (pk_config,str_key) " + + "VALUES " + + "(uuid_generate_v1(),'DISABLE_HISTORY');"; + + @Before + public void setTestMode() { + dispatcher.setTestMode(true); + } + + public void launchJob() { + jobLauncher.testMode = true; + jobLauncher.launch( + new File("src/test/resources/conf/jobspec/jobspec_gpus_test.xml")); + } + + @Before + public void createHost() { + RenderHost host = RenderHost.newBuilder() + .setName(HOSTNAME) + .setBootTime(1192369572) + .setFreeMcp(76020) + .setFreeMem((int) CueUtil.GB8) + .setFreeSwap(20760) + .setLoad(0) + .setTotalMcp(195430) + .setTotalMem(CueUtil.GB8) + .setTotalSwap(CueUtil.GB2) + .setNimbyEnabled(false) + .setNumProcs(40) + .setCoresPerProc(100) + .setState(HardwareState.UP) + .setFacility("spi") + .putAttributes("SP_OS", "Linux") + .setNumGpus(8) + .setFreeGpuMem(CueUtil.GB16 * 8) + .setTotalGpuMem(CueUtil.GB16 * 8) + .build(); + + hostManager.createHost(host, + adminManager.findAllocationDetail("spi", "general")); + } + + public DispatchHost getHost() { + return hostManager.findDispatchHost(HOSTNAME); + } + + public void launchAndDeleteJob() { + launchJob(); + + JobDetail job = jobManager.findJobDetail("pipe-default-testuser_test0"); + LayerDetail layer = layerDao.findLayerDetail(job, "layer0"); + jobManager.setJobPaused(job, false); + + DispatchHost host = getHost(); + List procs = dispatcher.dispatchHost(host); + VirtualProc proc = procs.get(0); + + RunningFrameInfo info = RunningFrameInfo.newBuilder() + .setJobId(proc.getJobId()) + .setLayerId(proc.getLayerId()) + .setFrameId(proc.getFrameId()) + .setResourceId(proc.getProcId()) + .build(); + FrameCompleteReport report = FrameCompleteReport.newBuilder() + .setFrame(info) + .setExitStatus(0) + .build(); + frameCompleteHandler.handleFrameCompleteReport(report); + + assertTrue(jobManager.isLayerComplete(layer)); + assertTrue(jobManager.isJobComplete(job)); + + jdbcTemplate.update("DELETE FROM job WHERE pk_job=?", job.getId()); + } + + @Test + @Transactional + @Rollback(true) + public void testEnabled() { + jdbcTemplate.update(DELETE_HISTORY); + assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( + "SELECT COUNT(*) FROM job_history", Integer.class)); + assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( + "SELECT COUNT(*) FROM frame_history", Integer.class)); + + launchAndDeleteJob(); + + assertEquals(Integer.valueOf(3), jdbcTemplate.queryForObject( + "SELECT COUNT(*) FROM job_history", Integer.class)); + assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( + "SELECT COUNT(*) FROM frame_history", Integer.class)); + } + + @Test + @Transactional + @Rollback(true) + public void testDisabled() { + jdbcTemplate.update(DELETE_HISTORY); + jdbcTemplate.update(DISABLE_HISTORY); + + assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( + "SELECT COUNT(*) FROM job_history", Integer.class)); + assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( + "SELECT COUNT(*) FROM frame_history", Integer.class)); + + launchAndDeleteJob(); + + assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( + "SELECT COUNT(*) FROM job_history", Integer.class)); + assertEquals(Integer.valueOf(0), jdbcTemplate.queryForObject( + "SELECT COUNT(*) FROM frame_history", Integer.class)); + } +} + From 686d55e19f9f274addc7ab84b01aa3f3aa5117ff Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Tue, 14 Dec 2021 15:13:50 -0800 Subject: [PATCH 156/277] [Cuebot] Add FIFO scheduling capability (#1060) * Support FIFO scheduling * unittests --- .../imageworks/spcue/dao/DispatcherDao.java | 22 +- .../spcue/dao/postgres/DispatchQuery.java | 16 ++ .../spcue/dao/postgres/DispatcherDaoJdbc.java | 55 ++-- .../spcue/dispatcher/CoreUnitDispatcher.java | 15 +- .../spcue/dispatcher/DispatchSupport.java | 10 +- .../dispatcher/DispatchSupportService.java | 8 +- cuebot/src/main/resources/opencue.properties | 2 + .../dao/postgres/DispatcherDaoFifoTests.java | 254 ++++++++++++++++++ .../test/dao/postgres/DispatcherDaoTests.java | 13 +- 9 files changed, 355 insertions(+), 40 deletions(-) create mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/DispatcherDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/DispatcherDao.java index 8ca8b07e4..a2eee3657 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/DispatcherDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/DispatcherDao.java @@ -82,7 +82,7 @@ public interface DispatcherDao { * @param numJobs * @return */ - Set findDispatchJobsForAllShows(DispatchHost host, int numJobs); + List findDispatchJobsForAllShows(DispatchHost host, int numJobs); /** * Return a list of jobs which could use resources of the specified @@ -92,7 +92,7 @@ public interface DispatcherDao { * @param numJobs * @return */ - Set findDispatchJobs(DispatchHost host, int numJobs); + List findDispatchJobs(DispatchHost host, int numJobs); /** * Return a list of jobs which could use resources of the specified @@ -102,7 +102,7 @@ public interface DispatcherDao { * @param numJobs * @return */ - Set findDispatchJobs(DispatchHost host, GroupInterface g); + List findDispatchJobs(DispatchHost host, GroupInterface g); /** * Finds an under proced job if one exists and returns it, @@ -131,7 +131,7 @@ public interface DispatcherDao { * @param numJobs * @return */ - Set findDispatchJobs(DispatchHost host, ShowInterface show, int numJobs); + List findDispatchJobs(DispatchHost host, ShowInterface show, int numJobs); /** * Find a list of local dispatch jobs. @@ -162,6 +162,20 @@ List findNextDispatchFrames(LayerInterface layer, VirtualProc pro */ List findNextDispatchFrames(LayerInterface layer, DispatchHost host, int limit); + + /** + * Return whether FIFO scheduling is enabled or not in the same priority for unittest. + * + * @return + */ + boolean getFifoSchedulingEnabled(); + + /** + * Set whether FIFO scheduling is enabled or not in the same priority for unittest. + * + * @param fifoSchedulingEnabled + */ + void setFifoSchedulingEnabled(boolean fifoSchedulingEnabled); } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java index fb267ddbd..1f08d2153 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java @@ -112,6 +112,22 @@ public class DispatchQuery { "AND job.pk_folder = ? "); + private static final String replaceQueryForFifo(String query) { + return query + .replace( + "JOBS_BY", + "JOBS_FIFO_BY") + .replace( + "ORDER BY job_resource.int_priority DESC", + "ORDER BY job_resource.int_priority DESC, job.ts_started ASC") + .replace( + "WHERE rank < ?", + "WHERE rank < ? ORDER BY rank"); + } + + public static final String FIND_JOBS_FIFO_BY_SHOW = replaceQueryForFifo(FIND_JOBS_BY_SHOW); + public static final String FIND_JOBS_FIFO_BY_GROUP = replaceQueryForFifo(FIND_JOBS_BY_GROUP); + /** * Dispatch a host in local booking mode. */ diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java index 10f506e9b..d33629f18 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java @@ -20,6 +20,7 @@ import java.sql.ResultSet; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashSet; import java.util.LinkedList; @@ -28,6 +29,8 @@ import java.util.concurrent.ConcurrentHashMap; import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; import org.springframework.jdbc.core.RowMapper; import org.springframework.jdbc.core.support.JdbcDaoSupport; @@ -52,6 +55,8 @@ import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_JOBS_BY_GROUP; import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_JOBS_BY_LOCAL; import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_JOBS_BY_SHOW; +import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_JOBS_FIFO_BY_GROUP; +import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_JOBS_FIFO_BY_SHOW; import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_LOCAL_DISPATCH_FRAME_BY_JOB_AND_HOST; import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_LOCAL_DISPATCH_FRAME_BY_JOB_AND_PROC; import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_LOCAL_DISPATCH_FRAME_BY_LAYER_AND_HOST; @@ -124,6 +129,27 @@ public List getShows() { private final ConcurrentHashMap bookableShows = new ConcurrentHashMap(); + /** + * Whether or not to enable FIFO scheduling in the same priority. + */ + private boolean fifoSchedulingEnabled; + + @Autowired + public DispatcherDaoJdbc(Environment env) { + fifoSchedulingEnabled = env.getProperty( + "dispatcher.fifo_scheduling_enabled", Boolean.class, false); + } + + @Override + public boolean getFifoSchedulingEnabled() { + return fifoSchedulingEnabled; + } + + @Override + public void setFifoSchedulingEnabled(boolean fifoSchedulingEnabled) { + this.fifoSchedulingEnabled = fifoSchedulingEnabled; + } + /** * Returns a sorted list of shows that have pending jobs * which could benefit from the specified allocation. @@ -149,8 +175,8 @@ else if (cached.isExpired()) { return bookableShows.get(key).shows; } - private Set findDispatchJobs(DispatchHost host, int numJobs, boolean shuffleShows) { - LinkedHashSet result = new LinkedHashSet(); + private List findDispatchJobs(DispatchHost host, int numJobs, boolean shuffleShows) { + ArrayList result = new ArrayList(); List shows = new LinkedList(getBookableShows(host)); // shows were sorted. If we want it in random sequence, we need to shuffle it. if (shuffleShows) { @@ -185,7 +211,7 @@ private Set findDispatchJobs(DispatchHost host, int numJobs, boolean shu } result.addAll(getJdbcTemplate().query( - FIND_JOBS_BY_SHOW, + fifoSchedulingEnabled ? FIND_JOBS_FIFO_BY_SHOW : FIND_JOBS_BY_SHOW, PKJOB_MAPPER, s.getShowId(), host.getFacilityId(), host.os, host.idleCores, host.idleMemory, @@ -208,27 +234,26 @@ private Set findDispatchJobs(DispatchHost host, int numJobs, boolean shu } @Override - public Set findDispatchJobsForAllShows(DispatchHost host, int numJobs) { + public List findDispatchJobsForAllShows(DispatchHost host, int numJobs) { return findDispatchJobs(host, numJobs, true); } @Override - public Set findDispatchJobs(DispatchHost host, int numJobs) { + public List findDispatchJobs(DispatchHost host, int numJobs) { return findDispatchJobs(host, numJobs, false); } @Override - public Set findDispatchJobs(DispatchHost host, GroupInterface g) { - LinkedHashSet result = new LinkedHashSet(5); - result.addAll(getJdbcTemplate().query( - FIND_JOBS_BY_GROUP, + public List findDispatchJobs(DispatchHost host, GroupInterface g) { + List result = getJdbcTemplate().query( + fifoSchedulingEnabled ? FIND_JOBS_FIFO_BY_GROUP : FIND_JOBS_BY_GROUP, PKJOB_MAPPER, g.getGroupId(),host.getFacilityId(), host.os, host.idleCores, host.idleMemory, threadMode(host.threadMode), host.idleGpus, (host.idleGpuMemory > 0) ? 1 : 0, host.idleGpuMemory, - host.getName(), 50)); + host.getName(), 50); return result; } @@ -378,19 +403,17 @@ public boolean higherPriorityJobExists(JobDetail baseJob, VirtualProc proc) { } @Override - public Set findDispatchJobs(DispatchHost host, + public List findDispatchJobs(DispatchHost host, ShowInterface show, int numJobs) { - LinkedHashSet result = new LinkedHashSet(numJobs); - - result.addAll(getJdbcTemplate().query( - FIND_JOBS_BY_SHOW, + List result = getJdbcTemplate().query( + fifoSchedulingEnabled ? FIND_JOBS_FIFO_BY_SHOW : FIND_JOBS_BY_SHOW, PKJOB_MAPPER, show.getShowId(), host.getFacilityId(), host.os, host.idleCores, host.idleMemory, threadMode(host.threadMode), host.idleGpus, (host.idleGpuMemory > 0) ? 1 : 0, host.idleGpuMemory, - host.getName(), numJobs * 10)); + host.getName(), numJobs * 10); return result; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java index beacefd97..7da8faf1d 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Set; import java.util.concurrent.TimeUnit; import com.google.common.cache.Cache; @@ -126,7 +125,7 @@ private Cache getOrCreateJobLock() { } - private List dispatchJobs(DispatchHost host, Set jobs) { + private List dispatchJobs(DispatchHost host, List jobs) { List procs = new ArrayList(); try { @@ -170,8 +169,8 @@ private List dispatchJobs(DispatchHost host, Set jobs) { return procs; } - private Set getGpuJobs(DispatchHost host, ShowInterface show) { - Set jobs = null; + private List getGpuJobs(DispatchHost host, ShowInterface show) { + List jobs = null; // TODO: GPU: make index with the 4 components instead of just 3, replace the just 3 @@ -200,7 +199,7 @@ private Set getGpuJobs(DispatchHost host, ShowInterface show) { @Override public List dispatchHostToAllShows(DispatchHost host) { - Set jobs = dispatchSupport.findDispatchJobsForAllShows( + List jobs = dispatchSupport.findDispatchJobsForAllShows( host, getIntProperty("dispatcher.job_query_max")); @@ -210,7 +209,7 @@ public List dispatchHostToAllShows(DispatchHost host) { @Override public List dispatchHost(DispatchHost host) { - Set jobs = getGpuJobs(host, null); + List jobs = getGpuJobs(host, null); if (jobs == null) jobs = dispatchSupport.findDispatchJobs(host, getIntProperty("dispatcher.job_query_max")); @@ -221,7 +220,7 @@ public List dispatchHost(DispatchHost host) { @Override public List dispatchHost(DispatchHost host, ShowInterface show) { - Set jobs = getGpuJobs(host, show); + List jobs = getGpuJobs(host, show); if (jobs == null) jobs = dispatchSupport.findDispatchJobs(host, show, @@ -233,7 +232,7 @@ public List dispatchHost(DispatchHost host, ShowInterface show) { @Override public List dispatchHost(DispatchHost host, GroupInterface group) { - Set jobs = getGpuJobs(host, null); + List jobs = getGpuJobs(host, null); if (jobs == null) jobs = dispatchSupport.findDispatchJobs(host, group); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java index bff3dd6af..ce261535c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java @@ -306,7 +306,7 @@ List findNextDispatchFrames(LayerInterface layer, VirtualProc pro * @param host * @return */ - Set findDispatchJobsForAllShows(DispatchHost host, int numJobs); + List findDispatchJobsForAllShows(DispatchHost host, int numJobs); /** * Returns the highest priority job that can utilize @@ -315,7 +315,7 @@ List findNextDispatchFrames(LayerInterface layer, VirtualProc pro * @param host * @return */ - Set findDispatchJobs(DispatchHost host, int numJobs); + List findDispatchJobs(DispatchHost host, int numJobs); /** * Returns the highest priority jobs that can utilize @@ -324,7 +324,7 @@ List findNextDispatchFrames(LayerInterface layer, VirtualProc pro * @param host * @return A set of unique job ids. */ - Set findDispatchJobs(DispatchHost host, GroupInterface p); + List findDispatchJobs(DispatchHost host, GroupInterface p); /** * @@ -523,14 +523,14 @@ void updateProcMemoryUsage(FrameInterface frame, long rss, long maxRss, long vsi void determineIdleCores(DispatchHost host, int load); /** - * Return a set of job IDs that can take the given host. + * Return a list of job IDs that can take the given host. * * @param host * @param show * @param numJobs * @return */ - Set findDispatchJobs(DispatchHost host, ShowInterface show, int numJobs); + List findDispatchJobs(DispatchHost host, ShowInterface show, int numJobs); /** * Return true of the job has pending frames. diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java index 4a871803e..fa34e7100 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java @@ -148,17 +148,17 @@ public boolean higherPriorityJobExists(JobDetail baseJob, VirtualProc proc) { } @Transactional(readOnly = true) - public Set findDispatchJobsForAllShows(DispatchHost host, int numJobs) { + public List findDispatchJobsForAllShows(DispatchHost host, int numJobs) { return dispatcherDao.findDispatchJobsForAllShows(host, numJobs); } @Transactional(readOnly = true) - public Set findDispatchJobs(DispatchHost host, int numJobs) { + public List findDispatchJobs(DispatchHost host, int numJobs) { return dispatcherDao.findDispatchJobs(host, numJobs); } @Transactional(readOnly = true) - public Set findDispatchJobs(DispatchHost host, GroupInterface g) { + public List findDispatchJobs(DispatchHost host, GroupInterface g) { return dispatcherDao.findDispatchJobs(host, g); } @@ -170,7 +170,7 @@ public Set findLocalDispatchJobs(DispatchHost host) { @Override @Transactional(readOnly = true) - public Set findDispatchJobs(DispatchHost host, ShowInterface show, + public List findDispatchJobs(DispatchHost host, ShowInterface show, int numJobs) { return dispatcherDao.findDispatchJobs(host, show, numJobs); } diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index 714d744a0..89db91c94 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -43,6 +43,8 @@ dispatcher.frame_query_max=20 dispatcher.job_frame_dispatch_max=8 # Maximum number of frames to dispatch from a host at one time. dispatcher.host_frame_dispatch_max=12 +# Whether or not to enable FIFO scheduling in the same priority. +dispatcher.fifo_scheduling_enabled=false # Number of threads to keep in the pool for launching job. dispatcher.launch_queue.core_pool_size=1 diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java new file mode 100644 index 000000000..a428cb3ab --- /dev/null +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java @@ -0,0 +1,254 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.imageworks.spcue.test.dao.postgres; + +import java.io.File; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import javax.annotation.Resource; + +import org.jdom.Document; +import org.jdom.Element; +import org.jdom.input.SAXBuilder; +import org.jdom.output.XMLOutputter; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.annotation.Rollback; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; +import org.springframework.test.context.support.AnnotationConfigContextLoader; +import org.springframework.transaction.annotation.Transactional; + +import com.imageworks.spcue.DispatchHost; +import com.imageworks.spcue.JobDetail; +import com.imageworks.spcue.config.TestAppConfig; +import com.imageworks.spcue.dao.DispatcherDao; +import com.imageworks.spcue.dao.HostDao; +import com.imageworks.spcue.dispatcher.Dispatcher; +import com.imageworks.spcue.grpc.host.HardwareState; +import com.imageworks.spcue.grpc.report.RenderHost; +import com.imageworks.spcue.service.AdminManager; +import com.imageworks.spcue.service.GroupManager; +import com.imageworks.spcue.service.HostManager; +import com.imageworks.spcue.service.JobLauncher; +import com.imageworks.spcue.service.JobManager; +import com.imageworks.spcue.test.AssumingPostgresEngine; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +@Transactional +@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) +public class DispatcherDaoFifoTests extends AbstractTransactionalJUnit4SpringContextTests { + + @Autowired + @Rule + public AssumingPostgresEngine assumingPostgresEngine; + + @Resource + DispatcherDao dispatcherDao; + + @Resource + HostDao hostDao; + + @Resource + JobManager jobManager; + + @Resource + HostManager hostManager; + + @Resource + AdminManager adminManager; + + @Resource + GroupManager groupManager; + + @Resource + Dispatcher dispatcher; + + @Resource + JobLauncher jobLauncher; + + private static final String HOSTNAME="beta"; + + public DispatchHost getHost() { + return hostDao.findDispatchHost(HOSTNAME); + } + + private void launchJobs(int count) throws Exception { + Document docTemplate = new SAXBuilder(true).build( + new File("src/test/resources/conf/jobspec/jobspec_simple.xml")); + docTemplate.getDocType().setSystemID("http://localhost:8080/spcue/dtd/cjsl-1.12.dtd"); + Element root = docTemplate.getRootElement(); + Element jobTemplate = root.getChild("job"); + Element depends = root.getChild("depends"); + assertEquals(jobTemplate.getAttributeValue("name"), "test"); + root.removeContent(jobTemplate); + root.removeContent(depends); + + long t = System.currentTimeMillis(); + for (int i = 0; i < count; i++) { + Document doc = (Document) docTemplate.clone(); + root = doc.getRootElement(); + Element job = (Element) jobTemplate.clone(); + job.setAttribute("name", "job" + i); + root.addContent(job); + root.addContent((Element) depends.clone()); + jobLauncher.launch(new XMLOutputter().outputString(doc)); + + // Force to set incremental ts_started to the jobs + // because current_timestamp is not updated during test. + jdbcTemplate.update("UPDATE job SET ts_started = ? WHERE str_name = ?", + new Timestamp(t + i), "pipe-default-testuser_job" + i); + } + } + + @Before + public void launchJob() { + dispatcherDao.setFifoSchedulingEnabled(true); + + dispatcher.setTestMode(true); + jobLauncher.testMode = true; + } + + @After + public void resetFifoScheduling() { + dispatcherDao.setFifoSchedulingEnabled(false); + } + + @Before + public void createHost() { + RenderHost host = RenderHost.newBuilder() + .setName(HOSTNAME) + .setBootTime(1192369572) + .setFreeMcp(76020) + .setFreeMem(53500) + .setFreeSwap(20760) + .setLoad(1) + .setTotalMcp(195430) + .setTotalMem(8173264) + .setTotalSwap(20960) + .setNimbyEnabled(false) + .setNumProcs(2) + .setCoresPerProc(100) + .addTags("test") + .setState(HardwareState.UP) + .setFacility("spi") + .putAttributes("SP_OS", "Linux") + .build(); + + hostManager.createHost(host, + adminManager.findAllocationDetail("spi", "general")); + } + + @Test + @Transactional + @Rollback(true) + public void testFifoSchedulingEnabled() { + assertTrue(dispatcherDao.getFifoSchedulingEnabled()); + dispatcherDao.setFifoSchedulingEnabled(false); + assertFalse(dispatcherDao.getFifoSchedulingEnabled()); + dispatcherDao.setFifoSchedulingEnabled(true); + assertTrue(dispatcherDao.getFifoSchedulingEnabled()); + } + + @Test + @Transactional + @Rollback(true) + public void testAllSorted() throws Exception { + int count = 10; + launchJobs(count); + + List jobs = dispatcherDao.findDispatchJobs(getHost(), count); + assertEquals(count, jobs.size()); + for (int i = 0; i < count; i++) { + assertEquals("pipe-default-testuser_job" + i, + jobManager.getJob(jobs.get(i)).getName()); + } + } + + @Test + @Transactional + @Rollback(true) + public void testPortionSorted() throws Exception { + int count = 100; + launchJobs(count); + + int portion = 19; + List jobs = dispatcherDao.findDispatchJobs(getHost(), (portion + 1) / 10); + assertEquals(portion, jobs.size()); + for (int i = 0; i < portion; i++) { + assertEquals("pipe-default-testuser_job" + i, + jobManager.getJob(jobs.get(i)).getName()); + } + } + + @Test + @Transactional + @Rollback(true) + public void testFifoSchedulingDisabled() throws Exception { + dispatcherDao.setFifoSchedulingEnabled(false); + assertFalse(dispatcherDao.getFifoSchedulingEnabled()); + + int count = 10; + launchJobs(count); + + List jobs = dispatcherDao.findDispatchJobs(getHost(), count); + assertEquals(count, jobs.size()); + + List sortedJobs = new ArrayList(jobs); + Collections.sort(sortedJobs, + Comparator.comparing(jobId -> jobManager.getJob(jobId).getName())); + assertNotEquals(jobs, sortedJobs); + + for (int i = 0; i < count; i++) { + assertEquals("pipe-default-testuser_job" + i, + jobManager.getJob(sortedJobs.get(i)).getName()); + } + } + + @Test + @Transactional + @Rollback(true) + public void testGroup() throws Exception { + int count = 10; + launchJobs(count); + + JobDetail job = jobManager.findJobDetail("pipe-default-testuser_job0"); + assertNotNull(job); + + List jobs = dispatcherDao.findDispatchJobs(getHost(), + groupManager.getGroupDetail(job)); + assertEquals(count, jobs.size()); + for (int i = 0; i < count; i++) { + assertEquals("pipe-default-testuser_job" + i, + jobManager.getJob(jobs.get(i)).getName()); + } + } +} diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java index a400bb26a..0a01478a4 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java @@ -327,7 +327,7 @@ public void testFindDispatchJobs() { assertTrue(jdbcTemplate.queryForObject( "SELECT COUNT(*) FROM job WHERE str_state='PENDING'", Integer.class) > 0); - Set jobs = dispatcherDao.findDispatchJobs(host, 10); + List jobs = dispatcherDao.findDispatchJobs(host, 10); assertTrue(jobs.size() > 0); } @@ -341,7 +341,7 @@ public void testFindDispatchJobsByGroup() { assertNotNull(job); assertNotNull(job.groupId); - Set jobs = dispatcherDao.findDispatchJobs(host, + List jobs = dispatcherDao.findDispatchJobs(host, groupManager.getGroupDetail(job)); assertTrue(jobs.size() > 0); } @@ -354,7 +354,7 @@ public void testFindDispatchJobsByShow() { final JobDetail job = getJob1(); assertNotNull(job); - Set jobs = dispatcherDao.findDispatchJobs(host, + List jobs = dispatcherDao.findDispatchJobs(host, adminManager.findShowEntity("pipe"), 5); assertTrue(jobs.size() > 0); } @@ -517,4 +517,11 @@ public void testHigherPriorityJobExistsMaxProcBound() { boolean isHigher = dispatcherDao.higherPriorityJobExists(job1, proc); assertFalse(isHigher); } + + @Test + @Transactional + @Rollback(true) + public void testFifoSchedulingEnabled() { + assertFalse(dispatcherDao.getFifoSchedulingEnabled()); + } } From aed5e8b07a5c37061c43bb6dc4f0706cfd90f96d Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Tue, 14 Dec 2021 16:06:10 -0800 Subject: [PATCH 157/277] [Cuebot][SQL] Create limit index (#1057) * Add limix index * Fix unittest for limit index * Bump up version --- .../migrations/V12__Add_limit_index.sql | 4 ++++ .../test/dao/postgres/LayerDaoTests.java | 21 +++++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V12__Add_limit_index.sql diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V12__Add_limit_index.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V12__Add_limit_index.sql new file mode 100644 index 000000000..a6ec10696 --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V12__Add_limit_index.sql @@ -0,0 +1,4 @@ +-- Add limit index +CREATE INDEX i_layer_limit_pk_layer ON layer_limit (pk_layer); +CREATE INDEX i_layer_limit_pk_limit_record ON layer_limit (pk_limit_record); +CREATE INDEX i_limit_record_pk_limit_record ON limit_record (pk_limit_record); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java index 06864a9bc..8ad2f5bac 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java @@ -20,6 +20,8 @@ package com.imageworks.spcue.test.dao.postgres; import java.io.File; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -715,10 +717,21 @@ public void testAddLimit() { LayerInterface layerResult = layerDao.getLayer(layer.getLayerId()); List limits = layerDao.getLimits(layerResult); assertEquals(limits.size(), 4); - assertEquals(limits.get(0).id, getTestLimitId(LIMIT_NAME)); - assertEquals(limits.get(1).id, getTestLimitId(LIMIT_TEST_A)); - assertEquals(limits.get(2).id, getTestLimitId(LIMIT_TEST_B)); - assertEquals(limits.get(3).id, getTestLimitId(LIMIT_TEST_C)); + List sourceIds = Arrays.asList( + getTestLimitId(LIMIT_NAME), + getTestLimitId(LIMIT_TEST_A), + getTestLimitId(LIMIT_TEST_B), + getTestLimitId(LIMIT_TEST_C) + ); + List resultIds = Arrays.asList( + limits.get(0).id, + limits.get(1).id, + limits.get(2).id, + limits.get(3).id + ); + Collections.sort(sourceIds); + Collections.sort(resultIds); + assertEquals(sourceIds, resultIds); } @Test From 607ff5f3cfd52d748cdf26820e9cfb433be3c479 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 5 Jan 2022 13:40:05 -0500 Subject: [PATCH 158/277] Check in notes from last 3 TSC meetings. (#1083) --- tsc/meetings/2021-09-15.md | 99 +++++++++++++++++++++++++++++++++++ tsc/meetings/2021-09-29.md | 96 ++++++++++++++++++++++++++++++++++ tsc/meetings/2021-12-08.md | 103 +++++++++++++++++++++++++++++++++++++ 3 files changed, 298 insertions(+) create mode 100644 tsc/meetings/2021-09-15.md create mode 100644 tsc/meetings/2021-09-29.md create mode 100644 tsc/meetings/2021-12-08.md diff --git a/tsc/meetings/2021-09-15.md b/tsc/meetings/2021-09-15.md new file mode 100644 index 000000000..be57b6e10 --- /dev/null +++ b/tsc/meetings/2021-09-15.md @@ -0,0 +1,99 @@ +# OpenCue TSC Meeting Notes 15 September 2021 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Completed 2021 Goals + * User survey + * ASWF graduation + * Drop Oracle support + * Rename demo_data.sql + * Expanded GPU support +* Todo 2021 Goals + * Open Source Days 2021 + * Any takeaways/followups? + * New user UX + * Publish PyPI packages + * Design + doc: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/ + * Proposal for config file + standardization: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/edit#heading=h.v85uo9klwqft + * Config cleanup for pycue done and ready for review. Tested on macOS and Windows. + * Next up will do config cleanup on other components. + * Fix cuebot:latest tag for sandbox setup + * Proposal: publish Docker images for every new patch version (i.e. every new commit to + master) + * Proposal accepted. + * No progress implementing this yet. + * Related: for PyPI design we may want to also publish packages for every new patch + version. + * Docs refresh + * User guide refresh + * Split into "install from latest release" and "install from master" + * Needed to assist transition to publish Docker images on all commits + * No progress. + * API reference + * We have the ability to generate HTML from any commit, but we need a new CI pipeline to + build and publish this on opencue.io for new releases. + * Proposal: for now, highlight existing docs for building reference locally + * Proposal accepted. + * Brian: cleaned up local build process, sent email with doc + link: https://www.opencue.io/contributing/opencue/build-docs/ + * pyoutline examples + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/177 + * No progress yet. + * User guide for CueGUI + * No progress. + * Configuration guide for client-side python + * In progress, will be done as part of PyPI work. + * Configuration guide for Cuebot + * In progress, will be done as part of PyPI work. + * Configuration guide for RQD + * In progress, will be done as part of PyPI work. + * Improve scheduler logic + * Diego has volunteered to own this. + * Email reply from Matt, looped in others. + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1001 + * Expand DCC plugins + * No progress. + * Logging solution + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/880 + * PR approved and merged. + * Proposal: write blog post on this topic + * Proposal accepted + * Tutorial is now published, presented at OSD 2021. + * CSP Terraform docs + * No progress yet. + * Auto-close Github issues + * Proposal: + * Any issues in a Project have been identified as important and will be ignored. + * Other issues: + * If no update for 60 days, will get the label "stale" and a comment notifying of + auto-close. + * If no further update for 7 days, will be closed. + * Any issue update will rest the timer. + * Action https://github.com/marketplace/actions/close-stale-issues will be used. + * May need another action to label issues if they are in a project, the above action doesn't + appear to have options for ignoring issues based on project membership. + * Expanded Cloud functionality + * Finalize GSoC plugin + * Test Azure support + * Add AWS support + * Design expanded functionality e.g. autoscaling for work in the queue. + * Last update from Greg: started to test / clean up azure support. Looking good so far. +* Other ongoing work + * Brian + * Python 3.8, 3.9 support + * VFX ref platform CY2021 how in CI pipeline + * Ready for CY2022 now that Python 3.9 is supported + * Published tutorial on Grafana integration + * CueGUI memory leak + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1025 + * Scheduling fixes + * Add thread pool properties https://github.com/AcademySoftwareFoundation/OpenCue/pull/1008 + * Schedule dispatch frame https://github.com/AcademySoftwareFoundation/OpenCue/pull/1012 + * Determine idle cores + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1021 + * Delete CORE_POINTS_RESERVED_MAX logic? + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1026 diff --git a/tsc/meetings/2021-09-29.md b/tsc/meetings/2021-09-29.md new file mode 100644 index 000000000..29a3be9d6 --- /dev/null +++ b/tsc/meetings/2021-09-29.md @@ -0,0 +1,96 @@ +# OpenCue TSC Meeting Notes 29 September 2021 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Completed 2021 Goals + * User survey + * ASWF graduation + * Drop Oracle support + * Rename demo_data.sql + * Expanded GPU support + * Open Source Days 2021 + * Logging/monitoring solution +* Ongoing work + * CueGUI memory leak + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1025 + * Consistent issue + * MonitorCue refresh tweak, reduce API calls + * Collecting feedback on this now + * Custom garbage collection + * Haven't tested yet + * Scheduling fixes + * Add thread pool properties https://github.com/AcademySoftwareFoundation/OpenCue/pull/1008 + * Schedule dispatch frame https://github.com/AcademySoftwareFoundation/OpenCue/pull/1012 + * SPI version https://github.com/AcademySoftwareFoundation/OpenCue/pull/1035 + * Tested >5 months + * User permissions + * default disable job options for jobs not owned by the current user + * let's start designing permission system, will make a github post to start gathering + requirements +* Todo 2021 Goals + * New user UX + * Publish PyPI packages + * Design + doc: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/ + * Proposal for config file + standardization: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/edit#heading=h.v85uo9klwqft + * Config cleanup for pycue done and ready for review. Tested on macOS and Windows. + * Next up will do config cleanup on other components. + * pyoutline, rqd up next + * Fix cuebot:latest tag for sandbox setup + * Proposal: publish Docker images for every new patch version (i.e. every new commit to + master) + * Proposal accepted. + * No progress implementing this yet. + * Related: for PyPI design we may want to also publish packages for every new patch + version. + * Docs refresh + * User guide refresh + * Split into "install from latest release" and "install from master" + * Needed to assist transition to publish Docker images on all commits + * No progress. + * API reference + * We have the ability to generate HTML from any commit, but we need a new CI pipeline to + build and publish this on opencue.io for new releases. + * Proposal: for now, highlight existing docs for building reference locally + * Proposal accepted. + * Brian: cleaned up local build process, sent email with doc + link: https://www.opencue.io/contributing/opencue/build-docs/ + * pyoutline examples + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/177 + * No progress yet. + * User guide for CueGUI + * No progress. + * Configuration guide for client-side python + * In progress, will be done as part of PyPI work. + * Configuration guide for Cuebot + * In progress, will be done as part of PyPI work. + * Configuration guide for RQD + * In progress, will be done as part of PyPI work. + * Improve scheduler logic + * Diego has volunteered to own this. + * Email reply from Matt, looped in others. + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1001 + * Expand DCC plugins + * No progress. + * CSP Terraform docs + * No progress yet. + * Auto-close Github issues + * Proposal: + * Any issues in a Project have been identified as important and will be ignored. + * Other issues: + * If no update for 60 days, will get the label "stale" and a comment notifying of + auto-close. + * If no further update for 7 days, will be closed. + * Any issue update will rest the timer. + * Action https://github.com/marketplace/actions/close-stale-issues will be used. + * May need another action to label issues if they are in a project, the above action doesn't + appear to have options for ignoring issues based on project membership. + * Expanded Cloud functionality + * Finalize GSoC plugin + * Test Azure support + * Add AWS support + * Design expanded functionality e.g. autoscaling for work in the queue. + * Last update from Greg: started to test / clean up azure support. Looking good so far. diff --git a/tsc/meetings/2021-12-08.md b/tsc/meetings/2021-12-08.md new file mode 100644 index 000000000..30528ec63 --- /dev/null +++ b/tsc/meetings/2021-12-08.md @@ -0,0 +1,103 @@ +# OpenCue TSC Meeting Notes 8 December 2021 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Completed 2021 Goals + * User survey + * ASWF graduation + * Drop Oracle support + * Rename demo_data.sql + * Expanded GPU support + * Open Source Days 2021 + * Logging/monitoring solution +* Ongoing work + * CueGUI memory leak + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1025 + * SPI: still researching and testing their fix. + * Scheduling fixes + * Thread pool properties + * Two versions: + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1008 + * SPI version https://github.com/AcademySoftwareFoundation/OpenCue/pull/1035 + * PRs need review. + * Let's start with 1035, then merge 1008 to fix conflicts to see what in 1008 is still + useful, if anything. + * Schedule dispatch frame + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1012 + * Adds database fields to eliminate multiple cuebots handing out the same frames. + * Might not want to merge while larger scheduler redesign is going on. + * RQD change, adding child proc info into log, storing in database + * For understanding memory usage of complex jobs +* Todo 2021 Goals + * New user UX + * Publish PyPI packages + * Design + doc: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/ + * Proposal for config file + standardization: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/edit#heading=h.v85uo9klwqft + * Config cleanup for pycue done, tested on macOS and Windows, merged. + * pyoutline, cuesubmit config cleanup in review. + * Fix cuebot:latest tag for sandbox setup + * Proposal: publish Docker images for every new patch version (i.e. every new commit to + master) + * Proposal accepted. + * No progress implementing this yet. + * Related: for PyPI design we may want to also publish packages for every new patch + version. + * Docs refresh + * User guide refresh + * Split into "install from latest release" and "install from master" + * Needed to assist transition to publish Docker images on all commits + * No progress. + * API reference + * We have the ability to generate HTML from any commit, but we need a new CI pipeline to + build and publish this on opencue.io for new releases. + * Proposal: for now, highlight existing docs for building reference locally + * Proposal accepted. + * Brian: cleaned up local build process, sent email with doc + link: https://www.opencue.io/contributing/opencue/build-docs/ + * pyoutline examples + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/177 + * No progress yet. + * User guide for CueGUI + * No progress. + * Configuration guide for client-side python + * In progress, will be done as part of PyPI work. + * Configuration guide for Cuebot + * In progress, will be done as part of PyPI work. + * Configuration guide for RQD + * In progress, will be done as part of PyPI work. + * Improve scheduler logic + * Diego has volunteered to own this. + * Email reply from Matt, looped in others. + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1001 + * Other PRs with improvements — see ongoing work above. + * Some feedback on discussion thread that needs review. + * SPI experimenting with redis to handle RQD host reports, but may make a good candidate for + the scheduler as well. PR with some initial changes likely around end of year. + * Expand DCC plugins + * No progress. + * CSP Terraform docs + * No progress yet. + * Auto-close Github issues + * Proposal: + * Any issues in a Project have been identified as important and will be ignored. + * Other issues: + * If no update for 60 days, will get the label "stale" and a comment notifying of + auto-close. + * If no further update for 7 days, will be closed. + * Any issue update will rest the timer. + * Action https://github.com/marketplace/actions/close-stale-issues will be used. + * May need another action to label issues if they are in a project, the above action doesn't + appear to have options for ignoring issues based on project membership. + * Expanded Cloud functionality + * Finalize GSoC plugin + * Test Azure support + * Add AWS support + * Design expanded functionality e.g. autoscaling for work in the queue. + * Last update from Greg: started to test / clean up azure support. Looking good so far. + * Let's send PR to add Kazuki (splhack) as committer + * Brian needs to catch up on importing meeting notes into the Github repo. + * Next meeting Dec 22, should probably cancel and hold next meeting Jan 5. From 7802947b3c8dd756cca8c35cf0af1ef1a63f2e5f Mon Sep 17 00:00:00 2001 From: Anti-Distinctlyminty Date: Fri, 7 Jan 2022 20:20:39 +0100 Subject: [PATCH 159/277] Changed psutil to 5.6.7 due to critical compilation error on Windows. (#1085) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cac91168b..ff5a63761 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ mock==2.0.0 packaging==20.9 pathlib==1.0.1;python_version<"3.4" protobuf==3.17.3;python_version<"3.0" -psutil==5.6.6 +psutil==5.6.7 pyfakefs==3.6 pylint==2.6.0;python_version>="3.7" PyYAML==5.1 From e8b71ba1d5682fa74727bd4a4e8f4efdfabb9d07 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 17 Jan 2022 14:23:33 -0500 Subject: [PATCH 160/277] Add new committer and clean up file. (#1084) --- CODEOWNERS | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index ff91076cd..09f526fc5 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,2 +1 @@ -* @bcipriano @gregdenton @jrray @smith1511 @larsbijl @DiegoTavares @IdrisMiles -/tsc/gsoc @bcipriano @gregdenton @shiva-kannan +* @bcipriano @gregdenton @jrray @smith1511 @larsbijl @DiegoTavares @IdrisMiles @splhack From 4200dbb38851ddcaf1346400ac989e3288695e97 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Mon, 17 Jan 2022 11:34:50 -0800 Subject: [PATCH 161/277] [cuebot] Fix Group GPU APIs. (#1064) --- .../spcue/dao/postgres/GroupDaoJdbc.java | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java index b18a67c93..948020f9f 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java @@ -235,11 +235,6 @@ public boolean isOverMinCores(JobInterface job) { @Override public void updateDefaultJobMaxGpus(GroupInterface group, int value) { if (value <= 0) { value = CueUtil.FEATURE_DISABLED; } - if (value < CueUtil.ONE_CORE && value != CueUtil.FEATURE_DISABLED) { - String msg = "The default max cores for a job must " + - "be greater than a single core"; - throw new IllegalArgumentException(msg); - } getJdbcTemplate().update( "UPDATE folder SET int_job_max_gpus=? WHERE pk_folder=?", value, group.getId()); @@ -248,11 +243,6 @@ public void updateDefaultJobMaxGpus(GroupInterface group, int value) { @Override public void updateDefaultJobMinGpus(GroupInterface group, int value) { if (value <= 0) { value = CueUtil.FEATURE_DISABLED; } - if (value < CueUtil.ONE_CORE && value != CueUtil.FEATURE_DISABLED) { - String msg = "The default min cores for a job must " + - "be greater than a single core"; - throw new IllegalArgumentException(msg); - } getJdbcTemplate().update( "UPDATE folder SET int_job_min_gpus=? WHERE pk_folder=?", value, group.getId()); @@ -261,11 +251,6 @@ public void updateDefaultJobMinGpus(GroupInterface group, int value) { @Override public void updateMaxGpus(GroupInterface group, int value) { if (value < 0) { value = CueUtil.FEATURE_DISABLED; } - if (value < CueUtil.ONE_CORE && value != CueUtil.FEATURE_DISABLED) { - String msg = "The group max cores feature must " + - "be a whole core or greater, pass in: " + value; - throw new IllegalArgumentException(msg); - } getJdbcTemplate().update( "UPDATE folder_resource SET int_max_gpus=? WHERE pk_folder=?", @@ -452,6 +437,10 @@ public GroupDetail mapRow(ResultSet rs, int rowNum) throws SQLException { group.jobMaxGpus = rs.getInt("int_job_max_gpus"); group.jobMinGpus = rs.getInt("int_job_min_gpus"); group.jobPriority = rs.getInt("int_job_priority"); + group.minCores = rs.getInt("int_min_cores"); + group.maxCores = rs.getInt("int_max_cores"); + group.minGpus = rs.getInt("int_min_gpus"); + group.maxGpus = rs.getInt("int_max_gpus"); group.name = rs.getString("str_name"); group.parentId = rs.getString("pk_parent_folder"); group.showId = rs.getString("pk_show"); From 9e03b9572d6389a4f3a37a8fb16c7fa340c0cae8 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Mon, 17 Jan 2022 11:36:13 -0800 Subject: [PATCH 162/277] Add Job.shutdownIfCompleted API method. (#1033) --- .../DispatchShutdownJobIfCompleted.java | 51 +++++++++++++++++++ .../imageworks/spcue/servant/ManageJob.java | 19 +++++++ .../spcue/service/JobManagerSupport.java | 4 ++ proto/job.proto | 11 ++++ pycue/opencue/wrappers/job.py | 5 ++ 5 files changed, 90 insertions(+) create mode 100644 cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchShutdownJobIfCompleted.java diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchShutdownJobIfCompleted.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchShutdownJobIfCompleted.java new file mode 100644 index 000000000..15ddc805e --- /dev/null +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchShutdownJobIfCompleted.java @@ -0,0 +1,51 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.imageworks.spcue.dispatcher.commands; + +import com.imageworks.spcue.JobInterface; +import com.imageworks.spcue.Source; +import com.imageworks.spcue.service.JobManagerSupport; + +/** + * A command for shutting down a job if it is completed. + * This is a workaround for when Cuebot failed to shutdown a job due to database access error. + * + * @category command + */ +public class DispatchShutdownJobIfCompleted implements Runnable { + private JobInterface job; + + private JobManagerSupport jobManagerSupport; + public DispatchShutdownJobIfCompleted(JobInterface job, JobManagerSupport jobManagerSupport) { + this.job = job; + this.jobManagerSupport = jobManagerSupport; + } + + public void run() { + new DispatchCommandTemplate() { + public void wrapDispatchCommand() { + if (jobManagerSupport.isJobComplete(job)) { + jobManagerSupport.shutdownJob(job, new Source("natural"), false); + } + } + }.execute(); + } +} + diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java index c13177a74..f7c765636 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java @@ -49,6 +49,7 @@ import com.imageworks.spcue.dispatcher.commands.DispatchReorderFrames; import com.imageworks.spcue.dispatcher.commands.DispatchRetryFrames; import com.imageworks.spcue.dispatcher.commands.DispatchSatisfyDepends; +import com.imageworks.spcue.dispatcher.commands.DispatchShutdownJobIfCompleted; import com.imageworks.spcue.dispatcher.commands.DispatchStaggerFrames; import com.imageworks.spcue.grpc.comment.Comment; import com.imageworks.spcue.grpc.job.FrameSeq; @@ -134,6 +135,8 @@ import com.imageworks.spcue.grpc.job.JobSetMinGpusResponse; import com.imageworks.spcue.grpc.job.JobSetPriorityRequest; import com.imageworks.spcue.grpc.job.JobSetPriorityResponse; +import com.imageworks.spcue.grpc.job.JobShutdownIfCompletedRequest; +import com.imageworks.spcue.grpc.job.JobShutdownIfCompletedResponse; import com.imageworks.spcue.grpc.job.JobStaggerFramesRequest; import com.imageworks.spcue.grpc.job.JobStaggerFramesResponse; import com.imageworks.spcue.grpc.job.LayerSeq; @@ -780,6 +783,22 @@ public void reorderFrames(JobReorderFramesRequest request, } } + @Override + public void shutdownIfCompleted(JobShutdownIfCompletedRequest request, + StreamObserver responseObserver) { + try { + setupJobData(request.getJob()); + manageQueue.execute(new DispatchShutdownJobIfCompleted(job, jobManagerSupport)); + responseObserver.onNext(JobShutdownIfCompletedResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find job data") + .asRuntimeException()); + } + } + @Override public void staggerFrames(JobStaggerFramesRequest request, StreamObserver responseObserver) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerSupport.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerSupport.java index e6c03c221..c73c30eb6 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerSupport.java @@ -212,6 +212,10 @@ public void satisfyWhatDependsOn(FrameSearchInterface request) { } } + public boolean isJobComplete(JobInterface job) { + return jobManager.isJobComplete(job); + } + /* * Destructive functions require a extra Source argument which contains * information about the user making the call. This information is diff --git a/proto/job.proto b/proto/job.proto index 240fba609..a07a93619 100644 --- a/proto/job.proto +++ b/proto/job.proto @@ -269,6 +269,10 @@ service JobInterface { // Sets the job priority rpc SetPriority(JobSetPriorityRequest) returns (JobSetPriorityResponse); + // Shutdown the job if it is completed. This is a workaround for when + // Cuebot failed to shutdown a job due to database access error. + rpc ShutdownIfCompleted(JobShutdownIfCompletedRequest) returns (JobShutdownIfCompletedResponse); + // Staggers the specified frame range rpc StaggerFrames(JobStaggerFramesRequest) returns (JobStaggerFramesResponse); } @@ -1427,6 +1431,13 @@ message JobSetPriorityRequest { message JobSetPriorityResponse {} // Empty +// ShutdownIfCompleted +message JobShutdownIfCompletedRequest { + Job job = 1; +} + +message JobShutdownIfCompletedResponse {} // Empty + // StaggerFrames message JobStaggerFramesRequest { Job job = 1; diff --git a/pycue/opencue/wrappers/job.py b/pycue/opencue/wrappers/job.py index 733c91e61..cbb1869d9 100644 --- a/pycue/opencue/wrappers/job.py +++ b/pycue/opencue/wrappers/job.py @@ -773,6 +773,11 @@ def maxRss(self): :return: most memory used by any frame in kB""" return self.data.job_stats.max_rss + def shutdownIfCompleted(self): + """Shutdown the job if it is completed.""" + self.stub.ShutdownIfCompleted(job_pb2.JobShutdownIfCompletedRequest(job=self.data), + timeout=Cuebot.Timeout) + class NestedJob(Job): """This class contains information and actions related to a nested job.""" From 6a91e73a201b2dca336ffe6e267e7b8783527f64 Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Mon, 17 Jan 2022 11:57:15 -0800 Subject: [PATCH 163/277] [cuebot] Fix HostSearch substring for loose search. (#1076) --- .../com/imageworks/spcue/dao/criteria/postgres/HostSearch.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/postgres/HostSearch.java b/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/postgres/HostSearch.java index 5d8de8586..9e84d3ed0 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/postgres/HostSearch.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/postgres/HostSearch.java @@ -44,7 +44,7 @@ public void filterByAlloc(AllocationInterface alloc) { public void buildWhereClause() { addPhrase("host.pk_host", criteria.getIdsList()); addPhrase("host.str_name", criteria.getHostsList()); - addPhrase("host.str_name", new HashSet<>(criteria.getSubstrList())); + addLikePhrase("host.str_name", new HashSet<>(criteria.getSubstrList())); addRegexPhrase("host.str_name", new HashSet<>(criteria.getRegexList())); addPhrase("alloc.str_name", criteria.getAllocsList()); Set items = new HashSet<>(criteria.getStates().getStateCount()); From 8037f2eea6a969433d8dbb8d9c203f89fb3628f3 Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Mon, 17 Jan 2022 11:59:11 -0800 Subject: [PATCH 164/277] [cuegui] Optimize CueMonitorTree processUpdate API call. (#1077) --- cuegui/cuegui/CueJobMonitorTree.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/cuegui/cuegui/CueJobMonitorTree.py b/cuegui/cuegui/CueJobMonitorTree.py index f61a416a1..5da23541e 100644 --- a/cuegui/cuegui/CueJobMonitorTree.py +++ b/cuegui/cuegui/CueJobMonitorTree.py @@ -31,6 +31,7 @@ import opencue import opencue.compiled_proto.job_pb2 import opencue.wrappers.group +from opencue.wrappers.job import Job import cuegui.AbstractTreeWidget import cuegui.AbstractWidgetItem @@ -476,16 +477,21 @@ def __processUpdateHandleNested(self, parent, groups): for nestedGroup in group.data.groups.nested_groups] self.__processUpdateHandleNested(groupItem, nestedGroups) - for jobId in group.data.jobs: - job = opencue.api.getJob(jobId) - try: - if job.id() in self._items: - self._items[job.id()].update(job, groupItem) - else: - self._items[job.id()] = JobWidgetItem(job, groupItem) - except RuntimeError: - logger.warning( - "Failed to create tree item. RootView might be closed", exc_info=True) + # empty list will return all jobs on the farm + # only search if list has jobs + if group.data.jobs: + jobSeq = opencue.search.JobSearch.byOptions(id=list(group.data.jobs)).jobs + jobsObject = [Job(j) for j in jobSeq.jobs] + + for job in jobsObject: + try: + if job.id() in self._items: + self._items[job.id()].update(job, groupItem) + else: + self._items[job.id()] = JobWidgetItem(job, groupItem) + except RuntimeError: + logger.warning( + "Failed to create tree item. RootView might be closed", exc_info=True) def mouseDoubleClickEvent(self, event): del event From 310af8ac5f29e3d2a99bc7dff4ffac3df39e170c Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 17 Jan 2022 15:55:52 -0500 Subject: [PATCH 165/277] [cuebot] Switch to new version of embedded Postgres for unit tests. (#1087) --- cuebot/build.gradle | 5 ++++- .../com/imageworks/spcue/test/TestDatabaseSetupPostgres.java | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cuebot/build.gradle b/cuebot/build.gradle index 4f93bd229..403cc893e 100644 --- a/cuebot/build.gradle +++ b/cuebot/build.gradle @@ -63,8 +63,11 @@ dependencies { testCompile group: 'junit', name: 'junit', version: '4.12' testCompile group: 'org.springframework.boot', name: 'spring-boot-starter-test' testCompile group: 'org.assertj', name: 'assertj-core', version: '3.8.0' - testCompile group: 'com.opentable.components', name: 'otj-pg-embedded', version: '0.13.3' + testCompile group: 'io.zonky.test', name: 'embedded-postgres', version: '1.3.1' testCompile group: 'org.flywaydb', name: 'flyway-core', version: '5.2.0' + + // Use newer version of Postgres for tests: https://github.com/zonkyio/embedded-postgres/issues/78 + implementation enforcedPlatform('io.zonky.test.postgres:embedded-postgres-binaries-bom:11.13.0') } compileJava { diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/TestDatabaseSetupPostgres.java b/cuebot/src/test/java/com/imageworks/spcue/test/TestDatabaseSetupPostgres.java index 713af7cbc..49513708b 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/TestDatabaseSetupPostgres.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/TestDatabaseSetupPostgres.java @@ -18,7 +18,7 @@ import com.google.common.base.Charsets; import com.google.common.io.Resources; -import com.opentable.db.postgres.embedded.EmbeddedPostgres; +import io.zonky.test.db.postgres.embedded.EmbeddedPostgres; import org.flywaydb.core.Flyway; import java.net.URL; From f7c12f78f4ffaba0384cb2fad2d74ce88e6c6c84 Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Tue, 18 Jan 2022 10:47:40 -0800 Subject: [PATCH 166/277] [cuebot] Introduce depend.satisfy_only_on_frame_success setting. (#1082) --- .../dispatcher/FrameCompleteHandler.java | 48 +++++++-- cuebot/src/main/resources/opencue.properties | 3 + .../dispatcher/FrameCompleteHandlerTests.java | 97 ++++++++++++++++++- .../test/dispatcher/HistoryControlTests.java | 2 +- .../conf/jobspec/jobspec_gpus_test.xml | 35 +++++++ 5 files changed, 173 insertions(+), 12 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java index fe2482720..f66b91097 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java @@ -25,6 +25,8 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; import org.springframework.dao.EmptyResultDataAccessException; import com.imageworks.spcue.DispatchFrame; @@ -92,6 +94,25 @@ public class FrameCompleteHandler { */ private boolean shutdown = false; + /** + * Whether or not to satisfy dependents (*_ON_FRAME and *_ON_LAYER) only on Frame success + */ + private boolean satisfyDependOnlyOnFrameSuccess; + + public boolean getSatisfyDependOnlyOnFrameSuccess() { + return satisfyDependOnlyOnFrameSuccess; + } + + public void setSatisfyDependOnlyOnFrameSuccess(boolean satisfyDependOnlyOnFrameSuccess) { + this.satisfyDependOnlyOnFrameSuccess = satisfyDependOnlyOnFrameSuccess; + } + + @Autowired + public FrameCompleteHandler(Environment env) { + satisfyDependOnlyOnFrameSuccess = env.getProperty( + "depend.satisfy_only_on_frame_success", Boolean.class, true); + } + /** * Handle the given FrameCompleteReport from RQD. * @@ -235,21 +256,28 @@ public void handlePostFrameCompleteOperations(VirtualProc proc, dispatchSupport.updateUsageCounters(frame, report.getExitStatus()); - if (newFrameState.equals(FrameState.SUCCEEDED)) { + boolean isLayerComplete = false; + + if (newFrameState.equals(FrameState.SUCCEEDED) + || (!satisfyDependOnlyOnFrameSuccess + && newFrameState.equals(FrameState.EATEN))) { jobManagerSupport.satisfyWhatDependsOn(frame); - if (jobManager.isLayerComplete(frame)) { + isLayerComplete = jobManager.isLayerComplete(frame); + if (isLayerComplete) { jobManagerSupport.satisfyWhatDependsOn((LayerInterface) frame); - } else { - /* - * If the layer meets some specific criteria then try to - * update the minimum memory and tags so it can run on a - * wider variety of cores, namely older hardware. - */ - jobManager.optimizeLayer(frame, report.getFrame().getNumCores(), - report.getFrame().getMaxRss(), report.getRunTime()); } } + if (newFrameState.equals(FrameState.SUCCEEDED) && !isLayerComplete) { + /* + * If the layer meets some specific criteria then try to + * update the minimum memory and tags so it can run on a + * wider variety of cores, namely older hardware. + */ + jobManager.optimizeLayer(frame, report.getFrame().getNumCores(), + report.getFrame().getMaxRss(), report.getRunTime()); + } + /* * The final frame can either be Succeeded or Eaten. If you only * check if the frame is Succeeded before doing an isJobComplete diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index 89db91c94..220cc2d5f 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -88,6 +88,9 @@ dispatcher.booking_queue.max_pool_size=6 # Queue capacity for booking. dispatcher.booking_queue.queue_capacity=1000 +# Whether or not to satisfy dependents (*_ON_FRAME and *_ON_LAYER) only on Frame success +depend.satisfy_only_on_frame_success=true + # Jobs will be archived to the history tables after being completed for this long. history.archive_jobs_cutoff_hours=72 diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java index 8888e0453..9228da36d 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java @@ -27,17 +27,23 @@ import org.junit.Test; import org.springframework.test.annotation.Rollback; import org.springframework.test.context.ContextConfiguration; +import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; +import com.imageworks.spcue.DispatchFrame; import com.imageworks.spcue.DispatchHost; -import com.imageworks.spcue.FrameInterface; +import com.imageworks.spcue.DispatchJob; +import com.imageworks.spcue.FrameDetail; import com.imageworks.spcue.JobDetail; import com.imageworks.spcue.LayerDetail; import com.imageworks.spcue.VirtualProc; +import com.imageworks.spcue.dao.FrameDao; import com.imageworks.spcue.dao.LayerDao; import com.imageworks.spcue.dispatcher.Dispatcher; +import com.imageworks.spcue.dispatcher.DispatchSupport; import com.imageworks.spcue.dispatcher.FrameCompleteHandler; import com.imageworks.spcue.grpc.host.HardwareState; +import com.imageworks.spcue.grpc.job.FrameState; import com.imageworks.spcue.grpc.report.FrameCompleteReport; import com.imageworks.spcue.grpc.report.RenderHost; import com.imageworks.spcue.grpc.report.RunningFrameInfo; @@ -70,12 +76,18 @@ public class FrameCompleteHandlerTests extends TransactionalTest { @Resource JobManager jobManager; + @Resource + FrameDao frameDao; + @Resource LayerDao layerDao; @Resource Dispatcher dispatcher; + @Resource + DispatchSupport dispatchSupport; + private static final String HOSTNAME = "beta"; @Before @@ -232,5 +244,88 @@ public void testGpuReportOver() { (jobManager.isJobComplete(job1) ? 1 : 0) + (jobManager.isJobComplete(job2) ? 1 : 0)); } + + private void executeDepend( + FrameState frameState, int exitStatus, int dependCount, FrameState dependState) { + JobDetail job = jobManager.findJobDetail("pipe-default-testuser_test_depend"); + LayerDetail layerFirst = layerDao.findLayerDetail(job, "layer_first"); + LayerDetail layerSecond = layerDao.findLayerDetail(job, "layer_second"); + FrameDetail frameFirst = frameDao.findFrameDetail(job, "0000-layer_first"); + FrameDetail frameSecond = frameDao.findFrameDetail(job, "0000-layer_second"); + + assertEquals(1, frameSecond.dependCount); + assertEquals(FrameState.DEPEND, frameSecond.state); + + jobManager.setJobPaused(job, false); + + DispatchHost host = getHost(); + List procs = dispatcher.dispatchHost(host); + assertEquals(1, procs.size()); + VirtualProc proc = procs.get(0); + assertEquals(job.getId(), proc.getJobId()); + assertEquals(layerFirst.getId(), proc.getLayerId()); + assertEquals(frameFirst.getId(), proc.getFrameId()); + + RunningFrameInfo info = RunningFrameInfo.newBuilder() + .setJobId(proc.getJobId()) + .setLayerId(proc.getLayerId()) + .setFrameId(proc.getFrameId()) + .setResourceId(proc.getProcId()) + .build(); + FrameCompleteReport report = FrameCompleteReport.newBuilder() + .setFrame(info) + .setExitStatus(exitStatus) + .build(); + + DispatchJob dispatchJob = jobManager.getDispatchJob(proc.getJobId()); + DispatchFrame dispatchFrame = jobManager.getDispatchFrame(report.getFrame().getFrameId()); + dispatchSupport.stopFrame(dispatchFrame, frameState, report.getExitStatus(), + report.getFrame().getMaxRss()); + frameCompleteHandler.handlePostFrameCompleteOperations(proc, + report, dispatchJob, dispatchFrame, frameState); + + assertTrue(jobManager.isLayerComplete(layerFirst)); + assertFalse(jobManager.isLayerComplete(layerSecond)); + + frameSecond = frameDao.findFrameDetail(job, "0000-layer_second"); + assertEquals(dependCount, frameSecond.dependCount); + assertEquals(dependState, frameSecond.state); + } + + @Test + @Transactional + @Rollback(true) + public void testDependOnSuccess() { + assertTrue(frameCompleteHandler.getSatisfyDependOnlyOnFrameSuccess()); + executeDepend(FrameState.SUCCEEDED, 0, 0, FrameState.WAITING); + } + + @Test + @Transactional + @Rollback(true) + public void testDependOnFailure() { + assertTrue(frameCompleteHandler.getSatisfyDependOnlyOnFrameSuccess()); + executeDepend(FrameState.EATEN, -1, 1, FrameState.DEPEND); + } + + @Test + @Transactional + @Rollback(true) + public void testDependOnSuccessSatifyOnAny() { + frameCompleteHandler.setSatisfyDependOnlyOnFrameSuccess(false); + assertFalse(frameCompleteHandler.getSatisfyDependOnlyOnFrameSuccess()); + executeDepend(FrameState.SUCCEEDED, 0, 0, FrameState.WAITING); + frameCompleteHandler.setSatisfyDependOnlyOnFrameSuccess(true); + } + + @Test + @Transactional + @Rollback(true) + public void testDependOnFailureSatisfyOnAny() { + frameCompleteHandler.setSatisfyDependOnlyOnFrameSuccess(false); + assertFalse(frameCompleteHandler.getSatisfyDependOnlyOnFrameSuccess()); + executeDepend(FrameState.EATEN, -1, 0, FrameState.WAITING); + frameCompleteHandler.setSatisfyDependOnlyOnFrameSuccess(true); + } } diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java index e8dc15517..a9787566b 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java @@ -169,7 +169,7 @@ public void testEnabled() { launchAndDeleteJob(); - assertEquals(Integer.valueOf(3), jdbcTemplate.queryForObject( + assertEquals(Integer.valueOf(4), jdbcTemplate.queryForObject( "SELECT COUNT(*) FROM job_history", Integer.class)); assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( "SELECT COUNT(*) FROM frame_history", Integer.class)); diff --git a/cuebot/src/test/resources/conf/jobspec/jobspec_gpus_test.xml b/cuebot/src/test/resources/conf/jobspec/jobspec_gpus_test.xml index 3b6b00ae7..9d9aa8245 100644 --- a/cuebot/src/test/resources/conf/jobspec/jobspec_gpus_test.xml +++ b/cuebot/src/test/resources/conf/jobspec/jobspec_gpus_test.xml @@ -73,4 +73,39 @@ + + + True + + + true + 0 + 1 + 1 + 1 + + shell + + + + true + 0 + 1 + 1 + 1 + + shell + + + + + + + + test_depend + layer_second + test_depend + layer_first + + From 66edf550c6fdc41be42b589823576ac1f880078a Mon Sep 17 00:00:00 2001 From: Kazuki Sakamoto Date: Tue, 1 Feb 2022 09:28:15 -0800 Subject: [PATCH 167/277] Fix %CPU for Windows (#1090) --- rqd/rqd/rqmachine.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 70a7dd2e6..a9afae757 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -222,7 +222,9 @@ def rssUpdate(self, frames): stat = stats[frame.pid] frame.rss = stat["rss"] // 1024 frame.maxRss = max(frame.rss, frame.maxRss) - frame.runFrame.attributes["pcpu"] = str(stat["pcpu"]) + frame.runFrame.attributes["pcpu"] = str( + stat["pcpu"] * self.__coreInfo.total_cores + ) return if platform.system() != 'Linux': From b0faee1d49f52dda076fe03467446f0a0c6ef327 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 1 Feb 2022 12:29:03 -0500 Subject: [PATCH 168/277] Notes from Jan 5 TSC meeting. (#1088) --- tsc/meetings/2022-01-05.md | 113 +++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 tsc/meetings/2022-01-05.md diff --git a/tsc/meetings/2022-01-05.md b/tsc/meetings/2022-01-05.md new file mode 100644 index 000000000..47636676b --- /dev/null +++ b/tsc/meetings/2022-01-05.md @@ -0,0 +1,113 @@ +# OpenCue TSC Meeting Notes 5 January 2022 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Completed 2022 Goals + * Cleared section for the new year. Nothing to be added here, yet. +* Ongoing work + * CueGUI memory leak + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1025 + * SPI: still researching and testing their fix. + * Scheduling fixes + * Thread pool properties + * Two versions: + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1008 + * SPI version https://github.com/AcademySoftwareFoundation/OpenCue/pull/1035 + * PRs need review. + * Let's start with 1035, then merge 1008 to fix conflicts to see what in 1008 is still + useful, if anything. + * Review of 1035 is almost complete, code looking good with a few minor suggestions. + * RQD change, adding child proc info into RQD logs as well as database for viewing in CueGUI + * For understanding memory usage of complex jobs. + * Still in development. + * log4j update + * PR needs another update. + * Let's post an update to the user group to let everyone know to update. +* Goals from 2021 + * Subsections here should be moved to the 2022 section or removed. + * New user UX + * Keep for 2022 + * Docs refresh + * Keep for 2022 + * Improve scheduler logic + * Keep for 2022 + * Expand DCC plugins + * No progress in 2021. Keep for 2022? + * We should keep it, but lower priority. Important for wider user adoption. + * High priority: Blender, Houdini + * Worth writing to the user group to see what folks have already? + * CSP Terraform docs + * No progress yet. + * Let's look at the Azure version of this. + * Let's roll this into the more general Cloud goal. + * Auto-close Github issues + * No need to keep this as its own goal — it's just a single piece of work. "Improving + development workflow" would be a more suitable goal but doesn't appear to be necessary at + the moment. + * Expanded Cloud functionality + * Keep for 2022. +* Todo 2022 Goals + * New user UX + * Publish PyPI packages + * Design + doc: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/ + * Proposal for config file + standardization: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/edit#heading=h.v85uo9klwqft + * Config cleanup for pycue done, tested on macOS and Windows, merged. + * pyoutline, cuesubmit config cleanup review done, ready to merge. + * Fix cuebot:latest tag for sandbox setup + * Proposal: publish Docker images for every new patch version (i.e. every new commit to + master) + * Proposal accepted. + * No progress implementing this yet. + * Related: for PyPI design we may want to also publish packages for every new patch + version. + * Docs refresh + * User guide refresh + * Split into "install from latest release" and "install from master" + * Needed to assist transition to publish Docker images on all commits + * No progress. + * API reference + * We have the ability to generate HTML from any commit, but we need a new CI pipeline to + build and publish this on opencue.io for new releases. + * Proposal: for now, highlight existing docs for building reference locally + * Proposal accepted. + * Brian: cleaned up local build process, sent email with doc + link: https://www.opencue.io/contributing/opencue/build-docs/ + * pyoutline examples + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/177 + * No progress yet. + * User guide for CueGUI + * No progress. + * Configuration guide for client-side python + * In progress, will be done as part of PyPI work. + * Configuration guide for Cuebot + * In progress, will be done as part of PyPI work. + * Configuration guide for RQD + * In progress, will be done as part of PyPI work. + * Improve scheduler logic + * Diego has volunteered to own this. + * Email reply from Matt, looped in others. + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1001 + * Other PRs with improvements — see ongoing work above. + * Prototype in progress. + * Expanded Cloud functionality + * Finalize GSoC plugin + * Test Azure support + * Add AWS support + * Design expanded functionality e.g. autoscaling for work in the queue. + * Last update from Greg: started to test / clean up azure support. Looking good so far. + * Terraform scripts? + * Let's look at the Azure version of this + * Generic k8s setup and cuebot pool size suggestions + * Let's write up a github issue to discuss requirements + * User permissions + * Need a github issue for discussion. + * Expand DCC plugins + * No progress. + * High priority: Blender, Houdini + * Important for wider user adoption + * Worth writing to the user group to see what folks have already? + From bdbd8c9add3ddecee71200c526bcf9ef289338ae Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Sun, 27 Feb 2022 17:24:54 -0500 Subject: [PATCH 169/277] Standardize config env var and paths. (#1075) --- cuesubmit/cuesubmit/Config.py | 6 +++- cuesubmit/tests/Config_tests.py | 57 ++++++++++++++++++++++----------- pycue/tests/config_test.py | 8 +++-- 3 files changed, 49 insertions(+), 22 deletions(-) diff --git a/cuesubmit/cuesubmit/Config.py b/cuesubmit/cuesubmit/Config.py index 99f76242c..d657f5402 100644 --- a/cuesubmit/cuesubmit/Config.py +++ b/cuesubmit/cuesubmit/Config.py @@ -27,6 +27,8 @@ import os import yaml +import opencue.config + CONFIG_FILE_ENV_VAR = 'CUESUBMIT_CONFIG_FILE' @@ -35,7 +37,9 @@ def getConfigValues(): """Reads the config file from disk and returns the values it defines.""" configData = {} configFile = os.environ.get(CONFIG_FILE_ENV_VAR) - if configFile and os.path.exists(configFile): + if not configFile: + configFile = os.path.join(opencue.config.config_base_directory(), 'cuesubmit.yaml') + if os.path.exists(configFile): with open(configFile, 'r') as data: try: configData = yaml.load(data, Loader=yaml.SafeLoader) diff --git a/cuesubmit/tests/Config_tests.py b/cuesubmit/tests/Config_tests.py index 12ce31fba..749d3e098 100644 --- a/cuesubmit/tests/Config_tests.py +++ b/cuesubmit/tests/Config_tests.py @@ -21,9 +21,11 @@ from __future__ import absolute_import import os -import tempfile import unittest +import mock +import pyfakefs.fake_filesystem_unittest + import cuesubmit.Config @@ -36,28 +38,47 @@ CONFIG_YAML_INVALID = b' " some text in an unclosed quote' -class ConfigTests(unittest.TestCase): +class ConfigTests(pyfakefs.fake_filesystem_unittest.TestCase): + def setUp(self): + self.setUpPyfakefs() + if 'CUESUBMIT_CONFIG_FILE' in os.environ: + del os.environ['CUESUBMIT_CONFIG_FILE'] + + def test__should_skip_missing_files_without_error(self): + configData = cuesubmit.Config.getConfigValues() + + self.assertDictEqual({}, configData) + + def test__should_load_config_from_env_var(self): + config_file_path = '/path/to/config.yaml' + self.fs.create_file(config_file_path, contents=CONFIG_YAML) + os.environ['CUESUBMIT_CONFIG_FILE'] = config_file_path + + configData = cuesubmit.Config.getConfigValues() + + self.assertEqual('OPENCUESUBMIT', configData.get('UI_NAME')) + self.assertEqual('OpenCue Submit', configData.get('SUBMIT_APP_WINDOW_TITLE')) + self.assertEqual(None, configData.get('SOME_UNKNOWN_SETTING')) - def testGetConfigValues(self): - with tempfile.NamedTemporaryFile() as fp: - fp.write(CONFIG_YAML) - fp.flush() - os.environ[cuesubmit.Config.CONFIG_FILE_ENV_VAR] = fp.name + @mock.patch('platform.system', new=mock.Mock(return_value='Linux')) + @mock.patch('os.path.expanduser', new=mock.Mock(return_value='/home/username')) + def test__should_load_config_from_user_profile(self): + config_file_path = '/home/username/.config/opencue/cuesubmit.yaml' + self.fs.create_file(config_file_path, contents=CONFIG_YAML) - configData = cuesubmit.Config.getConfigValues() + configData = cuesubmit.Config.getConfigValues() - self.assertEqual('OPENCUESUBMIT', configData.get('UI_NAME')) - self.assertEqual('OpenCue Submit', configData.get('SUBMIT_APP_WINDOW_TITLE')) - self.assertEqual(None, configData.get('SOME_UNKNOWN_SETTING')) + self.assertEqual('OPENCUESUBMIT', configData.get('UI_NAME')) + self.assertEqual('OpenCue Submit', configData.get('SUBMIT_APP_WINDOW_TITLE')) + self.assertEqual(None, configData.get('SOME_UNKNOWN_SETTING')) - def testFailOnInvalidYaml(self): - with tempfile.NamedTemporaryFile() as fp: - fp.write(CONFIG_YAML_INVALID) - fp.flush() - os.environ[cuesubmit.Config.CONFIG_FILE_ENV_VAR] = fp.name + def test__should_fail_on_invalid_yaml(self): + config_file_path = '/path/to/config.yaml' + self.fs.create_file(config_file_path, contents=CONFIG_YAML_INVALID) + os.environ['CUESUBMIT_CONFIG_FILE'] = config_file_path - with self.assertRaises(cuesubmit.Config.CuesubmitConfigError): - cuesubmit.Config.getConfigValues() + with self.assertRaises(cuesubmit.Config.CuesubmitConfigError): + cuesubmit.Config.getConfigValues() if __name__ == '__main__': diff --git a/pycue/tests/config_test.py b/pycue/tests/config_test.py index 9045365eb..15672cb04 100644 --- a/pycue/tests/config_test.py +++ b/pycue/tests/config_test.py @@ -61,8 +61,10 @@ def setUp(self): self.setUpPyfakefs() self.fs.add_real_file( os.path.join(os.path.dirname(opencue.__file__), 'default.yaml'), read_only=True) - os.unsetenv('OPENCUE_CONFIG_FILE') - os.unsetenv('OPENCUE_CONF') + if 'OPENCUE_CONFIG_FILE' in os.environ: + del os.environ['OPENCUE_CONFIG_FILE'] + if 'OPENCUE_CONF' in os.environ: + del os.environ['OPENCUE_CONF'] @mock.patch('platform.system', new=mock.Mock(return_value='Linux')) @mock.patch('os.path.expanduser', new=mock.Mock(return_value='/home/username')) @@ -106,7 +108,7 @@ def test__should_load_user_config(self): self.assertEqual(3, config['cuebot.exception_retries']) def test__should_load_user_config_from_legacy_var(self): - config_file_path = '/path/to/config.yaml' + config_file_path = '/path/to/legacy/config.yaml' self.fs.create_file(config_file_path, contents=USER_CONFIG) os.environ['OPENCUE_CONF'] = config_file_path From 6da47bab2ea621e0abf9d53348b3ccfd528b69d6 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Sun, 27 Feb 2022 17:25:16 -0500 Subject: [PATCH 170/277] [pyoutline] Standardize config env vars and paths. (#1074) --- pyoutline/outline/config.py | 104 ++++++++++++++++++++---- pyoutline/tests/config_test.py | 139 +++++++++++++++++++++++++++++++++ 2 files changed, 226 insertions(+), 17 deletions(-) create mode 100644 pyoutline/tests/config_test.py diff --git a/pyoutline/outline/config.py b/pyoutline/outline/config.py index 371793a3c..de74a8faa 100644 --- a/pyoutline/outline/config.py +++ b/pyoutline/outline/config.py @@ -28,8 +28,10 @@ # pylint: enable=wrong-import-position import getpass +import logging import os import pathlib +import platform import tempfile import six @@ -40,27 +42,95 @@ else: ConfigParser = configparser.ConfigParser - -__all__ = ["config"] +__all__ = ['config', 'read_config_from_disk'] __file_path__ = pathlib.Path(__file__) -PYOUTLINE_ROOT_DIR = __file_path__.parent.parent -DEFAULT_USER_DIR = pathlib.Path(tempfile.gettempdir()) / 'opencue' / 'outline' / getpass.getuser() +# Environment variables which can be used to define a custom config file. +__CONFIG_FILE_ENV_VARS = [ + # OUTLINE_CONFIG_FILE is the preferred setting to use. + 'OUTLINE_CONFIG_FILE', + # OL_CONFIG is deprecated, but kept for now for backwards compatibility. + 'OL_CONFIG', +] + + +logger = logging.getLogger("outline.config") + + +def config_base_directory(): + """Returns the OpenCue config base directory. + + This platform-dependent directory, stored within your user profile, is used by + OpenCue components as the default location for various configuration files. Typically + if you store your config files in this location, there is no need to set environment + variables to indicate where your config files are located -- OpenCue should recognize + them automatically. + + NOTE: This work is ongoing. Over time more OpenCue components will start using this + base directory. See https://github.com/AcademySoftwareFoundation/OpenCue/issues/785. + + :rtype: str + :return: config file base directory + """ + if platform.system() == 'Windows': + return os.path.join(os.path.expandvars('%APPDATA%'), 'opencue') + return os.path.join(os.path.expanduser('~'), '.config', 'opencue') + + +def read_config_from_disk(): + """Loads configuration settings from config file on the local system. + + The configuration file used is, in order of preference: + - Path defined by the OUTLINE_CONFIG_FILE environment variable. + - Path defined by the OL_CONFIG environment variable. + - Path within the config base directory (i.e. ~/.config/opencue/outline.cfg) + - The default outline.cfg file which is distributed with the outline library. + + :rtype: ConfigParser + :return: config settings + """ + pyoutline_root_dir = __file_path__.parent.parent + default_user_dir = pathlib.Path( + tempfile.gettempdir()) / 'opencue' / 'outline' / getpass.getuser() + + _config = ConfigParser() + config_file = None + + for config_file_env_var in __CONFIG_FILE_ENV_VARS: + logger.debug('Checking for outline config file path in %s', config_file_env_var) + config_file_from_env = os.environ.get(config_file_env_var) + if config_file_from_env and os.path.exists(config_file_from_env): + config_file = config_file_from_env + break + + if not config_file: + config_from_user_profile = os.path.join(config_base_directory(), 'outline.cfg') + logger.debug('Checking for outline config at %s', config_from_user_profile) + if os.path.exists(config_from_user_profile): + config_file = config_from_user_profile + + if not config_file: + default_config_paths = [__file_path__.parent.parent.parent / 'etc' / 'outline.cfg', + __file_path__.parent.parent / 'etc' / 'outline.cfg'] + for default_config_path in default_config_paths: + logger.info('Loading default outline config from %s', default_config_path) + if default_config_path.exists(): + config_file = default_config_path + break + + if not config_file: + raise FileNotFoundError('outline config file was not found') + + _config.read(str(config_file)) -config = ConfigParser() + # Add defaults to the config,if they were not specified. + if not _config.get('outline', 'home'): + _config.set('outline', 'home', str(pyoutline_root_dir)) -default_config_paths = [__file_path__.parent.parent.parent / 'etc' / 'outline.cfg', - __file_path__.parent.parent / 'etc' / 'outline.cfg'] -default_config_path = None -for default_config_path in default_config_paths: - if default_config_path.exists(): - break + if not _config.get('outline', 'user_dir'): + _config.set('outline', 'user_dir', str(default_user_dir)) -config.read(os.environ.get("OL_CONFIG", str(default_config_path))) + return _config -# Add defaults to the config,if they were not specified. -if not config.get('outline', 'home'): - config.set('outline', 'home', str(PYOUTLINE_ROOT_DIR)) -if not config.get('outline', 'user_dir'): - config.set('outline', 'user_dir', str(DEFAULT_USER_DIR)) +config = read_config_from_disk() diff --git a/pyoutline/tests/config_test.py b/pyoutline/tests/config_test.py new file mode 100644 index 000000000..5e0132d61 --- /dev/null +++ b/pyoutline/tests/config_test.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python + +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the outline.config module.""" + +import getpass +import os.path +import unittest + +import mock +import pyfakefs.fake_filesystem_unittest + +import opencue +import outline +# The local import is necessary as `outline.config` will point to the ConfigParser after the +# first import. +from outline.config import read_config_from_disk + + +USER_CONFIG = ''' +[outline] +home = /some/users/home/dir +session_dir = {HOME}/.opencue/sessions +wrapper_dir = %(home)s/wrappers +user_dir = /arbitrary/user/dir +spec_version = 1.9 +facility = cloud + +[plugin:local] +module=outline.plugins.local +enable=1 +''' + + +class ConfigTest(pyfakefs.fake_filesystem_unittest.TestCase): + def setUp(self): + self.setUpPyfakefs() + self.fs.add_real_file( + os.path.join(os.path.dirname(opencue.__file__), 'default.yaml'), read_only=True) + if 'OL_CONFIG' in os.environ: + del os.environ['OL_CONFIG'] + if 'OUTLINE_CONFIG_FILE' in os.environ: + del os.environ['OUTLINE_CONFIG_FILE'] + + @mock.patch('tempfile.gettempdir', new=mock.Mock(return_value='/path/to/tmp/dir')) + def test__should_load_default_values(self): + self.assertIsNone(os.environ.get('OL_CONF')) + self.assertIsNone(os.environ.get('OUTLINE_CONFIG_FILE')) + self.fs.add_real_file( + os.path.join(os.path.dirname(os.path.dirname(outline.__file__)), 'etc', 'outline.cfg'), + read_only=True) + + config = read_config_from_disk() + + default_home = os.path.dirname(os.path.dirname(__file__)) + self.assertEqual(default_home, config.get('outline', 'home')) + self.assertEqual('{HOME}/.opencue/sessions', config.get('outline', 'session_dir')) + self.assertEqual( + os.path.join(default_home, 'wrappers'), config.get('outline', 'wrapper_dir')) + self.assertEqual( + '/path/to/tmp/dir/opencue/outline/%s' % getpass.getuser(), + config.get('outline', 'user_dir')) + self.assertEqual( + os.path.join(default_home, 'bin'), config.get('outline', 'bin_dir')) + self.assertEqual('cue', config.get('outline', 'backend')) + self.assertEqual('local', config.get('outline', 'facility')) + self.assertEqual('example.com', config.get('outline', 'domain')) + self.assertEqual('2', config.get('outline', 'maxretries')) + self.assertEqual('testing', config.get('outline', 'default_show')) + self.assertEqual('default', config.get('outline', 'default_shot')) + self.assertEqual('outline.plugins.local', config.get('plugin:local', 'module')) + self.assertEqual('1', config.get('plugin:local', 'enable')) + + def test__should_load_user_config_from_env_var(self): + config_file_path = '/path/to/outline.cfg' + self.fs.create_file(config_file_path, contents=USER_CONFIG) + os.environ['OUTLINE_CONFIG_FILE'] = config_file_path + + config = read_config_from_disk() + + custom_home = '/some/users/home/dir' + self.assertEqual(custom_home, config.get('outline', 'home')) + self.assertEqual('{HOME}/.opencue/sessions', config.get('outline', 'session_dir')) + self.assertEqual( + os.path.join(custom_home, 'wrappers'), config.get('outline', 'wrapper_dir')) + self.assertEqual('/arbitrary/user/dir', config.get('outline', 'user_dir')) + self.assertEqual('1.9', config.get('outline', 'spec_version')) + self.assertEqual('cloud', config.get('outline', 'facility')) + + def test__should_load_user_config_from_legacy_env_var(self): + config_file_path = '/path/to/outline.cfg' + self.fs.create_file(config_file_path, contents=USER_CONFIG) + os.environ['OL_CONFIG'] = config_file_path + + config = read_config_from_disk() + + custom_home = '/some/users/home/dir' + self.assertEqual(custom_home, config.get('outline', 'home')) + self.assertEqual('{HOME}/.opencue/sessions', config.get('outline', 'session_dir')) + self.assertEqual( + os.path.join(custom_home, 'wrappers'), config.get('outline', 'wrapper_dir')) + self.assertEqual('/arbitrary/user/dir', config.get('outline', 'user_dir')) + self.assertEqual('1.9', config.get('outline', 'spec_version')) + self.assertEqual('cloud', config.get('outline', 'facility')) + + @mock.patch('platform.system', new=mock.Mock(return_value='Linux')) + @mock.patch('os.path.expanduser', new=mock.Mock(return_value='/home/username')) + def test__should_load_user_config_from_user_profile(self): + config_file_path = '/home/username/.config/opencue/outline.cfg' + self.fs.create_file(config_file_path, contents=USER_CONFIG) + os.environ['OL_CONFIG'] = config_file_path + + config = read_config_from_disk() + + custom_home = '/some/users/home/dir' + self.assertEqual(custom_home, config.get('outline', 'home')) + self.assertEqual('{HOME}/.opencue/sessions', config.get('outline', 'session_dir')) + self.assertEqual( + os.path.join(custom_home, 'wrappers'), config.get('outline', 'wrapper_dir')) + self.assertEqual('/arbitrary/user/dir', config.get('outline', 'user_dir')) + self.assertEqual('1.9', config.get('outline', 'spec_version')) + self.assertEqual('cloud', config.get('outline', 'facility')) + + +if __name__ == '__main__': + unittest.main() From 027d85363fdffe08d1e0ed5372e785b345ed5103 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Tue, 29 Mar 2022 09:46:12 -0700 Subject: [PATCH 171/277] Replace DispatchQueue and BookingQueue with HealthyThreadPool (#1035) * Update dispatchQuery to use min_cores Sorting jobs only by priority causes a situation where low priority jobs can get starved by a constant flow of high priority jobs. The new formula adds a modifier to the sorting rank to take into account the number of cores the job is requesting and also the number of days the job is waiting on the queue. Priorities numbers over 200 will mostly override the formula and work as a priority only based scheduling. sort = priority + (100 * (1 - (job.cores/job.int_min_cores))) + (age in days) Besides that, also take layer_int_cores_min into account when filtering folder_resourse limitations to avoid allocating more cores than the folder limits. (cherry picked from commit 566411aeeddc60983a30eabe121fd03263d05525) * Revert "Update dispatchQuery to use min_cores" This reverts commit 2eb4936c * Replace DispatchQueue and BookingQueue with HealthyThreadPool Queues will not inherit from ThreadPoolExecutor, instead they will manage an instance of HealthThreadPool, which is a threadPoolExecutor that handles healthChecks, termination and repeated tasks. With this the Booking queue should be able to self-heal when locked threads happen. * Remove trackit reference * Refactor HostReportQueue to use guava Cache Use a guava cache to store only the last version of a HostReport per host. * Configure HostReportQueue on opencue.properties * Fix unit tests * Fix unit tests * This unit tests is not actually testing anything useful Test doesn't make sense with the new threadpool and will also cause problems whenever an user changes a config property. Co-authored-by: Roula O'Regan --- .../imageworks/spcue/config/AppConfig.java | 1 - .../spcue/dispatcher/BookingQueue.java | 148 +++++------ .../spcue/dispatcher/DispatchQueue.java | 111 ++++----- .../dispatcher/FrameCompleteHandler.java | 10 +- .../spcue/dispatcher/HealthyThreadPool.java | 235 ++++++++++++++++++ .../spcue/dispatcher/HostReportQueue.java | 88 +++++-- .../spcue/dispatcher/QueueRejectCounter.java | 4 + .../dispatcher/commands/DispatchBookHost.java | 24 +- .../commands/DispatchBookHostLocal.java | 4 +- .../commands/DispatchDropDepends.java | 6 +- .../commands/DispatchEatFrames.java | 4 +- .../commands/DispatchHandleHostReport.java | 6 +- .../commands/DispatchJobComplete.java | 6 +- .../commands/DispatchKillFrames.java | 3 +- .../commands/DispatchKillProcs.java | 7 +- .../commands/DispatchLaunchJob.java | 3 +- .../dispatcher/commands/DispatchMoveJobs.java | 4 +- .../commands/DispatchNextFrame.java | 3 +- .../commands/DispatchReorderFrames.java | 12 +- .../commands/DispatchRetryFrames.java | 3 +- .../commands/DispatchRqdKillFrame.java | 5 +- .../commands/DispatchSatisfyDepends.java | 6 +- .../DispatchShutdownJobIfCompleted.java | 3 +- .../commands/DispatchStaggerFrames.java | 4 +- .../dispatcher/commands/KeyRunnable.java | 41 +++ .../commands/ManageReparentHosts.java | 4 +- .../imageworks/spcue/rqd/RqdClientGrpc.java | 2 +- .../imageworks/spcue/servant/CueStatic.java | 24 +- .../spcue/servant/ManageDepend.java | 4 +- .../spcue/servant/ManageFilter.java | 4 +- .../spring/applicationContext-service.xml | 187 ++++++++++---- .../conf/spring/healthServlet-servlet.xml | 24 ++ cuebot/src/main/resources/log4j.properties | 9 + cuebot/src/main/resources/opencue.properties | 25 +- .../test/dispatcher/TestBookingQueue.java | 9 +- .../test/dispatcher/ThreadPoolTests.java | 93 ------- cuebot/src/test/resources/opencue.properties | 18 ++ 37 files changed, 781 insertions(+), 363 deletions(-) create mode 100644 cuebot/src/main/java/com/imageworks/spcue/dispatcher/HealthyThreadPool.java create mode 100644 cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/KeyRunnable.java create mode 100644 cuebot/src/main/resources/conf/spring/healthServlet-servlet.xml delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/ThreadPoolTests.java diff --git a/cuebot/src/main/java/com/imageworks/spcue/config/AppConfig.java b/cuebot/src/main/java/com/imageworks/spcue/config/AppConfig.java index 4edac2826..adf6e0368 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/config/AppConfig.java +++ b/cuebot/src/main/java/com/imageworks/spcue/config/AppConfig.java @@ -65,6 +65,5 @@ public ServletRegistrationBean jobLaunchServlet() { b.setServlet(new JobLaunchServlet()); return b; } - } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java index 7428f0bca..347acd025 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java @@ -19,71 +19,69 @@ package com.imageworks.spcue.dispatcher; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import com.imageworks.spcue.dispatcher.commands.DispatchBookHost; -import com.imageworks.spcue.util.CueUtil; +import com.imageworks.spcue.dispatcher.commands.KeyRunnable; import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; -public class BookingQueue extends ThreadPoolExecutor { - - private static final Logger logger = Logger.getLogger(BookingQueue.class); - - private static final int THREADS_KEEP_ALIVE_SECONDS = 10; +public class BookingQueue { - private int queueCapacity; - private int baseSleepTimeMillis = 400; - private AtomicBoolean isShutdown = new AtomicBoolean(false); + private final int healthThreshold; + private final int minUnhealthyPeriodMin; + private final int queueCapacity; + private final int corePoolSize; + private final int maxPoolSize; + private static final int BASE_SLEEP_TIME_MILLIS = 300; - private QueueRejectCounter rejectCounter = new QueueRejectCounter(); + private static final Logger logger = Logger.getLogger("HEALTH"); + private HealthyThreadPool healthyThreadPool; - private Cache bookingCache = CacheBuilder.newBuilder() - .expireAfterWrite(3, TimeUnit.MINUTES) - // Invalidate entries that got executed by the threadpool and lost their reference - .weakValues() - .build(); - - private BookingQueue(int corePoolSize, int maxPoolSize, int queueCapacity, int sleepTimeMs) { - super(corePoolSize, maxPoolSize, THREADS_KEEP_ALIVE_SECONDS, - TimeUnit.SECONDS, new LinkedBlockingQueue(queueCapacity)); + public BookingQueue(int healthThreshold, int minUnhealthyPeriodMin, int queueCapacity, + int corePoolSize, int maxPoolSize) { + this.healthThreshold = healthThreshold; + this.minUnhealthyPeriodMin = minUnhealthyPeriodMin; this.queueCapacity = queueCapacity; - this.baseSleepTimeMillis = sleepTimeMs; - this.setRejectedExecutionHandler(rejectCounter); - logger.info("BookingQueue" + - " core:" + getCorePoolSize() + - " max:" + getMaximumPoolSize() + - " capacity:" + queueCapacity + - " sleepTimeMs:" + sleepTimeMs); + this.corePoolSize = corePoolSize; + this.maxPoolSize = maxPoolSize; + initThreadPool(); } - @Autowired - public BookingQueue(Environment env, String propertyKeyPrefix, int sleepTimeMs) { - this(CueUtil.getIntProperty(env, propertyKeyPrefix, "core_pool_size"), - CueUtil.getIntProperty(env, propertyKeyPrefix, "max_pool_size"), - CueUtil.getIntProperty(env, propertyKeyPrefix, "queue_capacity"), - sleepTimeMs); + public void initThreadPool() { + healthyThreadPool = new HealthyThreadPool( + "BookingQueue", + healthThreshold, + minUnhealthyPeriodMin, + queueCapacity, + corePoolSize, + maxPoolSize, + BASE_SLEEP_TIME_MILLIS); } - public void execute(DispatchBookHost r) { - if (isShutdown.get()) { - return; - } - if (bookingCache.getIfPresent(r.getKey()) == null){ - bookingCache.put(r.getKey(), r); - super.execute(r); + public boolean isHealthy() { + try { + if (!healthyThreadPool.isHealthyOrShutdown()) { + logger.warn("BookingQueue: Unhealthy queue terminated, starting a new one"); + initThreadPool(); + } + } catch (InterruptedException e) { + // TODO: evaluate crashing the whole springbook context here + // to force a container restart cycle + logger.error("Failed to restart BookingThreadPool", e); + return false; } + + return true; + } + + public void execute(KeyRunnable r) { + healthyThreadPool.execute(r); } public long getRejectedTaskCount() { - return rejectCounter.getRejectCount(); + return healthyThreadPool.getRejectedTaskCount(); } public int getQueueCapacity() { @@ -91,56 +89,32 @@ public int getQueueCapacity() { } public void shutdown() { - if (!isShutdown.getAndSet(true)) { - logger.info("clearing out booking queue: " + this.getQueue().size()); - this.getQueue().clear(); - } + healthyThreadPool.shutdown(); + } + public int getSize() { + return healthyThreadPool.getQueue().size(); } - /** - * Lowers the sleep time as the queue grows. - * - * @return - */ - public int sleepTime() { - if (!isShutdown.get()) { - int sleep = (int) (baseSleepTimeMillis - (((this.getQueue().size () / - (float) queueCapacity) * baseSleepTimeMillis)) * 2); - if (sleep < 0) { - sleep = 0; - } - return sleep; - } else { - return 0; - } + public int getRemainingCapacity() { + return healthyThreadPool.getQueue().remainingCapacity(); } - protected void beforeExecute(Thread t, Runnable r) { - super.beforeExecute(t, r); - if (isShutdown()) { - this.remove(r); - } else { - try { - Thread.sleep(sleepTime()); - } catch (InterruptedException e) { - logger.info("booking queue was interrupted."); - Thread.currentThread().interrupt(); - } - } + public int getActiveCount() { + return healthyThreadPool.getActiveCount(); } - protected void afterExecute(Runnable r, Throwable t) { - super.afterExecute(r, t); + public long getCompletedTaskCount() { + return healthyThreadPool.getCompletedTaskCount(); + } - // Invalidate cache to avoid having to wait for GC to mark processed entries collectible - DispatchBookHost h = (DispatchBookHost)r; - bookingCache.invalidate(h.getKey()); + public long getCorePoolSize() { + return corePoolSize; + } - if (sleepTime() < 100) { - logger.info("BookingQueue cleanup executed."); - getQueue().clear(); - } + public long getMaximumPoolSize() { + return maxPoolSize; } + } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchQueue.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchQueue.java index 81366dedc..3798bddaa 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchQueue.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchQueue.java @@ -23,93 +23,84 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.log4j.Logger; -import org.springframework.core.task.TaskExecutor; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import com.imageworks.spcue.dispatcher.commands.KeyRunnable; public class DispatchQueue { - private TaskExecutor dispatchPool; - private ThreadPoolTaskExecutor _dispatchPool; - private String name = "Default"; - private AtomicBoolean isShutdown = new AtomicBoolean(false); - - private final AtomicLong tasksRun = new AtomicLong(0); - private final AtomicLong tasksRejected = new AtomicLong(0); + private int healthThreshold; + private int minUnhealthyPeriodMin; + private int queueCapacity; + private int corePoolSize; + private int maxPoolSize; - private static final Logger logger = Logger.getLogger(DispatchQueue.class); - - public DispatchQueue() {} + private static final Logger logger = Logger.getLogger("HEALTH"); + private String name = "Default"; + private HealthyThreadPool healthyDispatchPool; - public DispatchQueue(String name) { + public DispatchQueue(String name, int healthThreshold, int minUnhealthyPeriodMin, int queueCapacity, + int corePoolSize, int maxPoolSize) { this.name = name; + this.healthThreshold = healthThreshold; + this.minUnhealthyPeriodMin = minUnhealthyPeriodMin; + this.queueCapacity = queueCapacity; + this.corePoolSize = corePoolSize; + this.maxPoolSize = maxPoolSize; + initThreadPool(); } - public void execute(Runnable r) { + public void initThreadPool() { + healthyDispatchPool = new HealthyThreadPool( + name, + healthThreshold, + minUnhealthyPeriodMin, + queueCapacity, + corePoolSize, + maxPoolSize); + } + + public boolean isHealthy() { try { - if (!isShutdown.get()) { - this.dispatchPool.execute(r); - tasksRun.addAndGet(1); + if (!healthyDispatchPool.isHealthyOrShutdown()) { + logger.warn("DispatchQueue_" + name + ": Unhealthy queue terminated, starting a new one"); + initThreadPool(); } - } catch (Exception e) { - long rejection = tasksRejected.addAndGet(1); - logger.warn("Warning, dispatch queue - [" + name + "] rejected, " + e); - throw new DispatchQueueTaskRejectionException( - "Warning, dispatch queue [" + name + " rejected task #" - + rejection); + } catch (InterruptedException e) { + // TODO: evaluate crashing the whole springbook context here + // to force a container restart cycle + logger.error("DispatchQueue_" + name + ":Failed to restart DispatchThreadPool", e); + return false; } - } - public int getMaxPoolSize() { - return _dispatchPool.getMaxPoolSize(); + return true; } - public int getActiveThreadCount() { - return _dispatchPool.getActiveCount(); + public void execute(KeyRunnable r) { + healthyDispatchPool.execute(r); } - public int getWaitingCount() { - return _dispatchPool.getThreadPoolExecutor().getQueue().size(); + public long getRejectedTaskCount() { + return healthyDispatchPool.getRejectedTaskCount(); } - public int getRemainingCapacity() { - return _dispatchPool.getThreadPoolExecutor().getQueue().remainingCapacity(); + public void shutdown() { + healthyDispatchPool.shutdown(); } - public long getTotalDispatched() { - return tasksRun.get(); + public int getSize() { + return healthyDispatchPool.getQueue().size(); } - public long getTotalRejected() { - return tasksRejected.get(); + public int getRemainingCapacity() { + return healthyDispatchPool.getQueue().remainingCapacity(); } - public TaskExecutor getDispatchPool() { - return dispatchPool; + public int getActiveCount() { + return healthyDispatchPool.getActiveCount(); } - public void setDispatchPool(TaskExecutor dispatchPool) { - this.dispatchPool = dispatchPool; - this._dispatchPool = (ThreadPoolTaskExecutor) dispatchPool; + public long getCompletedTaskCount() { + return healthyDispatchPool.getCompletedTaskCount(); } - public void shutdown() { - if (!isShutdown.getAndSet(true)) { - logger.info("Shutting down thread pool " + name + ", currently " - + getActiveThreadCount() + " active threads."); - final long startTime = System.currentTimeMillis(); - while (getWaitingCount() != 0 && getActiveThreadCount() != 0) { - try { - if (System.currentTimeMillis() - startTime > 10000) { - throw new InterruptedException(name - + " thread pool failed to shutdown properly"); - } - Thread.sleep(250); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; - } - } - } - } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java index f66b91097..b54757b69 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java @@ -39,6 +39,7 @@ import com.imageworks.spcue.VirtualProc; import com.imageworks.spcue.dispatcher.commands.DispatchBookHost; import com.imageworks.spcue.dispatcher.commands.DispatchNextFrame; +import com.imageworks.spcue.dispatcher.commands.KeyRunnable; import com.imageworks.spcue.grpc.host.LockState; import com.imageworks.spcue.grpc.job.FrameExitStatus; import com.imageworks.spcue.grpc.job.FrameState; @@ -158,10 +159,11 @@ public void handleFrameCompleteReport(final FrameCompleteReport report) { final LayerDetail layer = jobManager.getLayerDetail(report.getFrame().getLayerId()); final DispatchFrame frame = jobManager.getDispatchFrame(report.getFrame().getFrameId()); final FrameState newFrameState = determineFrameState(job, layer, frame, report); - + final String key = proc.getJobId() + "_" + report.getFrame().getLayerId() + + "_" + report.getFrame().getFrameId(); if (dispatchSupport.stopFrame(frame, newFrameState, report.getExitStatus(), report.getFrame().getMaxRss())) { - dispatchQueue.execute(new Runnable() { + dispatchQueue.execute(new KeyRunnable(key) { @Override public void run() { try { @@ -182,7 +184,7 @@ public void run() { * properties. */ if (redirectManager.hasRedirect(proc)) { - dispatchQueue.execute(new Runnable() { + dispatchQueue.execute(new KeyRunnable(key) { @Override public void run() { try { @@ -195,7 +197,7 @@ public void run() { }); } else { - dispatchQueue.execute(new Runnable() { + dispatchQueue.execute(new KeyRunnable(key) { @Override public void run() { try { diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HealthyThreadPool.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HealthyThreadPool.java new file mode 100644 index 000000000..13c96e776 --- /dev/null +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HealthyThreadPool.java @@ -0,0 +1,235 @@ +package com.imageworks.spcue.dispatcher; + +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; +import java.util.Date; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.LinkedBlockingQueue; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.imageworks.spcue.dispatcher.commands.DispatchBookHost; +import com.imageworks.spcue.dispatcher.commands.KeyRunnable; +import org.apache.log4j.Logger; + + +/*** + * A ThreadPoolExecutor with two additional features: + * - Handles repeated tasks by always keeping the latest version + * - With isHealthyOrShutdown, the threadpool will drain and clear resources when unhealthy + * + */ +public class HealthyThreadPool extends ThreadPoolExecutor { + // The service need s to be unhealthy for this period of time to report + private static final Logger logger = Logger.getLogger("HEALTH"); + // Threshold to consider healthy or unhealthy + private final int healthThreshold; + private final int poolSize; + private final int minUnhealthyPeriodMin; + private final QueueRejectCounter rejectCounter = new QueueRejectCounter(); + private final Cache taskCache; + private final String name; + private Date lastCheck = new Date(); + private boolean wasHealthy = true; + protected final AtomicBoolean isShutdown = new AtomicBoolean(false); + private final int baseSleepTimeMillis; + + /** + * Start a thread pool + * @param name For logging purposes + * @param healthThreshold Percentage that should be available to consider healthy + * @param minUnhealthyPeriodMin Period in min to consider a queue unhealthy + * @param poolSize how many jobs can be queued + * @param threadsMinimum Minimum number of threads + * @param threadsMaximum Maximum number of threads to grow to + */ + public HealthyThreadPool(String name, + int healthThreshold, + int minUnhealthyPeriodMin, + int poolSize, + int threadsMinimum, + int threadsMaximum) { + this(name, healthThreshold, minUnhealthyPeriodMin, poolSize, + threadsMinimum, threadsMaximum, 0); + } + + /** + * Start a thread pool + * + * @param name For logging purposes + * @param healthThreshold Percentage that should be available to consider healthy + * @param minUnhealthyPeriodMin Period in min to consider a queue unhealthy + * @param poolSize how many jobs can be queued + * @param threadsMinimum Minimum number of threads + * @param threadsMaximum Maximum number of threads to grow to + * @param baseSleepTimeMillis Time a thread should sleep when the service is not under pressure + */ + public HealthyThreadPool(String name, + int healthThreshold, + int minUnhealthyPeriodMin, + int poolSize, + int threadsMinimum, + int threadsMaximum, + int baseSleepTimeMillis) { + super(threadsMinimum, threadsMaximum, 10, + TimeUnit.SECONDS, new LinkedBlockingQueue(poolSize)); + + logger.debug(name + ": Starting a new HealthyThreadPool"); + this.name = name; + this.healthThreshold = healthThreshold; + this.poolSize = poolSize; + this.minUnhealthyPeriodMin = minUnhealthyPeriodMin; + this.baseSleepTimeMillis = baseSleepTimeMillis; + this.setRejectedExecutionHandler(rejectCounter); + + this.taskCache = CacheBuilder.newBuilder() + .expireAfterWrite(3, TimeUnit.MINUTES) + // Invalidate entries that got executed by the threadPool and lost their reference + .weakValues() + .concurrencyLevel(threadsMaximum) + .build(); + } + + public void execute(KeyRunnable r) { + if (isShutdown.get()) { + logger.info(name + ": Task ignored, queue on hold or shutdown"); + return; + } + if (taskCache.getIfPresent(r.getKey()) == null){ + taskCache.put(r.getKey(), r); + super.execute(r); + } + } + + public long getRejectedTaskCount() { + return rejectCounter.getRejectCount(); + } + + /** + * Monitor if the queue is unhealthy for MIN_UNHEALTHY_PERIOD_MIN + * + * If unhealthy, the service will start the shutdown process and the + * caller is responsible for starting a new instance after the lock on + * awaitTermination is released. + */ + protected boolean isHealthyOrShutdown() throws InterruptedException { + Date now = new Date(); + if (diffInMinutes(lastCheck, now) > minUnhealthyPeriodMin){ + this.wasHealthy = healthCheck(); + this.lastCheck = now; + } + + if(healthCheck() || wasHealthy) { + logger.debug(name + ": healthy (" + + "Remaining Capacity: " + this.getQueue().remainingCapacity() + + ", Running: " + this.getActiveCount() + + ", Total Executed: " + this.getCompletedTaskCount() + + ")"); + return true; + } + else if (isShutdown.get()) { + logger.warn("Queue shutting down"); + return false; + } + else { + logger.warn(name + ": unhealthy, starting shutdown)"); + threadDump(); + + isShutdown.set(true); + super.shutdownNow(); + logger.warn(name + ": Awaiting unhealthy queue termination"); + if (super.awaitTermination(1, TimeUnit.MINUTES)){ + logger.info(name + ": Terminated successfully"); + } + else { + logger.warn(name + ": Failed to terminate"); + } + // Threads will eventually terminate, proceed + taskCache.invalidateAll(); + return false; + } + } + + private void threadDump() { + ThreadMXBean mx = ManagementFactory.getThreadMXBean(); + for(ThreadInfo info : mx.dumpAllThreads(true, true)){ + logger.debug(info.toString()); + } + } + + private static long diffInMinutes(Date dateStart, Date dateEnd) { + return TimeUnit.MINUTES.convert( + dateEnd.getTime() - dateStart.getTime(), + TimeUnit.MILLISECONDS + ); + } + + /** + * Lowers the sleep time as the queue grows. + * + * @return + */ + public int sleepTime() { + if (!isShutdown.get()) { + int sleep = (int) (baseSleepTimeMillis - (((this.getQueue().size () / + (float) this.poolSize) * baseSleepTimeMillis)) * 2); + if (sleep < 0) { + sleep = 0; + } + return sleep; + } else { + return 0; + } + } + + protected void beforeExecute(Thread t, Runnable r) { + super.beforeExecute(t, r); + if (isShutdown()) { + this.remove(r); + } else { + if (baseSleepTimeMillis > 0) { + try { + Thread.sleep(sleepTime()); + } catch (InterruptedException e) { + logger.info(name + ": booking queue was interrupted."); + } + } + } + } + + protected void afterExecute(Runnable r, Throwable t) { + super.afterExecute(r, t); + + // Invalidate cache to avoid having to wait for GC to mark processed entries collectible + KeyRunnable h = (KeyRunnable)r; + taskCache.invalidate(h.getKey()); + } + + protected boolean healthCheck() { + return (this.getQueue().remainingCapacity() > 0) || + (getRejectedTaskCount() < this.poolSize / healthThreshold); + } + + public void shutdown() { + if (!isShutdown.getAndSet(true)) { + logger.info("Shutting down thread pool " + name + ", currently " + + getActiveCount() + " active threads."); + final long startTime = System.currentTimeMillis(); + while (this.getQueue().size() != 0 && this.getActiveCount() != 0) { + try { + if (System.currentTimeMillis() - startTime > 10000) { + throw new InterruptedException(name + + " thread pool failed to shutdown properly"); + } + Thread.sleep(250); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + } + } +} diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportQueue.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportQueue.java index 922172202..59b91abcc 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportQueue.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportQueue.java @@ -19,14 +19,16 @@ package com.imageworks.spcue.dispatcher; +import java.lang.ref.WeakReference; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import com.imageworks.spcue.grpc.report.HostReport; import org.apache.log4j.Logger; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.env.Environment; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.imageworks.spcue.dispatcher.commands.DispatchHandleHostReport; import com.imageworks.spcue.util.CueUtil; @@ -34,38 +36,76 @@ public class HostReportQueue extends ThreadPoolExecutor { private static final Logger logger = Logger.getLogger(HostReportQueue.class); - private QueueRejectCounter rejectCounter = new QueueRejectCounter(); private AtomicBoolean isShutdown = new AtomicBoolean(false); private int queueCapacity; - private HostReportQueue(String name, int corePoolSize, int maxPoolSize, int queueCapacity) { - super(corePoolSize, maxPoolSize, 10 , TimeUnit.SECONDS, - new LinkedBlockingQueue(queueCapacity)); - this.queueCapacity = queueCapacity; - this.setRejectedExecutionHandler(rejectCounter); - logger.info(name + - " core:" + getCorePoolSize() + - " max:" + getMaximumPoolSize() + - " capacity:" + queueCapacity); + private Cache hostMap = CacheBuilder.newBuilder() + .expireAfterWrite(1, TimeUnit.HOURS) + .build(); + + /** + * Wrapper around protobuf object HostReport to add reportTi + */ + private class HostReportWrapper{ + private final HostReport hostReport; + private final WeakReference reportTaskRef; + public long taskTime = System.currentTimeMillis(); + + public HostReportWrapper(HostReport hostReport, DispatchHandleHostReport reportTask) { + this.hostReport = hostReport; + this.reportTaskRef = new WeakReference<>(reportTask); + } + + public HostReport getHostReport() { + return hostReport; + } + + public DispatchHandleHostReport getReportTask() { + return reportTaskRef.get(); + } + + public long getTaskTime() { + return taskTime; + } } - @Autowired - public HostReportQueue(Environment env, String name, String propertyKeyPrefix) { - this(name, - CueUtil.getIntProperty(env, propertyKeyPrefix, "core_pool_size"), - CueUtil.getIntProperty(env, propertyKeyPrefix, "max_pool_size"), - CueUtil.getIntProperty(env, propertyKeyPrefix, "queue_capacity")); + public HostReportQueue(int threadPoolSizeInitial, int threadPoolSizeMax, int queueSize) { + super(threadPoolSizeInitial, threadPoolSizeMax, 10 , TimeUnit.SECONDS, + new LinkedBlockingQueue(queueSize)); + this.setRejectedExecutionHandler(rejectCounter); } - public void execute(DispatchHandleHostReport r) { + public void execute(DispatchHandleHostReport newReport) { if (isShutdown.get()) { return; } - if (getQueue().contains(r)) { - getQueue().remove(r); + HostReportWrapper oldWrappedReport = hostMap.getIfPresent(newReport.getKey()); + // If hostReport exists on the cache and there's also a task waiting to be executed + // replace the old report by the new on, but refrain from creating another task + if (oldWrappedReport != null) { + DispatchHandleHostReport oldReport = oldWrappedReport.getReportTask(); + if(oldReport != null) { + // Replace report, but keep the reference of the existing task + hostMap.put(newReport.getKey(), + new HostReportWrapper(newReport.getHostReport(), oldReport)); + return; + } + } + hostMap.put(newReport.getKey(), + new HostReportWrapper(newReport.getHostReport(), newReport)); + super.execute(newReport); + } + + public HostReport removePendingHostReport(String key) { + if (key != null) { + HostReportWrapper r = hostMap.getIfPresent(key); + if (r != null) { + hostMap.asMap().remove(key, r); + return r.getHostReport(); + } } - super.execute(r); + return null; } public long getRejectedTaskCount() { @@ -95,5 +135,9 @@ public void shutdown() { } } } + + public boolean isHealthy() { + return getQueue().remainingCapacity() > 0; + } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/QueueRejectCounter.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/QueueRejectCounter.java index a9e0809f9..bb3d00716 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/QueueRejectCounter.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/QueueRejectCounter.java @@ -36,5 +36,9 @@ public long getRejectCount() { return rejectCounter.get(); } + public void clear() { + rejectCounter.set(0); + } + } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java index 594444573..0b65257dd 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java @@ -19,62 +19,64 @@ package com.imageworks.spcue.dispatcher.commands; +import java.util.List; +import java.util.ArrayList; +import org.apache.log4j.Logger; + import com.imageworks.spcue.DispatchHost; import com.imageworks.spcue.GroupInterface; import com.imageworks.spcue.JobInterface; import com.imageworks.spcue.ShowInterface; import com.imageworks.spcue.dispatcher.Dispatcher; +import com.imageworks.spcue.VirtualProc; /** * A command for booking a host. * * @category command */ -public class DispatchBookHost implements Runnable { +public class DispatchBookHost extends KeyRunnable { + private static final Logger logger = + Logger.getLogger(DispatchBookHost.class); private ShowInterface show = null; private GroupInterface group = null; private JobInterface job = null; private DispatchHost host; private Dispatcher dispatcher; - private String key; public DispatchHost getDispatchHost() { - this.key = host.getId(); + this.setKey(host.getId()); return host; } public DispatchBookHost(DispatchHost host, Dispatcher d) { + super(host.getId()); this.host = host; - this.key = host.getId(); this.dispatcher = d; } public DispatchBookHost(DispatchHost host, JobInterface job, Dispatcher d) { + super(host.getId() + "_job_" + job.getJobId()); this.host = host; this.job = job; - this.key = host.getId() + "_job_" + job.getJobId(); this.dispatcher = d; } public DispatchBookHost(DispatchHost host, GroupInterface group, Dispatcher d) { + super(host.getId() + "_group_" + group.getGroupId()); this.host = host; this.group = group; - this.key = host.getId() + "_group_" + group.getGroupId(); this.dispatcher = d; } public DispatchBookHost(DispatchHost host, ShowInterface show, Dispatcher d) { + super(host.getId() + "_name_" + show.getName()); this.host = host; this.show = show; - this.key = host.getId() + "_name_" + show.getName(); this.dispatcher = d; } - public String getKey() { - return this.key; - } - public void run() { new DispatchCommandTemplate() { public void wrapDispatchCommand() { diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHostLocal.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHostLocal.java index 3a742504c..737541a08 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHostLocal.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHostLocal.java @@ -22,12 +22,14 @@ import com.imageworks.spcue.DispatchHost; import com.imageworks.spcue.dispatcher.Dispatcher; -public class DispatchBookHostLocal implements Runnable { +public class DispatchBookHostLocal extends KeyRunnable { private DispatchHost host; private Dispatcher dispatcher; public DispatchBookHostLocal(DispatchHost host, Dispatcher d) { + super(host.getId()); + this.host = host; this.dispatcher = d; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchDropDepends.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchDropDepends.java index ca6e4b8c2..cf6428f3a 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchDropDepends.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchDropDepends.java @@ -19,6 +19,7 @@ package com.imageworks.spcue.dispatcher.commands; + import com.imageworks.spcue.FrameInterface; import com.imageworks.spcue.JobInterface; import com.imageworks.spcue.LayerInterface; @@ -31,7 +32,7 @@ * * @category command */ -public class DispatchDropDepends implements Runnable { +public class DispatchDropDepends extends KeyRunnable { JobInterface job; LayerInterface layer; @@ -41,18 +42,21 @@ public class DispatchDropDepends implements Runnable { DependManager dependManager; public DispatchDropDepends(JobInterface job, DependTarget target, DependManager dependManager) { + super("disp_drop_dep_job_" + job.getJobId() + "_" + target.toString()); this.job = job; this.target = target; this.dependManager = dependManager; } public DispatchDropDepends(LayerInterface layer, DependTarget target, DependManager dependManager) { + super("disp_drop_dep_layer_" + layer.getLayerId() + "_" + target.toString()); this.layer = layer; this.target = target; this.dependManager = dependManager; } public DispatchDropDepends(FrameInterface frame, DependTarget target, DependManager dependManager) { + super("disp_drop_dep_frame_" + frame.getFrameId() + "_" + target.toString()); this.frame = frame; this.target = target; this.dependManager = dependManager; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchEatFrames.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchEatFrames.java index 34292db49..32a2acf69 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchEatFrames.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchEatFrames.java @@ -23,18 +23,20 @@ import com.imageworks.spcue.dao.criteria.FrameSearchInterface; import com.imageworks.spcue.service.JobManagerSupport; + /** * A command for eating an array of frames * * @category command */ -public class DispatchEatFrames implements Runnable { +public class DispatchEatFrames extends KeyRunnable { private FrameSearchInterface search; private Source source; private JobManagerSupport jobManagerSupport; public DispatchEatFrames(FrameSearchInterface search, Source source, JobManagerSupport jobManagerSupport) { + super("disp_eat_frames_job_" + search.hashCode() + "_" + jobManagerSupport.hashCode()); this.search = search; this.source = source; this.jobManagerSupport = jobManagerSupport; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchHandleHostReport.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchHandleHostReport.java index c8f41131d..1a18f06ad 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchHandleHostReport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchHandleHostReport.java @@ -28,7 +28,7 @@ * * @category command */ -public class DispatchHandleHostReport implements Runnable { +public class DispatchHandleHostReport extends KeyRunnable { private HostReport hostReport; private boolean isBootReport; @@ -36,12 +36,16 @@ public class DispatchHandleHostReport implements Runnable { public volatile int reportTime = (int) (System.currentTimeMillis() / 1000); public DispatchHandleHostReport(HostReport report, HostReportHandler rqdReportManager) { + super("disp_handle_host_report_" + report.hashCode() + + "_" + rqdReportManager.hashCode()); this.hostReport = report; this.isBootReport = false; this.hostReportHandler = rqdReportManager; } public DispatchHandleHostReport(BootReport report, HostReportHandler rqdReportManager) { + super("disp_handle_host_report_" + report.hashCode() + + "_" + rqdReportManager.hashCode()); HostReport hostReport = HostReport.newBuilder() .setHost(report.getHost()) .setCoreInfo(report.getCoreInfo()) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchJobComplete.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchJobComplete.java index 5cb0a0da7..1910321a8 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchJobComplete.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchJobComplete.java @@ -28,13 +28,15 @@ * * @category command */ -public class DispatchJobComplete implements Runnable { +public class DispatchJobComplete extends KeyRunnable { private JobInterface job; private Source source; private boolean isManualKill; private JobManagerSupport jobManagerSupport; - public DispatchJobComplete(JobInterface job, Source source, boolean isManualKill, JobManagerSupport jobManagerSupport) { + public DispatchJobComplete(JobInterface job, Source source, boolean isManualKill, + JobManagerSupport jobManagerSupport) { + super("disp_job_complete_" + job.getJobId() + "_" + source.toString()); this.job = job; this.source = source; this.isManualKill = isManualKill; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchKillFrames.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchKillFrames.java index 73ce9d97c..986d6bd05 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchKillFrames.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchKillFrames.java @@ -28,13 +28,14 @@ * * @category command */ -public class DispatchKillFrames implements Runnable { +public class DispatchKillFrames extends KeyRunnable { private FrameSearchInterface search; private JobManagerSupport jobManagerSupport; private Source source; public DispatchKillFrames(FrameSearchInterface search, Source source, JobManagerSupport jobManagerSupport) { + super("disp_kill_frames_" + source.toString() + "_" + jobManagerSupport.hashCode()); this.search = search; this.source = source; this.jobManagerSupport = jobManagerSupport; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchKillProcs.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchKillProcs.java index 7c6464eff..d97966139 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchKillProcs.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchKillProcs.java @@ -19,18 +19,19 @@ package com.imageworks.spcue.dispatcher.commands; -import java.util.Collection; - import com.imageworks.spcue.Source; import com.imageworks.spcue.VirtualProc; import com.imageworks.spcue.service.JobManagerSupport; +import java.util.Collection; -public class DispatchKillProcs implements Runnable { +public class DispatchKillProcs extends KeyRunnable { private Collection procs; private JobManagerSupport jobManagerSupport; private Source source; public DispatchKillProcs(Collection procs, Source source, JobManagerSupport jobManagerSupport) { + super("disp_kill_procs_" + procs.hashCode() + "_" + source.toString() + + "_" + jobManagerSupport.hashCode()); this.procs = procs; this.source = source; this.jobManagerSupport = jobManagerSupport; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchLaunchJob.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchLaunchJob.java index cfbcd3eb4..c3682866e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchLaunchJob.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchLaunchJob.java @@ -25,12 +25,13 @@ /** * @category DispatchCommand */ -public class DispatchLaunchJob implements Runnable { +public class DispatchLaunchJob extends KeyRunnable { private JobLauncher jobLauncher; private JobSpec spec; public DispatchLaunchJob(JobSpec spec, JobLauncher jobLauncher) { + super("disp_launch_job_" + spec.getShow() + "_" + spec.getShot() + "_" + spec.getUid()); this.spec = spec; this.jobLauncher = jobLauncher; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchMoveJobs.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchMoveJobs.java index b5c96740b..92ec0db29 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchMoveJobs.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchMoveJobs.java @@ -26,13 +26,15 @@ import com.imageworks.spcue.JobInterface; import com.imageworks.spcue.service.GroupManager; -public class DispatchMoveJobs implements Runnable { +public class DispatchMoveJobs extends KeyRunnable { private GroupDetail group; private List jobs; private GroupManager groupManager; public DispatchMoveJobs(GroupDetail group, List jobs, GroupManager groupManager) { + super("disp_move_jobs_" + group.getGroupId() + "_dept_" + group.getDepartmentId() + + "_show_" + group.getShowId()); this.group = group; this.jobs = jobs; this.groupManager = groupManager; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchNextFrame.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchNextFrame.java index c563a34c0..7e12eb90c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchNextFrame.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchNextFrame.java @@ -28,13 +28,14 @@ * * @category command */ -public class DispatchNextFrame implements Runnable { +public class DispatchNextFrame extends KeyRunnable { private VirtualProc proc; private DispatchJob job; private Dispatcher dispatcher; public DispatchNextFrame(DispatchJob j, VirtualProc p, Dispatcher d) { + super("disp_next_frame_" + j.getJobId() + "_" + p.getProcId()); this.job = j; this.proc = p; this.dispatcher = d; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchReorderFrames.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchReorderFrames.java index b17710f51..528474929 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchReorderFrames.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchReorderFrames.java @@ -25,7 +25,7 @@ import com.imageworks.spcue.service.JobManagerSupport; import com.imageworks.spcue.util.FrameSet; -public class DispatchReorderFrames implements Runnable { +public class DispatchReorderFrames extends KeyRunnable { private JobInterface job = null; private LayerInterface layer = null; @@ -33,14 +33,20 @@ public class DispatchReorderFrames implements Runnable { private Order order; private JobManagerSupport jobManagerSupport; - public DispatchReorderFrames(JobInterface job, FrameSet frameSet, Order order, JobManagerSupport jobManagerSupport) { + public DispatchReorderFrames(JobInterface job, FrameSet frameSet, Order order, + JobManagerSupport jobManagerSupport) { + super("disp_reorder_frames_job_" + job.getJobId() + + "_" + jobManagerSupport.toString()); this.job = job; this.frameSet = frameSet; this.order = order; this.jobManagerSupport = jobManagerSupport; } - public DispatchReorderFrames(LayerInterface layer, FrameSet frameSet, Order order, JobManagerSupport jobManagerSupport) { + public DispatchReorderFrames(LayerInterface layer, FrameSet frameSet, Order order, + JobManagerSupport jobManagerSupport) { + super("disp_reorder_frames_layer_" + layer.getLayerId() + + "_" + jobManagerSupport.toString()); this.layer = layer; this.frameSet = frameSet; this.order = order; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRetryFrames.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRetryFrames.java index 5d9037032..8546423dd 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRetryFrames.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRetryFrames.java @@ -28,7 +28,7 @@ * * @category command */ -public class DispatchRetryFrames implements Runnable { +public class DispatchRetryFrames extends KeyRunnable { private FrameSearchInterface search; private Source source; @@ -36,6 +36,7 @@ public class DispatchRetryFrames implements Runnable { public DispatchRetryFrames(FrameSearchInterface search, Source source, JobManagerSupport jobManagerSupport) { + super("disp_retry_frames_" + search.hashCode() + "_" + source.toString()); this.search = search; this.source = source; this.jobManagerSupport = jobManagerSupport; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrame.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrame.java index 4b7af8328..3a43c6fc3 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrame.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrame.java @@ -21,11 +21,12 @@ import org.apache.log4j.Logger; +import com.imageworks.spcue.dispatcher.commands.KeyRunnable; import com.imageworks.spcue.VirtualProc; import com.imageworks.spcue.rqd.RqdClient; import com.imageworks.spcue.rqd.RqdClientException; -public class DispatchRqdKillFrame implements Runnable { +public class DispatchRqdKillFrame extends KeyRunnable { private static final Logger logger = Logger.getLogger(DispatchRqdKillFrame.class); @@ -38,6 +39,7 @@ public class DispatchRqdKillFrame implements Runnable { private final RqdClient rqdClient; public DispatchRqdKillFrame(String hostname, String frameId, String message, RqdClient rqdClient) { + super("disp_rqd_kill_frame_" + hostname + "_" + frameId + "_" + rqdClient.toString()); this.hostname = hostname; this.frameId = frameId; this.message = message; @@ -45,6 +47,7 @@ public DispatchRqdKillFrame(String hostname, String frameId, String message, Rqd } public DispatchRqdKillFrame(VirtualProc proc, String message, RqdClient rqdClient) { + super("disp_rqd_kill_frame_" + proc.getProcId() + "_" + rqdClient.toString()); this.proc = proc; this.hostname = proc.hostName; this.message = message; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchSatisfyDepends.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchSatisfyDepends.java index 7016e526f..5294a203c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchSatisfyDepends.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchSatisfyDepends.java @@ -30,7 +30,7 @@ * * @category command */ -public class DispatchSatisfyDepends implements Runnable { +public class DispatchSatisfyDepends extends KeyRunnable { private JobInterface job = null; private LayerInterface layer = null; @@ -39,21 +39,25 @@ public class DispatchSatisfyDepends implements Runnable { private JobManagerSupport jobManagerSupport; public DispatchSatisfyDepends(JobInterface job, JobManagerSupport jobManagerSupport) { + super("disp_sat_deps_" + job.getJobId() + "_" + jobManagerSupport.toString()); this.job = job; this.jobManagerSupport = jobManagerSupport; } public DispatchSatisfyDepends(LayerInterface layer, JobManagerSupport jobManagerSupport) { + super("disp_sat_deps_" + layer.getLayerId() + "_" + jobManagerSupport.toString()); this.layer = layer; this.jobManagerSupport = jobManagerSupport; } public DispatchSatisfyDepends(FrameInterface frame, JobManagerSupport jobManagerSupport) { + super("disp_sat_deps_" + frame.getFrameId() + "_" + jobManagerSupport.toString()); this.frame = frame; this.jobManagerSupport = jobManagerSupport; } public DispatchSatisfyDepends(FrameSearchInterface search, JobManagerSupport jobManagerSupport) { + super("disp_sat_deps_" + search.hashCode() + "_" + jobManagerSupport.hashCode()); this.search = search; this.jobManagerSupport = jobManagerSupport; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchShutdownJobIfCompleted.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchShutdownJobIfCompleted.java index 15ddc805e..b4eb11a07 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchShutdownJobIfCompleted.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchShutdownJobIfCompleted.java @@ -29,11 +29,12 @@ * * @category command */ -public class DispatchShutdownJobIfCompleted implements Runnable { +public class DispatchShutdownJobIfCompleted extends KeyRunnable { private JobInterface job; private JobManagerSupport jobManagerSupport; public DispatchShutdownJobIfCompleted(JobInterface job, JobManagerSupport jobManagerSupport) { + super("disp_st_job_comp_" + job.getJobId()); this.job = job; this.jobManagerSupport = jobManagerSupport; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchStaggerFrames.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchStaggerFrames.java index d40e5e185..b0430b892 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchStaggerFrames.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchStaggerFrames.java @@ -23,7 +23,7 @@ import com.imageworks.spcue.LayerInterface; import com.imageworks.spcue.service.JobManagerSupport; -public class DispatchStaggerFrames implements Runnable { +public class DispatchStaggerFrames extends KeyRunnable { private JobInterface job = null; private LayerInterface layer = null; @@ -32,6 +32,7 @@ public class DispatchStaggerFrames implements Runnable { private JobManagerSupport jobManagerSupport; public DispatchStaggerFrames(JobInterface job, String range, int stagger, JobManagerSupport jobManagerSupport) { + super("disp_stag_frames_" + job.getJobId() + "_" + range); this.job = job; this.range = range; this.stagger = stagger; @@ -39,6 +40,7 @@ public DispatchStaggerFrames(JobInterface job, String range, int stagger, JobMan } public DispatchStaggerFrames(LayerInterface layer, String range, int stagger, JobManagerSupport jobManagerSupport) { + super("disp_stag_frames_" + layer.getLayerId() + "_" + range); this.layer = layer; this.range = range; this.stagger = stagger; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/KeyRunnable.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/KeyRunnable.java new file mode 100644 index 000000000..bdbdb87da --- /dev/null +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/KeyRunnable.java @@ -0,0 +1,41 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.imageworks.spcue.dispatcher.commands; + +import com.imageworks.spcue.DispatchHost; +import com.imageworks.spcue.dispatcher.Dispatcher; + +public abstract class KeyRunnable implements Runnable { + + private String key; + + public KeyRunnable(String key) { + this.key = key; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } +} + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/ManageReparentHosts.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/ManageReparentHosts.java index 1dfc2ca9d..15ab1384e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/ManageReparentHosts.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/ManageReparentHosts.java @@ -22,15 +22,17 @@ import java.util.List; import com.imageworks.spcue.AllocationInterface; +import com.imageworks.spcue.dispatcher.commands.KeyRunnable; import com.imageworks.spcue.HostInterface; import com.imageworks.spcue.service.HostManager; -public class ManageReparentHosts implements Runnable { +public class ManageReparentHosts extends KeyRunnable { AllocationInterface alloc; List hosts; HostManager hostManager; public ManageReparentHosts(AllocationInterface alloc, List hosts, HostManager hostManager) { + super(alloc.getAllocationId()); this.alloc = alloc; this.hosts = hosts; this.hostManager = hostManager; diff --git a/cuebot/src/main/java/com/imageworks/spcue/rqd/RqdClientGrpc.java b/cuebot/src/main/java/com/imageworks/spcue/rqd/RqdClientGrpc.java index d9ad91cfd..5e1e016fa 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/rqd/RqdClientGrpc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/rqd/RqdClientGrpc.java @@ -122,7 +122,7 @@ public void setHostLock(HostInterface host, LockState lock) { logger.debug("Locking RQD host"); lockHost(host); } else { - logger.debug("Unkown LockState passed to setHostLock."); + logger.debug("Unknown LockState passed to setHostLock."); } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/CueStatic.java b/cuebot/src/main/java/com/imageworks/spcue/servant/CueStatic.java index f7d5437af..d2c8b17e1 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/CueStatic.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/CueStatic.java @@ -44,17 +44,17 @@ public class CueStatic extends CueInterfaceGrpc.CueInterfaceImplBase { public void getSystemStats(CueGetSystemStatsRequest request, StreamObserver responseObserver) { SystemStats stats = SystemStats.newBuilder() - .setDispatchThreads(dispatchQueue.getActiveThreadCount()) - .setDispatchWaiting(dispatchQueue.getWaitingCount()) + .setDispatchThreads(dispatchQueue.getActiveCount()) + .setDispatchWaiting(dispatchQueue.getSize()) .setDispatchRemainingCapacity(dispatchQueue.getRemainingCapacity()) - .setDispatchExecuted(dispatchQueue.getTotalDispatched()) - .setDispatchRejected(dispatchQueue.getTotalRejected()) + .setDispatchExecuted(dispatchQueue.getCompletedTaskCount()) + .setDispatchRejected(dispatchQueue.getRejectedTaskCount()) - .setManageThreads(manageQueue.getActiveThreadCount()) - .setManageWaiting(manageQueue.getWaitingCount()) + .setManageThreads(manageQueue.getActiveCount()) + .setManageWaiting(manageQueue.getSize()) .setManageRemainingCapacity(manageQueue.getRemainingCapacity()) - .setManageExecuted(manageQueue.getTotalDispatched()) - .setManageRejected(manageQueue.getTotalRejected()) + .setManageExecuted(manageQueue.getCompletedTaskCount()) + .setManageRejected(manageQueue.getRejectedTaskCount()) .setReportThreads(reportQueue.getActiveCount()) .setReportWaiting(reportQueue.getQueue().size()) @@ -62,12 +62,12 @@ public void getSystemStats(CueGetSystemStatsRequest request, .setReportExecuted(reportQueue.getTaskCount()) .setReportRejected(reportQueue.getRejectedTaskCount()) - .setBookingWaiting(bookingQueue.getQueue().size()) - .setBookingRemainingCapacity(bookingQueue.getQueue().remainingCapacity()) + .setBookingWaiting(bookingQueue.getSize()) + .setBookingRemainingCapacity(bookingQueue.getRemainingCapacity()) .setBookingThreads(bookingQueue.getActiveCount()) .setBookingExecuted(bookingQueue.getCompletedTaskCount()) .setBookingRejected(bookingQueue.getRejectedTaskCount()) - .setBookingSleepMillis(bookingQueue.sleepTime()) + .setBookingSleepMillis(0) .setHostBalanceSuccess(DispatchSupport.balanceSuccess.get()) .setHostBalanceFailed(DispatchSupport.balanceFailed.get()) @@ -76,7 +76,7 @@ public void getSystemStats(CueGetSystemStatsRequest request, .setClearedProcs(DispatchSupport.clearedProcs.get()) .setBookingRetries(DispatchSupport.bookingRetries.get()) .setBookingErrors(DispatchSupport.bookingErrors.get()) - .setBookedProcs( DispatchSupport.bookedProcs.get()) + .setBookedProcs(DispatchSupport.bookedProcs.get()) // TODO(gregdenton) Reimplement these with gRPC. (Issue #69) // .setReqForData(IceServer.dataRequests.get()) diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageDepend.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageDepend.java index bb3837c26..5d1c2ab6e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageDepend.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageDepend.java @@ -26,6 +26,7 @@ import com.imageworks.spcue.LightweightDependency; import com.imageworks.spcue.dispatcher.DispatchQueue; +import com.imageworks.spcue.dispatcher.commands.KeyRunnable; import com.imageworks.spcue.grpc.depend.DependGetDependRequest; import com.imageworks.spcue.grpc.depend.DependGetDependResponse; import com.imageworks.spcue.grpc.depend.DependInterfaceGrpc; @@ -62,7 +63,8 @@ public void getDepend(DependGetDependRequest request, StreamObserver responseObserver) { LightweightDependency depend = dependManager.getDepend(request.getDepend().getId()); - manageQueue.execute(new Runnable() { + String key = "manage_dep_sat_req_" + request.getDepend().getId(); + manageQueue.execute(new KeyRunnable(key) { public void run() { try { logger.info("dropping dependency: " + depend.id); diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageFilter.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageFilter.java index 15f6163bc..8749a5787 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageFilter.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageFilter.java @@ -31,6 +31,7 @@ import com.imageworks.spcue.dao.FilterDao; import com.imageworks.spcue.dao.GroupDao; import com.imageworks.spcue.dispatcher.DispatchQueue; +import com.imageworks.spcue.dispatcher.commands.KeyRunnable; import com.imageworks.spcue.grpc.filter.Action; import com.imageworks.spcue.grpc.filter.ActionSeq; import com.imageworks.spcue.grpc.filter.Filter; @@ -125,7 +126,8 @@ public void createMatcher(FilterCreateMatcherRequest request, @Override public void delete(FilterDeleteRequest request, StreamObserver responseObserver) { FilterEntity filter = getFilterEntity(request.getFilter()); - manageQueue.execute(new Runnable() { + String key = "manage_filter_del_req_" + filter.getId(); + manageQueue.execute(new KeyRunnable(key) { public void run() { filterManager.deleteFilter(filter); } diff --git a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml index 521c74dd1..12e0889d8 100644 --- a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml +++ b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml @@ -54,33 +54,33 @@ - - - - LaunchQueue - - - dispatcher.launch_queue - + + + + - - - - DispatchPool - - - dispatcher.dispatch_pool - + + + + - - - - ManagePool + + + ${booking_queue.threadpool.health_threshold} - - dispatcher.manage_pool + + ${healthy_threadpool.min_unhealthy_period_min} + + + ${booking_queue.threadpool.queue_capacity} + + + ${booking_queue.threadpool.core_pool_size} + + + ${booking_queue.threadpool.max_pool_size} @@ -88,45 +88,67 @@ DispatchQueue - + + ${healthy_threadpool.health_threshold} + + + ${healthy_threadpool.min_unhealthy_period_min} + + + ${dispatch.threadpool.queue_capacity} + + + ${dispatch.threadpool.core_pool_size} + + + ${dispatch.threadpool.max_pool_size} + - + ManageQueue - + + ${healthy_threadpool.health_threshold} + + + ${healthy_threadpool.min_unhealthy_period_min} + + + ${dispatch.threadpool.queue_capacity} + + + ${dispatch.threadpool.core_pool_size} + + + ${dispatch.threadpool.max_pool_size} + - - - - ReportQueue + + ${report_queue.threadPoolSizeInitial} - - dispatcher.report_queue + + ${report_queue.threadPoolSizeMax} + + + ${report_queue.queueSize} - - - KillQueue - - - dispatcher.kill_queue + + ${kill_queue.threadPoolSizeInitial} - - - - - - dispatcher.booking_queue + + ${kill_queue.threadPoolSizeMax} - 300 + ${kill_queue.queueSize} + @@ -341,8 +363,8 @@ - + @@ -411,6 +433,76 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cuebot/src/main/resources/log4j.properties b/cuebot/src/main/resources/log4j.properties index 138d514ae..5f44dbee9 100644 --- a/cuebot/src/main/resources/log4j.properties +++ b/cuebot/src/main/resources/log4j.properties @@ -28,6 +28,15 @@ log4j.appender.API.MaxBackupIndex=20 log4j.appender.API.layout=org.apache.log4j.PatternLayout log4j.appender.API.layout.ConversionPattern=%d:%m%n +log4j.category.HEALTH=DEBUG, HEALTH +log4j.additivity.HEALTH=false +log4j.appender.HEALTH=org.apache.log4j.RollingFileAppender +log4j.appender.HEALTH.File=logs/health.log +log4j.appender.HEALTH.MaxFileSize=10MB +log4j.appender.HEALTH.MaxBackupIndex=20 +log4j.appender.HEALTH.layout=org.apache.log4j.PatternLayout +log4j.appender.HEALTH.layout.ConversionPattern=%d:%m%n + log4j.logger.org.apache.catalina=INFO log4j.logger.com.imageworks.spcue=DEBUG log4j.logger.com.imageworks.spcue.dispatcher.RqdReportManagerService=DEBUG diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index 220cc2d5f..dea35cd30 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -21,6 +21,27 @@ grpc.rqd_cache_concurrency=20 # RQD Channel task deadline in seconds grpc.rqd_task_deadline=10 +# Healthy Threadpool Executor +booking_queue.threadpool.health_threshold=10 +booking_queue.threadpool.core_pool_size=10 +booking_queue.threadpool.max_pool_size=14 +booking_queue.threadpool.queue_capacity=2000 +dispatch.threadpool.core_pool_size=6 +dispatch.threadpool.max_pool_size=8 +dispatch.threadpool.queue_capacity=2000 +healthy_threadpool.health_threshold=6 +healthy_threadpool.min_unhealthy_period_min=3 +report_queue.threadPoolSizeInitial=6 +report_queue.threadPoolSizeMax=12 +# The queue size should be bigger then the expected amount of hosts +report_queue.queueSize=5000 +kill_queue.threadPoolSizeInitial=2 +kill_queue.threadPoolSizeMax=6 +kill_queue.queueSize=1000 + +# Turn on/off jobCompletion mailing module +mailing.enabled=true + # Whether or not to enable publishing to a messaging topic. # Set to a boolean value. See com/imageworks/spcue/services/JmsMover.java. messaging.enabled=false @@ -34,9 +55,9 @@ log.frame-log-root=${CUE_FRAME_LOG_DIR:/shots} dispatcher.job_query_max=20 # Number of seconds before waiting to book the same job from a different host. # "0" disables the job_lock -dispatcher.job_lock_expire_seconds=0 +dispatcher.job_lock_expire_seconds=20 # Concurrency level to allow on the job lock cache -dispatcher.job_lock_concurrency_level=3 +dispatcher.job_lock_concurrency_level=14 # Maximum number of frames to query from the DB to attempt to dispatch. dispatcher.frame_query_max=20 # Maximum number of frames to book at one time on the same host. diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java index 319c4cfe1..74b21c102 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java @@ -89,11 +89,18 @@ public void create() { @Rollback(true) public void testBookingQueue() { + int healthThreshold = 10; + int minUnhealthyPeriodMin = 3; + int queueCapacity = 2000; + int corePoolSize = 10; + int maxPoolSize = 14; + DispatchHost host1 = hostDao.findDispatchHost(HOSTNAME); host1.idleCores = 500; DispatchHost host2 = hostDao.findDispatchHost(HOSTNAME); DispatchHost host3 = hostDao.findDispatchHost(HOSTNAME); - + BookingQueue queue = new BookingQueue(healthThreshold, minUnhealthyPeriodMin, queueCapacity, + corePoolSize, maxPoolSize); bookingQueue.execute(new DispatchBookHost(host2,dispatcher)); bookingQueue.execute(new DispatchBookHost(host3,dispatcher)); bookingQueue.execute(new DispatchBookHost(host1,dispatcher)); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/ThreadPoolTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/ThreadPoolTests.java deleted file mode 100644 index 76c9a3e0e..000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/ThreadPoolTests.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.imageworks.spcue.test.dispatcher; - -import java.util.concurrent.ThreadPoolExecutor; -import javax.annotation.Resource; - -import com.imageworks.spcue.config.TestAppConfig; -import com.imageworks.spcue.dispatcher.BookingQueue; -import com.imageworks.spcue.dispatcher.HostReportQueue; -import com.imageworks.spcue.dispatcher.ThreadPoolTaskExecutorWrapper; - -import org.junit.Test; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests; -import org.springframework.test.context.support.AnnotationConfigContextLoader; - -import static org.junit.Assert.assertEquals; - -@ContextConfiguration(classes=TestAppConfig.class, loader=AnnotationConfigContextLoader.class) -public class ThreadPoolTests extends AbstractTransactionalJUnit4SpringContextTests { - - @Resource - ThreadPoolTaskExecutor launchQueue; - - @Resource - ThreadPoolTaskExecutor dispatchPool; - - @Resource - ThreadPoolTaskExecutor managePool; - - @Resource - ThreadPoolExecutor reportQueue; - - @Resource - ThreadPoolExecutor killQueue; - - @Resource - BookingQueue bookingQueue; - - private int getQueueCapacity(ThreadPoolTaskExecutor t) { - return ((ThreadPoolTaskExecutorWrapper) t).getQueueCapacity(); - } - - private int getQueueCapacity(ThreadPoolExecutor t) { - return ((HostReportQueue) t).getQueueCapacity(); - } - - @Test - public void testPropertyValues() { - assertEquals(1, launchQueue.getCorePoolSize()); - assertEquals(1, launchQueue.getMaxPoolSize()); - assertEquals(100, getQueueCapacity(launchQueue)); - - assertEquals(4, dispatchPool.getCorePoolSize()); - assertEquals(4, dispatchPool.getMaxPoolSize()); - assertEquals(500, getQueueCapacity(dispatchPool)); - - assertEquals(8, managePool.getCorePoolSize()); - assertEquals(8, managePool.getMaxPoolSize()); - assertEquals(250, getQueueCapacity(managePool)); - - assertEquals(6, reportQueue.getCorePoolSize()); - assertEquals(8, reportQueue.getMaximumPoolSize()); - assertEquals(1000, getQueueCapacity(reportQueue)); - - assertEquals(6, killQueue.getCorePoolSize()); - assertEquals(8, killQueue.getMaximumPoolSize()); - assertEquals(1000, getQueueCapacity(killQueue)); - - assertEquals(6, bookingQueue.getCorePoolSize()); - assertEquals(6, bookingQueue.getMaximumPoolSize()); - assertEquals(1000, bookingQueue.getQueueCapacity()); - } -} - diff --git a/cuebot/src/test/resources/opencue.properties b/cuebot/src/test/resources/opencue.properties index d492f0a87..5c9264492 100644 --- a/cuebot/src/test/resources/opencue.properties +++ b/cuebot/src/test/resources/opencue.properties @@ -13,6 +13,24 @@ grpc.rqd_cache_concurrency=20 # RQD Channel task deadline in seconds grpc.rqd_task_deadline=10 +# Healthy Threadpool Executor +booking_queue.threadpool.health_threshold=10 +booking_queue.threadpool.core_pool_size=10 +booking_queue.threadpool.max_pool_size=14 +booking_queue.threadpool.queue_capacity=2000 +dispatch.threadpool.core_pool_size=6 +dispatch.threadpool.max_pool_size=8 +dispatch.threadpool.queue_capacity=2000 +healthy_threadpool.health_threshold=6 +healthy_threadpool.min_unhealthy_period_min=3 +report_queue.threadPoolSizeInitial=6 +report_queue.threadPoolSizeMax=12 +# The queue size should be bigger then the expected amount of hosts +report_queue.queueSize=5000 +kill_queue.threadPoolSizeInitial=2 +kill_queue.threadPoolSizeMax=6 +kill_queue.queueSize=1000 + log.frame-log-root=/arbitraryLogDirectory dispatcher.job_query_max=20 From 75c76394b2e53d48bb1a29bb88a0e1e8d2b56a01 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 30 Mar 2022 13:55:50 -0400 Subject: [PATCH 172/277] Upgrade Flyway and fix Dockerfile. (#1110) --- sandbox/flyway.Dockerfile | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/sandbox/flyway.Dockerfile b/sandbox/flyway.Dockerfile index 900c4449c..d3d45b998 100644 --- a/sandbox/flyway.Dockerfile +++ b/sandbox/flyway.Dockerfile @@ -1,16 +1,20 @@ FROM centos +ARG FLYWAY_VERSION=8.5.4 + # Get flyway -RUN ["curl", "-O", "https://repo1.maven.org/maven2/org/flywaydb/flyway-commandline/6.0.0/flyway-commandline-6.0.0-linux-x64.tar.gz"] -RUN ["yum", "install", "-y", "tar", "java-1.8.0-openjdk", "postgresql-jdbc", "nc", "postgresql"] -RUN ["tar", "-xzf", "flyway-commandline-6.0.0-linux-x64.tar.gz"] +RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* +RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* +RUN yum install -y tar java-1.8.0-openjdk postgresql-jdbc nc postgresql +RUN curl -O https://repo1.maven.org/maven2/org/flywaydb/flyway-commandline/${FLYWAY_VERSION}/flyway-commandline-${FLYWAY_VERSION}-linux-x64.tar.gz +RUN tar -xzf flyway-commandline-${FLYWAY_VERSION}-linux-x64.tar.gz -WORKDIR flyway-6.0.0 +WORKDIR flyway-${FLYWAY_VERSION} # Copy the postgres driver to its required location -RUN ["cp", "/usr/share/java/postgresql-jdbc.jar", "jars/"] -RUN ["mkdir", "/opt/migrations"] -RUN ["mkdir", "/opt/scripts"] +RUN cp /usr/share/java/postgresql-jdbc.jar jars/ +RUN mkdir /opt/migrations +RUN mkdir /opt/scripts COPY ./cuebot/src/main/resources/conf/ddl/postgres/migrations /opt/migrations COPY ./cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql /opt/scripts COPY ./sandbox/migrate.sh /opt/scripts/ From d899b71fea69fa1f3a589665a4681c5aff44ba0e Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 30 Mar 2022 13:59:00 -0400 Subject: [PATCH 173/277] [cuegui] Split config code into a new module. (#1095) --- cuegui/cuegui/Config.py | 63 ++++++++++++++++++++++++ cuegui/cuegui/Main.py | 27 +---------- cuegui/tests/Config_tests.py | 93 ++++++++++++++++++++++++++++++++++++ 3 files changed, 158 insertions(+), 25 deletions(-) create mode 100644 cuegui/cuegui/Config.py create mode 100644 cuegui/tests/Config_tests.py diff --git a/cuegui/cuegui/Config.py b/cuegui/cuegui/Config.py new file mode 100644 index 000000000..2c80f8492 --- /dev/null +++ b/cuegui/cuegui/Config.py @@ -0,0 +1,63 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Functions for loading application state and settings from disk.""" + +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import + +import os +import shutil + +from PySide2 import QtCore + +import cuegui.Constants +import cuegui.Logger + +logger = cuegui.Logger.getLogger(__file__) + + +def startup(app_name): + """ + Reads config from disk, restoring default config if necessary. + + :param app_name: application window name + :type app_name: str + :return: settings object containing the loaded settings + :rtype: QtCore.QSettings + """ + # read saved config from disk + # copy default config + config_path = "/.%s/config" % app_name.lower() + settings = QtCore.QSettings(QtCore.QSettings.IniFormat, QtCore.QSettings.UserScope, config_path) + logger.info('Reading config file from %s', settings.fileName()) + local = settings.fileName() + + # If the user has chose to revert the layout. delete the file and copy the default back. + if settings.value('RevertLayout'): + logger.warning('Found RevertLayout flag, will restore default config') + os.remove(local) + + # If the config file does not exist, copy over the default + if not os.path.exists(local): + default = os.path.join(cuegui.Constants.DEFAULT_INI_PATH, "%s.ini" % app_name.lower()) + logger.warning('Local config file not found at %s', local) + logger.warning('Copying %s to %s', default, local) + os.makedirs(os.path.dirname(local), exist_ok=True) + shutil.copy2(default, local) + settings.sync() + + return settings diff --git a/cuegui/cuegui/Main.py b/cuegui/cuegui/Main.py index b97e3d885..58bb3073d 100644 --- a/cuegui/cuegui/Main.py +++ b/cuegui/cuegui/Main.py @@ -20,14 +20,13 @@ from __future__ import print_function from __future__ import division -import os -import shutil import signal from PySide2 import QtCore from PySide2 import QtGui from PySide2 import QtWidgets +import cuegui.Config import cuegui.Constants import cuegui.Logger import cuegui.MainWindow @@ -92,33 +91,11 @@ def startup(app_name, app_version, argv): QtGui.qApp.threads = [] # pylint: enable=attribute-defined-outside-init - config_path = "/.%s/config" % app_name.lower() - settings = QtCore.QSettings(QtCore.QSettings.IniFormat, QtCore.QSettings.UserScope, config_path) - local = settings.fileName() - # If the user has chose to revert the layout. delete the file and copy the default back. - if settings.value('RevertLayout'): - os.remove(local) - + settings = cuegui.Config.startup(app_name) QtGui.qApp.settings = settings # pylint: disable=attribute-defined-outside-init cuegui.Style.init() - # If the config file does not exist, copy over the default - # pylint: disable=broad-except - if not os.path.exists(local): - default = os.path.join(cuegui.Constants.DEFAULT_INI_PATH, "%s.ini" % app_name.lower()) - logger.warning('Not found: %s\nCopying: %s', local, default) - try: - os.mkdir(os.path.dirname(local)) - except Exception as e: - logger.debug(e) - try: - shutil.copy2(default, local) - except Exception as e: - logger.debug(e) - settings.sync() - # pylint: enable=broad-except - mainWindow = cuegui.MainWindow.MainWindow(app_name, app_version, None) mainWindow.displayStartupNotice() mainWindow.show() diff --git a/cuegui/tests/Config_tests.py b/cuegui/tests/Config_tests.py new file mode 100644 index 000000000..61ab2c193 --- /dev/null +++ b/cuegui/tests/Config_tests.py @@ -0,0 +1,93 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Tests for cuegui.Config""" + + +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import + +import os +import shutil +import tempfile +import unittest + +from PySide2 import QtCore + +import cuegui.Config + + +CONFIG_INI = ''' +[General] +Version=0.14 + +[CueCommander] +Open=true +Title=CustomWindowTitle +OtherAttr=arbitrary-value +''' + +CONFIG_WITH_RESTORE_FLAG = ''' +[General] +Version=0.14 +RevertLayout=true + +[CueCommander] +OtherAttr=arbitrary-value +''' + + +class ConfigTests(unittest.TestCase): + def setUp(self): + self.config_dir = tempfile.mkdtemp() + QtCore.QSettings.setPath( + QtCore.QSettings.IniFormat, QtCore.QSettings.UserScope, self.config_dir) + + def tearDown(self): + shutil.rmtree(self.config_dir) + + def test__should_load_user_config(self): + app_name = 'arbitraryapp' + config_file_path = os.path.join(self.config_dir, '.%s' % app_name, 'config.ini') + os.mkdir(os.path.dirname(config_file_path)) + with open(config_file_path, 'w') as fp: + fp.write(CONFIG_INI) + + settings = cuegui.Config.startup(app_name) + + self.assertEqual('0.14', settings.value('Version')) + self.assertEqual('true', settings.value('CueCommander/Open')) + self.assertEqual('CustomWindowTitle', settings.value('CueCommander/Title')) + self.assertEqual('arbitrary-value', settings.value('CueCommander/OtherAttr')) + + def test__should_load_default_config(self): + settings = cuegui.Config.startup('CueCommander') + + self.assertEqual('false', settings.value('CueCommander/Open')) + self.assertEqual('CueCommander', settings.value('CueCommander/Title')) + self.assertFalse(settings.value('CueCommander/OtherAttr', False)) + + def test__should_restore_default_config(self): + config_file_path = os.path.join(self.config_dir, '.cuecommander', 'config.ini') + os.mkdir(os.path.dirname(config_file_path)) + with open(config_file_path, 'w') as fp: + fp.write(CONFIG_WITH_RESTORE_FLAG) + + settings = cuegui.Config.startup('CueCommander') + + self.assertEqual('false', settings.value('CueCommander/Open')) + self.assertEqual('CueCommander', settings.value('CueCommander/Title')) + self.assertFalse(settings.value('CueCommander/OtherAttr', False)) From 6993baa1bea1a32d8a0c6554f6b91bfecd774906 Mon Sep 17 00:00:00 2001 From: Akim Ruslanov Date: Fri, 1 Apr 2022 15:26:10 -0700 Subject: [PATCH 174/277] Set smtp_host as env variable (#1119) --- .../java/com/imageworks/spcue/util/CueUtil.java | 17 ++++++++++++++++- cuebot/src/main/resources/opencue.properties | 3 +++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java b/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java index 990511f47..2a7438f49 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java +++ b/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java @@ -35,6 +35,7 @@ import java.util.regex.Pattern; import javax.activation.DataHandler; import javax.activation.DataSource; +import javax.annotation.PostConstruct; import javax.mail.BodyPart; import javax.mail.Message; import javax.mail.Session; @@ -47,17 +48,24 @@ import org.apache.log4j.Logger; import org.springframework.core.env.Environment; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; import com.imageworks.spcue.LayerInterface; import com.imageworks.spcue.SpcueRuntimeException; import com.imageworks.spcue.dispatcher.Dispatcher; + /** * CueUtil is set of common methods used throughout the application. */ +@Component public final class CueUtil { private static final Logger logger = Logger.getLogger(CueUtil.class); + private static String smtpHost = ""; + @Autowired + private Environment env; /** * Commonly used macros for gigabyte values in KB. @@ -88,6 +96,11 @@ public final class CueUtil { */ public static final int ONE_HOUR = 3600; + @PostConstruct + public void init() { + CueUtil.smtpHost = this.env.getRequiredProperty("smtp_host", String.class); + } + /** * Return true if the given name is formatted as a valid * allocation name. Allocation names should be facility.unique_name. @@ -157,7 +170,7 @@ public static int findChunk(List dependOnFrames, int dependErFrame) { public static void sendmail(String to, String from, String subject, StringBuilder body, Map images) { try { Properties props = System.getProperties(); - props.put("mail.smtp.host", "smtp"); + props.put("mail.smtp.host", CueUtil.smtpHost); Session session = Session.getDefaultInstance(props, null); Message msg = new MimeMessage(session); msg.setFrom(new InternetAddress(from)); @@ -189,6 +202,8 @@ public static void sendmail(String to, String from, String subject, StringBuilde msg.setContent(mimeMultipart); msg.setHeader("X-Mailer", "OpenCueMailer"); msg.setSentDate(new Date()); + Transport transport = session.getTransport("smtp"); + transport.connect(CueUtil.smtpHost, null, null); Transport.send(msg); } catch (Exception e) { diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index dea35cd30..e64f6e417 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -117,3 +117,6 @@ history.archive_jobs_cutoff_hours=72 # Delete down hosts automatically. maintenance.auto_delete_down_hosts=false + +# Set hostname/IP of the smtp host. Will be used for mailing +smtp_host=smtp From 64366235416810a2fe8f9cde11b7028130fc32bb Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Fri, 8 Apr 2022 11:57:07 -0700 Subject: [PATCH 175/277] Fix NIMBY lock and refactor Nimby to Factory (#1106) * Fix NIMBY lock and refactor Nimby to Factory Nimby now has two modes, Select and Pynput. Windows hosts will activate the Pynput mode by default. Bug fix: Two changes in rqcore.py: start() under isDesktop to call onNimbyLock() if nimby override is true. Second in launchFrame, added another condition if Nimby override is True and user is active to not launch frame. Revamped logging, removed print statements. * Fix Pylint errors * Adding logging for pyput import failures --- rqd/rqd/__main__.py | 17 +-- rqd/rqd/cuerqd.py | 4 +- rqd/rqd/rqconstants.py | 15 +- rqd/rqd/rqcore.py | 17 ++- rqd/rqd/rqdservicers.py | 5 +- rqd/rqd/rqmachine.py | 3 +- rqd/rqd/rqnetwork.py | 7 +- rqd/rqd/rqnimby.py | 283 ++++++++++++++++++++++++++++------- rqd/rqd/rqswap.py | 3 +- rqd/rqd/rqutil.py | 3 +- rqd/tests/rqcore_tests.py | 4 +- rqd/tests/rqmachine_tests.py | 2 +- rqd/tests/rqnimby_tests.py | 8 +- 13 files changed, 293 insertions(+), 78 deletions(-) diff --git a/rqd/rqd/__main__.py b/rqd/rqd/__main__.py index 4c0a46bfe..32e6dd38b 100755 --- a/rqd/rqd/__main__.py +++ b/rqd/rqd/__main__.py @@ -61,15 +61,13 @@ def setupLogging(): """Sets up the logging for RQD. + Logs to /var/log/messages""" - Logs to /var/log/messages""" - # TODO(bcipriano) These should be config based. (Issue #72) - consoleFormat = '%(asctime)s %(levelname)-9s rqd3-%(module)-10s %(message)s' - consoleLevel = logging.DEBUG - fileFormat = '%(asctime)s %(levelname)-9s rqd3-%(module)-10s %(message)s' - fileLevel = logging.WARNING # Equal to or greater than the consoleLevel + consolehandler = logging.StreamHandler() + consolehandler.setLevel(rqd.rqconstants.CONSOLE_LOG_LEVEL) + consolehandler.setFormatter(logging.Formatter(rqd.rqconstants.LOG_FORMAT)) + logging.getLogger('').addHandler(consolehandler) - logging.basicConfig(level=consoleLevel, format=consoleFormat) if platform.system() in ('Linux', 'Darwin'): if platform.system() == 'Linux': syslogAddress = '/dev/log' @@ -83,9 +81,10 @@ def setupLogging(): logfile = logging.FileHandler(os.path.expandvars('%TEMP%/openrqd.log')) else: logfile = logging.handlers.SysLogHandler() - logfile.setLevel(fileLevel) - logfile.setFormatter(logging.Formatter(fileFormat)) + logfile.setLevel(rqd.rqconstants.FILE_LOG_LEVEL) + logfile.setFormatter(logging.Formatter(rqd.rqconstants.LOG_FORMAT)) logging.getLogger('').addHandler(logfile) + logging.getLogger('').setLevel(logging.DEBUG) def usage(): diff --git a/rqd/rqd/cuerqd.py b/rqd/rqd/cuerqd.py index 005c92d4f..c80098579 100755 --- a/rqd/rqd/cuerqd.py +++ b/rqd/rqd/cuerqd.py @@ -60,13 +60,13 @@ def getRunningFrame(self, frameId): def nimbyOff(self): """Disables Nimby on the host.""" - print(self.rqdHost, "Turning off Nimby") + log.info(self.rqdHost, "Turning off Nimby") log.info("rqd nimbyoff by %s", os.environ.get("USER")) self.stub.NimbyOff(rqd.compiled_proto.rqd_pb2.RqdStaticNimbyOffRequest()) def nimbyOn(self): """Enables Nimby on the host.""" - print(self.rqdHost, "Turning on Nimby") + log.info(self.rqdHost, "Turning on Nimby") log.info("rqd nimbyon by %s", os.environ.get("USER")) self.stub.NimbyOn(rqd.compiled_proto.rqd_pb2.RqdStaticNimbyOnRequest()) diff --git a/rqd/rqd/rqconstants.py b/rqd/rqd/rqconstants.py index 07a621456..14979ee95 100644 --- a/rqd/rqd/rqconstants.py +++ b/rqd/rqd/rqconstants.py @@ -116,10 +116,15 @@ OVERRIDE_PROCS = None # number of physical cpus. ex: None or 2 OVERRIDE_MEMORY = None # in Kb OVERRIDE_NIMBY = None # True to turn on, False to turn off +USE_NIMBY_PYNPUT = platform.system() == 'Windows' OVERRIDE_HOSTNAME = None # Force to use this hostname ALLOW_GPU = False LOAD_MODIFIER = 0 # amount to add/subtract from load +LOG_FORMAT = '%(asctime)s %(levelname)-9s openrqd-%(module)-10s %(message)s' +CONSOLE_LOG_LEVEL = logging.DEBUG +FILE_LOG_LEVEL = logging.WARNING # Equal to or greater than the consoleLevel + if subprocess.getoutput('/bin/su --help').find('session-command') != -1: SU_ARGUMENT = '--session-command' else: @@ -138,8 +143,8 @@ else: ConfigParser = configparser.RawConfigParser config = ConfigParser() - logging.info('Loading config %s', CONFIG_FILE) config.read(CONFIG_FILE) + logging.warning('Loading config %s', CONFIG_FILE) if config.has_option(__section, "RQD_GRPC_PORT"): RQD_GRPC_PORT = config.getint(__section, "RQD_GRPC_PORT") @@ -173,8 +178,16 @@ DEFAULT_FACILITY = config.get(__section, "DEFAULT_FACILITY") if config.has_option(__section, "LAUNCH_FRAME_USER_GID"): LAUNCH_FRAME_USER_GID = config.getint(__section, "LAUNCH_FRAME_USER_GID") + if config.has_option(__section, "CONSOLE_LOG_LEVEL"): + level = config.get(__section, "CONSOLE_LOG_LEVEL") + CONSOLE_LOG_LEVEL = logging.getLevelName(level) + if config.has_option(__section, "FILE_LOG_LEVEL"): + level = config.get(__section, "FILE_LOG_LEVEL") + FILE_LOG_LEVEL = logging.getLevelName(level) # pylint: disable=broad-except except Exception as e: logging.warning( "Failed to read values from config file %s due to %s at %s", CONFIG_FILE, e, traceback.extract_tb(sys.exc_info()[2])) + +logging.warning("CUEBOT_HOSTNAME: %s", CUEBOT_HOSTNAME) diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index beeca70ae..63ac828ec 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -22,7 +22,7 @@ from builtins import str from builtins import object -import logging as log +import logging import os import platform import random @@ -46,6 +46,8 @@ INT32_MAX = 2147483647 INT32_MIN = -2147483648 +log = logging.getLogger(__name__) + class FrameAttendantThread(threading.Thread): """Once a frame has been received and checked by RQD, this class handles @@ -582,7 +584,7 @@ def __init__(self, optNimbyoff=False): booked_cores=0, ) - self.nimby = rqd.rqnimby.Nimby(self) + self.nimby = rqd.rqnimby.NimbyFactory.getNimby(self) self.machine = rqd.rqmachine.Machine(self, self.cores) @@ -614,6 +616,7 @@ def start(self): log.warning('OVERRIDE_NIMBY is False, Nimby startup has been disabled') else: self.nimbyOn() + self.onNimbyLock() elif rqd.rqconstants.OVERRIDE_NIMBY: log.warning('Nimby startup has been triggered by OVERRIDE_NIMBY') self.nimbyOn() @@ -836,7 +839,12 @@ def launchFrame(self, runFrame): raise rqd.rqexceptions.CoreReservationFailureException(err) if self.nimby.locked and not runFrame.ignore_nimby: - err = "Not launching, rqd is lockNimby" + err = "Not launching, rqd is lockNimby and not Ignore Nimby" + log.info(err) + raise rqd.rqexceptions.CoreReservationFailureException(err) + + if rqd.rqconstants.OVERRIDE_NIMBY and self.nimby.isNimbyActive(): + err = "Not launching, rqd is lockNimby and User is Active" log.info(err) raise rqd.rqexceptions.CoreReservationFailureException(err) @@ -913,6 +921,7 @@ def shutdownRqdNow(self): def shutdownRqdIdle(self): """When machine is idle, shutdown RQD""" + log.info("shutdownRqdIdle") self.lockAll() self.__whenIdle = True self.sendStatusReport() @@ -921,11 +930,13 @@ def shutdownRqdIdle(self): def restartRqdNow(self): """Kill all running frames and restart RQD""" + log.info("RestartRqdNow") self.__respawn = True self.shutdownRqdNow() def restartRqdIdle(self): """When machine is idle, restart RQD""" + log.info("RestartRqdIdle") self.lockAll() self.__whenIdle = True self.__respawn = True diff --git a/rqd/rqd/rqdservicers.py b/rqd/rqd/rqdservicers.py index a56fdf442..98ab358ac 100644 --- a/rqd/rqd/rqdservicers.py +++ b/rqd/rqd/rqdservicers.py @@ -20,7 +20,7 @@ from __future__ import print_function from __future__ import division -import logging as log +import logging import grpc @@ -28,6 +28,9 @@ import rqd.compiled_proto.rqd_pb2_grpc +log = logging.getLogger(__name__) + + class RqdInterfaceServicer(rqd.compiled_proto.rqd_pb2_grpc.RqdInterfaceServicer): """Service interface for RqdStatic gRPC definition.""" diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index a9afae757..f9cdea412 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -31,7 +31,7 @@ import ctypes import errno -import logging as log +import logging import math import os import platform @@ -64,6 +64,7 @@ import rqd.rqutil +log = logging.getLogger(__name__) KILOBYTE = 1024 diff --git a/rqd/rqd/rqnetwork.py b/rqd/rqd/rqnetwork.py index 28018d5f5..94d350429 100644 --- a/rqd/rqd/rqnetwork.py +++ b/rqd/rqd/rqnetwork.py @@ -25,7 +25,7 @@ from random import shuffle import abc import atexit -import logging as log +import logging import os import platform import subprocess @@ -41,6 +41,9 @@ import rqd.rqutil +log = logging.getLogger(__name__) + + class RunningFrame(object): """Represents a running frame.""" @@ -101,7 +104,7 @@ def status(self): def kill(self, message=""): """Kills the frame""" - log.info("Request recieved: kill") + log.info("Request received: kill") if self.frameAttendantThread is None: log.warning( "Kill requested before frameAttendantThread is created for: %s", self.frameId) diff --git a/rqd/rqd/rqnimby.py b/rqd/rqd/rqnimby.py index ad006fca5..e57e39f23 100644 --- a/rqd/rqd/rqnimby.py +++ b/rqd/rqd/rqnimby.py @@ -20,18 +20,46 @@ from __future__ import print_function from __future__ import division +from abc import ABCMeta, abstractmethod import os import select import time import signal import threading -import logging as log +import logging +import platform import rqd.rqconstants import rqd.rqutil +log = logging.getLogger(__name__) -class Nimby(threading.Thread): +if platform.system() == 'Windows': + pynputIsAvailable = False + try: + import pynput + pynputIsAvailable = True + except ImportError as e: + log.error(e) + + +# compatible with Python 2 and 3: +ABC = ABCMeta('ABC', (object,), {'__slots__': ()}) + +class NimbyFactory(object): + """ Factory to handle Linux/Windows platforms """ + @staticmethod + def getNimby(rqCore): + """ assign platform dependent Nimby instance """ + nimbyInstance = None + if rqd.rqconstants.USE_NIMBY_PYNPUT and pynputIsAvailable: + nimbyInstance = NimbyPynput(rqCore) + else: + nimbyInstance = NimbySelect(rqCore) + return nimbyInstance + + +class Nimby(threading.Thread, ABC): """Nimby == Not In My Back Yard. If enabled, nimby will lock and kill all frames running on the host if keyboard or mouse activity is detected. If sufficient idle time has @@ -45,15 +73,18 @@ def __init__(self, rqCore): threading.Thread.__init__(self) self.rqCore = rqCore - self.locked = False self.active = False + log.warning("Locked state :%s", self.locked) + log.warning("Active state :%s", self.active) self.fileObjList = [] self.results = [[]] self.thread = None + self.interaction_detected = False + signal.signal(signal.SIGINT, self.signalHandler) def signalHandler(self, sig, frame): @@ -77,49 +108,113 @@ def unlockNimby(self, asOf=None): log.info("Unlocked nimby") self.rqCore.onNimbyUnlock(asOf=asOf) - def _openEvents(self): - """Opens the /dev/input/event* files so nimby can monitor them""" - self._closeEvents() + def run(self): + """Starts the Nimby thread""" + log.warning("Nimby Run") + self.active = True + self.locked = True + self.startListener() + self.unlockedIdle() rqd.rqutil.permissionsHigh() try: for device in os.listdir("/dev/input/"): if device.startswith("event") or device.startswith("mice"): - log.debug("Found device: %s", device) try: self.fileObjList.append(open("/dev/input/%s" % device, "rb")) except IOError as e: # Bad device found - log.debug("IOError: Failed to open %s, %s", "/dev/input/%s" % device, e) + log.warning("IOError: Failed to open %s, %s", "/dev/input/%s" % device, e) finally: rqd.rqutil.permissionsLow() - def _closeEvents(self): - """Closes the /dev/input/event* files""" - log.debug("_closeEvents") - if self.fileObjList: - for fileObj in self.fileObjList: - try: - fileObj.close() - # pylint: disable=broad-except - except Exception: - pass - self.fileObjList = [] + def stop(self): + """Stops the Nimby thread""" + log.warning("Stop Nimby") + if self.thread: + self.thread.cancel() + self.active = False + self.stopListener() + self.unlockNimby() + + @abstractmethod + def startListener(self): + """ start listening """ + + @abstractmethod + def stopListener(self): + """ stop listening """ + + @abstractmethod + def lockedInUse(self): + """Nimby State: Machine is in use, host is locked, + waiting for sufficient idle time""" + + @abstractmethod + def lockedIdle(self): + """Nimby State: Machine is idle, + waiting for sufficient idle time to unlock""" + + @abstractmethod + def unlockedIdle(self): + """Nimby State: Machine is idle, host is unlocked, + waiting for user activity""" + + @abstractmethod + def isNimbyActive(self): + """ Check if user is active + :return: boolean if events are logged and Nimby is active + """ + + +class NimbySelect(Nimby): + """ Nimby Linux """ + def startListener(self): + """ start listening """ + + def stopListener(self): + """ stop listening """ + self.closeEvents() def lockedInUse(self): """Nimby State: Machine is in use, host is locked, waiting for sufficient idle time""" - log.debug("lockedInUse") - self._openEvents() + log.warning("lockedInUse") + self.openEvents() try: self.results = select.select(self.fileObjList, [], [], 5) # pylint: disable=broad-except - except Exception: - pass + except Exception as e: + log.warning(e) if self.active and self.results[0] == []: self.lockedIdle() elif self.active: - self._closeEvents() + self.closeEvents() + self.thread = threading.Timer(rqd.rqconstants.CHECK_INTERVAL_LOCKED, + self.lockedInUse) + self.thread.start() + + def unlockedIdle(self): + """Nimby State: Machine is idle, host is unlocked, + waiting for user activity""" + log.warning("UnlockedIdle Nimby") + while self.active and \ + self.results[0] == [] and \ + self.rqCore.machine.isNimbySafeToRunJobs(): + try: + self.openEvents() + self.results = select.select(self.fileObjList, [], [], 5) + # pylint: disable=broad-except + except Exception: + pass + if not self.rqCore.machine.isNimbySafeToRunJobs(): + log.warning("memory threshold has been exceeded, locking nimby") + self.active = True + + if self.active: + log.warning("Is active, locking Nimby") + self.closeEvents() + self.lockNimby() self.thread = threading.Timer(rqd.rqconstants.CHECK_INTERVAL_LOCKED, self.lockedInUse) self.thread.start() @@ -127,21 +222,98 @@ def lockedInUse(self): def lockedIdle(self): """Nimby State: Machine is idle, waiting for sufficient idle time to unlock""" - self._openEvents() + log.warning("lockedIdle") + self.openEvents() waitStartTime = time.time() try: self.results = select.select(self.fileObjList, [], [], rqd.rqconstants.MINIMUM_IDLE) # pylint: disable=broad-except - except Exception: - pass + except Exception as e: + log.warning(e) if self.active and self.results[0] == [] and \ - self.rqCore.machine.isNimbySafeToUnlock(): - self._closeEvents() + self.rqCore.machine.isNimbySafeToUnlock(): + self.closeEvents() self.unlockNimby(asOf=waitStartTime) self.unlockedIdle() elif self.active: - self._closeEvents() + self.closeEvents() + self.thread = threading.Timer(rqd.rqconstants.CHECK_INTERVAL_LOCKED, + self.lockedInUse) + self.thread.start() + + def openEvents(self): + """Opens the /dev/input/event* files so nimby can monitor them""" + log.warning("openEvents") + self.closeEvents() + + rqd.rqutil.permissionsHigh() + try: + for device in os.listdir("/dev/input/"): + if device.startswith("event") or device.startswith("mice"): + try: + self.fileObjList.append(open("/dev/input/%s" % device, "rb")) + except IOError as e: + # Bad device found + msg = ('IOError: Failed to open %s, %s' + % ("/dev/input/%s" % device, e)) + log.warning(msg) + finally: + rqd.rqutil.permissionsLow() + + def closeEvents(self): + """Closes the /dev/input/event* files""" + log.warning("closeEvents") + if self.fileObjList: + for fileObj in self.fileObjList: + try: + fileObj.close() + # pylint: disable=broad-except + except Exception: + pass + self.fileObjList = [] + + def isNimbyActive(self): + """ Check if user is active + :return: boolean if events are logged and Nimby is active + """ + return self.active and self.results[0] == [] + +class NimbyPynput(Nimby): + """ Nimby Windows """ + def __init__(self, rqCore): + Nimby.__init__(self, rqCore) + + self.mouse_listener = pynput.mouse.Listener( + on_move=self.on_interaction, + on_click=self.on_interaction, + on_scroll=self.on_interaction) + self.keyboard_listener = pynput.keyboard.Listener(on_press=self.on_interaction) + + def on_interaction(self): + """ interaction detected """ + self.interaction_detected = True + + def startListener(self): + """ start listening """ + self.mouse_listener.start() + self.keyboard_listener.start() + + def stopListener(self): + """ stop listening """ + self.mouse_listener.stop() + self.keyboard_listener.stop() + + def lockedInUse(self): + """Nimby State: Machine is in use, host is locked, + waiting for sufficient idle time""" + self.interaction_detected = False + + time.sleep(5) + if self.active and not self.interaction_detected: + self.lockedIdle() + elif self.active: + self.thread = threading.Timer(rqd.rqconstants.CHECK_INTERVAL_LOCKED, self.lockedInUse) self.thread.start() @@ -149,35 +321,46 @@ def lockedIdle(self): def unlockedIdle(self): """Nimby State: Machine is idle, host is unlocked, waiting for user activity""" + log.warning("unlockedIdle") while self.active and \ - self.results[0] == [] and \ - self.rqCore.machine.isNimbySafeToRunJobs(): - try: - self._openEvents() - self.results = select.select(self.fileObjList, [], [], 5) - # pylint: disable=broad-except - except Exception: - pass + not self.interaction_detected and \ + self.rqCore.machine.isNimbySafeToRunJobs(): + + time.sleep(5) + if not self.rqCore.machine.isNimbySafeToRunJobs(): log.warning("memory threshold has been exceeded, locking nimby") self.active = True if self.active: - self._closeEvents() + log.warning("Is active, lock Nimby") self.lockNimby() self.thread = threading.Timer(rqd.rqconstants.CHECK_INTERVAL_LOCKED, self.lockedInUse) + log.warning("starting Thread") self.thread.start() - def run(self): - """Starts the Nimby thread""" - self.active = True - self.unlockedIdle() + def lockedIdle(self): + """Nimby State: Machine is idle, + waiting for sufficient idle time to unlock""" + log.warning("lockedIdle") + waitStartTime = time.time() - def stop(self): - """Stops the Nimby thread""" - if self.thread: - self.thread.cancel() - self.active = False - self._closeEvents() - self.unlockNimby() + time.sleep(rqd.rqconstants.MINIMUM_IDLE) + + if self.active and not self.interaction_detected and \ + self.rqCore.machine.isNimbySafeToUnlock(): + log.warning("Start wait time: %s", waitStartTime) + self.unlockNimby(asOf=waitStartTime) + self.unlockedIdle() + elif self.active: + + self.thread = threading.Timer(rqd.rqconstants.CHECK_INTERVAL_LOCKED, + self.lockedInUse) + self.thread.start() + + def isNimbyActive(self): + """ Check if user is active + :return: boolean if events are logged and Nimby is active + """ + return not self.active and self.interaction_detected diff --git a/rqd/rqd/rqswap.py b/rqd/rqd/rqswap.py index eae7259dd..733750702 100644 --- a/rqd/rqd/rqswap.py +++ b/rqd/rqd/rqswap.py @@ -24,12 +24,13 @@ from builtins import str from builtins import range from builtins import object -import logging as log +import logging import re import threading import time +log = logging.getLogger(__name__) PGPGOUT_RE = re.compile(r"^pgpgout (\d+)") diff --git a/rqd/rqd/rqutil.py b/rqd/rqd/rqutil.py index 0253dcff0..da627a2d5 100644 --- a/rqd/rqd/rqutil.py +++ b/rqd/rqd/rqutil.py @@ -24,7 +24,7 @@ from builtins import str from builtins import object import functools -import logging as log +import logging import os import platform import socket @@ -40,6 +40,7 @@ PERMISSIONS = threading.Lock() HIGH_PERMISSION_GROUPS = os.getgroups() +log = logging.getLogger(__name__) class Memoize(object): diff --git a/rqd/tests/rqcore_tests.py b/rqd/tests/rqcore_tests.py index bf357dddc..bb70c12a3 100644 --- a/rqd/tests/rqcore_tests.py +++ b/rqd/tests/rqcore_tests.py @@ -40,7 +40,7 @@ class RqCoreTests(unittest.TestCase): - @mock.patch('rqd.rqnimby.Nimby', autospec=True) + @mock.patch('rqd.rqnimby.NimbySelect', autospec=True) @mock.patch('rqd.rqnetwork.Network', autospec=True) @mock.patch('rqd.rqmachine.Machine', autospec=True) def setUp(self, machineMock, networkMock, nimbyMock): @@ -314,7 +314,7 @@ def test_launchFrameOnNimbyHost(self, frameThreadMock): frame = rqd.compiled_proto.rqd_pb2.RunFrame(uid=22, num_cores=10) frameIgnoreNimby = rqd.compiled_proto.rqd_pb2.RunFrame( uid=22, num_cores=10, ignore_nimby=True) - self.rqcore.nimby = mock.create_autospec(rqd.rqnimby.Nimby) + self.rqcore.nimby = mock.create_autospec(rqd.rqnimby.NimbySelect) self.rqcore.nimby.locked = True with self.assertRaises(rqd.rqexceptions.CoreReservationFailureException): diff --git a/rqd/tests/rqmachine_tests.py b/rqd/tests/rqmachine_tests.py index b95467d1d..123b797a3 100644 --- a/rqd/tests/rqmachine_tests.py +++ b/rqd/tests/rqmachine_tests.py @@ -173,7 +173,7 @@ def setUp(self): self.meminfo = self.fs.create_file('/proc/meminfo', contents=MEMINFO_MODERATE_USAGE) self.rqCore = mock.MagicMock(spec=rqd.rqcore.RqCore) - self.nimby = mock.MagicMock(spec=rqd.rqnimby.Nimby) + self.nimby = mock.MagicMock(spec=rqd.rqnimby.NimbySelect) self.rqCore.nimby = self.nimby self.nimby.active = False self.nimby.locked = False diff --git a/rqd/tests/rqnimby_tests.py b/rqd/tests/rqnimby_tests.py index aa6846f44..04cf3e765 100644 --- a/rqd/tests/rqnimby_tests.py +++ b/rqd/tests/rqnimby_tests.py @@ -41,10 +41,10 @@ def setUp(self): self.rqMachine = mock.MagicMock(spec=rqd.rqmachine.Machine) self.rqCore = mock.MagicMock(spec=rqd.rqcore.RqCore) self.rqCore.machine = self.rqMachine - self.nimby = rqd.rqnimby.Nimby(self.rqCore) + self.nimby = rqd.rqnimby.NimbyFactory.getNimby(self.rqCore) self.nimby.daemon = True - @mock.patch.object(rqd.rqnimby.Nimby, 'unlockedIdle') + @mock.patch.object(rqd.rqnimby.NimbySelect, 'unlockedIdle') def test_initialState(self, unlockedIdleMock): self.nimby.daemon = True @@ -70,7 +70,7 @@ def test_unlockedIdle(self, timerMock): timerMock.return_value.start.assert_called() @mock.patch('select.select', new=mock.MagicMock(return_value=[[], [], []])) - @mock.patch.object(rqd.rqnimby.Nimby, 'unlockedIdle') + @mock.patch.object(rqd.rqnimby.NimbySelect, 'unlockedIdle') @mock.patch('threading.Timer') def test_lockedIdleWhenIdle(self, timerMock, unlockedIdleMock): self.nimby.active = True @@ -96,7 +96,7 @@ def test_lockedIdleWhenInUse(self, timerMock): timerMock.return_value.start.assert_called() @mock.patch('select.select', new=mock.MagicMock(return_value=[[], [], []])) - @mock.patch.object(rqd.rqnimby.Nimby, 'lockedIdle') + @mock.patch.object(rqd.rqnimby.NimbySelect, 'lockedIdle') @mock.patch('threading.Timer') def test_lockedInUseWhenIdle(self, timerMock, lockedIdleMock): self.nimby.active = True From c570483f8ef08b850c12cb544b955ff6b1681858 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Fri, 8 Apr 2022 11:58:04 -0700 Subject: [PATCH 176/277] Post jobs should inherit parent environment variables (#1107) * Post jobs should inherit parent environment variables Environment variables set to parent jobs can be useful for post job frames. * Revert import * change --- .../com/imageworks/spcue/service/JobSpec.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java index db33729ee..293ce9730 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java @@ -341,6 +341,12 @@ private BuildableJob handleJobTag(Element jobTag) { job.priority = Integer.valueOf(jobTag.getChildTextTrim("priority")); } + + Element envTag = jobTag.getChild("env"); + if (envTag != null) { + handleEnvironmentTag(envTag, buildableJob.env); + } + handleLayerTags(buildableJob, jobTag); if (buildableJob.getBuildableLayers().size() > MAX_LAYERS) { @@ -353,11 +359,6 @@ private BuildableJob handleJobTag(Element jobTag) { + " has no layers"); } - Element envTag = jobTag.getChild("env"); - if (envTag != null) { - handleEnvironmentTag(envTag, buildableJob.env); - } - return buildableJob; } @@ -958,6 +959,11 @@ private BuildableJob initPostJob(BuildableJob parent) { job.deptName = parent.detail.deptName; BuildableJob postJob = new BuildableJob(job); + + for (String key : parent.env.keySet()) { + postJob.env.put(key, parent.env.get(key)); + } + return postJob; } From 351c9aed0f4e563cf83c99bbaefb9d3d48fb0cf0 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Mon, 11 Apr 2022 15:11:10 -0700 Subject: [PATCH 177/277] Add balanced mode to dispatch query (#1103) Dispatch balanced mode switch. When turned on, the dispatch query will use a formula that makes sure smaller jobs don't get starved by large jobs. This mode also disregards the layer_limit feature for performance implications. the formula takes into account the number of required cores and the time the job has been waiting on queue. Formula: ``` rank = priority + (100 * (1 - (job.cores/job.int_min_cores))) + (age in days) ``` This heuristic favours scenarios with a large amount of jobs and hosts and when turned off, only job priorities are taken into account, making the queue more predictable. --- .../imageworks/spcue/dao/DispatcherDao.java | 23 +++-- .../spcue/dao/postgres/DispatchQuery.java | 87 +++++++++++++++++-- .../spcue/dao/postgres/DispatcherDaoJdbc.java | 57 ++++++------ cuebot/src/main/resources/opencue.properties | 9 +- .../dao/postgres/DispatcherDaoFifoTests.java | 17 ++-- .../test/dao/postgres/DispatcherDaoTests.java | 2 +- 6 files changed, 145 insertions(+), 50 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/DispatcherDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/DispatcherDao.java index a2eee3657..c348bb2f4 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/DispatcherDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/DispatcherDao.java @@ -164,18 +164,31 @@ List findNextDispatchFrames(LayerInterface layer, DispatchHost ho int limit); /** - * Return whether FIFO scheduling is enabled or not in the same priority for unittest. + * Return Scheduling Mode selected * * @return */ - boolean getFifoSchedulingEnabled(); + SchedulingMode getSchedulingMode(); /** - * Set whether FIFO scheduling is enabled or not in the same priority for unittest. + * Set Scheduling Mode. * - * @param fifoSchedulingEnabled + * @param schedulingMode */ - void setFifoSchedulingEnabled(boolean fifoSchedulingEnabled); + void setSchedulingMode(SchedulingMode schedulingMode); + + /** + * - PRIORITY_ONLY: Sort by priority only + * - FIFO: Whether or not to enable FIFO scheduling in the same priority. + * - BALANCED: Use a rank formula that takes into account time waiting, and number + * of cores required: rank = priority + (100 * (1 - (job.cores/job.int_min_cores))) + age in days + */ + enum SchedulingMode { + PRIORITY_ONLY, + FIFO, + BALANCED + } } + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java index 1f08d2153..ee95feb82 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java @@ -21,8 +21,8 @@ public class DispatchQuery { - public static final String FIND_JOBS_BY_SHOW = - "/* FIND_JOBS_BY_SHOW */ " + + public static final String FIND_JOBS_BY_SHOW_PRIORITY_MODE = + "/* FIND_JOBS_BY_SHOW_PRIORITY_MODE */ " + "SELECT pk_job, int_priority, rank FROM ( " + "SELECT " + "ROW_NUMBER() OVER (ORDER BY job_resource.int_priority DESC) AS rank, " + @@ -101,9 +101,76 @@ public class DispatchQuery { ") " + ") AS t1 WHERE rank < ?"; + // sort = priority + (100 * (1 - (job.cores/job.int_min_cores))) + (age in days) */ + public static final String FIND_JOBS_BY_SHOW_BALANCED_MODE = + "/* FIND_JOBS_BY_SHOW_BALANCED_MODE */ " + + "SELECT pk_job, int_priority, rank FROM ( " + + "SELECT " + + "ROW_NUMBER() OVER (ORDER BY int_priority DESC) AS rank, " + + "pk_job, " + + "int_priority " + + "FROM ( " + + "SELECT DISTINCT " + + "job.pk_job as pk_job, " + + "CAST( " + + "job_resource.int_priority + ( " + + "100 * (CASE WHEN job_resource.int_min_cores <= 0 THEN 0 " + + "ELSE " + + "CASE WHEN job_resource.int_cores > job_resource.int_min_cores THEN 0 " + + "ELSE 1 - job_resource.int_cores/job_resource.int_min_cores " + + "END " + + "END) " + + ") + ( " + + "(DATE_PART('days', NOW()) - DATE_PART('days', job.ts_updated)) " + + ") as INT) as int_priority " + + "FROM " + + "job , " + + "job_resource , " + + "folder , " + + "folder_resource, " + + "point , " + + "layer , " + + "layer_stat , " + + "host " + + "WHERE " + + "job.pk_job = job_resource.pk_job " + + "AND job.pk_folder = folder.pk_folder " + + "AND folder.pk_folder = folder_resource.pk_folder " + + "AND folder.pk_dept = point.pk_dept " + + "AND folder.pk_show = point.pk_show " + + "AND job.pk_job = layer.pk_job " + + "AND job_resource.pk_job = job.pk_job " + + "AND (CASE WHEN layer_stat.int_waiting_count > 0 THEN layer_stat.pk_layer ELSE NULL END) = layer.pk_layer " + + "AND " + + "(" + + "folder_resource.int_max_cores = -1 " + + "OR " + + "folder_resource.int_cores + layer.int_cores_min < folder_resource.int_max_cores " + + ") " + + "AND job.str_state = 'PENDING' " + + "AND job.b_paused = false " + + "AND job.pk_show = ? " + + "AND job.pk_facility = ? " + + "AND " + + "(" + + "job.str_os IS NULL OR job.str_os = '' " + + "OR " + + "job.str_os = ? " + + ") " + + "AND (CASE WHEN layer_stat.int_waiting_count > 0 THEN 1 ELSE NULL END) = 1 " + + "AND layer.int_cores_min <= ? " + + "AND layer.int_mem_min <= ? " + + "AND (CASE WHEN layer.b_threadable = true THEN 1 ELSE 0 END) >= ? " + + "AND layer.int_gpus_min <= ? " + + "AND layer.int_gpu_mem_min BETWEEN ? AND ? " + + "AND job_resource.int_cores + layer.int_cores_min <= job_resource.int_max_cores " + + "AND host.str_tags ~* ('(?x)' || layer.str_tags) " + + "AND host.str_name = ? " + + ") AS t1 ) AS t2 WHERE rank < ?"; + - public static final String FIND_JOBS_BY_GROUP = - FIND_JOBS_BY_SHOW + public static final String FIND_JOBS_BY_GROUP_PRIORITY_MODE = + FIND_JOBS_BY_SHOW_PRIORITY_MODE .replace( "FIND_JOBS_BY_SHOW", "FIND_JOBS_BY_GROUP") @@ -111,6 +178,14 @@ public class DispatchQuery { "AND job.pk_show = ? ", "AND job.pk_folder = ? "); + public static final String FIND_JOBS_BY_GROUP_BALANCED_MODE = + FIND_JOBS_BY_SHOW_BALANCED_MODE + .replace( + "FIND_JOBS_BY_SHOW", + "FIND_JOBS_BY_GROUP") + .replace( + "AND job.pk_show = ? ", + "AND job.pk_folder = ? "); private static final String replaceQueryForFifo(String query) { return query @@ -125,8 +200,8 @@ private static final String replaceQueryForFifo(String query) { "WHERE rank < ? ORDER BY rank"); } - public static final String FIND_JOBS_FIFO_BY_SHOW = replaceQueryForFifo(FIND_JOBS_BY_SHOW); - public static final String FIND_JOBS_FIFO_BY_GROUP = replaceQueryForFifo(FIND_JOBS_BY_GROUP); + public static final String FIND_JOBS_BY_SHOW_FIFO_MODE = replaceQueryForFifo(FIND_JOBS_BY_SHOW_PRIORITY_MODE); + public static final String FIND_JOBS_BY_GROUP_FIFO_MODE = replaceQueryForFifo(FIND_JOBS_BY_GROUP_PRIORITY_MODE); /** * Dispatch a host in local booking mode. diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java index d33629f18..630ce2b0b 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java @@ -48,22 +48,7 @@ import com.imageworks.spcue.grpc.host.ThreadMode; import com.imageworks.spcue.util.CueUtil; -import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_DISPATCH_FRAME_BY_JOB_AND_HOST; -import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_DISPATCH_FRAME_BY_JOB_AND_PROC; -import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_DISPATCH_FRAME_BY_LAYER_AND_HOST; -import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_DISPATCH_FRAME_BY_LAYER_AND_PROC; -import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_JOBS_BY_GROUP; -import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_JOBS_BY_LOCAL; -import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_JOBS_BY_SHOW; -import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_JOBS_FIFO_BY_GROUP; -import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_JOBS_FIFO_BY_SHOW; -import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_LOCAL_DISPATCH_FRAME_BY_JOB_AND_HOST; -import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_LOCAL_DISPATCH_FRAME_BY_JOB_AND_PROC; -import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_LOCAL_DISPATCH_FRAME_BY_LAYER_AND_HOST; -import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_LOCAL_DISPATCH_FRAME_BY_LAYER_AND_PROC; -import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_SHOWS; -import static com.imageworks.spcue.dao.postgres.DispatchQuery.FIND_UNDER_PROCED_JOB_BY_FACILITY; -import static com.imageworks.spcue.dao.postgres.DispatchQuery.HIGHER_PRIORITY_JOB_BY_FACILITY_EXISTS; +import static com.imageworks.spcue.dao.postgres.DispatchQuery.*; /** @@ -130,24 +115,24 @@ public List getShows() { new ConcurrentHashMap(); /** - * Whether or not to enable FIFO scheduling in the same priority. + * Choose between different scheduling strategies */ - private boolean fifoSchedulingEnabled; + private SchedulingMode schedulingMode; @Autowired public DispatcherDaoJdbc(Environment env) { - fifoSchedulingEnabled = env.getProperty( - "dispatcher.fifo_scheduling_enabled", Boolean.class, false); + this.schedulingMode = SchedulingMode.valueOf(env.getProperty( + "dispatcher.scheduling_mode", String.class, "PRIORITY_ONLY")); } @Override - public boolean getFifoSchedulingEnabled() { - return fifoSchedulingEnabled; + public SchedulingMode getSchedulingMode() { + return schedulingMode; } @Override - public void setFifoSchedulingEnabled(boolean fifoSchedulingEnabled) { - this.fifoSchedulingEnabled = fifoSchedulingEnabled; + public void setSchedulingMode(SchedulingMode schedulingMode) { + this.schedulingMode = schedulingMode; } /** @@ -211,7 +196,7 @@ private List findDispatchJobs(DispatchHost host, int numJobs, boolean sh } result.addAll(getJdbcTemplate().query( - fifoSchedulingEnabled ? FIND_JOBS_FIFO_BY_SHOW : FIND_JOBS_BY_SHOW, + findByShowQuery(), PKJOB_MAPPER, s.getShowId(), host.getFacilityId(), host.os, host.idleCores, host.idleMemory, @@ -233,6 +218,24 @@ private List findDispatchJobs(DispatchHost host, int numJobs, boolean sh } + private String findByShowQuery() { + switch (schedulingMode) { + case PRIORITY_ONLY: return FIND_JOBS_BY_SHOW_PRIORITY_MODE; + case FIFO: return FIND_JOBS_BY_SHOW_FIFO_MODE; + case BALANCED: return FIND_JOBS_BY_SHOW_BALANCED_MODE; + default: return FIND_JOBS_BY_SHOW_PRIORITY_MODE; + } + } + + private String findByGroupQuery() { + switch (schedulingMode) { + case PRIORITY_ONLY: return FIND_JOBS_BY_GROUP_PRIORITY_MODE; + case FIFO: return FIND_JOBS_BY_GROUP_FIFO_MODE; + case BALANCED: return FIND_JOBS_BY_GROUP_BALANCED_MODE; + default: return FIND_JOBS_BY_GROUP_PRIORITY_MODE; + } + } + @Override public List findDispatchJobsForAllShows(DispatchHost host, int numJobs) { return findDispatchJobs(host, numJobs, true); @@ -246,7 +249,7 @@ public List findDispatchJobs(DispatchHost host, int numJobs) { @Override public List findDispatchJobs(DispatchHost host, GroupInterface g) { List result = getJdbcTemplate().query( - fifoSchedulingEnabled ? FIND_JOBS_FIFO_BY_GROUP : FIND_JOBS_BY_GROUP, + findByGroupQuery(), PKJOB_MAPPER, g.getGroupId(),host.getFacilityId(), host.os, host.idleCores, host.idleMemory, @@ -406,7 +409,7 @@ public boolean higherPriorityJobExists(JobDetail baseJob, VirtualProc proc) { public List findDispatchJobs(DispatchHost host, ShowInterface show, int numJobs) { List result = getJdbcTemplate().query( - fifoSchedulingEnabled ? FIND_JOBS_FIFO_BY_SHOW : FIND_JOBS_BY_SHOW, + findByShowQuery(), PKJOB_MAPPER, show.getShowId(), host.getFacilityId(), host.os, host.idleCores, host.idleMemory, diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index e64f6e417..d28925c15 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -64,8 +64,13 @@ dispatcher.frame_query_max=20 dispatcher.job_frame_dispatch_max=8 # Maximum number of frames to dispatch from a host at one time. dispatcher.host_frame_dispatch_max=12 -# Whether or not to enable FIFO scheduling in the same priority. -dispatcher.fifo_scheduling_enabled=false +# Choose between different scheduling strategies: +# - PRIORITY_ONLY: Sort by priority only +# - FIFO: Whether or not to enable FIFO scheduling in the same priority. +# - BALANCED: Use a rank formula that takes into account time waiting, and number +# of cores required: rank = priority + (100 * (1 - (job.cores/job.int_min_cores))) + age in days +# layer limiting is also disabled in this mode for performance reasons +dispatcher.scheduling_mode=PRIORITY_ONLY # Number of threads to keep in the pool for launching job. dispatcher.launch_queue.core_pool_size=1 diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java index a428cb3ab..5de19273e 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java @@ -131,7 +131,7 @@ private void launchJobs(int count) throws Exception { @Before public void launchJob() { - dispatcherDao.setFifoSchedulingEnabled(true); + dispatcherDao.setSchedulingMode(DispatcherDao.SchedulingMode.FIFO); dispatcher.setTestMode(true); jobLauncher.testMode = true; @@ -139,7 +139,7 @@ public void launchJob() { @After public void resetFifoScheduling() { - dispatcherDao.setFifoSchedulingEnabled(false); + dispatcherDao.setSchedulingMode(DispatcherDao.SchedulingMode.PRIORITY_ONLY); } @Before @@ -171,11 +171,11 @@ public void createHost() { @Transactional @Rollback(true) public void testFifoSchedulingEnabled() { - assertTrue(dispatcherDao.getFifoSchedulingEnabled()); - dispatcherDao.setFifoSchedulingEnabled(false); - assertFalse(dispatcherDao.getFifoSchedulingEnabled()); - dispatcherDao.setFifoSchedulingEnabled(true); - assertTrue(dispatcherDao.getFifoSchedulingEnabled()); + assertEquals(dispatcherDao.getSchedulingMode(), DispatcherDao.SchedulingMode.FIFO); + dispatcherDao.setSchedulingMode(DispatcherDao.SchedulingMode.PRIORITY_ONLY); + assertEquals(dispatcherDao.getSchedulingMode(), DispatcherDao.SchedulingMode.PRIORITY_ONLY); + dispatcherDao.setSchedulingMode(DispatcherDao.SchedulingMode.FIFO); + assertEquals(dispatcherDao.getSchedulingMode(), DispatcherDao.SchedulingMode.FIFO); } @Test @@ -213,8 +213,7 @@ public void testPortionSorted() throws Exception { @Transactional @Rollback(true) public void testFifoSchedulingDisabled() throws Exception { - dispatcherDao.setFifoSchedulingEnabled(false); - assertFalse(dispatcherDao.getFifoSchedulingEnabled()); + dispatcherDao.setSchedulingMode(DispatcherDao.SchedulingMode.PRIORITY_ONLY); int count = 10; launchJobs(count); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java index 0a01478a4..900f50afe 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java @@ -522,6 +522,6 @@ public void testHigherPriorityJobExistsMaxProcBound() { @Transactional @Rollback(true) public void testFifoSchedulingEnabled() { - assertFalse(dispatcherDao.getFifoSchedulingEnabled()); + assertEquals(dispatcherDao.getSchedulingMode(), DispatcherDao.SchedulingMode.PRIORITY_ONLY); } } From 64b382c93ce47a446a3806e3d481a1520906243c Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Wed, 20 Apr 2022 08:48:29 -0700 Subject: [PATCH 178/277] Swap RE with empty string in MonitorJobPlugin (#1132) Simplifies check in _regex_loadJobsHandle, if substring is not None, then load current matching jobs. --- cuegui/cuegui/plugins/MonitorJobsPlugin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cuegui/cuegui/plugins/MonitorJobsPlugin.py b/cuegui/cuegui/plugins/MonitorJobsPlugin.py index 6743a0717..406cd979d 100644 --- a/cuegui/cuegui/plugins/MonitorJobsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorJobsPlugin.py @@ -46,7 +46,6 @@ PLUGIN_CATEGORY = "Cuetopia" PLUGIN_DESCRIPTION = "Monitors a list of jobs" PLUGIN_PROVIDES = "MonitorJobsDockWidget" -REGEX_EMPTY_STRING = re.compile("^$") JOB_RESTORE_THRESHOLD_DAYS = 3 JOB_RESTORE_THRESHOLD_LIMIT = 200 @@ -199,7 +198,7 @@ def _regexLoadJobsHandle(self): self.jobMonitor.addJob(job) else: # Otherwise, just load current matching jobs (except for the empty string) - if not re.search(REGEX_EMPTY_STRING, substring): + if substring: for job in opencue.api.getJobs(regex=[substring]): self.jobMonitor.addJob(job) From 36f32fdc4f07a7c990245fc45146d69f913f90e1 Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Wed, 20 Apr 2022 08:50:28 -0700 Subject: [PATCH 179/277] Update CueJobMonitorTree only with new data (#1128) * Update CueJobMonitorTree only with new data Fixed processUpdate to only update in mem dict self._items when "new" jobs are retrieved during the API call. Calling API every 22 seconds with large number of jobs bogs down the gui. Fixed _getUpdate to include the root group and its' associated jobs, this fn returned a different count then the in mem ds. Fixed the recursive fn __getNestedIds, had previously returned only groups and not the associated jobs, so when processUpdate compared the len() of what is in self._items vs what was last retrieved ie current == set(rpcObjects[1]) line 417, it was never the same and continuously tried rebuilding self._items for each interval. * Fix Pylint errors * Fix Pylint errors * Fix method signature Pylint/unittests * Fix Pylint, removed unused import --- cuegui/cuegui/CueJobMonitorTree.py | 48 ++++++++++++++---------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/cuegui/cuegui/CueJobMonitorTree.py b/cuegui/cuegui/CueJobMonitorTree.py index 5da23541e..440933038 100644 --- a/cuegui/cuegui/CueJobMonitorTree.py +++ b/cuegui/cuegui/CueJobMonitorTree.py @@ -31,7 +31,6 @@ import opencue import opencue.compiled_proto.job_pb2 import opencue.wrappers.group -from opencue.wrappers.job import Job import cuegui.AbstractTreeWidget import cuegui.AbstractWidgetItem @@ -370,7 +369,7 @@ def getShowNames(self): return list(self.__shows.keys()) def __getCollapsed(self): - return [item.rpcObject for item in list(self._items.values()) if not item.isExpanded()] + return [item.rpcObject.id() for item in list(self._items.values()) if not item.isExpanded()] def __setCollapsed(self, collapsed): self.expandAll() @@ -384,13 +383,19 @@ def _getUpdate(self): @return: List that contains updated nested groups and a set of all updated item ideas""" self.currtime = time.time() + allIds = [] try: groups = [show.getJobWhiteboard() for show in self.getShows()] nestedGroups = [] allIds = [] for group in groups: + # add jobs and parent group to match self._items + allIds.append(group.id) + allIds.extend(group.jobs) nestedGroups.append(opencue.wrappers.group.NestedGroup(group)) - allIds.extend(self.__getNestedIds(group)) + # pylint: disable=no-value-for-parameter + allIds.extend(self.__getNestedIds(group, updated=[])) + # pylint: enable=no-value-for-parameter except opencue.exception.CueException as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) return None @@ -411,8 +416,9 @@ def _processUpdate(self, work, rpcObjects): try: current = set(self._items.keys()) if current == set(rpcObjects[1]): - # Only updates - self.__processUpdateHandleNested(self.invisibleRootItem(), rpcObjects[0]) + # Only updates if return rpcObjects doesn't equal current _items + collapsed = self.__getCollapsed() + self.__setCollapsed(collapsed) self.redraw() else: # (Something removed) or (Something added) @@ -431,25 +437,20 @@ def _processUpdate(self, work, rpcObjects): finally: self._itemsLock.unlock() - def __getNestedIds(self, group): - """Returns all the ids founds in the nested list + def __getNestedIds(self, group, updated): + """Returns all the ids founds in the nested list including + group and job ids. @type group: job_pb2.Group - @param group: A group that can contain groups and/or jobs + @param group: A group that can contain groups and their associated jobs @rtype: list @return: The list of all child ids""" - updated = [] - for innerGroup in group.groups.nested_groups: - updated.append(innerGroup.id) - - # If group has groups, recursively call this function - for g in innerGroup.groups.nested_groups: - updated_g = self.__getNestedIds(g) - if updated_g: - updated.extend(updated_g) - - # If group has jobs, update them - for jobId in innerGroup.jobs: - updated.append(jobId) + updated = updated if updated else [] + if group.groups.nested_groups: + for g in group.groups.nested_groups: + updated.append(g.id) + if g.jobs: + updated.extend(g.jobs) + self.__getNestedIds(g, updated) return updated @@ -477,11 +478,8 @@ def __processUpdateHandleNested(self, parent, groups): for nestedGroup in group.data.groups.nested_groups] self.__processUpdateHandleNested(groupItem, nestedGroups) - # empty list will return all jobs on the farm - # only search if list has jobs if group.data.jobs: - jobSeq = opencue.search.JobSearch.byOptions(id=list(group.data.jobs)).jobs - jobsObject = [Job(j) for j in jobSeq.jobs] + jobsObject = opencue.api.getJobs(id=list(group.data.jobs)) for job in jobsObject: try: From 76478411eb4a20cdde010c477e097d105a2e9200 Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Wed, 20 Apr 2022 08:52:09 -0700 Subject: [PATCH 180/277] Cuebot setChannel log supports old Python versions (#1126) --- pycue/opencue/cuebot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pycue/opencue/cuebot.py b/pycue/opencue/cuebot.py index cb36d9d64..b007f0efd 100644 --- a/pycue/opencue/cuebot.py +++ b/pycue/opencue/cuebot.py @@ -187,7 +187,9 @@ def setChannel(): else: connectStr = '%s:%s' % ( host, Cuebot.Config.get('cuebot.grpc_port', DEFAULT_GRPC_PORT)) - logger.debug('connecting to gRPC at %s', connectStr) + # pylint: disable=logging-not-lazy + logger.debug('connecting to gRPC at %s' % connectStr) + # pylint: enable=logging-not-lazy # TODO(bcipriano) Configure gRPC TLS. (Issue #150) try: Cuebot.RpcChannel = grpc.intercept_channel( From 55369047f1db67db6e69014c9ddfb6ab277d4430 Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Thu, 28 Apr 2022 16:17:21 -0700 Subject: [PATCH 181/277] Add Proc's child PIDs to Host report stats (#1130) * Add Proc's child PIDs to Host report stats Users require additional information about the running frame.data pid. For each parent process (frame.pid) add to report.proto the child process: name, rss, vsize, state, cmdline, pid. This additional info will get stored in the Proc table, while proc is running; users can view child proc stats via Cuegui and rqlog will output the highest recorded values for rss for each child pid. * Fix Proc Pylint errors * Add more exit codes to Frame state waiting When determining the frame state cuebot needs to make sure that certain exit statuses put the frame state back into waiting, this will help save time for users when a frame fails for host hardware issues. (cherry picked from commit cbeda9387b09a8713bb76d9075fdee72c3c795f1) * Add more exit codes to Frame state waiting * Removed unused method updateFrameHostDown * Removed deprecated oracle FrameDaoJdbc file * Remove log debugging from FrameCompleteHandler * Fixes to RQD and unittests * Remove unrelated changes from FrameDao * Fix pylint errors for rqmachine.py * Remove unrelated frame changes from pycue * Removing unrelated change from DispatchSupportService * Fix merged conflicts for ProcDaoTests --- VERSION.in | 2 +- .../com/imageworks/spcue/VirtualProc.java | 1 + .../com/imageworks/spcue/dao/ProcDao.java | 4 +- .../spcue/dao/postgres/FrameDaoJdbc.java | 2 +- .../spcue/dao/postgres/ProcDaoJdbc.java | 33 +++- .../spcue/dao/postgres/WhiteboardDaoJdbc.java | 6 + .../spcue/dispatcher/DispatchSupport.java | 5 +- .../dispatcher/DispatchSupportService.java | 6 +- .../spcue/dispatcher/HostReportHandler.java | 6 +- .../com/imageworks/spcue/util/SqlUtil.java | 11 ++ .../V14__Add_bytea_field_Proc_table.sql | 2 + .../spcue/test/dao/postgres/ProcDaoTests.java | 16 +- cuegui/cuegui/FrameMonitorTree.py | 1 + cuegui/cuegui/MenuActions.py | 13 ++ cuegui/cuegui/ProcChildren.py | 167 ++++++++++++++++++ proto/host.proto | 1 + proto/report.proto | 51 ++++++ proto/rqd.proto | 1 + pycue/opencue/api.py | 2 +- rqd/rqd/rqconstants.py | 7 + rqd/rqd/rqcore.py | 11 ++ rqd/rqd/rqmachine.py | 80 ++++++++- rqd/rqd/rqnetwork.py | 41 ++++- rqd/tests/cuerqd_tests.py | 1 + rqd/tests/rqcore_tests.py | 18 +- rqd/tests/rqmachine_tests.py | 6 + 26 files changed, 460 insertions(+), 34 deletions(-) create mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V14__Add_bytea_field_Proc_table.sql create mode 100644 cuegui/cuegui/ProcChildren.py diff --git a/VERSION.in b/VERSION.in index 2856407c0..50653ad0a 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.15 +0.17 diff --git a/cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java b/cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java index 28b54799d..a9032eb43 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java @@ -29,6 +29,7 @@ public class VirtualProc extends FrameEntity implements ProcInterface { public String frameId; public String hostName; public String os; + public byte[] childProcesses; public int coresReserved; public long memoryReserved; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/ProcDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/ProcDao.java index c96e8e28e..206b19e22 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/ProcDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/ProcDao.java @@ -150,8 +150,8 @@ public interface ProcDao { * @param maxKb */ void updateProcMemoryUsage(FrameInterface f, long rss, long maxRss, - long vsize, long maxVsize, - long usedGpuMemory, long maxUsedGpuMemory); + long vsize, long maxVsize, long usedGpuMemory, + long maxUsedGpuMemory, byte[] children); /** * get aq virual proc from its unique id diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java index a7e0dad47..ef3425a8f 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java @@ -67,7 +67,7 @@ public class FrameDaoJdbc extends JdbcDaoSupport implements FrameDao { "int_version = int_version + 1, " + "int_total_past_core_time = int_total_past_core_time + " + "round(INTERVAL_TO_SECONDS(current_timestamp - ts_started) * int_cores / 100)," + - "int_total_past_gpu_time = int_total_past_gpu_time + " + + "int_total_past_gpu_time = int_total_past_gpu_time + " + "round(INTERVAL_TO_SECONDS(current_timestamp - ts_started) * int_gpus) " + "WHERE " + "frame.pk_frame = ? " + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java index 43c7b081b..b68f8a74c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java @@ -19,6 +19,9 @@ package com.imageworks.spcue.dao.postgres; + +import java.sql.Connection; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; @@ -29,6 +32,7 @@ import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.RowMapper; import org.springframework.jdbc.core.support.JdbcDaoSupport; +import org.springframework.jdbc.core.PreparedStatementCreator; import com.imageworks.spcue.FrameInterface; import com.imageworks.spcue.HostInterface; @@ -236,13 +240,14 @@ public boolean clearVirtualProcAssignment(FrameInterface frame) { "int_virt_max_used = ?, " + "int_gpu_mem_used = ?, " + "int_gpu_mem_max_used = ?, " + + "bytea_children = ?, " + "ts_ping = current_timestamp " + "WHERE " + "pk_frame = ?"; @Override public void updateProcMemoryUsage(FrameInterface f, long rss, long maxRss, - long vss, long maxVss, long usedGpuMemory, long maxUsedGpuMemory) { + long vss, long maxVss, long usedGpuMemory, long maxUsedGpuMemory, byte[] children) { /* * This method is going to repeat for a proc every 1 minute, so * if the proc is being touched by another thread, then return @@ -261,7 +266,26 @@ public void updateProcMemoryUsage(FrameInterface f, long rss, long maxRss, rss, maxRss, vss, maxVss, usedGpuMemory, maxUsedGpuMemory, f.getFrameId()); } - } catch (DataAccessException dae) { + getJdbcTemplate().update(new PreparedStatementCreator() { + @Override + public PreparedStatement createPreparedStatement(Connection conn) + throws SQLException { + PreparedStatement updateProc = conn.prepareStatement( + UPDATE_PROC_MEMORY_USAGE); + updateProc.setLong(1, rss); + updateProc.setLong(2, maxRss); + updateProc.setLong(3, vss); + updateProc.setLong(4, maxVss); + updateProc.setLong(5, usedGpuMemory); + updateProc.setLong(6, maxUsedGpuMemory); + updateProc.setBytes(7, children); + updateProc.setString(8, f.getFrameId()); + return updateProc; + } + } + ); + } + catch (DataAccessException dae) { logger.info("The proc for frame " + f + " could not be updated with new memory stats: " + dae); } @@ -295,6 +319,7 @@ public VirtualProc mapRow(ResultSet rs, int rowNum) throws SQLException { proc.unbooked = rs.getBoolean("b_unbooked"); proc.isLocalDispatch = rs.getBoolean("b_local"); proc.os = rs.getString("str_os"); + proc.childProcesses = rs.getBytes("bytea_children"); return proc; } }; @@ -319,6 +344,7 @@ public VirtualProc mapRow(ResultSet rs, int rowNum) throws SQLException { "proc.int_gpu_mem_reserved,"+ "proc.int_gpu_mem_max_used,"+ "proc.int_gpu_mem_used,"+ + "proc.bytea_children,"+ "proc.int_virt_max_used,"+ "proc.int_virt_used,"+ "host.str_name AS host_name, " + @@ -571,7 +597,8 @@ public boolean increaseReservedMemory(ProcInterface p, long value) { "int_virt_max_used,"+ "int_virt_used,"+ "host_name, " + - "str_os " + + "str_os, " + + "bytea_children " + "FROM (" + GET_VIRTUAL_PROC + " " + "AND " + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index f632d0f73..bf8c23ebf 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -28,6 +28,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import com.google.protobuf.ByteString; import org.apache.log4j.Logger; import org.springframework.jdbc.core.RowMapper; import org.springframework.jdbc.core.support.JdbcDaoSupport; @@ -547,6 +548,7 @@ public ProcSeq getProcs(HostInterface host) { r.filterByHost(host); r.sortByHostName(); r.sortByDispatchedTime(); + logger.info("!!!! INSIDE getProcs Whiteboard!!! called getProcs !!! line 551"); return ProcSeq.newBuilder().addAllProcs(getProcs(r).getProcsList()).build(); } @@ -554,6 +556,7 @@ public ProcSeq getProcs(HostInterface host) { public ProcSeq getProcs(ProcSearchInterface p) { p.sortByHostName(); p.sortByDispatchedTime(); + logger.info("!!!! Inside getPROCS!!!!! line 559"); List procs = getJdbcTemplate().query(p.getFilteredQuery(GET_PROC), PROC_MAPPER, p.getValuesArray()); return ProcSeq.newBuilder().addAllProcs(procs).build(); @@ -969,9 +972,11 @@ public Proc mapRow(ResultSet rs, int row) throws SQLException { SqlUtil.getString(rs,"str_log_dir"), SqlUtil.getString(rs,"job_name"), SqlUtil.getString(rs,"frame_name"))) .setRedirectTarget(SqlUtil.getString(rs, "str_redirect")) + .setChildProcesses(SqlUtil.getByteString(rs, "bytea_children")) .addAllServices(Arrays.asList(SqlUtil.getString(rs,"str_services").split(","))) .build(); } +// logger.info("called ROW MAPPER!!! setChildProcesses!!!"); }; public static final RowMapper TASK_MAPPER = @@ -1609,6 +1614,7 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "proc.ts_booked, " + "proc.ts_dispatched, " + "proc.b_unbooked, " + + "proc.bytea_children, " + "redirect.str_name AS str_redirect " + "FROM proc " + "JOIN host ON proc.pk_host = host.pk_host " + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java index ce261535c..b7c0a65aa 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java @@ -439,8 +439,9 @@ void updateFrameMemoryUsageAndLluTime(FrameInterface frame, long rss, long maxRs * @param usedGpuMemory * @param maxUsedGpuMemory */ - void updateProcMemoryUsage(FrameInterface frame, long rss, long maxRss, long vsize, - long maxVsize, long usedGpuMemory, long maxUsedGpuMemory); + void updateProcMemoryUsage(FrameInterface frame, long rss, long maxRss, + long vsize, long maxVsize, long usedGpuMemory, + long maxUsedGpuMemory, byte[] children); /** * Return true if adding the given core units would put the show diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java index fa34e7100..d77cc03f3 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java @@ -535,10 +535,10 @@ public void lostProc(VirtualProc proc, String reason, int exitStatus) { @Override @Transactional(propagation = Propagation.REQUIRED) public void updateProcMemoryUsage(FrameInterface frame, long rss, long maxRss, - long vsize, long maxVsize, - long usedGpuMemory, long maxUsedGpuMemory) { + long vsize, long maxVsize, long usedGpuMemory, + long maxUsedGpuMemory, byte[] children) { procDao.updateProcMemoryUsage(frame, rss, maxRss, vsize, maxVsize, - usedGpuMemory, maxUsedGpuMemory); + usedGpuMemory, maxUsedGpuMemory, children); } @Override diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java index 33d7a3cf8..59c6b1ced 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java @@ -552,9 +552,9 @@ private void updateMemoryUsageAndLluTime(List rFrames) { dispatchSupport.updateFrameMemoryUsageAndLluTime(frame, rf.getRss(), rf.getMaxRss(), rf.getLluTime()); - dispatchSupport.updateProcMemoryUsage(frame, - rf.getRss(), rf.getMaxRss(), rf.getVsize(), rf.getMaxVsize(), - rf.getUsedGpuMemory(), rf.getMaxUsedGpuMemory()); + dispatchSupport.updateProcMemoryUsage(frame, rf.getRss(), rf.getMaxRss(), + rf.getVsize(), rf.getMaxVsize(), rf.getUsedGpuMemory(), + rf.getMaxUsedGpuMemory(), rf.getChildren().toByteArray()); } updateJobMemoryUsage(rFrames); diff --git a/cuebot/src/main/java/com/imageworks/spcue/util/SqlUtil.java b/cuebot/src/main/java/com/imageworks/spcue/util/SqlUtil.java index a7233777e..ddfbabc61 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/util/SqlUtil.java +++ b/cuebot/src/main/java/com/imageworks/spcue/util/SqlUtil.java @@ -19,6 +19,8 @@ package com.imageworks.spcue.util; +import com.google.protobuf.ByteString; + import java.sql.ResultSet; import java.sql.SQLException; import java.util.UUID; @@ -132,5 +134,14 @@ public static String getString(ResultSet rs, int index) throws SQLException { return value; } } + + public static ByteString getByteString(ResultSet rs, String field) throws SQLException { + byte[] data = rs.getBytes(field); + if (rs.wasNull()) { + return ByteString.copyFrom("".getBytes()); + } else { + return ByteString.copyFrom(data); + } + } } diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V14__Add_bytea_field_Proc_table.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V14__Add_bytea_field_Proc_table.sql new file mode 100644 index 000000000..9b013b8ed --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V14__Add_bytea_field_Proc_table.sql @@ -0,0 +1,2 @@ +ALTER TABLE proc +ADD COLUMN bytea_children BYTEA; \ No newline at end of file diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java index dbecca54d..ab95e7e1a 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java @@ -325,8 +325,9 @@ public void testUpdateProcMemoryUsage() { procDao.insertVirtualProc(proc); procDao.verifyRunningProc(proc.getId(), frame.getId()); + byte[] children = new byte[100]; - procDao.updateProcMemoryUsage(frame, 100, 100, 1000, 1000, 0, 0); + procDao.updateProcMemoryUsage(frame, 100, 100, 1000, 1000, 0, 0, children); } @@ -584,16 +585,17 @@ public void testFindReservedMemoryOffender() { int i = 1; List frames = dispatcherDao.findNextDispatchFrames(job, host, 6); assertEquals(6, frames.size()); - + byte[] children = new byte[100]; for (DispatchFrame frame: frames) { VirtualProc proc = VirtualProc.build(host, frame); + proc.childProcesses = children; frame.minMemory = Dispatcher.MEM_RESERVED_DEFAULT; dispatcher.dispatch(frame, proc); // Increase the memory usage as frames are added procDao.updateProcMemoryUsage(frame, - 1000*i, 1000*i, 1000*i, 1000*i, 0, 0); + 1000*i, 1000*i, 1000*i, 1000*i, 0, 0, children); i++; } @@ -666,7 +668,8 @@ public void testBalanceUnderUtilizedProcs() { proc1.frameId = frame1.id; procDao.insertVirtualProc(proc1); - procDao.updateProcMemoryUsage(frame1, 250000, 250000, 250000, 250000, 0, 0); + byte[] children = new byte[100]; + procDao.updateProcMemoryUsage(frame1, 250000, 250000, 250000, 250000, 0, 0, children); layerDao.updateLayerMaxRSS(frame1, 250000, true); FrameDetail frameDetail2 = frameDao.findFrameDetail(job, "0002-pass_1"); @@ -676,7 +679,7 @@ public void testBalanceUnderUtilizedProcs() { proc2.frameId = frame2.id; procDao.insertVirtualProc(proc2); - procDao.updateProcMemoryUsage(frame2, 255000, 255000,255000, 255000, 0, 0); + procDao.updateProcMemoryUsage(frame2, 255000, 255000,255000, 255000, 0, 0, children); layerDao.updateLayerMaxRSS(frame2, 255000, true); FrameDetail frameDetail3 = frameDao.findFrameDetail(job, "0003-pass_1"); @@ -686,7 +689,7 @@ public void testBalanceUnderUtilizedProcs() { proc3.frameId = frame3.id; procDao.insertVirtualProc(proc3); - procDao.updateProcMemoryUsage(frame3, 3145728, 3145728,3145728, 3145728, 0, 0); + procDao.updateProcMemoryUsage(frame3, 3145728, 3145728,3145728, 3145728, 0, 0, children); layerDao.updateLayerMaxRSS(frame3,300000, true); procDao.balanceUnderUtilizedProcs(proc3, 100000); @@ -797,6 +800,7 @@ public void getProcsBySearch() { proc.frameId = f.id; proc.layerId = f.layerId; proc.showId = f.showId; + proc.childProcesses = "".getBytes(); procDao.insertVirtualProc(proc); } diff --git a/cuegui/cuegui/FrameMonitorTree.py b/cuegui/cuegui/FrameMonitorTree.py index cfda922ec..fcbed14e1 100644 --- a/cuegui/cuegui/FrameMonitorTree.py +++ b/cuegui/cuegui/FrameMonitorTree.py @@ -907,3 +907,4 @@ def __init__(self, widget, filterSelectedLayersCallback): self.__menuActions.frames().addAction(self, "eat") self.__menuActions.frames().addAction(self, "kill") self.__menuActions.frames().addAction(self, "eatandmarkdone") + self.__menuActions.frames().addAction(self, "viewRunning") diff --git a/cuegui/cuegui/MenuActions.py b/cuegui/cuegui/MenuActions.py index b756d3f8e..2aadf86f2 100644 --- a/cuegui/cuegui/MenuActions.py +++ b/cuegui/cuegui/MenuActions.py @@ -50,6 +50,7 @@ import cuegui.LocalBooking import cuegui.Logger import cuegui.PreviewWidget +import cuegui.ProcChildren import cuegui.ServiceDialog import cuegui.ShowDialog import cuegui.TasksDialog @@ -871,6 +872,18 @@ def viewLastLog(self, rpcObjects=None): else: cuegui.Utils.popupView(path) + viewRunning_info = ["View Running", None, "viewRunning"] + + def viewRunning(self): + """ Display a Proc's child processes Host statistics.""" + job = self._getSource() + text = "Displaying host stats for each child process for job:\n%s" % job.name() + title = "View Running Child Proc Host Stats" + procDialog = cuegui.ProcChildren.ProcChildrenDialog(job=job, + text=text, + title=title) + procDialog.exec_() + useLocalCores_info = ["Use local cores...", "Set a single frame to use the local desktop cores.", "configure"] diff --git a/cuegui/cuegui/ProcChildren.py b/cuegui/cuegui/ProcChildren.py new file mode 100644 index 000000000..db5574981 --- /dev/null +++ b/cuegui/cuegui/ProcChildren.py @@ -0,0 +1,167 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""An interface for redirecting resources from one job to another job. + +The concept here is that there is a target job that needs procs. The user would choose the job. +The highest core/memory value would be detected and would populate 2 text boxes for cores and +memory. The user could then adjust these and hit search. The search will find all hosts that have +frames running that can be redirected to the target job.""" + + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +from builtins import str + +from PySide2 import QtGui +from PySide2 import QtWidgets + +import opencue + +import cuegui.Utils + + + +class ProcChildren(QtWidgets.QWidget): + """Widget for displaying Host statistics for a Proc's child processes.""" + + HEADERS = ["PID", "Name", "Start Time", "Rss (KB)", "VSize (KB)", + "Statm Rss (KB)", "Statm Size (KB)", "Cmd line"] + + def __init__(self, job, parent=None): + """ + Initializes the list of procs for a given job to display + + :param job: job Object for this item + :ptype job: opencue.wrappers.job.Job + :param parent: Optional parent for this item + """ + QtWidgets.QWidget.__init__(self, parent) + self._data = {} + + self._job = job + self._model = QtGui.QStandardItemModel(self) + self._model.setColumnCount(5) + self._model.setHorizontalHeaderLabels(ProcChildren.HEADERS) + + self._tree = QtWidgets.QTreeView(self) + self._tree.setEditTriggers(QtWidgets.QAbstractItemView.NoEditTriggers) + self._tree.setModel(self._model) + + layout = QtWidgets.QVBoxLayout(self) + layout.addWidget(self._tree) + + def update(self): + """ Updates visual representation with latest data""" + self._model.clear() + self._model.setHorizontalHeaderLabels(ProcChildren.HEADERS) + childrenProc = opencue.compiled_proto.report_pb2.ChildrenProcStats() + data = { } + + try: + procs = opencue.api.getProcs(job=[self._job.name()], + layer=[x.name() for x in self._job.getLayers()]) + for proc in procs: + data['children_processes'] =\ + childrenProc.FromString(proc.data.child_processes).children + + name = proc.data.name.split("/")[0] + if name not in data: + cue_host = opencue.api.findHost(name) + data['host'] = cue_host + self._addProc(data) + + self._data = data + + except opencue.exception.CueException: + msg = ('No Proc Data available: \n%s ' + % (self._job.name())) + cuegui.Utils.showErrorMessageBox(msg) + + def _addProc(self, entry): + host = entry["host"] + + checkbox = QtGui.QStandardItem(host.data.name) + + self._model.appendRow([checkbox]) + + for proc in entry['children_processes']: + checkbox.appendRow([QtGui.QStandardItem(proc.stat.pid), + QtGui.QStandardItem(proc.stat.name), + QtGui.QStandardItem(proc.start_time), + QtGui.QStandardItem(str(proc.stat.rss)), + QtGui.QStandardItem(str(proc.stat.vsize)), + QtGui.QStandardItem(str(proc.statm.rss)), + QtGui.QStandardItem(str(proc.statm.size)), + QtGui.QStandardItem(str(proc.cmdline))]) + + self._tree.setExpanded(self._model.indexFromItem(checkbox), True) + self._tree.resizeColumnToContents(0) + + +class ProcChildrenDialog(QtWidgets.QDialog): + """ + Dialog for displaying Host statistics for a Proc's child processes + """ + def __init__(self, job, text, title, parent=None): + """ + Initializes the data to be displayed + :ptype job: opencue.wrappers.job.Job + :param job: job Object for this item + :ptype text: str + :param text: Description of what is being displayed + :ptype title: str + :param title: Window Title + :param parent: AbstractActions + :param parent: The dialog's parent + """ + + QtWidgets.QDialog.__init__(self, parent) + self.parent = parent + self.job = job + self.text = text + self.title = title + self.setWindowTitle(self.title) + self._childProcStats = ProcChildren(job=job, parent=parent) + + _labelText = QtWidgets.QLabel(text, self) + _labelText.setWordWrap(True) + _btnUpdate = QtWidgets.QPushButton("Refresh", self) + _btnClose = QtWidgets.QPushButton("Close", self) + + _vlayout = QtWidgets.QVBoxLayout(self) + _vlayout.addWidget(_labelText) + _vlayout.addWidget(self._childProcStats) + + _hlayout = QtWidgets.QHBoxLayout() + _hlayout.addWidget(_btnUpdate) + _hlayout.addWidget(_btnClose) + _vlayout.addLayout(_hlayout) + + self._childProcStats.update() + # pylint: disable=no-member + _btnClose.clicked.connect(self.accept) + _btnUpdate.clicked.connect(self.refresh) + # pylint: enable=no-member + + def refresh(self): + """Update host report statistics""" + self._childProcStats.update() + + def accept(self): + """Exit""" + self.close() diff --git a/proto/host.proto b/proto/host.proto index f2193bdb9..e2e46e65f 100644 --- a/proto/host.proto +++ b/proto/host.proto @@ -375,6 +375,7 @@ message Proc { repeated string services = 17; int64 used_gpu_memory = 18; float reserved_gpus = 19; + bytes child_processes = 20; } message ProcSearchCriteria { diff --git a/proto/report.proto b/proto/report.proto index d53103663..78c8e4da6 100644 --- a/proto/report.proto +++ b/proto/report.proto @@ -93,11 +93,62 @@ message RunningFrameInfo { int32 num_gpus = 15; int64 max_used_gpu_memory = 16; // kB int64 used_gpu_memory = 17; // kB + ChildrenProcStats children = 18; //additional data about the running frame's child processes }; +message ChildrenProcStats { + repeated ProcStats children = 2; +} +message ProcStats { + Stat stat = 1; + Statm statm = 2; + Status status = 3; + string cmdline = 4; + string start_time = 5; +} + +message Stat { + int64 rss = 1; + int64 vsize = 2; + string state = 3; + string name = 4; + string pid = 5; +} + +message Statm { + int64 rss = 1; + int64 size = 2; +} + +message Status { + int64 vrss = 1; + int64 vsize = 2; +} // -------- Requests & Responses --------] +message ChildrenProcStatsRequest { + string frame_id = 1; + repeated ProcStats children = 2; +} + +message ChildrenProcStatsResponse { + ChildrenProcStats children_procs_stats = 1; +} + +message ProcStatsRequest { + string name = 1; + string stat = 2; + string statm = 3; + string cmdline = 4; + string state = 5; + string frame_id = 6; +} + +message ProcStatsResponse { + ProcStats proc_stats = 1; +} + // ReportRqdStartup message RqdReportRqdStartupRequest { BootReport boot_report = 1; diff --git a/proto/rqd.proto b/proto/rqd.proto index f67add41b..6d82ccaab 100644 --- a/proto/rqd.proto +++ b/proto/rqd.proto @@ -111,6 +111,7 @@ message RunFrame { map environment = 21; map attributes = 22; int32 num_gpus = 23; + report.ChildrenProcStats children = 24; } message RunFrameSeq { diff --git a/pycue/opencue/api.py b/pycue/opencue/api.py index 602170692..075d5c374 100644 --- a/pycue/opencue/api.py +++ b/pycue/opencue/api.py @@ -457,7 +457,7 @@ def getFrame(uniq): @util.grpcExceptionParser def getFrames(job, **options): - """Finds frames in a job that match the search critieria. + """Finds frames in a job that match the search criteria. :type job: str :param job: the job name diff --git a/rqd/rqd/rqconstants.py b/rqd/rqd/rqconstants.py index 14979ee95..b52d20afe 100644 --- a/rqd/rqd/rqconstants.py +++ b/rqd/rqd/rqconstants.py @@ -99,6 +99,13 @@ PATH_LOADAVG = "/proc/loadavg" PATH_STAT = "/proc/stat" PATH_MEMINFO = "/proc/meminfo" +# stat and statm are inaccurate because of kernel internal scability optimation +# stat/statm/status are inaccurate values, true values are in smaps +# but RQD user can't read smaps get: +# [Errno 13] Permission denied: '/proc/166289/smaps' +PATH_PROC_PID_STAT = "/proc/{0}/stat" +PATH_PROC_PID_STATM = "/proc/{0}/statm" +PATH_PROC_PID_CMDLINE = "/proc/{0}/cmdline" if platform.system() == 'Linux': SYS_HERTZ = os.sysconf('SC_CLK_TCK') diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index 63ac828ec..06c26e80f 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -22,6 +22,7 @@ from builtins import str from builtins import object +import datetime import logging import os import platform @@ -202,6 +203,16 @@ def __writeFooter(self): print("%-20s%s" % ("utime", self.frameInfo.utime), file=self.rqlog) print("%-20s%s" % ("stime", self.frameInfo.stime), file=self.rqlog) print("%-20s%s" % ("renderhost", self.rqCore.machine.getHostname()), file=self.rqlog) + + print("%-20s%s" % ("maxrss (KB)", self.frameInfo.maxRss), file=self.rqlog) + for child in sorted(self.frameInfo.childrenProcs.items(), + key=lambda item: item[1]['start_time']): + print("\t%-20s%s" % (child[1]['name'], child[1]['rss']), file=self.rqlog) + print("\t%-20s%s" % ("start_time", + datetime.timedelta(seconds=child[1]["start_time"])), + file=self.rqlog) + print("\t%-20s%s" % ("cmdline", " ".join(child[1]["cmd_line"])), file=self.rqlog) + print("="*59, file=self.rqlog) # pylint: disable=broad-except diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index f9cdea412..c712e5c41 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -207,6 +207,17 @@ def __updateGpuAndLlu(self, frame): stat = os.stat(frame.runFrame.log_dir_file).st_mtime frame.lluTime = int(stat) + def _getFields(self, filePath): + fields = [] + + try: + with open(filePath, "r") as statFile: + fields = statFile.read().split() + except rqd.rqexceptions.RqdException as e: + log.warning("Failed to read file: %s", e) + + return fields + def rssUpdate(self, frames): """Updates the rss and maxrss for all running frames""" if platform.system() == 'Windows' and winpsIsAvailable: @@ -235,13 +246,17 @@ def rssUpdate(self, frames): for pid in os.listdir("/proc"): if pid.isdigit(): try: - with open("/proc/%s/stat" % pid, "r") as statFile: + with open(rqd.rqconstants.PATH_PROC_PID_STAT + .format(pid), "r") as statFile: statFields = [None, None] + statFile.read().rsplit(")", 1)[-1].split() - # See "man proc" pids[pid] = { + "name": statFields[1], + "state": statFields[2], + "pgrp": statFields[4], "session": statFields[5], - "vsize": statFields[22], + # virtual memory size is in bytes convert to kb + "vsize": int(statFields[22]), "rss": statFields[23], # These are needed to compute the cpu used "utime": statFields[13], @@ -252,10 +267,28 @@ def rssUpdate(self, frames): # after system boot. "start_time": statFields[21], } + # cmdline: + p = psutil.Process(int(pid)) + pids[pid]["cmd_line"] = p.cmdline() + + try: + # 2. Collect Statm file: /proc/[pid]/statm (same as status vsize in kb) + # - size: "total program size" + # - rss: inaccurate, similar to VmRss in /proc/[pid]/status + child_statm_fields = self._getFields( + rqd.rqconstants.PATH_PROC_PID_STATM.format(pid)) + pids[pid]['statm_size'] = \ + int(re.search("\\d+", child_statm_fields[0]).group()) \ + if re.search("\\d+", child_statm_fields[0]) else -1 + pids[pid]['statm_rss'] = \ + int(re.search("\\d+", child_statm_fields[1]).group()) \ + if re.search("\\d+", child_statm_fields[1]) else -1 + except rqd.rqexceptions.RqdException as e: + log.warning("Failed to read statm file: %s", e) # pylint: disable=broad-except - except Exception: - log.exception('failed to read stat file for pid %s', pid) + except rqd.rqexceptions.RqdException: + log.exception('Failed to read stat file for pid %s', pid) # pylint: disable=too-many-nested-blocks try: @@ -267,10 +300,12 @@ def rssUpdate(self, frames): for frame in values: if frame.pid > 0: + session = str(frame.pid) rss = 0 vsize = 0 pcpu = 0 + # children pids share the same session id for pid, data in pids.items(): if data["session"] == session: try: @@ -303,19 +338,52 @@ def rssUpdate(self, frames): pidPcpu = totalTime / seconds pcpu += pidPcpu pidData[pid] = totalTime, seconds, pidPcpu + # only keep the highest recorded rss value + if pid in frame.childrenProcs: + childRss = (int(data["rss"]) * resource.getpagesize()) // 1024 + if childRss > frame.childrenProcs[pid]['rss']: + frame.childrenProcs[pid]['rss_page'] = int(data["rss"]) + frame.childrenProcs[pid]['rss'] = childRss + frame.childrenProcs[pid]['vsize'] = \ + int(data["vsize"]) // 1024 + frame.childrenProcs[pid]['statm_rss'] = \ + (int(data["statm_rss"]) \ + * resource.getpagesize()) // 1024 + frame.childrenProcs[pid]['statm_size'] = \ + (int(data["statm_size"]) * \ + resource.getpagesize()) // 1024 + else: + frame.childrenProcs[pid] = \ + {'name': data['name'], + 'rss_page': int(data["rss"]), + 'rss': (int(data["rss"]) * resource.getpagesize()) // 1024, + 'vsize': int(data["vsize"]) // 1024, + 'state': data['state'], + # statm reports in pages (~ 4kB) + # same as VmRss in /proc/[pid]/status (in KB) + 'statm_rss': (int(data["statm_rss"]) * \ + resource.getpagesize()) // 1024, + 'statm_size': (int(data["statm_size"]) * \ + resource.getpagesize()) // 1024, + 'cmd_line': data["cmd_line"], + 'start_time': seconds} # pylint: disable=broad-except except Exception as e: log.warning( 'Failure with pid rss update due to: %s at %s', e, traceback.extract_tb(sys.exc_info()[2])) - + # convert bytes to KB rss = (rss * resource.getpagesize()) // 1024 vsize = int(vsize/1024) frame.rss = rss frame.maxRss = max(rss, frame.maxRss) + if os.path.exists(frame.runFrame.log_dir_file): + stat = os.stat(frame.runFrame.log_dir_file).st_mtime + frame.lluTime = int(stat) + frame.vsize = vsize frame.maxVsize = max(vsize, frame.maxVsize) diff --git a/rqd/rqd/rqnetwork.py b/rqd/rqd/rqnetwork.py index 94d350429..39a65c750 100644 --- a/rqd/rqd/rqnetwork.py +++ b/rqd/rqd/rqnetwork.py @@ -25,6 +25,7 @@ from random import shuffle import abc import atexit +import datetime import logging import os import platform @@ -74,6 +75,7 @@ def __init__(self, rqCore, runFrame): self.stime = 0 self.lluTime = 0 + self.childrenProcs = {} def runningFrameInfo(self): """Returns the RunningFrameInfo object""" @@ -94,10 +96,47 @@ def runningFrameInfo(self): llu_time=self.lluTime, num_gpus=self.runFrame.num_gpus, max_used_gpu_memory=self.maxUsedGpuMemory, - used_gpu_memory=self.usedGpuMemory + used_gpu_memory=self.usedGpuMemory, + children=self._serializeChildrenProcs() ) return runningFrameInfo + def _serializeChildrenProcs(self): + """ Collect and serialize children proc stats for protobuf + Convert to Kilobytes: + * RSS (Resident set size) measured in pages + * Statm size measured in pages + * Stat size measured in bytes + + :param data: dictionary + :return: serialized children proc host stats + :rtype: rqd.compiled_proto.report_pb2.ChildrenProcStats + """ + childrenProc = rqd.compiled_proto.report_pb2.ChildrenProcStats() + for proc, values in self.childrenProcs.items(): + procStats = rqd.compiled_proto.report_pb2.ProcStats() + procStatFile = rqd.compiled_proto.report_pb2.Stat() + procStatmFile = rqd.compiled_proto.report_pb2.Statm() + + procStatFile.pid = proc + procStatFile.name = values["name"] if values["name"] else "" + procStatFile.state = values["state"] + procStatFile.vsize = values["vsize"] + procStatFile.rss = values["rss"] + + procStatmFile.size = values["statm_size"] + procStatmFile.rss = values["statm_rss"] + # pylint: disable=no-member + procStats.stat.CopyFrom(procStatFile) + procStats.statm.CopyFrom(procStatmFile) + procStats.cmdline = " ".join(values["cmd_line"]) + + startTime = datetime.datetime.now() - datetime.timedelta(seconds=values["start_time"]) + procStats.start_time = startTime.strftime("%Y-%m-%d %H:%M%S") + childrenProc.children.extend([procStats]) + # pylint: enable=no-member + return childrenProc + def status(self): """Returns the status of the frame""" return self.runningFrameInfo() diff --git a/rqd/tests/cuerqd_tests.py b/rqd/tests/cuerqd_tests.py index d1ef540a1..7fc574704 100644 --- a/rqd/tests/cuerqd_tests.py +++ b/rqd/tests/cuerqd_tests.py @@ -179,6 +179,7 @@ def test_launchFrame(self, stubMock, frameStubMock): runFrame.frame_id = "FD1S3I154O646UGSNN" runFrameRequest = rqd.compiled_proto.rqd_pb2.RqdStaticLaunchFrameRequest(run_frame=runFrame) rqdHost = rqd.cuerqd.RqdHost(RQD_HOSTNAME) + rqdHost.active = False rqdHost.launchFrame(runFrame) diff --git a/rqd/tests/rqcore_tests.py b/rqd/tests/rqcore_tests.py index bb70c12a3..aad307dac 100644 --- a/rqd/tests/rqcore_tests.py +++ b/rqd/tests/rqcore_tests.py @@ -286,6 +286,7 @@ def test_launchFrame(self, frameThreadMock): self.machineMock.return_value.state = rqd.compiled_proto.host_pb2.UP self.nimbyMock.return_value.locked = False frame = rqd.compiled_proto.rqd_pb2.RunFrame(uid=22, num_cores=10) + rqd.rqconstants.OVERRIDE_NIMBY = None self.rqcore.launchFrame(frame) @@ -331,6 +332,7 @@ def test_launchDuplicateFrame(self): frameId = 'arbitrary-frame-id' self.rqcore.storeFrame(frameId, rqd.compiled_proto.rqd_pb2.RunFrame(frame_id=frameId)) frameToLaunch = rqd.compiled_proto.rqd_pb2.RunFrame(frame_id=frameId) + rqd.rqconstants.OVERRIDE_NIMBY = None with self.assertRaises(rqd.rqexceptions.DuplicateFrameViolationException): self.rqcore.launchFrame(frameToLaunch) @@ -593,6 +595,7 @@ def test_runLinux(self, getTempDirMock, permsUser, timeMock, popenMock): # mkdir rqCore.machine.isDesktop.return_value = True rqCore.machine.getHostInfo.return_value = renderHost rqCore.nimby.locked = False + children = rqd.compiled_proto.report_pb2.ChildrenProcStats() runFrame = rqd.compiled_proto.rqd_pb2.RunFrame( frame_id=frameId, @@ -600,7 +603,8 @@ def test_runLinux(self, getTempDirMock, permsUser, timeMock, popenMock): # mkdir frame_name=frameName, uid=frameUid, user_name=frameUsername, - log_dir=logDir) + log_dir=logDir, + children=children) frameInfo = rqd.rqnetwork.RunningFrame(rqCore, runFrame) # when @@ -635,7 +639,7 @@ def test_runLinux(self, getTempDirMock, permsUser, timeMock, popenMock): # mkdir rqd.compiled_proto.report_pb2.FrameCompleteReport( host=renderHost, frame=rqd.compiled_proto.report_pb2.RunningFrameInfo( - job_name=jobName, frame_id=frameId, frame_name=frameName), + job_name=jobName, frame_id=frameId, frame_name=frameName, children=children), exit_status=returnCode)) # TODO(bcipriano) Re-enable this test once Windows is supported. The main sticking point here @@ -665,6 +669,7 @@ def disabled__test_runWindows(self, permsUser, timeMock, popenMock): rqCore.machine.isDesktop.return_value = True rqCore.machine.getHostInfo.return_value = renderHost rqCore.nimby.locked = False + children = rqd.compiled_proto.report_pb2.ChildrenProcStats() runFrame = rqd.compiled_proto.rqd_pb2.RunFrame( frame_id=frameId, @@ -674,6 +679,7 @@ def disabled__test_runWindows(self, permsUser, timeMock, popenMock): uid=frameUid, user_name=frameUsername, log_dir=logDir, + children=children, environment={'CUE_IFRAME': '2000'}) frameInfo = rqd.rqnetwork.RunningFrame(rqCore, runFrame) @@ -693,7 +699,7 @@ def disabled__test_runWindows(self, permsUser, timeMock, popenMock): rqd.compiled_proto.report_pb2.FrameCompleteReport( host=renderHost, frame=rqd.compiled_proto.report_pb2.RunningFrameInfo( - job_name=jobName, frame_id=frameId, frame_name=frameName), + job_name=jobName, frame_id=frameId, frame_name=frameName, children=children), exit_status=returnCode)) @mock.patch('platform.system', new=mock.Mock(return_value='Darwin')) @@ -726,6 +732,7 @@ def test_runDarwin(self, getTempDirMock, permsUser, timeMock, popenMock): rqCore.machine.isDesktop.return_value = True rqCore.machine.getHostInfo.return_value = renderHost rqCore.nimby.locked = False + children = rqd.compiled_proto.report_pb2.ChildrenProcStats() runFrame = rqd.compiled_proto.rqd_pb2.RunFrame( frame_id=frameId, @@ -733,7 +740,8 @@ def test_runDarwin(self, getTempDirMock, permsUser, timeMock, popenMock): frame_name=frameName, uid=frameUid, user_name=frameUsername, - log_dir=logDir) + log_dir=logDir, + children=children) frameInfo = rqd.rqnetwork.RunningFrame(rqCore, runFrame) # when @@ -765,7 +773,7 @@ def test_runDarwin(self, getTempDirMock, permsUser, timeMock, popenMock): rqd.compiled_proto.report_pb2.FrameCompleteReport( host=renderHost, frame=rqd.compiled_proto.report_pb2.RunningFrameInfo( - job_name=jobName, frame_id=frameId, frame_name=frameName), + job_name=jobName, frame_id=frameId, frame_name=frameName, children=children), exit_status=returnCode)) diff --git a/rqd/tests/rqmachine_tests.py b/rqd/tests/rqmachine_tests.py index 123b797a3..489cc35c8 100644 --- a/rqd/tests/rqmachine_tests.py +++ b/rqd/tests/rqmachine_tests.py @@ -156,6 +156,10 @@ PROC_PID_STAT_WITH_SPACES = '105 (test space)' + PROC_STAT_SUFFIX PROC_PID_STAT_WITH_BRACKETS = '105 (test) (brackets)' + PROC_STAT_SUFFIX +PROC_PID_STATM = '152510 14585 7032 9343 0 65453 0' + +PROC_PID_CMDLINE = ' sleep 20' + @mock.patch.object(rqd.rqutil.Memoize, 'isCached', new=mock.MagicMock(return_value=False)) @mock.patch('platform.system', new=mock.MagicMock(return_value='Linux')) @@ -282,6 +286,8 @@ def _test_rssUpdate(self, proc_stat): pid = 105 frameId = 'unused-frame-id' self.fs.create_file('/proc/%d/stat' % pid, contents=proc_stat) + self.fs.create_file('/proc/%s/cmdline' % pid, contents=PROC_PID_CMDLINE) + self.fs.create_file('/proc/%s/statm' % pid, contents=PROC_PID_STATM) runningFrame = rqd.rqnetwork.RunningFrame(self.rqCore, rqd.compiled_proto.rqd_pb2.RunFrame()) runningFrame.pid = pid From 671c895b452672b236f4dbd4d83fc5370a121a39 Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Mon, 2 May 2022 14:16:42 -0700 Subject: [PATCH 182/277] Add kill dependent and group dependent features (#1115) * Add kill dependent and group dependent features Migrate features from old cue. Kill dependent will suggest killing dependent job every time a job with dependencies gets killed. Group dependents adds a checkbox to cuetopia to allow grouping dependent jobs in the JobMonitor treeView. * Fix MenuActions unittest and pylint Co-authored-by: dtavares --- cuegui/cuegui/JobMonitorTree.py | 126 +++++++++++++++++++-- cuegui/cuegui/MenuActions.py | 69 ++++++++++- cuegui/cuegui/plugins/MonitorJobsPlugin.py | 44 +++++-- cuegui/tests/MenuActions_tests.py | 2 + 4 files changed, 221 insertions(+), 20 deletions(-) diff --git a/cuegui/cuegui/JobMonitorTree.py b/cuegui/cuegui/JobMonitorTree.py index 7552b2859..239aa1f2f 100644 --- a/cuegui/cuegui/JobMonitorTree.py +++ b/cuegui/cuegui/JobMonitorTree.py @@ -20,7 +20,7 @@ from __future__ import print_function from __future__ import division - +from future.utils import iteritems from builtins import map import time @@ -71,6 +71,7 @@ class JobMonitorTree(cuegui.AbstractTreeWidget.AbstractTreeWidget): """Tree widget to display a list of monitored jobs.""" __loadMine = True + __groupDependent = True view_object = QtCore.Signal(object) def __init__(self, parent): @@ -151,7 +152,9 @@ def __init__(self, parent): self.__jobTimeLoaded = {} self.__userColors = {} - + self.__dependentJobs = {} + self._dependent_items = {} + self.__reverseDependents = {} # Used to build right click context menus self.__menuActions = cuegui.MenuActions.MenuActions( self, self.updateSoon, self.selectedObjects) @@ -180,8 +183,21 @@ def tick(self): self._update() return + self.updateJobCount() self.ticksWithoutUpdate += 1 + def updateJobCount(self): + """Called at every tick. The total number of monitored + jobs is added to the column header + """ + count = 0 + iterator = QtWidgets.QTreeWidgetItemIterator(self) + while iterator.value(): + count += 1 + iterator += 1 + + self.headerItem().setText(0, "Job [Total Count: {}]".format(count)) + def __itemSingleClickedCopy(self, item, col): """Called when an item is clicked on. Copies selected object names to the middle click selection clip board. @@ -226,13 +242,21 @@ def setLoadMine(self, value): @type value: boolean or QtCore.Qt.Checked or QtCore.Qt.Unchecked""" self.__loadMine = (value is True or value == QtCore.Qt.Checked) - def addJob(self, job, timestamp=None): + def setGroupDependent(self, value): + """Enables or disables the auto grouping of the dependent jobs + @param value: New groupDependent state + @type value: boolean or QtCore.Qt.Checked or QtCore.Qt.Unchecked""" + self.__groupDependent = (value is True or value == QtCore.Qt.Checked) + self.updateRequest() + + def addJob(self, job, timestamp=None, loading_from_config=False): """Adds a job to the list. With locking" @param job: Job can be None, a job object, or a job name. @type job: job, string, None - @param timestamp: UTC time of the specific date the job was - added to be monitored - @type timestamp: float""" + @param loading_from_config: Whether or not this method is being called + for loading jobs found in user config + @type loading_from_config: bool + """ newJobObj = cuegui.Utils.findJob(job) self.ticksLock.lock() try: @@ -241,6 +265,42 @@ def addJob(self, job, timestamp=None): if not self.__groupDependent: self.__load[jobKey] = newJobObj self.__jobTimeLoaded[jobKey] = timestamp if timestamp else time.time() + else: + # We'll only add the new job if it's not already listed + # as a dependent on another job + if jobKey not in self.__reverseDependents.keys(): + self.__load[jobKey] = newJobObj + + # when we are adding jobs manually, we want to calculate + # all dependencies (active or not), so the user can see + # all the dependent jobs, even after the main/parent job + # has finished. + # When we're loading jobs from user config, we want to + # only include the active dependents. This is because + # the dependencies have already been calculated and + # listed in the config as a flat list, so attempting + # to re-add them will result in duplicates that will + # throw off the cleanup loop at the end of this method + active_only = not loading_from_config + dep = self.__menuActions.jobs( + ).getRecursiveDependentJobs([newJobObj], + active_only=active_only) + self.__dependentJobs[jobKey] = dep + # we'll also store a reversed dictionary for + # dependencies with the dependent as key and the main + # job as the value, this will be used in step 2 + # below to remove jobs that are added here + # as dependents + for j in dep: + depKey = cuegui.Utils.getObjectKey(j) + self.__reverseDependents[depKey] = newJobObj + self.__jobTimeLoaded[depKey] = time.time() + self.__jobTimeLoaded[jobKey] = time.time() + + for j in self.__reverseDependents: + if j in self.__load: + del self.__load[j] + finally: self.ticksLock.unlock() @@ -274,6 +334,20 @@ def _removeItem(self, item): # pylint: enable=no-member cuegui.AbstractTreeWidget.AbstractTreeWidget._removeItem(self, item) self.__jobTimeLoaded.pop(item.rpcObject, "") + try: + jobKey = cuegui.Utils.getObjectKey(item) + # Remove the item from the main _items dictionary as well as the + # __dependentJobs and the reverseDependent dictionaries + cuegui.AbstractTreeWidget.AbstractTreeWidget._removeItem(self, item) + dependent_jobs = self.__dependentJobs.get(jobKey, []) + for djob in dependent_jobs: + del self.__reverseDependents[djob] + del self.__reverseDependents[jobKey] + except KeyError: + # Dependent jobs are not stored in as keys the main self._items + # dictionary, trying to remove dependent jobs from self._items + # raises a KeyError, which we can safely ignore + pass def removeAllItems(self): """Notifies the other widgets of each item being unmonitored, then calls @@ -284,6 +358,8 @@ def removeAllItems(self): # pylint: enable=no-member if proxy in self.__jobTimeLoaded: del self.__jobTimeLoaded[proxy] + self.__dependentJobs.clear() + self.__reverseDependents.clear() cuegui.AbstractTreeWidget.AbstractTreeWidget.removeAllItems(self) def removeFinishedItems(self): @@ -296,6 +372,7 @@ def contextMenuEvent(self, e): @param e: Right click QEvent @type e: QEvent""" menu = QtWidgets.QMenu() + menu.setToolTipsVisible(True) __selectedObjects = self.selectedObjects() __count = len(__selectedObjects) @@ -304,6 +381,7 @@ def contextMenuEvent(self, e): self.__menuActions.jobs().addAction(menu, "unmonitor") self.__menuActions.jobs().addAction(menu, "view") self.__menuActions.jobs().addAction(menu, "emailArtist") + self.__menuActions.jobs().addAction(menu, "showProgBar") self.__menuActions.jobs().addAction(menu, "viewComments") self.__menuActions.jobs().addAction(menu, "useLocalCores") @@ -404,11 +482,21 @@ def _getUpdate(self): # Gather list of all other jobs to update monitored_proxies.append(objectKey) + # Refresh the dependent proxies for the next update + for job, dependents in iteritems(self.__dependentJobs): + ids = [d.id() for d in dependents] + # If the job has no dependents, then ids is an empty list, + # The getJobs call returns every job on the cue when called + # an empty list for the id argument! + if not ids: + continue + tmp = opencue.api.getJobs(id=ids, all=True) + self.__dependentJobs[job] = tmp + if self.__loadMine: # This auto-loads all the users jobs for job in opencue.api.getJobs(user=[cuegui.Utils.getUsername()]): - objectKey = cuegui.Utils.getObjectKey(job) - jobs[objectKey] = job + self.addJob(job) # Prune the users jobs from the remaining proxies to update for proxy, job in list(jobs.items()): @@ -438,30 +526,46 @@ def _processUpdate(self, work, rpcObjects): for proxy, item in list(self._items.items()): if not proxy in rpcObjects: rpcObjects[proxy] = item.rpcObject - + # pylint: disable=too-many-nested-blocks try: selectedKeys = [ cuegui.Utils.getObjectKey(item.rpcObject) for item in self.selectedItems()] scrolled = self.verticalScrollBar().value() + expanded = [cuegui.Utils.getObjectKey(item.rpcObject) + for item in self._items.values() if item.isExpanded()] # Store the creation time for the current item for item in list(self._items.values()): self.__jobTimeLoaded[cuegui.Utils.getObjectKey(item.rpcObject)] = item.created + # Store the creation time for the dependent jobs + for item in self._dependent_items.values(): + self.__jobTimeLoaded[cuegui.Utils.getObjectKey(item.rpcObject)] = item.created self._items = {} self.clear() - for proxy, job in list(rpcObjects.items()): + for proxy, job in iteritems(rpcObjects): self._items[proxy] = JobWidgetItem(job, self.invisibleRootItem(), self.__jobTimeLoaded.get(proxy, None)) if proxy in self.__userColors: self._items[proxy].setUserColor(self.__userColors[proxy]) + if self.__groupDependent: + dependent_jobs = self.__dependentJobs.get(proxy, []) + for djob in dependent_jobs: + item = JobWidgetItem(djob, + self._items[proxy], + self.__jobTimeLoaded.get(proxy, None)) + dkey = cuegui.Utils.getObjectKey(djob) + self._dependent_items[dkey] = item + if dkey in self.__userColors: + self._dependent_items[dkey].setUserColor( + self.__userColors[dkey]) self.verticalScrollBar().setRange(scrolled, len(rpcObjects.keys()) - scrolled) list(map(lambda key: self._items[key].setSelected(True), [key for key in selectedKeys if key in self._items])) - + list(self._items[key].setExpanded(True) for key in expanded if key in self._items) except opencue.exception.CueException as e: list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) finally: diff --git a/cuegui/cuegui/MenuActions.py b/cuegui/cuegui/MenuActions.py index 2aadf86f2..2b6f46076 100644 --- a/cuegui/cuegui/MenuActions.py +++ b/cuegui/cuegui/MenuActions.py @@ -360,13 +360,78 @@ def resume(self, rpcObjects=None): def kill(self, rpcObjects=None): jobs = self._getOnlyJobObjects(rpcObjects) if jobs: - if cuegui.Utils.questionBoxYesNo(self._caller, "Kill jobs?", - "Are you sure you want to kill these jobs?", + msg = ("Are you sure you want to kill these jobs?\n\n" + "** Note: This will stop all running frames and " + "permanently remove the jobs from the cue. " + "The jobs will NOT be able to return once killed.") + if cuegui.Utils.questionBoxYesNo(self._caller, "Kill jobs?", msg, [job.data.name for job in jobs]): for job in jobs: job.kill() + self.killDependents(jobs) self._update() + def killDependents(self, jobs): + dependents = self.getRecursiveDependentJobs(jobs) + if not dependents: + return + if cuegui.Utils.questionBoxYesNo(self._caller, + "Kill depending jobs?", + "The jobs have been killed. " + "Do you want to kill %s jobs that depend on it?" % + len(dependents), + sorted([dep.name() for dep in dependents])): + for depJob in dependents: + try: + depJob.kill() + except opencue.exception.CueException as e: + errMsg = "Failed to kill depending job: %s - %s" % (depJob.name(), e) + logger.warning(errMsg) + else: + # Drop only direct dependents. + for job in dependents: + try: + self.dropJobsDependingOnThis(job) + except opencue.exception.CueException as e: + logger.warning("Failed to drop dependencies: %s", e) + + def getRecursiveDependentJobs(self, jobs, seen=None, active_only=True): + seen = set() if seen is None else seen + dependents = [] + if not jobs: + return dependents + for job in jobs: + for dep in self.getExternalDependentNames(job, active_only): + if dep.data.name not in seen: + dependents.append(dep) + seen.add(dep.data.name) + return dependents + self.getRecursiveDependentJobs(dependents, + seen, + active_only) + + def getExternalDependentNames(self, job, active_only=True): + # pylint: disable=consider-using-set-comprehension + job_names = set([dep.dependErJob() + for dep in job.getWhatDependsOnThis() + if (not dep.isInternal()) + and (dep.isActive() if active_only else True)]) + + return [self.getJobByName(job_name) for job_name in job_names] + + def getJobByName(self, job_name): + jobs = opencue.api.getJobs(substr=[job_name], include_finished=True) + if not jobs: + raise Exception("Job %s not found" % job_name) + return jobs[0] + + def dropJobsDependingOnThis(self, job): + for dep in job.getWhatDependsOnThis(): + if not dep.isInternal(): + # pylint: disable=no-member + job = self.getJobByName(self, dep.dependOnJob()) + job.dropDepends(opencue.wrappers.depend.DependTarget.EXTERNAL) + # pylint: enable=no-member + eatDead_info = ["Eat dead frames", None, "eat"] def eatDead(self, rpcObjects=None): diff --git a/cuegui/cuegui/plugins/MonitorJobsPlugin.py b/cuegui/cuegui/plugins/MonitorJobsPlugin.py index 406cd979d..8b704b9ef 100644 --- a/cuegui/cuegui/plugins/MonitorJobsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorJobsPlugin.py @@ -91,7 +91,13 @@ def __init__(self, parent): self.jobMonitor.setColumnWidths), ("columnOrder", self.jobMonitor.getColumnOrder, - self.jobMonitor.setColumnOrder)]) + self.jobMonitor.setColumnOrder), + ("grpDependentCb", + self.getGrpDependent, + self.setGrpDependent), + ("autoLoadMineCb", + self.getAutoLoadMine, + self.setAutoLoadMine)]) def addJob(self, rpcObject): """Adds a job to be monitored.""" @@ -99,7 +105,7 @@ def addJob(self, rpcObject): rpcObject = cuegui.Utils.findJob(rpcObject.data.job_name) elif not cuegui.Utils.isJob(rpcObject): return - self.jobMonitor.addJob(rpcObject) + self.jobMonitor.addJob(rpcObject, loading_from_config=True) self.raise_() def getJobIds(self): @@ -202,6 +208,22 @@ def _regexLoadJobsHandle(self): for job in opencue.api.getJobs(regex=[substring]): self.jobMonitor.addJob(job) + def getGrpDependent(self): + """Is group dependent checked""" + return bool(self.grpDependentCb.isChecked()) + + def setGrpDependent(self, state): + """Set group dependent""" + self.grpDependentCb.setChecked(bool(state)) + + def getAutoLoadMine(self): + """Is autoload mine checked""" + return bool(self.autoLoadMineCb.isChecked()) + + def setAutoLoadMine(self, state): + """Set autoload mine""" + self.autoLoadMineCb.setChecked(bool(state)) + def _buttonSetup(self, layout): clearButton = QtWidgets.QPushButton("Clr") clearButton.setFocusPolicy(QtCore.Qt.NoFocus) @@ -213,14 +235,22 @@ def _buttonSetup(self, layout): spacer.setFixedWidth(20) layout.addWidget(spacer) - mineCheckbox = QtWidgets.QCheckBox("Autoload Mine") - mineCheckbox.setFocusPolicy(QtCore.Qt.NoFocus) - mineCheckbox.setChecked(True) - layout.addWidget(mineCheckbox) - mineCheckbox.stateChanged.connect(self.jobMonitor.setLoadMine) # pylint: disable=no-member + self.autoLoadMineCb = QtWidgets.QCheckBox("Autoload Mine") + self.autoLoadMineCb.setFocusPolicy(QtCore.Qt.NoFocus) + self.autoLoadMineCb.setChecked(True) + layout.addWidget(self.autoLoadMineCb) + self.autoLoadMineCb.stateChanged.connect(self.jobMonitor.setLoadMine) # pylint: disable=no-member self._loadFinishedJobsSetup(self.__toolbar) + self.grpDependentCb = QtWidgets.QCheckBox("Group Dependent") + self.grpDependentCb.setFocusPolicy(QtCore.Qt.NoFocus) + self.grpDependentCb.setChecked(True) + layout.addWidget(self.grpDependentCb) + # pylint: disable=no-member + self.grpDependentCb.stateChanged.connect(self.jobMonitor.setGroupDependent) + # pylint: enable=no-member + finishedButton = QtWidgets.QPushButton(QtGui.QIcon(":eject.png"), "Finished") finishedButton.setToolTip("Unmonitor finished jobs") finishedButton.setFocusPolicy(QtCore.Qt.NoFocus) diff --git a/cuegui/tests/MenuActions_tests.py b/cuegui/tests/MenuActions_tests.py index 87282eefe..87e31df0b 100644 --- a/cuegui/tests/MenuActions_tests.py +++ b/cuegui/tests/MenuActions_tests.py @@ -232,6 +232,8 @@ def test_resume(self): def test_kill(self, yesNoMock): job = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(name='job-name')) job.kill = mock.Mock() + job.getWhatDependsOnThis = mock.Mock() + job.getWhatDependsOnThis.return_value = [] self.job_actions.kill(rpcObjects=[job]) From f81070cf9514922f8cda9efe7adecd72fceaa4a2 Mon Sep 17 00:00:00 2001 From: Akim Ruslanov Date: Mon, 2 May 2022 15:56:50 -0700 Subject: [PATCH 183/277] Support different logging paths based on OS (#1133) * Support different logging paths based on OS * Added extra documentation * Support different logging paths based on OS * Added extra documentation --- .../com/imageworks/spcue/util/JobLogUtil.java | 17 +++++--- cuebot/src/main/resources/opencue.properties | 16 +++++-- .../spcue/test/util/JobLogUtilTests.java | 43 ++++++++++++++----- cuebot/src/test/resources/opencue.properties | 3 +- 4 files changed, 58 insertions(+), 21 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/util/JobLogUtil.java b/cuebot/src/main/java/com/imageworks/spcue/util/JobLogUtil.java index e7cfe1dff..c223ebbc0 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/util/JobLogUtil.java +++ b/cuebot/src/main/java/com/imageworks/spcue/util/JobLogUtil.java @@ -38,9 +38,9 @@ public boolean createJobLogDirectory(String path) { return f.isDirectory(); } - public String getJobLogDir(String show, String shot) { + public String getJobLogDir(String show, String shot, String os) { StringBuilder sb = new StringBuilder(512); - sb.append(getJobLogRootDir()); + sb.append(getJobLogRootDir(os)); sb.append("/"); sb.append(show); sb.append("/"); @@ -51,7 +51,7 @@ public String getJobLogDir(String show, String shot) { public String getJobLogPath(JobDetail job) { StringBuilder sb = new StringBuilder(512); - sb.append(getJobLogDir(job.showName, job.shot)); + sb.append(getJobLogDir(job.showName, job.shot, job.os)); sb.append("/"); sb.append(job.name); sb.append("--"); @@ -59,8 +59,11 @@ public String getJobLogPath(JobDetail job) { return sb.toString(); } - public String getJobLogRootDir() { - return env.getRequiredProperty("log.frame-log-root", String.class); + public String getJobLogRootDir(String os) { + try { + return env.getRequiredProperty(String.format("log.frame-log-root.%s", os), String.class); + } catch (IllegalStateException e) { + return env.getRequiredProperty("log.frame-log-root.default_os", String.class); + } } -} - +} \ No newline at end of file diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index d28925c15..e1c4fa402 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -46,10 +46,20 @@ mailing.enabled=true # Set to a boolean value. See com/imageworks/spcue/services/JmsMover.java. messaging.enabled=false -# Root directory for which logs will be stored. See com/imageworks/spcue/util/JobLogUtil.java. +# Default root directory for which logs will be stored if no other OS is defined. +# See com/imageworks/spcue/util/JobLogUtil.java. # Override this via environment variable (CUE_FRAME_LOG_DIR) or command line flag -# (--log.frame-log-root). Command line flag will be preferred if both are provided. -log.frame-log-root=${CUE_FRAME_LOG_DIR:/shots} +# (--log.frame-log-root.default_os). Command line flag will be preferred if both are provided. +log.frame-log-root.default_os=${CUE_FRAME_LOG_DIR:/shots} +# To set up root directories for other OS create new environment +# variable as `log.frame-log-root.[OS] where OS relates to str_os on the job table +# For example: +# - log.frame-log-root.linux=${CUE_FRAME_LOG_DIR:/shots} +# - log.frame-log-root.Windows=${CUE_FRAME_LOG_DIR:/S:} +# Note that for Windows, either forward or back slashes will work. However if CUE_FRAME_LOG_DIR is empty +# and S directory is in the root, the path will be broken due to the slash in front of S. Hence, if you +# are planning to use a folder in the root, use: +# - log.frame-log-root.Windows=${S:} # Maximum number of jobs to query. dispatcher.job_query_max=20 diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/util/JobLogUtilTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/util/JobLogUtilTests.java index bbb550ec2..417924b9b 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/util/JobLogUtilTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/util/JobLogUtilTests.java @@ -36,32 +36,55 @@ public class JobLogUtilTests extends AbstractTransactionalJUnit4SpringContextTes @Resource private JobLogUtil jobLogUtil; - private String logRoot; + private String logRootDefault; + private String logRootSomeOs; @Before public void setUp() { - // This value should match what's defined in test/resources/opencue.properties. - logRoot = "/arbitraryLogDirectory"; + // The values should match what's defined in test/resources/opencue.properties. + logRootDefault = "/arbitraryLogDirectory"; + logRootSomeOs = "/arbitrarySomeOsLogDirectory"; } @Test - public void testGetJobLogRootDir() { - assertEquals(logRoot, jobLogUtil.getJobLogRootDir()); + public void testGetJobLogRootDirDefault() { + assertEquals(logRootDefault, jobLogUtil.getJobLogRootDir("someUndefinedOs")); } @Test - public void testGetJobLogDir() { - assertEquals(logRoot + "/show/shot/logs", jobLogUtil.getJobLogDir("show", "shot")); + public void testGetJobLogRootSomeOs() { + assertEquals(logRootSomeOs, jobLogUtil.getJobLogRootDir("some_os")); } @Test - public void testGetJobLogPath() { + public void testGetJobLogDirDefault() { + assertEquals(logRootDefault + "/show/shot/logs", jobLogUtil.getJobLogDir("show", "shot", "someUndefinedOs")); + } + + @Test + public void testGetJobLogDirSomeOs() { + assertEquals(logRootSomeOs + "/show/shot/logs", jobLogUtil.getJobLogDir("show", "shot", "some_os")); + } + + @Test + public void testGetJobLogPathDefault() { JobDetail jobDetail = new JobDetail(); jobDetail.id = "id"; jobDetail.name = "name"; jobDetail.showName = "show"; jobDetail.shot = "shot"; - assertEquals(logRoot + "/show/shot/logs/name--id", jobLogUtil.getJobLogPath(jobDetail)); + jobDetail.os = "someUndefinedOs"; + assertEquals(logRootDefault + "/show/shot/logs/name--id", jobLogUtil.getJobLogPath(jobDetail)); } -} + @Test + public void testGetJobLogPathSomeOs() { + JobDetail jobDetail = new JobDetail(); + jobDetail.id = "id"; + jobDetail.name = "name"; + jobDetail.showName = "show"; + jobDetail.shot = "shot"; + jobDetail.os = "some_os"; + assertEquals(logRootSomeOs + "/show/shot/logs/name--id", jobLogUtil.getJobLogPath(jobDetail)); + } +} diff --git a/cuebot/src/test/resources/opencue.properties b/cuebot/src/test/resources/opencue.properties index 5c9264492..334408470 100644 --- a/cuebot/src/test/resources/opencue.properties +++ b/cuebot/src/test/resources/opencue.properties @@ -31,7 +31,8 @@ kill_queue.threadPoolSizeInitial=2 kill_queue.threadPoolSizeMax=6 kill_queue.queueSize=1000 -log.frame-log-root=/arbitraryLogDirectory +log.frame-log-root.default_os=/arbitraryLogDirectory +log.frame-log-root.some_os=/arbitrarySomeOsLogDirectory dispatcher.job_query_max=20 dispatcher.job_lock_expire_seconds=2 From eb477ff7e065c366ed1054a3d4dfe4049d65ec42 Mon Sep 17 00:00:00 2001 From: Alexis Oblet Date: Tue, 10 May 2022 20:33:03 +0200 Subject: [PATCH 184/277] [metrics] Add Limits info to Prometheus. (#1147) --- connectors/prometheus_metrics/metrics | 64 ++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 11 deletions(-) diff --git a/connectors/prometheus_metrics/metrics b/connectors/prometheus_metrics/metrics index 960132938..4db675a27 100755 --- a/connectors/prometheus_metrics/metrics +++ b/connectors/prometheus_metrics/metrics @@ -7,15 +7,22 @@ from prometheus_client import start_http_server from prometheus_client import Gauge -CLUE_HOSTS_HARDWARE = Gauge( - 'cue_hosts_hardware_total', 'hosts hardware status', ['status']) +WEB_SERVICE_PORT = 8302 +REPORT_INTERVAL_SECONDS = 30 + + +CLUE_HOSTS_HARDWARE = Gauge('cue_hosts_hardware_total', 'hosts hardware status', ['status']) CLUE_HOSTS_LOCK = Gauge('cue_hosts_lock_total', 'hosts lock status', ['status']) CLUE_PROCS = Gauge('cue_procs_total', 'number of Procs') CLUE_PROCS_USABLE = Gauge('cue_procs_usable_total', 'number of usable Procs') CLUE_PROCS_USED = Gauge('cue_procs_used_total', 'number of Procs currently in use') -CLUE_FRAMES = Gauge('cue_frames', 'number of frames ', ['status', 'show']) -CLUE_REMAIN = Gauge('cue_remain', 'remaining core seconds (estimated) ', ['show']) +CLUE_FRAMES_PER_SHOW = Gauge('cue_frames', 'number of frames ', ['status', 'show']) +CLUE_FRAMES_LIMIT_PER_SHOW = Gauge('cue_frames_limit', 'number of limits', ['limit', 'status', 'show']) +CLUE_REMAIN_FRAME_PER_SHOW = Gauge('cue_remain', 'remaining core seconds (estimated) ', ['show']) + +CLUE_LIMITS = Gauge('cue_limits', 'limit stats ', ['name', 'value']) +CLUE_LIMITS_CAPACITY = Gauge('cue_limits_capacity', 'limits capacity ', ['name']) MANAGE_WAITING = Gauge('cue_manage_waiting_total', '') MANAGE_REMAINING_CAPACITY = Gauge('cue_manage_remaining_capacity_total', '') @@ -55,20 +62,31 @@ STRANDED_CORES = Gauge('cue_stranded_cores_total', '') def main(): + default_frame_stats = { + 'pending': 0, + 'dead': 0, + 'eaten': 0, + 'succeeded': 0, + 'running': 0 + } + while True: jobs = opencue.api.getJobs() shows = {} shows_remaining = {} + limits = {} for job in jobs: show = job.show() if show not in shows: - shows[show] = {'pending': 0, 'dead': 0, - 'eaten': 0, 'succeeded': 0, 'running': 0} + shows[show] = default_frame_stats.copy() if show not in shows_remaining: shows_remaining[show] = 0 + if show not in limits: + limits[show] = {} + shows[show]['pending'] += job.pendingFrames() shows[show]['dead'] += job.deadFrames() shows[show]['eaten'] += job.eatenFrames() @@ -77,12 +95,36 @@ def main(): shows_remaining[show] += job.coreSecondsRemaining() + show_limits = limits[show] + for layer in job.getLayers(): + for limit in layer.limits(): + if limit not in show_limits: + show_limits[limit] = default_frame_stats.copy() + + show_limits[limit]['pending'] += layer.pendingFrames() + show_limits[limit]['dead'] += layer.deadFrames() + show_limits[limit]['eaten'] += layer.eatenFrames() + show_limits[limit]['succeeded'] += layer.succeededFrames() + show_limits[limit]['running'] += layer.runningFrames() + for show in shows: - for k, v in shows[show].items(): - CLUE_FRAMES.labels(status=k, show=show).set(v) + for frame_status, frame_count in shows[show].items(): + CLUE_FRAMES_PER_SHOW.labels(status=frame_status, show=show).set(frame_count) + + for limit, frame_stats in limits[show].items(): + for status, frame_count in frame_stats.items(): + CLUE_FRAMES_LIMIT_PER_SHOW.labels(limit=limit, status=status, show=show).set(frame_count) for show in shows_remaining: - CLUE_REMAIN.labels(show=show).set(shows_remaining[show]) + CLUE_REMAIN_FRAME_PER_SHOW.labels(show=show).set(shows_remaining[show]) + + for limit in opencue.api.getLimits(): + limit_name = limit.name() + current_running = limit.currentRunning() + max_value = limit.maxValue() + CLUE_LIMITS.labels(name=limit_name, value='current_running').set(current_running) + CLUE_LIMITS.labels(name=limit_name, value='max').set(max_value) + CLUE_LIMITS_CAPACITY.labels(name=limit_name).set(current_running/(max_value or 1) * 100.) # Handle the Host information hosts = opencue.api.getHosts() @@ -174,9 +216,9 @@ def main(): PICKED_UP_CORES.set(system_stats.picked_up_cores) STRANDED_CORES.set(system_stats.stranded_cores) - time.sleep(30) + time.sleep(REPORT_INTERVAL_SECONDS) if __name__ == '__main__': - start_http_server(8302) + start_http_server(WEB_SERVICE_PORT) main() From ab727e50a91a5d07204e1581a32e7f40170a846e Mon Sep 17 00:00:00 2001 From: Alexis Oblet Date: Tue, 10 May 2022 20:33:56 +0200 Subject: [PATCH 185/277] [cuegui] Fix cue monitor gpu min/max. (#1144) --- cuegui/cuegui/CueJobMonitorTree.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cuegui/cuegui/CueJobMonitorTree.py b/cuegui/cuegui/CueJobMonitorTree.py index 440933038..11ad7f9d2 100644 --- a/cuegui/cuegui/CueJobMonitorTree.py +++ b/cuegui/cuegui/CueJobMonitorTree.py @@ -559,8 +559,8 @@ def contextMenuEvent(self, e): menu.addSeparator() self.__menuActions.jobs().addAction(menu, "setMinCores") self.__menuActions.jobs().addAction(menu, "setMaxCores") - self.__menuActions.jobs().addAction(menu, "setMinGpu") - self.__menuActions.jobs().addAction(menu, "setMaxGpu") + self.__menuActions.jobs().addAction(menu, "setMinGpus") + self.__menuActions.jobs().addAction(menu, "setMaxGpus") self.__menuActions.jobs().addAction(menu, "setPriority") self.__menuActions.jobs().addAction(menu, "setMaxRetries") if counts["job"] == 1: From 187fe5696dbfac0c11bbd1b972eb8acb83e84cb7 Mon Sep 17 00:00:00 2001 From: prOOrc Date: Tue, 10 May 2022 21:48:29 +0300 Subject: [PATCH 186/277] [cuebot] Add LluTime field in WhiteboardDaoJdbc.FRAME_MAPPER. (#1146) --- .../imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index bf8c23ebf..aec665423 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -1424,6 +1424,13 @@ public Frame mapRow(ResultSet rs, int rowNum) throws SQLException { else { builder.setStopTime(0); } + java.sql.Timestamp ts_llu = rs.getTimestamp("ts_llu"); + if (ts_llu!= null) { + builder.setLluTime((int) (ts_llu.getTime() / 1000)); + } + else { + builder.setLluTime(0); + } builder.setTotalCoreTime(rs.getInt("int_total_past_core_time")); builder.setTotalGpuTime(rs.getInt("int_total_past_gpu_time")); From ae16a2f492fbf154375eaafc24560235f66d4a1d Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Tue, 10 May 2022 12:13:04 -0700 Subject: [PATCH 187/277] [rqd] Adjust shutdown behavior and use Nimby pynput as default. (#1142) --- rqd/deploy/rqd3_init.d | 16 +++--- rqd/rqd/cuerqd.py | 8 ++- rqd/rqd/rqconstants.py | 2 +- rqd/rqd/rqcore.py | 29 +++++----- rqd/rqd/rqmachine.py | 9 ++-- rqd/rqd/rqnetwork.py | 6 ++- rqd/rqd/rqnimby.py | 120 +++++++++++++++++++++++------------------ 7 files changed, 110 insertions(+), 80 deletions(-) diff --git a/rqd/deploy/rqd3_init.d b/rqd/deploy/rqd3_init.d index babc1e1f1..b146780f3 100644 --- a/rqd/deploy/rqd3_init.d +++ b/rqd/deploy/rqd3_init.d @@ -3,7 +3,7 @@ # RQD3: Start/stop rqd3 services # # chkconfig: 345 98 02 -# description: RQD for opencue +# description: Opencue RQD agent # # Source function library. @@ -18,7 +18,7 @@ RQD=${RQD_PATH}rqd.py start() { [ -f /usr/local/etc/sweatbox.csh ] && echo "Refusing to start RQD3 on a sweatbox" && exit 0 - echo -n $"Starting rqd3 services:" + echo -n $"Starting openrqd services:" cd ${RQD_PATH} daemon "${RQD}" -d echo "" @@ -26,20 +26,18 @@ start() idle_restart() { - echo -n "Requesting idle restart of rqd3 services:" + echo -n "Requesting idle restart of openrqd services:" cd ${RQD_PATH} - daemon "./cuerqd.py" -restart + daemon "rqd/cuerqd.py --restart &>/dev/null || :" echo "" } stop() { - echo -n "Stopping rqd3 services:" + echo -n "Stopping openrqd services:" cd ${RQD_PATH} - daemon "./cuerqd.py" -exit_now - sleep 2 - killproc ${RQD} >/dev/null 2>&1 || : - echo "" + daemon "rqd/cuerqd.py" --exit_now + echo "Stop Request completed" } case "$1" in diff --git a/rqd/rqd/cuerqd.py b/rqd/rqd/cuerqd.py index c80098579..f7aee88ef 100755 --- a/rqd/rqd/cuerqd.py +++ b/rqd/rqd/cuerqd.py @@ -100,7 +100,13 @@ def shutdownRqdIdle(self): def shutdownRqdNow(self): """Shuts down the host now.""" print(self.rqdHost, "Sending shutdownRqdNow command") - self.stub.ShutdownRqdNow(rqd.compiled_proto.rqd_pb2.RqdStaticShutdownNowRequest()) + try: + self.stub.ShutdownRqdNow(rqd.compiled_proto.rqd_pb2.RqdStaticShutdownNowRequest()) + # pylint: disable=broad-except + except Exception: + # Shutting down the service from inside means this request will receive + # a connection error response + pass def restartRqdIdle(self): """Restarts RQD on the host when idle.""" diff --git a/rqd/rqd/rqconstants.py b/rqd/rqd/rqconstants.py index b52d20afe..0d6968d27 100644 --- a/rqd/rqd/rqconstants.py +++ b/rqd/rqd/rqconstants.py @@ -123,7 +123,7 @@ OVERRIDE_PROCS = None # number of physical cpus. ex: None or 2 OVERRIDE_MEMORY = None # in Kb OVERRIDE_NIMBY = None # True to turn on, False to turn off -USE_NIMBY_PYNPUT = platform.system() == 'Windows' +USE_NIMBY_PYNPUT = True # True pynput, False select OVERRIDE_HOSTNAME = None # Force to use this hostname ALLOW_GPU = False LOAD_MODIFIER = 0 # amount to add/subtract from load diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index 06c26e80f..48ef7ccc9 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -817,15 +817,18 @@ def shutdown(self): log.warning("Rebooting machine by request") self.machine.reboot() else: - log.warning("Shutting down RQD by request") + log.warning("Shutting down RQD by request. pid(%s)", os.getpid()) + self.network.stopGrpc() + # Using sys.exit would raise SystemExit, giving exception handlers a chance + # to block this + # pylint: disable=protected-access + os._exit(0) def handleExit(self, signalnum, flag): """Shutdown threads and exit RQD.""" del signalnum del flag self.shutdown() - self.network.stopGrpc() - sys.exit() def launchFrame(self, runFrame): """This will setup for the launch the frame specified in the arguments. @@ -925,8 +928,12 @@ def reportStatus(self): def shutdownRqdNow(self): """Kill all running frames and shutdown RQD""" self.machine.state = rqd.compiled_proto.host_pb2.DOWN - self.lockAll() - self.killAllFrame("shutdownRqdNow Command") + try: + self.lockAll() + self.killAllFrame("shutdownRqdNow Command") + # pylint: disable=broad-except + except Exception: + log.exception("Failed to kill frames, stopping service anyways") if not self.__cache: self.shutdown() @@ -980,14 +987,12 @@ def rebootIdle(self): def nimbyOn(self): """Activates nimby, does not kill any running frames until next nimby event. Also does not unlock until sufficient idle time is reached.""" - if platform.system() != "Windows" and os.getuid() != 0: - log.warning("Not starting nimby, not running as root") - return - if not self.nimby.active: + if self.nimby and not self.nimby.active: try: self.nimby.run() - log.info("Nimby has been activated") - except: + log.warning("Nimby has been activated") + # pylint: disable=broad-except + except Exception: self.nimby.locked = False err = "Nimby is in the process of shutting down" log.exception(err) @@ -1007,7 +1012,7 @@ def onNimbyLock(self): self.sendStatusReport() def onNimbyUnlock(self, asOf=None): - """This is called by nimby when it unlocks the machine due to sufficent + """This is called by nimby when it unlocks the machine due to sufficient idle. A new report is sent to the cuebot. @param asOf: Time when idle state began, if known.""" del asOf diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index c712e5c41..bed78fdef 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -207,14 +207,15 @@ def __updateGpuAndLlu(self, frame): stat = os.stat(frame.runFrame.log_dir_file).st_mtime frame.lluTime = int(stat) - def _getFields(self, filePath): + def _getFields(self, pidFilePath): fields = [] try: - with open(filePath, "r") as statFile: + with open(pidFilePath, "r") as statFile: fields = statFile.read().split() - except rqd.rqexceptions.RqdException as e: - log.warning("Failed to read file: %s", e) + # pylint: disable=broad-except + except Exception: + log.warning("Not able to read pidFilePath: %s", pidFilePath) return fields diff --git a/rqd/rqd/rqnetwork.py b/rqd/rqd/rqnetwork.py index 39a65c750..be0208319 100644 --- a/rqd/rqd/rqnetwork.py +++ b/rqd/rqd/rqnetwork.py @@ -165,6 +165,7 @@ def kill(self, message=""): except OSError as e: log.warning( "kill() tried to kill a non-existant pid for: %s Error: %s", self.frameId, e) + # pylint: disable=broad-except except Exception as e: log.warning("kill() encountered an unknown error: %s", e) else: @@ -226,8 +227,8 @@ def serveForever(self): def shutdown(self): """Stops the gRPC server.""" - log.info('Stopping grpc server.') - self.server.stop(0) + log.warning('Stopping grpc server.') + self.server.stop(10) def stayAlive(self): """Runs forever until killed.""" @@ -255,6 +256,7 @@ def stopGrpc(self): """Stops the gRPC server.""" self.grpcServer.shutdown() del self.grpcServer + log.warning("Stopped grpc server") def closeChannel(self): """Closes the gRPC channel.""" diff --git a/rqd/rqd/rqnimby.py b/rqd/rqd/rqnimby.py index e57e39f23..e2cdbf214 100644 --- a/rqd/rqd/rqnimby.py +++ b/rqd/rqd/rqnimby.py @@ -20,43 +20,50 @@ from __future__ import print_function from __future__ import division -from abc import ABCMeta, abstractmethod +from abc import abstractmethod +import abc import os import select -import time import signal import threading +import time import logging -import platform import rqd.rqconstants import rqd.rqutil -log = logging.getLogger(__name__) - -if platform.system() == 'Windows': - pynputIsAvailable = False - try: - import pynput - pynputIsAvailable = True - except ImportError as e: - log.error(e) +log = logging.getLogger(__name__) # compatible with Python 2 and 3: -ABC = ABCMeta('ABC', (object,), {'__slots__': ()}) +ABC = abc.ABCMeta('ABC', (object,), {'__slots__': ()}) + class NimbyFactory(object): """ Factory to handle Linux/Windows platforms """ @staticmethod def getNimby(rqCore): """ assign platform dependent Nimby instance """ - nimbyInstance = None - if rqd.rqconstants.USE_NIMBY_PYNPUT and pynputIsAvailable: - nimbyInstance = NimbyPynput(rqCore) - else: - nimbyInstance = NimbySelect(rqCore) - return nimbyInstance + if rqd.rqconstants.USE_NIMBY_PYNPUT: + try: + # DISPLAY is required to import pynput internals + # and it's not automatically set depending on the + # environment rqd is running in + if "DISPLAY" not in os.environ: + os.environ['DISPLAY'] = ":0" + # pylint: disable=unused-import, import-error, unused-variable, import-outside-toplevel + import pynput + # pylint: disable=broad-except + except Exception: + # Ideally ImportError could be used here, but pynput + # can throw other kinds of exception while trying to + # access runpy components + log.exception("Failed to import pynput, falling back to Select module") + # Still enabling the application start as hosts can be manually locked + # using the API/GUI + return NimbyNop(rqCore) + return NimbyPynput(rqCore) + return NimbySelect(rqCore) class Nimby(threading.Thread, ABC): @@ -75,8 +82,6 @@ def __init__(self, rqCore): self.rqCore = rqCore self.locked = False self.active = False - log.warning("Locked state :%s", self.locked) - log.warning("Active state :%s", self.active) self.fileObjList = [] self.results = [[]] @@ -97,7 +102,7 @@ def lockNimby(self): """Activates the nimby lock, calls lockNimby() in rqcore""" if self.active and not self.locked: self.locked = True - log.info("Locked nimby") + log.warning("Locked nimby") self.rqCore.onNimbyLock() def unlockNimby(self, asOf=None): @@ -105,29 +110,15 @@ def unlockNimby(self, asOf=None): @param asOf: Time when idle state began, if known.""" if self.locked: self.locked = False - log.info("Unlocked nimby") + log.warning("Unlocked nimby") self.rqCore.onNimbyUnlock(asOf=asOf) def run(self): """Starts the Nimby thread""" - log.warning("Nimby Run") self.active = True - self.locked = True self.startListener() self.unlockedIdle() - rqd.rqutil.permissionsHigh() - try: - for device in os.listdir("/dev/input/"): - if device.startswith("event") or device.startswith("mice"): - try: - self.fileObjList.append(open("/dev/input/%s" % device, "rb")) - except IOError as e: - # Bad device found - log.warning("IOError: Failed to open %s, %s", "/dev/input/%s" % device, e) - finally: - rqd.rqutil.permissionsLow() - def stop(self): """Stops the Nimby thread""" log.warning("Stop Nimby") @@ -173,7 +164,6 @@ def startListener(self): """ start listening """ def stopListener(self): - """ stop listening """ self.closeEvents() def lockedInUse(self): @@ -206,7 +196,7 @@ def unlockedIdle(self): self.results = select.select(self.fileObjList, [], [], 5) # pylint: disable=broad-except except Exception: - pass + log.exception("failed to execute nimby check event") if not self.rqCore.machine.isNimbySafeToRunJobs(): log.warning("memory threshold has been exceeded, locking nimby") self.active = True @@ -244,7 +234,6 @@ def lockedIdle(self): def openEvents(self): """Opens the /dev/input/event* files so nimby can monitor them""" - log.warning("openEvents") self.closeEvents() rqd.rqutil.permissionsHigh() @@ -253,17 +242,15 @@ def openEvents(self): if device.startswith("event") or device.startswith("mice"): try: self.fileObjList.append(open("/dev/input/%s" % device, "rb")) - except IOError as e: + except IOError: # Bad device found - msg = ('IOError: Failed to open %s, %s' - % ("/dev/input/%s" % device, e)) - log.warning(msg) + log.exception("IOError: Failed to open /dev/input/%s", device) finally: rqd.rqutil.permissionsLow() def closeEvents(self): """Closes the /dev/input/event* files""" - log.warning("closeEvents") + log.info("closeEvents") if self.fileObjList: for fileObj in self.fileObjList: try: @@ -279,11 +266,14 @@ def isNimbyActive(self): """ return self.active and self.results[0] == [] + class NimbyPynput(Nimby): - """ Nimby Windows """ + """ Nimby using pynput """ def __init__(self, rqCore): Nimby.__init__(self, rqCore) + # pylint: disable=unused-import, import-error, import-outside-toplevel + import pynput self.mouse_listener = pynput.mouse.Listener( on_move=self.on_interaction, on_click=self.on_interaction, @@ -337,21 +327,19 @@ def unlockedIdle(self): self.lockNimby() self.thread = threading.Timer(rqd.rqconstants.CHECK_INTERVAL_LOCKED, self.lockedInUse) - log.warning("starting Thread") self.thread.start() def lockedIdle(self): """Nimby State: Machine is idle, waiting for sufficient idle time to unlock""" - log.warning("lockedIdle") - waitStartTime = time.time() + wait_start_time = time.time() time.sleep(rqd.rqconstants.MINIMUM_IDLE) if self.active and not self.interaction_detected and \ self.rqCore.machine.isNimbySafeToUnlock(): - log.warning("Start wait time: %s", waitStartTime) - self.unlockNimby(asOf=waitStartTime) + log.warning("Start wait time: %s", wait_start_time) + self.unlockNimby(asOf=wait_start_time) self.unlockedIdle() elif self.active: @@ -364,3 +352,33 @@ def isNimbyActive(self): :return: boolean if events are logged and Nimby is active """ return not self.active and self.interaction_detected + + +class NimbyNop(Nimby): + """Nimby option for when no option is available""" + def __init__(self, rqCore): + Nimby.__init__(self, rqCore) + self.warning_msg() + + @staticmethod + def warning_msg(): + """Just a helper to avoid duplication""" + log.warning("Using Nimby nop! Something went wrong on nimby's initialization.") + + def startListener(self): + self.warning_msg() + + def stopListener(self): + self.warning_msg() + + def lockedInUse(self): + self.warning_msg() + + def unlockedIdle(self): + self.warning_msg() + + def lockedIdle(self): + self.warning_msg() + + def isNimbyActive(self): + return False From 5fda6cfd4d236cb683214832d5dc9d067dd78398 Mon Sep 17 00:00:00 2001 From: Akim Ruslanov Date: Tue, 10 May 2022 12:27:22 -0700 Subject: [PATCH 188/277] [cuegui] New stuck frame plugin (#1120) --- cuegui/cuegui/images/add.png | Bin 0 -> 845 bytes cuegui/cuegui/plugins/StuckFramePlugin.py | 1834 +++++++++++++++++++++ 2 files changed, 1834 insertions(+) create mode 100755 cuegui/cuegui/images/add.png create mode 100644 cuegui/cuegui/plugins/StuckFramePlugin.py diff --git a/cuegui/cuegui/images/add.png b/cuegui/cuegui/images/add.png new file mode 100755 index 0000000000000000000000000000000000000000..783b35924be841d94c5297aed2904e85dbf0bd53 GIT binary patch literal 845 zcmV-T1G4;yP)Px#32;bRa{vGf6951U69E94oEQKA00(qQO+^RW0}>K55s2iGQ~&?~8FWQhbVF}# zZDnqB07G(RVRU6=Aa`kWXdp*PO;A^X4i^9b0=h{=K~y-)Ws+Zrlw}mgf9L(?`*zlu zVI12P)M+C&cJW{T7Ri(%ttf>suOx~=7xW@6A%eZwI}ste73o5uHWjamf)au-!s@0Z zK@C^Y1zB>%omO{tXZHKP_na=i-D$7`=jM6e=XYMt`-qv*S@N#P$)nF7Tv^=odaBJT zkOq<;pZMq7uRa|7`)eL4=>(j-y{T4;`o365>(pCU3=jZEWIcsvB@BvFx9Z<)8(H7_ z>wkdHC$=3GaijJ5tL9uN0BQ*!ILQH^8r!PUOLBki@|Ht`H-5|D!13`7Z+O+w`usQ$ ziK^hNo42cHd2`hXVjrU_K$lED{Kd-ofnN_?kU|^K<$BIc&)+n!4zGe&p%M=AWN|xd z%eyfR+qzjK_ZruDq3^Zh0Pe`8p3Ti<_AxXP?;X0>UjJK^cIw`ddYFr+_X6a~`KMlK zw5AXd^1*V}Ro=qzl*83+z%wh}L&O2_=lydS%0e1*daA6-!F00000NkvXXu0mjfjTMU! literal 0 HcmV?d00001 diff --git a/cuegui/cuegui/plugins/StuckFramePlugin.py b/cuegui/cuegui/plugins/StuckFramePlugin.py new file mode 100644 index 000000000..9a289b49b --- /dev/null +++ b/cuegui/cuegui/plugins/StuckFramePlugin.py @@ -0,0 +1,1834 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Plugin for managing stuck frames.""" + + +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import + +from builtins import str +from builtins import map +import datetime +import re +import os +from datetime import datetime +import time +import socket +import signal +import yaml + +from PySide2 import QtGui +from PySide2 import QtCore +from PySide2 import QtWidgets + +import opencue +import cuegui.AbstractDockWidget +import cuegui.Action +import cuegui.Constants +import cuegui.JobMonitorTree +import cuegui.Logger +import cuegui.Utils + +logger = cuegui.Logger.getLogger(__file__) + +PLUGIN_NAME = "Stuck Frame" +PLUGIN_CATEGORY = "Cuecommander" +PLUGIN_DESCRIPTION = "Work with stuck frames." +PLUGIN_REQUIRES = "CueCommander" +PLUGIN_PROVIDES = "StuckWidget" +CUE_SLEEP = 30 + +NAME_COLUMN = 0 +COMMENT_COLUMN = 1 +FRAME_COLUMN = 2 +LLU_COLUMN = 3 +RUNTIME_COLUMN = 4 +LASTLINE_COLUMN = 7 + + +class StuckWidget(cuegui.AbstractDockWidget.AbstractDockWidget): + """This builds what is displayed on the dock widget""" + + def __init__(self, parent): + cuegui.AbstractDockWidget.AbstractDockWidget.__init__(self, parent, PLUGIN_NAME) + self.__stuckWidget = StuckFrameWidget(self) + self.layout().addWidget(self.__stuckWidget) + + def pluginSaveState(self): + """Saves current state of the plugin and returns it as dict""" + filters = self.__stuckWidget.getControls().getFilters() + save = {} + for frame_filter in filters: + save[frame_filter.getServiceBoxText()] = [frame_filter.getRegexText(), + frame_filter.getTime(), + frame_filter.getMinLLu(), + frame_filter.getAvgCompTime(), + frame_filter.getRunTime(), + frame_filter.getEnabled().isChecked()] + return save + + def pluginRestoreState(self, saved_settings): + """Restores state based on the saved settings.""" + if saved_settings: + if len(saved_settings) > 1: + current_settings = saved_settings["All Other Types"] + frame_filter = self.__stuckWidget.getControls().getFilters()[0] + frame_filter.getServiceBox().setText("All Other Types") + frame_filter.getRegex().setText(current_settings[0]) + frame_filter.getEnabled().setChecked(current_settings[5]) + frame_filter.getLLUFilter().setValue(current_settings[2]) + frame_filter.getPercentFilter().setValue(current_settings[1]) + frame_filter.getCompletionFilter().setValue(current_settings[3]) + frame_filter.getRunFilter().setValue(current_settings[4]) + top_filter = self.__stuckWidget.getControls().getFilters()[0] + else: + settings_text = "All (Click + to Add Specific Filter)" + current_settings = saved_settings[settings_text] + frame_filter = self.__stuckWidget.getControls().getFilters()[0] + frame_filter.getServiceBox().setText(settings_text) + frame_filter.getRegex().setText(current_settings[0]) + frame_filter.getEnabled().setChecked(current_settings[5]) + frame_filter.getLLUFilter().setValue(current_settings[2]) + frame_filter.getPercentFilter().setValue(current_settings[1]) + frame_filter.getCompletionFilter().setValue(current_settings[3]) + frame_filter.getRunFilter().setValue(current_settings[4]) + return + + for frame_filter in saved_settings.keys(): + if (not frame_filter == "All Other Types" and + not frame_filter == "All (Click + to Add Specific Filter)"): + current_settings = saved_settings[frame_filter] + new_filter = top_filter.addFilter() + new_filter.getServiceBox().setText(frame_filter) + new_filter.getRegex().setText(current_settings[0]) + new_filter.getEnabled().setChecked(current_settings[5]) + new_filter.getLLUFilter().setValue(current_settings[2]) + new_filter.getPercentFilter().setValue(current_settings[1]) + new_filter.getCompletionFilter().setValue(current_settings[3]) + new_filter.getRunFilter().setValue(current_settings[4]) + return + + +class ShowCombo(QtWidgets.QComboBox): + """Combobox with show names""" + + def __init__(self, selected="pipe", parent=None): + QtWidgets.QComboBox.__init__(self, parent) + self.refresh() + self.setCurrentIndex(self.findText(selected)) + + def refresh(self): + """Refreshes the show list.""" + self.clear() + shows = opencue.api.getActiveShows() + shows.sort() + + for show in shows: + self.addItem(show.data.name, show) + + def getShow(self): + """Returns show name.""" + return str(self.setCurrentText()) + + +class StuckFrameControls(QtWidgets.QWidget): + """ + A widget that contains all search options for stuck frames + """ + + def __init__(self, parent=None): + QtWidgets.QWidget.__init__(self, parent) + + self.__current_show = opencue.api.findShow(os.getenv("SHOW", "pipe")) + self.__show_combo = ShowCombo(self.__current_show.data.name, self) + self.__show_label = QtWidgets.QLabel("Show:", self) + self.__show_label.setToolTip("The show you want to find stuck frames for.") + + self.__clear_btn = QtWidgets.QPushButton("Clear") + self.__clear_btn.setFocusPolicy(QtCore.Qt.NoFocus) + self.__clear_btn.setMaximumWidth(150) + self.__clear_btn.setMinimumWidth(150) + + self.__search_btn = QtWidgets.QPushButton("Refresh", self) + self.__search_btn.setMinimumWidth(150) + self.__search_btn.setMaximumWidth(150) + self.__search_btn.setFocusPolicy(QtCore.Qt.NoFocus) + + self.__auto_refresh_btn = QtWidgets.QCheckBox("Auto-refresh", self) + self.__auto_refresh_btn.setToolTip("""Automatically get a new set of + frames approximately every 30 minutes.""") + self.__notification_btn = QtWidgets.QCheckBox("Notification", self) + self.__notification_btn.setEnabled(False) + self.__notification_btn.setToolTip("Get a notification when an auto-refresh has completed.") + + self.__progress = QtWidgets.QProgressBar(self) + self.__progress.setRange(0, 1000) + self.__progress.setMaximumWidth(150) + self.__progress.setMinimumWidth(150) + self.__progress.setMinimumHeight(20) + self.__progress.setMaximumHeight(20) + self.__progress.setFocusPolicy(QtCore.Qt.NoFocus) + + self.__progressLabel = QtWidgets.QLabel(self) + self.__progressLabel.setText("Progress: ") + + self.__group_filters = QtWidgets.QGroupBox("Search Filters") + + controls = QtWidgets.QHBoxLayout() + controls.addSpacing(10) + controls.addWidget(self.__show_label) + controls.addWidget(self.__show_combo) + controls.addWidget(self.__search_btn) + controls.addWidget(self.__clear_btn) + controls.addWidget(self.__auto_refresh_btn) + controls.addWidget(self.__notification_btn) + controls.addStretch() + controls.addWidget(self.__progressLabel) + controls.addWidget(self.__progress) + controls.addSpacing(10) + + self.__service_label = QtWidgets.QLabel("Layer Service", self) + self.__service_label.setToolTip("Apply filters to only this service.") + + self.__percent_label = QtWidgets.QLabel("% of Run Since LLU", self) + self.__percent_label.setToolTip("Percentage of the frame's running time spent" + + " with the same last log update.") + + self.__llu_label = QtWidgets.QLabel("Min LLU", self) + self.__llu_label.setToolTip("Only show frames whose last log update is more " + + "than this many minutes ago.") + + self.__completion_label = QtWidgets.QLabel("% of Average Completion Time ", self) + self.__completion_label.setToolTip(""" + Only show frames who are running at this percentage of + the average completion time for the same layer. If there is no + average yet, all frames will qualify. + """) + + self.__run_label = QtWidgets.QLabel("Total Runtime", self) + self.__run_label.setToolTip("Only show frames running for this long") + + self.__exclude_label = QtWidgets.QLabel("Exclude Keywords", self) + self.__exclude_label.setToolTip("Keywords to exclude certain layers or jobs. " + + "Separate by commas.") + + self.__enable_label = QtWidgets.QLabel("Enable", self) + self.__enable_label.setToolTip("Uncheck to disable a filter") + + self.__remove_btn = QtWidgets.QPushButton(QtGui.QIcon(":up.png"), "") + self.__remove_btn.setToolTip("Remove Filter") + self.__remove_btn.setFocusPolicy(QtCore.Qt.NoFocus) + self.__remove_btn.setFlat(True) + + self.__labels = QtWidgets.QHBoxLayout() + self.__labels.addSpacing(30) + self.__labels.addWidget(self.__service_label) + self.__labels.addSpacing(90) + self.__labels.addWidget(self.__exclude_label) + self.__labels.addSpacing(10) + self.__labels.addWidget(self.__percent_label) + self.__labels.addWidget(self.__llu_label) + self.__labels.addSpacing(30) + self.__labels.addWidget(self.__completion_label) + self.__labels.addWidget(self.__run_label) + self.__labels.addWidget(self.__enable_label) + self.__labels.addWidget(self.__remove_btn) + self.__labels.addStretch() + + filters = StuckFrameBar(True, self) + self.__all_filters = [filters] + self.showing = True + + filters3 = QtWidgets.QVBoxLayout() + self.filters4 = QtWidgets.QVBoxLayout() + self.filters4.addLayout(self.__labels) + self.filters4.addWidget(filters) + self.filters4.setSpacing(0) + + filters3.addLayout(self.filters4) + + self.__group_filters.setLayout(filters3) + layout = QtWidgets.QVBoxLayout(self) + layout.addWidget(self.__group_filters) + layout.addLayout(controls) + + self.connect(self.__show_combo, + QtCore.SIGNAL("currentIndexChanged(QString)"), + self.showChanged) + + self.connect(self.__remove_btn, + QtCore.SIGNAL("clicked()"), + self.hideButtonRequest) + + def addFilter(self): + """Adds new filter.""" + newFilter = StuckFrameBar(self) + self.__all_filters.append(newFilter) + self.filters4.addWidget(newFilter) + + def showChanged(self, show): + """Sets current show the one provided.""" + self.__current_show = opencue.api.findShow(str(show)) + + def getFilterBar(self): + """Returns filter bar.""" + return self.filters4 + + def getAllFilters(self): + """Returns all filters.""" + return self.__all_filters + + def hideButtonRequest(self): + """If filters are showed, hides all filters except the first and sets the remove + button icon as downward facing arrow. Otherwise, shows all filters.""" + if self.showing: + self.showing = False + for frame_filter in self.__all_filters: + if not frame_filter.isFirst(): + frame_filter.hide() + self.__remove_btn.setIcon(QtGui.QIcon(":down.png")) + else: + self.openAll() + + def openAll(self): + """Shows all filters and sets the remove button icon as upward facing arrow.""" + self.showing = True + for frame_filter in self.__all_filters: + frame_filter.show() + self.__remove_btn.setIcon(QtGui.QIcon(":up.png")) + + def getRegexString(self): + """Returns regex string.""" + return str(self.__exclude_regex.text()).strip() + + def getSearchButton(self): + """Returns search button.""" + return self.__search_btn + + def getProgress(self): + """Returns progress bar.""" + return self.__progress + + def getClearButton(self): + """Returns clear button.""" + return self.__clear_btn + + def getAutoRefresh(self): + """Returns auto refresh button.""" + return self.__auto_refresh_btn + + def getNotification(self): + """Returns notification button.""" + return self.__notification_btn + + def getShow(self): + """Returns current show.""" + return self.__current_show + + def getFilters(self): + """Returns all filters""" + return self.__all_filters + + def add(self): + """Adds new filter""" + # TODO: check if this is the correct implementation + return self.__all_filters + + +class StuckFrameBar(QtWidgets.QWidget): + """Bar with filters""" + + def __init__(self, first, parent=None): + self.defaults = {'preprocess': [1, 1, 115, 10], 'nuke': [50, 5, 115, 10], + 'arnold': [50, 60, 115, 120]} + + QtWidgets.QWidget.__init__(self, parent) + + self.__percent_spin = QtWidgets.QSpinBox(self) + self.__percent_spin.setRange(1, 100) + self.__percent_spin.setValue(50) + self.__percent_spin.setMaximumWidth(100) + self.__percent_spin.setMinimumWidth(100) + self.__percent_spin.setSuffix("%") + self.__percent_spin.setAlignment(QtCore.Qt.AlignRight) + + self.__run_sping = QtWidgets.QSpinBox(self) + self.__run_sping.setRange(1, 50000) + self.__run_sping.setValue(60) + self.__run_sping.setSuffix("min") + self.__run_sping.setAlignment(QtCore.Qt.AlignRight) + + self.__llu_spin = QtWidgets.QSpinBox(self) + self.__llu_spin.setRange(1, 50000) + self.__llu_spin.setValue(30) + self.__llu_spin.setSuffix(" min") + self.__llu_spin.setAlignment(QtCore.Qt.AlignRight) + + self.__completion_spin = QtWidgets.QSpinBox(self) + self.__completion_spin.setRange(1, 50000) + self.__completion_spin.setValue(115) + self.__completion_spin.setMaximumWidth(175) + self.__completion_spin.setMinimumWidth(175) + self.__completion_spin.setSuffix("%") + self.__completion_spin.setAlignment(QtCore.Qt.AlignRight) + + self.__exclude_regex = QtWidgets.QLineEdit(self) + self.__exclude_regex.setMaximumWidth(150) + self.__exclude_regex.setMinimumWidth(150) + + self.__service_type = ServiceBox(self) + self.__service_type.setMaximumWidth(200) + self.__service_type.setMinimumWidth(200) + self.__service_type.setTextMargins(5, 0, 0, 0) + + self.__enable = QtWidgets.QCheckBox(self) + self.__enable.setChecked(True) + + self.__filters = QtWidgets.QHBoxLayout(self) + self.__filters.addWidget(self.__service_type) + self.__filters.addWidget(self.__exclude_regex) + self.__filters.addWidget(self.__percent_spin) + self.__filters.addSpacing(30) + self.__filters.addWidget(self.__llu_spin) + self.__filters.addWidget(self.__completion_spin) + self.__filters.addSpacing(35) + self.__filters.addWidget(self.__run_sping) + self.__filters.addSpacing(25) + self.__filters.addWidget(self.__enable) + + if not first: + self.__remove_btn = QtWidgets.QPushButton(QtGui.QIcon(":kill.png"), "") + self.__remove_btn.setToolTip("Remove Filter") + self.__remove_btn.setFocusPolicy(QtCore.Qt.NoFocus) + self.__remove_btn.setFlat(True) + + self.connect(self.__remove_btn, + QtCore.SIGNAL("clicked()"), + self.removeFilter) + self.__filters.addWidget(self.__remove_btn) + self.__isFirst = False + else: + self.__service_type.setText("All (Click + to Add Specific Filter)") + self.__service_type.setReadOnly(True) + self.__add_btn = QtWidgets.QPushButton(QtGui.QIcon('%s/add.png' % + cuegui.Constants.RESOURCE_PATH), "") + self.__add_btn.setToolTip("Add Filter") + self.__add_btn.setFocusPolicy(QtCore.Qt.NoFocus) + self.__add_btn.setFlat(True) + + self.connect(self.__add_btn, + QtCore.SIGNAL("clicked()"), + self.addFilter) + self.__filters.addWidget(self.__add_btn) + self.__isFirst = True + + self.__filters.addStretch() + + def getServiceBox(self): + """Returns service box.""" + return self.__service_type + + def getRegex(self): + """Returns regex.""" + return self.__exclude_regex + + def getServiceBoxText(self): + """Returns service box text.""" + return str(self.__service_type.text()).strip() + + def getRegexText(self): + """Returns regex text.""" + return str(self.__exclude_regex.text()).strip() + + def getEnabled(self): + """Returns enable checkbox.""" + return self.__enable + + def removeFilter(self): + """Removes filter.""" + self.parent().parent().getFilterBar().removeWidget(self) + self.parent().parent().getAllFilters().remove(self) + if len(self.parent().parent().getAllFilters()) == 1: + self.parent().parent().getAllFilters()[0]\ + .getServiceBox().setText("All (Click + to Add Specific Filter)") + self.hide() + + def addFilter(self): + """Adds new filter.""" + newFilter = StuckFrameBar(False, self.parent()) + self.parent().parent().getFilterBar().addWidget(newFilter) + self.parent().parent().getAllFilters().append(newFilter) + self.parent().parent().getAllFilters()[0].getServiceBox().setText("All Other Types") + self.parent().parent().openAll() + self.parent().parent().parent().addConnections(newFilter) + return newFilter + + def isFirst(self): + """Returns true if first.""" + return self.__isFirst + + def getFilters(self): + """Returns filters.""" + return self.__filters + + def getTime(self): + """Returns time value as int""" + return int(self.__percent_spin.value()) + + def getMinLLu(self): + """Returns min LLU.""" + return int(self.__llu_spin.value()) + + def getAvgCompTime(self): + """Returns average completion time as int.""" + return int(self.__completion_spin.value()) + + def getMVTime(self): + """Returns MV time as int.""" + return int(self.__mkvid_spin.value()) + + def getPrepTime(self): + """Returns preparation time as int.""" + return int(self.__prep_spin.value()) + + def getLLUFilter(self): + """Returns LLU filter.""" + return self.__llu_spin + + def getPercentFilter(self): + """Returns percent filter.""" + return self.__percent_spin + + def getCompletionFilter(self): + """Return completion filter.""" + return self.__completion_spin + + def getRunFilter(self): + """Returns run filter.""" + return self.__run_sping + + def getRunTime(self): + """Returns run time as int.""" + return int(self.__run_sping.value()) + + def enable(self): + """Enables filters.""" + self.__percent_spin.setEnabled(not self.__percent_spin.isEnabled()) + self.__run_sping.setEnabled(not self.__run_sping.isEnabled()) + self.__llu_spin.setEnabled(not self.__llu_spin.isEnabled()) + self.__completion_spin.setEnabled(not self.__completion_spin.isEnabled()) + self.__exclude_regex.setEnabled(not self.__exclude_regex.isEnabled()) + self.__service_type.setEnabled(not self.__service_type.isEnabled()) + + def checkForDefaults(self): + """If service is in defaults, the filter values will be set to the service.""" + service = str(self.__service_type.text()).strip() + if service in self.defaults: + self.__percent_spin.setValue(self.defaults[service][0]) + self.__llu_spin.setValue(self.defaults[service][1]) + self.__completion_spin.setValue(self.defaults[service][2]) + self.__run_sping.setValue(self.defaults[service][3]) + + +class ServiceBox(QtWidgets.QLineEdit): + """ + A text box that auto-completes job names. + """ + + def __init__(self, parent=None): + QtWidgets.QLineEdit.__init__(self, parent) + self.__c = None + self.refresh() + + def refresh(self): + """Refreshes the show list.""" + slist = opencue.api.getDefaultServices() + slist.sort() + self.__c = QtWidgets.QCompleter(slist, self) + self.__c.setCaseSensitivity(QtCore.Qt.CaseInsensitive) + self.setCompleter(self.__c) + + +class StuckFrameWidget(QtWidgets.QWidget): + """ + Displays controls for finding stuck frames and a tree of the findings. + """ + + def __init__(self, parent): + QtWidgets.QWidget.__init__(self, parent) + + self.controls = StuckFrameControls(self) + self.tree = StuckFrameMonitorTree(self) + + layout = QtWidgets.QVBoxLayout(self) + layout.addWidget(self.controls) + layout.addWidget(self.tree) + + if self.tree.enableRefresh: + self.controls.getAutoRefresh().setCheckState(QtCore.Qt.Checked) + + self.connect(self.controls.getAutoRefresh(), + QtCore.SIGNAL('stateChanged(int)'), + self.__refreshToggleCheckBoxHandle) + + self.connect(self.controls.getNotification(), + QtCore.SIGNAL('stateChanged(int)'), + self.__refreshNotificationCheckBoxHandle) + + self.connect(self.tree, + QtCore.SIGNAL("updated()"), + self._refreshButtonDisableHandle) + + self.connect(self.controls.getSearchButton(), + QtCore.SIGNAL("clicked()"), + self.updateRequest) + filters = self.controls.getFilters() + for frame_filter in filters: + self.addConnections(frame_filter) + + self.connect(self.controls.getClearButton(), + QtCore.SIGNAL("clicked()"), + self.clearButtonRequest) + + def addConnections(self, frame_filter): + """Connects to the widget based on the filter provided""" + self.connect(frame_filter.getLLUFilter(), + QtCore.SIGNAL("valueChanged(int)"), + self.updateFilters) + + self.connect(frame_filter.getPercentFilter(), + QtCore.SIGNAL("valueChanged(int)"), + self.updateFilters) + + self.connect(frame_filter.getCompletionFilter(), + QtCore.SIGNAL("valueChanged(int)"), + self.updateFilters) + + self.connect(frame_filter.getRunFilter(), + QtCore.SIGNAL("valueChanged(int)"), + self.updateFilters) + + self.connect(frame_filter.getRegex(), + QtCore.SIGNAL("textChanged(QString)"), + self.updateFilters) + + self.connect(frame_filter.getServiceBox(), + QtCore.SIGNAL("textChanged(QString)"), + self.updateFilters) + + self.connect(frame_filter.getEnabled(), + QtCore.SIGNAL("stateChanged(int)"), + self.updateFilters) + + self.connect(frame_filter.getEnabled(), + QtCore.SIGNAL("stateChanged(int)"), + frame_filter.enable) + + self.connect(frame_filter.getServiceBox(), + QtCore.SIGNAL("textChanged(QString)"), + frame_filter.checkForDefaults) + + def __refreshToggleCheckBoxHandle(self, state): + self.tree.enableRefresh = bool(state) + self.controls.getNotification().setEnabled(bool(state)) + + def __refreshNotificationCheckBoxHandle(self, state): + self.tree.enableNotification = bool(state) + + def _refreshButtonEnableHandle(self): + self.controls.getSearchButton().setEnabled(True) + + def _refreshButtonDisableHandle(self): + self.controls.getSearchButton().setEnabled(False) + QtCore.QTimer.singleShot(5000, self._refreshButtonEnableHandle) + + def updateRequest(self): + """Updates filter list with only enabled filters and then updates the tree widget.""" + allFilters = {} + filters = self.controls.getFilters() + for frame_filter in filters: + if frame_filter.getEnabled().isChecked(): + allFilters[frame_filter.getServiceBoxText()] = [frame_filter.getRegexText(), + frame_filter.getTime(), + frame_filter.getMinLLu(), + frame_filter.getAvgCompTime(), + frame_filter.getRunTime()] + + self.tree.updateFilters(allFilters, self.controls.getShow()) + self.tree.setCompleteRefresh(True) + self.tree.updateRequest() + + def updateFilters(self): + """Updates filter list with only enabled filters.""" + allFilters = {} + filters = self.controls.getFilters() + for frame_filter in filters: + if frame_filter.getEnabled().isChecked(): + allFilters[frame_filter.getServiceBoxText()] = [ + frame_filter.getRegexText(), + frame_filter.getTime(), + frame_filter.getMinLLu(), + frame_filter.getAvgCompTime(), + frame_filter.getRunTime() + ] + self.tree.updateFilters(allFilters, self.controls.getShow()) + + def clearButtonRequest(self): + """Clears tree widget.""" + self.tree.clearItems() + self.tree.enableRefresh = False + self.controls.getAutoRefresh().setCheckState(QtCore.Qt.Unchecked) + + def getControls(self): + """Returns controls.""" + return self.controls + + +class StuckFrameMonitorTree(cuegui.AbstractTreeWidget.AbstractTreeWidget): + """Tree widget with stuck frames""" + + _updateProgress = QtCore.Signal(int) + _updateProgressMax = QtCore.Signal(int) + _itemSingleClickedComment = QtCore.Signal(QtWidgets.QTreeWidgetItem, int) + + def __init__(self, parent): + self.parent = parent + self.startColumnsForType(cuegui.Constants.TYPE_FRAME) + self.addColumn("Name", 300, id=1, + data=lambda item: (item.data.name or ""), + tip="The job name.") + self.addColumn("_Comment", 20, id=2, + tip="A comment icon will appear if a job has a comment. You\n" + "may click on it to view the comments.") + self.addColumn("Frame", 40, id=3, + data=lambda item: (item.number or ""), + tip="Frame number") + self.addColumn("Host", 120, id=4, + data=lambda item: (item.lastResource or ""), + tip="Host the frame is currently running on") + self.addColumn("LLU", 60, id=5, + data=lambda item: (self.numFormat(item.lastLogUpdate, "t") or ""), + tip="Last Log Update") + self.addColumn("Runtime", 60, id=6, + data=lambda item: (self.numFormat(item.timeRunning, "t") or ""), + tip="Length the Frame has been running") + self.addColumn("% Stuck", 50, id=7, + data=lambda item: (self.numFormat(item.stuckness, "f") or ""), + tip="Percent of frame's total runtime that the log has not been updated") + self.addColumn("Average", 60, id=8, + data=lambda item: (self.numFormat(item.averageFrameTime, "t") or ""), + tip="Average time for a frame of this type to complete") + self.addColumn("Last Line", 250, id=9, + data=lambda item: (cuegui.Utils.getLastLine(item.log_path) or ""), + tip="The last line of a running frame's log file.") + + self.startColumnsForType(cuegui.Constants.TYPE_GROUP) + self.addColumn("", 0, id=1, + data=lambda group: (group.data.name), sort=lambda group: (group.data.name)) + self.addColumn("", 0, id=2) + self.addColumn("", 0, id=3) + self.addColumn("", 0, id=4) + self.addColumn("", 0, id=5) + self.addColumn("", 0, id=6) + self.addColumn("", 0, id=7) + self.addColumn("", 0, id=8) + self.addColumn("", 0, id=9) + + cuegui.AbstractTreeWidget.AbstractTreeWidget.__init__(self, parent) + self.procSearch = opencue.search.ProcSearch() + + # Used to build right click context menus + self.__menuActions = cuegui.MenuActions.MenuActions(self, self.updateSoon, + self.selectedObjects, self.getJob) + + self.setDropIndicatorShown(True) + self.setDragEnabled(True) + + self.layer_cache = {} + self.jobs_created = {} + self.groups_created = {} + self.currentHosts = [] + + # Bring Up a comment if it exists + self._itemSingleClickedComment.connect(self.__itemSingleClickedComment) + # Set progress bar current value + self._updateProgress.connect(self.updateProgress) + + # Set total number of procs for the progress bar max + self._updateProgressMax.connect(self.updateProgressMax) + + # Don't use the standard space bar to refres + self.disconnect(QtGui.qApp, + QtCore.SIGNAL('request_update()'), + self.updateRequest) + + self.run_log = LogFinal() + self.frames = {} + self.ticksSinceLogFlush = 0 + self.startTicksUpdate(2000) + + # Don't start refreshing until the user sets a filter or hits refresh + self.ticksWithoutUpdate = -1 + self.enableRefresh = False + self.completeRefresh = False + self.enableNotification = False + + self.runtime_filter = None + self.min_llu_filter = None + self.time_filter = None + self.avg_comp_filter = None + self.excludes = None + self.groups = None + self.showData = None + self.filters = None + + def logIt(self): + """Logs cache to a file.""" + if hasattr(QtGui.qApp, "threadpool"): + print("Stuck Frame Log cache is being written to file.") + QtGui.qApp.threadpool.queue(self.run_log.finalize, self.logResult, + "Writing out log", self.frames, self.show) + else: + logger.warning("threadpool not found, doing work in gui thread") + + # pylint: disable=missing-function-docstring,unused-argument + def logResult(self, work, rpcObjects): + self.frames = {} + + # pylint: disable=redefined-builtin,inconsistent-return-statements,no-self-use + def numFormat(self, num, type): + """Returns string formatting based on the number""" + if num == "" or num < .001 or num is None: + return "" + if type == "t": + return cuegui.Utils.secondsToHHMMSS(int(num)) + if type == "f": + return "%.2f" % float(num) + + def setCompleteRefresh(self, value): + """Sets complete refresh based on given value.""" + self.completeRefresh = value + + def tick(self): + """Handles update on single tick.""" + if self.ticksSinceLogFlush >= 400 and len(self.frames) > 0: + self.ticksSinceLogFlush = 0 + self.logIt() + + if self.completeRefresh: + self.ticksWithoutUpdate = 0 + self.completeRefresh = False + self._update() + return + + if (self.ticksWithoutUpdate % 40 == 0 and + self.ticksWithoutUpdate != self.updateInterval and not self.window().isMinimized()): + self.ticksWithoutUpdate += 1 + if len(self.currentHosts) > 0: + self.confirm(1) + return + + if (self.ticksWithoutUpdate >= self.updateInterval and + self.enableRefresh and not self.window().isMinimized()): + self.ticksWithoutUpdate = 0 + self._update() + if self.enableNotification: + message = QtWidgets.QMessageBox(self) + message.setText("Stuck Frames have refreshed!.") + message.exec_() + return + + self.ticksSinceLogFlush += 1 + if not self.window().isMinimized(): + self.ticksWithoutUpdate += 1 + + def __itemSingleClickedComment(self, item, col): + """If the comment column is clicked on, and there is a comment on the + host, this pops up the comments dialog + @type item: QTreeWidgetItem + @param item: The item clicked on + @type col: int + @param col: The column clicked on""" + commentItem = item.rpcObject + if (col == COMMENT_COLUMN and + cuegui.Utils.isJob(commentItem) and commentItem.data.hasComment): + self.__menuActions.jobs().viewComments([commentItem]) + self.update() + + def updateProgressMax(self, newMax): + """Send an update to the progress bar of the new maximum value""" + self.parent.getControls().getProgress().setMaximum(newMax) + + def updateProgress(self, currentValue): + """Send an update of the current value for the progress bar""" + self.parent.getControls().getProgress().setValue(currentValue) + + def updateSoon(self): + """Returns immediately. Causes an update to happen + Constants.AFTER_ACTION_UPDATE_DELAY after calling this function.""" + QtCore.QTimer.singleShot(cuegui.Constants.AFTER_ACTION_UPDATE_DELAY, + self.updateRequest) + + def getJob(self): + """Returns the current job + @return: The current job + @rtype: job""" + return cuegui.Utils.findJob(self.selectedObjects()[0].data.name) + + def clearItems(self): + """Clears all items""" + self.clearSelection() + self.removeAllItems() + self.currentHosts = [] + + def updateRequest(self): + """Updates the items in the TreeWidget if sufficient time has passed + since last updated""" + self.ticksWithoutUpdate = 999 + self.completeRefresh = True + + # pylint: disable=no-self-use + def get_frame_run_time(self, item): + """Returns frame run time.""" + if cuegui.Utils.isProc(item): + start_time = item.data.dispatch_time + elif cuegui.Utils.isFrame(item): + start_time = item.data.start_time + else: + return "" + current_time = time.time() + run_time = current_time - start_time + return run_time + + # pylint: disable=no-self-use + def get_llu_time(self, item): + """Returns LLU time.""" + if cuegui.Utils.isProc(item): + log_file = item.data.log_path + elif cuegui.Utils.isFrame(item): + log_file = item.log_path + else: + return "" + # pylint: disable=broad-except + try: + stat_info = os.path.getmtime(log_file) + except Exception: + return "None" + current_time = time.time() + llu_time = current_time - stat_info + + return llu_time + + def find_layer(self, proc): + """Return layer based on proc.""" + jobName = proc.data.job_name + layerName = proc.data.frame_name.split("-")[1] + key = "%s/%s" % (jobName, layerName) + + if not self.layer_cache.get(key, None): + # pylint: disable=broad-except + try: + self.layer_cache[key] = proc.getLayer() + except Exception: + return "None" + + return self.layer_cache[key] + + def confirm(self, update): + """Confirm frame filter.""" + currentHostsNew = [] + nextIndex = 2 + for index in range(len(self.currentHosts)): + if index == nextIndex: + frame = self.currentHosts[index] + nextIndex = nextIndex + 3 + + if frame.service in self.filters.keys(): + self.runtime_filter = self.filters[frame.service][4] + self.min_llu_filter = self.filters[frame.service][2] + self.time_filter = self.filters[frame.service][1] + self.avg_comp_filter = self.filters[frame.service][3] + self.excludes = [x.strip() for x in self.filters[frame.service][0].split(',') + if x != ""] + else: + if "All (Click + to Add Specific Filter)" in self.filters.keys(): + key = "All (Click + to Add Specific Filter)" + elif "All Other Types" in self.filters.keys(): + key = "All Other Types" + else: + continue + self.runtime_filter = self.filters[key][4] + self.min_llu_filter = self.filters[key][2] + self.time_filter = self.filters[key][1] + self.avg_comp_filter = self.filters[key][3] + self.excludes = [x.strip() for x in self.filters[key][0].split(',') if x != ""] + + # layerName = frame.data.layer_name + frameRunTime = self.get_frame_run_time(frame) + # jobName = frame.data.name + lluTime = self.get_llu_time(frame) + avgFrameTime = frame.averageFrameTime + percentStuck = lluTime / frameRunTime + + frame.stuckness = percentStuck + frame.lastLogUpdate = lluTime + frame.timeRunning = frameRunTime + + if ((lluTime > (self.min_llu_filter * 60)) and + (percentStuck * 100 > self.time_filter) and + (frameRunTime > (avgFrameTime * self.avg_comp_filter / 100) and + percentStuck < 1.1 and frameRunTime > 500)): + currentHostsNew.append(self.currentHosts[index - 2]) + currentHostsNew.append(self.currentHosts[index - 1]) + currentHostsNew.append(frame) + + self.currentHosts[:] = [] + self.currentHosts = currentHostsNew + + if update == 1: + self._processUpdate(None, self.currentHosts) + + def _getUpdate(self): + """Returns the proper data from the cuebot""" + # pylint: disable=broad-except,too-many-nested-blocks + try: + treeItems = [] + procs = [] + self.groups = [] + self.procSearch.hosts = [] + self.procSearch.shows = [self.show] + procs = opencue.api.getProcs() + + current_prog = 0 + self.emit(QtCore.SIGNAL("updatedProgressMax"), (len(procs))) + self._updateProgressMax.emit(len(procs)) + for proc in procs: + if proc.data.services[0] in self.filters.keys(): + self.runtime_filter = self.filters[proc.data.services[0]][4] + self.min_llu_filter = self.filters[proc.data.services[0]][2] + self.time_filter = self.filters[proc.data.services[0]][1] + self.avg_comp_filter = self.filters[proc.data.services[0]][3] + self.excludes = [x.strip() + for x in self.filters[proc.data.services[0]][0].split(',') + if x != ""] + else: + if "All (Click + to Add Specific Filter)" in self.filters.keys(): + key = "All (Click + to Add Specific Filter)" + elif "All Other Types" in self.filters.keys(): + key = "All Other Types" + else: + continue + self.runtime_filter = self.filters[key][4] + self.min_llu_filter = self.filters[key][2] + self.time_filter = self.filters[key][1] + self.avg_comp_filter = self.filters[key][3] + self.excludes = [x.strip() for x in self.filters[key][0].split(',') if x != ""] + + jobName = proc.data.job_name + (frameNumber, layerName) = proc.data.frame_name.split("-") + + frameRunTime = self.get_frame_run_time(proc) + # frameResource = proc.data.name + + if frameRunTime >= self.runtime_filter * 60: + # Get average completion time of the layer + layer = self.find_layer(proc) + # Skip processing if the layer obj doesn't exist. i.e frame finished + if layer == "None": + continue + avgFrameTime = layer.avgFrameTimeSeconds() + + if frameRunTime > (avgFrameTime * self.avg_comp_filter / 100): + # log_path = proc.data.log_path # Get the log file path for last line + lluTime = self.get_llu_time(proc) + if lluTime == "None": + # Skip processing if there was any error with reading + # the log file(path did not exist,permissions) + continue + if lluTime > self.min_llu_filter * 60: + percentStuck = 0 + if frameRunTime > 0: + percentStuck = lluTime / frameRunTime + + if percentStuck * 100 > self.time_filter and percentStuck < 1.1: + please_exclude = False + for exclude in self.excludes: + if (layerName.__contains__(exclude) or + jobName.__contains__(exclude)): + please_exclude = True + continue + + if please_exclude: + continue + + # Job may have finished/killed put in a try + # Injecting into rpcObjects extra data not available via client API + # to support cue3 iceObject backwards capability + try: + frame = opencue.api.findFrame(jobName, + layerName, int(frameNumber)) + frame.data.layer_name = layerName + frame.__dict__['job_name'] = jobName + frame.__dict__['log_path'] = proc.data.log_path + frame.__dict__['number'] = frame.data.number + frame.__dict__['lastLogUpdate'] = lluTime + frame.__dict__['averageFrameTime'] = avgFrameTime + frame.__dict__['stuckness'] = percentStuck + frame.__dict__['timeRunning'] = frameRunTime + frame.__dict__['lastResource'] = frame.data.last_resource + frame.__dict__['service'] = proc.data.services[0] + + job = opencue.api.findJob(jobName) + job.__dict__['log_path'] = job.data.log_dir + job.__dict__['lastLogUpdate'] = "" + job.__dict__['averageFrameTime'] = "" + job.__dict__['number'] = "" + job.__dict__['stuckness'] = "" + job.__dict__['timeRunning'] = "" + job.__dict__['lastResource'] = "" + job.__dict__['hostUsage'] = "" + job.__dict__['service'] = proc.data.services[0] + if self.show == job.data.show: + group = opencue.api.findGroup(self.show, job.data.group) + + treeItems.append(group) + treeItems.append(job) + treeItems.append(frame) + except Exception: + # Can safely ignore if a Job has already completed + pass + + current_prog = current_prog + 1 + self._updateProgress.emit(current_prog) + + self._updateProgress.emit(len(procs)) + self.currentHosts[:] = [] + self.currentHosts = treeItems + + self.confirm(0) + + return self.currentHosts + except Exception as e: + print(cuegui.Utils.exceptionOutput(e)) + map(logger.warning, cuegui.Utils.exceptionOutput(e)) + return [] + + def _createItem(self, object, parent=None): + """Creates and returns the proper item + @type object: Host + @param object: The object for this item + @type parent: QTreeWidgetItem + @param parent: Optional parent for this item + @rtype: QTreeWidgetItem + @return: The created item""" + + if cuegui.Utils.isGroup(object): + groupWidget = GroupWidgetItem(object, self) + self.groups_created[object.data.name] = groupWidget # Store parents created + groupWidget.setExpanded(True) + return groupWidget + if cuegui.Utils.isJob(object): + jobWidget = HostWidgetItem(object, self.groups_created[object.data.group]) + self.jobs_created[object.data.name] = jobWidget # Store parents created + jobWidget.setExpanded(True) + return jobWidget + if cuegui.Utils.isFrame(object): + frameWidget = HostWidgetItem(object, + # Find the Job to serve as its parent + self.jobs_created[object.job_name]) + return frameWidget + + def contextMenuEvent(self, e): + """When right clicking on an item, this raises a context menu""" + + menu = QtWidgets.QMenu() + + # Since we want different menu options based on what is chosen, we need to figure this out + isJob = False + isFrame = False + sameJob = True + jobName = None + # isGroup = True + for item in self.selectedObjects(): + if cuegui.Utils.isJob(item): + isJob = True + elif cuegui.Utils.isFrame(item): + isFrame = True + # elif cuegui.Utils.isGroup(item): + # isGroup = True + if not jobName: + jobName = item.data.name + else: + if item.data.name != jobName: + sameJob = False + + if isJob and not isFrame and sameJob: + self.__menuActions.jobs().addAction(menu, "viewComments") + self.__menuActions.jobs().addAction(menu, "emailArtist") + menu.addAction( + cuegui.Action.create(self, "Email and Comment", "Email and Comment", + self.emailComment, "mail")) + menu.addSeparator() + menu.addAction(cuegui.Action.create(self, "Job Not Stuck", "Job Not Stuck", + self.RemoveJob, "warning")) + menu.addAction( + cuegui.Action.create(self, "Add Job to Excludes", "Add Job to Excludes", + self.AddJobToExcludes, "eject")) + menu.addAction(cuegui.Action.create(self, "Exclude and Remove Job", + "Exclude and Remove Job", + self.AddJobToExcludesandRemove, "unbookkill")) + menu.addSeparator() + menu.addAction(cuegui.Action.create(self, "Core Up", "Core Up", self.coreup, "up")) + menu.exec_(e.globalPos()) + + if isFrame and not isJob and sameJob: + count = len(self.selectedItems()) + + self.__menuActions.frames().addAction(menu, "tail") + self.__menuActions.frames().addAction(menu, "view") + + if count == 1: + if self.selectedObjects()[0].data.retry_count >= 1: + self.__menuActions.frames().addAction(menu, "viewLastLog") + + if count >= 3: + self.__menuActions.frames().addAction(menu, "xdiff3") + elif count == 2: + self.__menuActions.frames().addAction(menu, "xdiff2") + + if count == 1: + menu.addSeparator() + menu.addAction(cuegui.Action.create(self, "Top Machine", "Top Machine", + self.topMachine, "up")) + if QtGui.qApp.applicationName() == "CueCommander3": + self.__menuActions.frames().addAction(menu, "viewHost") + + menu.addSeparator() + menu.addAction(cuegui.Action.create(self, "Retry", "Retry", self.retryFrame, "retry")) + menu.addAction(cuegui.Action.create(self, "Eat", "Eat", self.eatFrame, "eat")) + menu.addAction(cuegui.Action.create(self, "Kill", "Kill", self.killFrame, "kill")) + menu.addSeparator() + if count == 1: + menu.addAction(cuegui.Action.create(self, "Log Stuck Frame", "Log Stuck Frame", + self.log, "loglast")) + elif count > 1: + menu.addAction(cuegui.Action.create(self, "Log Stuck Frames", "Log Stuck Frames", + self.log, "loglast")) + menu.addAction(cuegui.Action.create(self, "Log and Retry", "Log and Retry", + self.logRetry, "retry")) + menu.addAction(cuegui.Action.create(self, "Log and Eat", "Log and Eat", + self.logEat, "eat")) + menu.addAction(cuegui.Action.create(self, "Log and Kill", "Log and Kill", + self.logKill, "kill")) + menu.addSeparator() + menu.addAction(cuegui.Action.create(self, "Frame Not Stuck", "Frame Not Stuck", + self.remove, "warning")) + menu.addAction( + cuegui.Action.create(self, "Add Job to Excludes", "Add Job to Excludes", + self.AddJobToExcludes, "eject")) + menu.addAction(cuegui.Action.create(self, "Exclude and Remove Job", + "Exclude and Remove Job", + self.AddJobToExcludesandRemove, "unbookkill")) + menu.addSeparator() + menu.addAction(cuegui.Action.create(self, "Core Up", "Core Up", self.coreup, "up")) + + menu.exec_(e.globalPos()) + + def coreup(self): + """PST Menu Plugin entry point.""" + job = self.getJob() + win = CoreUpWindow(self, {job: job.getLayers()}) + win.show() + + def _processUpdate(self, work, rpcObjects): + """A generic function that Will: + Create new TreeWidgetItems if an item does not exist for the object. + Update existing TreeWidgetItems if an item already exists for the object. + Remove items that were not updated with rpcObjects. + @param work: + @type work: from ThreadPool + @param rpcObjects: A list of ice objects + @type rpcObjects: list """ + self._itemsLock.lockForWrite() + try: + updated = [] + for rpcObject in rpcObjects: + updated.append(cuegui.Utils.getObjectKey(rpcObject)) # rpcObject) + # If id already exists, update it + if cuegui.Utils.getObjectKey(rpcObject) in self._items: + self._items[cuegui.Utils.getObjectKey(rpcObject)].update(rpcObject) + # If id does not exist, create it + else: + self._items[cuegui.Utils.getObjectKey(rpcObject)] = self._createItem(rpcObject) + # Remove any items that were not updated + for rpcObject in list(set(self._items.keys()) - set(updated)): + self._removeItem(rpcObject) + self.redraw() + finally: + self._itemsLock.unlock() + + def topMachine(self): + signal.signal(signal.SIGALRM, self.handler) + signal.alarm(int(30)) + + job = self.selectedObjects()[0] + + command = (' xhost ' + job.lastResource.split('/')[0] + '; rsh ' + + job.lastResource.split('/')[0] + ' \"setenv DISPLAY ' + + str(socket.gethostname()).split('.', maxsplit=1)[0] + ':0; xterm -e top\" &') + os.system(command) + signal.alarm(0) + + def remove(self): + currentHostsNew = [] + nextIndex = 2 + for index in range(len(self.currentHosts)): + if index == nextIndex: + + nextIndex = nextIndex + 3 + + if self.currentHosts[index] not in self.selectedObjects(): + currentHostsNew.append(self.currentHosts[index - 2]) + currentHostsNew.append(self.currentHosts[index - 1]) + currentHostsNew.append(self.currentHosts[index]) + self.currentHosts[:] = [] + self.currentHosts = currentHostsNew + # self.currentHosts = [x for x in self.currentHosts if x not in self.selectedObjects() ] - + + self._processUpdate(None, self.currentHosts) + + def emailComment(self): + job = self.getJob() + self.__menuActions.jobs().emailArtist([job]) + job.addComment("Emailed artists", "Emailed Artist but took no further action") + + def logRetry(self): + names = [frame.name() for frame in self.selectedObjects()] + + if cuegui.Utils.questionBoxYesNo(self, "Confirm", "Retry selected frames?", names): + self.log() + for frame in self.selectedObjects(): + frame.retry() + self.remove() + + def logEat(self): + names = [frame.name() for frame in self.selectedObjects()] + + if cuegui.Utils.questionBoxYesNo(self, "Confirm", "Eat selected frames?", names): + self.log() + for frame in self.selectedObjects(): + frame.eat() + self.remove() + + def logKill(self): + names = [frame.name() for frame in self.selectedObjects()] + + if cuegui.Utils.questionBoxYesNo(self, "Confirm", "Kill selected frames?", names): + self.log() + for frame in self.selectedObjects(): + frame.kill() + self.remove() + + def retryFrame(self): + names = [frame.name() for frame in self.selectedObjects()] + + if cuegui.Utils.questionBoxYesNo(self, "Confirm", "Retry selected frames?", names): + for frame in self.selectedObjects(): + frame.retry() + self.remove() + + def eatFrame(self): + names = [frame.name() for frame in self.selectedObjects()] + + if cuegui.Utils.questionBoxYesNo(self, "Confirm", "Eat selected frames?", names): + for frame in self.selectedObjects(): + frame.eat() + self.remove() + + def killFrame(self): + names = [frame.name() for frame in self.selectedObjects()] + + if cuegui.Utils.questionBoxYesNo(self, "Confirm", "Kill selected frames?", names): + for frame in self.selectedObjects(): + frame.kill() + self.remove() + + def handler(self): + message = QtWidgets.QMessageBox(self) + message.setText("""Unable to connect to host after 30 sec. + It may need to be put into repair state. """) + message.exec_() + + def log(self): + self.ticksSinceLogFlush = 0 + currentJob = self.selectedObjects()[0].data.name + framesForJob = {} + for frame in self.selectedObjects(): + frameData = {} + frameData['layer'] = frame.job_name + frameData['host'] = frame.lastResource + frameData['llu'] = self.get_llu_time(frame) + frameData['runtime'] = self.get_frame_run_time(frame) + frameData['average'] = frame.averageFrameTime + frameData['log'] = cuegui.Utils.getLastLine(frame.log_path) + framesForJob[str(frame.data.number) + '-' + str(time.time())] = frameData + + self.frames[currentJob] = framesForJob + + def AddJobToExcludes(self): + currentJob = self.selectedObjects()[0] + currentJobName = currentJob.data.name + currentJobService = currentJob.service + filters = self.parent().getControls().getFilters() + + key = "" + for filter in filters: + if currentJobService == filter.getServiceBoxText(): + key = currentJobService + filterChange = filter + break + if key == "": + for filter in filters: + if filter.getServiceBoxText() == "All (Click + to Add Specific Filter)": + key = "All (Click + to Add Specific Filter)" + filterChange = filter + break + if key == "": + for filter in filters: + if filter.getServiceBoxText() == "All Other Types": + key = "All Other Types" + filterChange = filter + break + + if len(filterChange.getRegexText()) > 0: + filterChange.getRegex().setText(filterChange.getRegexText() + ", " + currentJobName) + else: + filterChange.getRegex().setText(currentJobName) + + return currentJobName + + def AddJobToExcludesandRemove(self): + self.AddJobToExcludes() + self.RemoveJob() + + def RemoveJob(self): + jobName = self.selectedObjects()[0].data.name + currentHostsNew = [] + nextIndex = 2 + for index in range(len(self.currentHosts)): + if index == nextIndex: + + nextIndex = nextIndex + 3 + + if self.currentHosts[index].data.name != jobName: + currentHostsNew.append(self.currentHosts[index - 2]) + currentHostsNew.append(self.currentHosts[index - 1]) + currentHostsNew.append(self.currentHosts[index]) + self.currentHosts[:] = [] + self.currentHosts = currentHostsNew + + self._processUpdate(None, self.currentHosts) + + def startDrag(self, dropActions): + cuegui.Utils.startDrag(self, dropActions, self.selectedObjects()) + + def dragEnterEvent(self, event): + cuegui.Utils.dragEnterEvent(event, "application/x-host-ids") + + def dragMoveEvent(self, event): + cuegui.Utils.dragMoveEvent(event, "application/x-host-ids") + + def updateFilters(self, filters, show): + self.showData = show + self.show = show.data.name + self.filters = filters + + def _removeItem(self, item): + """Removes an item from the TreeWidget without locking + @type item: AbstractTreeWidgetItem or String + @param item: A tree widget item or the string with the id of the item""" + + if item in self._items: + item = self._items[item] + elif not isinstance(item, cuegui.AbstractWidgetItem.AbstractWidgetItem): + # if the parent was already deleted, then this one was too + return + + # If it has children, they must be deleted first + if item.childCount() > 0: + for child in item.takeChildren(): + self._removeItem(child) + + if item.isSelected(): + item.setSelected(False) + + if item.parent(): + self.invisibleRootItem().removeChild(item) + self.takeTopLevelItem(self.indexOfTopLevelItem(item)) + objectClass = item.rpcObject.__class__.__name__ + objectId = item.rpcObject.id() + try: + del self._items['{}.{}'.format(objectClass, objectId)] + except KeyError: + # Dependent jobs are not stored in as keys the main self._items + # dictionary, trying to remove dependent jobs from self._items + # raises a KeyError, which we can safely ignore + pass + + def finalize(self, frames): + + dict = frames + yaml_path = "/shots/" + self.show + "/home/etc/stuck_frames_db.yaml" + + if not os.path.exists(yaml_path): + yaml_ob = open(yaml_path, 'w') + yaml.dump(dict, yaml_ob) + yaml_ob.close() + + else: + yaml_ob = open(yaml_path, 'r') + old_dict = yaml.load(yaml_ob) + yaml_ob.close() + + yaml_ob = open(yaml_path, 'w') + + for key in dict: # updates old dict + old_dict[key] = dict[key] + + yaml.dump(old_dict, yaml_ob) + yaml_ob.close() + + +class CommentWidget(QtWidgets.QWidget): + """Represents a comment.""" + def __init__(self, subject, message, parent=None): + QtWidgets.QWidget.__init__(self, parent) + self.__textSubject = subject + self.__textMessage = message + + # def getSubject(self): + # return str(self.__textSubject.text()) + # + # def getMessage(self): + # return str(self.__textMessage.toPlainText()) + + +class GroupWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): + """Represents a group entry in the MonitorCue widget.""" + __initialized = False + + def __init__(self, rpcObject, parent): + # pylint: disable=protected-access + if not self.__initialized: + self.__class__.__initialized = True + self.__class__.__icon = QtGui.QIcon(":group.png") + self.__class__.__foregroundColor = cuegui.Style.ColorTheme.COLOR_GROUP_FOREGROUND + self.__class__.__backgroundColor = cuegui.Style.ColorTheme.COLOR_GROUP_BACKGROUND + self.__class__.__type = cuegui.Constants.TYPE_GROUP + + cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( + self, cuegui.Constants.TYPE_GROUP, rpcObject, parent) + + def data(self, col, role): + """Returns the proper display data for the given column and role + @type col: int + @param col: The column being displayed + @type role: QtCore.Qt.ItemDataRole + @param role: The role being displayed + @rtype: QtCore.QVariant + @return: The desired data wrapped in a QVariant""" + if role == QtCore.Qt.DisplayRole: + return self.column_info[col][cuegui.Constants.COLUMN_INFO_DISPLAY](self.rpcObject) + + if role == QtCore.Qt.ForegroundRole: + return self.__foregroundColor + + if role == QtCore.Qt.BackgroundRole: + return self.__backgroundColor + + if role == QtCore.Qt.DecorationRole and col == 0: + return self.__icon + + if role == QtCore.Qt.UserRole: + return self.__type + + return cuegui.Constants.QVARIANT_NULL + + +class LogFinal(): + """Utility class for logging to yaml.""" + # pylint: disable=no-self-use + def finalize(self, frames, show): + """Saves logs to yaml. If file not created, will create one.""" + frames_dict = frames + + yaml_path = "/shots/" + show + "/home/etc/stuck_frames_db.yaml" + if not os.path.exists(yaml_path): + yaml_ob = open(yaml_path, 'w') + yaml.dump(frames_dict, yaml_ob) + yaml_ob.close() + else: + yaml_ob = open(yaml_path, 'r') + old_dict = yaml.load(yaml_ob) + yaml_ob.close() + + yaml_ob = open(yaml_path, 'w') + + for key in frames_dict: # updates old dict + old_dict[key] = frames_dict[key] + + yaml.dump(old_dict, yaml_ob) + yaml_ob.close() + + +class HostWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): + """Represents a host widget.""" + __initialized = False + # pylint: disable=redefined-builtin,protected-access + def __init__(self, object, parent): + if not self.__initialized: + self.__class__.__initialized = True + self.__class__.__commentIcon = QtGui.QIcon(":comment.png") + self.__class__.__backgroundColor = QtGui.qApp.palette().color(QtGui.QPalette.Base) + self.__class__.__foregroundColor = cuegui.Style.ColorTheme.COLOR_JOB_FOREGROUND + + cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__(self, cuegui.Constants.TYPE_FRAME, + object, parent) + + def data(self, col, role): + if role == QtCore.Qt.DisplayRole: + if col not in self._cache: + self._cache[col] = self.column_info[col][cuegui.Constants.COLUMN_INFO_DISPLAY]( + self.rpcObject) + return self._cache.get(col, cuegui.Constants.QVARIANT_NULL) + if role == QtCore.Qt.DecorationRole: + # todo: get rpcOject comment!! + if col == COMMENT_COLUMN and cuegui.Utils.isJob(self.rpcObject): + # and self.rpcObject.hasComment: + return self.__commentIcon + elif role == QtCore.Qt.ForegroundRole: + return self.__foregroundColor + return cuegui.Constants.QVARIANT_NULL + + +class CoreUpWindow(QtWidgets.QDialog): + """A dialog box for adding more cores to a job.""" + + # pylint: disable=non-parent-init-called,super-init-not-called + def __init__(self, parent, jobs, selected=False): + QtWidgets.QWidget.__init__(self, parent) + self.setWindowTitle('Core Up') + self.jobs = jobs + self.dj = DJArnold() + self.setupUI(selected) + + def setupUI(self, selected=False): + """Setup the initial dialog box layout.""" + # Create initial layout + build_times = {} + for job, layers in self.jobs.iteritems(): + build_times[job] = self.dj.getBuildTimes(job, layers) + layout = QtWidgets.QVBoxLayout() + self.setLayout(layout) + + self.listWidget = QtWidgets.QListWidget(self) + self._layers = {} + for job, layers in self.jobs.iteritems(): + for layer in layers: + self._layers[layer.name()] = (job, layer) + layer_label = layer.name() + # if build_times.has_key(layer.name()): + # layer_label += ' - %s' % build_times[layer.name()] + listItem = QtWidgets.QListWidgetItem(layer_label) + self.listWidget.addItem(listItem) + self.listWidget.setSelectionMode(3) # Multi Selection Mode + layout.addWidget(self.listWidget) + + buttonLayout = QtWidgets.QHBoxLayout() + self.core2btn = QtWidgets.QPushButton('2 Cores') + self.connect(self.core2btn, QtCore.SIGNAL('clicked()'), self.core2btn_callback) + self.core4btn = QtWidgets.QPushButton('4 Cores') + self.connect(self.core4btn, QtCore.SIGNAL('clicked()'), self.core4btn_callback) + self.core8btn = QtWidgets.QPushButton('8 Cores') + self.connect(self.core8btn, QtCore.SIGNAL('clicked()'), self.core8btn_callback) + buttonLayout.addWidget(self.core2btn) + buttonLayout.addWidget(self.core4btn) + buttonLayout.addWidget(self.core8btn) + layout.addLayout(buttonLayout) + + coreLayout = QtWidgets.QHBoxLayout() + self.coreSpinner = QtWidgets.QSpinBox() + self.coreSpinner.setRange(1, 16) + self.coreSpinner.setWrapping(True) + self.coreSpinner.setSingleStep(1) + self.coreUpButton = QtWidgets.QPushButton('Core Up') + self.connect(self.coreUpButton, QtCore.SIGNAL('clicked()'), self.coreUpbtn_callback) + coreLayout.addWidget(self.coreSpinner) + coreLayout.addWidget(self.coreUpButton) + layout.addLayout(coreLayout) + + controlLayout = QtWidgets.QHBoxLayout() + self.retryFramesCB = QtWidgets.QCheckBox('Retry Frames') + self.retryThresholdSpinner = QtWidgets.QSpinBox() + self.retryThresholdSpinner.setRange(0, 100) + self.retryThresholdSpinner.setWrapping(True) + self.retryThresholdSpinner.setSingleStep(5) + self.retryThresholdSpinner.setSuffix('%') + self.retryThresholdSpinner.setEnabled(False) + self.retryThresholdSpinner.setValue(70) + controlLayout.addWidget(self.retryFramesCB) + controlLayout.addWidget(self.retryThresholdSpinner) + layout.addLayout(controlLayout) + + self.connect(self.retryFramesCB, QtCore.SIGNAL('stateChanged(int)'), + self.retryFrameCB_callback) + + if selected: + self.listWidget.selectAll() + + def selectedLayers(self): + """Return a list of selected layer rpcObjects.""" + indexs = map(lambda x: str(x.text()), self.listWidget.selectedItems()) + return [self._layers[index] for index in indexs] + + def coreup(self, cores): + """Set Min Cores to cores for all selected layers of job.""" + for job, layer in self.selectedLayers(): + print("Setting max cores to %d for %s" % (cores, layer.name())) + layer.setMinCores(cores * 1.0) + time.sleep(CUE_SLEEP) + if self.retryFramesCB.isChecked(): + fs = opencue.search.FrameSearch() + fs.state = [opencue.wrappers.frame.Frame().FrameState(2)] + frames = layer.getFrames(fs) + for frame in frames: + precentage = self.dj.getCompletionAmount(job, frame) + if precentage >= 0: + if precentage < self.retryThresholdSpinner.value(): + print('Retrying frame %s %s' % (job.name(), frame.frame())) + frame.kill() + time.sleep(CUE_SLEEP) + self.close() + + def core2btn_callback(self): + """2 Core Button Callback.""" + self.coreup(2) + + def core4btn_callback(self): + """4 Core Button Callback.""" + self.coreup(4) + + def core8btn_callback(self): + """8 Core Button Callback.""" + self.coreup(8) + + def coreUpbtn_callback(self): + """Core Up Button Callback.""" + cores = int(self.coreSpinner.value()) + self.coreup(cores) + + def retryFrameCB_callback(self, value): + """Retries frame if value is given.""" + if value: + self.retryThresholdSpinner.setEnabled(True) + else: + self.retryThresholdSpinner.setEnabled(False) + + +class DJArnold(object): + """Represents arnold engine.""" + completion_pattern = re.compile( + # pylint: disable=line-too-long + r'[INFO BatchMain]: [0-9][0-9]:[0-9][0-9]:[0-9][0-9] [0-9]{1,8}mb | (?P[0-9]{1,3})% done - [0-9]{1,5} rays/pixel') + + def __init__(self, show=None): + if not show: + show = os.environ.get('SHOW') + self.show = show + + # pylint: disable=no-self-use + def getLog(self, job, frame): + """Return the contents of a log given a job and a frame.""" + log_dir = job.logDir() + log_name = '%s.%s.rqlog' % (job.name(), frame.data.name) + log_file = os.path.join(log_dir, log_name) + if not os.path.exists(log_file): + return [] + f = open(log_file, 'r') + log_lines = [line.strip() for line in f.readlines() if line.strip()] + f.close() + return log_lines + + def getBuildTimes(self, job, layers=None): + """Return a dictionary with layer names as keys, and build tiems as + values. + """ + results = {} + if not layers: + layers = job.getLayers() + for layer in layers: + if isinstance(layer, str): + layer = job.getLayer(layer) + if 'preprocess' in layer.name(): + continue + built_frames = [] + cores = 0 + cores_list = [] + fs = opencue.search.FrameSearch() + fs.states = [opencue.wrappers.frame.Frame().FrameState(3)] + frames = layer.getFrames(fs) + if not frames: + fs.states = [opencue.wrappers.frame.Frame().FrameState(2)] + frames = layer.getFrames(fs) + for frame in frames: + frame_cores = float(frame.lastResource.split('/')[1]) + if frame_cores != cores: + if frame_cores not in cores_list: + built_frames.append((frame, frame_cores)) + cores_list.append(frame_cores) + build_times = [] + for frame, cores in built_frames: + log_lines = self.getLog(job, frame) + for line in log_lines: + if "[kat] Building scene done." in line: + line = line.replace('[INFO BatchMain]: ', '') + build_time = line.split()[0] + hours, minutes, seconds = build_time.split(':') + seconds = int(seconds) + seconds += (int(minutes) * 60) + seconds += (int(hours) * 360) + build_times.append(seconds) + if build_times: + avg = sum(build_times) / len(build_times) + seconds = int(avg % 60) + minutes = int((avg / 60) % 60) + hours = int(avg / 3600) + results[layer.name()] = (layer, datetime.time(hours, minutes, seconds)) + return results + + def getCompletionAmount(self, job, frame): + """Return a integer representing the last reported completed percenatge on arnold job.""" + log_lines = self.getLog(job, frame) + log_lines.reverse() + complete = -1 + for line in log_lines: + if line.startswith('[INFO BatchMain]:'): + matches = self.completion_pattern.search(line) + if matches: + complete = int(matches.group('total')) + break + return complete From 870b0b591c27beba6765a5339b5ee162a4ac2db2 Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Tue, 10 May 2022 13:03:04 -0700 Subject: [PATCH 189/277] [cuegui] Improvements and new features for the Redirect plugin. (#1113) --- cuegui/cuegui/Redirect.py | 340 +++++++++++++++++++++++++++++--------- cuegui/cuegui/Utils.py | 47 ++++++ 2 files changed, 311 insertions(+), 76 deletions(-) diff --git a/cuegui/cuegui/Redirect.py b/cuegui/cuegui/Redirect.py index ffdbfb534..49fb6b7a5 100644 --- a/cuegui/cuegui/Redirect.py +++ b/cuegui/cuegui/Redirect.py @@ -37,8 +37,14 @@ import opencue +import cuegui.Logger import cuegui.Utils +logger = cuegui.Logger.getLogger(__file__) + +MEMORY_PATTERN = re.compile("[0-9]+(?:TB|GB|MB|KB)") +MEMORY_BTYPE = "TB|GB|MB|KB" + class ShowCombo(QtWidgets.QComboBox): """ @@ -216,8 +222,12 @@ def __init__(self, parent=None): self.__cores_spin.setRange(1, self._cfg().get('max_cores', 32)) self.__cores_spin.setValue(1) + self.__max_cores_spin = QtWidgets.QSpinBox(self) + self.__max_cores_spin.setRange(1, self._cfg().get('max_cores', 32)) + self.__max_cores_spin.setValue(32) + self.__mem_spin = QtWidgets.QDoubleSpinBox(self) - self.__mem_spin.setRange(1, self._cfg().get('max_memory', 200)) + self.__mem_spin.setRange(1, self._cfg().get('max_memory', 250)) self.__mem_spin.setDecimals(1) self.__mem_spin.setValue(4) self.__mem_spin.setSuffix("GB") @@ -227,9 +237,10 @@ def __init__(self, parent=None): self.__limit_spin.setValue(10) self.__prh_spin = QtWidgets.QDoubleSpinBox(self) - self.__prh_spin.setRange(1, self._cfg().get('max_proc_hour_cutoff', 30)) + # increase Proc Hour upper bound limit + self.__prh_spin.setRange(1, 500) self.__prh_spin.setDecimals(1) - self.__prh_spin.setValue(10) + self.__prh_spin.setValue(20) self.__prh_spin.setSuffix("PrcHrs") # Job Filters @@ -243,7 +254,7 @@ def __init__(self, parent=None): self.__clear_btn = QtWidgets.QPushButton("Clr", self) self.__group = QtWidgets.QGroupBox("Resource Filters") - self.__group_filter = QtWidgets.QGroupBox("Job Filters") + self.__groupFilter = QtWidgets.QGroupBox("Job Filters") layout1 = QtWidgets.QHBoxLayout() layout1.addWidget(self.__update_btn) @@ -257,6 +268,8 @@ def __init__(self, parent=None): layout2.addWidget(self.__alloc_filter) layout2.addWidget(QtWidgets.QLabel("Minimum Cores:", self)) layout2.addWidget(self.__cores_spin) + layout2.addWidget(QtWidgets.QLabel("Max Cores:", self)) + layout2.addWidget(self.__max_cores_spin) layout2.addWidget(QtWidgets.QLabel("Minimum Memory:", self)) layout2.addWidget(self.__mem_spin) layout2.addWidget(QtWidgets.QLabel("Result Limit:", self)) @@ -274,10 +287,10 @@ def __init__(self, parent=None): layout3.addWidget(self.__exclude_regex) self.__group.setLayout(layout2) - self.__group_filter.setLayout(layout3) + self.__groupFilter.setLayout(layout3) layout = QtWidgets.QVBoxLayout(self) - layout.addWidget(self.__group_filter) + layout.addWidget(self.__groupFilter) layout.addWidget(self.__group) layout.addLayout(layout1) @@ -335,6 +348,10 @@ def getCores(self): """Gets the core count.""" return int(self.__cores_spin.value()) + def getMaxCores(self): + """Gets the max core count.""" + return int(self.__max_cores_spin.value()) + def getMemory(self): """Gets the memory amount.""" return int(self.__mem_spin.value() * 1048576.0) @@ -393,7 +410,8 @@ class RedirectWidget(QtWidgets.QWidget): Displays a table of procs that can be selected for redirect. """ - HEADERS = ["Name", "Cores", "Memory", "PrcTime", "Group", "Service"] + HEADERS = ["Name", "Cores", "Memory", "PrcTime", "Group", "Service", + "Job Cores", "Pending", "LLU", "Log"] def __init__(self, parent=None): QtWidgets.QWidget.__init__(self, parent) @@ -402,12 +420,16 @@ def __init__(self, parent=None): self.__controls = RedirectControls(self) self.__model = QtGui.QStandardItemModel(self) - self.__model.setColumnCount(5) + self.__model.setColumnCount(7) self.__model.setHorizontalHeaderLabels(RedirectWidget.HEADERS) + self.__proxyModel = ProxyModel(self) + self.__proxyModel.setSourceModel(self.__model) + self.__tree = QtWidgets.QTreeView(self) self.__tree.setEditTriggers(QtWidgets.QAbstractItemView.NoEditTriggers) - self.__tree.setModel(self.__model) + self.__tree.setSortingEnabled(True) + self.__tree.setModel(self.__proxyModel) layout = QtWidgets.QVBoxLayout(self) layout.addWidget(self.__controls) @@ -418,29 +440,57 @@ def __init__(self, parent=None): self.__controls.getRedirectButton().pressed.connect(self.redirect) self.__controls.getSelectAllButton().pressed.connect(self.selectAll) self.__controls.getClearButton().pressed.connect(self.clearTarget) - # pylint: enable=no-member - def __get_selected_procs_by_alloc(self, selected_items): + self.__tree.doubleClicked.connect(self.mouseDoubleClickEvent) + self.__tree.clicked.connect(self.mousePressEvent) + # pylint: disable=no-member + + @QtCore.Slot("QModelIndex") + def mousePressEvent(self, item): + """Called when an item is clicked on. Copies selected object names to + the middle click selection clip board. + """ + try: + QtWidgets.QApplication.clipboard().setText(item.data(), QtGui.QClipboard.Selection) + except AttributeError as e: + logger.info("Error item no longer available %s", e) + + @QtCore.Slot("QModelIndex") + def mouseDoubleClickEvent(self, index): + """ emit proc to Job Monitor Tree """ + attr = getattr(index, 'data', None) + if attr is not None: + try: + jobObject = opencue.api.getJobs(job=[index.data()]) + if jobObject: + if cuegui.Utils.isJob(jobObject[0]): + QtGui.qApp.view_object.emit(jobObject[0]) + except opencue.exception.CueException as e: + text = ('Not able to add job to Job Monitor Tree. ' + 'Error Message:\n %s' % e) + self.__warn(text) + + def __getSelectedProcsByAlloc(self, selectedItems): """ Gathers and returns the selected procs, grouped by allocation their allocation names - @param selected_items: The selected rows to analyze - @type selected_items: list> + @param selectedItems: The selected rows to analyze + @type selectedItems: list> @return: A dictionary with the allocation names are the keys and the selected procs are the values. @rtype: dict """ - procs_by_alloc = {} - for item in selected_items: + procByAlloc = {} + for item in selectedItems: entry = self.__hosts.get(str(item.text())) alloc = entry.get('alloc') - alloc_procs = procs_by_alloc.get(alloc, []) - alloc_procs.extend(list(entry["procs"])) - procs_by_alloc[alloc] = alloc_procs - return procs_by_alloc + allocProcs = procByAlloc.get(alloc, []) + allocProcs.extend(list(entry["procs"])) + procByAlloc[alloc] = allocProcs + return procByAlloc def __warn(self, msg): """ @@ -454,7 +504,7 @@ def __warn(self, msg): message.setText(msg) message.exec_() - def __is_cross_show_safe(self, procs, target_show): + def __isCrossShowSafe(self, procs, targetShow): """ Determines whether or not it's safe to redirect cores from a show to another, based on user response to the warning message @@ -462,30 +512,31 @@ def __is_cross_show_safe(self, procs, target_show): @param procs: The procs to redirect @type procs: L{opencue.wrappers.proc.Proc} - @param target_show: The name of the target show - @type target_show: str + @param targetShow: The name of the target show + @type targetShow: str @return: Whether or not it's safe to redirect the given procs to the target show @rtype: bool """ - xshow_jobs = [proc.getJob() for proc in procs if not - proc.getJob().show() == target_show] - if not xshow_jobs: + xshowJobs = [proc.getJob() for proc in procs if + proc.getJob().show() != targetShow] + + if not xshowJobs: return True # No cross-show procs msg = ('Redirecting the selected procs to the target will result ' 'in killing frames on other show/s.\nDo you have approval ' 'from (%s) to redirect cores from the following jobs?' - % ', '.join([j.show().upper() for j in xshow_jobs])) + % ', '.join([j.show().upper() for j in xshowJobs])) return cuegui.Utils.questionBoxYesNo(parent=self, title="Cross-show Redirect!", text=msg, items=[j.name() for j - in xshow_jobs]) + in xshowJobs]) - def __is_burst_safe(self, alloc, procs, show): + def __isBurstSafe(self, alloc, procs, show): """ Determines whether or not it's safe to redirect cores by checking the burst target show burst and the number of cores being redirected. If @@ -511,33 +562,34 @@ def __is_burst_safe(self, alloc, procs, show): # pylint: disable=protected-access cfg = self.__controls._cfg() # pylint: enable=protected-access - wc_ok = cfg.get('wasted_cores_threshold', 100) - if wc_ok < 0: + wcThreshold = cfg.get('redirect_wasted_cores_threshold', 100) + if wcThreshold < 0: return True - show_obj = opencue.api.findShow(show) - show_subs = dict((s.data.name.rstrip('.%s' % show), s) - for s in show_obj.getSubscriptions() + showObj = opencue.api.findShow(show) + stripShowRegex = '\\.%s' % show + showSubs = dict((re.sub(stripShowRegex, "", s.data.name), s) + for s in showObj.getSubscriptions() if s.data.allocation_name in alloc) try: - procs_to_burst = (show_subs.get(alloc).data.burst - - show_subs.get(alloc).data.reserved_cores) - procs_to_redirect = int(sum([p.data.reserved_cores + procsBurst = (showSubs.get(alloc).data.burst - + showSubs.get(alloc).data.reserved_cores) + procsRedirect = int(sum([p.data.reserved_cores for p in procs])) - wasted_cores = int(procs_to_redirect - procs_to_burst) - if wasted_cores <= wc_ok: + wastedCores = int(procsRedirect - procsBurst) + if wastedCores <= wcThreshold: return True # wasted cores won't exceed threshold - status = ('at burst' if procs_to_burst == 0 else + status = ('at burst' if procsBurst == 0 else '%d cores %s burst' - % (procs_to_burst, - 'below' if procs_to_burst > 0 else 'above')) + % (abs(procsBurst), + 'below' if procsBurst > 0 else 'above')) msg = ('Target show\'s %s subscription is %s. Redirecting ' 'the selected procs will kill frames to free up %d ' 'cores. You will be killing %d cores ' 'that the target show will not be able to use. ' 'Do you want to redirect anyway?' - % (alloc, status, int(procs_to_redirect), wasted_cores)) + % (alloc, status, int(procsRedirect), wastedCores)) return cuegui.Utils.questionBoxYesNo(parent=self, title=status.title(), text=msg) @@ -547,6 +599,67 @@ def __is_burst_safe(self, alloc, procs, show): % (alloc, show, show, alloc)) return False + @classmethod + def __isAllowed(cls, procs, targetJob): + """Checks if the follow criteria are met to allow redirect to target job: + - if source/target job have pending frames + - if target job hasn't reached maximum cores + - check if adding frames will push target job over it's max cores + + @param procs: The (source) procs to be redirected + @type procs: L{opencue.wrappers.proc.Proc} + @param targetJob: target job to move procs to + @return: true/false of whether criteria are met + and error message if any + @rtype: tuple(boolean, string) + """ + errMsg = "" + allowed = False + + # Case 1: Check if target job hasn't reached it's max cores + if targetJob.coresReserved() < targetJob.maxCores(): + allowed = True + errMsg = "Target job %s cores reserved %s \ + reached max cores %s " %(targetJob.name(), + targetJob.coresReserved(), + targetJob.maxCores()) + + # Case 2: 1. Check target job for pending frames + # 2. Check source procs for pending frames + if allowed and targetJob.waitingFrames() <= 0: + allowed = False + errMsg = "Target job %s has no pending (waiting) frames" % targetJob.name() + + if allowed: + for proc in procs: + job = proc.getJob() + if job.waitingFrames() <= 0: + allowed = False + errMsg = "Source job %s has no pending (waiting) frames" % job.name() + break + + # Case 3: Check if each proc or summed up procs will + # push targetJob over it's max cores + if allowed: + totalProcCores = 0 + for proc in procs: + totalProcCores += proc.coresReserved() + msg = ('proc cores reserved of %s will push %s ' + 'over it\'s max cores limit of %s') + if (proc.coresReserved() + targetJob.coresReserved()) > targetJob.maxCores() or \ + (totalProcCores + targetJob.coresReserved()) > targetJob.maxCores(): + errMsg = msg % (str(proc.coresReserved() + targetJob.coresReserved()), + targetJob.name(), str(targetJob.maxCores())) + allowed = False + break + + if totalProcCores > targetJob.maxCores(): + errMsg = msg % (totalProcCores, targetJob.name(), + str(targetJob.maxCores())) + allowed = False + + return allowed, errMsg + def redirect(self): """ Redirect the selected procs to the target job, after running a few @@ -558,37 +671,58 @@ def redirect(self): # Get selected items items = [self.__model.item(row) for row in range(0, self.__model.rowCount())] - selected_items = [item for item in items + selectedItems = [item for item in items if item.checkState() == QtCore.Qt.Checked] - if not selected_items: # Nothing selected, exit + if not selectedItems: # Nothing selected, exit self.__warn('You have not selected anything to redirect.') return # Get the Target Job - job_name = self.__controls.getJob() - if not job_name: # No target job, exit + jobName = self.__controls.getJob() + if not jobName: # No target job, exit self.__warn('You must have a job name selected.') return job = None try: - job = opencue.api.findJob(job_name) + job = opencue.api.findJob(jobName) except opencue.EntityNotFoundException: # Target job finished, exit - self.__warn_and_stop('The job you\'re trying to redirect to ' - 'appears to be no longer in the cue!') + text = ('The job you\'re trying to redirect to ' + 'appears to be no longer in the cue!') + cuegui.Utils.showErrorMessageBox(text, title="ERROR!") + return # Gather Selected Procs - procs_by_alloc = self.__get_selected_procs_by_alloc(selected_items) - show_name = job.show() - for alloc, procs in list(procs_by_alloc.items()): - if not self.__is_cross_show_safe(procs, show_name): # Cross-show - return - if not self.__is_burst_safe(alloc, procs, show_name): # At burst - return + procByAlloc = self.__getSelectedProcsByAlloc(selectedItems) + showName = job.show() + # Check if safe to redirect + # 1. don't redirect if target job's reserved cores reached job max + # 2. at burst + # 3. cross-show safe + warning = "" + try: + for alloc, procs in list(procByAlloc.items()): + if not self.__isCrossShowSafe(procs, showName): + warning = "Is not cross show safe" + break + if not self.__isBurstSafe(alloc, procs, showName): + warning = "Is not burst safe" + break + allowed, errMsg = self.__isAllowed(procs, targetJob=job) + if not allowed: + warning = errMsg + break + except opencue.exception.CueException as e: + warning = str(e) + + if warning: + warning = "Failed to Redirect:\n" + warning + self.__warn(warning) + return # Redirect errors = [] - for item in selected_items: + for item in selectedItems: entry = self.__hosts.get(str(item.text())) procs = entry["procs"] # pylint: disable=broad-except @@ -601,7 +735,12 @@ def redirect(self): item.setEnabled(False) if errors: # Something went wrong! - self.__warn('Some procs failed to redirect.') + stackTrace = "\n".join(errors) + text = 'Some procs failed to redirect with errors:\n' + stackTrace + self.__warn(text) + else: + text = 'Redirect To Job Request sent for:\n' + job.name() + self.__warn(text) def selectAll(self): """ @@ -618,15 +757,16 @@ def clearTarget(self): self.__controls.getJobBox().clear() def update(self): + """ Update the model """ self.__model.clear() self.__model.setHorizontalHeaderLabels(RedirectWidget.HEADERS) hosts = { } ok = 0 - service_filter = self.__controls.getRequiredService() - group_filter = self.__controls.getIncludedGroups() - job_regex = self.__controls.getJobNameExcludeRegex() + serviceFilter = self.__controls.getRequiredService() + groupFilter = self.__controls.getIncludedGroups() + jobRegexFilter = self.__controls.getJobNameExcludeRegex() show = self.__controls.getShow() alloc = self.__controls.getAllocFilter() @@ -654,19 +794,23 @@ def update(self): if ok >= self.__controls.getLimit(): break - if job_regex: - if re.match(job_regex, proc.data.job_name): + if jobRegexFilter: + if re.match(jobRegexFilter, proc.data.job_name): continue - if service_filter: - if service_filter not in proc.data.services: + if serviceFilter: + if serviceFilter not in proc.data.services: continue - if group_filter: - if proc.data.group_name not in group_filter: + if groupFilter: + if proc.data.group_name not in groupFilter: continue name = proc.data.name.split("/")[0] + lluTime = cuegui.Utils.getLLU(proc) + job = proc.getJob() + logLines = cuegui.Utils.getLastLine(proc.data.log_path) or "" + if name not in hosts: cue_host = opencue.api.findHost(name) hosts[name] = { @@ -684,10 +828,15 @@ def update(self): host["procs"].append(proc) host["mem"] = host["mem"] + proc.data.reserved_memory - host["cores"] = host["cores"] + proc.data.reserved_cores + host["cores"] = int(host["cores"]) + int(proc.data.reserved_cores) host["time"] = host["time"] + (int(time.time()) - proc.data.dispatch_time) + host["llu"] = cuegui.Utils.numFormat(lluTime, "t") + host["log"] = logLines + host['job_cores'] = job.data.job_stats.reserved_cores + host['waiting'] = job.pendingFrames() or 0 if host["cores"] >= self.__controls.getCores() and \ + host["cores"] <= self.__controls.getMaxCores() and \ host["mem"] >= self.__controls.getMemory() and \ host["time"] < self.__controls.getCutoffTime(): self.__addHost(host) @@ -700,6 +849,7 @@ def update(self): self.__hosts = hosts def __addHost(self, entry): + """ Add Host to ProxyModel """ host = entry["host"] procs = entry["procs"] rtime = entry["time"] @@ -707,10 +857,10 @@ def __addHost(self, entry): checkbox = QtGui.QStandardItem(host.data.name) checkbox.setCheckable(True) - self.__model.appendRow([checkbox, - QtGui.QStandardItem(str(entry["cores"])), - QtGui.QStandardItem("%0.2fGB" % (entry["mem"] / 1048576.0)), - QtGui.QStandardItem(cuegui.Utils.secondsToHHMMSS(rtime))]) + self.__proxyModel.sourceModel().appendRow([checkbox, + QtGui.QStandardItem(str(entry["cores"])), + QtGui.QStandardItem("%0.2fGB" % (entry["mem"] / 1048576.0)), + QtGui.QStandardItem(cuegui.Utils.secondsToHHMMSS(rtime))]) for proc in procs: checkbox.appendRow([QtGui.QStandardItem(proc.data.job_name), @@ -718,9 +868,47 @@ def __addHost(self, entry): QtGui.QStandardItem( "%0.2fGB" % (proc.data.reserved_memory / 1048576.0)), QtGui.QStandardItem(cuegui.Utils.secondsToHHMMSS(time.time() - - proc.data.dispatch_time)), + proc.data.dispatch_time)), QtGui.QStandardItem(proc.data.group_name), - QtGui.QStandardItem(",".join(proc.data.services))]) + QtGui.QStandardItem(",".join(proc.data.services)), + QtGui.QStandardItem(str(entry["job_cores"])), + QtGui.QStandardItem(str(entry["waiting"])), + QtGui.QStandardItem(str(entry["llu"])), + QtGui.QStandardItem(str(entry["log"])) + ]) + + proxy = self.__tree.model() + model = proxy.sourceModel() + for row in range(model.rowCount()): + index = model.index(row, 0) + self.__tree.expand(proxy.mapFromSource(index)) + self.__tree.resizeColumnToContents(0) + self.__tree.setWordWrap(True) + + +class ProxyModel(QtCore.QSortFilterProxyModel): + """Provides support for sorting data passed between the model and the tree view""" - self.__tree.setExpanded(self.__model.indexFromItem(checkbox), True) - self.__tree.resizeColumnToContents(0) + def lessThan(self, left, right): + + leftData = self.sourceModel().data(left) + rightData = self.sourceModel().data(right) + + try: + return int(leftData) < int(rightData) + except ValueError: + if re.search(MEMORY_PATTERN, leftData): + # strip memory type to compare + leftDataBtype = re.search(MEMORY_BTYPE, leftData).group() + leftDataMem = re.sub(MEMORY_BTYPE, "", leftData) + leftBtyes = cuegui.Utils.byteConversion(float(leftDataMem), leftDataBtype) + + rightDataBtype = re.search(MEMORY_BTYPE, rightData).group() + rightDataMem = re.sub(MEMORY_BTYPE, "", rightData) + rightBytes = cuegui.Utils.byteConversion(float(rightDataMem), rightDataBtype) + return float(leftBtyes) < float(rightBytes) + + return leftData < rightData + + except TypeError: + return leftData < rightData diff --git a/cuegui/cuegui/Utils.py b/cuegui/cuegui/Utils.py index e931ab98b..4c18f7233 100644 --- a/cuegui/cuegui/Utils.py +++ b/cuegui/cuegui/Utils.py @@ -625,3 +625,50 @@ def shutdownThread(thread): """Shuts down a WorkerThread.""" thread.stop() return thread.wait(1500) + +def getLLU(item): + """ LLU time from log_path """ + if isProc(item): + logFile = item.data.log_path + elif isFrame(item): + logFile = item.log_path + else: + return "" + try: + statInfo = os.path.getmtime(logFile) + except Exception as e: + logger.info("not able to extract LLU: %s", e) + return None + + lluTime = time.time() - statInfo + + return lluTime + +def numFormat(num, _type): + """ format LLU time """ + if num == "" or num < .001 or num is None: + return "" + if _type == "t": + return secondsToHHMMSS(int(num)) + if _type == "f": + return "%.2f" % float(num) + +def byteConversion(amount, btype): + """ convert unit of memory size into bytes for comparing different + unit measures + + :param amount: unit of memory size + :ptype amount: float + :param btype: unit type + :ptype btype: string + :return: unit in bytes + :rtype: float + """ + n = 1 + conversionMap = {"KB": 1, "TB": 4, "GB": 3, "MB": 2} + _bytes = amount + if btype.upper() in conversionMap: + n = conversionMap[btype.upper()] + for _ in range(n): + _bytes *= 1024 + return _bytes From 02cb67f50b90e69f8494b931f5e7fefea296fd09 Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Tue, 10 May 2022 15:41:10 -0700 Subject: [PATCH 190/277] Add more exit codes to Frame state waiting (#1131) * Add more exit codes to Frame state waiting When determining the frame state cuebot needs to make sure that certain exit statuses put the frame state back into waiting, this will help save time for users when a frame fails for host hardware issues. (cherry picked from commit cbeda9387b09a8713bb76d9075fdee72c3c795f1) * Add more exit codes to Frame state waiting * Removed unused method updateFrameHostDown * Removed deprecated oracle FrameDaoJdbc file * Remove log debugging from FrameCompleteHandler * Reverting, adding updateFrameHostDown FrameDao method * Verion bump, add missing interface checkRetries method Co-authored-by: Diego Tavares da Silva --- .../com/imageworks/spcue/dao/FrameDao.java | 17 ++++ .../spcue/dao/postgres/FrameDaoJdbc.java | 85 ++++++++++++++----- .../dispatcher/DispatchSupportService.java | 31 +++++-- .../spcue/dispatcher/Dispatcher.java | 4 +- .../dispatcher/FrameCompleteHandler.java | 23 +++-- proto/job.proto | 4 +- pycue/opencue/wrappers/frame.py | 2 +- pycue/tests/wrappers/frame_test.py | 6 +- 8 files changed, 128 insertions(+), 44 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java index 7c22c3e07..7c2e3c050 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java @@ -71,6 +71,15 @@ public interface FrameDao { */ public FrameDetail findLongestFrame(JobInterface job); + /** + * Checks to see how many retries a frame has. If that number + * is greater than or equal to the jobs max retries, the frame + * is marked as dead. + * + * @param frame + */ + void checkRetries(FrameInterface frame); + /** * Batch inserts a frameSet of frames. * @@ -194,6 +203,14 @@ boolean updateFrameStopped(FrameInterface frame, FrameState state, int exitStatu */ boolean updateFrameCleared(FrameInterface frame); + /** + * Sets a frame to an unreserved waiting state. + * + * @param frame + * @return + */ + boolean updateFrameHostDown(FrameInterface frame); + /** * Returns a DispatchFrame object from the frame's uinique ID. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java index ef3425a8f..3c752ad96 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java @@ -27,6 +27,7 @@ import java.sql.Timestamp; import java.util.Optional; +import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.RowMapper; import org.springframework.jdbc.core.support.JdbcDaoSupport; @@ -116,7 +117,7 @@ public boolean updateFrameStopped(FrameInterface frame, FrameState state, frame.getVersion()) == 1; } - private static final String UPDATE_FRAME_CLEARED = + private static final String UPDATE_FRAME_REASON = "UPDATE "+ "frame "+ "SET " + @@ -132,17 +133,26 @@ public boolean updateFrameStopped(FrameInterface frame, FrameState state, "(SELECT proc.pk_frame FROM " + "proc WHERE proc.pk_frame=?)"; - @Override - public boolean updateFrameCleared(FrameInterface frame) { + private int updateFrame(FrameInterface frame, int exitStatus) { int result = getJdbcTemplate().update( - UPDATE_FRAME_CLEARED, - FrameState.WAITING.toString(), - Dispatcher.EXIT_STATUS_FRAME_CLEARED, - frame.getFrameId(), - frame.getFrameId()); + UPDATE_FRAME_REASON, + FrameState.WAITING.toString(), + exitStatus, + frame.getFrameId(), + frame.getFrameId()); - return result > 0; + return result; + } + + @Override + public boolean updateFrameHostDown(FrameInterface frame) { + return updateFrame(frame, Dispatcher.EXIT_STATUS_DOWN_HOST) > 0; + } + + @Override + public boolean updateFrameCleared(FrameInterface frame) { + return updateFrame(frame, Dispatcher.EXIT_STATUS_FRAME_CLEARED) > 0; } private static final String UPDATE_FRAME_STARTED = @@ -196,31 +206,45 @@ public boolean updateFrameCleared(FrameInterface frame) { "WHERE " + "pk_frame = ? " + "AND " + - "int_exit_status NOT IN (?,?,?) "; + "int_exit_status NOT IN (?,?,?,?,?,?,?) "; @Override public void updateFrameStarted(VirtualProc proc, FrameInterface frame) { lockFrameForUpdate(frame, FrameState.WAITING); - int result = getJdbcTemplate().update(UPDATE_FRAME_STARTED, - FrameState.RUNNING.toString(), proc.hostName, proc.coresReserved, - proc.memoryReserved, proc.gpusReserved, proc.gpuMemoryReserved, frame.getFrameId(), - FrameState.WAITING.toString(), frame.getVersion()); - - if (result == 0) { - String error_msg = "the frame " + - frame + " was updated by another thread."; - throw new FrameReservationException(error_msg); + try { + int result = getJdbcTemplate().update(UPDATE_FRAME_STARTED, + FrameState.RUNNING.toString(), proc.hostName, proc.coresReserved, + proc.memoryReserved, proc.gpusReserved, proc.gpuMemoryReserved, frame.getFrameId(), + FrameState.WAITING.toString(), frame.getVersion()); + if (result == 0) { + String error_msg = "the frame " + + frame + " was updated by another thread."; + throw new FrameReservationException(error_msg); + } + } catch (DataAccessException e) { + /* + * This usually happens when the folder's max cores + * limit has exceeded + */ + throw new FrameReservationException(e.getCause()); } /* * Frames that were killed via nimby or hardware errors not attributed to - * the software do not increment the retry counter. + * the software do not increment the retry counter. Like failed launch, + * orphaned frame, failed kill or down host. */ - getJdbcTemplate().update(UPDATE_FRAME_RETRIES, - frame.getFrameId(), -1, FrameExitStatus.SKIP_RETRY_VALUE, - Dispatcher.EXIT_STATUS_FRAME_CLEARED); + try { + getJdbcTemplate().update(UPDATE_FRAME_RETRIES, + frame.getFrameId(), -1, FrameExitStatus.SKIP_RETRY_VALUE, + FrameExitStatus.FAILED_LAUNCH_VALUE, Dispatcher.EXIT_STATUS_FRAME_CLEARED, + Dispatcher.EXIT_STATUS_FRAME_ORPHAN, Dispatcher.EXIT_STATUS_FAILED_KILL, + Dispatcher.EXIT_STATUS_DOWN_HOST); + } catch (DataAccessException e) { + throw new FrameReservationException(e.getCause()); + } } private static final String UPDATE_FRAME_FIXED = @@ -657,6 +681,21 @@ public FrameInterface findFrame(JobInterface job, String name) { FRAME_MAPPER, name, job.getJobId()); } + @Override + public void checkRetries(FrameInterface frame) { + int max_retries = getJdbcTemplate().queryForObject( + "SELECT int_max_retries FROM job WHERE pk_job=?", Integer.class, + frame.getJobId()); + + if (getJdbcTemplate().queryForObject( + "SELECT int_retries FROM frame WHERE pk_frame=?", Integer.class, + frame.getFrameId()) >= max_retries) { + getJdbcTemplate().update( + "UPDATE frame SET str_state=? WHERE pk_frame=?", + FrameState.DEAD.toString(), frame.getFrameId()); + } + } + private static final String UPDATE_FRAME_STATE = "UPDATE " + "frame "+ diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java index d77cc03f3..d6a30fdba 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java @@ -23,15 +23,11 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import org.apache.log4j.Logger; -import org.springframework.dao.EmptyResultDataAccessException; -import org.springframework.transaction.annotation.Propagation; -import org.springframework.transaction.annotation.Transactional; - import com.imageworks.spcue.AllocationInterface; import com.imageworks.spcue.DispatchFrame; import com.imageworks.spcue.DispatchHost; import com.imageworks.spcue.FacilityInterface; +import com.imageworks.spcue.FrameDetail; import com.imageworks.spcue.FrameInterface; import com.imageworks.spcue.GroupInterface; import com.imageworks.spcue.HostInterface; @@ -43,6 +39,11 @@ import com.imageworks.spcue.ShowInterface; import com.imageworks.spcue.StrandedCores; import com.imageworks.spcue.VirtualProc; +import org.apache.log4j.Logger; +import org.springframework.dao.EmptyResultDataAccessException; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; + import com.imageworks.spcue.dao.BookingDao; import com.imageworks.spcue.dao.DispatcherDao; import com.imageworks.spcue.dao.FrameDao; @@ -522,13 +523,29 @@ public void lostProc(VirtualProc proc, String reason, int exitStatus) { frameDao.updateFrameCheckpointState(f, CheckpointState.DISABLED); /* * If the proc has a frame, stop the frame. Frames - * can only be stopped that are running, so if the frame - * is not running it will remain untouched. + * can only be stopped that are running. */ if (frameDao.updateFrameStopped(f, FrameState.WAITING, exitStatus)) { updateUsageCounters(proc, exitStatus); } + /* + * If the frame is not running, check if frame is in dead state, + * frames that died due to host going down should be put back + * into WAITING status. + */ + else { + FrameDetail frameDetail = frameDao.getFrameDetail(f); + if ((frameDetail.state == FrameState.DEAD) && + (Dispatcher.EXIT_STATUS_DOWN_HOST == exitStatus)) { + if (frameDao.updateFrameHostDown(f)) { + logger.info("update frame " + f.getFrameId() + + "to WAITING status for down host"); + } + } + } + } else { + logger.info("Frame ID is NULL, not updating Frame state"); } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java index 848220fcd..6e06bdc44 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java @@ -18,7 +18,6 @@ package com.imageworks.spcue.dispatcher; - import java.util.List; import com.imageworks.spcue.DispatchFrame; @@ -34,6 +33,9 @@ public interface Dispatcher { + // Maximum number of core points that can be assigned to a frame + public static final int CORE_POINTS_RESERVED_MAX = 2400; + // The default number of core points assigned to a frame, if no core // point value is specified public static final int CORE_POINTS_RESERVED_DEFAULT = 100; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java index b54757b69..7d6fad4fb 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java @@ -321,17 +321,17 @@ public void handlePostFrameCompleteOperations(VirtualProc proc, } /* - * An exit status of NO_RETRY (256) indicates that the frame could + * An exit status of FAILED_LAUNCH (256) indicates that the frame could * not be launched due to some unforeseen unrecoverable error that * is not checked when the launch command is given. The most common * cause of this is when the job log directory is removed before the * job is complete. * - * Frames that return a 256 are not automatically retried. + * Frames that return a 256 are put Frame back into WAITING status */ - else if (report.getExitStatus() == FrameExitStatus.NO_RETRY_VALUE) { - logger.info("unbooking " + proc + " frame status was no-retry."); + else if (report.getExitStatus() == FrameExitStatus.FAILED_LAUNCH_VALUE) { + logger.info("unbooking " + proc + " frame status was failed frame launch."); unbookProc = true; } @@ -544,7 +544,8 @@ else if (report.getHost().getNimbyLocked()) { * @param report * @return */ - public static final FrameState determineFrameState(DispatchJob job, LayerDetail layer, DispatchFrame frame, FrameCompleteReport report) { + public static final FrameState determineFrameState(DispatchJob job, LayerDetail layer, + DispatchFrame frame, FrameCompleteReport report) { if (EnumSet.of(FrameState.WAITING, FrameState.EATEN).contains( frame.state)) { @@ -567,17 +568,25 @@ else if (frame.state.equals(FrameState.DEAD)) { || (job.maxRetries != 0 && report.getExitSignal() == 119)) { report = FrameCompleteReport.newBuilder(report).setExitStatus(FrameExitStatus.SKIP_RETRY_VALUE).build(); newState = FrameState.WAITING; + // exemption code 256 + } else if ((report.getExitStatus() == FrameExitStatus.FAILED_LAUNCH_VALUE || + report.getExitSignal() == FrameExitStatus.FAILED_LAUNCH_VALUE) && + (frame.retries < job.maxRetries)) { + report = FrameCompleteReport.newBuilder(report).setExitStatus(report.getExitStatus()).build(); + newState = FrameState.WAITING; } else if (job.autoEat) { newState = FrameState.EATEN; // ETC Time out and LLU timeout - } else if (layer.timeout_llu != 0 && report.getFrame().getLluTime() != 0 && lastUpdate > (layer.timeout_llu -1)) { + } else if (layer.timeout_llu != 0 && report.getFrame().getLluTime() != 0 + && lastUpdate > (layer.timeout_llu -1)) { newState = FrameState.DEAD; } else if (layer.timeout != 0 && report.getRunTime() > layer.timeout * 60) { newState = FrameState.DEAD; } else if (report.getRunTime() > Dispatcher.FRAME_TIME_NO_RETRY) { newState = FrameState.DEAD; } else if (frame.retries >= job.maxRetries) { - if (!(report.getExitStatus() == Dispatcher.EXIT_STATUS_MEMORY_FAILURE || report.getExitSignal() == Dispatcher.EXIT_STATUS_MEMORY_FAILURE)) + if (!(report.getExitStatus() == Dispatcher.EXIT_STATUS_MEMORY_FAILURE + || report.getExitSignal() == Dispatcher.EXIT_STATUS_MEMORY_FAILURE)) newState = FrameState.DEAD; } diff --git a/proto/job.proto b/proto/job.proto index a07a93619..79a0dc234 100644 --- a/proto/job.proto +++ b/proto/job.proto @@ -400,8 +400,8 @@ enum CheckpointState { enum FrameExitStatus { // The frame was a success SUCCESS = 0; - // The frame should not be retried - NO_RETRY = 256; + // The frame should be automatically retried + FAILED_LAUNCH = 256; // Retries should not be incremented SKIP_RETRY = 286; } diff --git a/pycue/opencue/wrappers/frame.py b/pycue/opencue/wrappers/frame.py index 0455cbd7c..5aa528b5d 100644 --- a/pycue/opencue/wrappers/frame.py +++ b/pycue/opencue/wrappers/frame.py @@ -36,7 +36,7 @@ class CheckpointState(enum.IntEnum): class FrameExitStatus(enum.IntEnum): """Possible frame exit statuses.""" SUCCESS = job_pb2.SUCCESS - NO_RETRY = job_pb2.NO_RETRY + FAILED_LAUNCH = job_pb2.FAILED_LAUNCH SKIP_RETRY = job_pb2.SKIP_RETRY class FrameState(enum.IntEnum): diff --git a/pycue/tests/wrappers/frame_test.py b/pycue/tests/wrappers/frame_test.py index 8be9ff0fd..d0e7c7ba6 100644 --- a/pycue/tests/wrappers/frame_test.py +++ b/pycue/tests/wrappers/frame_test.py @@ -206,9 +206,9 @@ def testCheckpointState(self): self.assertEqual(opencue.api.Frame.CheckpointState.DISABLED, 0) def testFrameExitStatus(self): - self.assertEqual(opencue.api.Frame.FrameExitStatus.NO_RETRY, - opencue.compiled_proto.job_pb2.NO_RETRY) - self.assertEqual(opencue.api.Frame.FrameExitStatus.NO_RETRY, 256) + self.assertEqual(opencue.api.Frame.FrameExitStatus.FAILED_LAUNCH, + opencue.compiled_proto.job_pb2.FAILED_LAUNCH) + self.assertEqual(opencue.api.Frame.FrameExitStatus.FAILED_LAUNCH, 256) def testFrameState(self): self.assertEqual(opencue.api.Frame.FrameState.RUNNING, From 596088e26b9dda702fabbfdf01a266c87ca4cf99 Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Wed, 11 May 2022 10:42:16 -0700 Subject: [PATCH 191/277] [cuegui] MonitorCue displays all jobs/groups to move. (#979) --- cuegui/cuegui/CueJobMonitorTree.py | 125 ++++++++++++++++++++++------- cuegui/cuegui/MenuActions.py | 26 +++--- cuegui/tests/MenuActions_tests.py | 47 ++++++++--- 3 files changed, 147 insertions(+), 51 deletions(-) diff --git a/cuegui/cuegui/CueJobMonitorTree.py b/cuegui/cuegui/CueJobMonitorTree.py index 11ad7f9d2..45a9fb07e 100644 --- a/cuegui/cuegui/CueJobMonitorTree.py +++ b/cuegui/cuegui/CueJobMonitorTree.py @@ -22,6 +22,7 @@ from builtins import str from builtins import map +from collections import namedtuple import time from PySide2 import QtCore @@ -43,6 +44,7 @@ logger = cuegui.Logger.getLogger(__file__) +Body = namedtuple("Body", "group_names, group_ids, job_names, job_ids") COLUMN_COMMENT = 1 COLUMN_EAT = 2 @@ -291,35 +293,23 @@ def dropEvent(self, event): if item and item.type() in (cuegui.Constants.TYPE_ROOTGROUP, cuegui.Constants.TYPE_GROUP): job_ids = cuegui.Utils.dropEvent(event, "application/x-job-ids") group_ids = cuegui.Utils.dropEvent(event, "application/x-group-ids") + job_names = cuegui.Utils.dropEvent(event, "application/x-job-names") + group_names = cuegui.Utils.dropEvent(event, "application/x-group-names") if job_ids or group_ids: - body = "" - if group_ids: - body += "Groups:\n" + "\n".join( - cuegui.Utils.dropEvent(event, "application/x-group-names")) - if group_ids and job_ids: - body += "\n\n" - if job_ids: - body += "Jobs:\n" + "\n".join( - cuegui.Utils.dropEvent(event, "application/x-job-names")) - - result = QtWidgets.QMessageBox.question( - self, - "Move groups/jobs?", - "Move the following into the group: " + - "\"%s\"?\n\n%s" % ( - item.rpcObject.data.name, body), - QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No) - - if result == QtWidgets.QMessageBox.Yes: - if job_ids: - jobs = [opencue.api.getJob(id_) for id_ in job_ids] - item.rpcObject.asGroup().reparentJobs(jobs) - - if group_ids: - item.rpcObject.asGroup().reparentGroupIds(group_ids) - - self.updateRequest() + body_content = Body(group_names=group_names, + group_ids=group_ids, + job_names=job_names, + job_ids=job_ids) + + dialog = MoveDialog(title="Move Groups/Jobs", + text="Move the following into the group: %s?" \ + % item.rpcObject.data.name, + event_item=item, + items=body_content, + dist_groups={}, + parent=self) + dialog.exec_() def addShow(self, show, update=True): """Adds a show to the list of monitored shows @@ -812,3 +802,84 @@ def data(self, col, role): return self._cache.get("FST", cuegui.Constants.QVARIANT_NULL) return cuegui.Constants.QVARIANT_NULL + + +class MoveDialog(QtWidgets.QDialog): + """ + A dialog for moving selected Jobs/Groups into another Group + """ + def __init__(self, title, text, event_item, items, dst_groups, + send_to_groups=False, parent=None): + """ + Initializes the list of jobs/groups to move + @type title: str + @param title: Window Title + @type text: str + @param text: Confirmation question to the user + @type event_item: rpcObject + @param event_item: the rpcObject to act on + @type items: namedtuple + @param items: object that holds job_ids, group_ids, group_names, job_names to act on + @type dst_groups: dict + @param dst_groups: dict of destination groups to move jobs/groups to + @type parent: AbstractTreeWidget + @param parent: The dialog's parent + """ + QtWidgets.QDialog.__init__(self, parent) + self.parent = parent + self.items = items + self.event_item = event_item + self.send_to_groups = send_to_groups + self.dst_groups = dst_groups + _btn_accept = QtWidgets.QPushButton("Ok", self) + _btn_cancel = QtWidgets.QPushButton("Cancel", self) + _label_text = QtWidgets.QLabel(text, self) + _label_text.setWordWrap(True) + + _vlayout = QtWidgets.QVBoxLayout(self) + _vlayout.addWidget(_label_text) + + self._listView = QtWidgets.QListView(self) + _vlayout.addWidget(self._listView) + _model = QtGui.QStandardItemModel(self._listView) + self.setWindowTitle(title) + for item in self.items.job_names: + standard_item = QtGui.QStandardItem(item) + _model.appendRow(standard_item) + for item in self.items.group_names: + _standard_item = QtGui.QStandardItem(item) + _model.appendRow(_standard_item) + self._listView.setModel(_model) + + if self.send_to_groups: + self.combo = QtWidgets.QComboBox(self) + self.combo.addItems(sorted(self.dst_groups.keys())) + self.layout().addWidget(self.combo) + + _hlayout = QtWidgets.QHBoxLayout() + _hlayout.addWidget(_btn_accept) + _hlayout.addWidget(_btn_cancel) + _vlayout.addLayout(_hlayout) + + self.connect(_btn_accept, + QtCore.SIGNAL("clicked()"), + self.move_items) + self.connect(_btn_cancel, + QtCore.SIGNAL("clicked()"), + self.reject) + + def move_items(self): + """Reparent jobs to new group""" + + if not self.send_to_groups: + if self.items.job_ids: + jobs = [opencue.api.getJob(id_) for id_ in self.items.job_ids] + self.event_item.rpcObject.asGroup().reparentJobs(jobs) + + if self.items.group_ids: + self.event_item.rpcObject.asGroup().reparentGroupIds(self.items.group_ids) + self.parent.updateRequest() + else: + selected_group = self.combo.currentText() + self.dst_groups[str(selected_group)].reparentJobs(self.items.job_ids) + self.accept() diff --git a/cuegui/cuegui/MenuActions.py b/cuegui/cuegui/MenuActions.py index 2b6f46076..058ea2a39 100644 --- a/cuegui/cuegui/MenuActions.py +++ b/cuegui/cuegui/MenuActions.py @@ -594,17 +594,21 @@ def sendToGroup(self, rpcObjects=None): return title = "Send jobs to group" - groups = { - group.data.name: group for group in opencue.api.findShow(jobs[0].data.show).getGroups()} - body = "What group should these jobs move to?\n" + \ - "\n".join([job.data.name for job in jobs]) - - (group, choice) = QtWidgets.QInputDialog.getItem( - self._caller, title, body, sorted(groups.keys()), 0, False) - if not choice: - return - - groups[str(group)].reparentJobs(jobs) + groups = {group.data.name: group for group in opencue.api.findShow( + jobs[0].data.show).getGroups()} + + body_content = cuegui.CueJobMonitorTree.Body(group_names=[], + group_ids=[], + job_names=[job.name() for job in jobs], + job_ids=jobs) + + dialog = cuegui.CueJobMonitorTree.MoveDialog(title=title, + text="What group should these jobs move to?", + event_item=None, + items=body_content, + dst_groups=groups, + send_to_groups=True) + dialog.exec_() self._update() useLocalCores_info = [ diff --git a/cuegui/tests/MenuActions_tests.py b/cuegui/tests/MenuActions_tests.py index 87e31df0b..b15a2cc9a 100644 --- a/cuegui/tests/MenuActions_tests.py +++ b/cuegui/tests/MenuActions_tests.py @@ -445,34 +445,55 @@ def test_unbook(self, unbookDialogMock): unbookDialogMock.assert_called_with(jobs, self.widgetMock) unbookDialogMock.return_value.exec_.assert_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('cuegui.CueJobMonitorTree.MoveDialog.move_items') @mock.patch('opencue.api.findShow') - def test_sendToGroup(self, findShowMock, getItemMock): + def test_sendToGroup(self, findShowMock, move_itemsMock): + + move_dialogMock = mock.Mock() + + move_dialogMock.open() group_name = 'arbitrary-group-name' job = opencue.wrappers.job.Job( opencue.compiled_proto.job_pb2.Job( name='arbitrary-job-name', show='arbitrary-show-name')) - show = opencue.wrappers.show.Show() + body_content = cuegui.CueJobMonitorTree.Body(group_names=[], + group_ids=[], + job_names=[job.name()], + job_ids=[job]) + group = opencue.wrappers.group.Group(opencue.compiled_proto.job_pb2.Group(name=group_name)) group.reparentJobs = mock.Mock() + + show = opencue.wrappers.show.Show() findShowMock.return_value = show show.getGroups = mock.Mock(return_value=[group]) - getItemMock.return_value = (group_name, True) - self.job_actions.sendToGroup(rpcObjects=[job]) + move_dialogMock.dst_groups = {str(group_name): group} + move_itemsMock.return_value = move_dialogMock.dst_groups[str(group_name)].reparentJobs( + body_content.job_ids) + move_dialogMock.accept() - group.reparentJobs.assert_called_with([job]) + group.reparentJobs.assert_called_with(body_content.job_ids) - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('cuegui.CueJobMonitorTree.MoveDialog.move_items') @mock.patch('opencue.api.findShow') - def test_sendToGroupCanceled(self, findShowMock, getItemMock): - job = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(name='job-name')) - group = opencue.wrappers.group.Group() + def test_sendToGroupCanceled(self, findShowMock, move_itemsMock): + + move_dialogMock = mock.Mock() + + move_dialogMock.open() + group_name = 'arbitrary-group-name' + job = opencue.wrappers.job.Job( + opencue.compiled_proto.job_pb2.Job( + name='arbitrary-job-name', show='arbitrary-show-name')) + group = opencue.wrappers.group.Group(opencue.compiled_proto.job_pb2.Group(name=group_name)) group.reparentJobs = mock.Mock() - findShowMock.getGroups.return_value = [] - getItemMock.return_value = (None, False) - self.job_actions.sendToGroup(rpcObjects=[job]) + show = opencue.wrappers.show.Show() + findShowMock.return_value = show + show.getGroups = mock.Mock(return_value=[group]) + move_itemsMock.return_value = (None, False) + move_dialogMock.reject() group.reparentJobs.assert_not_called() From 101cdbf1a5c0bd0d6cff50ce819cd1f2aaf9fd46 Mon Sep 17 00:00:00 2001 From: prOOrc Date: Wed, 11 May 2022 21:04:08 +0300 Subject: [PATCH 192/277] [cuebot] Fix gRPC GetHost method. (#1148) --- .../src/main/java/com/imageworks/spcue/servant/ManageHost.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageHost.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageHost.java index 77998e416..7728c4cb1 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageHost.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageHost.java @@ -140,7 +140,7 @@ public void getHost(HostGetHostRequest request, StreamObserver responseObserver) { try { responseObserver.onNext(HostGetHostResponse.newBuilder() - .setHost(whiteboard.findHost(request.getId())) + .setHost(whiteboard.getHost(request.getId())) .build()); responseObserver.onCompleted(); } catch (EmptyResultDataAccessException e) { From b414abc6edd428e3a2c8c30845fa6d1143f44b99 Mon Sep 17 00:00:00 2001 From: Akim Ruslanov Date: Tue, 17 May 2022 10:32:34 -0700 Subject: [PATCH 193/277] Add scheduled task to update show statuses (#1151) --- .../com/imageworks/spcue/dao/ShowDao.java | 6 +++ .../spcue/dao/postgres/ShowDaoJdbc.java | 37 +++++++++++++++++++ .../spcue/service/AdminManager.java | 1 + .../spcue/service/AdminManagerService.java | 5 +++ .../spring/applicationContext-service.xml | 14 +++++++ cuebot/src/main/resources/opencue.properties | 3 ++ 6 files changed, 66 insertions(+) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/ShowDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/ShowDao.java index f0cdcbba7..b12c0b097 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/ShowDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/ShowDao.java @@ -146,5 +146,11 @@ public interface ShowDao { * @param emails */ void updateShowCommentEmail(ShowInterface s, String[] emails); + + /** + * Scheduled task to update shows. Set show as inactive if it has at + * least 1 job in job_history service th + */ + void updateShowsStatus(); } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java index add49a178..2c4182122 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java @@ -21,8 +21,13 @@ import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Arrays; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; import org.springframework.dao.DataAccessException; import org.springframework.dao.EmptyResultDataAccessException; import org.springframework.jdbc.core.RowMapper; @@ -35,6 +40,8 @@ import com.imageworks.spcue.util.SqlUtil; public class ShowDaoJdbc extends JdbcDaoSupport implements ShowDao { + @Autowired + private Environment env; private static final RowMapper SHOW_MAPPER = new RowMapper() { @@ -228,6 +235,36 @@ public void updateShowCommentEmail(ShowInterface s, String[] email) { StringUtils.join(email, ","), s.getShowId()); } + @Override + public void updateShowsStatus() { + Stream protectedShowsRaw = Arrays.stream(env.getProperty("protected_shows", String.class).split(",")); + String protectedShows = protectedShowsRaw.map(show -> "'" + show + "'") + .collect(Collectors.joining(",")); + getJdbcTemplate().update("UPDATE " + + "show " + + "SET " + + "b_active=false " + + "WHERE " + + "pk_show NOT IN (" + + "SELECT " + + "pk_show " + + "FROM (" + + "SELECT " + + "pk_show, count(pk_job) " + + "FROM " + + "job_history " + + "WHERE " + + "(DATE_PART('days', NOW()) - DATE_PART('days', dt_last_modified)) < 30 " + + "GROUP BY " + + "pk_show " + + "HAVING COUNT(pk_job)>0 " + + ") pk_show" + + ") " + + "AND " + + "str_name NOT IN (?)", + protectedShows); + } + @Override public void updateFrameCounters(ShowInterface s, int exitStatus) { String col = "int_frame_success_count = int_frame_success_count + 1"; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/AdminManager.java b/cuebot/src/main/java/com/imageworks/spcue/service/AdminManager.java index f6d2220df..755b39317 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/AdminManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/AdminManager.java @@ -40,6 +40,7 @@ public interface AdminManager { ShowEntity getShowEntity(String id); void setShowActive(ShowInterface show, boolean value); void updateShowCommentEmail(ShowInterface s, String[] emails); + void updateShowsStatus(); /* * Facilities diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java index f99107270..a824dfb09 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java @@ -135,6 +135,11 @@ public void updateShowCommentEmail(ShowInterface s, String[] emails) { showDao.updateShowCommentEmail(s, emails); } + @Override + public void updateShowsStatus() { + showDao.updateShowsStatus(); + } + public SubscriptionInterface createSubscription(SubscriptionEntity sub) { subscriptionDao.insertSubscription(sub); return sub; diff --git a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml index 12e0889d8..6c3a9f2bd 100644 --- a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml +++ b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml @@ -543,6 +543,19 @@ + + + + + + + + + + + + + false @@ -557,6 +570,7 @@ + diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index e1c4fa402..6d541bd00 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -135,3 +135,6 @@ maintenance.auto_delete_down_hosts=false # Set hostname/IP of the smtp host. Will be used for mailing smtp_host=smtp + +# These shows won't be deactivated by the scheduled tasks +protected_shows=pipe,swtest,edu From 681bf7a073bdd9910cc5f497eb95320cddd795d6 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 17 May 2022 16:51:41 -0400 Subject: [PATCH 194/277] Notes from last two TSC meetings. (#1094) --- tsc/meetings/2022-01-19.md | 94 ++++++++++++++++++++++++++++++++++ tsc/meetings/2022-02-16.md | 102 +++++++++++++++++++++++++++++++++++++ 2 files changed, 196 insertions(+) create mode 100644 tsc/meetings/2022-01-19.md create mode 100644 tsc/meetings/2022-02-16.md diff --git a/tsc/meetings/2022-01-19.md b/tsc/meetings/2022-01-19.md new file mode 100644 index 000000000..6d0cd0ab6 --- /dev/null +++ b/tsc/meetings/2022-01-19.md @@ -0,0 +1,94 @@ +# OpenCue TSC Meeting Notes 19 January 2022 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Completed 2022 Goals + * Nothing yet. +* Ongoing work + * CueGUI memory leak + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1025 + * SPI: still researching and testing their fix. + * Scheduling fixes + * Thread pool properties + * Two versions: + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1008 + * SPI version https://github.com/AcademySoftwareFoundation/OpenCue/pull/1035 + * PRs need review. + * Let's start with 1035, then merge 1008 to fix conflicts to see what in 1008 is still + useful, if anything. + * Schedule dispatch frame + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1012 + * Adds database fields to eliminate multiple cuebots handing out the same frames. + * Might not want to merge while larger scheduler redesign is going on. + * RQD change, adding child proc info into log, storing in database + * For understanding memory usage of complex jobs + * Entering testing soon. + * Replaced embedded Postgres server. + * log4j update + * Was blocked on embedded Postgres server, now good to resume. + * Should post announcement to opencue-users before merging. +* 2022 Goals + * New user UX + * Publish PyPI packages + * Design + doc: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/ + * Proposal for config file + standardization: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/edit#heading=h.v85uo9klwqft + * Config cleanup for pycue done, tested on macOS and Windows, merged. + * pyoutline, cuesubmit config cleanup review done, ready to merge. + * CueGUI/RQD still to do. + * Fix cuebot:latest tag for sandbox setup + * Proposal: publish Docker images for every new patch version (i.e. every new commit to + master) + * Proposal accepted. + * No progress implementing this yet. + * Related: for PyPI design we may want to also publish packages for every new patch + version. + * Docs refresh + * User guide refresh + * Split into "install from latest release" and "install from master" + * Needed to assist transition to publish Docker images on all commits + * No progress. + * API reference + * We have the ability to generate HTML from any commit, but we need a new CI pipeline to + build and publish this on opencue.io for new releases. + * Proposal: for now, highlight existing docs for building reference locally + * Proposal accepted. + * Brian: cleaned up local build process, sent email with doc + link: https://www.opencue.io/contributing/opencue/build-docs/ + * pyoutline examples + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/177 + * No progress yet. + * User guide for CueGUI + * No progress. + * Configuration guides for Python/Cuebot/RQD + * In progress, will be done as part of PyPI work. + * Improve scheduler logic + * Diego has volunteered to own this. + * Email reply from Matt, looped in others. + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1001 + * Other PRs with improvements — see ongoing work above. + * Prototype in progress. + * Expanded Cloud functionality + * Finalize GSoC plugin + * Test Azure support + * Add AWS support + * Design expanded functionality e.g. autoscaling for work in the queue. + * Last update from Greg: started to test / clean up azure support. Looking good so far. + * Terraform scripts? + * Let's look at the Azure version of this + * Generic k8s setup and cuebot pool size suggestions + * Let's write up a github issue to discuss requirements + * User permissions + * Need a github issue for discussion. + * Added initial + thoughts: https://github.com/AcademySoftwareFoundation/OpenCue/issues/218#issuecomment-1016904933 + * Need more research into integration with Active Directory. + * Will try to find an example. + * Expand DCC plugins + * No progress. + * High priority: Blender, Houdini + * Important for wider user adoption + * Worth writing to the user group to see what folks have already? diff --git a/tsc/meetings/2022-02-16.md b/tsc/meetings/2022-02-16.md new file mode 100644 index 000000000..ebe212c29 --- /dev/null +++ b/tsc/meetings/2022-02-16.md @@ -0,0 +1,102 @@ +# OpenCue TSC Meeting Notes 16 February 2022 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Completed 2022 Goals + * Nothing yet. +* ASWF open source forum +* Ongoing work + * CueGUI memory leak + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1025 + * What we know: bug in job monitor tree creates unnecessary refreshing. Probably present in + all tree plugins, but most apparent when monitoring 100s/1000s of jobs. + * SPI: researching and testing their fix. + * Scheduling fixes + * Thread pool properties + * Two versions: + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1008 + * SPI version https://github.com/AcademySoftwareFoundation/OpenCue/pull/1035 + * Let's start with 1035, then merge 1008 to fix conflicts to see what in 1008 is still + useful, if anything. + * Initial review of 1035 done. Pretty close, a few last things to wrap up. + * RQD change, adding child proc info into log, storing in database + * For understanding memory usage of complex jobs. + * Testing ongoing, feedback has been positive so far. + * Avoiding complex merges from upstream. + * Problem: users will maintain their own OpenCue forks with many preferred customizations. + When it comes time to merge in changes from upstream, a complex merge is needed. This is + time consuming and results in long gaps between upstream merges. + * How do others solve this? + * Suggestion: let's identify the various areas needing customization, and discuss how we can + generalize a solution by creating options/plugins/hooks so customization can live outside + the main codebase. + * SPI to start compiling a list of these locations. +* 2022 Goals + * New user UX + * Publish PyPI packages + * Design + doc: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/ + * Proposal for config file + standardization: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/edit#heading=h.v85uo9klwqft + * Config cleanup for pycue done, tested on macOS and Windows, merged. + * pyoutline, cuesubmit config cleanup review done, ready to merge. + * CueGUI/RQD still to do. + * Fix cuebot:latest tag for sandbox setup + * Proposal: publish Docker images for every new patch version (i.e. every new commit to + master) + * Proposal accepted. + * No progress implementing this yet. + * Related: for PyPI design we may want to also publish packages for every new patch + version. + * Docs refresh + * Tech writing help + * Discussing with John Mertic at LF and OCIO reps who are also interested. + * User guide refresh + * Split into "install from latest release" and "install from master" + * Needed to assist transition to publish Docker images on all commits + * No progress. + * API reference + * We have the ability to generate HTML from any commit, but we need a new CI pipeline to + build and publish this on opencue.io for new releases. + * Proposal: for now, highlight existing docs for building reference locally + * Proposal accepted. + * Brian: cleaned up local build process, sent email with doc + link: https://www.opencue.io/contributing/opencue/build-docs/ + * pyoutline examples + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/177 + * No progress yet. + * User guide for CueGUI + * No progress. + * Configuration guides for Python/Cuebot/RQD + * In progress, will be done as part of PyPI work. + * Improve scheduler logic + * Diego has volunteered to own this. + * Email reply from Matt, looped in others. + * Other PRs with improvements — see ongoing work above. + * Prototype in progress. + * High-level prototype details using + Redis: https://github.com/AcademySoftwareFoundation/OpenCue/issues/1001 + * Expanded Cloud functionality + * Finalize GSoC plugin + * Test Azure support + * Add AWS support + * Design expanded functionality e.g. autoscaling for work in the queue. + * Last update from Greg: started to test / clean up azure support. Looking good so far. + * Terraform scripts? + * Let's look at the Azure version of this + * Generic k8s setup and cuebot pool size suggestions + * Let's write up a github issue to discuss requirements + * https://github.com/Azure/Avere/tree/main/src/terraform/examples/vfxt/opencue + * User permissions + * Need a github issue for discussion. + * Added initial + thoughts: https://github.com/AcademySoftwareFoundation/OpenCue/issues/218#issuecomment-1016904933 + * Need more research into integration with Active Directory. + * Permissions model: proposal to be written up. + * Expand DCC plugins + * High priority: Blender, Houdini + * Important for wider user adoption. + * Worth writing to the user group to see what folks have already. + * No further progress. From 48f36cd433d2fef995c8e5a958f804d8c8d44c42 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 25 May 2022 13:28:43 -0400 Subject: [PATCH 195/277] TSC notes from last 3 meetings. (#1152) --- tsc/meetings/2022-03-02.md | 114 ++++++++++++++++++++++++++++++++++ tsc/meetings/2022-03-30.md | 121 ++++++++++++++++++++++++++++++++++++ tsc/meetings/2022-05-11.md | 123 +++++++++++++++++++++++++++++++++++++ 3 files changed, 358 insertions(+) create mode 100644 tsc/meetings/2022-03-02.md create mode 100644 tsc/meetings/2022-03-30.md create mode 100644 tsc/meetings/2022-05-11.md diff --git a/tsc/meetings/2022-03-02.md b/tsc/meetings/2022-03-02.md new file mode 100644 index 000000000..4f5e54d2e --- /dev/null +++ b/tsc/meetings/2022-03-02.md @@ -0,0 +1,114 @@ +# OpenCue TSC Meeting Notes 2 March 2022 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Completed 2022 Goals + * ASWF Open Source Forum + * Skip boring stuff like CL/issue stats, highlight items that show why it's good to be an ASWF member. + * Ideas: + * Extra resources provided through ASWF/LF: + * Tech writing resources + * CI system + * New features that benefit from TSC with varied perspectives/experience: + * Scheduler rewrite + * Cloud functionality + * Users/permissions + * Window support, feedback + * Visibility as a member, for new projects +* Ongoing work + * CueGUI memory leak + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1025 + * What we know: bug in job monitor tree creates unnecessary refreshing. Probably present in all tree plugins, + but most apparent when monitoring 100s/1000s of jobs. + * SPI: researching and testing their fix. + * Scheduling fixes + * Thread pool properties + * Two versions: + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1008 + * SPI version https://github.com/AcademySoftwareFoundation/OpenCue/pull/1035 + * Let's start with 1035, then merge 1008 to fix conflicts to see what in 1008 is still useful, if anything. + * Initial review of 1035 done. Pretty close, a few last things to wrap up. + * RQD change, adding child proc info into log, storing in database + * For understanding memory usage of complex jobs. + * Testing ongoing, feedback has been positive so far. + * Linux only right now. + * Avoiding complex merges from upstream. + * Problem: users will maintain their own OpenCue forks with many preferred customizations. When it comes time to + merge in changes from upstream, a complex merge is needed. This is time consuming and results in long gaps + between upstream merges. + * How do others solve this? + * Suggestion: let's identify the various areas needing customization, and discuss how we can generalize a + solution by creating options/plugins/hooks so customization can live outside the main codebase. + * SPI to start compiling a list of these locations. + * List in progress, will report soon. + * OS-dependent log root + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1096 +* 2022 Goals + * New user UX + * Publish PyPI packages + * Design doc: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/ + * Proposal for config file + standardization: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/edit#heading=h.v85uo9klwqft + * Config cleanup for pycue, pyoutline, cuesubmit done. + * CueGUI in progress. + * RQD still todo. + * Fix cuebot:latest tag for sandbox setup + * Proposal: publish Docker images for every new patch version (i.e. every new commit to master) + * Proposal accepted. + * No progress implementing this yet. + * Related: for PyPI design we may want to also publish packages for every new patch version. + * Docs refresh + * Tech writing help + * Discussing with John Mertic at LF and OCIO reps who are also interested. + * User guide refresh + * Split into "install from latest release" and "install from master" + * Needed to assist transition to publish Docker images on all commits + * No progress. + * API reference + * We have the ability to generate HTML from any commit, but we need a new CI pipeline to build and publish + this on opencue.io for new releases. + * Proposal: for now, highlight existing docs for building reference locally + * Proposal accepted. + * Brian: cleaned up local build process, sent email with doc + link: https://www.opencue.io/contributing/opencue/build-docs/ + * pyoutline examples + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/177 + * No progress yet. + * User guide for CueGUI + * No progress. + * Configuration guides for Python/Cuebot/RQD + * In progress, will be done as part of PyPI work. + * Improve scheduler logic + * Diego has volunteered to own this. + * Email reply from Matt, looped in others. + * Other PRs with improvements — see ongoing work above. + * Prototype in progress. + * High-level prototype details using Redis: https://github.com/AcademySoftwareFoundation/OpenCue/issues/1001 + * Expanded Cloud functionality + * Finalize GSoC plugin + * Test Azure support + * Add AWS support + * Design expanded functionality e.g. autoscaling for work in the queue. + * Last update from Greg: started to test / clean up azure support. Looking good so far. + * Terraform scripts? + * Let's look at the Azure version of this + * Generic k8s setup and cuebot pool size suggestions + * Existing Azure work on this: https://github.com/Azure/Avere/tree/main/src/terraform/examples/vfxt/opencue + * Issue for discussion: https://github.com/AcademySoftwareFoundation/OpenCue/issues/1097 + * Use Kubernetes for rqd? Kubernetes adds some overhead. A very small amount but some people care. + * A Helm chart could be a good option. Easy to use and get started. + * Let's review the Github issue, leave notes. Short, rough draft of design doc for auto-scaling would be good + for further discussion. + * User permissions + * Need a github issue for discussion. + * Added initial + thoughts: https://github.com/AcademySoftwareFoundation/OpenCue/issues/218#issuecomment-1016904933 + * Need more research into integration with Active Directory. + * Permissions model: proposal to be written up. + * Expand DCC plugins + * High priority: Blender, Houdini + * Important for wider user adoption. + * Worth writing to the user group to see what folks have already. + * No further progress. diff --git a/tsc/meetings/2022-03-30.md b/tsc/meetings/2022-03-30.md new file mode 100644 index 000000000..798fbbf89 --- /dev/null +++ b/tsc/meetings/2022-03-30.md @@ -0,0 +1,121 @@ +# OpenCue TSC Meeting Notes 30 March 2022 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Completed 2022 Goals + * +* New business + * Opencue Discord channel + * seems like a good alternative to slack/mailing lists + * let's look into free version limitations and consider it + * Nimby changes: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1106 + * supersedes older windows nimby PR, let's close out the old one with a note + * Stuck frame plugin: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1109 + * Dynamic facility / OS: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1108 + * some pipelines need to be able to have complex logic to determine these settings + * some concerns around the custom module loading + * let's post discussion to the github PR to settle on a solution + * Sandbox fix: need to have scheduled tests for some things, to account for upstream changes. + * New dispatcher query modes: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1103 +* Ongoing work + * CueGUI memory leak + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1025 + * What we know: bug in job monitor tree creates unnecessary refreshing. Probably present in all tree plugins, + but most apparent when monitoring 100s/1000s of jobs. + * SPI: researching and testing their fix. + * some improvements were successful, still needs some more work. problems redrawing when more threads are + running + * Scheduling fixes + * Thread pool properties + * Two versions: + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1008 + * SPI version https://github.com/AcademySoftwareFoundation/OpenCue/pull/1035 + * Let's start with 1035, then merge 1008 to fix conflicts to see what in 1008 is still useful, if anything. + * 1035 merged. Ready to proceed to 1008. + * RQD change, adding child proc info into log, storing in database + * For understanding memory usage of complex jobs. + * Linux-only. + * Testing ongoing, feedback has been positive so far. + * feedback still good, making some minor improvements in response to user feedback + * Avoiding complex merges from upstream. + * Problem: users will maintain their own OpenCue forks with many preferred customizations. When it comes time to + merge in changes from upstream, a complex merge is needed. This is time consuming and results in long gaps + between upstream merges. + * How do others solve this? + * Suggestion: let's identify the various areas needing customization, and discuss how we can generalize a + solution by creating options/plugins/hooks so customization can live outside the main codebase. + * SPI to start compiling a list of these locations. + * List in progress, will report soon. + * OS-dependent log root + * Issue: https://github.com/AcademySoftwareFoundation/OpenCue/issues/1096 + * PR: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1102 + * Almost ready to merge. +* 2022 Goals + * New user UX + * Publish PyPI packages + * Design doc: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/ + * Proposal for config file + standardization: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/edit#heading=h.v85uo9klwqft + * Config cleanup for pycue, pyoutline, cuesubmit done. + * CueGUI in progress. + * RQD still todo. + * Fix cuebot:latest tag for sandbox setup + * Proposal: publish Docker images for every new patch version (i.e. every new commit to master) + * Proposal accepted. + * No progress implementing this yet. + * Related: for PyPI design we may want to also publish packages for every new patch version. + * Docs refresh + * Tech writing help + * Discussing with John Mertic at LF and OCIO reps who are also interested. + * User guide refresh + * Split into "install from latest release" and "install from master" + * Needed to assist transition to publish Docker images on all commits + * No progress. + * API reference + * We have the ability to generate HTML from any commit, but we need a new CI pipeline to build and publish + this on opencue.io for new releases. + * Proposal: for now, highlight existing docs for building reference locally + * Proposal accepted. + * Brian: cleaned up local build process, sent email with doc + link: https://www.opencue.io/contributing/opencue/build-docs/ + * pyoutline examples + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/177 + * No progress yet. + * User guide for CueGUI + * No progress. + * Configuration guides for Python/Cuebot/RQD + * In progress, will be done as part of PyPI work. + * Improve scheduler logic + * Diego has volunteered to own this. + * Email reply from Matt, looped in others. + * Other PRs with improvements — see ongoing work above. + * Prototype in progress. + * High-level prototype details using Redis: https://github.com/AcademySoftwareFoundation/OpenCue/issues/1001 + * Expanded Cloud functionality + * Finalize GSoC plugin + * Test Azure support + * Add AWS support + * Design expanded functionality e.g. autoscaling for work in the queue. + * Last update from Greg: started to test / clean up azure support. Looking good so far. + * Terraform scripts? + * Let's look at the Azure version of this + * Generic k8s setup and cuebot pool size suggestions + * Existing Azure work on this: https://github.com/Azure/Avere/tree/main/src/terraform/examples/vfxt/opencue + * Issue for discussion: https://github.com/AcademySoftwareFoundation/OpenCue/issues/1097 + * Use Kubernetes for rqd? Kubernetes adds some overhead. A very small amount but some people care. + * A Helm chart could be a good option. Easy to use and get started. + * Let's review the Github issue, leave notes. Short, rough draft of design doc for auto-scaling would be good + for further discussion. + * User permissions + * Need a github issue for discussion. + * Added initial + thoughts: https://github.com/AcademySoftwareFoundation/OpenCue/issues/218#issuecomment-1016904933 + * Need more research into integration with Active Directory. + * Permissions model: proposal to be written up. + * Expand DCC plugins + * High priority: Blender, Houdini + * Important for wider user adoption. + * Worth writing to the user group to see what folks have already. + * No further progress. diff --git a/tsc/meetings/2022-05-11.md b/tsc/meetings/2022-05-11.md new file mode 100644 index 000000000..4d633af9a --- /dev/null +++ b/tsc/meetings/2022-05-11.md @@ -0,0 +1,123 @@ +# OpenCue TSC Meeting Notes 11 May 2022 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Completed 2022 Goals +* New business + * Kill dependent https://github.com/AcademySoftwareFoundation/OpenCue/pull/1115 + * AllowDeeding: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1114 + * Job monitor filter behavior: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1139 + * open comment, need to resolve before merging? + * Nimby in new thread: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1063 + * not reproducible? needs input + * SQL query adjustments: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1078 + * ready to merge? + * Sentry logging in CueGUI: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1127 + * looks ok, looking for more detail +* Ongoing work + * Footage for ASWF reel + * https://www.youtube.com/user/ImageworksVFX/videos + * Brian to follow up with ASWF + * CueGUI memory leak + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1025 + * What we know: bug in job monitor tree creates unnecessary refreshing. Probably present in all tree plugins, + but most apparent when monitoring 100s/1000s of jobs. + * SPI: researching and testing their fix. + * Scheduling fixes + * Thread pool properties + * Two versions: + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1008 + * SPI version https://github.com/AcademySoftwareFoundation/OpenCue/pull/1035 + * Let's start with 1035, then merge 1008 to fix conflicts to see what in 1008 is still useful, if anything. + * 1035 merged. Ready to proceed to 1008. + * RQD change, adding child proc info into log, storing in database + * For understanding memory usage of complex jobs. + * Linux-only. + * Testing ongoing, feedback has been positive so far. + * Avoiding complex merges from upstream. + * Problem: users will maintain their own OpenCue forks with many preferred customizations. When it comes time to + merge in changes from upstream, a complex merge is needed. This is time consuming and results in long gaps + between upstream merges. + * How do others solve this? + * Suggestion: let's identify the various areas needing customization, and discuss how we can generalize a + solution by creating options/plugins/hooks so customization can live outside the main codebase. + * SPI to start compiling a list of these locations. + * List in progress, will report soon. + * Some PRs to resolve this, will get update from Diego next time + * OS-dependent log root + * Issue: https://github.com/AcademySoftwareFoundation/OpenCue/issues/1096 + * PR: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1133 + * PR merged. + * New PR to fix log paths on the CueGUI side: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1138 +* 2022 Goals + * New user UX + * Publish PyPI packages + * Design doc: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/ + * Proposal for config file + standardization: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/edit#heading=h.v85uo9klwqft + * Config cleanup for pycue, pyoutline, cuesubmit done. + * CueGUI in progress. + * constants file in particular needs attention + * RQD still todo. + * Fix cuebot:latest tag for sandbox setup + * Proposal: publish Docker images for every new patch version (i.e. every new commit to master) + * Proposal accepted. + * No progress implementing this yet. + * Related: for PyPI design we may want to also publish packages for every new patch version. + * Docs refresh + * Tech writing help + * Discussing with John Mertic at LF and OCIO reps who are also interested. + * Need to check in with ASWF + * Let's ping Sharif to see what his plan was for publishing API reference docs + * User guide refresh + * Split into "install from latest release" and "install from master" + * Needed to assist transition to publish Docker images on all commits + * No progress. + * API reference + * We have the ability to generate HTML from any commit, but we need a new CI pipeline to build and publish + this on opencue.io for new releases. + * Proposal: for now, highlight existing docs for building reference locally + * Proposal accepted. + * Brian: cleaned up local build process, sent email with doc + link: https://www.opencue.io/contributing/opencue/build-docs/ + * pyoutline examples + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/177 + * No progress yet. + * User guide for CueGUI + * No progress. + * Configuration guides for Python/Cuebot/RQD + * In progress, will be done as part of PyPI work. + * Improve scheduler logic + * Diego has volunteered to own this. + * Email reply from Matt, looped in others. + * Other PRs with improvements — see ongoing work above. + * Prototype in progress. + * High-level prototype details using Redis: https://github.com/AcademySoftwareFoundation/OpenCue/issues/1001 + * Expanded Cloud functionality + * Finalize GSoC plugin + * Test Azure support + * Add AWS support + * Design expanded functionality e.g. autoscaling for work in the queue. + * Last update from Greg: started to test / clean up azure support. Looking good so far. + * Terraform scripts? + * Let's look at the Azure version of this + * Generic k8s setup and cuebot pool size suggestions + * Existing Azure work on this: https://github.com/Azure/Avere/tree/main/src/terraform/examples/vfxt/opencue + * Issue for discussion: https://github.com/AcademySoftwareFoundation/OpenCue/issues/1097 + * Use Kubernetes for rqd? Kubernetes adds some overhead. A very small amount but some people care. + * A Helm chart could be a good option. Easy to use and get started. + * Let's review the Github issue, leave notes. Short, rough draft of design doc for auto-scaling would be good + for further discussion. + * User permissions + * Need a github issue for discussion. + * Added initial + thoughts: https://github.com/AcademySoftwareFoundation/OpenCue/issues/218#issuecomment-1016904933 + * Need more research into integration with Active Directory. + * Permissions model: proposal to be written up. + * Expand DCC plugins + * High priority: Blender, Houdini + * Important for wider user adoption. + * Worth writing to the user group to see what folks have already. + * No further progress. From 747c6970e057e073c1f056aaf0d86dac0b16d43d Mon Sep 17 00:00:00 2001 From: prOOrc Date: Wed, 25 May 2022 20:34:03 +0300 Subject: [PATCH 196/277] Fix: Failed tests due to cache (#1112) * fix/cache-in-tests * fix/cache-in-tests review edits * removed redundant code --- .../main/java/com/imageworks/spcue/dao/DispatcherDao.java | 7 +++++++ .../imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java | 7 +++++++ .../imageworks/spcue/dispatcher/CoreUnitDispatcher.java | 2 +- .../com/imageworks/spcue/dispatcher/DispatchSupport.java | 7 +++++++ .../spcue/dispatcher/DispatchSupportService.java | 5 +++++ 5 files changed, 27 insertions(+), 1 deletion(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/DispatcherDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/DispatcherDao.java index c348bb2f4..ce74224b1 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/DispatcherDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/DispatcherDao.java @@ -188,6 +188,13 @@ enum SchedulingMode { FIFO, BALANCED } + + /** + * Clear bookableShows cache + * + * @return + */ + void clearCache(); } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java index 630ce2b0b..4c9570d9a 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java @@ -114,6 +114,8 @@ public List getShows() { private final ConcurrentHashMap bookableShows = new ConcurrentHashMap(); + public boolean testMode = false; + /** * Choose between different scheduling strategies */ @@ -432,5 +434,10 @@ public Set findLocalDispatchJobs(DispatchHost host) { return result; } + + @Override + public void clearCache() { + bookableShows.clear(); + } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java index 7da8faf1d..4d556589e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java @@ -387,6 +387,7 @@ public boolean isTestMode() { @Override public void setTestMode(boolean enabled) { testMode = enabled; + dispatchSupport.clearCache(); } /** @@ -544,4 +545,3 @@ public boolean execute() { } } } - diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java index b7c0a65aa..4accd0f8d 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java @@ -549,5 +549,12 @@ void updateProcMemoryUsage(FrameInterface frame, long rss, long maxRss, */ boolean hasPendingFrames(LayerInterface layer); + /** + * Clear bookableShows cache + * + * @return + */ + void clearCache(); + } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java index d6a30fdba..dd8f71cf0 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java @@ -689,5 +689,10 @@ public BookingDao getBookingDao() { public void setBookingDao(BookingDao bookingDao) { this.bookingDao = bookingDao; } + + @Override + public void clearCache() { + dispatcherDao.clearCache(); + } } From 4402c1ac5065c2efd346cd911c38985369695272 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Wed, 25 May 2022 15:48:32 -0700 Subject: [PATCH 197/277] Convert log paths between OSs (#1138) * Convert log paths between OSs As windows job logs are now having their path saved to the database, the GUI needs to be able to convert paths so that multiple OSs can read the logs using the GUI * pylint pass * Apply suggestions from code review Still using replace, but only targeting the first occurrence to avoid unintended substitutions Added a macos example --- cuegui/cuegui/Constants.py | 8 ++++++++ cuegui/cuegui/Utils.py | 15 +++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/cuegui/cuegui/Constants.py b/cuegui/cuegui/Constants.py index bb38e9495..a988538c3 100644 --- a/cuegui/cuegui/Constants.py +++ b/cuegui/cuegui/Constants.py @@ -136,3 +136,11 @@ 'no licenses could be found', 'killMessage'] LOG_HIGHLIGHT_WARN = ['warning', 'not found'] LOG_HIGHLIGHT_INFO = ['info:', 'rqd cmd:'] + + +LOG_ROOT_OS = { + "rhel7": "/shots", + "linux": "/shots", + "windows": "S:", + "mac": "/Users/shots" +} diff --git a/cuegui/cuegui/Utils.py b/cuegui/cuegui/Utils.py index 4c18f7233..17df4e559 100644 --- a/cuegui/cuegui/Utils.py +++ b/cuegui/cuegui/Utils.py @@ -416,8 +416,19 @@ def getResourceConfig(path=None): ################################################################################ def getFrameLogFile(job, frame): - """Get the log file associated with a frame.""" - return os.path.join(job.data.log_dir, "%s.%s.rqlog" % (job.data.name, frame.data.name)) + """Get the log file associated with a frame. Return path based on the + current OS path using Constants.LOG_ROOT_OS to translate paths.""" + my_os = platform.system().lower() + job_os = job.data.os.lower() + + log_dir = job.data.log_dir + if my_os != job_os and \ + my_os in cuegui.Constants.LOG_ROOT_OS and \ + job_os in cuegui.Constants.LOG_ROOT_OS: + log_dir = log_dir.replace(cuegui.Constants.LOG_ROOT_OS[job_os], + cuegui.Constants.LOG_ROOT_OS[my_os], 1) + + return os.path.join(log_dir, "%s.%s.rqlog" % (job.data.name, frame.data.name)) def getFrameLLU(job, frame): From 76536a6c08194976877483544a89a145664e4b35 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Wed, 25 May 2022 15:51:24 -0700 Subject: [PATCH 198/277] Fix None.shutdown bug on rqd (#1145) * Fix None.shutdown bug on rqd Besides that, also add variable args to on_interation as some versions of pynput will call this methot passing interaction arguments. * fix lint error --- rqd/rqd/rqnetwork.py | 6 +++--- rqd/rqd/rqnimby.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/rqd/rqd/rqnetwork.py b/rqd/rqd/rqnetwork.py index be0208319..9de2cf452 100644 --- a/rqd/rqd/rqnetwork.py +++ b/rqd/rqd/rqnetwork.py @@ -254,9 +254,9 @@ def start_grpc(self): def stopGrpc(self): """Stops the gRPC server.""" - self.grpcServer.shutdown() - del self.grpcServer - log.warning("Stopped grpc server") + if self.grpcServer: + self.grpcServer.shutdown() + del self.grpcServer def closeChannel(self): """Closes the gRPC channel.""" diff --git a/rqd/rqd/rqnimby.py b/rqd/rqd/rqnimby.py index e2cdbf214..4784ba90f 100644 --- a/rqd/rqd/rqnimby.py +++ b/rqd/rqd/rqnimby.py @@ -280,7 +280,8 @@ def __init__(self, rqCore): on_scroll=self.on_interaction) self.keyboard_listener = pynput.keyboard.Listener(on_press=self.on_interaction) - def on_interaction(self): + # pylint: disable=unused-argument + def on_interaction(self, *args): """ interaction detected """ self.interaction_detected = True From 34c2bded6d959e2f5dc406b872bd4f07cd8b4807 Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Fri, 27 May 2022 08:46:05 -0700 Subject: [PATCH 199/277] Create feature flag to allow deeding (#1114) * Create feature flag to allow deeding - Set AllowDeeding=1 in general config to be able to use local cores - Use Local Cores option disabled by default * Update cuegui/cuegui/FrameMonitorTree.py Fix merging conflict Co-authored-by: Diego Tavares da Silva --- cuegui/cuegui/FrameMonitorTree.py | 6 +++++- cuegui/cuegui/JobMonitorTree.py | 4 +++- cuegui/cuegui/LayerMonitorTree.py | 4 +++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/cuegui/cuegui/FrameMonitorTree.py b/cuegui/cuegui/FrameMonitorTree.py index fcbed14e1..b930d54f4 100644 --- a/cuegui/cuegui/FrameMonitorTree.py +++ b/cuegui/cuegui/FrameMonitorTree.py @@ -212,6 +212,9 @@ def __init__(self, parent): cuegui.AbstractTreeWidget.AbstractTreeWidget.__init__(self, parent) + # Used to build right click context menus + self.__menuActions = cuegui.MenuActions.MenuActions( + self, self.updateSoon, self.selectedObjects, self.getJob) self.__sortByColumnCache = {} self.ticksWithoutUpdate = 999 self.__lastUpdateTime = None @@ -876,7 +879,8 @@ def __init__(self, widget, filterSelectedLayersCallback): elif count == 2: self.__menuActions.frames().addAction(self, "xdiff2") - self.__menuActions.frames().addAction(self, "useLocalCores") + if bool(int(QtGui.qApp.settings.value("AllowDeeding", 0))): + self.__menuActions.frames().addAction(self, "useLocalCores") # pylint: disable=no-member if QtGui.qApp.applicationName() == "CueCommander": diff --git a/cuegui/cuegui/JobMonitorTree.py b/cuegui/cuegui/JobMonitorTree.py index 239aa1f2f..dc35fc521 100644 --- a/cuegui/cuegui/JobMonitorTree.py +++ b/cuegui/cuegui/JobMonitorTree.py @@ -383,7 +383,9 @@ def contextMenuEvent(self, e): self.__menuActions.jobs().addAction(menu, "emailArtist") self.__menuActions.jobs().addAction(menu, "showProgBar") self.__menuActions.jobs().addAction(menu, "viewComments") - self.__menuActions.jobs().addAction(menu, "useLocalCores") + + if bool(int(QtGui.qApp.settings.value("AllowDeeding", 0))): + self.__menuActions.jobs().addAction(menu, "useLocalCores") depend_menu = QtWidgets.QMenu("&Dependencies",self) self.__menuActions.jobs().addAction(depend_menu, "viewDepends") diff --git a/cuegui/cuegui/LayerMonitorTree.py b/cuegui/cuegui/LayerMonitorTree.py index 5b15450b6..a8665c1a0 100644 --- a/cuegui/cuegui/LayerMonitorTree.py +++ b/cuegui/cuegui/LayerMonitorTree.py @@ -21,6 +21,7 @@ from __future__ import division from PySide2 import QtCore +from PySide2 import QtGui from PySide2 import QtWidgets from opencue.exception import EntityNotFoundException @@ -231,7 +232,8 @@ def contextMenuEvent(self, e): if len(__selectedObjects) == 1: menu.addSeparator() - self.__menuActions.layers().addAction(menu, "useLocalCores") + if bool(int(QtGui.qApp.settings.value("AllowDeeding", 0))): + self.__menuActions.layers().addAction(menu, "useLocalCores") if len({layer.data.range for layer in __selectedObjects}) == 1: self.__menuActions.layers().addAction(menu, "reorder") self.__menuActions.layers().addAction(menu, "stagger") From 3893b87db78d3489e1164431739ce12e9804c411 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Fri, 27 May 2022 08:47:09 -0700 Subject: [PATCH 200/277] Improve view running procs (#1141) * Improve view running procs This changes the widget to only display info about the selected frame instead of all the running frames for the job. * pylint pass * pylint pass * Update VERSION.in Co-authored-by: Brian Cipriano Co-authored-by: Brian Cipriano --- cuegui/cuegui/FrameMonitorTree.py | 5 +++-- cuegui/cuegui/MenuActions.py | 28 +++++++++++++++++++--------- cuegui/cuegui/ProcChildren.py | 23 +++++++++++++++-------- proto/host.proto | 2 +- 4 files changed, 38 insertions(+), 20 deletions(-) diff --git a/cuegui/cuegui/FrameMonitorTree.py b/cuegui/cuegui/FrameMonitorTree.py index b930d54f4..e7ee6119c 100644 --- a/cuegui/cuegui/FrameMonitorTree.py +++ b/cuegui/cuegui/FrameMonitorTree.py @@ -349,7 +349,8 @@ def __itemSingleClickedViewLog(self, item, col): key=lambda l: int(l.split('rqlog.')[-1]), reverse=True) except ValueError: - pass + old_log_files = [] + # pylint: disable=no-member QtGui.qApp.display_log_file_content.emit([current_log_file] + old_log_files) # pylint: enable=no-member @@ -911,4 +912,4 @@ def __init__(self, widget, filterSelectedLayersCallback): self.__menuActions.frames().addAction(self, "eat") self.__menuActions.frames().addAction(self, "kill") self.__menuActions.frames().addAction(self, "eatandmarkdone") - self.__menuActions.frames().addAction(self, "viewRunning") + self.__menuActions.frames().addAction(self, "viewProcesses") diff --git a/cuegui/cuegui/MenuActions.py b/cuegui/cuegui/MenuActions.py index 058ea2a39..7bc592b8f 100644 --- a/cuegui/cuegui/MenuActions.py +++ b/cuegui/cuegui/MenuActions.py @@ -941,17 +941,27 @@ def viewLastLog(self, rpcObjects=None): else: cuegui.Utils.popupView(path) - viewRunning_info = ["View Running", None, "viewRunning"] + viewProcesses_info = ["View Processes", None, "viewProcesses"] - def viewRunning(self): + def viewProcesses(self, rpcObjects=None): """ Display a Proc's child processes Host statistics.""" - job = self._getSource() - text = "Displaying host stats for each child process for job:\n%s" % job.name() - title = "View Running Child Proc Host Stats" - procDialog = cuegui.ProcChildren.ProcChildrenDialog(job=job, - text=text, - title=title) - procDialog.exec_() + frames = self._getOnlyFrameObjects(rpcObjects) + hosts = list({frame.data.last_resource.split("/")[0] + for frame in frames if frame.data.last_resource}) + if hosts: + layers = self._getSource().getLayers() + layer = [l for l in layers if l.data.name == frames[0].data.layer_name] + + if len(layer) > 0: + job = self._getSource() + text = "Displaying host stats for each child process for job:\n%s" % job.name() + title = "View Running Child Proc Host Stats" + procDialog = cuegui.ProcChildren.ProcChildrenDialog(job=job, + layer=layer[0], + hosts=hosts, + text=text, + title=title) + procDialog.exec_() useLocalCores_info = ["Use local cores...", "Set a single frame to use the local desktop cores.", diff --git a/cuegui/cuegui/ProcChildren.py b/cuegui/cuegui/ProcChildren.py index db5574981..7a4190e88 100644 --- a/cuegui/cuegui/ProcChildren.py +++ b/cuegui/cuegui/ProcChildren.py @@ -34,26 +34,27 @@ import cuegui.Utils - - class ProcChildren(QtWidgets.QWidget): """Widget for displaying Host statistics for a Proc's child processes.""" HEADERS = ["PID", "Name", "Start Time", "Rss (KB)", "VSize (KB)", "Statm Rss (KB)", "Statm Size (KB)", "Cmd line"] - def __init__(self, job, parent=None): + def __init__(self, job, layer, hosts, parent=None): """ Initializes the list of procs for a given job to display - :param job: job Object for this item - :ptype job: opencue.wrappers.job.Job + :param job: job Object for this item (opencue.wrappers.job.Job) + :param layer: job Object for this item (opencue.wrappers.layer.Layer) + :param hosts: list of host Object for this item (List[opencue.wrappers.host.Host]) :param parent: Optional parent for this item """ QtWidgets.QWidget.__init__(self, parent) self._data = {} self._job = job + self._layer = layer + self._hosts = hosts self._model = QtGui.QStandardItemModel(self) self._model.setColumnCount(5) self._model.setHorizontalHeaderLabels(ProcChildren.HEADERS) @@ -74,7 +75,8 @@ def update(self): try: procs = opencue.api.getProcs(job=[self._job.name()], - layer=[x.name() for x in self._job.getLayers()]) + layer=[self._layer.name()], + host=self._hosts) for proc in procs: data['children_processes'] =\ childrenProc.FromString(proc.data.child_processes).children @@ -117,11 +119,15 @@ class ProcChildrenDialog(QtWidgets.QDialog): """ Dialog for displaying Host statistics for a Proc's child processes """ - def __init__(self, job, text, title, parent=None): + def __init__(self, job, layer, hosts, text, title, parent=None): """ Initializes the data to be displayed :ptype job: opencue.wrappers.job.Job :param job: job Object for this item + :ptype layer: opencue.wrappers.layer.Layer + :param layer: layer Object for this item + :ptype hosts: List[opencue.wrappers.host.Host] + :param hosts: list of hosts Object for this item :ptype text: str :param text: Description of what is being displayed :ptype title: str @@ -136,7 +142,8 @@ def __init__(self, job, text, title, parent=None): self.text = text self.title = title self.setWindowTitle(self.title) - self._childProcStats = ProcChildren(job=job, parent=parent) + self._childProcStats = ProcChildren(job, layer, hosts, parent=parent) + self.resize(920, 420) _labelText = QtWidgets.QLabel(text, self) _labelText.setWordWrap(True) diff --git a/proto/host.proto b/proto/host.proto index e2e46e65f..9a7e30917 100644 --- a/proto/host.proto +++ b/proto/host.proto @@ -385,7 +385,7 @@ message ProcSearchCriteria { // An array of job names to match. repeated string jobs = 2; - // An arra of layer names to match. + // An array of layer names to match. repeated string layers = 3; // An array of show names to match. From a929b90cc41778f3cd64d297db4224adabbdd114 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Sat, 4 Jun 2022 11:09:42 -0700 Subject: [PATCH 201/277] [cuegui] Remove invalid menu action. (#1154) --- cuegui/cuegui/JobMonitorTree.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cuegui/cuegui/JobMonitorTree.py b/cuegui/cuegui/JobMonitorTree.py index dc35fc521..0519a9e9d 100644 --- a/cuegui/cuegui/JobMonitorTree.py +++ b/cuegui/cuegui/JobMonitorTree.py @@ -381,7 +381,6 @@ def contextMenuEvent(self, e): self.__menuActions.jobs().addAction(menu, "unmonitor") self.__menuActions.jobs().addAction(menu, "view") self.__menuActions.jobs().addAction(menu, "emailArtist") - self.__menuActions.jobs().addAction(menu, "showProgBar") self.__menuActions.jobs().addAction(menu, "viewComments") if bool(int(QtGui.qApp.settings.value("AllowDeeding", 0))): From 81a272a18c298ac1a3bbcd53b2cf9d7daad03248 Mon Sep 17 00:00:00 2001 From: Akim Ruslanov Date: Sat, 4 Jun 2022 11:14:16 -0700 Subject: [PATCH 202/277] Add layer max cores. (#1125) --- VERSION.in | 2 +- .../com/imageworks/spcue/dao/LayerDao.java | 12 +++++++++- .../spcue/dao/postgres/LayerDaoJdbc.java | 7 ++++++ .../spcue/service/FilterManagerService.java | 6 ++++- .../test/service/FilterManagerTests.java | 2 +- cuegui/cuegui/FilterDialog.py | 24 +++++++++++++++---- proto/filter.proto | 9 +++++-- pycue/opencue/wrappers/filter.py | 6 +++-- 8 files changed, 55 insertions(+), 13 deletions(-) diff --git a/VERSION.in b/VERSION.in index 50653ad0a..a4d2aceeb 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.17 +0.18 diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/LayerDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/LayerDao.java index ba8295462..9343c3aa0 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/LayerDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/LayerDao.java @@ -124,7 +124,7 @@ public interface LayerDao { LayerInterface findLayer(JobInterface job, String name); /** - * update the number of cores the layer requires + * update the number of min cores the layer requires * * @param layer * @param val @@ -270,6 +270,16 @@ public interface LayerDao { */ void updateMinGpuMemory(JobInterface job, long mem, LayerType type); + /** + * Update all layers of the set type in the specified job + * with the new max cores requirement. + * + * @param job + * @param cores + * @param type + */ + void updateMaxCores(JobInterface job, int cores, LayerType type); + /** * Update all layers of the set type in the specified job * with the new min cores requirement. diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/LayerDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/LayerDaoJdbc.java index 212963519..15941a196 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/LayerDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/LayerDaoJdbc.java @@ -641,6 +641,13 @@ public void updateMinCores(JobInterface job, int cores, LayerType type) { cores, job.getJobId(), type.toString()); } + @Override + public void updateMaxCores(JobInterface job, int cores, LayerType type) { + getJdbcTemplate().update( + "UPDATE layer SET int_cores_max=? WHERE pk_job=? AND str_type=?", + cores, job.getJobId(), type.toString()); + } + @Override public void updateMinGpus(JobInterface job, int gpus, LayerType type) { getJdbcTemplate().update( diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/FilterManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/FilterManagerService.java index f27541884..6d4382714 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/FilterManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/FilterManagerService.java @@ -396,10 +396,14 @@ public boolean applyAction(ActionEntity action, JobDetail job, Context context) layerDao.updateMinMemory(job, (int) action.intValue, LayerType.RENDER); break; - case SET_ALL_RENDER_LAYER_CORES: + case SET_ALL_RENDER_LAYER_MIN_CORES: layerDao.updateMinCores(job, Convert.coresToCoreUnits(action.floatValue), LayerType.RENDER); break; + case SET_ALL_RENDER_LAYER_MAX_CORES: + layerDao.updateMaxCores(job, Convert.coresToCoreUnits(action.floatValue), LayerType.RENDER); + break; + case SET_MEMORY_OPTIMIZER: List layers = layerDao.getLayers(job); for (LayerInterface layer : layers) { diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/FilterManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/FilterManagerTests.java index 4bce07507..13da1c7de 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/FilterManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/FilterManagerTests.java @@ -359,7 +359,7 @@ public void testApplyActionSetRenderCoreLayers() { filterDao.insertFilter(f); ActionEntity a1 = new ActionEntity(); - a1.type = ActionType.SET_ALL_RENDER_LAYER_CORES; + a1.type = ActionType.SET_ALL_RENDER_LAYER_MIN_CORES; a1.filterId = f.getFilterId(); a1.valueType = ActionValueType.FLOAT_TYPE; a1.floatValue = 40f; diff --git a/cuegui/cuegui/FilterDialog.py b/cuegui/cuegui/FilterDialog.py index 73b4153da..46126050b 100644 --- a/cuegui/cuegui/FilterDialog.py +++ b/cuegui/cuegui/FilterDialog.py @@ -480,11 +480,22 @@ def createAction(self): 2) value = int(value * 1048576) - elif actionType in (opencue.api.filter_pb2.SET_ALL_RENDER_LAYER_CORES,): + elif actionType in (opencue.api.filter_pb2.SET_ALL_RENDER_LAYER_MIN_CORES,): (value, choice) = QtWidgets.QInputDialog.getDouble( self, "Create Action", - "How many cores should every render layer require?", + "How many min cores should every render layer require?", + 1, + 0.1, + 100, + 2) + value = float(value) + + elif actionType in (opencue.api.filter_pb2.SET_ALL_RENDER_LAYER_MAX_CORES,): + (value, choice) = QtWidgets.QInputDialog.getDouble( + self, + "Create Action", + "How many max cores should every render layer require?", 1, 0.1, 100, @@ -728,7 +739,8 @@ def __setValue(self, value=None): elif self.rpcObject.type() in (opencue.api.filter_pb2.SET_JOB_MAX_CORES, opencue.api.filter_pb2.SET_JOB_MIN_CORES, - opencue.api.filter_pb2.SET_ALL_RENDER_LAYER_CORES): + opencue.api.filter_pb2.SET_ALL_RENDER_LAYER_MIN_CORES, + opencue.api.filter_pb2.SET_ALL_RENDER_LAYER_MAX_CORES): value = float(widget.value()) elif self.rpcObject.type() in (opencue.api.filter_pb2.SET_ALL_RENDER_LAYER_TAGS,): @@ -765,7 +777,8 @@ def updateWidgets(self): widget.editingFinished.connect(self.__setValue) # pylint: disable=no-member elif self.rpcObject.type() in (opencue.api.filter_pb2.SET_ALL_RENDER_LAYER_MEMORY, - opencue.api.filter_pb2.SET_ALL_RENDER_LAYER_CORES): + opencue.api.filter_pb2.SET_ALL_RENDER_LAYER_MIN_CORES, + opencue.api.filter_pb2.SET_ALL_RENDER_LAYER_MAX_CORES): widget = NoWheelDoubleSpinBox(self.parent()) widget.setDecimals(2) widget.setSingleStep(.10) @@ -817,7 +830,8 @@ def updateWidgets(self): elif self.rpcObject.type() in (opencue.api.filter_pb2.SET_ALL_RENDER_LAYER_TAGS,): self.__widgets["ActionValue"].setText(self.rpcObject.value()) - elif self.rpcObject.type() in (opencue.api.filter_pb2.SET_ALL_RENDER_LAYER_CORES, + elif self.rpcObject.type() in (opencue.api.filter_pb2.SET_ALL_RENDER_LAYER_MIN_CORES, + opencue.api.filter_pb2.SET_ALL_RENDER_LAYER_MAX_CORES, opencue.api.filter_pb2.SET_JOB_MAX_CORES, opencue.api.filter_pb2.SET_JOB_MIN_CORES): self.__widgets["ActionValue"].setValue(float(str(self.rpcObject.value()))) diff --git a/proto/filter.proto b/proto/filter.proto index fdfae12e6..184097782 100644 --- a/proto/filter.proto +++ b/proto/filter.proto @@ -96,10 +96,15 @@ enum ActionType { SET_ALL_RENDER_LAYER_TAGS = 6; // Sets all layer minimum memory for any layer with the type "Render" SET_ALL_RENDER_LAYER_MEMORY = 7; - // Sets all min cores for any layer with the type "Render" - SET_ALL_RENDER_LAYER_CORES = 8; + // This field is deprecated, use SET_ALL_RENDER_LAYER_MIN_CORES and + // SET_ALL_RENDER_LAYER_MAX_CORES instead. + SET_ALL_RENDER_LAYER_CORES = 8 [deprecated = true]; // Set memory optimizer SET_MEMORY_OPTIMIZER = 9; + // Sets all min cores for any layer with the type "Render" + SET_ALL_RENDER_LAYER_MIN_CORES = 10; + // Sets all max cores for any layer with the type "Render" + SET_ALL_RENDER_LAYER_MAX_CORES = 11; }; enum ActionValueType { diff --git a/pycue/opencue/wrappers/filter.py b/pycue/opencue/wrappers/filter.py index 45e3949ca..e18194a1c 100644 --- a/pycue/opencue/wrappers/filter.py +++ b/pycue/opencue/wrappers/filter.py @@ -270,7 +270,8 @@ class ActionType(enum.IntEnum): SET_JOB_PRIORITY = filter_pb2.SET_JOB_PRIORITY SET_ALL_RENDER_LAYER_TAGS = filter_pb2.SET_ALL_RENDER_LAYER_TAGS SET_ALL_RENDER_LAYER_MEMORY = filter_pb2.SET_ALL_RENDER_LAYER_MEMORY - SET_ALL_RENDER_LAYER_CORES = filter_pb2.SET_ALL_RENDER_LAYER_CORES + SET_ALL_RENDER_LAYER_MIN_CORES = filter_pb2.SET_ALL_RENDER_LAYER_MIN_CORES + SET_ALL_RENDER_LAYER_MAX_CORES = filter_pb2.SET_ALL_RENDER_LAYER_MAX_CORES SET_MEMORY_OPTIMIZER = filter_pb2.SET_MEMORY_OPTIMIZER class ActionValueType(enum.IntEnum): @@ -390,7 +391,8 @@ def setTypeAndValue(self, actionType, value): elif actionType in (filter_pb2.SET_JOB_MIN_CORES, filter_pb2.SET_JOB_MAX_CORES, - filter_pb2.SET_ALL_RENDER_LAYER_CORES): + filter_pb2.SET_ALL_RENDER_LAYER_MIN_CORES, + filter_pb2.SET_ALL_RENDER_LAYER_MAX_CORES): self.data.float_value = float(value) self.data.value_type = filter_pb2.FLOAT_TYPE From 8c2c4acf0c5c728220b5ad526b522e0a65786ad9 Mon Sep 17 00:00:00 2001 From: roulaoregan-spi <65304316+roulaoregan-spi@users.noreply.github.com> Date: Wed, 6 Jul 2022 12:25:15 -0700 Subject: [PATCH 203/277] [cuebot] Fix PSQL Booking Next Frame RE word boundary match (#1078) --- .../spcue/dao/postgres/DispatchQuery.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java index ee95feb82..865472d1a 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java @@ -76,7 +76,7 @@ public class DispatchQuery { "AND layer.int_gpu_mem_min BETWEEN ? AND ? " + "AND job_resource.int_cores + layer.int_cores_min < job_resource.int_max_cores " + "AND job_resource.int_gpus + layer.int_gpus_min < job_resource.int_max_gpus " + - "AND host.str_tags ~* ('(?x)' || layer.str_tags) " + + "AND host.str_tags ~* ('(?x)' || layer.str_tags || '\\y') " + "AND host.str_name = ? " + "AND layer.pk_layer IN (" + "SELECT " + @@ -164,7 +164,7 @@ public class DispatchQuery { "AND layer.int_gpus_min <= ? " + "AND layer.int_gpu_mem_min BETWEEN ? AND ? " + "AND job_resource.int_cores + layer.int_cores_min <= job_resource.int_max_cores " + - "AND host.str_tags ~* ('(?x)' || layer.str_tags) " + + "AND host.str_tags ~* ('(?x)' || layer.str_tags || '\\y') " + "AND host.str_name = ? " + ") AS t1 ) AS t2 WHERE rank < ?"; @@ -368,7 +368,7 @@ private static final String replaceQueryForFifo(String query) { "AND " + "l.int_gpu_mem_min = ? " + "AND " + - "h.str_tags ~* ('(?x)' || l.str_tags) " + + "h.str_tags ~* ('(?x)' || l.str_tags || '\\y') " + "AND " + "h.str_name = ? " + "AND " + @@ -471,7 +471,7 @@ private static final String replaceQueryForFifo(String query) { "AND " + "l.int_gpu_mem_min = ? " + "AND " + - "h.str_tags ~* ('(?x)' || l.str_tags) " + + "h.str_tags ~* ('(?x)' || l.str_tags || '\\y') " + "AND " + "h.str_name = ? " + "AND " + @@ -593,7 +593,7 @@ private static final String replaceQueryForFifo(String query) { "l.pk_layer " + "FROM " + "layer l " + - "JOIN host h ON (h.str_tags ~* ('(?x)' || l.str_tags) AND h.str_name = ?) " + + "JOIN host h ON (h.str_tags ~* ('(?x)' || l.str_tags || '\\y') AND h.str_name = ?) " + "LEFT JOIN layer_limit ON layer_limit.pk_layer = l.pk_layer " + "LEFT JOIN limit_record ON limit_record.pk_limit_record = layer_limit.pk_limit_record " + "LEFT JOIN (" + @@ -711,7 +711,7 @@ private static final String replaceQueryForFifo(String query) { "l.pk_layer " + "FROM " + "layer l " + - "JOIN host h ON (h.str_tags ~* ('(?x)' || l.str_tags) AND h.str_name = ?) " + + "JOIN host h ON (h.str_tags ~* ('(?x)' || l.str_tags || '\\y') AND h.str_name = ?) " + "LEFT JOIN layer_limit ON layer_limit.pk_layer = l.pk_layer " + "LEFT JOIN limit_record ON limit_record.pk_limit_record = layer_limit.pk_limit_record " + "LEFT JOIN (" + @@ -1045,7 +1045,7 @@ private static final String replaceQueryForFifo(String query) { "l.pk_layer " + "FROM " + "layer l " + - "JOIN host h ON (h.str_tags ~* ('(?x)' || l.str_tags) AND h.str_name = ?) " + + "JOIN host h ON (h.str_tags ~* ('(?x)' || l.str_tags || '\\y') AND h.str_name = ?) " + "LEFT JOIN layer_limit ON layer_limit.pk_layer = l.pk_layer " + "LEFT JOIN limit_record ON limit_record.pk_limit_record = layer_limit.pk_limit_record " + "LEFT JOIN (" + @@ -1163,7 +1163,7 @@ private static final String replaceQueryForFifo(String query) { "l.pk_layer " + "FROM " + "layer l " + - "JOIN host h ON (h.str_tags ~* ('(?x)' || l.str_tags) AND h.str_name = ?) " + + "JOIN host h ON (h.str_tags ~* ('(?x)' || l.str_tags || '\\y') AND h.str_name = ?) " + "LEFT JOIN layer_limit ON layer_limit.pk_layer = l.pk_layer " + "LEFT JOIN limit_record ON limit_record.pk_limit_record = layer_limit.pk_limit_record " + "LEFT JOIN (" + From de10306aea5a78f535e104c3a30b9758c70d286b Mon Sep 17 00:00:00 2001 From: Fermi Perumal Date: Wed, 6 Jul 2022 12:34:09 -0700 Subject: [PATCH 204/277] Add error handling for host allocation failures (#1149) --- cuegui/cuegui/MenuActions.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/cuegui/cuegui/MenuActions.py b/cuegui/cuegui/MenuActions.py index 7bc592b8f..2cd284f6d 100644 --- a/cuegui/cuegui/MenuActions.py +++ b/cuegui/cuegui/MenuActions.py @@ -1575,10 +1575,28 @@ def changeAllocation(self, rpcObjects=None): self._caller, title, body, sorted(allocations.keys()), 0, False) if choice: allocation = allocations[str(allocationName)] + error_hosts = [] for host in hosts: - self.cuebotCall(host.setAllocation, - "Set Allocation on %s Failed" % host.data.name, - allocation) + # pylint: disable=broad-except + try: + self.cuebotCall(host.setAllocation, + "Set Allocation on %s Failed" % host.data.name, + allocation) + except Exception as e: + # Handle allocation modification errors separately + # pylint: disable=no-member + if (hasattr(e, "details") and + "EntityModificationError" in str(e.details())): + error_hosts.append(host.name()) + else: + raise + if error_hosts: + error_msg = "{hosts} not moved.\nHosts with reserved cores " \ + "cannot be moved between allocations." + QtWidgets.QMessageBox.warning(self._caller, + "Warning", + error_msg.format(hosts=", ".join(error_hosts)), + QtWidgets.QMessageBox.Ok) self._update() setRepair_info = ["Set Repair State", None, "configure"] From 132569b090899e45cd8f41ccec8240866e6fdcc9 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Fri, 8 Jul 2022 10:19:21 -0700 Subject: [PATCH 205/277] Fix bug affecting REBOOT_WHEN_IDLE hosts (#1153) When a host marked as REBOOT_WHEN_IDLE was still reporting a running frame, it would automatically revert the host's status to UP. The logic has been fixed to only trigger the status update on boot reports. --- .../spcue/dispatcher/HostReportHandler.java | 53 +++++++++---------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java index 59c6b1ced..a63da07e8 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java @@ -155,7 +155,7 @@ public void handleHostReport(HostReport report, boolean isBoot) { rhost.getLoad(), new Timestamp(rhost.getBootTime() * 1000l), rhost.getAttributesMap().get("SP_OS")); - changeHardwareState(host, report.getHost().getState()); + changeHardwareState(host, report.getHost().getState(), isBoot); changeNimbyState(host, report.getHost()); /** @@ -304,46 +304,45 @@ else if (!dispatchSupport.isCueBookable(host)) { * * If a host pings in with a different hardware state than what * is currently in the DB, the state is updated. If the hardware - * state is Rebooting RebootWhenIdle, then state can only be + * state is Rebooting or RebootWhenIdle, then state can only be * updated with a boot report. If the state is Repair, then state is * never updated via RQD. * * @param host * @param reportState + * @param isBoot */ private void changeHardwareState(DispatchHost host, - HardwareState reportState) { + HardwareState reportState, boolean isBoot) { - /* - * If the states are the same there is no reason - * to do this update. - */ + + // If the states are the same there is no reason to do this update. if (host.hardwareState.equals(reportState)) { return; } - /* - * Do not change the state of the host if its in a - * repair state. Removing the repair state must - * be done manually. - */ - if (host.hardwareState.equals(HardwareState.REPAIR)) { - return; - } + switch (host.hardwareState) { + case DOWN: + hostManager.setHostState(host, HardwareState.UP); + host.hardwareState = HardwareState.UP; + break; + case REBOOTING: + case REBOOT_WHEN_IDLE: + // Rebooting hosts only change to UP when processing a boot report + if (isBoot) { + hostManager.setHostState(host, HardwareState.UP); + host.hardwareState = HardwareState.UP; + } + break; + case REPAIR: + // Do not change the state of the host if its in a repair state. + break; + default: + hostManager.setHostState(host, reportState); + host.hardwareState = reportState; + break; - /* - * Hosts in these states always change to Up. - */ - if (reportState.equals(HardwareState.UP) && EnumSet.of(HardwareState.DOWN, - HardwareState.REBOOTING, - HardwareState.REBOOT_WHEN_IDLE).contains(host.hardwareState)) { - hostManager.setHostState(host, HardwareState.UP); } - else { - hostManager.setHostState(host, reportState); - } - - host.hardwareState = reportState; } /** From 3f974d291a6321936fd02b98946f3158d9736aa7 Mon Sep 17 00:00:00 2001 From: Alexis Oblet Date: Fri, 8 Jul 2022 19:19:43 +0200 Subject: [PATCH 206/277] [cuegui] fix job monitor filter behavior (#1139) * cuegui: job monitor fix call to undefined member Signed-off-by: Oblet Alexis * cuegui: job monitor fix datetime python import Signed-off-by: Oblet Alexis * cuegui: monitor job search - allow regex when searching Signed-off-by: Oblet Alexis * cuegui: monitor jobs search - remove items before each search Signed-off-by: Oblet Alexis * cuegui: monitor job - save loadFinished plugin settings Signed-off-by: Oblet Alexis Co-authored-by: Diego Tavares da Silva --- cuegui/cuegui/JobMonitorTree.py | 1 - cuegui/cuegui/plugins/MonitorJobsPlugin.py | 11 ++++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cuegui/cuegui/JobMonitorTree.py b/cuegui/cuegui/JobMonitorTree.py index 0519a9e9d..a35d02897 100644 --- a/cuegui/cuegui/JobMonitorTree.py +++ b/cuegui/cuegui/JobMonitorTree.py @@ -300,7 +300,6 @@ def addJob(self, job, timestamp=None, loading_from_config=False): for j in self.__reverseDependents: if j in self.__load: del self.__load[j] - finally: self.ticksLock.unlock() diff --git a/cuegui/cuegui/plugins/MonitorJobsPlugin.py b/cuegui/cuegui/plugins/MonitorJobsPlugin.py index 8b704b9ef..94b081b13 100644 --- a/cuegui/cuegui/plugins/MonitorJobsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorJobsPlugin.py @@ -22,7 +22,7 @@ from builtins import str from builtins import map -from datetime import datetime +import datetime import re import weakref @@ -89,9 +89,12 @@ def __init__(self, parent): ("columnWidths", self.jobMonitor.getColumnWidths, self.jobMonitor.setColumnWidths), - ("columnOrder", + ("columnOrder", self.jobMonitor.getColumnOrder, self.jobMonitor.setColumnOrder), + ("loadFinished", + self.__loadFinishedJobsCheckBox.isChecked, + self.__loadFinishedJobsCheckBox.setChecked), ("grpDependentCb", self.getGrpDependent, self.setGrpDependent), @@ -194,13 +197,15 @@ def _regexLoadJobsHandle(self): substring = str(self.__regexLoadJobsEditBox.text()).strip() load_finished_jobs = self.__loadFinishedJobsCheckBox.isChecked() + self.jobMonitor.removeAllItems() + if cuegui.Utils.isStringId(substring): # If a uuid is provided, load it self.jobMonitor.addJob(substring) elif load_finished_jobs or re.search( r"^([a-z0-9_]+)\-([a-z0-9\.]+)\-", substring, re.IGNORECASE): # If show and shot is provided, or if "load finished" checkbox is checked, load all jobs - for job in opencue.api.getJobs(substr=[substring], include_finished=True): + for job in opencue.api.getJobs(regex=[substring], include_finished=True): self.jobMonitor.addJob(job) else: # Otherwise, just load current matching jobs (except for the empty string) From 6fafd62947f4d63951353e48478882ef18b6a619 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Wed, 13 Jul 2022 12:22:46 -0700 Subject: [PATCH 207/277] [rqd] Core affinity for cache optimization (#1171) When possible, try to book frames from the same Layer on the same core to leverage shared cache --- VERSION.in | 2 +- .../spcue/dao/postgres/ProcDaoJdbc.java | 2 + cuegui/cuegui/ProcChildren.py | 2 +- proto/report.proto | 7 + pycue/opencue/wrappers/host.py | 2 +- rqd/rqd/rqcore.py | 9 + rqd/rqd/rqmachine.py | 191 +++++++++++------- rqd/tests/rqmachine_tests.py | 96 ++++++++- 8 files changed, 228 insertions(+), 83 deletions(-) diff --git a/VERSION.in b/VERSION.in index a4d2aceeb..caa4836d8 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.18 +0.19 diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java index b68f8a74c..32c454e0f 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java @@ -29,6 +29,8 @@ import java.util.List; import java.util.Map; +import com.imageworks.spcue.dispatcher.AbstractDispatcher; +import org.apache.log4j.Logger; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.RowMapper; import org.springframework.jdbc.core.support.JdbcDaoSupport; diff --git a/cuegui/cuegui/ProcChildren.py b/cuegui/cuegui/ProcChildren.py index 7a4190e88..5faa0415e 100644 --- a/cuegui/cuegui/ProcChildren.py +++ b/cuegui/cuegui/ProcChildren.py @@ -75,7 +75,7 @@ def update(self): try: procs = opencue.api.getProcs(job=[self._job.name()], - layer=[self._layer.name()], + layer=[x.name() for x in self._job.getLayers()], host=self._hosts) for proc in procs: data['children_processes'] =\ diff --git a/proto/report.proto b/proto/report.proto index 78c8e4da6..441f810c3 100644 --- a/proto/report.proto +++ b/proto/report.proto @@ -36,6 +36,13 @@ message CoreDetail { int32 idle_cores = 2; int32 locked_cores = 3; int32 booked_cores = 4; + //map + map reserved_cores = 5; +} + +message CoreId { + repeated int64 coreid = 1; + } message FrameCompleteReport { diff --git a/pycue/opencue/wrappers/host.py b/pycue/opencue/wrappers/host.py index 83dac252f..aa54d44d0 100644 --- a/pycue/opencue/wrappers/host.py +++ b/pycue/opencue/wrappers/host.py @@ -279,7 +279,7 @@ def coresReserved(self): :rtype: float :return: number of cores reserved """ - return self.data.cores - self.data.idle_ores + return self.data.cores - self.data.idle_cores def coresIdle(self): """Returns the number of cores the host currently has idel. diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index 48ef7ccc9..cb74f9d34 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -593,6 +593,7 @@ def __init__(self, optNimbyoff=False): idle_cores=0, locked_cores=0, booked_cores=0, + reserved_cores=[], ) self.nimby = rqd.rqnimby.NimbyFactory.getNimby(self) @@ -732,6 +733,14 @@ def deleteFrame(self, frameId): try: if frameId in self.__cache: del self.__cache[frameId] + # pylint: disable=no-member + if not self.__cache and self.cores.reserved_cores: + # pylint: disable=no-member + log.error( + 'No running frames but reserved_cores is not empty: %s', + self.cores.reserved_cores) + # pylint: disable=no-member + self.cores.reserved_cores.clear() finally: self.__threadLock.release() diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index bed78fdef..37e1f5771 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -79,9 +79,16 @@ def __init__(self, rqCore, coreInfo): """ self.__rqCore = rqCore self.__coreInfo = coreInfo - self.__tasksets = set() self.__gpusets = set() + # A dictionary built from /proc/cpuinfo containing + # { : { : set([, , ...]), ... }, ... } + self.__procs_by_physid_and_coreid = {} + + # A reverse mapping of the above. + # { : (, ), ... } + self.__physid_and_coreid_by_proc = {} + if platform.system() == 'Linux': self.__vmstat = rqd.rqswap.VmStat() @@ -103,8 +110,8 @@ def __init__(self, rqCore, coreInfo): self.__pidHistory = {} - self.setupHT() self.setupGpu() + self.setupTaskset() def isNimbySafeToRunJobs(self): """Returns False if nimby should be triggered due to resource limits""" @@ -207,17 +214,9 @@ def __updateGpuAndLlu(self, frame): stat = os.stat(frame.runFrame.log_dir_file).st_mtime frame.lluTime = int(stat) - def _getFields(self, pidFilePath): - fields = [] - - try: - with open(pidFilePath, "r") as statFile: - fields = statFile.read().split() - # pylint: disable=broad-except - except Exception: - log.warning("Not able to read pidFilePath: %s", pidFilePath) - - return fields + def _getStatFields(self, pidFilePath): + with open(pidFilePath, "r") as statFile: + return [None, None] + statFile.read().rsplit(")", 1)[-1].split() def rssUpdate(self, frames): """Updates the rss and maxrss for all running frames""" @@ -247,10 +246,8 @@ def rssUpdate(self, frames): for pid in os.listdir("/proc"): if pid.isdigit(): try: - with open(rqd.rqconstants.PATH_PROC_PID_STAT - .format(pid), "r") as statFile: - statFields = [None, None] + statFile.read().rsplit(")", 1)[-1].split() - + statFields = self._getStatFields(rqd.rqconstants.PATH_PROC_PID_STAT + .format(pid)) pids[pid] = { "name": statFields[1], "state": statFields[2], @@ -272,24 +269,21 @@ def rssUpdate(self, frames): p = psutil.Process(int(pid)) pids[pid]["cmd_line"] = p.cmdline() - try: - # 2. Collect Statm file: /proc/[pid]/statm (same as status vsize in kb) - # - size: "total program size" - # - rss: inaccurate, similar to VmRss in /proc/[pid]/status - child_statm_fields = self._getFields( - rqd.rqconstants.PATH_PROC_PID_STATM.format(pid)) - pids[pid]['statm_size'] = \ - int(re.search("\\d+", child_statm_fields[0]).group()) \ - if re.search("\\d+", child_statm_fields[0]) else -1 - pids[pid]['statm_rss'] = \ - int(re.search("\\d+", child_statm_fields[1]).group()) \ - if re.search("\\d+", child_statm_fields[1]) else -1 - except rqd.rqexceptions.RqdException as e: - log.warning("Failed to read statm file: %s", e) + # 2. Collect Statm file: /proc/[pid]/statm (same as status vsize in kb) + # - size: "total program size" + # - rss: inaccurate, similar to VmRss in /proc/[pid]/status + child_statm_fields = self._getStatFields( + rqd.rqconstants.PATH_PROC_PID_STATM.format(pid)) + pids[pid]['statm_size'] = \ + int(re.search(r"\d+", child_statm_fields[0]).group()) \ + if re.search(r"\d+", child_statm_fields[0]) else -1 + pids[pid]['statm_rss'] = \ + int(re.search(r"\d+", child_statm_fields[1]).group()) \ + if re.search(r"\d+", child_statm_fields[1]) else -1 # pylint: disable=broad-except - except rqd.rqexceptions.RqdException: - log.exception('Failed to read stat file for pid %s', pid) + except (OSError, IOError): + log.exception('Failed to read stat/statm file for pid %s', pid) # pylint: disable=too-many-nested-blocks try: @@ -298,10 +292,8 @@ def rssUpdate(self, frames): bootTime = self.getBootTime() values = list(frames.values()) - for frame in values: if frame.pid > 0: - session = str(frame.pid) rss = 0 vsize = 0 @@ -573,33 +565,50 @@ def __initMachineStats(self, pathCpuInfo=None): mcpStat = os.statvfs(self.getTempPath()) self.__renderHost.total_mcp = mcpStat.f_blocks * mcpStat.f_frsize // KILOBYTE + # Reset mappings + self.__procs_by_physid_and_coreid = {} + self.__physid_and_coreid_by_proc = {} + # Reads static information from /proc/cpuinfo with open(pathCpuInfo or rqd.rqconstants.PATH_CPUINFO, "r") as cpuinfoFile: - singleCore = {} + currCore = {} procsFound = [] for line in cpuinfoFile: - lineList = line.strip().replace("\t","").split(": ") + lineList = line.strip().replace("\t", "").split(": ") # A normal entry added to the singleCore dictionary if len(lineList) >= 2: - singleCore[lineList[0]] = lineList[1] + currCore[lineList[0]] = lineList[1] # The end of a processor block elif lineList == ['']: # Check for hyper-threading - hyperthreadingMultiplier = (int(singleCore.get('siblings', '1')) - // int(singleCore.get('cpu cores', '1'))) + hyperthreadingMultiplier = (int(currCore.get('siblings', '1')) + // int(currCore.get('cpu cores', '1'))) __totalCores += rqd.rqconstants.CORE_VALUE - if "core id" in singleCore \ - and "physical id" in singleCore \ - and not singleCore["physical id"] in procsFound: - procsFound.append(singleCore["physical id"]) + if "core id" in currCore \ + and "physical id" in currCore \ + and not currCore["physical id"] in procsFound: + procsFound.append(currCore["physical id"]) __numProcs += 1 - elif "core id" not in singleCore: + elif "core id" not in currCore: __numProcs += 1 - singleCore = {} + + if 'physical id' in currCore and 'core id' in currCore: + # Keep track of what processors are on which core on + # which physical socket. + procid, physid, coreid = ( + currCore['processor'], + currCore['physical id'], + currCore['core id']) + self.__procs_by_physid_and_coreid \ + .setdefault(physid, {}) \ + .setdefault(coreid, set()).add(procid) + self.__physid_and_coreid_by_proc[procid] = physid, coreid + currCore = {} + # An entry without data elif len(lineList) == 1: - singleCore[lineList[0]] = "" + currCore[lineList[0]] = "" else: hyperthreadingMultiplier = 1 @@ -782,50 +791,76 @@ def __enabledHT(self): def __getHyperthreadingMultiplier(self): return int(self.__renderHost.attributes['hyperthreadingMultiplier']) - def setupHT(self): + def setupTaskset(self): """ Setup rqd for hyper-threading """ - - if self.__enabledHT(): - self.__tasksets = set(range(self.__coreInfo.total_cores // 100)) + self.__coreInfo.reserved_cores.clear() def setupGpu(self): """ Setup rqd for Gpus """ self.__gpusets = set(range(self.getGpuCount())) - def reserveHT(self, reservedCores): + def reserveHT(self, frameCores): """ Reserve cores for use by taskset taskset -c 0,1,8,9 COMMAND Not thread save, use with locking. - @type reservedCores: int - @param reservedCores: The total physical cores reserved by the frame. + @type frameCores: int + @param frameCores: The total physical cores reserved by the frame. @rtype: string @return: The cpu-list for taskset -c """ - if not self.__enabledHT(): - return None - - if reservedCores % 100: - log.debug('Taskset: Can not reserveHT with fractional cores') + if frameCores % 100: + log.warning('Taskset: Can not reserveHT with fractional cores') return None + log.warning('Taskset: Requesting reserve of %d', (frameCores // 100)) + + # Look for the most idle physical cpu. + # Prefer to assign cores from the same physical cpu. + # Spread different frames around on different physical cpus. + avail_cores = {} + avail_cores_count = 0 + reserved_cores = self.__coreInfo.reserved_cores + + for physid, cores in self.__procs_by_physid_and_coreid.items(): + for coreid in cores.keys(): + if int(physid) in reserved_cores and \ + int(coreid) in reserved_cores[int(physid)].coreid: + continue + avail_cores.setdefault(physid, set()).add(coreid) + avail_cores_count += 1 - log.debug('Taskset: Requesting reserve of %d', (reservedCores // 100)) + remaining_cores = frameCores / 100 - if len(self.__tasksets) < reservedCores // 100: + if avail_cores_count < remaining_cores: err = ('Not launching, insufficient hyperthreading cores to reserve ' - 'based on reservedCores') + 'based on frameCores (%s < %s)') \ + % (avail_cores_count, remaining_cores) log.critical(err) raise rqd.rqexceptions.CoreReservationFailureException(err) - hyperthreadingMultiplier = self.__getHyperthreadingMultiplier() tasksets = [] - for _ in range(reservedCores // 100): - core = self.__tasksets.pop() - tasksets.append(str(core)) - if hyperthreadingMultiplier > 1: - tasksets.append(str(core + self.__coreInfo.total_cores // 100)) - log.debug('Taskset: Reserving cores - %s', ','.join(tasksets)) + for physid, cores in sorted( + avail_cores.items(), + # Return the physical socket that has + # the most idle cores first. + key=lambda tup: len(tup[1]), + reverse=True): + + while remaining_cores > 0 and len(cores) > 0: + coreid = cores.pop() + # Give all the hyperthreads on this core. + # This counts as one core. + reserved_cores[int(physid)].coreid.extend([int(coreid)]) + remaining_cores -= 1 + + for procid in self.__procs_by_physid_and_coreid[physid][coreid]: + tasksets.append(procid) + + if remaining_cores == 0: + break + + log.warning('Taskset: Reserving procs - %s', ','.join(tasksets)) return ','.join(tasksets) @@ -838,13 +873,21 @@ def releaseHT(self, reservedHT): @param: The cpu-list used for taskset to release. ex: '0,8,1,9' """ - if not self.__enabledHT(): - return None - log.debug('Taskset: Releasing cores - %s', reservedHT) + + # Remove these cores from the reserved set. + # Silently ignore any that weren't really reserved or + # aren't valid core identities. + reserved_cores = self.__coreInfo.reserved_cores for core in reservedHT.split(','): - if int(core) < self.__coreInfo.total_cores // 100: - self.__tasksets.add(int(core)) + physical_id_str, core_id_str = self.__physid_and_coreid_by_proc.get(core) + physical_id = int(physical_id_str) + core_id = int(core_id_str) + + if physical_id in reserved_cores and core_id in reserved_cores[physical_id].coreid: + reserved_cores[physical_id].coreid.remove(core_id) + if len(reserved_cores[physical_id].coreid) == 0: + del reserved_cores[physical_id] def reserveGpus(self, reservedGpus): """ Reserve gpus diff --git a/rqd/tests/rqmachine_tests.py b/rqd/tests/rqmachine_tests.py index 489cc35c8..1b1bdaf4a 100644 --- a/rqd/tests/rqmachine_tests.py +++ b/rqd/tests/rqmachine_tests.py @@ -160,7 +160,6 @@ PROC_PID_CMDLINE = ' sleep 20' - @mock.patch.object(rqd.rqutil.Memoize, 'isCached', new=mock.MagicMock(return_value=False)) @mock.patch('platform.system', new=mock.MagicMock(return_value='Linux')) @mock.patch('os.statvfs', new=mock.MagicMock()) @@ -447,18 +446,103 @@ def test_getBootReport(self): self.assertEqual(25699176, bootReport.host.free_mem) def test_reserveHT(self): + """ + Total 2 physical(ph) processors with 4 cores each with 2 threads each + step1 - taskset1: Reserve 3 cores (ph1) + step2 - taskset0: Reserve 4 cores (ph0) + step3 - Release cores on taskset0 + step4 - taskset3: Reserve 2 cores (ph0) + step5 - taskset4: 3 remaining, Reserve 3 cores (ph0+ph1) + step5 - taskset5: No more cores + """ cpuInfo = os.path.join(os.path.dirname(__file__), 'cpuinfo', '_cpuinfo_shark_ht_8-4-2-2') self.fs.add_real_file(cpuInfo) self.machine.testInitMachineStats(cpuInfo) - self.machine.setupHT() - tasksets = self.machine.reserveHT(300) + self.machine.setupTaskset() + + # ------------------------step1------------------------- + # phys_id 1 + # - core_id 0 + # - process_id 4 + # - process_id 12 + # - core_id 1 + # - process_id 5 + # - process_id 13 + # - core_id 3 + # - process_id 7 + # - process_id 15 + tasksets1 = self.machine.reserveHT(300) + # pylint: disable=no-member + self.assertItemsEqual(['4', '5', '7', '12', '13', '15'], sorted(tasksets1.split(','))) + + # ------------------------step2------------------------- + # phys_id 0 + # - core_id 0 + # - process_id 0 + # - process_id 8 + # - core_id 1 + # - process_id 1 + # - process_id 9 + # - core_id 2 + # - process_id 2 + # - process_id 10 + # - core_id 3 + # - process_id 3 + # - process_id 11 + tasksets0 = self.machine.reserveHT(400) + # pylint: disable=no-member + self.assertItemsEqual(['0', '1', '2', '3', '8', '9', '10', '11'], + sorted(tasksets0.split(','))) + + # reserved cores got updated properly + # pylint: disable=no-member + self.assertItemsEqual([0, 1, 2, 3], self.coreDetail.reserved_cores[0].coreid) + + # Make sure tastsets don't overlap + self.assertTrue(set(tasksets0.split(',')).isdisjoint(tasksets1.split(','))) - self.assertEqual('0,8,1,9,2,10', tasksets) + # ------------------------step3------------------------- + # Releasing a physcore shouldn't impact other physcores + self.machine.releaseHT(tasksets0) + # pylint: disable=no-member + self.assertTrue(1 in self.coreDetail.reserved_cores) + # pylint: disable=no-member + self.assertItemsEqual([0, 1, 3], self.coreDetail.reserved_cores[1].coreid) + + # ------------------------step4------------------------- + # phys_id 0 + # - core_id 0 + # - process_id 0 + # - process_id 8 + # - core_id 1 + # - process_id 1 + # - process_id 9 + tasksets3 = self.machine.reserveHT(200) + # pylint: disable=no-member + self.assertItemsEqual(['0', '1', '8', '9'], sorted(tasksets3.split(','))) + + # ------------------------step5------------------------- + # phys_id 0 + # - core_id 2 + # - process_id 2 + # - process_id 10 + # - core_id 3 + # - process_id 3 + # - process_id 11 + # phys_id 1 + # - core_id 2 + # - process_id 6 + # - process_id 14 + tasksets4 = self.machine.reserveHT(300) + # pylint: disable=no-member + self.assertItemsEqual(['2', '10', '3', '11', '6', '14'], sorted(tasksets4.split(','))) - self.machine.releaseHT(tasksets) + # ------------------------step6------------------------- + # No cores available + with self.assertRaises(rqd.rqexceptions.CoreReservationFailureException): + self.machine.reserveHT(300) - self.assertEqual({0, 1, 2, 3, 4, 5, 6, 7}, self.machine._Machine__tasksets) def test_tags(self): tags = ["test1", "test2", "test3"] From d9d24f0c9aa4a9303d45354ba40fd484b1c7f0a6 Mon Sep 17 00:00:00 2001 From: Rosa Behrens Camp Date: Wed, 13 Jul 2022 14:27:01 -0700 Subject: [PATCH 208/277] Min mem increase (#1157) Added the ability for admins to set the minimum memory increase value on a service. OpenCue increases the minimum memory requirements every time a job is retried when it fails from running out of memory but the old memory increase (hardcoded to 2G) was causing some jobs to be retried many times before it finally succeeded. --- VERSION.in | 2 +- cuebot/{build.gradle => build.gradle.orig} | 0 .../com/imageworks/spcue/ServiceEntity.java | 2 + .../spcue/dao/postgres/ServiceDaoJdbc.java | 33 ++++-- .../spcue/dao/postgres/WhiteboardDaoJdbc.java | 8 +- .../spcue/dispatcher/Dispatcher.java | 4 + .../dispatcher/FrameCompleteHandler.java | 64 +++++++++- .../spcue/servant/ManageService.java | 2 + .../spcue/servant/ManageServiceOverride.java | 1 + .../imageworks/spcue/servant/ManageShow.java | 1 + .../V15__Add_min_memory_increase_service.sql | 5 + .../spring/applicationContext-service.xml | 3 + .../test/dao/postgres/ServiceDaoTests.java | 11 ++ .../test/dao/postgres/WhiteboardDaoTests.java | 1 + .../dispatcher/FrameCompleteHandlerTests.java | 110 ++++++++++++++++-- .../test/dispatcher/HistoryControlTests.java | 2 +- .../resources/conf/ddl/postgres/test_data.sql | 2 + .../conf/jobspec/jobspec_gpus_test.xml | 17 +++ proto/service.proto | 1 + pycue/opencue/wrappers/service.py | 19 +++ pycue/opencue/wrappers/show.py | 5 + pycue/tests/api_test.py | 4 +- pycue/tests/wrappers/service_test.py | 31 ++++- pycue/tests/wrappers/show_test.py | 6 + 24 files changed, 305 insertions(+), 29 deletions(-) rename cuebot/{build.gradle => build.gradle.orig} (100%) create mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V15__Add_min_memory_increase_service.sql diff --git a/VERSION.in b/VERSION.in index caa4836d8..9f4eca259 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.19 +0.20 diff --git a/cuebot/build.gradle b/cuebot/build.gradle.orig similarity index 100% rename from cuebot/build.gradle rename to cuebot/build.gradle.orig diff --git a/cuebot/src/main/java/com/imageworks/spcue/ServiceEntity.java b/cuebot/src/main/java/com/imageworks/spcue/ServiceEntity.java index 16d03c5c5..0bb47c02c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/ServiceEntity.java +++ b/cuebot/src/main/java/com/imageworks/spcue/ServiceEntity.java @@ -70,5 +70,7 @@ public class ServiceEntity extends Entity { public int timeout_llu = 0; + public long minMemoryIncrease = Dispatcher.MINIMUM_MEMORY_INCREASE; + } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ServiceDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ServiceDaoJdbc.java index 6330cc8cb..a637b34d6 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ServiceDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ServiceDaoJdbc.java @@ -67,6 +67,7 @@ public ServiceEntity mapRow(ResultSet rs, int rowNum) throws SQLException { s.tags = splitTags(rs.getString("str_tags")); s.timeout = rs.getInt("int_timeout"); s.timeout_llu = rs.getInt("int_timeout_llu"); + s.minMemoryIncrease = rs.getLong("int_min_memory_increase"); return s; } }; @@ -89,6 +90,7 @@ public ServiceOverrideEntity mapRow(ResultSet rs, int rowNum) s.showId = rs.getString("pk_show"); s.timeout = rs.getInt("int_timeout"); s.timeout_llu = rs.getInt("int_timeout_llu"); + s.minMemoryIncrease = rs.getLong("int_min_memory_increase"); return s; } }; @@ -106,7 +108,8 @@ public ServiceOverrideEntity mapRow(ResultSet rs, int rowNum) "service.int_gpu_mem_min," + "service.str_tags, " + "service.int_timeout, " + - "service.int_timeout_llu " + + "service.int_timeout_llu, " + + "service.int_min_memory_increase " + "FROM " + "service "; @@ -131,6 +134,7 @@ public ServiceEntity get(String id) { "show_service.str_tags," + "show_service.int_timeout," + "show_service.int_timeout_llu," + + "show_service.int_min_memory_increase," + "show.pk_show " + "FROM " + "show_service," + @@ -180,8 +184,9 @@ public boolean isOverridden(String service, String show) { "int_gpu_mem_min," + "str_tags," + "int_timeout," + - "int_timeout_llu " + - ") VALUES (?,?,?,?,?,?,?,?,?,?,?,?)"; + "int_timeout_llu, " + + "int_min_memory_increase " + + ") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)"; @Override public void insert(ServiceEntity service) { @@ -191,7 +196,8 @@ public void insert(ServiceEntity service) { service.maxCores, service.minMemory, service.minGpus, service.maxGpus, service.minGpuMemory, StringUtils.join(service.tags.toArray(), " | "), - service.timeout, service.timeout_llu); + service.timeout, service.timeout_llu, + service.minMemoryIncrease); } private static final String INSERT_SERVICE_WITH_SHOW = @@ -210,8 +216,9 @@ public void insert(ServiceEntity service) { "int_gpu_mem_min," + "str_tags," + "int_timeout," + - "int_timeout_llu " + - ") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)"; + "int_timeout_llu, " + + "int_min_memory_increase " + + ") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; @Override public void insert(ServiceOverrideEntity service) { @@ -220,7 +227,7 @@ public void insert(ServiceOverrideEntity service) { service.showId, service.name, service.threadable, service.minCores, service.maxCores, service.minMemory, service.minGpus, service.maxGpus, service.minGpuMemory, joinTags(service.tags), - service.timeout, service.timeout_llu); + service.timeout, service.timeout_llu, service.minMemoryIncrease); } private static final String UPDATE_SERVICE = @@ -237,7 +244,8 @@ service.minGpus, service.maxGpus, service.minGpuMemory, joinTags(service.tags), "int_gpu_mem_min=?," + "str_tags=?," + "int_timeout=?," + - "int_timeout_llu=? " + + "int_timeout_llu=?, " + + "int_min_memory_increase=? " + "WHERE " + "pk_service = ?"; @@ -246,7 +254,8 @@ public void update(ServiceEntity service) { getJdbcTemplate().update(UPDATE_SERVICE, service.name, service.threadable, service.minCores, service.maxCores, service.minMemory, service.minGpus, service.maxGpus, service.minGpuMemory, joinTags(service.tags), - service.timeout, service.timeout_llu, service.getId()); + service.timeout, service.timeout_llu, service.minMemoryIncrease, + service.getId()); } private static final String UPDATE_SERVICE_WITH_SHOW = @@ -263,7 +272,8 @@ service.minMemory, service.minGpus, service.maxGpus, service.minGpuMemory, joinT "int_gpu_mem_min=?," + "str_tags=?," + "int_timeout=?," + - "int_timeout_llu=? " + + "int_timeout_llu=?, " + + "int_min_memory_increase=? " + "WHERE " + "pk_show_service = ?"; @@ -272,7 +282,8 @@ public void update(ServiceOverrideEntity service) { getJdbcTemplate().update(UPDATE_SERVICE_WITH_SHOW, service.name, service.threadable, service.minCores, service.maxCores, service.minMemory, service.minGpus, service.maxGpus, service.minGpuMemory, joinTags(service.tags), - service.timeout, service.timeout_llu, service.getId()); + service.timeout, service.timeout_llu, service.minMemoryIncrease, + service.getId()); } @Override diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index aec665423..7a1a85351 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -1461,6 +1461,7 @@ public Service mapRow(ResultSet rs, int rowNum) throws SQLException { SqlUtil.getString(rs,"str_tags")))) .setTimeout(rs.getInt("int_timeout")) .setTimeoutLlu(rs.getInt("int_timeout_llu")) + .setMinMemoryIncrease(rs.getInt("int_min_memory_increase")) .build(); } }; @@ -1482,6 +1483,7 @@ public ServiceOverride mapRow(ResultSet rs, int rowNum) throws SQLException { SqlUtil.getString(rs,"str_tags")))) .setTimeout(rs.getInt("int_timeout")) .setTimeoutLlu(rs.getInt("int_timeout_llu")) + .setMinMemoryIncrease(rs.getInt("int_min_memory_increase")) .build(); return ServiceOverride.newBuilder() .setId(SqlUtil.getString(rs,"pk_show_service")) @@ -2087,7 +2089,8 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "service.int_gpu_mem_min," + "service.str_tags," + "service.int_timeout," + - "service.int_timeout_llu " + + "service.int_timeout_llu," + + "service.int_min_memory_increase " + "FROM "+ "service "; @@ -2104,7 +2107,8 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "show_service.int_gpu_mem_min," + "show_service.str_tags," + "show_service.int_timeout," + - "show_service.int_timeout_llu " + + "show_service.int_timeout_llu," + + "show_service.int_min_memory_increase " + "FROM "+ "show_service, " + "show " + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java index 6e06bdc44..6ac703a41 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java @@ -132,6 +132,10 @@ public interface Dispatcher { // The default operating system assigned to host that don't report one. public static final String OS_DEFAULT = "rhel40"; + // The default minimum memory increase for when jobs fail due to not enough + // memory + public static final long MINIMUM_MEMORY_INCREASE = CueUtil.GB2; + /** * Dispatch a host to the facility. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java index 7d6fad4fb..d4e619894 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java @@ -24,6 +24,7 @@ import java.util.Random; import java.util.concurrent.atomic.AtomicLong; +import com.imageworks.spcue.*; import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; @@ -53,6 +54,12 @@ import com.imageworks.spcue.util.CueExceptionUtil; import com.imageworks.spcue.util.CueUtil; +import com.imageworks.spcue.dao.WhiteboardDao; +import com.imageworks.spcue.dao.ShowDao; +import com.imageworks.spcue.dao.ServiceDao; +import com.imageworks.spcue.grpc.service.Service; +import com.imageworks.spcue.grpc.service.ServiceOverride; + /** * The FrameCompleteHandler encapsulates all logic necessary for processing * FrameComplete reports from RQD. @@ -75,6 +82,10 @@ public class FrameCompleteHandler { private DispatchSupport dispatchSupport; private JmsMover jsmMover; + private WhiteboardDao whiteboardDao; + private ServiceDao serviceDao; + private ShowDao showDao; + /* * The last time a proc was unbooked for subscription or job balancing. * Since there are so many more dispatch threads than booking threads, the @@ -298,14 +309,45 @@ public void handlePostFrameCompleteOperations(VirtualProc proc, /* * An exit status of 33 indicates that the frame was killed by the * application due to a memory issue and should be retried. In this - * case, disable the optimizer and raise the memory by 2GB. + * case, disable the optimizer and raise the memory by what is + * specified in the show's service override, service or 2GB. */ if (report.getExitStatus() == Dispatcher.EXIT_STATUS_MEMORY_FAILURE || report.getExitSignal() == Dispatcher.EXIT_STATUS_MEMORY_FAILURE) { + long increase = CueUtil.GB2; + + // since there can be multiple services, just going for the + // first service (primary) + String serviceName = ""; + try { + serviceName = frame.services.split(",")[0]; + ServiceOverride showService = whiteboardDao.getServiceOverride( + showDao.findShowDetail(frame.show), serviceName); + // increase override is stored in Kb format so convert to Mb + // for easier reading. Note: Kb->Mb conversion uses 1024 blocks + increase = showService.getData().getMinMemoryIncrease(); + logger.info("Using " + serviceName + " service show " + + "override for memory increase: " + + Math.floor(increase / 1024) + "Mb."); + } + catch (NullPointerException e) { + logger.info("Frame has no associated services"); + } + catch (EmptyResultDataAccessException e) { + logger.info(frame.show + " has no service override for " + + serviceName + "."); + Service service = whiteboardDao.findService(serviceName); + increase = service.getMinMemoryIncrease(); + logger.info("Using service default for mem increase: " + + Math.floor(increase / 1024) + "Mb."); + } + unbookProc = true; jobManager.enableMemoryOptimizer(frame, false); jobManager.increaseLayerMemoryRequirement(frame, - proc.memoryReserved + CueUtil.GB2); + proc.memoryReserved + increase); + logger.info("Increased mem usage to: " + + (proc.memoryReserved + increase)); } /* @@ -501,7 +543,6 @@ else if (report.getHost().getNimbyLocked()) { dispatchSupport.unbookProc(proc, "frame state was " + newFrameState.toString()); } - } catch (Exception e) { /* * At this point, the proc has no place to go. Since we've run into @@ -692,5 +733,22 @@ public JmsMover getJmsMover() { public void setJmsMover(JmsMover jsmMover) { this.jsmMover = jsmMover; } + + public WhiteboardDao getWhiteboardDao() { return whiteboardDao; } + + public void setWhiteboardDao(WhiteboardDao whiteboardDao) { + this.whiteboardDao = whiteboardDao; } + + public ServiceDao getServiceDao() { return serviceDao; } + + public void setServiceDao(ServiceDao serviceDao) { + this.serviceDao = serviceDao; } + + public ShowDao getShowDao() { return showDao; } + + public void setShowDao(ShowDao showDao) { + this.showDao = showDao; } + } + diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageService.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageService.java index 70a15f3bf..8cd9029c8 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageService.java @@ -62,6 +62,7 @@ public void createService(ServiceCreateServiceRequest request, service.threadable = request.getData().getThreadable(); service.timeout = request.getData().getTimeout(); service.timeout_llu = request.getData().getTimeoutLlu(); + service.minMemoryIncrease = request.getData().getMinMemoryIncrease(); serviceManager.createService(service); responseObserver.onNext(ServiceCreateServiceResponse.newBuilder() .setService(whiteboard.getService(service.getId())) @@ -138,6 +139,7 @@ private ServiceEntity toServiceEntity(Service service) { entity.threadable = service.getThreadable(); entity.timeout = service.getTimeout(); entity.timeout_llu = service.getTimeoutLlu(); + entity.minMemoryIncrease = service.getMinMemoryIncrease(); return entity; } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageServiceOverride.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageServiceOverride.java index ed3d46107..16afb3bf2 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageServiceOverride.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageServiceOverride.java @@ -74,6 +74,7 @@ private ServiceEntity toServiceEntity(Service service) { entity.threadable = service.getThreadable(); entity.timeout = service.getTimeout(); entity.timeout_llu = service.getTimeoutLlu(); + entity.minMemoryIncrease = service.getMinMemoryIncrease(); return entity; } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageShow.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageShow.java index 6e5fbcbe8..0c496b25d 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageShow.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageShow.java @@ -388,6 +388,7 @@ public void createServiceOverride(ShowCreateServiceOverrideRequest request, service.minGpuMemory = requestService.getMinGpuMemory(); service.tags = Sets.newLinkedHashSet(requestService.getTagsList()); service.threadable = requestService.getThreadable(); + service.minMemoryIncrease = requestService.getMinMemoryIncrease(); serviceManager.createService(service); ServiceOverride serviceOverride = whiteboard.getServiceOverride(show, service.name); responseObserver.onNext(ShowCreateServiceOverrideResponse.newBuilder() diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V15__Add_min_memory_increase_service.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V15__Add_min_memory_increase_service.sql new file mode 100644 index 000000000..0b0655521 --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V15__Add_min_memory_increase_service.sql @@ -0,0 +1,5 @@ + +-- Add minimum memory increase - default 2G + +ALTER TABLE show_service ADD COLUMN int_min_memory_increase INT DEFAULT 2097152 NOT NULL; +ALTER TABLE service ADD COLUMN int_min_memory_increase INT DEFAULT 2097152 NOT NULL; diff --git a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml index 6c3a9f2bd..35bd5cbb8 100644 --- a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml +++ b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml @@ -369,6 +369,9 @@ + + + diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ServiceDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ServiceDaoTests.java index 16168f245..811cb129d 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ServiceDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ServiceDaoTests.java @@ -73,6 +73,7 @@ public void testInsertService() { s.minGpuMemory = CueUtil.GB; s.threadable = false; s.tags.addAll(Sets.newHashSet(new String[] { "general"})); + s.minMemoryIncrease = CueUtil.GB4; serviceDao.insert(s); assertEquals(s, serviceDao.get("dillweed")); @@ -91,6 +92,7 @@ public void testUpdateService() { s.minGpuMemory = CueUtil.GB; s.threadable = false; s.tags.addAll(Sets.newHashSet(new String[] { "general"})); + s.minMemoryIncrease = CueUtil.GB; serviceDao.insert(s); assertEquals(s, serviceDao.get("dillweed")); @@ -104,6 +106,7 @@ public void testUpdateService() { s.threadable = true; s.tags = Sets.newLinkedHashSet(); s.tags.add("linux"); + s.minMemoryIncrease = CueUtil.GB4 + CueUtil.GB2; serviceDao.update(s); ServiceEntity s1 = serviceDao.get(s.getId()); @@ -113,6 +116,7 @@ public void testUpdateService() { assertEquals(s.minMemory, s1.minMemory); assertEquals(s.threadable, s1.threadable); assertEquals(s.tags.toArray()[0], s1.tags.toArray()[0]); + assertEquals(s.minMemoryIncrease, s1.minMemoryIncrease); } @Test @@ -128,6 +132,7 @@ public void testDeleteService() { s.minGpuMemory = CueUtil.GB; s.threadable = false; s.tags.addAll(Sets.newHashSet(new String[] { "general"})); + s.minMemoryIncrease = CueUtil.GB2; serviceDao.insert(s); assertEquals(s, serviceDao.get("dillweed")); @@ -153,6 +158,7 @@ public void testInsertServiceOverride() { s.threadable = false; s.tags.addAll(Sets.newHashSet(new String[] { "general"})); s.showId = "00000000-0000-0000-0000-000000000000"; + s.minMemoryIncrease = CueUtil.GB2; serviceDao.insert(s); assertEquals(s, serviceDao.getOverride("dillweed")); @@ -172,6 +178,7 @@ public void testUpdateServiceOverride() { s.threadable = false; s.tags.addAll(Sets.newHashSet(new String[] { "general"})); s.showId = "00000000-0000-0000-0000-000000000000"; + s.minMemoryIncrease = CueUtil.GB2; serviceDao.insert(s); assertEquals(s, serviceDao.getOverride("dillweed")); @@ -186,6 +193,7 @@ public void testUpdateServiceOverride() { s.threadable = true; s.tags = Sets.newLinkedHashSet(); s.tags.add("linux"); + s.minMemoryIncrease = CueUtil.GB4; serviceDao.update(s); ServiceEntity s1 = serviceDao.getOverride(s.getId()); @@ -198,6 +206,8 @@ public void testUpdateServiceOverride() { assertEquals(s.minGpuMemory, s1.minGpuMemory); assertEquals(s.threadable, s1.threadable); assertEquals(s.tags.toArray()[0], s1.tags.toArray()[0]); + assertEquals(s.minMemoryIncrease, s1.minMemoryIncrease); + assertEquals(s1.minMemoryIncrease, CueUtil.GB4); } @Test @@ -214,6 +224,7 @@ public void testDeleteServiceOverride() { s.threadable = false; s.tags.addAll(Sets.newHashSet(new String[] { "general"})); s.showId = "00000000-0000-0000-0000-000000000000"; + s.minMemoryIncrease = CueUtil.GB2; serviceDao.insert(s); assertEquals(s, serviceDao.getOverride("dillweed")); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java index 8807514d4..d419b6ce9 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java @@ -330,6 +330,7 @@ public void getServiceOverride() { s.tags.add("general"); s.threadable = false; s.showId = show.getId(); + s.minMemoryIncrease = CueUtil.GB4; serviceManager.createService(s); whiteboardDao.getServiceOverride(getShow(), "test"); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java index 9228da36d..f022fc687 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java @@ -22,6 +22,7 @@ import java.io.File; import java.util.List; import javax.annotation.Resource; +import java.util.LinkedHashSet; import org.junit.Before; import org.junit.Test; @@ -36,6 +37,7 @@ import com.imageworks.spcue.FrameDetail; import com.imageworks.spcue.JobDetail; import com.imageworks.spcue.LayerDetail; +import com.imageworks.spcue.ServiceOverrideEntity; import com.imageworks.spcue.VirtualProc; import com.imageworks.spcue.dao.FrameDao; import com.imageworks.spcue.dao.LayerDao; @@ -51,6 +53,7 @@ import com.imageworks.spcue.service.HostManager; import com.imageworks.spcue.service.JobLauncher; import com.imageworks.spcue.service.JobManager; +import com.imageworks.spcue.service.ServiceManager; import com.imageworks.spcue.test.TransactionalTest; import com.imageworks.spcue.util.CueUtil; @@ -88,10 +91,15 @@ public class FrameCompleteHandlerTests extends TransactionalTest { @Resource DispatchSupport dispatchSupport; + @Resource + ServiceManager serviceManager; + private static final String HOSTNAME = "beta"; + private static final String HOSTNAME2 = "zeta"; @Before public void setTestMode() { + dispatcher.setTestMode(true); } @@ -127,10 +135,31 @@ public void createHost() { hostManager.createHost(host, adminManager.findAllocationDetail("spi", "general")); + + RenderHost host2 = RenderHost.newBuilder() + .setName(HOSTNAME2) + .setBootTime(1192369572) + .setFreeMcp(76020) + .setFreeMem((int) CueUtil.GB4) + .setFreeSwap((int) CueUtil.GB4) + .setLoad(0) + .setTotalMcp(195430) + .setTotalMem((int) CueUtil.GB8) + .setTotalSwap((int) CueUtil.GB8) + .setNimbyEnabled(false) + .setNumProcs(8) + .setCoresPerProc(100) + .setState(HardwareState.UP) + .setFacility("spi") + .putAttributes("SP_OS", "Linux") + .build(); + + hostManager.createHost(host2, + adminManager.findAllocationDetail("spi", "general")); } - public DispatchHost getHost() { - return hostManager.findDispatchHost(HOSTNAME); + public DispatchHost getHost(String hostname) { + return hostManager.findDispatchHost(hostname); } @Test @@ -141,7 +170,7 @@ public void testGpuReport() { LayerDetail layer = layerDao.findLayerDetail(job, "layer0"); jobManager.setJobPaused(job, false); - DispatchHost host = getHost(); + DispatchHost host = getHost(HOSTNAME); List procs = dispatcher.dispatchHost(host); assertEquals(1, procs.size()); VirtualProc proc = procs.get(0); @@ -177,7 +206,7 @@ public void testGpuReportMultiple() { LayerDetail layer1_0 = layerDao.findLayerDetail(job1, "layer0"); jobManager.setJobPaused(job1, false); - DispatchHost host = getHost(); + DispatchHost host = getHost(HOSTNAME); List procs = dispatcher.dispatchHost(host); assertEquals(2, procs.size()); @@ -216,7 +245,7 @@ public void testGpuReportOver() { LayerDetail layer2_0 = layerDao.findLayerDetail(job2, "layer0"); jobManager.setJobPaused(job2, false); - DispatchHost host = getHost(); + DispatchHost host = getHost(HOSTNAME); List procs = dispatcher.dispatchHost(host); assertEquals(1, procs.size()); @@ -258,7 +287,7 @@ private void executeDepend( jobManager.setJobPaused(job, false); - DispatchHost host = getHost(); + DispatchHost host = getHost(HOSTNAME); List procs = dispatcher.dispatchHost(host); assertEquals(1, procs.size()); VirtualProc proc = procs.get(0); @@ -327,5 +356,72 @@ public void testDependOnFailureSatisfyOnAny() { executeDepend(FrameState.EATEN, -1, 0, FrameState.WAITING); frameCompleteHandler.setSatisfyDependOnlyOnFrameSuccess(true); } -} + private void executeMinMemIncrease(int expected, boolean override) { + if (override) { + ServiceOverrideEntity soe = new ServiceOverrideEntity(); + soe.showId = "00000000-0000-0000-0000-000000000000"; + soe.name = "apitest"; + soe.threadable = false; + soe.minCores = 10; + soe.minMemory = (int) CueUtil.GB2; + soe.tags = new LinkedHashSet<>(); + soe.tags.add("general"); + soe.minMemoryIncrease = (int) CueUtil.GB8; + + serviceManager.createService(soe); + } + + String jobName = "pipe-default-testuser_min_mem_test"; + JobDetail job = jobManager.findJobDetail(jobName); + LayerDetail layer = layerDao.findLayerDetail(job, "test_layer"); + FrameDetail frame = frameDao.findFrameDetail(job, "0000-test_layer"); + jobManager.setJobPaused(job, false); + + DispatchHost host = getHost(HOSTNAME2); + List procs = dispatcher.dispatchHost(host); + assertEquals(1, procs.size()); + VirtualProc proc = procs.get(0); + assertEquals(job.getId(), proc.getJobId()); + assertEquals(layer.getId(), proc.getLayerId()); + assertEquals(frame.getId(), proc.getFrameId()); + + RunningFrameInfo info = RunningFrameInfo.newBuilder() + .setJobId(proc.getJobId()) + .setLayerId(proc.getLayerId()) + .setFrameId(proc.getFrameId()) + .setResourceId(proc.getProcId()) + .build(); + FrameCompleteReport report = FrameCompleteReport.newBuilder() + .setFrame(info) + .setExitStatus(Dispatcher.EXIT_STATUS_MEMORY_FAILURE) + .build(); + + DispatchJob dispatchJob = jobManager.getDispatchJob(proc.getJobId()); + DispatchFrame dispatchFrame = jobManager.getDispatchFrame(report.getFrame().getFrameId()); + dispatchSupport.stopFrame(dispatchFrame, FrameState.DEAD, report.getExitStatus(), + report.getFrame().getMaxRss()); + frameCompleteHandler.handlePostFrameCompleteOperations(proc, + report, dispatchJob, dispatchFrame, FrameState.WAITING); + + assertFalse(jobManager.isLayerComplete(layer)); + + JobDetail ujob = jobManager.findJobDetail(jobName); + LayerDetail ulayer = layerDao.findLayerDetail(ujob, "test_layer"); + assertEquals(expected, ulayer.getMinimumMemory()); + } + + @Test + @Transactional + @Rollback(true) + public void testMinMemIncrease() { + executeMinMemIncrease(6291456, false); + } + + @Test + @Transactional + @Rollback(true) + public void testMinMemIncreaseShowOverride() { + executeMinMemIncrease(10485760, true); + } +} diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java index a9787566b..138a3f33c 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java @@ -169,7 +169,7 @@ public void testEnabled() { launchAndDeleteJob(); - assertEquals(Integer.valueOf(4), jdbcTemplate.queryForObject( + assertEquals(Integer.valueOf(5), jdbcTemplate.queryForObject( "SELECT COUNT(*) FROM job_history", Integer.class)); assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( "SELECT COUNT(*) FROM frame_history", Integer.class)); diff --git a/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql b/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql index 14c56afcc..ae7b77354 100644 --- a/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql +++ b/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql @@ -104,6 +104,8 @@ Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN, Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS,INT_CORES_MAX,INT_GPU_MEM_MIN) values ('488c75f0-eae4-4dd0-83e0-29b982adbbff','cuda',true,100,3354624,'cuda',0,262144) +Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS,INT_MIN_MEMORY_INCREASE) values ('123c75f0-eie4-4cc0-84e0-46b982abcdef','apitest',false,10,2097152,'general',4194304) + Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000005','MAX_FRAME_RETRIES',16,0,null,false) diff --git a/cuebot/src/test/resources/conf/jobspec/jobspec_gpus_test.xml b/cuebot/src/test/resources/conf/jobspec/jobspec_gpus_test.xml index 9d9aa8245..cfa6e694c 100644 --- a/cuebot/src/test/resources/conf/jobspec/jobspec_gpus_test.xml +++ b/cuebot/src/test/resources/conf/jobspec/jobspec_gpus_test.xml @@ -74,6 +74,22 @@ + + true + 3 + + + /shots/pipe/usr_testuser/logs/help.py + 0 + 1 + false + + apitest + + + + + True @@ -108,4 +124,5 @@ layer_first + diff --git a/proto/service.proto b/proto/service.proto index 23633bc40..57110bdaf 100644 --- a/proto/service.proto +++ b/proto/service.proto @@ -53,6 +53,7 @@ message Service { int32 timeout_llu = 10; int32 min_gpus = 11; int32 max_gpus = 12; + int32 min_memory_increase = 13; } message ServiceSeq { diff --git a/pycue/opencue/wrappers/service.py b/pycue/opencue/wrappers/service.py index 3878d9b3d..b3f6563d2 100644 --- a/pycue/opencue/wrappers/service.py +++ b/pycue/opencue/wrappers/service.py @@ -29,6 +29,10 @@ def __init__(self, service=None): def create(self): """Creates a service in the database using the current instance data.""" + # min_memory_increase has to be greater than 0. + if self.data.min_memory_increase <= 0: + raise ValueError("Minimum memory increase must be > 0") + response = self.stub.CreateService( service_pb2.ServiceCreateServiceRequest(data=self.data), timeout=Cuebot.Timeout) @@ -69,6 +73,10 @@ def getService(name): def update(self): """Updates the service database record with the current instance data.""" + # min_memory_increase has to be greater than 0. + if self.data.min_memory_increase <= 0: + raise ValueError("Minimum memory increase must be > 0") + return self.stub.Update( service_pb2.ServiceUpdateRequest(service=self.data), timeout=Cuebot.Timeout) @@ -240,3 +248,14 @@ def timeoutLLU(self): def setTimeoutLLU(self, timeout_llu): """Sets the default service LLU timeout.""" self.data.timeout_llu = timeout_llu + + def minMemoryIncrease(self): + """Gets the default service minimum memory increment""" + return self.data.min_memory_increase + + def setMinMemoryIncrease(self, min_memory_increase): + """Sets the default service minimum memory increment""" + if min_memory_increase > 0: + self.data.min_memory_increase = min_memory_increase + else: + raise ValueError("Minimum memory increase must be > 0") diff --git a/pycue/opencue/wrappers/show.py b/pycue/opencue/wrappers/show.py index 90094e757..5dad1a000 100644 --- a/pycue/opencue/wrappers/show.py +++ b/pycue/opencue/wrappers/show.py @@ -70,6 +70,11 @@ def createServiceOverride(self, data): :type data: service_pb2.Service :param data: service data, typically from opencue.wrappers.service.Service.data """ + + # min_memory_increase has to be greater than 0. + if data.min_memory_increase <= 0: + raise ValueError("Minimum memory increase must be > 0") + self.stub.CreateServiceOverride( show_pb2.ShowCreateServiceOverrideRequest(show=self.data, service=data), timeout=Cuebot.Timeout) diff --git a/pycue/tests/api_test.py b/pycue/tests/api_test.py index b20acfff9..759c09da1 100644 --- a/pycue/tests/api_test.py +++ b/pycue/tests/api_test.py @@ -357,11 +357,13 @@ class ServiceTests(unittest.TestCase): testThreadable = False testMinCores = 1000 testMaxCores = 2000 + minMemoryIncrease = 2068378 def setUp(self): self.service = service_pb2.Service( name=self.testName, threadable=self.testThreadable, min_cores=self.testMinCores, - max_cores=self.testMaxCores, tags=self.testTags) + max_cores=self.testMaxCores, tags=self.testTags, + min_memory_increase=self.minMemoryIncrease) @mock.patch('opencue.cuebot.Cuebot.getStub') def testCreate(self, getStubMock): diff --git a/pycue/tests/wrappers/service_test.py b/pycue/tests/wrappers/service_test.py index e46b29bdf..3500eec5c 100644 --- a/pycue/tests/wrappers/service_test.py +++ b/pycue/tests/wrappers/service_test.py @@ -51,16 +51,23 @@ def testDelete(self, getStubMock): def testCreateService(self, getStubMock): stubMock = mock.Mock() stubMock.CreateService.return_value = service_pb2.ServiceCreateServiceResponse( - service=service_pb2.Service(name=TEST_SERVICE_NAME)) + service=service_pb2.Service(name=TEST_SERVICE_NAME, min_memory_increase=2097152)) getStubMock.return_value = stubMock wrapper = opencue.wrappers.service.Service( - service_pb2.Service(name=TEST_SERVICE_NAME)) + service_pb2.Service(name=TEST_SERVICE_NAME, min_memory_increase=2097152)) service = wrapper.create() stubMock.CreateService.assert_called_with( service_pb2.ServiceCreateServiceRequest(data=wrapper.data), timeout=mock.ANY) self.assertEqual(wrapper.name(), service.name()) + self.assertEqual(wrapper.minMemoryIncrease(), service.minMemoryIncrease()) + + def testCreateServiceMemError(self, getStubMock): + service = opencue.wrappers.service.Service(service_pb2.Service( + name=TEST_SERVICE_NAME)) + + self.assertRaises(ValueError, service.create) def testGetDefaultServices(self, getStubMock): stubMock = mock.Mock() @@ -96,7 +103,7 @@ def testUpdate(self, getStubMock): getStubMock.return_value = stubMock wrapper = opencue.wrappers.service.Service(service=service_pb2.Service( - name=TEST_SERVICE_NAME)) + name=TEST_SERVICE_NAME, min_memory_increase=302)) wrapper.update() stubMock.Update.assert_called_with( @@ -124,5 +131,23 @@ def testGpus(self, getStubMock): service_pb2.ServiceGetServiceRequest(name=TEST_SERVICE_NAME), timeout=mock.ANY) self.assertEqual(service.name(), TEST_SERVICE_NAME) + def testUpdateMemError(self, getStubMock): + service = opencue.wrappers.service.Service(service=service_pb2.Service( + name=TEST_SERVICE_NAME)) + + self.assertRaises(ValueError, service.update) + + def testSetMinMemIncrease(self, getStubMock): + service = opencue.wrappers.service.Service( + service_pb2.Service(name=TEST_SERVICE_NAME, + min_memory_increase=2097152)) + + self.assertRaises(ValueError, service.setMinMemoryIncrease, -1) + self.assertRaises(ValueError, service.setMinMemoryIncrease, 0) + + service.setMinMemoryIncrease(12345678) + self.assertEqual(service.minMemoryIncrease(), 12345678) + + if __name__ == '__main__': unittest.main() diff --git a/pycue/tests/wrappers/show_test.py b/pycue/tests/wrappers/show_test.py index b8fb26285..cb73a4b5d 100644 --- a/pycue/tests/wrappers/show_test.py +++ b/pycue/tests/wrappers/show_test.py @@ -324,6 +324,12 @@ def testEnableDispatching(self, getStubMock): show_pb2.ShowEnableDispatchingRequest(show=show.data, enabled=TEST_ENABLE_VALUE), timeout=mock.ANY) + def testCreateServiceOverrideMemError(self, getStubMock): + service = service_pb2.Service(name=TEST_SERVICE_NAME) + show = opencue.wrappers.show.Show(show_pb2.Show(name=TEST_SHOW_NAME)) + + self.assertRaises(ValueError, show.createServiceOverride, service) + if __name__ == '__main__': unittest.main() From 3aca86f8752c6ac7d90e6e575e3fc674b4b07d28 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 20 Jul 2022 14:16:19 -0400 Subject: [PATCH 209/277] Notes from July 6 TSC meeting. (#1176) --- tsc/meetings/2022-07-06.md | 125 +++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 tsc/meetings/2022-07-06.md diff --git a/tsc/meetings/2022-07-06.md b/tsc/meetings/2022-07-06.md new file mode 100644 index 000000000..9b52bfb9b --- /dev/null +++ b/tsc/meetings/2022-07-06.md @@ -0,0 +1,125 @@ +# OpenCue TSC Meeting Notes 6 July 2022 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* New business + * API changes to fail on incorrect parameters + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1165/files + * no issues, good to merge when code/tests are ready + * Make frames and layers readonly after job finishes + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1164 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1163 + * solution seems ok, but should be a cuebot config flag for now + * Siggraph/Open Source Days + * Few folks attending Siggraph likely. Brian will be OOO and unable to attend. + * Will hybrid in-person/virtual sessions be supported? + * How to design the session? + * Brian to do some research and start email thread. + * Integration tests + * Integration test script being developed at SPI. + * We will want to look at turning it into a github action. + * This will help speed up the release process a lot. Most of the time spent making a release is QAing it. + * SPI to update soon. +* Ongoing work + * CueGUI memory leak + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1025 + * What we know: bug in job monitor tree creates unnecessary refreshing. Probably present in all tree plugins, + but most apparent when monitoring 100s/1000s of jobs. + * SPI: researching and testing their fix. + * RQD change, adding child proc info into log, storing in database + * For understanding memory usage of complex jobs. + * Linux-only. + * Testing ongoing, feedback has been positive so far. + * Change was deployed to production, found some issues, fixed. + * Work is done. Has it been merged to github yet? + * Avoiding complex merges from upstream. + * Problem: users will maintain their own OpenCue forks with many preferred customizations. When it comes time to + merge in changes from upstream, a complex merge is needed. This is time consuming and results in long gaps + between upstream merges. + * How do others solve this? + * Suggestion: let's identify the various areas needing customization, and discuss how we can generalize a + solution by creating options/plugins/hooks so customization can live outside the main codebase. + * Most spots identified, PRs have been sent + * Upstream merge almost complete, will resume that soon. + * OS-dependent log root + * Issue: https://github.com/AcademySoftwareFoundation/OpenCue/issues/1096 + * PR: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1102 + * PR merged. + * New PR to fix log paths on the CueGUI side: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1138 + * All PRs merged. Done now. + * Nimby in new thread: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1063 + * Not reproducible? We need input from SPI. + * Reproduced now. PR coming with several rqd/nimby fixes. + * pynput will become the default for detecting system activity. + * Sentry logging in CueGUI: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1127 + * Change looks ok, looking for more detail. + * Let's follow up on that PR again, review, figure out if we need to clone to new PR. +* 2022 Goals + * New user UX + * Publish PyPI packages + * Design doc: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/ + * Proposal for config file + standardization: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/edit#heading=h.v85uo9klwqft + * Config cleanup for pycue, pyoutline, cuesubmit done. + * CueGUI in progress. + * RQD still todo. + * Fix cuebot:latest tag for sandbox setup + * Proposal: publish Docker images for every new patch version (i.e. every new commit to master) + * Proposal accepted. + * No progress implementing this yet. + * Related: for PyPI design we may want to also publish packages for every new patch version. + * Docs refresh + * Tech writing help + * Discussing with John Mertic at LF and OCIO reps who are also interested. + * User guide refresh + * Split into "install from latest release" and "install from master" + * Needed to assist transition to publish Docker images on all commits + * No progress. + * API reference + * We have the ability to generate HTML from any commit, but we need a new CI pipeline to build and publish + this on opencue.io for new releases. + * Proposal: for now, highlight existing docs for building reference locally + * Proposal accepted. + * Brian: cleaned up local build process, sent email with doc + link: https://www.opencue.io/contributing/opencue/build-docs/ + * pyoutline examples + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/177 + * No progress yet. + * User guide for CueGUI + * No progress. + * Configuration guides for Python/Cuebot/RQD + * In progress, will be done as part of PyPI work. + * Improve scheduler logic + * Diego has volunteered to own this. + * Email reply from Matt, looped in others. + * Other PRs with improvements — see ongoing work above. + * Prototype in progress. + * High-level prototype details using Redis: https://github.com/AcademySoftwareFoundation/OpenCue/issues/1001 + * Expanded Cloud functionality + * Finalize GSoC plugin + * Test Azure support + * Add AWS support + * Design expanded functionality e.g. autoscaling for work in the queue. + * Last update from Greg: started to test / clean up azure support. Looking good so far. + * Terraform scripts? + * Let's look at the Azure version of this + * Generic k8s setup and cuebot pool size suggestions + * Existing Azure work on this: https://github.com/Azure/Avere/tree/main/src/terraform/examples/vfxt/opencue + * Issue for discussion: https://github.com/AcademySoftwareFoundation/OpenCue/issues/1097 + * Use Kubernetes for rqd? Kubernetes adds some overhead. A very small amount but some people care. + * A Helm chart could be a good option. Easy to use and get started. + * Let's review the Github issue, leave notes. Short, rough draft of design doc for auto-scaling would be good + for further discussion. + * User permissions + * Need a github issue for discussion. + * Added initial + thoughts: https://github.com/AcademySoftwareFoundation/OpenCue/issues/218#issuecomment-1016904933 + * Need more research into integration with Active Directory. + * Permissions model: proposal to be written up. + * Expand DCC plugins + * High priority: Blender, Houdini + * Important for wider user adoption. + * Worth writing to the user group to see what folks have already. + * No further progress. From b2d4fc220c9440d958c3141d56b06473bad1fe40 Mon Sep 17 00:00:00 2001 From: Yaash Jain Date: Thu, 21 Jul 2022 11:13:01 -0700 Subject: [PATCH 210/277] Add OOM Increase field to Service Properties (#1160) * Add OOM Increase field to Service Properties Allow setting minimum memory increase for services through Cuecommander * Make range for OOM increase field more accurate Previously, the min. value for the input was 512. This caused confusion when db value for min_memory_increase was 0. Also includes check to prevent users from setting the field to 0. --- cuegui/cuegui/ServiceDialog.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/cuegui/cuegui/ServiceDialog.py b/cuegui/cuegui/ServiceDialog.py index bdeec4dd6..b29a13c23 100644 --- a/cuegui/cuegui/ServiceDialog.py +++ b/cuegui/cuegui/ServiceDialog.py @@ -72,6 +72,9 @@ def __init__(self, parent=None): self.timeout_llu = QtWidgets.QSpinBox(self) self.timeout_llu.setRange(0, 4320) self.timeout_llu.setValue(0) + self.min_memory_increase = QtWidgets.QSpinBox(self) + self.min_memory_increase.setRange(0, int(self._cfg().get('max_memory', 48)) * 1024) + self.min_memory_increase.setValue(0) layout = QtWidgets.QGridLayout(self) layout.addWidget(QtWidgets.QLabel("Name:", self), 0, 0) layout.addWidget(self.name, 0, 1) @@ -89,15 +92,17 @@ def __init__(self, parent=None): layout.addWidget(self.timeout, 6, 1) layout.addWidget(QtWidgets.QLabel("Timeout LLU (in minutes):", self), 7, 0) layout.addWidget(self.timeout_llu, 7, 1) + layout.addWidget(QtWidgets.QLabel("OOM Increase MB:", self), 8, 0) + layout.addWidget(self.min_memory_increase, 8, 1) self._tags_w = cuegui.TagsWidget.TagsWidget(allowed_tags=cuegui.Constants.ALLOWED_TAGS) - layout.addWidget(self._tags_w, 8, 0, 1, 2) + layout.addWidget(self._tags_w, 9, 0, 1, 2) self.__buttons = QtWidgets.QDialogButtonBox(QtWidgets.QDialogButtonBox.Save, QtCore.Qt.Horizontal, self) self.__buttons.setDisabled(True) - layout.addWidget(self.__buttons, 9, 1) + layout.addWidget(self.__buttons, 10, 1) self.__buttons.accepted.connect(self.save) # pylint: disable=no-member @@ -128,6 +133,8 @@ def setService(self, service): self._tags_w.set_tags(service.data.tags) self.timeout.setValue(service.data.timeout) self.timeout_llu.setValue(service.data.timeout_llu) + self.min_memory_increase.setValue(service.data.min_memory_increase // 1024) + self.__service = service.data def new(self): """ @@ -144,6 +151,7 @@ def new(self): self.min_gpu_memory.setValue(self.gpu_min_mb) self.timeout.setValue(0) self.timeout_llu.setValue(0) + self.min_memory_increase.setValue(2048) self._tags_w.set_tags(['general']) def save(self): @@ -160,6 +168,11 @@ def save(self): QtWidgets.QMessageBox.critical(self, "Error", "The service name must alphanumeric.") return + if self.min_memory_increase.value() <= 0: + QtWidgets.QMessageBox.critical(self, "Error", + "The minimum memory increase must be more than 0 MB") + return + service = opencue.wrappers.service.Service() if self.__service: service.data.id = self.__service.data.id @@ -171,6 +184,7 @@ def save(self): service.setMinGpuMemory(self.min_gpu_memory.value() * 1024) service.setTimeout(self.timeout.value()) service.setTimeoutLLU(self.timeout_llu.value()) + service.setMinMemoryIncrease(self.min_memory_increase.value() * 1024) service.setTags(self._tags_w.get_tags()) self.saved.emit(service) @@ -330,4 +344,4 @@ def __init__(self, show, parent=None): self.setWindowTitle("Services") self.setAttribute(QtCore.Qt.WA_DeleteOnClose) self.setSizeGripEnabled(True) - self.resize(620, 420) + self.resize(700, 700) From 1d59497ac6bc5b8a8756839f59e74fa9b7172658 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 1 Aug 2022 15:47:20 -0400 Subject: [PATCH 211/277] Notes from July 20 TSC meeting. (#1179) --- tsc/meetings/2022-07-20.md | 45 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tsc/meetings/2022-07-20.md diff --git a/tsc/meetings/2022-07-20.md b/tsc/meetings/2022-07-20.md new file mode 100644 index 000000000..99037e579 --- /dev/null +++ b/tsc/meetings/2022-07-20.md @@ -0,0 +1,45 @@ +# OpenCue TSC Meeting Notes 20 July 2022 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Open Source Days + * Sections + * New stuff + * Improved Windows support, including hybrid farms with multiple OSes (multiple log roots, path conversion + in CueGUI) + * Improved GPU support + * New scheduler modes + * Scitech investigations + * Github stats + * Last year 34 -> 41 unique contributors from 10+ organizations + * Grown committer pool to 8 committers + * 102 PRs merged = 2/week + * Issues + * Over past year, 27 unique people have submitted issues. + * Would be nice to count people who have contributed to issues as well, many folks will chime in but not + necessarily open their own issues. + * Any way to do this? + * Contributor pool significantly more diverse + * In the past year 50% of commits came from "founding" organizations. + * In past years / all time this was ~70%. + * Good to highlight project is not dependent on the original creators. + * SPI update + * Fixing things from migration + * Working on upstream merge issues + * When was oracle removed? Apr 2021 + * Potential good story: migration cue3 -> opencue -> opencue master branch + * Stats on shows that have used opencue + * Anything usable from scitech presentation? + * Q&A + * Let's collect potential warmup questions + * We could also maybe use the LF poll tool. For example to vote on feature requests. + * Brian to create new doc to polish this outline. + * Presenters + * Diego: in-person, run through the project update then open up Q&A + * Ben: virtual + * Greg: in-person + * Brian to email LF organizers with contact info. +* Record video on using opencue from wrangler perspective? From Noel? Let's check in after OSD event. +* Yaash: introductions. Waiting on a code review. From e8660d99c08ea0ca17e7fc9b6bb1eb351a879c59 Mon Sep 17 00:00:00 2001 From: Akim Ruslanov Date: Thu, 11 Aug 2022 17:03:41 -0700 Subject: [PATCH 212/277] Open log files in binary mode for Python 3 comp (#1182) --- cuegui/cuegui/Utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cuegui/cuegui/Utils.py b/cuegui/cuegui/Utils.py index 17df4e559..8d41df3cf 100644 --- a/cuegui/cuegui/Utils.py +++ b/cuegui/cuegui/Utils.py @@ -459,17 +459,17 @@ def getLastLine(path): ansiEscape = r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]' try: - fp = open(path, 'r') + fp = open(path, 'rb') fp.seek(0, 2) backseek = min(4096, fp.tell()) fp.seek(-backseek, 1) buf = fp.read(4096) - newline_pos = buf.rfind("\n",0,len(buf)-1) + newline_pos = buf.rfind(b'\n', 0, len(buf)-1) fp.close() - line = buf[newline_pos+1:].strip() + line = buf[newline_pos+1:].strip().decode("utf-8") return re.sub(ansiEscape, "", line) except IOError: From d530cc74a7fcb9ebaea636a570d6d44ce6c31011 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 15 Aug 2022 16:25:51 -0400 Subject: [PATCH 213/277] Restore build.gradle original filename. (#1184) --- cuebot/{build.gradle.orig => build.gradle} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cuebot/{build.gradle.orig => build.gradle} (100%) diff --git a/cuebot/build.gradle.orig b/cuebot/build.gradle similarity index 100% rename from cuebot/build.gradle.orig rename to cuebot/build.gradle From e7c38c6bf5a39eec650a1f7a3851dabcc10d6f48 Mon Sep 17 00:00:00 2001 From: Akim Ruslanov Date: Tue, 23 Aug 2022 09:27:45 -0700 Subject: [PATCH 214/277] cuebot: Make frames and layers readonly after job is done (#1164) * Make frames and layers readonly after job is done * Add prop to toggle viewing done jobs as readonly * Encapsulate editing response observer * Refactor attemptChange method into utils --- .../imageworks/spcue/servant/ManageJob.java | 249 +++++++++------- .../imageworks/spcue/servant/ManageLayer.java | 269 +++++++++++------- .../imageworks/spcue/servant/ServantUtil.java | 28 ++ cuebot/src/main/resources/opencue.properties | 6 + 4 files changed, 343 insertions(+), 209 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java index f7c765636..01601998a 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java @@ -25,6 +25,8 @@ import io.grpc.Status; import io.grpc.stub.StreamObserver; import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; import org.springframework.dao.EmptyResultDataAccessException; import com.imageworks.spcue.BuildableJob; @@ -156,6 +158,8 @@ import com.imageworks.spcue.util.Convert; import com.imageworks.spcue.util.FrameSet; +import static com.imageworks.spcue.servant.ServantUtil.attemptChange; + public class ManageJob extends JobInterfaceGrpc.JobInterfaceImplBase { private static final Logger logger = Logger.getLogger(ManageJob.class); private Whiteboard whiteboard; @@ -173,6 +177,9 @@ public class ManageJob extends JobInterfaceGrpc.JobInterfaceImplBase { private JobInterface job; private FrameSearchFactory frameSearchFactory; private JobSearchFactory jobSearchFactory; + private final String property = "frame.finished_jobs_readonly"; + @Autowired + private Environment env; @Override public void findJob(JobFindJobRequest request, StreamObserver responseObserver) { @@ -357,9 +364,11 @@ public void resume(JobResumeRequest request, StreamObserver r public void setMaxCores(JobSetMaxCoresRequest request, StreamObserver responseObserver) { try{ setupJobData(request.getJob()); - jobDao.updateMaxCores(job, Convert.coresToWholeCoreUnits(request.getVal())); - responseObserver.onNext(JobSetMaxCoresResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + jobDao.updateMaxCores(job, Convert.coresToWholeCoreUnits(request.getVal())); + responseObserver.onNext(JobSetMaxCoresResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -417,9 +426,11 @@ public void setMinGpus(JobSetMinGpusRequest request, StreamObserver responseObserver) { try{ setupJobData(request.getJob()); - jobDao.updatePriority(job, request.getVal()); - responseObserver.onNext(JobSetPriorityResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + jobDao.updatePriority(job, request.getVal()); + responseObserver.onNext(JobSetPriorityResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -449,13 +460,15 @@ public void getCurrent(JobGetCurrentRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - manageQueue.execute( - new DispatchEatFrames( - frameSearchFactory.create(job, request.getReq()), - new Source(request.toString()), - jobManagerSupport)); - responseObserver.onNext(JobEatFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + manageQueue.execute( + new DispatchEatFrames( + frameSearchFactory.create(job, request.getReq()), + new Source(request.toString()), + jobManagerSupport)); + responseObserver.onNext(JobEatFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -468,13 +481,15 @@ public void eatFrames(JobEatFramesRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - manageQueue.execute( - new DispatchKillFrames( - frameSearchFactory.create(job, request.getReq()), - new Source(request.toString()), - jobManagerSupport)); - responseObserver.onNext(JobKillFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + manageQueue.execute( + new DispatchKillFrames( + frameSearchFactory.create(job, request.getReq()), + new Source(request.toString()), + jobManagerSupport)); + responseObserver.onNext(JobKillFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -488,11 +503,13 @@ public void markDoneFrames(JobMarkDoneFramesRequest request, StreamObserver responseObserver) { try{ setupJobData(request.getJob()); - manageQueue.execute( - new DispatchSatisfyDepends( - frameSearchFactory.create(job, request.getReq()), jobManagerSupport)); - responseObserver.onNext(JobMarkDoneFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + manageQueue.execute( + new DispatchSatisfyDepends( + frameSearchFactory.create(job, request.getReq()), jobManagerSupport)); + responseObserver.onNext(JobMarkDoneFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -505,13 +522,15 @@ public void markDoneFrames(JobMarkDoneFramesRequest request, public void retryFrames(JobRetryFramesRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - manageQueue.execute( - new DispatchRetryFrames( - frameSearchFactory.create(job, request.getReq()), - new Source(request.toString()), - jobManagerSupport)); - responseObserver.onNext(JobRetryFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + manageQueue.execute( + new DispatchRetryFrames( + frameSearchFactory.create(job, request.getReq()), + new Source(request.toString()), + jobManagerSupport)); + responseObserver.onNext(JobRetryFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -524,9 +543,11 @@ public void retryFrames(JobRetryFramesRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - jobDao.updateAutoEat(job, request.getValue()); - responseObserver.onNext(JobSetAutoEatResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + jobDao.updateAutoEat(job, request.getValue()); + responseObserver.onNext(JobSetAutoEatResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -540,13 +561,15 @@ public void createDependencyOnFrame(JobCreateDependencyOnFrameRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - JobOnFrame depend = new JobOnFrame(job, - jobManager.getFrameDetail(request.getFrame().getId())); - dependManager.createDepend(depend); - responseObserver.onNext(JobCreateDependencyOnFrameResponse.newBuilder() - .setDepend(whiteboard.getDepend(depend)) - .build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + JobOnFrame depend = new JobOnFrame(job, + jobManager.getFrameDetail(request.getFrame().getId())); + dependManager.createDepend(depend); + responseObserver.onNext(JobCreateDependencyOnFrameResponse.newBuilder() + .setDepend(whiteboard.getDepend(depend)) + .build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -560,13 +583,15 @@ public void createDependencyOnJob(JobCreateDependencyOnJobRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - JobOnJob depend = new JobOnJob(job, - jobManager.getJobDetail(request.getOnJob().getId())); - dependManager.createDepend(depend); - responseObserver.onNext(JobCreateDependencyOnJobResponse.newBuilder() - .setDepend(whiteboard.getDepend(depend)) - .build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + JobOnJob depend = new JobOnJob(job, + jobManager.getJobDetail(request.getOnJob().getId())); + dependManager.createDepend(depend); + responseObserver.onNext(JobCreateDependencyOnJobResponse.newBuilder() + .setDepend(whiteboard.getDepend(depend)) + .build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -580,13 +605,15 @@ public void createDependencyOnLayer(JobCreateDependencyOnLayerRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - JobOnLayer depend = new JobOnLayer(job, - jobManager.getLayerDetail(request.getLayer().getId())); - dependManager.createDepend(depend); - responseObserver.onNext(JobCreateDependencyOnLayerResponse.newBuilder() - .setDepend(whiteboard.getDepend(depend)) - .build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + JobOnLayer depend = new JobOnLayer(job, + jobManager.getLayerDetail(request.getLayer().getId())); + dependManager.createDepend(depend); + responseObserver.onNext(JobCreateDependencyOnLayerResponse.newBuilder() + .setDepend(whiteboard.getDepend(depend)) + .build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -671,9 +698,11 @@ public void getUpdatedFrames(JobGetUpdatedFramesRequest request, StreamObserver< public void setMaxRetries(JobSetMaxRetriesRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - jobDao.updateMaxFrameRetries(job, request.getMaxRetries()); - responseObserver.onNext(JobSetMaxRetriesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + jobDao.updateMaxFrameRetries(job, request.getMaxRetries()); + responseObserver.onNext(JobSetMaxRetriesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -723,9 +752,11 @@ public void getComments(JobGetCommentsRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - manageQueue.execute(new DispatchDropDepends(job, request.getTarget(), dependManager)); - responseObserver.onNext(JobDropDependsResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + manageQueue.execute(new DispatchDropDepends(job, request.getTarget(), dependManager)); + responseObserver.onNext(JobDropDependsResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -738,9 +769,11 @@ public void dropDepends(JobDropDependsRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - jobDao.updateParent(job, groupManager.getGroupDetail(request.getGroupId())); - responseObserver.onNext(JobSetGroupResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + jobDao.updateParent(job, groupManager.getGroupDetail(request.getGroupId())); + responseObserver.onNext(JobSetGroupResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -754,10 +787,12 @@ public void markAsWaiting(JobMarkAsWaitingRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - jobManagerSupport.markFramesAsWaiting( - frameSearchFactory.create(job, request.getReq()), new Source(request.toString())); - responseObserver.onNext(JobMarkAsWaitingResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + jobManagerSupport.markFramesAsWaiting( + frameSearchFactory.create(job, request.getReq()), new Source(request.toString())); + responseObserver.onNext(JobMarkAsWaitingResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -771,10 +806,12 @@ public void reorderFrames(JobReorderFramesRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - manageQueue.execute(new DispatchReorderFrames(job, - new FrameSet(request.getRange()), request.getOrder(), jobManagerSupport)); - responseObserver.onNext(JobReorderFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + manageQueue.execute(new DispatchReorderFrames(job, + new FrameSet(request.getRange()), request.getOrder(), jobManagerSupport)); + responseObserver.onNext(JobReorderFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -804,10 +841,12 @@ public void staggerFrames(JobStaggerFramesRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - manageQueue.execute( - new DispatchStaggerFrames(job, request.getRange(), request.getStagger(), jobManagerSupport)); - responseObserver.onNext(JobStaggerFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + manageQueue.execute( + new DispatchStaggerFrames(job, request.getRange(), request.getStagger(), jobManagerSupport)); + responseObserver.onNext(JobStaggerFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -820,31 +859,33 @@ public void staggerFrames(JobStaggerFramesRequest request, public void addRenderPartition(JobAddRenderPartRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - LocalHostAssignment lha = new LocalHostAssignment(); - lha.setJobId(job.getId()); - lha.setThreads(request.getThreads()); - lha.setMaxCoreUnits(request.getMaxCores() * 100); - lha.setMaxMemory(request.getMaxMemory()); - lha.setMaxGpuUnits(request.getMaxGpus()); - lha.setMaxGpuMemory(request.getMaxGpuMemory()); - lha.setType(RenderPartitionType.JOB_PARTITION); - - if (localBookingSupport.bookLocal(job, request.getHost(), request.getUsername(), lha)) { - try { - RenderPartition renderPart = whiteboard.getRenderPartition(lha); - responseObserver.onNext(JobAddRenderPartResponse.newBuilder() - .setRenderPartition(renderPart) - .build()); - responseObserver.onCompleted(); - } catch (EmptyResultDataAccessException e) { + if (attemptChange(env, property, jobManager, job, responseObserver)) { + LocalHostAssignment lha = new LocalHostAssignment(); + lha.setJobId(job.getId()); + lha.setThreads(request.getThreads()); + lha.setMaxCoreUnits(request.getMaxCores() * 100); + lha.setMaxMemory(request.getMaxMemory()); + lha.setMaxGpuUnits(request.getMaxGpus()); + lha.setMaxGpuMemory(request.getMaxGpuMemory()); + lha.setType(RenderPartitionType.JOB_PARTITION); + + if (localBookingSupport.bookLocal(job, request.getHost(), request.getUsername(), lha)) { + try { + RenderPartition renderPart = whiteboard.getRenderPartition(lha); + responseObserver.onNext(JobAddRenderPartResponse.newBuilder() + .setRenderPartition(renderPart) + .build()); + responseObserver.onCompleted(); + } catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to allocate render partition to host.") + .asRuntimeException()); + } + } else { responseObserver.onError(Status.INTERNAL - .withDescription("Failed to allocate render partition to host.") + .withDescription("Failed to find suitable frames.") .asRuntimeException()); } - } else { - responseObserver.onError(Status.INTERNAL - .withDescription("Failed to find suitable frames.") - .asRuntimeException()); } } catch (EmptyResultDataAccessException e) { @@ -858,10 +899,12 @@ public void addRenderPartition(JobAddRenderPartRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - JobDetail jobDetail = jobManager.getJobDetail(job.getJobId()); - filterManager.runFiltersOnJob(jobDetail); - responseObserver.onNext(JobRunFiltersResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + JobDetail jobDetail = jobManager.getJobDetail(job.getJobId()); + filterManager.runFiltersOnJob(jobDetail); + responseObserver.onNext(JobRunFiltersResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageLayer.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageLayer.java index 5bb5a022f..7f61bc287 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageLayer.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageLayer.java @@ -22,6 +22,8 @@ import com.google.protobuf.Descriptors; import io.grpc.Status; import io.grpc.stub.StreamObserver; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; import org.springframework.dao.EmptyResultDataAccessException; import com.imageworks.spcue.LayerDetail; @@ -123,6 +125,8 @@ import com.imageworks.spcue.util.Convert; import com.imageworks.spcue.util.FrameSet; +import static com.imageworks.spcue.servant.ServantUtil.attemptChange; + public class ManageLayer extends LayerInterfaceGrpc.LayerInterfaceImplBase { private LayerDetail layer; @@ -135,6 +139,9 @@ public class ManageLayer extends LayerInterfaceGrpc.LayerInterfaceImplBase { private Whiteboard whiteboard; private LocalBookingSupport localBookingSupport; private FrameSearchFactory frameSearchFactory; + private final String property = "layer.finished_jobs_readonly"; + @Autowired + private Environment env; @Override public void findLayer(LayerFindLayerRequest request, StreamObserver responseObserver) { @@ -169,10 +176,12 @@ public void getLayer(LayerGetLayerRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - manageQueue.execute(new DispatchEatFrames(frameSearch, - new Source(request.toString()), jobManagerSupport)); - responseObserver.onNext(LayerEatFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + manageQueue.execute(new DispatchEatFrames(frameSearch, + new Source(request.toString()), jobManagerSupport)); + responseObserver.onNext(LayerEatFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override @@ -188,118 +197,142 @@ public void getFrames(LayerGetFramesRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - manageQueue.execute(new DispatchKillFrames(frameSearch, - new Source(request.toString()), jobManagerSupport)); - responseObserver.onNext(LayerKillFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + manageQueue.execute(new DispatchKillFrames(frameSearch, + new Source(request.toString()), jobManagerSupport)); + responseObserver.onNext(LayerKillFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void markdoneFrames(LayerMarkdoneFramesRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - manageQueue.execute(new DispatchSatisfyDepends(layer, jobManagerSupport)); - responseObserver.onNext(LayerMarkdoneFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + manageQueue.execute(new DispatchSatisfyDepends(layer, jobManagerSupport)); + responseObserver.onNext(LayerMarkdoneFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void retryFrames(LayerRetryFramesRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - manageQueue.execute(new DispatchRetryFrames(frameSearch, - new Source(request.toString()), jobManagerSupport)); - responseObserver.onNext(LayerRetryFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + manageQueue.execute(new DispatchRetryFrames(frameSearch, + new Source(request.toString()), jobManagerSupport)); + responseObserver.onNext(LayerRetryFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void setTags(LayerSetTagsRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - layerDao.updateLayerTags(layer, new HashSet<>(request.getTagsList())); - responseObserver.onNext(LayerSetTagsResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + layerDao.updateLayerTags(layer, new HashSet<>(request.getTagsList())); + responseObserver.onNext(LayerSetTagsResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void setMinCores(LayerSetMinCoresRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - jobManager.setLayerMinCores(layer, Convert.coresToCoreUnits(request.getCores())); - responseObserver.onNext(LayerSetMinCoresResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + jobManager.setLayerMinCores(layer, Convert.coresToCoreUnits(request.getCores())); + responseObserver.onNext(LayerSetMinCoresResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void setMinGpus(LayerSetMinGpusRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - jobManager.setLayerMinGpus(layer, request.getMinGpus()); - responseObserver.onNext(LayerSetMinGpusResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + jobManager.setLayerMinGpus(layer, request.getMinGpus()); + responseObserver.onNext(LayerSetMinGpusResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void setMinMemory(LayerSetMinMemoryRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - layerDao.updateLayerMinMemory(layer, request.getMemory()); - responseObserver.onNext(LayerSetMinMemoryResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + layerDao.updateLayerMinMemory(layer, request.getMemory()); + responseObserver.onNext(LayerSetMinMemoryResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void setMinGpuMemory(LayerSetMinGpuMemoryRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - layerDao.updateLayerMinGpuMemory(layer, request.getGpuMemory()); - responseObserver.onNext(LayerSetMinGpuMemoryResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + layerDao.updateLayerMinGpuMemory(layer, request.getGpuMemory()); + responseObserver.onNext(LayerSetMinGpuMemoryResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void createDependencyOnFrame(LayerCreateDependOnFrameRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - LayerOnFrame depend = new LayerOnFrame(layer, jobManager.getFrameDetail(request.getFrame().getId())); - dependManager.createDepend(depend); - responseObserver.onNext(LayerCreateDependOnFrameResponse.newBuilder() - .setDepend(whiteboard.getDepend(depend)) - .build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + LayerOnFrame depend = new LayerOnFrame(layer, jobManager.getFrameDetail(request.getFrame().getId())); + dependManager.createDepend(depend); + responseObserver.onNext(LayerCreateDependOnFrameResponse.newBuilder() + .setDepend(whiteboard.getDepend(depend)) + .build()); + responseObserver.onCompleted(); + } } @Override public void createDependencyOnJob(LayerCreateDependOnJobRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - LayerOnJob depend = new LayerOnJob(layer, jobManager.getJobDetail(request.getJob().getId())); - dependManager.createDepend(depend); - responseObserver.onNext(LayerCreateDependOnJobResponse.newBuilder() - .setDepend(whiteboard.getDepend(depend)) - .build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + LayerOnJob depend = new LayerOnJob(layer, jobManager.getJobDetail(request.getJob().getId())); + dependManager.createDepend(depend); + responseObserver.onNext(LayerCreateDependOnJobResponse.newBuilder() + .setDepend(whiteboard.getDepend(depend)) + .build()); + responseObserver.onCompleted(); + } } @Override public void createDependencyOnLayer(LayerCreateDependOnLayerRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - LayerOnLayer depend = new LayerOnLayer(layer, jobManager.getLayerDetail(request.getDependOnLayer().getId())); - dependManager.createDepend(depend); - responseObserver.onNext(LayerCreateDependOnLayerResponse.newBuilder() - .setDepend(whiteboard.getDepend(depend)) - .build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + LayerOnLayer depend = new LayerOnLayer(layer, jobManager.getLayerDetail(request.getDependOnLayer().getId())); + dependManager.createDepend(depend); + responseObserver.onNext(LayerCreateDependOnLayerResponse.newBuilder() + .setDepend(whiteboard.getDepend(depend)) + .build()); + responseObserver.onCompleted(); + } } @Override public void createFrameByFrameDependency(LayerCreateFrameByFrameDependRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - FrameByFrame depend = new FrameByFrame(layer, jobManager.getLayerDetail(request.getDependLayer().getId())); - dependManager.createDepend(depend); - responseObserver.onNext(LayerCreateFrameByFrameDependResponse.newBuilder() - .setDepend(whiteboard.getDepend(depend)) - .build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + FrameByFrame depend = new FrameByFrame(layer, jobManager.getLayerDetail(request.getDependLayer().getId())); + dependManager.createDepend(depend); + responseObserver.onNext(LayerCreateFrameByFrameDependResponse.newBuilder() + .setDepend(whiteboard.getDepend(depend)) + .build()); + responseObserver.onCompleted(); + } } @Override @@ -326,94 +359,112 @@ public void getWhatThisDependsOn(LayerGetWhatThisDependsOnRequest request, public void dropDepends(LayerDropDependsRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - manageQueue.execute(new DispatchDropDepends(layer, request.getTarget(), dependManager)); - responseObserver.onNext(LayerDropDependsResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + manageQueue.execute(new DispatchDropDepends(layer, request.getTarget(), dependManager)); + responseObserver.onNext(LayerDropDependsResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void dropLimit(LayerDropLimitRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - layerDao.dropLimit(layer, request.getLimitId()); - responseObserver.onNext(LayerDropLimitResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + layerDao.dropLimit(layer, request.getLimitId()); + responseObserver.onNext(LayerDropLimitResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void reorderFrames(LayerReorderFramesRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - manageQueue.execute(new DispatchReorderFrames(layer, new FrameSet(request.getRange()), request.getOrder(), - jobManagerSupport)); - responseObserver.onNext(LayerReorderFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + manageQueue.execute(new DispatchReorderFrames(layer, new FrameSet(request.getRange()), request.getOrder(), + jobManagerSupport)); + responseObserver.onNext(LayerReorderFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void staggerFrames(LayerStaggerFramesRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - manageQueue.execute(new DispatchStaggerFrames(layer, request.getRange(), request.getStagger(), - jobManagerSupport)); - responseObserver.onNext(LayerStaggerFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + manageQueue.execute(new DispatchStaggerFrames(layer, request.getRange(), request.getStagger(), + jobManagerSupport)); + responseObserver.onNext(LayerStaggerFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void setThreadable(LayerSetThreadableRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - layerDao.updateThreadable(layer, request.getThreadable()); - responseObserver.onNext(LayerSetThreadableResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + layerDao.updateThreadable(layer, request.getThreadable()); + responseObserver.onNext(LayerSetThreadableResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void setTimeout(LayerSetTimeoutRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - layerDao.updateTimeout(layer, request.getTimeout()); - responseObserver.onNext(LayerSetTimeoutResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + layerDao.updateTimeout(layer, request.getTimeout()); + responseObserver.onNext(LayerSetTimeoutResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void setTimeoutLLU(LayerSetTimeoutLLURequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - layerDao.updateTimeoutLLU(layer, request.getTimeoutLlu()); - responseObserver.onNext(LayerSetTimeoutLLUResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + layerDao.updateTimeoutLLU(layer, request.getTimeoutLlu()); + responseObserver.onNext(LayerSetTimeoutLLUResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void addLimit(LayerAddLimitRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - layerDao.addLimit(layer, request.getLimitId()); - responseObserver.onNext(LayerAddLimitResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + layerDao.addLimit(layer, request.getLimitId()); + responseObserver.onNext(LayerAddLimitResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void addRenderPartition(LayerAddRenderPartitionRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - LocalHostAssignment lha = new LocalHostAssignment(); - lha.setThreads(request.getThreads()); - lha.setMaxCoreUnits(request.getMaxCores() * 100); - lha.setMaxMemory(request.getMaxMemory()); - lha.setMaxGpuUnits(request.getMaxGpus()); - lha.setMaxGpuMemory(request.getMaxGpuMemory()); - lha.setType(RenderPartitionType.LAYER_PARTITION); - if (localBookingSupport.bookLocal(layer, request.getHost(), request.getUsername(), lha)) { - RenderPartition partition = whiteboard.getRenderPartition(lha); - responseObserver.onNext(LayerAddRenderPartitionResponse.newBuilder() - .setRenderPartition(partition) - .build()); - responseObserver.onCompleted(); - } else { - responseObserver.onError(Status.INTERNAL - .withDescription("Failed to find suitable frames.") - .asRuntimeException()); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + LocalHostAssignment lha = new LocalHostAssignment(); + lha.setThreads(request.getThreads()); + lha.setMaxCoreUnits(request.getMaxCores() * 100); + lha.setMaxMemory(request.getMaxMemory()); + lha.setMaxGpuUnits(request.getMaxGpus()); + lha.setMaxGpuMemory(request.getMaxGpuMemory()); + lha.setType(RenderPartitionType.LAYER_PARTITION); + if (localBookingSupport.bookLocal(layer, request.getHost(), request.getUsername(), lha)) { + RenderPartition partition = whiteboard.getRenderPartition(lha); + responseObserver.onNext(LayerAddRenderPartitionResponse.newBuilder() + .setRenderPartition(partition) + .build()); + responseObserver.onCompleted(); + } else { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find suitable frames.") + .asRuntimeException()); + } } } @@ -422,9 +473,11 @@ public void addRenderPartition(LayerAddRenderPartitionRequest request, public void registerOutputPath(LayerRegisterOutputPathRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - jobManager.registerLayerOutput(layer, request.getSpec()); - responseObserver.onNext(LayerRegisterOutputPathResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + jobManager.registerLayerOutput(layer, request.getSpec()); + responseObserver.onNext(LayerRegisterOutputPathResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override @@ -450,17 +503,21 @@ public void getOutputPaths(LayerGetOutputPathsRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - jobManager.enableMemoryOptimizer(layer, request.getValue()); - responseObserver.onNext(LayerEnableMemoryOptimizerResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + jobManager.enableMemoryOptimizer(layer, request.getValue()); + responseObserver.onNext(LayerEnableMemoryOptimizerResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void setMaxCores(LayerSetMaxCoresRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - jobManager.setLayerMaxCores(layer, Convert.coresToWholeCoreUnits(request.getCores())); - responseObserver.onNext(LayerSetMaxCoresResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + jobManager.setLayerMaxCores(layer, Convert.coresToWholeCoreUnits(request.getCores())); + responseObserver.onNext(LayerSetMaxCoresResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ServantUtil.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ServantUtil.java index a5146947e..a88c8e87d 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ServantUtil.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ServantUtil.java @@ -21,10 +21,19 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; +import com.imageworks.spcue.JobDetail; +import com.imageworks.spcue.JobInterface; import com.imageworks.spcue.LayerInterface; +import com.imageworks.spcue.grpc.job.JobState; import com.imageworks.spcue.grpc.job.Layer; import com.imageworks.spcue.grpc.job.LayerSeq; +import com.imageworks.spcue.service.JobManager; + +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import org.springframework.core.env.Environment; public class ServantUtil { @@ -44,5 +53,24 @@ public static List convertLayerFilterList(LayerSeq layers) { } return result; } + + private static boolean isJobFinished(Environment env, String property, JobManager jobManager, JobInterface job) { + if (env.getProperty(property, String.class) != null && + Objects.equals(env.getProperty(property, String.class), "true")) { + JobDetail jobDetail = jobManager.getJobDetail(job.getJobId()); + return jobDetail.state == JobState.FINISHED; + } + return false; + } + + public static boolean attemptChange(Environment env, String property, JobManager jobManager, JobInterface job, StreamObserver responseObserver) { + if (ServantUtil.isJobFinished(env, property, jobManager, job)) { + responseObserver.onError(Status.FAILED_PRECONDITION + .withDescription("Finished jobs are readonly") + .asRuntimeException()); + return false; + } + return true; + } } diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index 6d541bd00..7f35eb90c 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -138,3 +138,9 @@ smtp_host=smtp # These shows won't be deactivated by the scheduled tasks protected_shows=pipe,swtest,edu + +# These flags determine whether or not layers/frames will be readonly when job is finished. +# If flags are set as true, layers/frames cannot be retried, eaten, edited dependency on, etc. +# In order to toggle the same functionility on cuegui side, set flags in cue_resources.yaml +layer.finished_jobs_readonly=false +frame.finished_jobs_readonly=false \ No newline at end of file From cd0fc3d9c32c1af30efe02409c765b26e9947ce4 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Sat, 17 Sep 2022 23:29:19 -0400 Subject: [PATCH 215/277] TSC notes from Aug 17 and Sep 14. (#1200) --- tsc/meetings/2022-08-17.md | 39 +++++++++++++++++++++++++++ tsc/meetings/2022-09-14.md | 54 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 tsc/meetings/2022-08-17.md create mode 100644 tsc/meetings/2022-09-14.md diff --git a/tsc/meetings/2022-08-17.md b/tsc/meetings/2022-08-17.md new file mode 100644 index 000000000..af88fcbe5 --- /dev/null +++ b/tsc/meetings/2022-08-17.md @@ -0,0 +1,39 @@ +# OpenCue TSC Meeting Notes 17 Aug 2022 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Siggraph/OSD debrief + * Event attendance ~20 in room, 50-60 total + * Q: who is using opencue? + * Should we change the TSC meeting time? To account for EU participants. + * OpenCue on GKE + * Some peculiarities there as both OpenCue and GKE act as schedulers, in different ways but with some + overlapping features. + * How can we help decide which pods can be killed? + * Could maybe use metrics/prometheus data as a source. + * We should add a helm chart for the repo. + * Both kubernetes and OpenCue want to do bin packing, maybe we could find a way to help them play better + together. + * PyPI packages still a need + * Look at "poetry"? + * Protobuf compiling is an obstacle as it breaks modularity. + * Can we make compiled proto its own package that other components depend on? + * Not getting consistent responses to issues / threads. + * Currently this is best effort. + * TSC should do an issue/PR cleanout. + * Let's try to make a standard process to keep up with it. + * Finish setting up stale issue closer to help with cleanout. + * Bring up with TAC to see if other projects have similar issues or thoughts. +* M1 issues + * Having some difficulties running OpenCue on a local M1 macbook, which is primary development machine for an + increasing number of people. + * Need to upgrade some dependencies like protoc. + * Docker build issues due to architecture mismatch. + * PySide2 might not be supported anymore. + * Has VFX reference platform dealt with this? +* Unit test failures + * Coming from code that hasn't changed + * Possibly due to changes in ref platform docker images? + * More investigation needed. diff --git a/tsc/meetings/2022-09-14.md b/tsc/meetings/2022-09-14.md new file mode 100644 index 000000000..5b4df0279 --- /dev/null +++ b/tsc/meetings/2022-09-14.md @@ -0,0 +1,54 @@ +# OpenCue TSC Meeting Notes 14 Sep 2022 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Upgrade Cuebot dependencies + * M1 has issues with grpc and protoc. + * PR to upgrade, needs review: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1185 + * Will review soon. +* PySide qApp problem + * This is causing unit test / CI failures. + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1192 + * Fix in progress https://github.com/AcademySoftwareFoundation/OpenCue/pull/1193 +* PySide6 + * PySide2 is deprecated, wheels are no longer published for newer architectures like M1. + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1194 + * Fix in progress https://github.com/AcademySoftwareFoundation/OpenCue/pull/1195 + * Will need some help testing. + * VFX reference platform still uses Qt 5 and will until at least 2024. This may create an issue if we drop support + for PySide2 which depends on Qt 5. PySide6 uses Qt 6. + * Can the pyside2/6 versions coexist? + * We could also create a wrapper / compatibility layer. This will likely be useful in the future as PySide will + change again. + * OpenTimelineIO has dealt with the same issue. + * See Github issues: + * https://github.com/AcademySoftwareFoundation/OpenTimelineIO/issues/1215 + * https://github.com/AcademySoftwareFoundation/OpenTimelineIO/issues/1220 + * https://github.com/AcademySoftwareFoundation/OpenTimelineIO/issues/1261 + * Appears they've fixed it, fix looks very similar to our in-progress fix. + * Has the VFX reference platform dealt with architectures at all? arm is getting more popular, and the switch to arm + created/exposed these issues. +* Redis + * Considering adding Redis into the mix to help with Cuebot/database load and Cuebot/RQD communication issues. + * Draft PR: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1187 + * Brian has left some initial comments on that PR. + * This will break the connection between RQD and a single Cuebot. Currently RQD must communicate with a single + Cuebot via gRPC and must be restarted to switch to a different one. + * Using Redis is highly experimental, not production tested. + * The PR must be updated to be completely optional, coexist with the existing code. We need to collect performance + data and decide whether we want to proceed with this project. + * Redis should probably be optional in perpetuity. Default install should not require it, but it can be used to + increase performance for larger deployments. +* Docker image issues + * Can't build on M1 mac. + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1196 + * Also slowing down development. +* Need to do a release + * Blocked on qApp issue as this causes CI failures, we can't use our release pipeline. + * Any update on integration test script? + * This would help speed up the release by automating the QA procedure. + * There was a basic script that had some issues, not much progress. + * Work may resume soon. + * If the in-progress script is sharable, that would save some time. From bce47b590fa097c577ba694fb69f985d54251561 Mon Sep 17 00:00:00 2001 From: Mark Sisson <5761292+marksisson@users.noreply.github.com> Date: Tue, 20 Sep 2022 14:51:11 -0500 Subject: [PATCH 216/277] Add .envrc to .gitignore (#1207) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9de17bdf1..12d65dce4 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ venv*/ coverage.xml htmlcov/ /.env +.envrc From 3bb4528472287ad802de478c0f734d51c5d3c3ca Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Thu, 29 Sep 2022 13:36:15 -0400 Subject: [PATCH 217/277] Upgrade gRPC and protobuf dependencies. (#1185) --- VERSION.in | 2 +- cuebot/build.gradle | 8 ++++---- requirements.txt | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/VERSION.in b/VERSION.in index 9f4eca259..5320adc1c 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.20 +0.21 diff --git a/cuebot/build.gradle b/cuebot/build.gradle index 403cc893e..fe5abf3e2 100644 --- a/cuebot/build.gradle +++ b/cuebot/build.gradle @@ -44,7 +44,7 @@ dependencies { compile group: 'com.google.guava', name: 'guava', version: '26.0-android' compile group: 'com.sun.mail', name: 'mailapi', version: '1.5.4' compile group: 'commons-lang', name: 'commons-lang', version: '2.6' - compile group: 'io.grpc', name: 'grpc-all', version: '1.36.2' + compile group: 'io.grpc', name: 'grpc-all', version: '1.47.0' compile group: 'org.apache.activemq', name: 'activemq-pool', version: activemqVersion compile group: 'org.apache.velocity', name: 'velocity', version: '1.7' compile group: 'org.jdom', name: 'jdom', version: '1.1.3' @@ -54,7 +54,7 @@ dependencies { compile group: 'org.springframework', name: 'spring-jms' compile group: 'org.quartz-scheduler', name: 'quartz', version: '2.2.1', { exclude group: 'c3p0', module: 'c3p0' } compile group: 'org.postgresql', name: 'postgresql', version: '42.2.2' - compile group: 'com.google.protobuf', name: 'protobuf-java', version: '3.13.0' + compile group: 'com.google.protobuf', name: 'protobuf-java', version: '3.21.2' compile group: 'log4j', name: 'log4j', version: '1.2.17' compile group: 'org.slf4j', name: 'slf4j-log4j12', version: '1.7.26' @@ -81,12 +81,12 @@ compileTestJava { protobuf { protoc { // The protoc compiler - artifact = 'com.google.protobuf:protoc:3.13.0' + artifact = 'com.google.protobuf:protoc:3.21.2' } plugins { grpc { // Generate gRPC stubs. - artifact = 'io.grpc:protoc-gen-grpc-java:1.7.0' + artifact = 'io.grpc:protoc-gen-grpc-java:1.47.0' } } generateProtoTasks { diff --git a/requirements.txt b/requirements.txt index ff5a63761..f465c5910 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,8 +4,8 @@ future==0.17.1 futures==3.2.0;python_version<"3.0" grpcio==1.26.0;python_version<"3.0" grpcio-tools==1.26.0;python_version<"3.0" -grpcio==1.39.0;python_version>="3.0" -grpcio-tools==1.39.0;python_version>="3.0" +grpcio==1.47.0;python_version>="3.0" +grpcio-tools==1.47.0;python_version>="3.0" mock==2.0.0 packaging==20.9 pathlib==1.0.1;python_version<"3.4" From 3859a5da29f98ed426d0e5b3ca312a4be02fec61 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Thu, 29 Sep 2022 13:37:21 -0400 Subject: [PATCH 218/277] [rqd] Reduce the log level when failing to read a stat file. (#1191) --- rqd/rqd/rqmachine.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 37e1f5771..d4cf918ca 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -283,7 +283,9 @@ def rssUpdate(self, frames): # pylint: disable=broad-except except (OSError, IOError): - log.exception('Failed to read stat/statm file for pid %s', pid) + # Many Linux processes are ephemeral and will disappear before we're able + # to read them. This is not typically indicative of a problem. + log.debug('Failed to read stat/statm file for pid %s', pid) # pylint: disable=too-many-nested-blocks try: From a6ceb2fe3b66aaf7e84fb1af77b994264da95830 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Thu, 29 Sep 2022 13:53:27 -0400 Subject: [PATCH 219/277] Update Dockerfile base images to fix compatibility issues. (#1198) --- cueadmin/Dockerfile | 62 ++++++++------------------------------------ cuebot/Dockerfile | 16 +++--------- cuegui/Dockerfile | 30 ++++----------------- cuesubmit/Dockerfile | 45 +++++--------------------------- pycue/Dockerfile | 56 +++++++-------------------------------- pyoutline/Dockerfile | 58 ++++++++--------------------------------- rqd/Dockerfile | 15 +++-------- 7 files changed, 49 insertions(+), 233 deletions(-) diff --git a/cueadmin/Dockerfile b/cueadmin/Dockerfile index 37caf1d5f..74b318d46 100644 --- a/cueadmin/Dockerfile +++ b/cueadmin/Dockerfile @@ -1,35 +1,14 @@ -# ----------------- -# BUILD -# ----------------- -FROM centos:7 as build - -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "CueAdmin build stage" +FROM python:3.9.14 WORKDIR /src -RUN yum -y install \ - epel-release \ - gcc \ - python-devel - -RUN yum -y install \ - python-pip \ - python36 \ - python36-devel \ - python36-pip - -RUN python -m pip install --upgrade 'pip<21' -RUN python3.6 -m pip install --upgrade pip - -RUN python -m pip install --upgrade 'setuptools<45' -RUN python3.6 -m pip install --upgrade setuptools +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools COPY LICENSE ./ COPY requirements.txt ./ -RUN python -m pip install -r requirements.txt -RUN python3.6 -m pip install -r requirements.txt +RUN python3 -m pip install -r requirements.txt COPY proto/ ./proto COPY pycue/README.md ./pycue/ @@ -37,7 +16,7 @@ COPY pycue/setup.py ./pycue/ COPY pycue/opencue ./pycue/opencue COPY pycue/FileSequence ./pycue/FileSequence -RUN python -m grpc_tools.protoc \ +RUN python3 -m grpc_tools.protoc \ -I=./proto \ --python_out=./pycue/opencue/compiled_proto \ --grpc_python_out=./pycue/opencue/compiled_proto \ @@ -55,32 +34,13 @@ COPY cueadmin/cueadmin ./cueadmin/cueadmin COPY VERSION.in VERSIO[N] ./ RUN test -e VERSION || echo "$(cat VERSION.in)-custom" | tee VERSION -RUN cd pycue && python setup.py install - -RUN cd pycue && python3.6 setup.py install - -# TODO(bcipriano) Lint the code here. (Issue #78) - -RUN cd cueadmin && python setup.py test - -RUN cd cueadmin && python3.6 setup.py test +RUN cd pycue && python3 setup.py install +RUN cd cueadmin && python3 setup.py test RUN cp LICENSE requirements.txt VERSION cueadmin/ RUN versioned_name="cueadmin-$(cat ./VERSION)-all" \ - && mv cueadmin $versioned_name \ - && tar -cvzf $versioned_name.tar.gz $versioned_name/* - - -# ----------------- -# RUN -# ----------------- -FROM centos:7 - -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "CueAdmin runtime stage" - -WORKDIR /opt/opencue - -COPY --from=build /src/cueadmin-*-all.tar.gz ./ - + && mv cueadmin "${versioned_name}" \ + && tar -cvzf "${versioned_name}.tar.gz" ${versioned_name}/* \ + && mkdir -p /opt/opencue \ + && cp "${versioned_name}.tar.gz" /opt/opencue/ diff --git a/cuebot/Dockerfile b/cuebot/Dockerfile index 2dd5ae960..d82b861cc 100644 --- a/cuebot/Dockerfile +++ b/cuebot/Dockerfile @@ -1,19 +1,16 @@ # ----------------- # BUILD # ----------------- -FROM gradle:6.0.1-jdk11 AS build +FROM gradle:6.0.1-jdk13 AS build USER gradle -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "Cuebot build stage" - COPY --chown=gradle:gradle ./cuebot /home/gradle/cuebot/ COPY --chown=gradle:gradle ./proto /home/gradle/proto/ WORKDIR /home/gradle/cuebot -RUN gradle build --stacktrace +RUN gradle build --info --stacktrace COPY --chown=gradle:gradle VERSION.in VERSIO[N] ./ RUN test -e VERSION || echo "$(cat VERSION.in)-custom" | tee VERSION @@ -25,12 +22,8 @@ RUN mv ./build/libs/cuebot.jar ./build/libs/cuebot-$(cat ./VERSION)-all.jar # ----------------- FROM jc21/rpmbuild-centos7:latest AS rpm -# Random first line after from - USER rpmbuilder -RUN echo "Cuebot RPM Stage" - COPY --chown=rpmbuilder:rpmbuilder LICENSE ./ COPY --from=build \ @@ -49,10 +42,7 @@ RUN chmod +x create_rpm.sh && ./create_rpm.sh cuebot "$(cat VERSION)" # ----------------- # RUN # ----------------- -FROM openjdk:11-jre-slim-buster - -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "Cuebot runtime stage" +FROM openjdk:18-slim-buster ARG CUEBOT_GRPC_CUE_PORT=8443 ARG CUEBOT_GRPC_RQD_PORT=8444 diff --git a/cuegui/Dockerfile b/cuegui/Dockerfile index f3cd788ce..8b0b189f9 100644 --- a/cuegui/Dockerfile +++ b/cuegui/Dockerfile @@ -1,10 +1,4 @@ -# ----------------- -# BUILD -# ----------------- -FROM centos:7 as build - -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "CueGUI build stage" +FROM --platform=linux/x86_64 centos:7 WORKDIR /src @@ -26,13 +20,11 @@ RUN yum -y install \ xcb-util-wm.x86_64 RUN yum -y install \ - python-pip \ python36 \ python36-devel \ python36-pip RUN python3.6 -m pip install --upgrade pip - RUN python3.6 -m pip install --upgrade setuptools RUN dbus-uuidgen > /etc/machine-id @@ -74,19 +66,7 @@ RUN cd cuegui && xvfb-run -d python3.6 setup.py test RUN cp LICENSE requirements.txt requirements_gui.txt VERSION cuegui/ RUN versioned_name="cuegui-$(cat ./VERSION)-all" \ - && mv cuegui $versioned_name \ - && tar -cvzf $versioned_name.tar.gz $versioned_name/* - - -# ----------------- -# RUN -# ----------------- -FROM centos:7 - -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "CueGUI runtime stage" - -WORKDIR /opt/opencue - -COPY --from=build /src/cuegui-*-all.tar.gz ./ - + && mv cuegui "${versioned_name}" \ + && tar -cvzf "${versioned_name}.tar.gz" ${versioned_name}/* \ + && mkdir -p /opt/opencue \ + && cp "${versioned_name}.tar.gz" /opt/opencue/ diff --git a/cuesubmit/Dockerfile b/cuesubmit/Dockerfile index 252936fa9..cc8f51178 100644 --- a/cuesubmit/Dockerfile +++ b/cuesubmit/Dockerfile @@ -1,10 +1,4 @@ -# ----------------- -# BUILD -# ----------------- -FROM centos:7 as build - -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "CueSubmit build stage" +FROM --platform=linux/x86_64 centos:7 WORKDIR /src @@ -15,22 +9,17 @@ RUN yum -y install \ python-devel RUN yum -y install \ - python-pip \ python36 \ python36-devel \ python36-pip -RUN python -m pip install --upgrade 'pip<21' RUN python3.6 -m pip install --upgrade pip - -RUN python -m pip install --upgrade 'setuptools<45' RUN python3.6 -m pip install --upgrade setuptools COPY LICENSE ./ COPY requirements.txt ./ COPY requirements_gui.txt ./ -RUN python -m pip install -r requirements.txt -r requirements_gui.txt RUN python3.6 -m pip install -r requirements.txt -r requirements_gui.txt COPY proto/ ./proto @@ -39,7 +28,7 @@ COPY pycue/setup.py ./pycue/ COPY pycue/opencue ./pycue/opencue COPY pycue/FileSequence ./pycue/FileSequence -RUN python -m grpc_tools.protoc \ +RUN python3.6 -m grpc_tools.protoc \ -I=./proto \ --python_out=./pycue/opencue/compiled_proto \ --grpc_python_out=./pycue/opencue/compiled_proto \ @@ -65,36 +54,14 @@ COPY cuesubmit/cuesubmit ./cuesubmit/cuesubmit COPY VERSION.in VERSIO[N] ./ RUN test -e VERSION || echo "$(cat VERSION.in)-custom" | tee VERSION -RUN cd pycue && python setup.py install - RUN cd pycue && python3.6 setup.py install - -RUN cd pyoutline && python setup.py install - RUN cd pyoutline && python3.6 setup.py install - -# TODO(bcipriano) Lint the code here. (Issue #78) - -RUN cd cuesubmit && python setup.py test - RUN cd cuesubmit && python3.6 setup.py test RUN cp LICENSE requirements.txt requirements_gui.txt VERSION cuesubmit/ RUN versioned_name="cuesubmit-$(cat ./VERSION)-all" \ - && mv cuesubmit $versioned_name \ - && tar -cvzf $versioned_name.tar.gz $versioned_name/* - - -# ----------------- -# RUN -# ----------------- -FROM centos:7 - -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "CueSubmit runtime stage" - -WORKDIR /opt/opencue - -COPY --from=build /src/cuesubmit-*-all.tar.gz ./ - + && mv cuesubmit "${versioned_name}" \ + && tar -cvzf "${versioned_name}.tar.gz" ${versioned_name}/* \ + && mkdir -p /opt/opencue \ + && cp "${versioned_name}.tar.gz" /opt/opencue/ diff --git a/pycue/Dockerfile b/pycue/Dockerfile index 54f8a2dc2..e93f9526a 100644 --- a/pycue/Dockerfile +++ b/pycue/Dockerfile @@ -1,35 +1,14 @@ -# ----------------- -# BUILD -# ----------------- -FROM centos:7 as build - -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "PyCue build stage" +FROM python:3.9.14 WORKDIR /src -RUN yum -y install \ - epel-release \ - gcc \ - python-devel - -RUN yum -y install \ - python-pip \ - python36 \ - python36-devel \ - python36-pip - -RUN python -m pip install --upgrade 'pip<21' -RUN python3.6 -m pip install --upgrade pip - -RUN python -m pip install --upgrade 'setuptools<45' -RUN python3.6 -m pip install --upgrade setuptools +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools COPY LICENSE ./ COPY requirements.txt ./ -RUN python -m pip install -r requirements.txt -RUN python3.6 -m pip install -r requirements.txt +RUN python3 -m pip install -r requirements.txt COPY proto/ ./proto COPY pycue/README.md ./pycue/ @@ -38,7 +17,7 @@ COPY pycue/FileSequence ./pycue/FileSequence COPY pycue/tests/ ./pycue/tests COPY pycue/opencue ./pycue/opencue -RUN python -m grpc_tools.protoc \ +RUN python3 -m grpc_tools.protoc \ -I=./proto \ --python_out=./pycue/opencue/compiled_proto \ --grpc_python_out=./pycue/opencue/compiled_proto \ @@ -48,30 +27,15 @@ RUN python -m grpc_tools.protoc \ # for more info. RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py -# TODO(bcipriano) Lint the code here. (Issue #78) - COPY VERSION.in VERSIO[N] ./ RUN test -e VERSION || echo "$(cat VERSION.in)-custom" | tee VERSION -RUN cd pycue && python setup.py test - -RUN cd pycue && python3.6 setup.py test +RUN cd pycue && python3 setup.py test RUN cp LICENSE requirements.txt VERSION pycue/ RUN versioned_name="pycue-$(cat ./VERSION)-all" \ - && mv pycue $versioned_name \ - && tar -cvzf $versioned_name.tar.gz $versioned_name/* - - -# ----------------- -# RUN -# ----------------- -FROM centos:7 - -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "PyCue runtime stage" - -WORKDIR /opt/opencue - -COPY --from=build /src/pycue-*-all.tar.gz ./ + && mv pycue "${versioned_name}" \ + && tar -cvzf "${versioned_name}.tar.gz" ${versioned_name}/* \ + && mkdir /opt/opencue \ + && cp "${versioned_name}.tar.gz" /opt/opencue/ diff --git a/pyoutline/Dockerfile b/pyoutline/Dockerfile index fb24d1136..e898452e1 100644 --- a/pyoutline/Dockerfile +++ b/pyoutline/Dockerfile @@ -1,35 +1,14 @@ -# ----------------- -# BUILD -# ----------------- -FROM centos:7 as build - -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "PyCue build stage" +FROM python:3.9.14 WORKDIR /src -RUN yum -y install \ - epel-release \ - gcc \ - python-devel - -RUN yum -y install \ - python-pip \ - python36 \ - python36-devel \ - python36-pip - -RUN python -m pip install --upgrade 'pip<21' -RUN python3.6 -m pip install --upgrade pip - -RUN python -m pip install --upgrade 'setuptools<45' -RUN python3.6 -m pip install --upgrade setuptools +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools COPY LICENSE ./ COPY requirements.txt ./ -RUN python -m pip install -r requirements.txt -RUN python3.6 -m pip install -r requirements.txt +RUN python3 -m pip install -r requirements.txt COPY proto/ ./proto COPY pycue/README.md ./pycue/ @@ -37,7 +16,7 @@ COPY pycue/setup.py ./pycue/ COPY pycue/opencue ./pycue/opencue COPY pycue/FileSequence ./pycue/FileSequence -RUN python -m grpc_tools.protoc \ +RUN python3 -m grpc_tools.protoc \ -I=./proto \ --python_out=./pycue/opencue/compiled_proto \ --grpc_python_out=./pycue/opencue/compiled_proto \ @@ -58,28 +37,13 @@ COPY pyoutline/outline ./pyoutline/outline COPY VERSION.in VERSIO[N] ./ RUN test -e VERSION || echo "$(cat VERSION.in)-custom" | tee VERSION -RUN cd pycue && python setup.py install -RUN cd pycue && python3.6 setup.py install - -RUN cd pyoutline && python setup.py test -RUN cd pyoutline && python3.6 setup.py test +RUN cd pycue && python3 setup.py install +RUN cd pyoutline && python3 setup.py test RUN cp LICENSE requirements.txt VERSION pyoutline/ RUN versioned_name="pyoutline-$(cat ./VERSION)-all" \ - && mv pyoutline $versioned_name \ - && tar -cvzf $versioned_name.tar.gz $versioned_name/* - - -# ----------------- -# RUN -# ----------------- -FROM centos:7 - -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "PyCue runtime stage" - -WORKDIR /opt/opencue - -COPY --from=build /src/pyoutline-*-all.tar.gz ./ - + && mv pyoutline "${versioned_name}" \ + && tar -cvzf "${versioned_name}.tar.gz" ${versioned_name}/* \ + && mkdir -p /opt/opencue \ + && cp "${versioned_name}.tar.gz" /opt/opencue/ diff --git a/rqd/Dockerfile b/rqd/Dockerfile index fd772c59e..15f5ac21c 100644 --- a/rqd/Dockerfile +++ b/rqd/Dockerfile @@ -1,4 +1,6 @@ -FROM centos:7 +FROM --platform=linux/x86_64 centos:7 + +WORKDIR /opt/opencue RUN yum -y install \ epel-release \ @@ -7,23 +9,16 @@ RUN yum -y install \ time RUN yum -y install \ - python-pip \ python36 \ python36-devel \ python36-pip -RUN python -m pip install --upgrade 'pip<21' RUN python3.6 -m pip install --upgrade pip - -RUN python -m pip install --upgrade 'setuptools<45' RUN python3.6 -m pip install --upgrade setuptools -WORKDIR /opt/opencue - COPY LICENSE ./ COPY requirements.txt ./ -RUN python -m pip install -r requirements.txt RUN python3.6 -m pip install -r requirements.txt COPY proto/ ./proto @@ -43,13 +38,9 @@ RUN python3.6 -m grpc_tools.protoc \ # for more info. RUN 2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py -# TODO(bcipriano) Lint the code here. (Issue #78) - COPY VERSION.in VERSIO[N] ./ RUN test -e VERSION || echo "$(cat VERSION.in)-custom" | tee VERSION -# Test in Python 2 and 3, but only install in the Python 3 environment. -RUN cd rqd && python setup.py test RUN cd rqd && python3.6 setup.py test RUN cd rqd && python3.6 setup.py install From d458bc2d2f2f3dc8867547e5d4240a8c0f9ea537 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Thu, 29 Sep 2022 14:27:30 -0400 Subject: [PATCH 220/277] [pyoutline] Replace use of execfile() with exec(). (#1201) --- pyoutline/outline/loader.py | 5 +++-- pyoutline/outline/modules/shell.py | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pyoutline/outline/loader.py b/pyoutline/outline/loader.py index 3db6b2c4f..0b0d43a18 100644 --- a/pyoutline/outline/loader.py +++ b/pyoutline/outline/loader.py @@ -25,7 +25,6 @@ import os import logging import json -from past.builtins import execfile import time import uuid import yaml @@ -173,7 +172,9 @@ def parse_outline_script(path): """ try: logger.info("parsing outline file %s", path) - execfile(path, {}) + with open(path) as fp: + code = compile(fp.read(), path, 'exec') + exec(code) # pylint: disable=exec-used except Exception as exp: logger.warning("failed to parse as python file, %s", exp) raise outline.exception.OutlineException( diff --git a/pyoutline/outline/modules/shell.py b/pyoutline/outline/modules/shell.py index e564e2162..482fcda4a 100644 --- a/pyoutline/outline/modules/shell.py +++ b/pyoutline/outline/modules/shell.py @@ -22,7 +22,6 @@ import logging import os -from past.builtins import execfile import outline.layer import outline.util @@ -56,7 +55,10 @@ def _setup(self): self.__code = None def _execute(self, frames): - execfile(self.get_file("script")) + path = self.get_file("script") + with open(path) as fp: + code = compile(fp.read(), path, 'exec') + exec(code) # pylint: disable=exec-used class Shell(outline.layer.Layer): From 962b22701c4ff76a503fe60639d7478629c032a4 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Thu, 29 Sep 2022 11:49:31 -0700 Subject: [PATCH 221/277] [cuebot] Add a new property max_show_stale_days to control show expiration rule. (#1208) --- .../spcue/dao/postgres/ShowDaoJdbc.java | 38 +++++++------------ cuebot/src/main/resources/opencue.properties | 9 ++++- 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java index 2c4182122..5d674ed82 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java @@ -237,32 +237,22 @@ public void updateShowCommentEmail(ShowInterface s, String[] email) { @Override public void updateShowsStatus() { - Stream protectedShowsRaw = Arrays.stream(env.getProperty("protected_shows", String.class).split(",")); + Stream protectedShowsRaw = Arrays + .stream(env.getProperty("protected_shows", String.class, "").split(",")); String protectedShows = protectedShowsRaw.map(show -> "'" + show + "'") .collect(Collectors.joining(",")); - getJdbcTemplate().update("UPDATE " + - "show " + - "SET " + - "b_active=false " + - "WHERE " + - "pk_show NOT IN (" + - "SELECT " + - "pk_show " + - "FROM (" + - "SELECT " + - "pk_show, count(pk_job) " + - "FROM " + - "job_history " + - "WHERE " + - "(DATE_PART('days', NOW()) - DATE_PART('days', dt_last_modified)) < 30 " + - "GROUP BY " + - "pk_show " + - "HAVING COUNT(pk_job)>0 " + - ") pk_show" + - ") " + - "AND " + - "str_name NOT IN (?)", - protectedShows); + int maxShowStaleDays = env.getProperty("max_show_stale_days", Integer.class, -1); + + if (maxShowStaleDays > 0) { + getJdbcTemplate().update("UPDATE show SET b_active=false " + + "WHERE pk_show NOT IN (SELECT pk_show " + + " FROM (SELECT pk_show, count(pk_job) FROM job_history " + + " WHERE " + + " (DATE_PART('days', NOW()) - DATE_PART('days', dt_last_modified)) < ? " + + "GROUP BY pk_show HAVING COUNT(pk_job) > 0) pk_show) " + + " AND str_name NOT IN (?)", + maxShowStaleDays, protectedShows); + } } @Override diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index 7f35eb90c..145bbb352 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -136,8 +136,13 @@ maintenance.auto_delete_down_hosts=false # Set hostname/IP of the smtp host. Will be used for mailing smtp_host=smtp -# These shows won't be deactivated by the scheduled tasks -protected_shows=pipe,swtest,edu +# Flags related to a job that runs periodically to deactivate shows that haven't been +# receiving jobs. +# A comma separated list of shows that won't be deactivated by the scheduled tasks +protected_shows=testing +# Number of days a show needs to be stale before it gets deactivated. +# -1 means shows should not get deactivated at all. +max_show_stale_days=-1 # These flags determine whether or not layers/frames will be readonly when job is finished. # If flags are set as true, layers/frames cannot be retried, eaten, edited dependency on, etc. From 234a9cef483bbf35599841ce7771609adc7d6da2 Mon Sep 17 00:00:00 2001 From: Akim Ruslanov Date: Tue, 25 Oct 2022 11:30:32 -0700 Subject: [PATCH 222/277] [pycue] Add missing exception parsing to setAllocation. (#1172) --- pycue/opencue/wrappers/host.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pycue/opencue/wrappers/host.py b/pycue/opencue/wrappers/host.py index aa54d44d0..860847bb0 100644 --- a/pycue/opencue/wrappers/host.py +++ b/pycue/opencue/wrappers/host.py @@ -19,6 +19,7 @@ import time from opencue import Cuebot +from opencue import util from opencue.compiled_proto import comment_pb2 from opencue.compiled_proto import host_pb2 import opencue.wrappers.comment @@ -149,6 +150,7 @@ def renameTag(self, oldTag, newTag): host_pb2.HostRenameTagRequest(host=self.data, old_tag=oldTag, new_tag=newTag), timeout=Cuebot.Timeout) + @util.grpcExceptionParser def setAllocation(self, allocation): """Sets the host to the given allocation. From 3742ff0281d9ff61a411a018bde8ab3843c7b007 Mon Sep 17 00:00:00 2001 From: Olivier Evers Date: Thu, 27 Oct 2022 18:03:25 +0200 Subject: [PATCH 223/277] [rqd] On Windows, pass environment and properly escape command chars. (#1215) --- rqd/rqd/rqcore.py | 22 ++++++++++++++++++---- rqd/rqd/rqmachine.py | 2 ++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index cb74f9d34..646c4f73d 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -83,8 +83,6 @@ def __createEnvVariables(self): self.frameEnv["TZ"] = self.rqCore.machine.getTimezone() self.frameEnv["USER"] = self.runFrame.user_name self.frameEnv["LOGNAME"] = self.runFrame.user_name - self.frameEnv["MAIL"] = "/usr/mail/%s" % self.runFrame.user_name - self.frameEnv["HOME"] = "/net/homedirs/%s" % self.runFrame.user_name self.frameEnv["mcp"] = "1" self.frameEnv["show"] = self.runFrame.show self.frameEnv["shot"] = self.runFrame.shot @@ -99,8 +97,18 @@ def __createEnvVariables(self): self.frameEnv["CUE_GPU_MEMORY"] = str(self.rqCore.machine.getGpuMemoryFree()) self.frameEnv["SP_NOMYCSHRC"] = "1" - for key in self.runFrame.environment: - self.frameEnv[key] = self.runFrame.environment[key] + if platform.system() in ("Linux", "Darwin"): + self.frameEnv["MAIL"] = "/usr/mail/%s" % self.runFrame.user_name + self.frameEnv["HOME"] = "/net/homedirs/%s" % self.runFrame.user_name + elif platform.system() == "Windows": + self.frameEnv["APPDATA"] = os.environ["APPDATA"] + self.frameEnv["SYSTEMROOT"] = os.environ["SYSTEMROOT"] + + for key, value in self.runFrame.environment.items(): + if key == 'PATH': + self.frameEnv[key] += os.pathsep + value + else: + self.frameEnv[key] = value # Add threads to use all assigned hyper-threading cores if 'CPU_LIST' in self.runFrame.attributes and 'CUE_THREADS' in self.frameEnv: @@ -128,6 +136,11 @@ def _createCommandFile(self, command): except OSError: pass # okay, already exists + # Windows Batch needs some characters escaped: + command = command.replace('%', '%%') + for char in '^&<>|': + command = command.replace(char, '^' + char) + commandFile = os.path.join( rqd_tmp_dir, 'cmd-%s-%s.bat' % (self.runFrame.frame_id, time.time())) @@ -358,6 +371,7 @@ def runWindows(self): tempCommand = [self._createCommandFile(runFrame.command)] frameInfo.forkedCommand = subprocess.Popen(tempCommand, + env=self.frameEnv, stdin=subprocess.PIPE, stdout=self.rqlog, stderr=self.rqlog) diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index d4cf918ca..27295ed92 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -509,6 +509,8 @@ def getPathEnv(self): """Returns the correct path environment for the given machine""" if platform.system() == 'Linux': return '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + if platform.system() == 'Windows': + return 'C:/Windows/system32;C:/Windows;C:/Windows/System32/Wbem' return '' @rqd.rqutil.Memoize From 6500fe2859a52641774499eda820c154b557f696 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Thu, 27 Oct 2022 12:08:28 -0400 Subject: [PATCH 224/277] [cuegui] Replace qApp with a new cuegui.App library. (#1193) --- cuegui/cuegui/AbstractTreeWidget.py | 19 ++--- cuegui/cuegui/AbstractWidgetItem.py | 1 + cuegui/cuegui/App.py | 78 +++++++++++++++++++ cuegui/cuegui/Comments.py | 11 +-- cuegui/cuegui/CueJobMonitorTree.py | 6 +- cuegui/cuegui/CueStateBarWidget.py | 5 +- cuegui/cuegui/DarkPalette.py | 11 ++- cuegui/cuegui/Exception.py | 35 +++++++++ cuegui/cuegui/FrameMonitorTree.py | 27 +++---- cuegui/cuegui/HostMonitor.py | 14 +--- cuegui/cuegui/HostMonitorTree.py | 12 +-- cuegui/cuegui/JobMonitorTree.py | 14 +--- cuegui/cuegui/LayerDialog.py | 6 +- cuegui/cuegui/LayerMonitorTree.py | 3 +- cuegui/cuegui/LimitsWidget.py | 7 +- cuegui/cuegui/Main.py | 32 ++------ cuegui/cuegui/MainWindow.py | 39 +++------- cuegui/cuegui/MenuActions.py | 27 +++---- cuegui/cuegui/Plugins.py | 23 ++---- cuegui/cuegui/PreviewWidget.py | 7 +- cuegui/cuegui/ProcMonitor.py | 14 ++-- cuegui/cuegui/ProcMonitorTree.py | 13 +--- cuegui/cuegui/ProgressDialog.py | 11 +-- cuegui/cuegui/Redirect.py | 3 +- cuegui/cuegui/ShowsWidget.py | 7 +- cuegui/cuegui/Style.py | 10 +-- cuegui/cuegui/SubscriptionsWidget.py | 6 +- cuegui/cuegui/ThreadPool.py | 6 +- cuegui/cuegui/Utils.py | 7 +- cuegui/cuegui/__init__.py | 8 ++ cuegui/cuegui/plugins/AllocationsPlugin.py | 7 +- cuegui/cuegui/plugins/AttributesPlugin.py | 16 ++-- cuegui/cuegui/plugins/LogViewPlugin.py | 11 +-- cuegui/cuegui/plugins/MonitorCuePlugin.py | 14 +--- .../cuegui/plugins/MonitorJobDetailsPlugin.py | 9 +-- cuegui/cuegui/plugins/MonitorJobsPlugin.py | 6 +- cuegui/cuegui/plugins/ServicePlugin.py | 6 +- cuegui/cuegui/plugins/StuckFramePlugin.py | 19 +++-- cuegui/tests/CueJobMonitorTree_tests.py | 4 +- cuegui/tests/DependWizard_tests.py | 4 +- cuegui/tests/FilterDialog_tests.py | 12 +-- cuegui/tests/FrameMonitorTree_tests.py | 5 +- cuegui/tests/LayerDialog_tests.py | 4 +- cuegui/tests/MenuActions_tests.py | 44 +++++++---- cuegui/tests/Redirect_tests.py | 4 +- cuegui/tests/UnbookDialog_tests.py | 12 +-- cuegui/tests/plugins/LogViewPlugin_tests.py | 16 ++-- cuegui/tests/test_utils.py | 13 +--- 48 files changed, 328 insertions(+), 340 deletions(-) create mode 100644 cuegui/cuegui/App.py create mode 100644 cuegui/cuegui/Exception.py diff --git a/cuegui/cuegui/AbstractTreeWidget.py b/cuegui/cuegui/AbstractTreeWidget.py index 96250453e..88e98ad4f 100644 --- a/cuegui/cuegui/AbstractTreeWidget.py +++ b/cuegui/cuegui/AbstractTreeWidget.py @@ -75,6 +75,7 @@ def __init__(self, parent): @type parent: QWidget @param parent: The widget to set as the parent""" QtWidgets.QTreeWidget.__init__(self, parent) + self.app = cuegui.app() self._items = {} self._lastUpdate = 0 @@ -104,8 +105,8 @@ def __init__(self, parent): self.itemClicked.connect(self.__itemSingleClickedEmitToApp) self.itemDoubleClicked.connect(self.__itemDoubleClickedEmitToApp) self._timer.timeout.connect(self.updateRequest) - QtGui.qApp.request_update.connect(self.updateRequest) # pylint: enable=no-member + self.app.request_update.connect(self.updateRequest) self.updateRequest() self.setUpdateInterval(10) @@ -279,9 +280,7 @@ def __itemSingleClickedEmitToApp(item, col): @type col: int @param col: Column number single clicked on""" del col - # pylint: disable=no-member - QtGui.qApp.single_click.emit(item.rpcObject) - # pylint: enable=no-member + cuegui.app().single_click.emit(item.rpcObject) @staticmethod def __itemDoubleClickedEmitToApp(item, col): @@ -293,10 +292,8 @@ def __itemDoubleClickedEmitToApp(item, col): @type col: int @param col: Column number double clicked on""" del col - # pylint: disable=no-member - QtGui.qApp.view_object.emit(item.rpcObject) - QtGui.qApp.double_click.emit(item.rpcObject) - # pylint: enable=no-member + cuegui.app().view_object.emit(item.rpcObject) + cuegui.app().double_click.emit(item.rpcObject) def addObject(self, rpcObject): """Adds or updates an rpcObject in the list using the _createItem function @@ -385,11 +382,9 @@ def _update(self): """Updates the items in the TreeWidget without checking when it was last updated""" self._lastUpdate = time.time() - if hasattr(QtGui.qApp, "threadpool"): - # pylint: disable=no-member - QtGui.qApp.threadpool.queue( + if self.app.threadpool is not None: + self.app.threadpool.queue( self._getUpdate, self._processUpdate, "getting data for %s" % self.__class__) - # pylint: enable=no-member else: logger.warning("threadpool not found, doing work in gui thread") self._processUpdate(None, self._getUpdate()) diff --git a/cuegui/cuegui/AbstractWidgetItem.py b/cuegui/cuegui/AbstractWidgetItem.py index 548dde72a..301a3aa59 100644 --- a/cuegui/cuegui/AbstractWidgetItem.py +++ b/cuegui/cuegui/AbstractWidgetItem.py @@ -47,6 +47,7 @@ class AbstractWidgetItem(QtWidgets.QTreeWidgetItem): def __init__(self, itemType, rpcObject, parent, source=None): QtWidgets.QTreeWidgetItem.__init__(self, parent, itemType) + self.app = cuegui.app() self.column_info = self.treeWidget().getColumnInfo(itemType) self._cache = {} self._source = source diff --git a/cuegui/cuegui/App.py b/cuegui/cuegui/App.py new file mode 100644 index 000000000..e161420fe --- /dev/null +++ b/cuegui/cuegui/App.py @@ -0,0 +1,78 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module for CueGUI's custom QApplication and associated helper functions.""" + +from PySide2 import QtCore +from PySide2 import QtWidgets + +import cuegui.Exception + +__QAPPLICATION_SINGLETON = None + + +class CueGuiApplication(QtWidgets.QApplication): + """The CueGUI application.""" + + # Settings + settings = None + + # Global signals + display_log_file_content = QtCore.Signal(object) + double_click = QtCore.Signal(object) + facility_changed = QtCore.Signal() + single_click = QtCore.Signal(object) + unmonitor = QtCore.Signal(object) + view_hosts = QtCore.Signal(object) + view_object = QtCore.Signal(object) + view_procs = QtCore.Signal(object) + request_update = QtCore.Signal() + status = QtCore.Signal() + quit = QtCore.Signal() + + # Thread pool + threadpool = None + threads = [] + + # Shutdown signal + closingApp = False + + +def create_app(argv): + """ + Create an instance of the CueGUI application. + + :param argv: user-provided commandline arguments + :type argv: list + :return: the application instance + :rtype: CueGuiApplication + """ + # pylint: disable=global-statement + global __QAPPLICATION_SINGLETON + if __QAPPLICATION_SINGLETON is None: + __QAPPLICATION_SINGLETON = CueGuiApplication(argv) + return __QAPPLICATION_SINGLETON + + +def app(): + """Returns the current application instance. + + :return: the current application instance + :rtype: CueGuiApplication + :raises: opencue.exception.ApplicationNotRunningException: the application has not been + initialized yet + """ + if __QAPPLICATION_SINGLETON is None: + raise cuegui.Exception.ApplicationNotRunningException() + return __QAPPLICATION_SINGLETON diff --git a/cuegui/cuegui/Comments.py b/cuegui/cuegui/Comments.py index c24a9802e..628c1b740 100644 --- a/cuegui/cuegui/Comments.py +++ b/cuegui/cuegui/Comments.py @@ -24,7 +24,6 @@ import pickle from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import cuegui.Utils @@ -50,6 +49,8 @@ def __init__(self, source, parent=None): @type parent: QWidget @param parent: The dialog's parent""" QtWidgets.QDialog.__init__(self, parent) + self.app = cuegui.app() + self.__source = source self.__labelTitle = QtWidgets.QLabel(self.__source.data.name, self) @@ -208,10 +209,8 @@ def refreshComments(self): def __macroLoad(self): """Loads the defined comment macros from settings""" - # pylint: disable=no-member self.__macroList = pickle.loads( - str(QtGui.qApp.settings.value("Comments", pickle.dumps({})))) - # pylint: enable=no-member + str(self.app.settings.value("Comments", pickle.dumps({})))) self.__macroRefresh() def __macroRefresh(self): @@ -225,9 +224,7 @@ def __macroRefresh(self): def __macroSave(self): """Saves the current comment macros to settings""" - # pylint: disable=no-member - QtGui.qApp.settings.setValue("Comments", pickle.dumps(self.__macroList)) - # pylint: enable=no-member + self.app.settings.setValue("Comments", pickle.dumps(self.__macroList)) def __macroHandle(self, selection): """Called when the comment macro combo box is selected diff --git a/cuegui/cuegui/CueJobMonitorTree.py b/cuegui/cuegui/CueJobMonitorTree.py index 45a9fb07e..b7f32ebec 100644 --- a/cuegui/cuegui/CueJobMonitorTree.py +++ b/cuegui/cuegui/CueJobMonitorTree.py @@ -223,8 +223,8 @@ def __init__(self, parent): self.__menuActions = cuegui.MenuActions.MenuActions( self, self.updateSoon, self.selectedObjects) + self.app.facility_changed.connect(self.removeAllShows) # pylint: disable=no-member - QtGui.qApp.facility_changed.connect(self.removeAllShows) self.itemClicked.connect(self.__itemSingleClickedCopy) self.itemClicked.connect(self.__itemSingleClickedComment) # pylint: enable=no-member @@ -726,9 +726,7 @@ def __init__(self, rpcObject, parent): self.__class__.__initialized = True self.__class__.__commentIcon = QtGui.QIcon(":comment.png") self.__class__.__eatIcon = QtGui.QIcon(":eat.png") - # pylint: disable=no-member - self.__class__.__backgroundColor = QtGui.qApp.palette().color(QtGui.QPalette.Base) - # pylint: enable=no-member + self.__class__.__backgroundColor = cuegui.app().palette().color(QtGui.QPalette.Base) self.__class__.__foregroundColor = cuegui.Style.ColorTheme.COLOR_JOB_FOREGROUND self.__class__.__pausedColor = cuegui.Style.ColorTheme.COLOR_JOB_PAUSED_BACKGROUND self.__class__.__finishedColor = cuegui.Style.ColorTheme.COLOR_JOB_FINISHED_BACKGROUND diff --git a/cuegui/cuegui/CueStateBarWidget.py b/cuegui/cuegui/CueStateBarWidget.py index 780ea319d..c35622f17 100644 --- a/cuegui/cuegui/CueStateBarWidget.py +++ b/cuegui/cuegui/CueStateBarWidget.py @@ -47,6 +47,7 @@ def __init__(self, sourceTree, parent=None): @type parent: QWidget @param parent: The parent widget""" QtWidgets.QWidget.__init__(self, parent) + self.app = cuegui.app() self.__background = None @@ -55,9 +56,7 @@ def __init__(self, sourceTree, parent=None): self.__sourceTree = weakref.proxy(sourceTree) self.__colors = [] - # pylint: disable=no-member - self.__baseColor = QtGui.qApp.palette().color(QtGui.QPalette.Base) - # pylint: enable=no-member + self.__baseColor = self.app.palette().color(QtGui.QPalette.Base) self.__colorsLock = QtCore.QReadWriteLock() self.__timer = QtCore.QTimer(self) self.__lastUpdate = 0 diff --git a/cuegui/cuegui/DarkPalette.py b/cuegui/cuegui/DarkPalette.py index 2b15bd66a..22b89bf68 100644 --- a/cuegui/cuegui/DarkPalette.py +++ b/cuegui/cuegui/DarkPalette.py @@ -32,20 +32,19 @@ def init(): """Convenience function that takes the QApplication object for the application and configures the palette and style for the Plastique color scheme""" - # pylint: disable=no-member - QtGui.qApp.setPalette(DarkPalette()) + app = cuegui.app() + app.setPalette(DarkPalette()) if platform.system() in ['Darwin', 'Linux']: setDarkStyleSheet() elif platform.system() == 'Windows': - QtGui.qApp.setStyle('Fusion') + app.setStyle('Fusion') else: - QtGui.qApp.setStyle(QtWidgets.QStyleFactory.create(cuegui.Constants.COLOR_THEME)) + app.setStyle(QtWidgets.QStyleFactory.create(cuegui.Constants.COLOR_THEME)) def setDarkStyleSheet(): """Sets the stylesheet.""" - # pylint: disable=no-member - QtGui.qApp.setStyleSheet(open(cuegui.Constants.DARK_STYLE_SHEET).read()) + cuegui.app().setStyleSheet(open(cuegui.Constants.DARK_STYLE_SHEET).read()) def DarkPalette(): diff --git a/cuegui/cuegui/Exception.py b/cuegui/cuegui/Exception.py new file mode 100644 index 000000000..151386b32 --- /dev/null +++ b/cuegui/cuegui/Exception.py @@ -0,0 +1,35 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Custom exception classes for CueGUI application errors.""" + + +class CueGuiException(Exception): + """Base class for all CueGUI exceptions. + + Note that this class does NOT inherit from opencue.exception.CueException, so that error + handling code can easily distinguish between API errors and CueGUI errors. + """ + + +class ApplicationNotRunningException(CueGuiException): + """Raised when the CueGUI application has not been instantiated but is required to be.""" + + default_message = ( + 'attempted to access the CueGUI application before cuegui.create_app() was called') + + def __init__(self, message=None): + if message is None: + message = self.default_message + super().__init__(message) diff --git a/cuegui/cuegui/FrameMonitorTree.py b/cuegui/cuegui/FrameMonitorTree.py index e7ee6119c..b546be2d5 100644 --- a/cuegui/cuegui/FrameMonitorTree.py +++ b/cuegui/cuegui/FrameMonitorTree.py @@ -351,9 +351,7 @@ def __itemSingleClickedViewLog(self, item, col): except ValueError: old_log_files = [] - # pylint: disable=no-member - QtGui.qApp.display_log_file_content.emit([current_log_file] + old_log_files) - # pylint: enable=no-member + self.app.display_log_file_content.emit([current_log_file] + old_log_files) def __itemDoubleClickedViewLog(self, item, col): """Called when a frame is double clicked, views the frame log in a popup @@ -447,11 +445,9 @@ def _update(self): updated""" logger.info("_update") self._lastUpdate = time.time() - if hasattr(QtGui.qApp, "threadpool"): - # pylint: disable=no-member - QtGui.qApp.threadpool.queue( + if self.app.threadpool is not None: + self.app.threadpool.queue( self._getUpdate, self._processUpdate, "getting data for %s" % self.__class__) - # pylint: enable=no-member else: logger.warning("threadpool not found, doing work in gui thread") self._processUpdate(None, self._getUpdate()) @@ -461,12 +457,10 @@ def _updateChanged(self): updated""" logger.info("_updateChanged") self._lastUpdate = time.time() - if hasattr(QtGui.qApp, "threadpool"): - # pylint: disable=no-member - QtGui.qApp.threadpool.queue( + if self.app.threadpool is not None: + self.app.threadpool.queue( self._getUpdateChanged, self._processUpdateChanged, "getting data for %s" % self.__class__) - # pylint: enable=no-member else: logger.warning("threadpool not found, doing work in gui thread") self._processUpdateChanged(None, self._getUpdateChanged()) @@ -596,9 +590,7 @@ class FrameWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): def __init__(self, rpcObject, parent, job): if not self.__initialized: self.__class__.__initialized = True - # pylint: disable=no-member - self.__class__.__backgroundColor = QtGui.qApp.palette().color(QtGui.QPalette.Base) - # pylint: enable=no-member + self.__class__.__backgroundColor = cuegui.app().palette().color(QtGui.QPalette.Base) self.__class__.__foregroundColor = cuegui.Style.ColorTheme.COLOR_JOB_FOREGROUND self.__class__.__foregroundColorBlack = QCOLOR_BLACK self.__class__.__foregroundColorGreen = QCOLOR_GREEN @@ -862,6 +854,7 @@ class FrameContextMenu(QtWidgets.QMenu): def __init__(self, widget, filterSelectedLayersCallback): super(FrameContextMenu, self).__init__() + self.app = cuegui.app() self.__menuActions = cuegui.MenuActions.MenuActions( widget, widget.updateSoon, widget.selectedObjects, widget.getJob) @@ -880,13 +873,11 @@ def __init__(self, widget, filterSelectedLayersCallback): elif count == 2: self.__menuActions.frames().addAction(self, "xdiff2") - if bool(int(QtGui.qApp.settings.value("AllowDeeding", 0))): + if bool(int(self.app.settings.value("AllowDeeding", 0))): self.__menuActions.frames().addAction(self, "useLocalCores") - # pylint: disable=no-member - if QtGui.qApp.applicationName() == "CueCommander": + if self.app.applicationName() == "CueCommander": self.__menuActions.frames().addAction(self, "viewHost") - # pylint: enable=no-member depend_menu = QtWidgets.QMenu("&Dependencies", self) self.__menuActions.frames().addAction(depend_menu, "viewDepends") diff --git a/cuegui/cuegui/HostMonitor.py b/cuegui/cuegui/HostMonitor.py index ad6c16a5e..cd7ffee99 100644 --- a/cuegui/cuegui/HostMonitor.py +++ b/cuegui/cuegui/HostMonitor.py @@ -23,7 +23,6 @@ from builtins import str from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import opencue @@ -43,6 +42,7 @@ class HostMonitor(QtWidgets.QWidget): def __init__(self, parent): QtWidgets.QWidget.__init__(self, parent) + self.app = cuegui.app() self.__filterByHostNameLastInput = None self.hostMonitorTree = cuegui.HostMonitorTree.HostMonitorTree(self) @@ -70,10 +70,8 @@ def __init__(self, parent): self.__viewHostsSetup() - # pylint: disable=no-member - if bool(int(QtGui.qApp.settings.value("AutoRefreshMonitorHost", 1))): + if bool(int(self.app.settings.value("AutoRefreshMonitorHost", 1))): self.updateRequest() - # pylint: enable=no-member def updateRequest(self): """Requests an update of the displayed information.""" @@ -279,9 +277,7 @@ def __refreshToggleCheckBoxSetup(self, layout): def __refreshToggleCheckBoxHandle(self, state): self.hostMonitorTree.enableRefresh = bool(state) - # pylint: disable=no-member - QtGui.qApp.settings.setValue("AutoRefreshMonitorHost", int(bool(state))) - # pylint: enable=no-member + self.app.settings.setValue("AutoRefreshMonitorHost", int(bool(state))) # ============================================================================== # Button to refresh @@ -332,9 +328,7 @@ def __clearButtonHandle(self): # Monitors and handles the view_hosts signal # ============================================================================== def __viewHostsSetup(self): - # pylint: disable=no-member - QtGui.qApp.view_hosts.connect(self.__viewHostsHandle) - # pylint: enable=no-member + self.app.view_hosts.connect(self.__viewHostsHandle) def __viewHostsHandle(self, hosts): self.__clearButtonHandle() diff --git a/cuegui/cuegui/HostMonitorTree.py b/cuegui/cuegui/HostMonitorTree.py index 06143769d..1c247e2c6 100644 --- a/cuegui/cuegui/HostMonitorTree.py +++ b/cuegui/cuegui/HostMonitorTree.py @@ -169,17 +169,13 @@ def __init__(self, parent): # pylint: enable=no-member # Don't use the standard space bar to refresh - # pylint: disable=no-member - QtGui.qApp.request_update.connect(self.updateRequest) - # pylint: enable=no-member + self.app.request_update.connect(self.updateRequest) self.startTicksUpdate(40) # Don't start refreshing until the user sets a filter or hits refresh self.ticksWithoutUpdate = -1 - # pylint: disable=no-member - self.enableRefresh = bool(int(QtGui.qApp.settings.value("AutoRefreshMonitorHost", 1))) - # pylint: enable=no-member + self.enableRefresh = bool(int(self.app.settings.value("AutoRefreshMonitorHost", 1))) def tick(self): if self.ticksWithoutUpdate >= self.updateInterval and \ @@ -301,9 +297,7 @@ def __init__(self, rpcObject, parent): cuegui.Style.init() self.__class__.__initialized = True self.__class__.__commentIcon = QtGui.QIcon(":comment.png") - # pylint: disable=no-member - self.__class__.__backgroundColor = QtGui.qApp.palette().color(QtGui.QPalette.Base) - # pylint: enable=no-member + self.__class__.__backgroundColor = cuegui.app().palette().color(QtGui.QPalette.Base) self.__class__.__foregroundColor = cuegui.Style.ColorTheme.COLOR_JOB_FOREGROUND self.__class__.__pausedColor = cuegui.Style.ColorTheme.COLOR_JOB_PAUSED_BACKGROUND self.__class__.__dyingColor = cuegui.Style.ColorTheme.COLOR_JOB_DYING_BACKGROUND diff --git a/cuegui/cuegui/JobMonitorTree.py b/cuegui/cuegui/JobMonitorTree.py index a35d02897..3bf15373b 100644 --- a/cuegui/cuegui/JobMonitorTree.py +++ b/cuegui/cuegui/JobMonitorTree.py @@ -328,9 +328,7 @@ def _removeItem(self, item): """Removes an item from the TreeWidget without locking @param item: A tree widget item @type item: AbstractTreeWidgetItem""" - # pylint: disable=no-member - QtGui.qApp.unmonitor.emit(item.rpcObject) - # pylint: enable=no-member + self.app.unmonitor.emit(item.rpcObject) cuegui.AbstractTreeWidget.AbstractTreeWidget._removeItem(self, item) self.__jobTimeLoaded.pop(item.rpcObject, "") try: @@ -352,9 +350,7 @@ def removeAllItems(self): """Notifies the other widgets of each item being unmonitored, then calls the the AbstractTreeWidget.removeAllItems like normal""" for proxy in list(self._items.keys()): - # pylint: disable=no-member - QtGui.qApp.unmonitor.emit(proxy) - # pylint: enable=no-member + self.app.unmonitor.emit(proxy) if proxy in self.__jobTimeLoaded: del self.__jobTimeLoaded[proxy] self.__dependentJobs.clear() @@ -382,7 +378,7 @@ def contextMenuEvent(self, e): self.__menuActions.jobs().addAction(menu, "emailArtist") self.__menuActions.jobs().addAction(menu, "viewComments") - if bool(int(QtGui.qApp.settings.value("AllowDeeding", 0))): + if bool(int(self.app.settings.value("AllowDeeding", 0))): self.__menuActions.jobs().addAction(menu, "useLocalCores") depend_menu = QtWidgets.QMenu("&Dependencies",self) @@ -597,9 +593,7 @@ def __init__(self, rpcObject, parent, created): self.__class__.__initialized = True self.__class__.__commentIcon = QtGui.QIcon(":comment.png") self.__class__.__eatIcon = QtGui.QIcon(":eat.png") - # pylint: disable=no-member - self.__class__.__backgroundColor = QtGui.qApp.palette().color(QtGui.QPalette.Base) - # pylint: enable=no-member + self.__class__.__backgroundColor = cuegui.app().palette().color(QtGui.QPalette.Base) self.__class__.__foregroundColor = cuegui.Style.ColorTheme.COLOR_JOB_FOREGROUND self.__class__.__pausedColor = cuegui.Style.ColorTheme.COLOR_JOB_PAUSED_BACKGROUND self.__class__.__dyingColor = cuegui.Style.ColorTheme.COLOR_JOB_DYING_BACKGROUND diff --git a/cuegui/cuegui/LayerDialog.py b/cuegui/cuegui/LayerDialog.py index 3efd36de3..5f25be5f6 100644 --- a/cuegui/cuegui/LayerDialog.py +++ b/cuegui/cuegui/LayerDialog.py @@ -21,7 +21,6 @@ from __future__ import print_function from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import opencue @@ -106,6 +105,7 @@ class LayerPropertiesDialog(QtWidgets.QDialog): def __init__(self, layers, parent=None): QtWidgets.QDialog.__init__(self, parent) + self.app = cuegui.app() self.__layers = [opencue.api.getLayer(opencue.id(layer)) for layer in layers] self.setWindowTitle("Layer Properties") @@ -147,10 +147,8 @@ def __init__(self, layers, parent=None): self.__max_cores.setSingleStep(1) # Disable this for everything except commander. - # pylint: disable=no-member - if QtGui.qApp.applicationName() != "CueCommander": + if self.app.applicationName() != "CueCommander": self.__core.setDisabled(True) - # pylint: enable=no-member # Threads self.__thread = QtWidgets.QCheckBox(self) diff --git a/cuegui/cuegui/LayerMonitorTree.py b/cuegui/cuegui/LayerMonitorTree.py index a8665c1a0..7663d3276 100644 --- a/cuegui/cuegui/LayerMonitorTree.py +++ b/cuegui/cuegui/LayerMonitorTree.py @@ -21,7 +21,6 @@ from __future__ import division from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets from opencue.exception import EntityNotFoundException @@ -232,7 +231,7 @@ def contextMenuEvent(self, e): if len(__selectedObjects) == 1: menu.addSeparator() - if bool(int(QtGui.qApp.settings.value("AllowDeeding", 0))): + if bool(int(self.app.settings.value("AllowDeeding", 0))): self.__menuActions.layers().addAction(menu, "useLocalCores") if len({layer.data.range for layer in __selectedObjects}) == 1: self.__menuActions.layers().addAction(menu, "reorder") diff --git a/cuegui/cuegui/LimitsWidget.py b/cuegui/cuegui/LimitsWidget.py index ec0753f2e..26f871de1 100644 --- a/cuegui/cuegui/LimitsWidget.py +++ b/cuegui/cuegui/LimitsWidget.py @@ -21,7 +21,6 @@ from __future__ import division from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import opencue @@ -110,10 +109,8 @@ def __init__(self, parent): self.__menuActions = cuegui.MenuActions.MenuActions( self, self.updateSoon, self.selectedObjects) - # pylint: disable=no-member - self.itemClicked.connect(self.__itemSingleClickedToDouble) - QtGui.qApp.facility_changed.connect(self.__facilityChanged) - # pylint: enable=no-member + self.itemClicked.connect(self.__itemSingleClickedToDouble) # pylint: disable=no-member + self.app.facility_changed.connect(self.__facilityChanged) self.setUpdateInterval(60) diff --git a/cuegui/cuegui/Main.py b/cuegui/cuegui/Main.py index 58bb3073d..7712a4cdb 100644 --- a/cuegui/cuegui/Main.py +++ b/cuegui/cuegui/Main.py @@ -22,10 +22,9 @@ import signal -from PySide2 import QtCore from PySide2 import QtGui -from PySide2 import QtWidgets +import cuegui import cuegui.Config import cuegui.Constants import cuegui.Logger @@ -40,23 +39,6 @@ logger = cuegui.Logger.getLogger(__file__) -class CueGuiApplication(QtWidgets.QApplication): - """The CueGUI application.""" - - # Global signals - display_log_file_content = QtCore.Signal(object) - double_click = QtCore.Signal(object) - facility_changed = QtCore.Signal() - single_click = QtCore.Signal(object) - unmonitor = QtCore.Signal(object) - view_hosts = QtCore.Signal(object) - view_object = QtCore.Signal(object) - view_procs = QtCore.Signal(object) - request_update = QtCore.Signal() - status = QtCore.Signal() - quit = QtCore.Signal() - - def cuetopia(argv): """Starts the Cuetopia window.""" startup("Cuetopia", cuegui.Constants.VERSION, argv) @@ -70,8 +52,7 @@ def cuecommander(argv): def startup(app_name, app_version, argv): """Starts an application window.""" - app = CueGuiApplication(argv) - QtGui.qApp = app + app = cuegui.create_app(argv) # Start splash screen splash = cuegui.SplashWindow.SplashWindow( @@ -86,13 +67,10 @@ def startup(app_name, app_version, argv): app.setApplicationName(app_name) app.lastWindowClosed.connect(app.quit) # pylint: disable=no-member - # pylint: disable=attribute-defined-outside-init - QtGui.qApp.threadpool = cuegui.ThreadPool.ThreadPool(3, parent=app) - QtGui.qApp.threads = [] - # pylint: enable=attribute-defined-outside-init + app.threadpool = cuegui.ThreadPool.ThreadPool(3, parent=app) settings = cuegui.Config.startup(app_name) - QtGui.qApp.settings = settings # pylint: disable=attribute-defined-outside-init + app.settings = settings cuegui.Style.init() @@ -118,6 +96,6 @@ def startup(app_name, app_version, argv): def closingTime(): """Window close callback.""" logger.info("Closing all threads...") - threads = QtGui.qApp.threads # pylint: disable=no-member + threads = cuegui.app().threads for thread in threads: cuegui.Utils.shutdownThread(thread) diff --git a/cuegui/cuegui/MainWindow.py b/cuegui/cuegui/MainWindow.py index 9fd5c5492..50555f647 100644 --- a/cuegui/cuegui/MainWindow.py +++ b/cuegui/cuegui/MainWindow.py @@ -53,16 +53,14 @@ class MainWindow(QtWidgets.QMainWindow): def __init__(self, app_name, app_version, window_name, parent = None): QtWidgets.QMainWindow.__init__(self, parent) + self.app = cuegui.app() self.__actions_facility = {} self.facility_default = None self.facility_dict = None self.windowMenu = None - self.qApp = QtGui.qApp - # pylint: disable=no-member - self.settings = QtGui.qApp.settings - # pylint: enable=no-member + self.settings = self.app.settings self.windows_names = [app_name] + ["%s_%s" % (app_name, num) for num in range(2, 5)] self.app_name = app_name self.app_version = app_version @@ -96,10 +94,7 @@ def __init__(self, app_name, app_version, window_name, parent = None): # Restore saved settings self.__restoreSettings() - # pylint: disable=no-member - QtGui.qApp.status.connect(self.showStatusBarMessage) - # pylint: enable=no-member - + self.app.status.connect(self.showStatusBarMessage) self.showStatusBarMessage("Ready") def displayStartupNotice(self): @@ -184,9 +179,7 @@ def __facilityMenuHandle(self, action): for facility in list(self.__actions_facility.values()): if facility.isChecked(): opencue.Cuebot.setFacility(str(facility.text())) - # pylint: disable=no-member - QtGui.qApp.facility_changed.emit() - # pylint: enable=no-member + self.app.facility_changed.emit() return ################################################################################ @@ -359,24 +352,18 @@ def windowMenuOpenWindow(self, name): def __windowOpened(self): """Called from __init__ on window creation""" - # pylint: disable=no-member - self.qApp.quit.connect(self.close) + self.app.quit.connect(self.close) self.windows.append(self) - self.qApp.closingApp = False - # pylint: enable=no-member + self.app.closingApp = False def __windowClosed(self): """Called from closeEvent on window close""" # Disconnect to avoid multiple attempts to close a window - # pylint: disable=no-member - self.qApp.quit.connect(self.close) - # pylint: enable=no-member + self.app.quit.connect(self.close) # Save the fact that this window is open or not when the app closed - # pylint: disable=no-member - self.settings.setValue("%s/Open" % self.name, self.qApp.closingApp) - # pylint: enable=no-member + self.settings.setValue("%s/Open" % self.name, self.app.closingApp) # pylint: disable=bare-except try: @@ -392,10 +379,8 @@ def __windowCloseWindow(self): def __windowCloseApplication(self): """Called when the entire application should exit. Signals other windows to exit.""" - # pylint: disable=no-member - self.qApp.closingApp = True - self.qApp.quit.emit() - # pylint: enable=no-member + self.app.closingApp = True + self.app.quit.emit() ################################################################################ @@ -420,9 +405,7 @@ def __toggleFullscreen(self): def keyPressEvent(self, event): if event.key() == QtCore.Qt.Key_Space: - # pylint: disable=no-member - QtGui.qApp.request_update.emit() - # pylint: enable=no-member + self.app.request_update.emit() event.accept() def closeEvent(self, event): diff --git a/cuegui/cuegui/MenuActions.py b/cuegui/cuegui/MenuActions.py index 2cd284f6d..2f6771baa 100644 --- a/cuegui/cuegui/MenuActions.py +++ b/cuegui/cuegui/MenuActions.py @@ -35,12 +35,14 @@ import FileSequence import opencue import opencue.compiled_proto.job_pb2 +import opencue.wrappers.depend # pylint: disable=cyclic-import import cuegui.Action import cuegui.Comments import cuegui.Constants import cuegui.CreatorDialog +import cuegui.CueJobMonitorTree import cuegui.DependDialog import cuegui.DependWizard import cuegui.EmailDialog @@ -78,6 +80,7 @@ def __init__(self, caller, updateCallable, selectedRpcObjectsCallable, sourceCal self.__selectedRpcObjects = selectedRpcObjectsCallable self._getSource = sourceCallable self._update = updateCallable + self.app = cuegui.app() self.__actionCache = {} @@ -218,9 +221,7 @@ def unmonitor(self, rpcObjects=None): def view(self, rpcObjects=None): for job in self._getOnlyJobObjects(rpcObjects): - # pylint: disable=no-member - QtGui.qApp.view_object.emit(job) - # pylint: enable=no-member + self.app.view_object.emit(job) viewDepends_info = ["&View Dependencies...", None, "log"] @@ -427,10 +428,8 @@ def getJobByName(self, job_name): def dropJobsDependingOnThis(self, job): for dep in job.getWhatDependsOnThis(): if not dep.isInternal(): - # pylint: disable=no-member - job = self.getJobByName(self, dep.dependOnJob()) - job.dropDepends(opencue.wrappers.depend.DependTarget.EXTERNAL) - # pylint: enable=no-member + job = self.getJobByName(dep.dependOnJob()) + job.dropDepends(opencue.wrappers.depend.Depend.DependTarget.EXTERNAL) eatDead_info = ["Eat dead frames", None, "eat"] @@ -995,10 +994,8 @@ def viewHost(self, rpcObjects=None): hosts = list({frame.data.last_resource.split("/")[0] for frame in frames if frame.data.last_resource}) if hosts: - # pylint: disable=no-member - QtGui.qApp.view_hosts.emit(hosts) - QtGui.qApp.single_click.emit(opencue.api.findHost(hosts[0])) - # pylint: enable=no-member + self.app.view_hosts.emit(hosts) + self.app.single_click.emit(opencue.api.findHost(hosts[0])) getWhatThisDependsOn_info = ["print getWhatThisDependsOn", None, "log"] @@ -1452,9 +1449,7 @@ def viewProc(self, rpcObjects=None): hosts = self._getOnlyHostObjects(rpcObjects) hosts = list({host.data.name for host in hosts}) if hosts: - # pylint: disable=no-member - QtGui.qApp.view_procs.emit(hosts) - # pylint: enable=no-member + self.app.view_procs.emit(hosts) lock_info = ["Lock Host", None, "lock"] @@ -1650,9 +1645,7 @@ def __init__(self, *args): def view(self, rpcObjects=None): for job in list({proc.data.job_name for proc in self._getOnlyProcObjects(rpcObjects)}): try: - # pylint: disable=no-member - QtGui.qApp.view_object.emit(opencue.api.findJob(job)) - # pylint: enable=no-member + self.app.view_object.emit(opencue.api.findJob(job)) except opencue.exception.CueException: logger.warning("Unable to load: %s", job) diff --git a/cuegui/cuegui/Plugins.py b/cuegui/cuegui/Plugins.py index 2b67ebd7e..f4e6e3524 100644 --- a/cuegui/cuegui/Plugins.py +++ b/cuegui/cuegui/Plugins.py @@ -58,7 +58,6 @@ import pickle from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import cuegui.Constants @@ -98,13 +97,12 @@ def __init__(self, mainWindow, name): self.__running = [] self.name = name self.mainWindow = mainWindow + self.app = cuegui.app() self.__menu_separator = " \t-> " # Load plugin paths from the config file - # pylint: disable=no-member - __pluginPaths = QtGui.qApp.settings.value("Plugin_Paths", []) - # pylint: enable=no-member + __pluginPaths = self.app.settings.value("Plugin_Paths", []) for path in cuegui.Constants.DEFAULT_PLUGIN_PATHS + __pluginPaths: self.loadPluginPath(str(path)) @@ -123,9 +121,7 @@ def loadConfigFilePlugins(self, configGroup): The imported module must have an init function and a QMainWindow will be passed to it. """ - # pylint: disable=no-member - __plugins = QtGui.qApp.settings.value("%s/Plugins" % configGroup, []) - # pylint: enable=no-member + __plugins = self.app.settings.value("%s/Plugins" % configGroup, []) for plugin in __plugins: path = os.path.dirname(str(plugin)) @@ -181,26 +177,20 @@ def saveState(self): opened.append("%s::%s" % (plugin[0], json.dumps(plugin[1].pluginSaveState()))) except Exception as e: logger.warning("Error saving plugin state for: %s\n%s", plugin[0], e) - # pylint: disable=no-member - QtGui.qApp.settings.setValue("%s/Plugins_Opened" % self.name, opened) - # pylint: enable=no-member + self.app.settings.setValue("%s/Plugins_Opened" % self.name, opened) def restoreState(self): """Loads any user defined plugin directories and restores all open plugins. Calls .restoreSettings (if available) on all plugins.""" # Loads any user defined plugin directories - # pylint: disable=no-member - pluginPaths = QtGui.qApp.settings.value("Plugins/Paths", []) - # pylint: enable=no-member + pluginPaths = self.app.settings.value("Plugins/Paths", []) for path in pluginPaths: self.loadPluginPath(str(path)) # Runs any plugins that were saved to the settings - # pylint: disable=no-member - openPlugins = QtGui.qApp.settings.value("%s/Plugins_Opened" % self.name) or [] - # pylint: enable=no-member + openPlugins = self.app.settings.value("%s/Plugins_Opened" % self.name) or [] for plugin in openPlugins: if '::' in plugin: plugin_name, plugin_state = str(plugin).split("::") @@ -357,6 +347,7 @@ class Plugin(object): def __init__(self): self.__settings = [] + self.app = cuegui.app() def pluginRestoreState(self, saved_settings): """Called on plugin start with any previously saved state. diff --git a/cuegui/cuegui/PreviewWidget.py b/cuegui/cuegui/PreviewWidget.py index 6e60a0a6a..56888188e 100644 --- a/cuegui/cuegui/PreviewWidget.py +++ b/cuegui/cuegui/PreviewWidget.py @@ -34,7 +34,6 @@ import xml.etree.ElementTree as Et from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import cuegui.Logger @@ -59,6 +58,8 @@ def __init__(self, job, frame, aovs=False, parent=None): :param parent: the parent widget """ QtWidgets.QDialog.__init__(self, parent) + self.app = cuegui.app() + self.__job = job self.__frame = frame self.__aovs = aovs @@ -93,9 +94,7 @@ def process(self): self.__itvFile = self.__writePlaylist(playlist) self.__previewThread = PreviewProcessorWatchThread(items, self) - # pylint: disable=no-member - QtGui.qApp.threads.append(self.__previewThread) - # pylint: enable=no-member + self.app.threads.append(self.__previewThread) self.__previewThread.start() self.__progbar.setRange(0, len(items)) diff --git a/cuegui/cuegui/ProcMonitor.py b/cuegui/cuegui/ProcMonitor.py index 84e01fd4b..4221a2c92 100644 --- a/cuegui/cuegui/ProcMonitor.py +++ b/cuegui/cuegui/ProcMonitor.py @@ -23,7 +23,6 @@ from builtins import str from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import cuegui.Logger @@ -42,6 +41,7 @@ class ProcMonitor(QtWidgets.QWidget): def __init__(self, parent): QtWidgets.QWidget.__init__(self, parent) + self.app = cuegui.app() self.__filterByHostNameLastInput = None @@ -71,7 +71,7 @@ def __init__(self, parent): self.__viewHostsSetup() - if bool(int(QtGui.qApp.settings.value("AutoRefreshMonitorProc", 1))): # pylint: disable=no-member + if bool(int(self.app.settings.value("AutoRefreshMonitorProc", 1))): self.updateRequest() def updateRequest(self): @@ -141,9 +141,7 @@ def __refreshToggleCheckBoxSetup(self, layout): def __refreshToggleCheckBoxHandle(self, state): self.procMonitorTree.enableRefresh = bool(state) - # pylint: disable=no-member - QtGui.qApp.settings.setValue("AutoRefreshMonitorProc", int(bool(state))) - # pylint: enable=no-member + self.app.settings.setValue("AutoRefreshMonitorProc", int(bool(state))) # ============================================================================== # Button to refresh @@ -192,7 +190,7 @@ def __clearButtonHandle(self): # Monitors and handles the view_procs signal # ============================================================================== def __viewProcsSetup(self): - QtGui.qApp.view_procs.connect(self.__viewProcsHandle) # pylint: disable=no-member + self.app.view_procs.connect(self.__viewProcsHandle) def __viewProcsHandle(self, hosts): self.procMonitorTree.procSearch.options['host'] = hosts @@ -202,7 +200,7 @@ def __viewProcsHandle(self, hosts): # Views procs when a host is double clicked # ============================================================================== def __hostDoubleClickedSetup(self): - QtGui.qApp.view_object.connect(self.__hostDoubleClickedHandle) # pylint: disable=no-member + self.app.view_object.connect(self.__hostDoubleClickedHandle) def __hostDoubleClickedHandle(self, rpcObject): if cuegui.Utils.isHost(rpcObject): @@ -213,7 +211,7 @@ def __hostDoubleClickedHandle(self, rpcObject): # Monitors and handles the view_hosts signal # ============================================================================== def __viewHostsSetup(self): - QtGui.qApp.view_hosts.connect(self.__viewHostsHandle) # pylint: disable=no-member + self.app.view_hosts.connect(self.__viewHostsHandle) def __viewHostsHandle(self, hosts): if hosts: diff --git a/cuegui/cuegui/ProcMonitorTree.py b/cuegui/cuegui/ProcMonitorTree.py index 97171e08f..cf2658de5 100644 --- a/cuegui/cuegui/ProcMonitorTree.py +++ b/cuegui/cuegui/ProcMonitorTree.py @@ -24,7 +24,6 @@ import time from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import opencue @@ -89,17 +88,13 @@ def __init__(self, parent): self.itemDoubleClicked.connect(self.__itemDoubleClickedViewLog) # Don't use the standard space bar to refresh - # pylint: disable=no-member - QtGui.qApp.request_update.connect(self.updateRequest) - # pylint: enable=no-member + self.app.request_update.connect(self.updateRequest) self.startTicksUpdate(40) # Don't start refreshing until the user sets a filter or hits refresh self.ticksWithoutUpdate = -1 - # pylint: disable=no-member - self.enableRefresh = bool(int(QtGui.qApp.settings.value("AutoRefreshMonitorProc", 1))) - # pylint: enable=no-member + self.enableRefresh = bool(int(self.app.settings.value("AutoRefreshMonitorProc", 1))) def tick(self): if self.ticksWithoutUpdate >= self.updateInterval and \ @@ -140,9 +135,7 @@ def __itemDoubleClickedViewLog(self, item, col): @param col: Column number double clicked on""" del col job_name = item.rpcObject.data.job_name - # pylint: disable=no-member - QtGui.qApp.view_object.emit(opencue.api.findJob(job_name)) - # pylint: enable=no-member + self.app.view_object.emit(opencue.api.findJob(job_name)) def clearFilters(self): """Removes all sorting and filtering to restore default state.""" diff --git a/cuegui/cuegui/ProgressDialog.py b/cuegui/cuegui/ProgressDialog.py index 375e2a189..2f507fda7 100644 --- a/cuegui/cuegui/ProgressDialog.py +++ b/cuegui/cuegui/ProgressDialog.py @@ -24,7 +24,6 @@ from builtins import range from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import cuegui.Logger @@ -57,6 +56,7 @@ def __init__(self, title, function, work, concurrent, cancelTitle, @type parent: QObject @param parent: The parent for this object""" QtWidgets.QDialog.__init__(self, parent) + self.app = cuegui.app() self.__work = work self.__function = function @@ -161,12 +161,9 @@ def _submitWork(self): """Submits a new unit of work to threadpool""" self.__count += 1 - if hasattr(QtGui.qApp, "threadpool"): - # pylint: disable=no-member - QtGui.qApp.threadpool.queue(self.__doWork, - self.__doneWork, - "getting data for %s" % self.__class__) - # pylint: enable=no-member + if self.app.threadpool is not None: + self.app.threadpool.queue( + self.__doWork, self.__doneWork, "getting data for %s" % self.__class__) else: logger.warning("threadpool not found, doing work in gui thread") self.__doneWork(None, self.__doWork()) diff --git a/cuegui/cuegui/Redirect.py b/cuegui/cuegui/Redirect.py index 49fb6b7a5..ae46814f2 100644 --- a/cuegui/cuegui/Redirect.py +++ b/cuegui/cuegui/Redirect.py @@ -415,6 +415,7 @@ class RedirectWidget(QtWidgets.QWidget): def __init__(self, parent=None): QtWidgets.QWidget.__init__(self, parent) + self.app = cuegui.app() self.__hosts = {} self.__controls = RedirectControls(self) @@ -464,7 +465,7 @@ def mouseDoubleClickEvent(self, index): jobObject = opencue.api.getJobs(job=[index.data()]) if jobObject: if cuegui.Utils.isJob(jobObject[0]): - QtGui.qApp.view_object.emit(jobObject[0]) + self.app.view_object.emit(jobObject[0]) except opencue.exception.CueException as e: text = ('Not able to add job to Job Monitor Tree. ' 'Error Message:\n %s' % e) diff --git a/cuegui/cuegui/ShowsWidget.py b/cuegui/cuegui/ShowsWidget.py index a91685cd1..f10fa08e3 100644 --- a/cuegui/cuegui/ShowsWidget.py +++ b/cuegui/cuegui/ShowsWidget.py @@ -21,7 +21,6 @@ from __future__ import division from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import opencue @@ -62,10 +61,8 @@ def __init__(self, parent): self.__menuActions = cuegui.MenuActions.MenuActions( self, self.updateSoon, self.selectedObjects) - # pylint: disable=no-member - self.itemClicked.connect(self.__itemSingleClickedToDouble) - QtGui.qApp.facility_changed.connect(self.__facilityChanged) - # pylint: enable=no-member + self.itemClicked.connect(self.__itemSingleClickedToDouble) # pylint: disable=no-member + self.app.facility_changed.connect(self.__facilityChanged) self.setUpdateInterval(60) diff --git a/cuegui/cuegui/Style.py b/cuegui/cuegui/Style.py index 3f25bbed0..d95b80dd8 100644 --- a/cuegui/cuegui/Style.py +++ b/cuegui/cuegui/Style.py @@ -24,6 +24,8 @@ from PySide2 import QtGui +import cuegui + DEFAULT_COLOR = "DarkPalette" DEFAULT_ICON = "crystal" @@ -56,16 +58,12 @@ def setFont(font): """Sets the application font.""" global Font Font = font - # pylint: disable=no-member - QtGui.qApp.setFont(font) - # pylint: enable=no-member + cuegui.app().setFont(font) def init(): """Initializes the global style settings.""" - # pylint: disable=no-member - settings = QtGui.qApp.settings - # pylint: enable=no-member + settings = cuegui.app().settings loadColorTheme(settings.value("Style/colorTheme", DEFAULT_COLOR)) setIconTheme(settings.value("Style/iconTheme", DEFAULT_ICON)) diff --git a/cuegui/cuegui/SubscriptionsWidget.py b/cuegui/cuegui/SubscriptionsWidget.py index 8be93ad00..575742f4b 100644 --- a/cuegui/cuegui/SubscriptionsWidget.py +++ b/cuegui/cuegui/SubscriptionsWidget.py @@ -25,7 +25,6 @@ import opencue from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import cuegui.AbstractTreeWidget @@ -41,6 +40,7 @@ class SubscriptionsWidget(QtWidgets.QWidget): def __init__(self, parent): QtWidgets.QWidget.__init__(self, parent) + self.app = cuegui.app() self.__show = None self.__shows = None @@ -68,9 +68,9 @@ def __init__(self, parent): self.__btnShowProperties.clicked.connect(self.__showProperties) self.__btnAddSubscription.clicked.connect(self.__addSubscription) self.__comboShows.currentIndexChanged.connect(self.setShow) - QtGui.qApp.view_object.connect(self.setShow) - QtGui.qApp.facility_changed.connect(self.changeFacility) # pylint: enable=no-member + self.app.view_object.connect(self.setShow) + self.app.facility_changed.connect(self.changeFacility) self.__menuActions = cuegui.MenuActions.MenuActions( self, self.updateSoon, self.selectedObjects) diff --git a/cuegui/cuegui/ThreadPool.py b/cuegui/cuegui/ThreadPool.py index 73d791d50..ba03ddc8f 100644 --- a/cuegui/cuegui/ThreadPool.py +++ b/cuegui/cuegui/ThreadPool.py @@ -50,7 +50,6 @@ def someWorkCallback(work, result): import os from PySide2 import QtCore -from PySide2 import QtGui import cuegui.Logger @@ -74,6 +73,7 @@ class ThreadPool(QtCore.QObject): def __init__(self, num_threads, max_queue=20, parent=None): QtCore.QObject.__init__(self, parent=parent) + self.app = cuegui.app() self.__threads = [] self.__started = False self.__max_queue = max_queue @@ -90,9 +90,7 @@ def start(self): self.__started = True for i in range(0, self.__num_threads): thread = ThreadPool.WorkerThread(i, self) - # pylint: disable=no-member - QtGui.qApp.threads.append(thread) - # pylint: enable=no-member + self.app.threads.append(thread) self.__threads.append(thread) self.__threads[i].start() self.__threads[i].workComplete.connect(self.runCallback, diff --git a/cuegui/cuegui/Utils.py b/cuegui/cuegui/Utils.py index 8d41df3cf..07cb5c025 100644 --- a/cuegui/cuegui/Utils.py +++ b/cuegui/cuegui/Utils.py @@ -489,14 +489,13 @@ def popupView(file, facility=None): """Opens the given file in your editor.""" if file and not popupWeb(file, facility): editor_from_env = os.getenv('EDITOR') - # pylint: disable=no-member + app = cuegui.app() if editor_from_env: job_log_cmd = editor_from_env.split() - elif QtGui.qApp.settings.contains('LogEditor'): - job_log_cmd = QtGui.qApp.settings.value("LogEditor") + elif app.settings.contains('LogEditor'): + job_log_cmd = app.settings.value("LogEditor") else: job_log_cmd = cuegui.Constants.DEFAULT_EDITOR.split() - # pylint: enable=no-member job_log_cmd.append(str(file)) checkShellOut(job_log_cmd) diff --git a/cuegui/cuegui/__init__.py b/cuegui/cuegui/__init__.py index d9c80f13d..ce0dbd72e 100644 --- a/cuegui/cuegui/__init__.py +++ b/cuegui/cuegui/__init__.py @@ -11,3 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +"""Top level module for the CueGUI application. + +See __main__.py and Main.py for the application entrypoint. +""" + +from .App import create_app +from .App import app diff --git a/cuegui/cuegui/plugins/AllocationsPlugin.py b/cuegui/cuegui/plugins/AllocationsPlugin.py index b181a350b..9e7d1a647 100644 --- a/cuegui/cuegui/plugins/AllocationsPlugin.py +++ b/cuegui/cuegui/plugins/AllocationsPlugin.py @@ -22,7 +22,7 @@ from builtins import map -from PySide2 import QtGui, QtWidgets +from PySide2 import QtWidgets import opencue @@ -104,10 +104,7 @@ def __init__(self, parent): self.setDragEnabled(True) self.setDragDropMode(QtWidgets.QAbstractItemView.DragDrop) - # Signals are defined in code, so pylint thinks they don't exist. - # pylint: disable=no-member - QtGui.qApp.facility_changed.connect(self._update) - # pylint: enable=no-member + self.app.facility_changed.connect(self._update) self.setUpdateInterval(60) diff --git a/cuegui/cuegui/plugins/AttributesPlugin.py b/cuegui/cuegui/plugins/AttributesPlugin.py index b20825204..26f9350de 100644 --- a/cuegui/cuegui/plugins/AttributesPlugin.py +++ b/cuegui/cuegui/plugins/AttributesPlugin.py @@ -24,7 +24,6 @@ from builtins import str import time -from PySide2 import QtGui from PySide2 import QtCore from PySide2 import QtWidgets @@ -89,6 +88,8 @@ class Attributes(QtWidgets.QWidget): """ def __init__(self, parent=None): QtWidgets.QWidget.__init__(self, parent) + self.app = cuegui.app() + layout = QtWidgets.QVBoxLayout(self) layout.setContentsMargins(0, 0, 0, 0) @@ -106,9 +107,7 @@ def __init__(self, parent=None): self.__scrollWidget.layout().addWidget(self.__stack) self.__scrollArea.setWidget(self.__scrollWidget) layout.addWidget(self.__scrollArea) - # pylint: disable=no-member - QtGui.qApp.single_click.connect(self.setWidget) - # pylint: enable=no-member + self.app.single_click.connect(self.setWidget) self.__load = None @@ -138,13 +137,10 @@ def setWidget(self, item): # called in a worker thread prior to the creation of the widget. # Otherwise the widget will just be created now. if hasattr(function, "preload"): - if hasattr(QtGui.qApp, "threadpool"): + if self.app.threadpool is not None: self.__load = {"item": item, "function": function} - # pylint: disable=no-member - QtGui.qApp.threadpool.queue(self.__getUpdate, - self.__processResults, - "getting data for %s" % self.__class__) - # pylint: enable=no-member + self.app.threadpool.queue( + self.__getUpdate, self.__processResults, "getting data for %s" % self.__class__) else: logger.warning("threadpool not found, doing work in gui thread") return self.__createItemAttribute(item, function, function.preload(item)) diff --git a/cuegui/cuegui/plugins/LogViewPlugin.py b/cuegui/cuegui/plugins/LogViewPlugin.py index 8e8486a1b..caeacf449 100644 --- a/cuegui/cuegui/plugins/LogViewPlugin.py +++ b/cuegui/cuegui/plugins/LogViewPlugin.py @@ -300,9 +300,10 @@ def __init__(self, parent=None): """ Create the UI elements """ + QtWidgets.QWidget.__init__(self, parent) + self.app = cuegui.app() # Main Widget - QtWidgets.QWidget.__init__(self, parent) layout = QtWidgets.QVBoxLayout(self) layout.setContentsMargins(0, 0, 0, 0) self._scrollArea = QtWidgets.QScrollArea() @@ -428,9 +429,7 @@ def __init__(self, parent=None): pos = QtCore.QPoint(0, 0) self._highlight_cursor = self._content_box.cursorForPosition(pos) # Signals are defined in code, so pylint thinks they don't exist. - # pylint: disable=no-member - QtGui.qApp.display_log_file_content.connect(self._set_log_files) - # pylint: enable=no-member + self.app.display_log_file_content.connect(self._set_log_files) self._log_scrollbar = self._content_box.verticalScrollBar() self._log_scrollbar.valueChanged.connect(self._set_scrollbar_value) @@ -856,9 +855,7 @@ def _update_log(self): else: self._content_box.appendPlainText(new_text) self._content_timestamp = time.time() - # pylint: disable=no-member - QtGui.qApp.processEvents() - # pylint: enable=no-member + self.app.processEvents() # Adjust scrollbar value (if necessary) self._scrollbar_max = self._log_scrollbar.maximum() diff --git a/cuegui/cuegui/plugins/MonitorCuePlugin.py b/cuegui/cuegui/plugins/MonitorCuePlugin.py index 8edbda25b..c0b3e6fac 100644 --- a/cuegui/cuegui/plugins/MonitorCuePlugin.py +++ b/cuegui/cuegui/plugins/MonitorCuePlugin.py @@ -78,9 +78,7 @@ def __init__(self, parent): self.layout().addLayout(self.__hlayout) - # pylint: disable=no-member - self.__monitorCue.view_object.connect(QtGui.qApp.view_object.emit) - # pylint: enable=no-member + self.__monitorCue.view_object.connect(self.app.view_object.emit) self.pluginRegisterSettings([("shows", self.__monitorCue.getShowNames, @@ -98,9 +96,7 @@ def __init__(self, parent): self.addShows([os.getenv('SHOW')]) def __cueStateBarSetup(self, layout): - # pylint: disable=no-member - cueStateBarEnabled = QtGui.qApp.settings.value("CueStateBar", False) - # pylint: enable=no-member + cueStateBarEnabled = self.app.settings.value("CueStateBar", False) if cueStateBarEnabled: self.__cueStateBar = cuegui.CueStateBarWidget.CueStateBarWidget(self.__monitorCue, self) layout.addWidget(self.__cueStateBar) @@ -175,10 +171,8 @@ def __showMenuSetup(self): self.__showMenuBtn.setMenu(self.__showMenu) self.__showMenuBtn.setFocusPolicy(QtCore.Qt.NoFocus) self.__showMenu.setFont(cuegui.Constants.STANDARD_FONT) - # pylint: disable=no-member - self.__showMenu.triggered.connect(self.__showMenuHandle) - QtGui.qApp.facility_changed.connect(self.__showMenuUpdate) - # pylint: enable=no-member + self.__showMenu.triggered.connect(self.__showMenuHandle) # pylint: disable=no-member + self.app.facility_changed.connect(self.__showMenuUpdate) self.__showMenuUpdate() diff --git a/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py b/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py index 809cefbfc..270a74b23 100644 --- a/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py @@ -22,7 +22,6 @@ from __future__ import division from __future__ import absolute_import -from PySide2 import QtGui from PySide2 import QtCore from PySide2 import QtWidgets @@ -64,11 +63,9 @@ def __init__(self, parent): self.__splitter.addWidget(self.__monitorLayers) self.__splitter.addWidget(self.__monitorFrames) - # pylint: disable=no-member - QtGui.qApp.view_object.connect(self.__setJob) - QtGui.qApp.unmonitor.connect(self.__unmonitor) - QtGui.qApp.facility_changed.connect(self.__setJob) - # pylint: enable=no-member + self.app.view_object.connect(self.__setJob) + self.app.unmonitor.connect(self.__unmonitor) + self.app.facility_changed.connect(self.__setJob) self.__monitorLayers.handle_filter_layers_byLayer.connect(self.handleLayerFilter) self.__splitter.splitterMoved.connect(self.__splitterMoved) # pylint: disable=no-member diff --git a/cuegui/cuegui/plugins/MonitorJobsPlugin.py b/cuegui/cuegui/plugins/MonitorJobsPlugin.py index 94b081b13..f9ab677c2 100644 --- a/cuegui/cuegui/plugins/MonitorJobsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorJobsPlugin.py @@ -69,10 +69,8 @@ def __init__(self, parent): self.layout().addWidget(self.jobMonitor) # Signals in - # pylint: disable=no-member - QtGui.qApp.view_object.connect(self.addJob) - QtGui.qApp.facility_changed.connect(self.jobMonitor.removeAllItems) - # pylint: enable=no-member + self.app.view_object.connect(self.addJob) + self.app.facility_changed.connect(self.jobMonitor.removeAllItems) # Signals out self.jobMonitor.view_object.connect(self.view_object.emit) diff --git a/cuegui/cuegui/plugins/ServicePlugin.py b/cuegui/cuegui/plugins/ServicePlugin.py index cc2a99bef..009ed450b 100644 --- a/cuegui/cuegui/plugins/ServicePlugin.py +++ b/cuegui/cuegui/plugins/ServicePlugin.py @@ -20,8 +20,6 @@ from __future__ import division from __future__ import absolute_import -from PySide2 import QtGui - import cuegui.AbstractDockWidget import cuegui.ServiceDialog @@ -41,6 +39,4 @@ def __init__(self, parent): self.setWindowTitle("Facility Service Defaults") self.__serviceManager = cuegui.ServiceDialog.ServiceManager(None, self) self.layout().addWidget(self.__serviceManager) - # pylint: disable=no-member - QtGui.qApp.facility_changed.connect(self.__serviceManager.refresh) - # pylint: enable=no-member + self.app.facility_changed.connect(self.__serviceManager.refresh) diff --git a/cuegui/cuegui/plugins/StuckFramePlugin.py b/cuegui/cuegui/plugins/StuckFramePlugin.py index 9a289b49b..f3d5e2904 100644 --- a/cuegui/cuegui/plugins/StuckFramePlugin.py +++ b/cuegui/cuegui/plugins/StuckFramePlugin.py @@ -35,12 +35,17 @@ from PySide2 import QtCore from PySide2 import QtWidgets -import opencue +import opencue.wrappers.frame + import cuegui.AbstractDockWidget +import cuegui.AbstractTreeWidget +import cuegui.AbstractWidgetItem import cuegui.Action import cuegui.Constants import cuegui.JobMonitorTree import cuegui.Logger +import cuegui.MenuActions +import cuegui.Style import cuegui.Utils logger = cuegui.Logger.getLogger(__file__) @@ -772,7 +777,7 @@ def __init__(self, parent): self._updateProgressMax.connect(self.updateProgressMax) # Don't use the standard space bar to refres - self.disconnect(QtGui.qApp, + self.disconnect(self.app, QtCore.SIGNAL('request_update()'), self.updateRequest) @@ -798,10 +803,10 @@ def __init__(self, parent): def logIt(self): """Logs cache to a file.""" - if hasattr(QtGui.qApp, "threadpool"): + if self.app.threadpool is not None: print("Stuck Frame Log cache is being written to file.") - QtGui.qApp.threadpool.queue(self.run_log.finalize, self.logResult, - "Writing out log", self.frames, self.show) + self.app.threadpool.queue( + self.run_log.finalize, self.logResult, "Writing out log", self.frames, self.show) else: logger.warning("threadpool not found, doing work in gui thread") @@ -1214,7 +1219,7 @@ def contextMenuEvent(self, e): menu.addSeparator() menu.addAction(cuegui.Action.create(self, "Top Machine", "Top Machine", self.topMachine, "up")) - if QtGui.qApp.applicationName() == "CueCommander3": + if self.app.applicationName() == "CueCommander3": self.__menuActions.frames().addAction(menu, "viewHost") menu.addSeparator() @@ -1603,7 +1608,7 @@ def __init__(self, object, parent): if not self.__initialized: self.__class__.__initialized = True self.__class__.__commentIcon = QtGui.QIcon(":comment.png") - self.__class__.__backgroundColor = QtGui.qApp.palette().color(QtGui.QPalette.Base) + self.__class__.__backgroundColor = cuegui.app().palette().color(QtGui.QPalette.Base) self.__class__.__foregroundColor = cuegui.Style.ColorTheme.COLOR_JOB_FOREGROUND cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__(self, cuegui.Constants.TYPE_FRAME, diff --git a/cuegui/tests/CueJobMonitorTree_tests.py b/cuegui/tests/CueJobMonitorTree_tests.py index 87dd0a635..2cf93d0d5 100644 --- a/cuegui/tests/CueJobMonitorTree_tests.py +++ b/cuegui/tests/CueJobMonitorTree_tests.py @@ -39,8 +39,8 @@ class CueJobMonitorTreeTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, get_stub_mock): - test_utils.createApplication() - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + app = test_utils.createApplication() + app.settings = PySide2.QtCore.QSettings() cuegui.Style.init() self.show_name = 'arbitrary-show-name' diff --git a/cuegui/tests/DependWizard_tests.py b/cuegui/tests/DependWizard_tests.py index 0e8667460..de1632502 100644 --- a/cuegui/tests/DependWizard_tests.py +++ b/cuegui/tests/DependWizard_tests.py @@ -40,8 +40,8 @@ class DependWizardTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): - test_utils.createApplication() - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + app = test_utils.createApplication() + app.settings = PySide2.QtCore.QSettings() cuegui.Style.init() self.parentWidget = PySide2.QtWidgets.QWidget() diff --git a/cuegui/tests/FilterDialog_tests.py b/cuegui/tests/FilterDialog_tests.py index 071a2dab9..bfe4c5832 100644 --- a/cuegui/tests/FilterDialog_tests.py +++ b/cuegui/tests/FilterDialog_tests.py @@ -41,8 +41,8 @@ class FilterDialogTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, getStubMock): - test_utils.createApplication() - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + app = test_utils.createApplication() + app.settings = PySide2.QtCore.QSettings() cuegui.Style.init() self.show = opencue.wrappers.show.Show(opencue.compiled_proto.show_pb2.Show(name='fooShow')) @@ -154,8 +154,8 @@ class FilterMonitorTreeTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, getStubMock): - test_utils.createApplication() - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + app = test_utils.createApplication() + app.settings = PySide2.QtCore.QSettings() cuegui.Style.init() show = opencue.wrappers.show.Show(opencue.compiled_proto.show_pb2.Show(name='fooShow')) @@ -199,8 +199,8 @@ class MatcherMonitorTreeTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, getStubMock): - test_utils.createApplication() - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + app = test_utils.createApplication() + app.settings = PySide2.QtCore.QSettings() cuegui.Style.init() self.matchers = [ diff --git a/cuegui/tests/FrameMonitorTree_tests.py b/cuegui/tests/FrameMonitorTree_tests.py index c3c2b4963..fc77e864c 100644 --- a/cuegui/tests/FrameMonitorTree_tests.py +++ b/cuegui/tests/FrameMonitorTree_tests.py @@ -43,9 +43,8 @@ class FrameMonitorTreeTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): - test_utils.createApplication() - - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + app = test_utils.createApplication() + app.settings = PySide2.QtCore.QSettings() cuegui.Style.init() self.parentWidget = PySide2.QtWidgets.QWidget() self.frameMonitorTree = cuegui.FrameMonitorTree.FrameMonitorTree(self.parentWidget) diff --git a/cuegui/tests/LayerDialog_tests.py b/cuegui/tests/LayerDialog_tests.py index 5e515775d..78d4d5060 100644 --- a/cuegui/tests/LayerDialog_tests.py +++ b/cuegui/tests/LayerDialog_tests.py @@ -47,8 +47,8 @@ class LayerPropertiesDialogTests(unittest.TestCase): @mock.patch('opencue.api.getLayer') @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, get_stub_mock, get_layer_mock, get_limits_mock): - test_utils.createApplication() - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + app = test_utils.createApplication() + app.settings = PySide2.QtCore.QSettings() cuegui.Style.init() self.layers = { diff --git a/cuegui/tests/MenuActions_tests.py b/cuegui/tests/MenuActions_tests.py index b15a2cc9a..25033c79b 100644 --- a/cuegui/tests/MenuActions_tests.py +++ b/cuegui/tests/MenuActions_tests.py @@ -52,7 +52,7 @@ import cuegui.CueJobMonitorTree import cuegui.Main import cuegui.MenuActions - +from . import test_utils _GB_TO_KB = 1024 * 1024 @@ -60,6 +60,7 @@ @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) class JobActionsTests(unittest.TestCase): def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.job_actions = cuegui.MenuActions.JobActions(self.widgetMock, mock.Mock(), None, None) @@ -71,14 +72,14 @@ def test_unmonitor(self): self.widgetMock.actionRemoveSelectedItems.assert_called_with() - @mock.patch('PySide2.QtGui.qApp') - def test_view(self, qAppMock): + def test_view(self): + self.app.view_object = mock.Mock() job_name = 'arbitrary-name' job = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(name=job_name)) self.job_actions.view(rpcObjects=[job, opencue.wrappers.frame.Frame()]) - qAppMock.view_object.emit.assert_called_once_with(job) + self.app.view_object.emit.assert_called_once_with(job) @mock.patch('cuegui.DependDialog.DependDialog') def test_viewDepends(self, dependDialogMock): @@ -545,6 +546,7 @@ class LayerActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.job = mock.create_autospec(opencue.wrappers.job.Job()) self.layer_actions = cuegui.MenuActions.LayerActions( @@ -793,6 +795,7 @@ class FrameActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.job = mock.create_autospec(opencue.wrappers.job.Job()) self.frame_actions = cuegui.MenuActions.FrameActions( @@ -870,8 +873,9 @@ def test_xdiff3(self, popupFrameXdiffMock): popupFrameXdiffMock.assert_called_with(self.job, frame1, frame2, frame3) @mock.patch('opencue.api.findHost') - @mock.patch('PySide2.QtGui.qApp') - def test_viewHost(self, qAppMock, findHostMock): + def test_viewHost(self, findHostMock): + self.app.view_hosts = mock.Mock() + self.app.single_click = mock.Mock() host_name = 'arbitrary-host-name' host = opencue.wrappers.host.Host( opencue.compiled_proto.host_pb2.Host(id='arbitrary-id', name=host_name)) @@ -881,8 +885,8 @@ def test_viewHost(self, qAppMock, findHostMock): self.frame_actions.viewHost(rpcObjects=[frame]) - qAppMock.view_hosts.emit.assert_called_with([host_name]) - qAppMock.single_click.emit.assert_called_with(host) + self.app.view_hosts.emit.assert_called_with([host_name]) + self.app.single_click.emit.assert_called_with(host) def test_getWhatThisDependsOn(self): frame = opencue.wrappers.frame.Frame() @@ -1049,6 +1053,7 @@ class ShowActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.show_actions = cuegui.MenuActions.ShowActions( self.widgetMock, mock.Mock(), None, None) @@ -1085,6 +1090,7 @@ class GroupActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.group_actions = cuegui.MenuActions.GroupActions( self.widgetMock, mock.Mock(), None, None) @@ -1122,6 +1128,7 @@ class SubscriptionActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.subscription_actions = cuegui.MenuActions.SubscriptionActions( self.widgetMock, mock.Mock(), None, None) @@ -1175,6 +1182,7 @@ class HostActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.host_actions = cuegui.MenuActions.HostActions( self.widgetMock, mock.Mock(), None, None) @@ -1188,15 +1196,15 @@ def test_viewComments(self, commentListDialogMock): commentListDialogMock.assert_called_with(host, mock.ANY) commentListDialogMock.return_value.show.assert_called() - @mock.patch('PySide2.QtGui.qApp') - def test_viewProc(self, qAppMock): + def test_viewProc(self): + self.app.view_procs = mock.Mock() hostName = 'arbitrary-name' host = opencue.wrappers.host.Host( opencue.compiled_proto.host_pb2.Host(id='arbitrary-id', name=hostName)) self.host_actions.viewProc(rpcObjects=[opencue.wrappers.layer.Layer, host, host]) - qAppMock.view_procs.emit.assert_called_with([hostName]) + self.app.view_procs.emit.assert_called_with([hostName]) def test_lock(self): host = opencue.wrappers.host.Host( @@ -1337,13 +1345,14 @@ class ProcActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.proc_actions = cuegui.MenuActions.ProcActions( self.widgetMock, mock.Mock(), None, None) - @mock.patch('PySide2.QtGui.qApp') @mock.patch('opencue.api.findJob') - def test_view(self, findJobMock, qAppMock): + def test_view(self, findJobMock): + self.app.view_object = mock.Mock() jobName = 'arbitraryJobName' job = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(name=jobName)) proc = opencue.wrappers.proc.Proc(opencue.compiled_proto.host_pb2.Proc(job_name=jobName)) @@ -1351,7 +1360,7 @@ def test_view(self, findJobMock, qAppMock): self.proc_actions.view(rpcObjects=[opencue.wrappers.layer.Layer, proc]) - qAppMock.view_object.emit.assert_called_once_with(job) + self.app.view_object.emit.assert_called_once_with(job) @mock.patch('cuegui.Utils.questionBoxYesNo', new=mock.Mock(return_value=True)) def test_kill(self): @@ -1386,6 +1395,7 @@ class DependenciesActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.dep_actions = cuegui.MenuActions.DependenciesActions( self.widgetMock, mock.Mock(), None, None) @@ -1412,6 +1422,7 @@ class FilterActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.filter_actions = cuegui.MenuActions.FilterActions( self.widgetMock, mock.Mock(), None, None) @@ -1485,6 +1496,7 @@ class MatcherActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.matcher_actions = cuegui.MenuActions.MatcherActions( self.widgetMock, mock.Mock(), None, None) @@ -1515,6 +1527,7 @@ class ActionActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.action_actions = cuegui.MenuActions.ActionActions( self.widgetMock, mock.Mock(), None, None) @@ -1534,6 +1547,7 @@ class TaskActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.task_actions = cuegui.MenuActions.TaskActions( self.widgetMock, mock.Mock(), None, None) @@ -1572,6 +1586,7 @@ class LimitActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.limit_actions = cuegui.MenuActions.LimitActions( self.widgetMock, mock.Mock(), None, None) @@ -1622,6 +1637,7 @@ def test_rename(self, getTextMock): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) class MenuActionsTests(unittest.TestCase): def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.args = [self.widgetMock, lambda: None, lambda: None, lambda: None] self.menuActions = cuegui.MenuActions.MenuActions(*self.args) diff --git a/cuegui/tests/Redirect_tests.py b/cuegui/tests/Redirect_tests.py index d1e7ce5dd..5e1f6ad27 100644 --- a/cuegui/tests/Redirect_tests.py +++ b/cuegui/tests/Redirect_tests.py @@ -36,8 +36,8 @@ class RedirectTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, getStubMock): - test_utils.createApplication() - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + app = test_utils.createApplication() + app.settings = PySide2.QtCore.QSettings() cuegui.Style.init() getStubMock.return_value.GetActiveShows.return_value = \ diff --git a/cuegui/tests/UnbookDialog_tests.py b/cuegui/tests/UnbookDialog_tests.py index 2482aa201..a62b390e7 100644 --- a/cuegui/tests/UnbookDialog_tests.py +++ b/cuegui/tests/UnbookDialog_tests.py @@ -46,8 +46,8 @@ class UnbookDialogTests(unittest.TestCase): @mock.patch('opencue.api.findShow') @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, get_stub_mock, find_show_mock): - test_utils.createApplication() - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + app = test_utils.createApplication() + app.settings = PySide2.QtCore.QSettings() cuegui.Style.init() show_name = 'showname' @@ -202,8 +202,8 @@ def test__should_redirect_proc_to_job( class SelectItemsWithSearchDialogTests(unittest.TestCase): def setUp(self): - test_utils.createApplication() - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + app = test_utils.createApplication() + app.settings = PySide2.QtCore.QSettings() cuegui.Style.init() def test__should_display_all_items(self): @@ -245,8 +245,8 @@ def test__should_return_selected_items(self): class KillConfirmationDialogTests(unittest.TestCase): def setUp(self): - test_utils.createApplication() - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + app = test_utils.createApplication() + app.settings = PySide2.QtCore.QSettings() cuegui.Style.init() @mock.patch('PySide2.QtWidgets.QMessageBox.information', new=mock.Mock()) diff --git a/cuegui/tests/plugins/LogViewPlugin_tests.py b/cuegui/tests/plugins/LogViewPlugin_tests.py index 9ceb4a5b6..8206176f8 100644 --- a/cuegui/tests/plugins/LogViewPlugin_tests.py +++ b/cuegui/tests/plugins/LogViewPlugin_tests.py @@ -64,26 +64,26 @@ def setUp(self): self.fs.create_file(self.logPath2, contents=_LOG_TEXT_2) test_utils.createApplication() - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + cuegui.app().settings = PySide2.QtCore.QSettings() cuegui.Style.init() self.parentWidget = PySide2.QtWidgets.QMainWindow() self.logViewPlugin = cuegui.plugins.LogViewPlugin.LogViewPlugin(self.parentWidget) def test_shouldDisplayFirstLogFile(self): - PySide2.QtGui.qApp.display_log_file_content.emit([self.logPath1, self.logPath2]) + cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) self.assertEqual(_LOG_TEXT_1, self.logViewPlugin.logview_widget._content_box.toPlainText()) def test_shouldUpdateLogFile(self): - PySide2.QtGui.qApp.display_log_file_content.emit([self.logPath1, self.logPath2]) + cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) new_contents = _LOG_TEXT_1 + '\nanother line at the end' self.log1.set_contents(new_contents) - PySide2.QtGui.qApp.display_log_file_content.emit([self.logPath1, self.logPath2]) + cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) self.assertEqual(new_contents, self.logViewPlugin.logview_widget._content_box.toPlainText()) def test_shouldHighlightAllSearchResults(self): - PySide2.QtGui.qApp.display_log_file_content.emit([self.logPath1, self.logPath2]) + cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( PySide2.QtCore.Qt.CheckState.Unchecked) @@ -100,7 +100,7 @@ def test_shouldHighlightAllSearchResults(self): self.logViewPlugin.logview_widget._content_box, matches[1][0], matches[1][1])) def test_shouldMoveCursorToSecondSearchResult(self): - PySide2.QtGui.qApp.display_log_file_content.emit([self.logPath1, self.logPath2]) + cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( PySide2.QtCore.Qt.CheckState.Unchecked) @@ -114,7 +114,7 @@ def test_shouldMoveCursorToSecondSearchResult(self): self.assertEqual(132, self.logViewPlugin.logview_widget._cursor.position()) def test_shouldMoveCursorLastSearchResult(self): - PySide2.QtGui.qApp.display_log_file_content.emit([self.logPath1, self.logPath2]) + cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( PySide2.QtCore.Qt.CheckState.Unchecked) @@ -128,7 +128,7 @@ def test_shouldMoveCursorLastSearchResult(self): self.assertEqual(132, self.logViewPlugin.logview_widget._cursor.position()) def test_shouldPerformCaseInsensitiveSearch(self): - PySide2.QtGui.qApp.display_log_file_content.emit([self.logPath1, self.logPath2]) + cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( PySide2.QtCore.Qt.CheckState.Checked) diff --git a/cuegui/tests/test_utils.py b/cuegui/tests/test_utils.py index dfcb446db..dd07840d4 100644 --- a/cuegui/tests/test_utils.py +++ b/cuegui/tests/test_utils.py @@ -16,17 +16,8 @@ """Common utility functions for CueGUI test code.""" -import PySide2.QtGui +import cuegui -import cuegui.Main - -__QAPPLICATION_SINGLETON = None - - -# pylint: disable=global-statement def createApplication(): - global __QAPPLICATION_SINGLETON - if __QAPPLICATION_SINGLETON is None: - __QAPPLICATION_SINGLETON = cuegui.Main.CueGuiApplication() - PySide2.QtGui.qApp = __QAPPLICATION_SINGLETON + return cuegui.create_app([]) From c8eb6cad0cc1ce61d863afe2ea0ee7af10306751 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 7 Nov 2022 12:24:57 -0500 Subject: [PATCH 225/277] Notes from last two TSC meetings. (#1216) --- tsc/meetings/2022-09-28.md | 31 +++++++++++++++++++++++++++++++ tsc/meetings/2022-10-26.md | 24 ++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 tsc/meetings/2022-09-28.md create mode 100644 tsc/meetings/2022-10-26.md diff --git a/tsc/meetings/2022-09-28.md b/tsc/meetings/2022-09-28.md new file mode 100644 index 000000000..f22833839 --- /dev/null +++ b/tsc/meetings/2022-09-28.md @@ -0,0 +1,31 @@ +# OpenCue TSC Meeting Notes 28 Sep 2022 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Update on M1 issues / CI failures + * All issues now have proposed fixes. Working our way through review. +* Preparing for next release + * qApp PR needs review/merge. This is the highest priority to unbreak our CI pipelines. + * Logging flags. Dropping --log.frame-log-root is a breaking change. Continue supporting it for + now? Add a deprecation warning? + * Possible solution: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1203 + * Could also just rename the new flags. + * Will follow up on the PR. + * Integration test script + * Existing script is very basic and SPI specific. Won't really save us time vs writing from + scratch. + * CueGUI is the challenge here. Automating API calls is trivial, automating GUI tests much + more involved. + * Script to test our sandbox setup is a good place to start. This is critical for ensuring + new users have a good experience. + * "testing" show deactivated? + * Reported in https://github.com/AcademySoftwareFoundation/OpenCue/issues/1205 + * Culprit PR: ​​https://github.com/AcademySoftwareFoundation/OpenCue/pull/1151 + * Does automated show cleanup, testing show is not in the allowlist. + * Quick fix: allowlist testing show in properties file. + * Permanent fix: change 30 day hardcoded limit to come from properties files. -1 means + feature is disabled. Disable this by default. +* Redis + * No update yet, work to resume later. diff --git a/tsc/meetings/2022-10-26.md b/tsc/meetings/2022-10-26.md new file mode 100644 index 000000000..d4c21d39f --- /dev/null +++ b/tsc/meetings/2022-10-26.md @@ -0,0 +1,24 @@ +# OpenCue TSC Meeting Notes 26 Oct 2022 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Intros + * Holovisn.com + * Intro from team, discussion of OpenCue use case, demo. +* RQD completely ignores env on Windows + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1211 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1215 + * LGTM +* RQD doesn't Nimby with Remote users on machine + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/564 + * Diego: this is fixed by using pynput. PR for this has been merged, release is needed. +* Release blockers + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1193 + * Testing/review will commence shortly. + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1203 + * Let's merge the PR. Not ideal but lesson is learned for the future. + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1199 + * Brian to try again to see if problem persists. + * Diego to ask DBA about the issue and potential performance issues created by the fix. From 35aa4612d70fc15c1b2b6a97bde2f0f3aed7a29a Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 6 Dec 2022 12:29:29 -0500 Subject: [PATCH 226/277] [cuebot] Automatically replace --log.frame-log-root with the new flag. (#1203) --- .../imageworks/spcue/CuebotApplication.java | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/CuebotApplication.java b/cuebot/src/main/java/com/imageworks/spcue/CuebotApplication.java index a5747e558..12087f93e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/CuebotApplication.java +++ b/cuebot/src/main/java/com/imageworks/spcue/CuebotApplication.java @@ -16,18 +16,48 @@ */ - package com.imageworks.spcue; +import java.util.Arrays; +import java.util.Optional; +import java.util.stream.Stream; + +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @SpringBootApplication public class CuebotApplication extends SpringApplication { + private static String[] checkArgs(String[] args) { + Optional deprecatedFlag = Arrays.stream(args) + .filter(arg -> arg.startsWith("--log.frame-log-root=")).findFirst(); + if (deprecatedFlag.isPresent()) { + // Log a deprecation warning. + Logger warning_logger = Logger.getLogger(CuebotApplication.class); + warning_logger.warn("`--log.frame-log-root` is deprecated and will be removed in an " + + "upcoming release. It has been replaced with `--log.frame-log-root.default_os`. " + + "See opencue.properties for details on OpenCue's new OS-dependent root directories."); + // If new flags are not present, swap in the value provided using the new flag. + // If the new flags are already present, don't do anything. + Optional newFlags = Arrays.stream(args) + .filter(arg -> arg.startsWith("--log.frame-log-root.")).findAny(); + if (!newFlags.isPresent()) { + String fixedFlag = "--log.frame-log-root.default_os=" + + StringUtils.substringAfter(deprecatedFlag.get(), "="); + args = Stream.concat( + Arrays.stream(args).filter(arg -> !arg.startsWith("--log.frame-log-root=")), + Stream.of(fixedFlag)) + .toArray(String[]::new); + } + } + return args; + } public static void main(String[] args) { // Cuebot startup - SpringApplication.run(CuebotApplication.class, args); + String[] filteredArgs = checkArgs(args); + SpringApplication.run(CuebotApplication.class, filteredArgs); } } From 529d606c67012a6630ac34838eb626fe03b69871 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 6 Dec 2022 12:36:37 -0500 Subject: [PATCH 227/277] [cuebot] Remove some stray debugging log messages. (#1209) --- .../com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index 7a1a85351..9ffe56beb 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -548,7 +548,6 @@ public ProcSeq getProcs(HostInterface host) { r.filterByHost(host); r.sortByHostName(); r.sortByDispatchedTime(); - logger.info("!!!! INSIDE getProcs Whiteboard!!! called getProcs !!! line 551"); return ProcSeq.newBuilder().addAllProcs(getProcs(r).getProcsList()).build(); } @@ -556,7 +555,6 @@ public ProcSeq getProcs(HostInterface host) { public ProcSeq getProcs(ProcSearchInterface p) { p.sortByHostName(); p.sortByDispatchedTime(); - logger.info("!!!! Inside getPROCS!!!!! line 559"); List procs = getJdbcTemplate().query(p.getFilteredQuery(GET_PROC), PROC_MAPPER, p.getValuesArray()); return ProcSeq.newBuilder().addAllProcs(procs).build(); @@ -976,7 +974,6 @@ public Proc mapRow(ResultSet rs, int row) throws SQLException { .addAllServices(Arrays.asList(SqlUtil.getString(rs,"str_services").split(","))) .build(); } -// logger.info("called ROW MAPPER!!! setChildProcesses!!!"); }; public static final RowMapper TASK_MAPPER = From 91f2244ff68c5a0682aa2520831275004195182d Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 12 Dec 2022 14:30:07 -0500 Subject: [PATCH 228/277] Upgrade Log4j to 2.16. (#1080) --- cuebot/build.gradle | 5 +- .../spring/remoting/CueServerInterceptor.java | 7 +- .../common/spring/remoting/GrpcServer.java | 5 +- .../imageworks/spcue/CuebotApplication.java | 5 +- .../com/imageworks/spcue/SortableShow.java | 5 +- .../dao/criteria/postgres/FrameSearch.java | 5 +- .../spcue/dao/postgres/DispatcherDaoJdbc.java | 5 +- .../spcue/dao/postgres/ProcDaoJdbc.java | 2 - .../spcue/dao/postgres/WhiteboardDaoJdbc.java | 5 +- .../spcue/dispatcher/AbstractDispatcher.java | 5 +- .../spcue/dispatcher/BookingQueue.java | 5 +- .../spcue/dispatcher/CoreUnitDispatcher.java | 5 +- .../spcue/dispatcher/DispatchQueue.java | 5 +- .../dispatcher/DispatchSupportService.java | 5 +- .../dispatcher/FrameCompleteHandler.java | 6 +- .../spcue/dispatcher/HealthyThreadPool.java | 5 +- .../spcue/dispatcher/HostReportHandler.java | 5 +- .../spcue/dispatcher/HostReportQueue.java | 5 +- .../spcue/dispatcher/LocalDispatcher.java | 5 +- .../spcue/dispatcher/RedirectManager.java | 5 +- .../ThreadPoolTaskExecutorWrapper.java | 5 +- .../dispatcher/commands/DispatchBookHost.java | 5 +- .../commands/DispatchRqdKillFrame.java | 5 +- .../imageworks/spcue/rqd/RqdClientGrpc.java | 5 +- .../spcue/servant/ManageDepend.java | 5 +- .../imageworks/spcue/servant/ManageJob.java | 5 +- .../spcue/service/AdminManagerService.java | 5 +- .../spcue/service/BookingManagerService.java | 5 +- .../spcue/service/DependManagerService.java | 5 +- .../spcue/service/EmailSupport.java | 5 +- .../spcue/service/FilterManagerService.java | 5 +- .../spcue/service/HistoricalSupport.java | 5 +- .../spcue/service/HostManagerService.java | 5 +- .../imageworks/spcue/service/JmsMover.java | 5 +- .../imageworks/spcue/service/JobLauncher.java | 5 +- .../spcue/service/JobManagerService.java | 5 +- .../spcue/service/JobManagerSupport.java | 5 +- .../com/imageworks/spcue/service/JobSpec.java | 5 +- .../spcue/service/LocalBookingSupport.java | 5 +- .../service/MaintenanceManagerSupport.java | 5 +- .../spcue/service/RedirectService.java | 5 +- .../spcue/service/WhiteboardService.java | 5 +- .../spcue/servlet/JobLaunchServlet.java | 5 +- .../spcue/util/CueExceptionUtil.java | 5 +- .../com/imageworks/spcue/util/CueUtil.java | 5 +- cuebot/src/main/resources/log4j.properties | 51 --------- cuebot/src/main/resources/log4j2.properties | 103 ++++++++++++++++++ 47 files changed, 236 insertions(+), 143 deletions(-) delete mode 100644 cuebot/src/main/resources/log4j.properties create mode 100644 cuebot/src/main/resources/log4j2.properties diff --git a/cuebot/build.gradle b/cuebot/build.gradle index fe5abf3e2..a9b7cfaa5 100644 --- a/cuebot/build.gradle +++ b/cuebot/build.gradle @@ -55,8 +55,9 @@ dependencies { compile group: 'org.quartz-scheduler', name: 'quartz', version: '2.2.1', { exclude group: 'c3p0', module: 'c3p0' } compile group: 'org.postgresql', name: 'postgresql', version: '42.2.2' compile group: 'com.google.protobuf', name: 'protobuf-java', version: '3.21.2' - compile group: 'log4j', name: 'log4j', version: '1.2.17' - compile group: 'org.slf4j', name: 'slf4j-log4j12', version: '1.7.26' + compile group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.16.0' + compile group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.16.0' + compile group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: '2.16.0' protobuf fileTree("../proto/") diff --git a/cuebot/src/main/java/com/imageworks/common/spring/remoting/CueServerInterceptor.java b/cuebot/src/main/java/com/imageworks/common/spring/remoting/CueServerInterceptor.java index 69c448b11..31ebeb12e 100644 --- a/cuebot/src/main/java/com/imageworks/common/spring/remoting/CueServerInterceptor.java +++ b/cuebot/src/main/java/com/imageworks/common/spring/remoting/CueServerInterceptor.java @@ -7,13 +7,14 @@ import io.grpc.ServerCallHandler; import io.grpc.ServerInterceptor; import io.grpc.Status; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; public class CueServerInterceptor implements ServerInterceptor { - private static final Logger logger = Logger.getLogger(CueServerInterceptor.class); - private static final Logger accessLogger = Logger.getLogger("API"); + private static final Logger logger = LogManager.getLogger(CueServerInterceptor.class); + private static final Logger accessLogger = LogManager.getLogger("API"); @Override public ServerCall.Listener interceptCall( diff --git a/cuebot/src/main/java/com/imageworks/common/spring/remoting/GrpcServer.java b/cuebot/src/main/java/com/imageworks/common/spring/remoting/GrpcServer.java index 37c1c408d..a5038f82c 100644 --- a/cuebot/src/main/java/com/imageworks/common/spring/remoting/GrpcServer.java +++ b/cuebot/src/main/java/com/imageworks/common/spring/remoting/GrpcServer.java @@ -6,7 +6,8 @@ import io.grpc.Server; import io.grpc.ServerBuilder; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.beans.BeansException; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; @@ -40,7 +41,7 @@ public class GrpcServer implements ApplicationContextAware { - private static final Logger logger = Logger.getLogger(GrpcServer.class); + private static final Logger logger = LogManager.getLogger(GrpcServer.class); private static final String DEFAULT_NAME = "CueGrpcServer"; private static final String DEFAULT_PORT = "8443"; diff --git a/cuebot/src/main/java/com/imageworks/spcue/CuebotApplication.java b/cuebot/src/main/java/com/imageworks/spcue/CuebotApplication.java index 12087f93e..6ef64080c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/CuebotApplication.java +++ b/cuebot/src/main/java/com/imageworks/spcue/CuebotApplication.java @@ -23,7 +23,8 @@ import java.util.stream.Stream; import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -34,7 +35,7 @@ private static String[] checkArgs(String[] args) { .filter(arg -> arg.startsWith("--log.frame-log-root=")).findFirst(); if (deprecatedFlag.isPresent()) { // Log a deprecation warning. - Logger warning_logger = Logger.getLogger(CuebotApplication.class); + Logger warning_logger = LogManager.getLogger(CuebotApplication.class); warning_logger.warn("`--log.frame-log-root` is deprecated and will be removed in an " + "upcoming release. It has been replaced with `--log.frame-log-root.default_os`. " + "See opencue.properties for details on OpenCue's new OS-dependent root directories."); diff --git a/cuebot/src/main/java/com/imageworks/spcue/SortableShow.java b/cuebot/src/main/java/com/imageworks/spcue/SortableShow.java index 83fe7da5d..f13fbaae2 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/SortableShow.java +++ b/cuebot/src/main/java/com/imageworks/spcue/SortableShow.java @@ -24,11 +24,12 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; public class SortableShow implements Comparable { - private static final Logger logger = Logger.getLogger(SortableShow.class); + private static final Logger logger = LogManager.getLogger(SortableShow.class); private String show; private float tier; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/postgres/FrameSearch.java b/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/postgres/FrameSearch.java index ee3a1f841..de33e29cd 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/postgres/FrameSearch.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/criteria/postgres/FrameSearch.java @@ -23,7 +23,8 @@ import java.util.stream.Collectors; import com.google.common.collect.ImmutableList; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import com.imageworks.spcue.FrameInterface; import com.imageworks.spcue.JobInterface; @@ -36,7 +37,7 @@ public class FrameSearch extends Criteria implements FrameSearchInterface { private static final int MAX_RESULTS = 1000; - private static final Logger logger = Logger.getLogger(FrameSearch.class); + private static final Logger logger = LogManager.getLogger(FrameSearch.class); private static final Pattern PATTERN_SINGLE_FRAME = Pattern.compile("^([0-9]+)$"); private static final Pattern PATTERN_RANGE = Pattern.compile("^([0-9]+)\\-([0-9]+)$"); private static final Pattern PATTERN_FLOAT_RANGE = Pattern.compile("^([0-9\\.]+)\\-([0-9\\.]+)$"); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java index 4c9570d9a..032c90dac 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java @@ -28,7 +28,8 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.jdbc.core.RowMapper; @@ -58,7 +59,7 @@ */ public class DispatcherDaoJdbc extends JdbcDaoSupport implements DispatcherDao { - private static final Logger logger = Logger.getLogger(DispatcherDaoJdbc.class); + private static final Logger logger = LogManager.getLogger(DispatcherDaoJdbc.class); public static final RowMapper PKJOB_MAPPER = new RowMapper() { diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java index 32c454e0f..b68f8a74c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java @@ -29,8 +29,6 @@ import java.util.List; import java.util.Map; -import com.imageworks.spcue.dispatcher.AbstractDispatcher; -import org.apache.log4j.Logger; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.RowMapper; import org.springframework.jdbc.core.support.JdbcDaoSupport; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index 9ffe56beb..38566470a 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -29,7 +29,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.protobuf.ByteString; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.jdbc.core.RowMapper; import org.springframework.jdbc.core.support.JdbcDaoSupport; @@ -129,7 +130,7 @@ public class WhiteboardDaoJdbc extends JdbcDaoSupport implements WhiteboardDao { @SuppressWarnings("unused") - private static final Logger logger = Logger.getLogger(WhiteboardDaoJdbc.class); + private static final Logger logger = LogManager.getLogger(WhiteboardDaoJdbc.class); private FrameSearchFactory frameSearchFactory; private ProcSearchFactory procSearchFactory; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/AbstractDispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/AbstractDispatcher.java index 73f5aef73..355c64175 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/AbstractDispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/AbstractDispatcher.java @@ -19,7 +19,8 @@ package com.imageworks.spcue.dispatcher; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import com.imageworks.spcue.DispatchFrame; import com.imageworks.spcue.VirtualProc; @@ -33,7 +34,7 @@ */ public abstract class AbstractDispatcher { - private static final Logger logger = Logger.getLogger(AbstractDispatcher.class); + private static final Logger logger = LogManager.getLogger(AbstractDispatcher.class); public DispatchSupport dispatchSupport; public RqdClient rqdClient; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java index 347acd025..69a961977 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/BookingQueue.java @@ -23,7 +23,8 @@ import com.google.common.cache.CacheBuilder; import com.imageworks.spcue.dispatcher.commands.KeyRunnable; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; @@ -36,7 +37,7 @@ public class BookingQueue { private final int maxPoolSize; private static final int BASE_SLEEP_TIME_MILLIS = 300; - private static final Logger logger = Logger.getLogger("HEALTH"); + private static final Logger logger = LogManager.getLogger("HEALTH"); private HealthyThreadPool healthyThreadPool; public BookingQueue(int healthThreshold, int minUnhealthyPeriodMin, int queueCapacity, diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java index 4d556589e..a9a8b918a 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java @@ -25,7 +25,8 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; @@ -85,7 +86,7 @@ */ public class CoreUnitDispatcher implements Dispatcher { private static final Logger logger = - Logger.getLogger(CoreUnitDispatcher.class); + LogManager.getLogger(CoreUnitDispatcher.class); private DispatchSupport dispatchSupport; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchQueue.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchQueue.java index 3798bddaa..00e552a05 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchQueue.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchQueue.java @@ -22,7 +22,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import com.imageworks.spcue.dispatcher.commands.KeyRunnable; public class DispatchQueue { @@ -33,7 +34,7 @@ public class DispatchQueue { private int corePoolSize; private int maxPoolSize; - private static final Logger logger = Logger.getLogger("HEALTH"); + private static final Logger logger = LogManager.getLogger("HEALTH"); private String name = "Default"; private HealthyThreadPool healthyDispatchPool; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java index dd8f71cf0..c935024d6 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java @@ -39,7 +39,8 @@ import com.imageworks.spcue.ShowInterface; import com.imageworks.spcue.StrandedCores; import com.imageworks.spcue.VirtualProc; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.dao.EmptyResultDataAccessException; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -64,7 +65,7 @@ @Transactional(propagation = Propagation.REQUIRED) public class DispatchSupportService implements DispatchSupport { - private static final Logger logger = Logger.getLogger(DispatchSupportService.class); + private static final Logger logger = LogManager.getLogger(DispatchSupportService.class); private JobDao jobDao; private FrameDao frameDao; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java index d4e619894..55336aaf4 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java @@ -24,8 +24,8 @@ import java.util.Random; import java.util.concurrent.atomic.AtomicLong; -import com.imageworks.spcue.*; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.dao.EmptyResultDataAccessException; @@ -66,7 +66,7 @@ */ public class FrameCompleteHandler { - private static final Logger logger = Logger.getLogger(FrameCompleteHandler.class); + private static final Logger logger = LogManager.getLogger(FrameCompleteHandler.class); private static final Random randomNumber = new Random(); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HealthyThreadPool.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HealthyThreadPool.java index 13c96e776..5c2efabc4 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HealthyThreadPool.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HealthyThreadPool.java @@ -13,7 +13,8 @@ import com.google.common.cache.CacheBuilder; import com.imageworks.spcue.dispatcher.commands.DispatchBookHost; import com.imageworks.spcue.dispatcher.commands.KeyRunnable; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; /*** @@ -24,7 +25,7 @@ */ public class HealthyThreadPool extends ThreadPoolExecutor { // The service need s to be unhealthy for this period of time to report - private static final Logger logger = Logger.getLogger("HEALTH"); + private static final Logger logger = LogManager.getLogger("HEALTH"); // Threshold to consider healthy or unhealthy private final int healthThreshold; private final int poolSize; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java index a63da07e8..d763cce53 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java @@ -27,7 +27,8 @@ import java.util.Map; import java.util.concurrent.ThreadPoolExecutor; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.core.task.TaskRejectedException; import org.springframework.dao.DataAccessException; import org.springframework.dao.EmptyResultDataAccessException; @@ -64,7 +65,7 @@ public class HostReportHandler { - private static final Logger logger = Logger.getLogger(HostReportHandler.class); + private static final Logger logger = LogManager.getLogger(HostReportHandler.class); private BookingManager bookingManager; private HostManager hostManager; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportQueue.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportQueue.java index 59b91abcc..6326086ae 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportQueue.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportQueue.java @@ -26,16 +26,17 @@ import java.util.concurrent.atomic.AtomicBoolean; import com.imageworks.spcue.grpc.report.HostReport; -import org.apache.log4j.Logger; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import com.imageworks.spcue.dispatcher.commands.DispatchHandleHostReport; import com.imageworks.spcue.util.CueUtil; public class HostReportQueue extends ThreadPoolExecutor { - private static final Logger logger = Logger.getLogger(HostReportQueue.class); + private static final Logger logger = LogManager.getLogger(HostReportQueue.class); private QueueRejectCounter rejectCounter = new QueueRejectCounter(); private AtomicBoolean isShutdown = new AtomicBoolean(false); private int queueCapacity; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LocalDispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LocalDispatcher.java index 23bf6f73a..288965a04 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LocalDispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LocalDispatcher.java @@ -22,7 +22,8 @@ import java.util.ArrayList; import java.util.List; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.dao.EmptyResultDataAccessException; import com.imageworks.spcue.DispatchFrame; @@ -42,7 +43,7 @@ public class LocalDispatcher extends AbstractDispatcher implements Dispatcher { private static final Logger logger = - Logger.getLogger(LocalDispatcher.class); + LogManager.getLogger(LocalDispatcher.class); private BookingManager bookingManager; private JobManager jobManager; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/RedirectManager.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/RedirectManager.java index 24b1681e9..e665345cd 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/RedirectManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/RedirectManager.java @@ -21,7 +21,8 @@ import java.util.ArrayList; import java.util.List; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import com.imageworks.spcue.DispatchHost; import com.imageworks.spcue.GroupInterface; @@ -46,7 +47,7 @@ public class RedirectManager { - private static final Logger logger = Logger.getLogger(RedirectManager.class); + private static final Logger logger = LogManager.getLogger(RedirectManager.class); private JobDao jobDao; private ProcDao procDao; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/ThreadPoolTaskExecutorWrapper.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/ThreadPoolTaskExecutorWrapper.java index 8b749cdb1..0090d3619 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/ThreadPoolTaskExecutorWrapper.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/ThreadPoolTaskExecutorWrapper.java @@ -19,7 +19,8 @@ package com.imageworks.spcue.dispatcher; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; @@ -31,7 +32,7 @@ */ public class ThreadPoolTaskExecutorWrapper extends ThreadPoolTaskExecutor { - private static final Logger logger = Logger.getLogger(ThreadPoolTaskExecutorWrapper.class); + private static final Logger logger = LogManager.getLogger(ThreadPoolTaskExecutorWrapper.class); private static final long serialVersionUID = -2977068663355369141L; private int queueCapacity; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java index 0b65257dd..64329fc0f 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHost.java @@ -21,7 +21,8 @@ import java.util.List; import java.util.ArrayList; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import com.imageworks.spcue.DispatchHost; import com.imageworks.spcue.GroupInterface; @@ -37,7 +38,7 @@ */ public class DispatchBookHost extends KeyRunnable { private static final Logger logger = - Logger.getLogger(DispatchBookHost.class); + LogManager.getLogger(DispatchBookHost.class); private ShowInterface show = null; private GroupInterface group = null; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrame.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrame.java index 3a43c6fc3..61258a824 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrame.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrame.java @@ -19,7 +19,8 @@ package com.imageworks.spcue.dispatcher.commands; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import com.imageworks.spcue.dispatcher.commands.KeyRunnable; import com.imageworks.spcue.VirtualProc; @@ -28,7 +29,7 @@ public class DispatchRqdKillFrame extends KeyRunnable { - private static final Logger logger = Logger.getLogger(DispatchRqdKillFrame.class); + private static final Logger logger = LogManager.getLogger(DispatchRqdKillFrame.class); private VirtualProc proc = null; private String message; diff --git a/cuebot/src/main/java/com/imageworks/spcue/rqd/RqdClientGrpc.java b/cuebot/src/main/java/com/imageworks/spcue/rqd/RqdClientGrpc.java index 5e1e016fa..40554904b 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/rqd/RqdClientGrpc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/rqd/RqdClientGrpc.java @@ -23,7 +23,8 @@ import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.grpc.StatusRuntimeException; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -49,7 +50,7 @@ import com.imageworks.spcue.grpc.rqd.RunningFrameStatusResponse; public final class RqdClientGrpc implements RqdClient { - private static final Logger logger = Logger.getLogger(RqdClientGrpc.class); + private static final Logger logger = LogManager.getLogger(RqdClientGrpc.class); private final int rqdCacheSize; private final int rqdCacheExpiration; diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageDepend.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageDepend.java index 5d1c2ab6e..9fad37ef1 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageDepend.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageDepend.java @@ -21,7 +21,8 @@ import io.grpc.Status; import io.grpc.stub.StreamObserver; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.dao.EmptyResultDataAccessException; import com.imageworks.spcue.LightweightDependency; @@ -39,7 +40,7 @@ public class ManageDepend extends DependInterfaceGrpc.DependInterfaceImplBase { - private static final Logger logger = Logger.getLogger(ManageDepend.class); + private static final Logger logger = LogManager.getLogger(ManageDepend.class); private DependManager dependManager; private DispatchQueue manageQueue; diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java index 01601998a..e3cfa5178 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java @@ -24,7 +24,8 @@ import io.grpc.Status; import io.grpc.stub.StreamObserver; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.dao.EmptyResultDataAccessException; @@ -161,7 +162,7 @@ import static com.imageworks.spcue.servant.ServantUtil.attemptChange; public class ManageJob extends JobInterfaceGrpc.JobInterfaceImplBase { - private static final Logger logger = Logger.getLogger(ManageJob.class); + private static final Logger logger = LogManager.getLogger(ManageJob.class); private Whiteboard whiteboard; private JobManager jobManager; private GroupManager groupManager; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java index a824dfb09..8f1f66133 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java @@ -19,7 +19,8 @@ package com.imageworks.spcue.service; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -46,7 +47,7 @@ public class AdminManagerService implements AdminManager { @SuppressWarnings("unused") - private static final Logger logger = Logger.getLogger(AdminManagerService.class); + private static final Logger logger = LogManager.getLogger(AdminManagerService.class); private ShowDao showDao; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/BookingManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/BookingManagerService.java index 1a2b6cee2..1322b622d 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/BookingManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/BookingManagerService.java @@ -21,7 +21,8 @@ import java.util.List; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.dao.EmptyResultDataAccessException; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -49,7 +50,7 @@ public class BookingManagerService implements BookingManager { @SuppressWarnings("unused") private static final Logger logger = - Logger.getLogger(BookingManagerService.class); + LogManager.getLogger(BookingManagerService.class); private BookingQueue bookingQueue; private BookingDao bookingDao; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/DependManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/DependManagerService.java index 187e81331..2a82c099d 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/DependManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/DependManagerService.java @@ -23,7 +23,8 @@ import java.util.List; import java.util.Set; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.dao.DataIntegrityViolationException; import org.springframework.dao.DataRetrievalFailureException; import org.springframework.transaction.annotation.Propagation; @@ -63,7 +64,7 @@ @Transactional public class DependManagerService implements DependManager { - private static final Logger logger = Logger.getLogger(DependManagerService.class); + private static final Logger logger = LogManager.getLogger(DependManagerService.class); private DependDao dependDao; private JobDao jobDao; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/EmailSupport.java b/cuebot/src/main/java/com/imageworks/spcue/service/EmailSupport.java index ef9c2d32a..b25e7d520 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/EmailSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/EmailSupport.java @@ -33,7 +33,8 @@ import java.util.Map; import java.util.Properties; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.apache.velocity.Template; import org.apache.velocity.VelocityContext; import org.apache.velocity.app.VelocityEngine; @@ -65,7 +66,7 @@ public class EmailSupport { private final Map imageMap; - private static final Logger logger = Logger.getLogger(EmailSupport.class); + private static final Logger logger = LogManager.getLogger(EmailSupport.class); public EmailSupport() { diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/FilterManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/FilterManagerService.java index 6d4382714..eb49a9c51 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/FilterManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/FilterManagerService.java @@ -23,7 +23,8 @@ import java.util.List; import java.util.regex.Pattern; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -62,7 +63,7 @@ @Transactional public class FilterManagerService implements FilterManager { - private static final Logger logger = Logger.getLogger(FilterManagerService.class); + private static final Logger logger = LogManager.getLogger(FilterManagerService.class); private ActionDao actionDao; private MatcherDao matcherDao; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/HistoricalSupport.java b/cuebot/src/main/java/com/imageworks/spcue/service/HistoricalSupport.java index 0eb6d1213..74c256729 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/HistoricalSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/HistoricalSupport.java @@ -21,12 +21,13 @@ import java.util.List; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import com.imageworks.spcue.JobInterface; public class HistoricalSupport { - private static final Logger logger = Logger.getLogger(HistoricalSupport.class); + private static final Logger logger = LogManager.getLogger(HistoricalSupport.class); private HistoricalManager historicalManager; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java index ccd355889..a7c5b0729 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java @@ -22,7 +22,8 @@ import java.sql.Timestamp; import java.util.List; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.dao.EmptyResultDataAccessException; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -57,7 +58,7 @@ @Transactional public class HostManagerService implements HostManager { - private static final Logger logger = Logger.getLogger(HostManagerService.class); + private static final Logger logger = LogManager.getLogger(HostManagerService.class); private HostDao hostDao; private RqdClient rqdClient; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JmsMover.java b/cuebot/src/main/java/com/imageworks/spcue/service/JmsMover.java index bc5be1779..ce231331b 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JmsMover.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JmsMover.java @@ -29,7 +29,8 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.jms.JmsException; @@ -39,7 +40,7 @@ import com.imageworks.spcue.util.CueExceptionUtil; public class JmsMover extends ThreadPoolExecutor { - private static final Logger logger = Logger.getLogger(JmsMover.class); + private static final Logger logger = LogManager.getLogger(JmsMover.class); private final Gson gson = new GsonBuilder().serializeNulls().create(); @Autowired diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobLauncher.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobLauncher.java index 14f2a5741..f46616115 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobLauncher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobLauncher.java @@ -23,7 +23,8 @@ import java.util.HashSet; import java.util.Set; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.beans.BeansException; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; @@ -43,7 +44,7 @@ * Job launching functions. */ public class JobLauncher implements ApplicationContextAware { - private static final Logger logger = Logger.getLogger(JobLauncher.class); + private static final Logger logger = LogManager.getLogger(JobLauncher.class); private ApplicationContext context; private JobManager jobManager; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java index 6da593712..c1ca1bdfc 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java @@ -22,7 +22,8 @@ import java.util.List; import com.google.common.collect.Sets; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.dao.DataAccessException; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -70,7 +71,7 @@ @Transactional public class JobManagerService implements JobManager { - private static final Logger logger = Logger.getLogger(JobManagerService.class); + private static final Logger logger = LogManager.getLogger(JobManagerService.class); private JobDao jobDao; private ShowDao showDao; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerSupport.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerSupport.java index c73c30eb6..b2db74d59 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerSupport.java @@ -22,7 +22,8 @@ import java.util.Collection; import java.util.List; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.dao.DataAccessException; import org.springframework.dao.EmptyResultDataAccessException; @@ -55,7 +56,7 @@ * A non-transaction support class for managing jobs. */ public class JobManagerSupport { - private static final Logger logger = Logger.getLogger(JobManagerSupport.class); + private static final Logger logger = LogManager.getLogger(JobManagerSupport.class); private JobManager jobManager; private DependManager dependManager; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java index 293ce9730..269a9f4af 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java @@ -33,7 +33,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.jdom.Document; import org.jdom.Element; import org.jdom.input.SAXBuilder; @@ -57,7 +58,7 @@ import com.imageworks.spcue.util.CueUtil; public class JobSpec { - private static final Logger logger = Logger.getLogger(JobSpec.class); + private static final Logger logger = LogManager.getLogger(JobSpec.class); private String facility; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/LocalBookingSupport.java b/cuebot/src/main/java/com/imageworks/spcue/service/LocalBookingSupport.java index a242382bb..7bf7db136 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/LocalBookingSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/LocalBookingSupport.java @@ -19,7 +19,8 @@ package com.imageworks.spcue.service; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import com.imageworks.spcue.DispatchHost; import com.imageworks.spcue.FrameInterface; @@ -37,7 +38,7 @@ */ public class LocalBookingSupport { - private static final Logger logger = Logger.getLogger(LocalBookingSupport.class); + private static final Logger logger = LogManager.getLogger(LocalBookingSupport.class); private HostManager hostManager; private LocalDispatcher localDispatcher; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/MaintenanceManagerSupport.java b/cuebot/src/main/java/com/imageworks/spcue/service/MaintenanceManagerSupport.java index fdc44eddd..84049bd61 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/MaintenanceManagerSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/MaintenanceManagerSupport.java @@ -21,7 +21,8 @@ import java.util.List; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.jdbc.CannotGetJdbcConnectionException; @@ -42,7 +43,7 @@ public class MaintenanceManagerSupport { - private static final Logger logger = Logger.getLogger(MaintenanceManagerSupport.class); + private static final Logger logger = LogManager.getLogger(MaintenanceManagerSupport.class); @Autowired private Environment env; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/RedirectService.java b/cuebot/src/main/java/com/imageworks/spcue/service/RedirectService.java index 23f164c53..51bf4211d 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/RedirectService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/RedirectService.java @@ -20,7 +20,8 @@ import javax.annotation.Resource; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.dao.CannotSerializeTransactionException; import org.springframework.dao.DuplicateKeyException; import org.springframework.transaction.PlatformTransactionManager; @@ -37,7 +38,7 @@ public class RedirectService { private static final Logger logger = - Logger.getLogger(RedirectService.class); + LogManager.getLogger(RedirectService.class); @Resource private PlatformTransactionManager txManager; diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/WhiteboardService.java b/cuebot/src/main/java/com/imageworks/spcue/service/WhiteboardService.java index 694a58643..5fe08e1f6 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/WhiteboardService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/WhiteboardService.java @@ -21,7 +21,8 @@ import java.util.List; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -111,7 +112,7 @@ public class WhiteboardService implements Whiteboard { @SuppressWarnings("unused") - private static final Logger logger = Logger.getLogger(WhiteboardService.class); + private static final Logger logger = LogManager.getLogger(WhiteboardService.class); private WhiteboardDao whiteboardDao; diff --git a/cuebot/src/main/java/com/imageworks/spcue/servlet/JobLaunchServlet.java b/cuebot/src/main/java/com/imageworks/spcue/servlet/JobLaunchServlet.java index 284b6739a..76040b7bd 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servlet/JobLaunchServlet.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servlet/JobLaunchServlet.java @@ -25,7 +25,8 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.web.servlet.FrameworkServlet; import com.imageworks.spcue.BuildableJob; @@ -39,7 +40,7 @@ @SuppressWarnings("serial") public class JobLaunchServlet extends FrameworkServlet { - private static final Logger logger = Logger.getLogger(JobLaunchServlet.class); + private static final Logger logger = LogManager.getLogger(JobLaunchServlet.class); private JobLauncher jobLauncher; diff --git a/cuebot/src/main/java/com/imageworks/spcue/util/CueExceptionUtil.java b/cuebot/src/main/java/com/imageworks/spcue/util/CueExceptionUtil.java index e947815a2..3879914b1 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/util/CueExceptionUtil.java +++ b/cuebot/src/main/java/com/imageworks/spcue/util/CueExceptionUtil.java @@ -23,7 +23,8 @@ import java.io.StringWriter; import java.io.Writer; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; /** * Utility class for handling and logging exceptions @@ -52,7 +53,7 @@ public static String getStackTrace(Throwable aThrowable) { * @return String */ public static void logStackTrace(String msg, Throwable aThrowable) { - Logger error_logger = Logger.getLogger(CueExceptionUtil.class); + Logger error_logger = LogManager.getLogger(CueExceptionUtil.class); error_logger.info("Caught unexpected exception caused by: " + aThrowable); error_logger.info("StackTrace: \n" + getStackTrace(aThrowable)); if (aThrowable.getCause() != null) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java b/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java index 2a7438f49..88b325483 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java +++ b/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java @@ -46,7 +46,8 @@ import javax.mail.internet.MimeMultipart; import javax.mail.util.ByteArrayDataSource; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.core.env.Environment; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -62,7 +63,7 @@ @Component public final class CueUtil { - private static final Logger logger = Logger.getLogger(CueUtil.class); + private static final Logger logger = LogManager.getLogger(CueUtil.class); private static String smtpHost = ""; @Autowired private Environment env; diff --git a/cuebot/src/main/resources/log4j.properties b/cuebot/src/main/resources/log4j.properties deleted file mode 100644 index 5f44dbee9..000000000 --- a/cuebot/src/main/resources/log4j.properties +++ /dev/null @@ -1,51 +0,0 @@ -############################################################## -# SpCue Logging Configuration -############################################################## - -############################################################### -# Root Logger -# Logs Application wide INFO messages / Tomcat messges -############################################################### - -log4j.rootLogger=INFO, STDOUT, FILE -log4j.appender.STDOUT=org.apache.log4j.ConsoleAppender -log4j.appender.STDOUT.Threshold=WARN -log4j.appender.STDOUT.layout=org.apache.log4j.PatternLayout -log4j.appender.STDOUT.layout.ConversionPattern=%d %p %t %c - %m%n -log4j.appender.FILE=org.apache.log4j.RollingFileAppender -log4j.appender.FILE.File=logs/spcue.log -log4j.appender.FILE.MaxFileSize=10MB -log4j.appender.FILE.MaxBackupIndex=10 -log4j.appender.FILE.layout=org.apache.log4j.PatternLayout -log4j.appender.FILE.layout.ConversionPattern=%d %p %t %c - %m%n - -log4j.category.API=INFO, API -log4j.additivity.API=false -log4j.appender.API=org.apache.log4j.RollingFileAppender -log4j.appender.API.File=logs/api.log -log4j.appender.API.MaxFileSize=10MB -log4j.appender.API.MaxBackupIndex=20 -log4j.appender.API.layout=org.apache.log4j.PatternLayout -log4j.appender.API.layout.ConversionPattern=%d:%m%n - -log4j.category.HEALTH=DEBUG, HEALTH -log4j.additivity.HEALTH=false -log4j.appender.HEALTH=org.apache.log4j.RollingFileAppender -log4j.appender.HEALTH.File=logs/health.log -log4j.appender.HEALTH.MaxFileSize=10MB -log4j.appender.HEALTH.MaxBackupIndex=20 -log4j.appender.HEALTH.layout=org.apache.log4j.PatternLayout -log4j.appender.HEALTH.layout.ConversionPattern=%d:%m%n - -log4j.logger.org.apache.catalina=INFO -log4j.logger.com.imageworks.spcue=DEBUG -log4j.logger.com.imageworks.spcue.dispatcher.RqdReportManagerService=DEBUG -log4j.logger.com.imageworks.spcue.service.HostManagerService=TRACE -log4j.logger.com.imageworks.spcue.dispatcher=TRACE - -#log4j.logger.org.springframework=DEBUG - -# Very verbose sql output: -#log4j.logger.org.springframework.jdbc.core=DEBUG -#log4j.logger.org.springframework.jdbc.core.JdbcTemplate=DEBUG -#log4j.logger.org.springframework.jdbc.core.StatementCreatorUtils=TRACE diff --git a/cuebot/src/main/resources/log4j2.properties b/cuebot/src/main/resources/log4j2.properties new file mode 100644 index 000000000..b924a762f --- /dev/null +++ b/cuebot/src/main/resources/log4j2.properties @@ -0,0 +1,103 @@ +############################################################## +# OpenCue Logging Configuration +############################################################## + +# Log4j uses "appenders" and "loggers". Loggers define the logging behavior within the +# application. Appenders deliver the log messages to the intended targets. Loggers must +# be associated with appenders in order for log messages to be written out. + +# Stdout. +appender.console.type = Console +appender.console.name = STDOUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d %p %t %c - %m%n +appender.console.filter.threshold.type = ThresholdFilter +appender.console.filter.threshold.level = warn + +# Main log file. +appender.rolling.type = RollingFile +appender.rolling.name = FILE +appender.rolling.fileName = logs/spcue.log +appender.rolling.filePattern = logs/spcue.log.%i +appender.rolling.layout.type = PatternLayout +appender.rolling.layout.pattern = %d %p %t %c - %m%n +appender.rolling.policies.type = Policies +appender.rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling.policies.size.size=10MB +appender.rolling.strategy.type = DefaultRolloverStrategy +appender.rolling.strategy.max = 10 + +# API log file, for logging API requests only. +appender.api.type = RollingFile +appender.api.name = API +appender.api.fileName = logs/api.log +appender.api.filePattern = logs/api.log.%i +appender.api.layout.type = PatternLayout +appender.api.layout.pattern = %d:%m%n +appender.api.policies.type = Policies +appender.api.policies.size.type = SizeBasedTriggeringPolicy +appender.api.policies.size.size = 10MB +appender.api.strategy.type = DefaultRolloverStrategy +appender.api.strategy.max = 20 +appender.api.filter.threshold.type = ThresholdFilter +appender.api.filter.threshold.level = info + +# HEALTH log file +appender.health.type = RollingFile +appender.health.name = HEALTH +appender.health.fileName = logs/health.log +appender.health.filePattern = logs/health.log.%i +appender.health.layout.type = PatternLayout +appender.health.layout.pattern = %d:%m%n +appender.health.policies.type = Policies +appender.health.policies.size.type = SizeBasedTriggeringPolicy +appender.health.policies.size.size = 10MB +appender.health.strategy.type = DefaultRolloverStrategy +appender.health.strategy.max = 20 +appender.health.filter.threshold.type = ThresholdFilter +appender.health.filter.threshold.level = debug + +# Root-level logger. All messages will go to both stdout and the main log file, though they +# may not appear there if the appender filters based on log level. For example INFO messages +# will not appear by default in stdout as the default log level for that appender is WARN. +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = FILE + +# API logger. Does not inherit from the root logger, so only API requests will be logged. +logger.api.name = API +logger.api.level = info +logger.api.additivity = false +logger.api.appenderRef.api.ref = API + +# HEALTH logger. Does not inherit from the root logger, so only HEALTH requests will be logged. +logger.health.name = HEALTH +logger.health.level = debug +logger.health.additivity = false +logger.health.appenderRef.health.ref = HEALTH + +# Child loggers. These inherit from the root logger, so will be sent to the relevant appenders. +# This allows us to increase verbosity for specific modules. + +logger.catalina.name = org.apache.catalina +logger.catalina.level = info + +logger.spcue.name = com.imageworks.spcue +logger.spcue.level = debug + +logger.rqdReport.name = com.imageworks.spcue.dispatcher.RqdReportManagerService +logger.rqdReport.level = debug + +logger.hostManager.name = com.imageworks.spcue.service.HostManagerService +logger.hostManager.level = trace + +logger.dispatcher.name = com.imageworks.spcue.dispatcher +logger.dispatcher.level = trace + +# For very verbose sql output: +# logger.sql.name = org.springframework.jdbc.core +# logger.sql.level = debug +# logger.sqlJdbcTemplate.name = org.springframework.jdbc.core.JdbcTemplate +# logger.sqlJdbcTemplate.level = debug +# logger.sqlStatementCreator.name = org.springframework.jdbc.core.StatementCreatorUtils +# logger.sqlStatementCreator.level = trace From 939811a87ac83b8f6985af07216a38219375c370 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Fri, 16 Dec 2022 15:04:00 -0500 Subject: [PATCH 229/277] Notes from Dec 7 TSC meeting. (#1231) --- tsc/meetings/2022-12-07.md | 51 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 tsc/meetings/2022-12-07.md diff --git a/tsc/meetings/2022-12-07.md b/tsc/meetings/2022-12-07.md new file mode 100644 index 000000000..6fc9286cf --- /dev/null +++ b/tsc/meetings/2022-12-07.md @@ -0,0 +1,51 @@ +# OpenCue TSC Meeting Notes 7 Dec 2022 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Postgres upgrade/fixes + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1199 + * PR updated with some new research. + * DBA contact has reproduced problem but doesn't support Docker deployments. Confirmed this is + embedded-postgres issue only. + * Filed a ticket with the upstream project. Got a few suggestions. + * Will be hard to verify performance issues until it's in production and hard to roll back. + * Conclusion: Mac+Docker for production deployments is uncommon, this is mostly for developers. + Let's work around this for now, and look into the suggestions from the upstream ticket. + Hopefully we can track down the problem and avoid having to merge this PR. +* Log4j update + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1080 + * Older PR, requested by user who tested and verified. + * Confirmed, we are good to merge this now. +* PySide6 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1195 + * SPI: difficult to production test, PySide6 does not fit nicely into packaging system. + * Conclusion: we'll add the compatibility layer now, test as best we can, and merge as an + experimental feature. Issues may still be present but we can fix these as we go, and it will + be a better situation than we currently have, where CueGUI is not available at all for users + who can't access PySide2. +* Postgres query issues + * New issue with very slow queries on the database side. + * Upgrade happened 3-4 months ago but symptoms didn't present until heavy production load. + * Debugged issue, found culprit is the `show` table, particularly a few stats columns. These + columns are updated multiple times per second under heavy load, and many other critical + queries join to the show table. This slows down the whole system. + * PR coming soon to separate these columns out into their own table/view. +* Removing dead code + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1178/files + * Can we ignore the version bump here? + * Diego will look into it offline. +* akim-ruslanov PRs need update + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1168 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1167 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1165 + * Diego will look into these offline. +* Blender update + * Blender has made progress on improved Python bindings. Should make it easier to avoid + compatibility issues in the future. + * They will support a build using VFX reference platform versions. + * March release for the initial implementation, this will be improved over the next few cycles. + * JT looking to hand this off soon. + * Nuwan is interested in working on the plugin as well. He should proceed, and we'll sync his + and JT's work if needed, or maybe JT will just use it as the base for his own work. From 133687d6328bcf42b8c350b8fbdae6605d98e43d Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 9 Jan 2023 15:19:28 -0500 Subject: [PATCH 230/277] Remove obsolete CREDITS file. (#1234) --- CREDITS | 37 ------------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 CREDITS diff --git a/CREDITS b/CREDITS deleted file mode 100644 index 988a755e9..000000000 --- a/CREDITS +++ /dev/null @@ -1,37 +0,0 @@ -cue3bot - 3720 Matt Chambers - 167 John Welborn - 123 Michael Zhang - 50 J Robert Ray - 27 Bond-Jay Ting - 22 Kasra Faghihi - 14 Blair Zajac - 2 Kevin Coats - 1 Jordon Phillips - -rqd - 473 John Welborn - 25 J Robert Ray - 6 Yudi Xue - 2 Blair Zajac - 2 Michael Zhang - 2 Jordon Phillips - 1 Kasra Faghihi - -spi_cue - 252 John Welborn - 226 Matt Chambers - 12 Jordon Phillips - 8 Yudi Xue - 5 J Robert Ray - 4 Michael Zhang - 3 Blair Zajac - -python_ice_server - 27 Blair Zajac - 8 Cottalango Leon - 5 J Robert Ray - 3 John Welborn - 2 Michael Zhang - 1 Geo Snelling - 1 Sam Richards From a4a9cd1da02ed1de6aeeb42af8d6553378c1e104 Mon Sep 17 00:00:00 2001 From: Olivier Evers Date: Tue, 10 Jan 2023 21:59:16 +0100 Subject: [PATCH 231/277] [rqd] Add some missing env vars on Windows. (#1225) --- rqd/rqd/rqcore.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index 646c4f73d..224485f2b 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -101,8 +101,9 @@ def __createEnvVariables(self): self.frameEnv["MAIL"] = "/usr/mail/%s" % self.runFrame.user_name self.frameEnv["HOME"] = "/net/homedirs/%s" % self.runFrame.user_name elif platform.system() == "Windows": - self.frameEnv["APPDATA"] = os.environ["APPDATA"] - self.frameEnv["SYSTEMROOT"] = os.environ["SYSTEMROOT"] + for variable in ["SYSTEMROOT", "APPDATA", "TMP", "COMMONPROGRAMFILES"]: + if variable in os.environ: + self.frameEnv[variable] = os.environ[variable] for key, value in self.runFrame.environment.items(): if key == 'PATH': From 58fcb4a1472e717b14175231bcb5241b2395958f Mon Sep 17 00:00:00 2001 From: romainf-ubi <117918548+romainf-ubi@users.noreply.github.com> Date: Thu, 12 Jan 2023 12:15:14 -0500 Subject: [PATCH 232/277] [cuebot] Fix malformed SQL queries. (#1222) --- .../spcue/dao/postgres/DispatchQuery.java | 14 +++++++------- .../spcue/dao/postgres/FilterDaoJdbc.java | 4 ++-- .../imageworks/spcue/dao/postgres/JobDaoJdbc.java | 4 ++-- .../imageworks/spcue/dao/postgres/ShowDaoJdbc.java | 2 +- .../spcue/test/dao/postgres/LayerDaoTests.java | 4 ++-- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java index 865472d1a..1d1f7210f 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatchQuery.java @@ -230,7 +230,7 @@ private static final String replaceQueryForFifo(String query) { "AND " + "job.pk_facility = ? " + "AND " + - "(job.str_os = ? OR job.str_os IS NULL)" + + "(job.str_os = ? OR job.str_os IS NULL) " + "AND " + "job.pk_job IN ( " + "SELECT " + @@ -256,7 +256,7 @@ private static final String replaceQueryForFifo(String query) { "AND " + "j.pk_facility = ? " + "AND " + - "(j.str_os = ? OR j.str_os IS NULL)" + + "(j.str_os = ? OR j.str_os IS NULL) " + "AND " + "(CASE WHEN lst.int_waiting_count > 0 THEN lst.pk_layer ELSE NULL END) = l.pk_layer " + "AND " + @@ -333,7 +333,7 @@ private static final String replaceQueryForFifo(String query) { "AND " + "job.pk_facility = ? " + "AND " + - "(job.str_os = ? OR job.str_os IS NULL)" + + "(job.str_os = ? OR job.str_os IS NULL) " + "AND " + "job.pk_job IN ( " + "SELECT /* index (h i_str_host_tag) */ " + @@ -354,7 +354,7 @@ private static final String replaceQueryForFifo(String query) { "AND " + "j.pk_facility = ? " + "AND " + - "(j.str_os = ? OR j.str_os IS NULL)" + + "(j.str_os = ? OR j.str_os IS NULL) " + "AND " + "(CASE WHEN lst.int_waiting_count > 0 THEN lst.pk_layer ELSE NULL END) = l.pk_layer " + "AND " + @@ -426,7 +426,7 @@ private static final String replaceQueryForFifo(String query) { "AND " + "(folder_resource.int_max_gpus = -1 OR folder_resource.int_gpus < folder_resource.int_max_gpus) " + "AND " + - "job_resource.int_priority > ?" + + "job_resource.int_priority > ? " + "AND " + "job_resource.int_cores < job_resource.int_max_cores " + "AND " + @@ -438,7 +438,7 @@ private static final String replaceQueryForFifo(String query) { "AND " + "job.pk_facility = ? " + "AND " + - "(job.str_os = ? OR job.str_os IS NULL)" + + "(job.str_os = ? OR job.str_os IS NULL) " + "AND " + "job.pk_job IN ( " + "SELECT /* index (h i_str_host_tag) */ " + @@ -457,7 +457,7 @@ private static final String replaceQueryForFifo(String query) { "AND " + "j.pk_facility = ? " + "AND " + - "(j.str_os = ? OR j.str_os IS NULL)" + + "(j.str_os = ? OR j.str_os IS NULL) " + "AND " + "(CASE WHEN lst.int_waiting_count > 0 THEN lst.pk_layer ELSE NULL END) = l.pk_layer " + "AND " + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FilterDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FilterDaoJdbc.java index a1cb7d2cf..ee30d5f8e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FilterDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FilterDaoJdbc.java @@ -54,7 +54,7 @@ public class FilterDaoJdbc extends JdbcDaoSupport implements FilterDao { private static final String GET_ACTIVE_FILTERS = "SELECT " + - "filter.*" + + "filter.* " + "FROM " + "filter " + "WHERE " + @@ -66,7 +66,7 @@ public class FilterDaoJdbc extends JdbcDaoSupport implements FilterDao { private static final String GET_FILTERS = "SELECT " + - "filter.*" + + "filter.* " + "FROM " + "filter " + "WHERE " + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java index 890f5ea81..d32a5a259 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java @@ -437,7 +437,7 @@ public void updateMaxRSS(JobInterface job, long value) { "str_visible_name = NULL, " + "ts_stopped = current_timestamp "+ "WHERE " + - "str_state = 'PENDING'" + + "str_state = 'PENDING' " + "AND " + "pk_job = ?"; @@ -945,7 +945,7 @@ public void updateParent(JobInterface job, GroupDetail dest, Inherit[] inherits) "AND " + "job.b_auto_book = true " + "AND " + - "job_stat.int_waiting_count != 0" + + "job_stat.int_waiting_count != 0 " + "AND " + "job_resource.int_cores < job_resource.int_max_cores " + "AND " + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java index 5d674ed82..8361a6c5f 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java @@ -147,7 +147,7 @@ public void insertShow(ShowEntity show) { "SELECT " + "COUNT(show.pk_show) " + "FROM " + - "show LEFT JOIN show_alias ON (show.pk_show = show_alias.pk_show )" + + "show LEFT JOIN show_alias ON (show.pk_show = show_alias.pk_show) " + "WHERE " + "(show.str_name = ? OR show_alias.str_name = ?) "; public boolean showExists(String name) { diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java index 8ad2f5bac..83c8a466b 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java @@ -522,7 +522,7 @@ public void isOptimizable() { layer.getLayerId()); jdbcTemplate.update( - "UPDATE layer_usage SET int_core_time_success = 3600 * 6" + + "UPDATE layer_usage SET int_core_time_success = 3600 * 6 " + "WHERE pk_layer=?", layer.getLayerId()); assertFalse(layerDao.isOptimizable(layer, 5, 3600)); @@ -532,7 +532,7 @@ public void isOptimizable() { * Assert True */ jdbcTemplate.update( - "UPDATE layer_usage SET int_core_time_success = 3500 * 5" + + "UPDATE layer_usage SET int_core_time_success = 3500 * 5 " + "WHERE pk_layer=?", layer.getLayerId()); assertTrue(layerDao.isOptimizable(layer, 5, 3600)); From 3b2326ebbb3d1f4395cdcb66d0b41a556ee57e17 Mon Sep 17 00:00:00 2001 From: romainf-ubi <117918548+romainf-ubi@users.noreply.github.com> Date: Thu, 12 Jan 2023 12:30:15 -0500 Subject: [PATCH 233/277] [rqd] Add pynput to requirements.txt. (#1230) --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index f465c5910..92b375d60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,5 +13,6 @@ protobuf==3.17.3;python_version<"3.0" psutil==5.6.7 pyfakefs==3.6 pylint==2.6.0;python_version>="3.7" +pynput==1.7.6 PyYAML==5.1 six==1.11.0 From 1a920dcb7b79668777388c69c4806e2e6d29e69b Mon Sep 17 00:00:00 2001 From: romainf-ubi <117918548+romainf-ubi@users.noreply.github.com> Date: Thu, 12 Jan 2023 13:09:14 -0500 Subject: [PATCH 234/277] Upgrade PySide2 to 5.15.2.1. (#1226) --- requirements_gui.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_gui.txt b/requirements_gui.txt index d9aa53b35..c589c3517 100644 --- a/requirements_gui.txt +++ b/requirements_gui.txt @@ -1 +1 @@ -PySide2==5.15.2 +PySide2==5.15.2.1 From 27bc9e16713d437b92cee87a7c148c77ddaf2cec Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 16 Jan 2023 14:25:08 -0500 Subject: [PATCH 235/277] [sandbox] Stability improvements. (#1244) --- docker-compose.yml | 2 +- sandbox/flyway.Dockerfile | 6 ++---- sandbox/get-latest-release-tag.sh | 12 ++++++++++++ sandbox/migrate.sh | 3 ++- 4 files changed, 17 insertions(+), 6 deletions(-) create mode 100755 sandbox/get-latest-release-tag.sh diff --git a/docker-compose.yml b/docker-compose.yml index 554735581..656dec4bc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: db: - image: postgres + image: postgres:15.1 environment: - POSTGRES_USER=cuebot - POSTGRES_PASSWORD=cuebot_password diff --git a/sandbox/flyway.Dockerfile b/sandbox/flyway.Dockerfile index d3d45b998..086d47bbd 100644 --- a/sandbox/flyway.Dockerfile +++ b/sandbox/flyway.Dockerfile @@ -1,10 +1,8 @@ -FROM centos +FROM almalinux:8.7 -ARG FLYWAY_VERSION=8.5.4 +ARG FLYWAY_VERSION=9.9.0 # Get flyway -RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* -RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* RUN yum install -y tar java-1.8.0-openjdk postgresql-jdbc nc postgresql RUN curl -O https://repo1.maven.org/maven2/org/flywaydb/flyway-commandline/${FLYWAY_VERSION}/flyway-commandline-${FLYWAY_VERSION}-linux-x64.tar.gz RUN tar -xzf flyway-commandline-${FLYWAY_VERSION}-linux-x64.tar.gz diff --git a/sandbox/get-latest-release-tag.sh b/sandbox/get-latest-release-tag.sh new file mode 100755 index 000000000..0aa552efa --- /dev/null +++ b/sandbox/get-latest-release-tag.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Script for fetching the latest release version of OpenCue. +# - `curl` fetches all of the metadata for the latest release, in JSON format. +# - `grep` filters for just the `"tag_name": "v1.2.3"` line. +# - `cut` extracts the `v1.2.3` value from the `tag_name` line. +# - `tr` removes the `v` to leave us with the final version number e.g. `1.2.3`. + +curl -s https://api.github.com/repos/AcademySoftwareFoundation/OpenCue/releases/latest \ + | grep tag_name \ + | cut -d \" -f 4 \ + | tr -d v diff --git a/sandbox/migrate.sh b/sandbox/migrate.sh index 529dca834..b7af1cdfd 100755 --- a/sandbox/migrate.sh +++ b/sandbox/migrate.sh @@ -1,5 +1,6 @@ #!/bin/bash +set -e until nc --send-only $PGHOST $PGPORT < /dev/null do @@ -13,7 +14,7 @@ do sleep 2 done -# Apply the flyway database migrations. +echo "Applying database migrations..." ./flyway migrate -user=${PGUSER} -password=${PGPASSWORD} -url="jdbc:postgresql://${PGHOST}:${PGPORT}/${PGDATABASE}" -locations='filesystem:/opt/migrations' # Check if a show exists, if not apply demo data From 14dddd36f16348c9c6778435136c7904302a0ad9 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Fri, 20 Jan 2023 13:00:14 -0500 Subject: [PATCH 236/277] Notes from Jan 18 TSC meeting. (#1247) --- tsc/meetings/2023-01-18.md | 67 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 tsc/meetings/2023-01-18.md diff --git a/tsc/meetings/2023-01-18.md b/tsc/meetings/2023-01-18.md new file mode 100644 index 000000000..1f34a4726 --- /dev/null +++ b/tsc/meetings/2023-01-18.md @@ -0,0 +1,67 @@ +# OpenCue TSC Meeting Notes 18 Jan 2023 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* New release! + * CI / dev environment issues mostly resolved now, release unblocked. + * Getting back to our monthly release cycle. +* Postgres upgrade + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1199 + * Upstream: https://github.com/zonkyio/embedded-postgres/issues/99 + * Cuebot Docker image currently does not build on M1 Mac due to the old embedded-postgres + library. + * Newer embedded-postgres binaries produce weird results that fail tests. M1+Docker only. + * [Breakthrough!](https://github.com/zonkyio/embedded-postgres/issues/99#issuecomment-1378159242) + The issue was coming from an old Ubuntu version used for the embedded-postgres build. + * Brian sent https://github.com/zonkyio/embedded-postgres-binaries/pull/64 upstream with a + proposed fix, which upgrades their build process to use a newer Ubuntu version. + * Waiting for review, then waiting for new binaries to be published. But we are able to build + embedded-postgres locally now, and modify Cuebot to use those binaries rather than pull from + Maven. +* PySide6 + * New proposed PR: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1238 + * Use QtPy library as the compatibility layer, works in PySide2 and 6. + * Would like to get more testing in a PySide2 environment. + * This breaks the dependency chain slightly, as the code would now depend on QtPy but not + PySide. However we can specify whatever other dependencies we want in + setup.py/requirements.txt. + * Proposal: master branch will continue to specify PySide 2 as a dependency. Packaging/release + pipelines will also create a PySide 6 version. + * Idea: setup.py is a Python script, so it could use custom logic to specify the PySide6 + dependency if on an M1 mac. setup.py is executed not just at build/packaging time but at + install time as well. + * Will test more using Pyside2 before merging. +* Migrate stats columns + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1228 + * Test failures to be resolved. + * Diego will take a look. + * Maybe related to some intermittent CI failures we've seen recently, those should be mostly + resolved now. +* CueGUI new config file, cuegui.yaml + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1242 + * Moves most constants to be editable via an external YAML file. No longer any need to edit + code, and exposes many useful settings to users. + * YAML file can be specified via env var or standard config directories. Sysadmins can + distribute their own cuegui.yaml to all local users. + * Need docs update that includes config guides for all components, including this update. + * This should be the last client-side piece needing a config update. We can now move on to the + main PyPI work. +* Integration tests + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1245 + * Initial version out for review. Stands up Docker compose environment, tests database, RQD + registration, API, cueadmin. + * Next will need to add launching a job and verifying. +* Batch of GUI bug fixes coming soon. +* RQD systemd changes + * Previously using init.d, now migrating to systemd. + * OOM manager was sometimes killing the parent RQD process rather than the job itself. This + would take the RQD host offline and it would not report on failure cause. Cuebot would then + distribute the culprit job to other hosts, and the problem could proliferate. + * systemd has a feature to help reduce likelihood of this happening. + * Once that's done, it would be good to publish rpm packages as part of packaging/release. + Cuebot does this already, this would standardize among the server-side components. + * RQD pip package may need to include initd/systemd scripts, or docs to help register RQD with + the system, i.e. start on host boot. + * Sysadmins also seem to prefer rpms to pip install. From ad28df3f8b0f7975db15f0c72d8b90e8f742b555 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Fri, 20 Jan 2023 14:13:12 -0500 Subject: [PATCH 237/277] Add wrapper script for cuegui tests. (#1236) --- ci/run_gui_test.sh | 27 +++++++++++++++++++++++++++ ci/run_python_tests.sh | 2 +- cuegui/tests/MenuActions_tests.py | 2 +- 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100755 ci/run_gui_test.sh diff --git a/ci/run_gui_test.sh b/ci/run_gui_test.sh new file mode 100755 index 000000000..b80505bdb --- /dev/null +++ b/ci/run_gui_test.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Wrapper script for CueGUI tests. +# +# xvfb-run sometimes crashes on exit, we haven't been able to figure out why yet. +# This means that tests may pass but the xvfb-run crash will generate a non-zero exit code +# and cause our CI pipeline to fail. +# +# We work around this by capturing unit test output and looking for the text that indicates +# tests have passed: +# +# > Ran 209 tests in 4.394s +# > +# > OK +# + +test_log="/tmp/cuegui_result.log" +PYTHONPATH=pycue xvfb-run -d python cuegui/setup.py test | tee ${test_log} + +grep -Pz 'Ran \d+ tests in [0-9\.]+s\n\nOK' ${test_log} +if [ $? -eq 0 ]; then + echo "Detected passing tests" + exit 0 +fi + +echo "Detected test failure" +exit 1 diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index 093ab9c62..c37b539ad 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -24,5 +24,5 @@ python rqd/setup.py test # Xvfb no longer supports Python 2. if [[ "$python_version" =~ "Python 3" ]]; then - PYTHONPATH=pycue xvfb-run -d python cuegui/setup.py test + ci/run_gui_test.sh fi diff --git a/cuegui/tests/MenuActions_tests.py b/cuegui/tests/MenuActions_tests.py index 25033c79b..0127cc7ce 100644 --- a/cuegui/tests/MenuActions_tests.py +++ b/cuegui/tests/MenuActions_tests.py @@ -65,7 +65,7 @@ def setUp(self): self.job_actions = cuegui.MenuActions.JobActions(self.widgetMock, mock.Mock(), None, None) def test_jobs(self): - print(cuegui.MenuActions.MenuActions(self.widgetMock, None, None, None).jobs()) + cuegui.MenuActions.MenuActions(self.widgetMock, None, None, None).jobs() def test_unmonitor(self): self.job_actions.unmonitor() From 64a4dfd29827f3797fa458d4ad54d31394f9ee7b Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Fri, 20 Jan 2023 14:14:04 -0500 Subject: [PATCH 238/277] Replace deprecated CI actions. (#1239) --- .github/workflows/packaging-pipeline.yml | 20 ++++++++----- .github/workflows/release-pipeline.yml | 29 +++++++++++------- .github/workflows/sonar-cloud-pipeline.yml | 4 +-- .github/workflows/testing-pipeline.yml | 34 +++++++++++----------- 4 files changed, 49 insertions(+), 38 deletions(-) diff --git a/.github/workflows/packaging-pipeline.yml b/.github/workflows/packaging-pipeline.yml index 920d92ca7..6fcfa5676 100644 --- a/.github/workflows/packaging-pipeline.yml +++ b/.github/workflows/packaging-pipeline.yml @@ -43,7 +43,7 @@ jobs: name: Build ${{ matrix.NAME }} steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: # Fetch all Git history, otherwise the current version number will # not be correctly calculated. @@ -58,6 +58,12 @@ jobs: role-to-assume: ${{ secrets.AWS_S3_ROLE }} role-duration-seconds: 1800 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_PASS }} + - name: Set build ID run: | set -e @@ -66,13 +72,11 @@ jobs: echo "BUILD_ID=$(cat ./VERSION)" >> ${GITHUB_ENV} - name: Build Docker image - uses: docker/build-push-action@v1 + uses: docker/build-push-action@v3 with: - username: ${{ secrets.DOCKER_USER }} - password: ${{ secrets.DOCKER_PASS }} - dockerfile: ${{ matrix.component }}/Dockerfile - repository: opencuebuild/${{ matrix.component }} - tags: ${{ env.BUILD_ID }} + file: ${{ matrix.component }}/Dockerfile + tags: opencuebuild/${{ matrix.component }}:${{ env.BUILD_ID }} + push: true - name: Extract Artifacts run: | @@ -101,7 +105,7 @@ jobs: name: Create Other Build Artifacts steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: # Fetch all Git history, otherwise the current version number will # not be correctly calculated. diff --git a/.github/workflows/release-pipeline.yml b/.github/workflows/release-pipeline.yml index d87637d61..1111bec54 100644 --- a/.github/workflows/release-pipeline.yml +++ b/.github/workflows/release-pipeline.yml @@ -12,7 +12,7 @@ jobs: name: Preflight steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -45,7 +45,7 @@ jobs: name: Release ${{ matrix.component }} Docker image steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -61,14 +61,18 @@ jobs: set -e docker pull opencuebuild/${{ matrix.component }}:${BUILD_ID} - - name: Rebuild and push Docker image - uses: docker/build-push-action@v1 + - name: Login to Docker Hub + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_USER }} password: ${{ secrets.DOCKER_PASS }} - dockerfile: ${{ matrix.component }}/Dockerfile - repository: opencue/${{ matrix.component }} - tags: ${{ env.BUILD_ID }}, latest + + - name: Rebuild and push Docker image + uses: docker/build-push-action@v3 + with: + file: ${{ matrix.component }}/Dockerfile + tags: opencue/${{ matrix.component }}:${{ env.BUILD_ID }},opencue/${{ matrix.component }}:latest + push: true create_release: needs: preflight @@ -76,7 +80,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -103,7 +107,7 @@ jobs: run: | mkdir -p "${GITHUB_WORKSPACE}/artifacts/" aws s3 sync "s3://${S3_BUCKET}/opencue/${BUILD_ID}/" "${GITHUB_WORKSPACE}/artifacts/" - echo "::set-output name=filenames::$(ls "${GITHUB_WORKSPACE}/artifacts/" | xargs)" + echo "filenames=$(ls "${GITHUB_WORKSPACE}/artifacts/" | xargs)" >> ${GITHUB_OUTPUT} - name: List artifacts run: | @@ -114,9 +118,12 @@ jobs: run: | last_tagged_version=$(git describe --tags --abbrev=0 $(git rev-list --tags --skip=1 --max-count=1)) commits_since_last_release=$(git log --reverse --pretty="* %H %s" ${last_tagged_version}..HEAD) + # Use a delimiter to preserve the multiline string. # See https://github.community/t/set-output-truncates-multiline-strings/16852 - commits_since_last_release="${commits_since_last_release//$'\n'/'%0A'}" - echo "::set-output name=commits::${commits_since_last_release}" + delimiter="$(openssl rand -hex 8)" + echo "commits<<${delimiter}" >> ${GITHUB_OUTPUT} + echo "${commits_since_last_release}" >> ${GITHUB_OUTPUT} + echo "${delimiter}" >> ${GITHUB_OUTPUT} - name: Create release id: create_release diff --git a/.github/workflows/sonar-cloud-pipeline.yml b/.github/workflows/sonar-cloud-pipeline.yml index ec330da8c..1319f828b 100644 --- a/.github/workflows/sonar-cloud-pipeline.yml +++ b/.github/workflows/sonar-cloud-pipeline.yml @@ -12,7 +12,7 @@ jobs: name: Analyze Python Components steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: # Fetch all Git history, otherwise the current version number will # not be correctly calculated. @@ -33,7 +33,7 @@ jobs: name: Analyze Cuebot steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: # Fetch all Git history, otherwise the current version number will # not be correctly calculated. diff --git a/.github/workflows/testing-pipeline.yml b/.github/workflows/testing-pipeline.yml index d3583bd0e..98e5ca166 100644 --- a/.github/workflows/testing-pipeline.yml +++ b/.github/workflows/testing-pipeline.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest container: aswf/ci-opencue:2019 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Run Python Tests run: ci/run_python_tests.sh @@ -22,7 +22,7 @@ jobs: container: image: aswf/ci-opencue:2019 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Build with Gradle run: | chown -R aswfuser:aswfgroup . @@ -33,7 +33,7 @@ jobs: runs-on: ubuntu-latest container: aswf/ci-opencue:2020 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Run Python Tests run: ci/run_python_tests.sh @@ -43,7 +43,7 @@ jobs: container: image: aswf/ci-opencue:2020 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Build with Gradle run: | chown -R aswfuser:aswfgroup . @@ -54,7 +54,7 @@ jobs: runs-on: ubuntu-latest container: aswf/ci-opencue:2021 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Run Python Tests run: ci/run_python_tests.sh @@ -64,7 +64,7 @@ jobs: container: image: aswf/ci-opencue:2021 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Build with Gradle run: | chown -R aswfuser:aswfgroup . @@ -75,7 +75,7 @@ jobs: runs-on: ubuntu-latest container: aswf/ci-opencue:2022 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Run Python Tests run: ci/run_python_tests.sh @@ -85,7 +85,7 @@ jobs: container: image: aswf/ci-opencue:2022 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Build with Gradle run: | chown -R aswfuser:aswfgroup . @@ -96,7 +96,7 @@ jobs: runs-on: ubuntu-latest container: aswf/ci-opencue:2022 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Lint Python Code run: ci/run_python_lint.sh @@ -106,7 +106,7 @@ jobs: container: image: aswf/ci-opencue:2020 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Run Sphinx build run: ci/build_sphinx_docs.sh @@ -114,18 +114,18 @@ jobs: name: Check Changed Files runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Get Changed Files id: get_changed_files - uses: jitterbit/get-changed-files@v1 + uses: tj-actions/changed-files@v35 - name: Check for Version Change - run: ci/check_changed_files.py ${{ steps.get_changed_files.outputs.modified }} ${{ steps.get_changed_files.outputs.removed }} + run: ci/check_changed_files.py ${{ steps.get_changed_files.outputs.modified_files }} ${{ steps.get_changed_files.outputs.deleted_files }} check_migration_files: name: Check Database Migration Files runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Check Migration Files run: ci/check_database_migrations.py @@ -133,9 +133,9 @@ jobs: name: Check for Version Bump runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Get Changed Files id: get_changed_files - uses: jitterbit/get-changed-files@v1 + uses: tj-actions/changed-files@v35 - name: Check for Version Change - run: ci/check_version_bump.py ${{ steps.get_changed_files.outputs.all }} + run: ci/check_version_bump.py ${{ steps.get_changed_files.outputs.all_changed_and_modified_files }} From df96a2f962ec14ad1e004c241891037e734c01c1 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Fri, 20 Jan 2023 15:06:19 -0500 Subject: [PATCH 239/277] Fix Docker build in the packaging pipeline. (#1250) --- .github/workflows/packaging-pipeline.yml | 1 + .github/workflows/release-pipeline.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/packaging-pipeline.yml b/.github/workflows/packaging-pipeline.yml index 6fcfa5676..6e21eab65 100644 --- a/.github/workflows/packaging-pipeline.yml +++ b/.github/workflows/packaging-pipeline.yml @@ -76,6 +76,7 @@ jobs: with: file: ${{ matrix.component }}/Dockerfile tags: opencuebuild/${{ matrix.component }}:${{ env.BUILD_ID }} + context: . push: true - name: Extract Artifacts diff --git a/.github/workflows/release-pipeline.yml b/.github/workflows/release-pipeline.yml index 1111bec54..48aeb9f5a 100644 --- a/.github/workflows/release-pipeline.yml +++ b/.github/workflows/release-pipeline.yml @@ -72,6 +72,7 @@ jobs: with: file: ${{ matrix.component }}/Dockerfile tags: opencue/${{ matrix.component }}:${{ env.BUILD_ID }},opencue/${{ matrix.component }}:latest + context: . push: true create_release: From 04660512566565814a1b36a975ac77e7a1cdfea5 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Fri, 20 Jan 2023 23:46:42 -0500 Subject: [PATCH 240/277] Lock evdev dependency for Python 2. (#1248) --- ci/run_python_tests.sh | 2 +- requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index c37b539ad..c6a51a015 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -2,7 +2,7 @@ set -e -python_version=$(python -V) +python_version=$(python -V 2>&1) echo "Will run tests using ${python_version}" pip install --user -r requirements.txt -r requirements_gui.txt diff --git a/requirements.txt b/requirements.txt index 92b375d60..44adbe5b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ 2to3==1.0 enum34==1.1.6 +evdev==1.4.0;python_version<"3.0" and "linux" in sys_platform future==0.17.1 futures==3.2.0;python_version<"3.0" grpcio==1.26.0;python_version<"3.0" From 633908735f81aa1e503c1618ddb1b8eb7d058f4d Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Fri, 20 Jan 2023 23:47:51 -0500 Subject: [PATCH 241/277] [rqd] Raise exception during startup if CUEBOT_HOSTNAME is empty. (#1237) --- rqd/rqd/rqnetwork.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/rqd/rqd/rqnetwork.py b/rqd/rqd/rqnetwork.py index 9de2cf452..9f33e64a2 100644 --- a/rqd/rqd/rqnetwork.py +++ b/rqd/rqd/rqnetwork.py @@ -38,6 +38,7 @@ import rqd.compiled_proto.report_pb2_grpc import rqd.compiled_proto.rqd_pb2_grpc import rqd.rqconstants +import rqd.rqexceptions import rqd.rqdservicers import rqd.rqutil @@ -278,12 +279,13 @@ def __getChannel(self): ), ) - cuebots = rqd.rqconstants.CUEBOT_HOSTNAME.split() + cuebots = rqd.rqconstants.CUEBOT_HOSTNAME.strip().split() + if len(cuebots) == 0: + raise rqd.rqexceptions.RqdException("CUEBOT_HOSTNAME is empty") shuffle(cuebots) - if len(cuebots) > 0: - self.channel = grpc.insecure_channel('%s:%s' % (cuebots[0], - rqd.rqconstants.CUEBOT_GRPC_PORT)) - self.channel = grpc.intercept_channel(self.channel, *interceptors) + self.channel = grpc.insecure_channel('%s:%s' % (cuebots[0], + rqd.rqconstants.CUEBOT_GRPC_PORT)) + self.channel = grpc.intercept_channel(self.channel, *interceptors) atexit.register(self.closeChannel) def __getReportStub(self): From 5d5ef8f73d03f43b8a9c955f6cd0a9173f4b6f82 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Fri, 20 Jan 2023 23:59:50 -0500 Subject: [PATCH 242/277] [cuebot] Fix a few database query and test issues. (#1232) --- .../spcue/dao/postgres/ProcDaoJdbc.java | 39 ++++++++----------- .../dao/postgres/DispatcherDaoFifoTests.java | 1 - .../test/dao/postgres/LayerDaoTests.java | 24 +++++++----- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java index b68f8a74c..5af292fb3 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java @@ -262,30 +262,25 @@ public void updateProcMemoryUsage(FrameInterface f, long rss, long maxRss, "SELECT pk_frame FROM proc WHERE pk_frame=? FOR UPDATE", String.class, f.getFrameId()).equals(f.getFrameId())) { - getJdbcTemplate().update(UPDATE_PROC_MEMORY_USAGE, - rss, maxRss, vss, maxVss, - usedGpuMemory, maxUsedGpuMemory, f.getFrameId()); - } getJdbcTemplate().update(new PreparedStatementCreator() { - @Override - public PreparedStatement createPreparedStatement(Connection conn) - throws SQLException { - PreparedStatement updateProc = conn.prepareStatement( - UPDATE_PROC_MEMORY_USAGE); - updateProc.setLong(1, rss); - updateProc.setLong(2, maxRss); - updateProc.setLong(3, vss); - updateProc.setLong(4, maxVss); - updateProc.setLong(5, usedGpuMemory); - updateProc.setLong(6, maxUsedGpuMemory); - updateProc.setBytes(7, children); - updateProc.setString(8, f.getFrameId()); - return updateProc; - } - } - ); + @Override + public PreparedStatement createPreparedStatement(Connection conn) + throws SQLException { + PreparedStatement updateProc = conn.prepareStatement( + UPDATE_PROC_MEMORY_USAGE); + updateProc.setLong(1, rss); + updateProc.setLong(2, maxRss); + updateProc.setLong(3, vss); + updateProc.setLong(4, maxVss); + updateProc.setLong(5, usedGpuMemory); + updateProc.setLong(6, maxUsedGpuMemory); + updateProc.setBytes(7, children); + updateProc.setString(8, f.getFrameId()); + return updateProc; + } + }); } - catch (DataAccessException dae) { + } catch (DataAccessException dae) { logger.info("The proc for frame " + f + " could not be updated with new memory stats: " + dae); } diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java index 5de19273e..c34396709 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java @@ -224,7 +224,6 @@ public void testFifoSchedulingDisabled() throws Exception { List sortedJobs = new ArrayList(jobs); Collections.sort(sortedJobs, Comparator.comparing(jobId -> jobManager.getJob(jobId).getName())); - assertNotEquals(jobs, sortedJobs); for (int i = 0; i < count; i++) { assertEquals("pipe-default-testuser_job" + i, diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java index 83c8a466b..189178689 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java @@ -20,6 +20,7 @@ package com.imageworks.spcue.test.dao.postgres; import java.io.File; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -44,7 +45,6 @@ import com.imageworks.spcue.LayerDetail; import com.imageworks.spcue.LayerInterface; import com.imageworks.spcue.LimitEntity; -import com.imageworks.spcue.LimitInterface; import com.imageworks.spcue.ResourceUsage; import com.imageworks.spcue.config.TestAppConfig; import com.imageworks.spcue.dao.DepartmentDao; @@ -63,9 +63,11 @@ import com.imageworks.spcue.util.FrameSet; import com.imageworks.spcue.util.JobLogUtil; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @Transactional @@ -116,7 +118,11 @@ public void testMode() { } public LayerDetail getLayer() { + List layers = getLayers(); + return layers.get(layers.size()-1); + } + public List getLayers() { JobSpec spec = jobLauncher.parse(new File("src/test/resources/conf/jobspec/jobspec.xml")); JobDetail job = spec.getJobs().get(0).detail; job.groupId = ROOT_FOLDER; @@ -126,14 +132,13 @@ public LayerDetail getLayer() { job.facilityId = facilityDao.getDefaultFacility().getId(); jobDao.insertJob(job, jobLogUtil); - LayerDetail lastLayer= null; + List result = new ArrayList<>(); String limitId = limitDao.createLimit(LIMIT_NAME, LIMIT_MAX_VALUE); limitDao.createLimit(LIMIT_TEST_A, 1); limitDao.createLimit(LIMIT_TEST_B, 2); limitDao.createLimit(LIMIT_TEST_C, 3); for (BuildableLayer buildableLayer: spec.getJobs().get(0).getBuildableLayers()) { - LayerDetail layer = buildableLayer.layerDetail; FrameSet frameSet = new FrameSet(layer.range); int num_frames = frameSet.size(); @@ -147,10 +152,10 @@ public LayerDetail getLayer() { layerDao.insertLayerDetail(layer); layerDao.insertLayerEnvironment(layer, buildableLayer.env); layerDao.addLimit(layer, limitId); - lastLayer = layer; + result.add(layer); } - return lastLayer; + return result; } public JobDetail getJob() { @@ -202,16 +207,17 @@ public void testGetLayerDetail() { LayerDetail l2 = layerDao.getLayerDetail(layer); LayerDetail l3 = layerDao.getLayerDetail(layer.id); - assertEquals(l2, l3); + assertEquals(layer, l2); + assertEquals(layer, l3); } @Test @Transactional @Rollback(true) public void testGetLayerDetails() { - LayerDetail layer = getLayer(); - List ld = layerDao.getLayerDetails(getJob()); - assertEquals(ld.get(0).name, LAYER_NAME); + List wantLayers = getLayers(); + List gotLayers = layerDao.getLayerDetails(getJob()); + assertThat(gotLayers, containsInAnyOrder(wantLayers.toArray())); } @Test From f7cf0bdea1e75c9c56da7ae5c7ccdf7a5aab5677 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Sat, 21 Jan 2023 00:06:19 -0500 Subject: [PATCH 243/277] Create initial integration test script. (#1245) --- .github/workflows/packaging-pipeline.yml | 22 ++- ci/run_integration_test.sh | 234 +++++++++++++++++++++++ sandbox/flyway.Dockerfile | 6 +- sandbox/install-client-sources.sh | 9 +- 4 files changed, 261 insertions(+), 10 deletions(-) create mode 100755 ci/run_integration_test.sh diff --git a/.github/workflows/packaging-pipeline.yml b/.github/workflows/packaging-pipeline.yml index 6e21eab65..c7ab1f85c 100644 --- a/.github/workflows/packaging-pipeline.yml +++ b/.github/workflows/packaging-pipeline.yml @@ -6,8 +6,25 @@ on: branches: [ master ] jobs: - build_components: + integration_test: + name: Run Integration Test runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Run test + run: ci/run_integration_test.sh + + - name: Archive log files + uses: actions/upload-artifact@v3 + if: ${{ always() }} + with: + name: test-logs + path: /tmp/opencue-test/*.log + + build_components: + needs: integration_test strategy: matrix: component: [cuebot, rqd] @@ -41,6 +58,7 @@ jobs: ARTIFACTS: cueadmin-${BUILD_ID}-all.tar.gz name: Build ${{ matrix.NAME }} + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 @@ -101,9 +119,9 @@ jobs: done create_other_artifacts: + name: Create Other Build Artifacts needs: build_components runs-on: ubuntu-latest - name: Create Other Build Artifacts steps: - name: Checkout uses: actions/checkout@v3 diff --git a/ci/run_integration_test.sh b/ci/run_integration_test.sh new file mode 100755 index 000000000..ee0a37fe6 --- /dev/null +++ b/ci/run_integration_test.sh @@ -0,0 +1,234 @@ +#!/bin/bash + +set -e + +RQD_ROOT="/tmp/rqd" +TEST_LOGS="/tmp/opencue-test" +DOCKER_COMPOSE_LOG="${TEST_LOGS}/docker-compose.log" +DB_DATA_DIR="sandbox/db-data" +VENV="/tmp/opencue-integration-venv" + +log() { + echo "$(date "+%Y-%m-%d %H:%M:%S") $1 $2" +} + +kill_descendant_processes() { + local pid="$1" + local and_self="${2:-false}" + if children="$(pgrep -P "$pid")"; then + for child in $children; do + kill_descendant_processes "$child" true + done + fi + if [[ "$and_self" == true ]]; then + kill "$pid" 2>/dev/null || true + fi +} + +verify_command_exists() { + if ! command -v $1 &> /dev/null; then + log ERROR "command \"$1\" was not found" + exit 1 + fi +} + +verify_no_database() { + if [ -e "${DB_DATA_DIR}" ]; then + log ERROR "Postgres data directory ${DB_DATA_DIR} already exists" + exit 1 + fi +} + +verify_no_containers() { + num_containers=$(docker compose ps --format json | jq length) + if [[ $num_containers -gt 0 ]]; then + log ERROR "Found ${num_containers} Docker compose containers, clean these up with \`docker compose rm\` before continuing" + exit 1 + fi +} + +create_rqd_root() { + if [ -e "$RQD_ROOT" ]; then + log ERROR "log root ${RQD_ROOT} already exists" + exit 1 + fi + + mkdir -p "${RQD_ROOT}/logs" + mkdir "${RQD_ROOT}/shots" +} + +wait_for_service_state() { + log INFO "Waiting for service \"$1\" to have state \"$2\"..." + while true; do + current_time=$(date +%s) + if [[ $current_time -gt $3 ]]; then + log ERROR "Timed out waiting for Docker compose to come up" + exit 1 + fi + container=$(docker compose ps --all --format json | jq ".[] | select(.Service==\"$1\")") + if [[ ${container} = "" ]]; then + log INFO "Service \"$1\": no container yet" + else + container_name=$(echo "$container" | jq -r '.Name') + current_state=$(echo "$container" | jq -r '.State') + log INFO "Service \"$1\": container \"${container_name}\" state = ${current_state}" + if [[ ${current_state} = $2 ]]; then + break + fi + fi + sleep 5 + done +} + +verify_flyway_success() { + container=$(docker compose ps --all --format json | jq '.[] | select(.Service=="flyway")') + container_name=$(echo "$container" | jq -r '.Name') + exit_code=$(echo "$container" | jq -r '.ExitCode') + if [[ ${exit_code} = 0 ]]; then + log INFO "Service \"flyway\": container \"${container_name}\" exit code = 0 (PASS)" + else + log ERROR "Service \"flyway\": container \"${container_name}\" exit code = ${exit_code} (FAIL)" + exit 1 + fi +} + +verify_migration_versions() { + migrations_in_db=$(docker compose exec -e PGUSER=cuebot db psql -Aqtc "SELECT COUNT(*) FROM flyway_schema_history") + migrations_in_code=$(ls cuebot/src/main/resources/conf/ddl/postgres/migrations/ | wc -l | tr -d ' ') + if [[ ${migrations_in_db} = ${migrations_in_code} ]]; then + log INFO "Database and code both contain ${migrations_in_db} migrations (PASS)" + else + log ERROR "Database contains ${migrations_in_db} migrations, code contains ${migrations_in_code} (FAIL)" + exit 1 + fi +} + +create_and_activate_venv() { + if [[ -d "${VENV}" ]]; then + rm -rf "${VENV}" + fi + python3 -m venv "${VENV}" + source "${VENV}/bin/activate" +} + +test_pycue() { + want_shows="['testing']" + got_shows=$(python -c 'import opencue; print([show.name() for show in opencue.api.getShows()])') + if [[ "${got_shows}" = "${want_shows}" ]]; then + log INFO "(pycue) Got expected show list (PASS)" + else + log ERROR "(pycue) Got unexpected show list (FAIL)" + log ERROR "got: ${got_shows}, want: ${want_shows}" + exit 1 + fi + + rqd_name=$(docker compose ps --format json | jq -r '.[] | select(.Service=="rqd") | .Name') + rqd_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${rqd_name}") + want_hosts="['${rqd_ip}']" + got_hosts=$(python -c 'import opencue; print([host.name() for host in opencue.api.getHosts()])') + if [[ "${got_hosts}" = "${want_hosts}" ]]; then + log INFO "(pycue) Got expected host list (PASS)" + else + log ERROR "(pycue) Got unexpected host list (FAIL)" + log ERROR "got: ${got_hosts}, want: ${want_hosts}" + exit 1 + fi +} + +test_cueadmin() { + want_show="testing" + ls_response=$(cueadmin -ls) + got_show=$(echo "${ls_response}" | tail -n 1 | cut -d ' ' -f 1) + if [[ "${got_show}" = "${want_show}" ]]; then + log INFO "(cueadmin) Got expected -ls response (PASS)" + else + log ERROR "(cueadmin) Got unexpected -ls response (FAIL)" + log ERROR "got show: ${got_show}, want show: ${want_show}" + log ERROR "full response: ${ls_response}" + exit 1 + fi + + rqd_name=$(docker compose ps --format json | jq -r '.[] | select(.Service=="rqd") | .Name') + want_host=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${rqd_name}") + lh_response=$(cueadmin -lh) + got_host=$(echo "${lh_response}" | tail -n 1 | cut -d ' ' -f 1) + if [[ "${got_host}" = "${want_host}" ]]; then + log INFO "(cueadmin) Got expected -lh response (PASS)" + else + log ERROR "(cueadmin) Got unexpected -lh response (FAIL)" + log ERROR "got host: ${got_host}, want host: ${want_host}" + log ERROR "full response: ${lh_response}" + exit 1 + fi +} + +cleanup() { + docker compose rm --stop --force >>"${DOCKER_COMPOSE_LOG}" 2>&1 + rm -rf "${RQD_ROOT}" || true + rm -rf "${DB_DATA_DIR}" || true + rm -rf "${VENV}" || true +} + +main() { + # Ensure all subshells in the background are terminated when the main script exits. + trap "{ kill_descendant_processes $$; exit; }" SIGINT SIGTERM EXIT + + mkdir -p "${TEST_LOGS}" + if [[ "${CI:-false}" == true ]]; then + log INFO "More logs can be found under the test-logs artifact attached to this workflow execution" + else + log INFO "More logs can be found at ${TEST_LOGS}" + fi + + CI_DIRECTORY=$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd) + OPENCUE_ROOT=$(dirname "${CI_DIRECTORY}") + log INFO "OpenCue project is located at ${OPENCUE_ROOT}" + cd "${OPENCUE_ROOT}" + + verify_command_exists docker + verify_command_exists "docker compose" + verify_command_exists jq + verify_no_database + verify_no_containers + create_rqd_root + + log INFO "$(docker --version)" + log INFO "$(docker compose version)" + + log INFO "Building Cuebot image..." + docker build -t opencue/cuebot -f cuebot/Dockerfile . &>"${TEST_LOGS}/docker-build-cuebot.log" + log INFO "Building RQD image..." + docker build -t opencue/rqd -f rqd/Dockerfile . &>"${TEST_LOGS}/docker-build-rqd.log" + + log INFO "Starting Docker compose..." + docker compose up &>"${DOCKER_COMPOSE_LOG}" & + if [[ "$(uname -s)" == "Darwin" ]]; then + docker_timeout=$(date -v +5M +%s) + else + docker_timeout=$(date -d '5 min' +%s) + fi + wait_for_service_state "db" "running" $docker_timeout + wait_for_service_state "flyway" "exited" $docker_timeout + wait_for_service_state "cuebot" "running" $docker_timeout + wait_for_service_state "rqd" "running" $docker_timeout + + verify_flyway_success + verify_migration_versions + log INFO "Creating Python virtual environment..." + create_and_activate_venv + log INFO "Installing OpenCue Python libraries..." + install_log="${TEST_LOGS}/install-client-sources.log" + sandbox/install-client-sources.sh &>"${install_log}" + log INFO "Testing pycue library..." + test_pycue + log INFO "Testing cueadmin..." + test_cueadmin + + # TODO Launch a job and verify it finishes. + + cleanup + + log INFO "Success" +} + +main diff --git a/sandbox/flyway.Dockerfile b/sandbox/flyway.Dockerfile index 086d47bbd..fffcbd3c2 100644 --- a/sandbox/flyway.Dockerfile +++ b/sandbox/flyway.Dockerfile @@ -1,11 +1,11 @@ FROM almalinux:8.7 -ARG FLYWAY_VERSION=9.9.0 +ARG FLYWAY_VERSION=9.11.0 # Get flyway RUN yum install -y tar java-1.8.0-openjdk postgresql-jdbc nc postgresql -RUN curl -O https://repo1.maven.org/maven2/org/flywaydb/flyway-commandline/${FLYWAY_VERSION}/flyway-commandline-${FLYWAY_VERSION}-linux-x64.tar.gz -RUN tar -xzf flyway-commandline-${FLYWAY_VERSION}-linux-x64.tar.gz +RUN curl -O https://repo1.maven.org/maven2/org/flywaydb/flyway-commandline/${FLYWAY_VERSION}/flyway-commandline-${FLYWAY_VERSION}.tar.gz +RUN tar -xzf flyway-commandline-${FLYWAY_VERSION}.tar.gz WORKDIR flyway-${FLYWAY_VERSION} diff --git a/sandbox/install-client-sources.sh b/sandbox/install-client-sources.sh index 1030f492b..7e15ed018 100755 --- a/sandbox/install-client-sources.sh +++ b/sandbox/install-client-sources.sh @@ -4,14 +4,13 @@ set -e pip install -r requirements.txt -r requirements_gui.txt -# Compile the proto used to communicate with the Cuebot server +# Compile the proto used to communicate with the Cuebot server. cd proto python -m grpc_tools.protoc -I=. \ --python_out=../pycue/opencue/compiled_proto \ --grpc_python_out=../pycue/opencue/compiled_proto ./*.proto cd .. +2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py -# Install the OpenCue client packages -# You also need to set the OL_CONFIG environment variable -# to pyoutline/etc/outline.cfg to run Cuesubmit -pip install pycue/ pyoutline/ cuesubmit/ cuegui/ cueadmin/ +# Install all client packages. +pip install pycue/ pyoutline/ cueadmin/ cuesubmit/ cuegui/ From 8a33d8394ec390e75761f0798d3b403a3b7e7936 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Thu, 26 Jan 2023 18:23:09 -0500 Subject: [PATCH 244/277] [cuegui] Move constants to a YAML config file. (#1242) --- cuebot/src/main/resources/opencue.properties | 3 +- cuegui/cuegui/Constants.py | 214 +++++++++++------- cuegui/cuegui/{Config.py => Layout.py} | 5 +- cuegui/cuegui/Main.py | 4 +- cuegui/cuegui/Redirect.py | 2 +- cuegui/cuegui/Utils.py | 18 +- cuegui/cuegui/config/cue_resources.yaml | 28 --- cuegui/cuegui/config/cuegui.yaml | 112 +++++++++ cuegui/tests/Constants_tests.py | 166 ++++++++++++++ .../{Config_tests.py => Layout_tests.py} | 18 +- cuegui/tests/Utils_tests.py | 12 + 11 files changed, 442 insertions(+), 140 deletions(-) rename cuegui/cuegui/{Config.py => Layout.py} (94%) delete mode 100644 cuegui/cuegui/config/cue_resources.yaml create mode 100644 cuegui/cuegui/config/cuegui.yaml create mode 100644 cuegui/tests/Constants_tests.py rename cuegui/tests/{Config_tests.py => Layout_tests.py} (86%) diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index 145bbb352..a08522eb1 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -144,8 +144,7 @@ protected_shows=testing # -1 means shows should not get deactivated at all. max_show_stale_days=-1 -# These flags determine whether or not layers/frames will be readonly when job is finished. +# These flags determine whether layers/frames will be readonly when job is finished. # If flags are set as true, layers/frames cannot be retried, eaten, edited dependency on, etc. -# In order to toggle the same functionility on cuegui side, set flags in cue_resources.yaml layer.finished_jobs_readonly=false frame.finished_jobs_readonly=false \ No newline at end of file diff --git a/cuegui/cuegui/Constants.py b/cuegui/cuegui/Constants.py index a988538c3..4dcef529d 100644 --- a/cuegui/cuegui/Constants.py +++ b/cuegui/cuegui/Constants.py @@ -17,91 +17,165 @@ Application constants. """ - from __future__ import print_function from __future__ import division from __future__ import absolute_import +import logging import os import platform -from PySide2 import QtCore from PySide2 import QtGui from PySide2 import QtWidgets +import yaml import opencue +import opencue.config -possible_version_path = os.path.join( - os.path.abspath(os.path.join(__file__ , "../../..")), 'VERSION.in') -if os.path.exists(possible_version_path): - with open(possible_version_path) as fp: - VERSION = fp.read().strip() -else: - VERSION = "1.3.0" +__CONFIG_FILE_ENV_VAR = 'CUEGUI_CONFIG_FILE' +__DEFAULT_INI_PATH_ENV_VAR = 'CUEGUI_DEFAULT_INI_PATH' +__DEFAULT_CONFIG_FILE_NAME = 'cuegui.yaml' +__DEFAULT_CONFIG_FILE = os.path.join( + os.path.dirname(__file__), 'config', __DEFAULT_CONFIG_FILE_NAME) + + +def __getLogger(): + """Other code should use cuegui.Logger to get a logger; we avoid using that module here + to avoid creating a circular dependency.""" + logger_format = logging.Formatter("%(levelname)-9s %(module)-10s %(message)s") + logger_stream = logging.StreamHandler() + logger_stream.setLevel(logging.INFO) + logger_stream.setFormatter(logger_format) + logger = logging.getLogger(__file__) + logger.addHandler(logger_stream) + return logger + + +def __loadConfigFromFile(): + logger = __getLogger() + with open(__DEFAULT_CONFIG_FILE) as fp: + config = yaml.load(fp, Loader=yaml.SafeLoader) + + user_config_file = None + + logger.debug('Checking for cuegui config file path in %s', __CONFIG_FILE_ENV_VAR) + config_file_from_env = os.environ.get(__CONFIG_FILE_ENV_VAR) + if config_file_from_env and os.path.exists(config_file_from_env): + user_config_file = config_file_from_env + + if not user_config_file: + config_file_from_user_profile = os.path.join( + opencue.config.config_base_directory(), __DEFAULT_CONFIG_FILE_NAME) + logger.debug('Checking for cuegui config at %s', config_file_from_user_profile) + if os.path.exists(config_file_from_user_profile): + user_config_file = config_file_from_user_profile + + if user_config_file: + logger.info('Loading cuegui config from %s', user_config_file) + with open(user_config_file, 'r') as fp: + config.update(yaml.load(fp, Loader=yaml.SafeLoader)) + + return config + + +def __packaged_version(): + possible_version_path = os.path.join( + os.path.abspath(os.path.join(__file__, "../../..")), 'VERSION.in') + if os.path.exists(possible_version_path): + with open(possible_version_path) as fp: + default_version = fp.read().strip() + return default_version + return "1.3.0" -STARTUP_NOTICE_DATE = 0 -STARTUP_NOTICE_MSG = "" -JOB_UPDATE_DELAY = 10000 # msec -LAYER_UPDATE_DELAY = 10000 # msec -FRAME_UPDATE_DELAY = 10000 # msec -HOST_UPDATE_DELAY = 20000 # msec -AFTER_ACTION_UPDATE_DELAY = 1000 # msec +__config = __loadConfigFromFile() -MAX_LOG_POPUPS = 5 -MINIMUM_UPDATE_INTERVAL = 5 # sec +VERSION = __config.get('version', __packaged_version()) -FONT_SIZE = 10 # 8 -STANDARD_FONT = QtGui.QFont("Luxi Sans", FONT_SIZE) -STANDARD_ROW_HEIGHT = 16 # 14 +STARTUP_NOTICE_DATE = __config.get('startup_notice.date') +STARTUP_NOTICE_MSG = __config.get('startup_notice.msg') -MEMORY_WARNING_LEVEL = 5242880 +JOB_UPDATE_DELAY = __config.get('refresh.job_update_delay') +LAYER_UPDATE_DELAY = __config.get('refresh.layer_update_delay') +FRAME_UPDATE_DELAY = __config.get('refresh.frame_update_delay') +HOST_UPDATE_DELAY = __config.get('refresh.host_update_delay') +AFTER_ACTION_UPDATE_DELAY = __config.get('refresh.after_action_update_delay') +MINIMUM_UPDATE_INTERVAL = __config.get('refresh.min_update_interval') // 1000 -RESOURCE_PATH = os.path.dirname(__file__) + "/images" -DEFAULT_INI_PATH = os.getenv('CUEGUI_DEFAULT_INI_PATH', os.path.dirname(__file__) + '/config') +FONT_FAMILY = __config.get('style.font.family') +FONT_SIZE = __config.get('style.font.size') +STANDARD_FONT = QtGui.QFont(FONT_FAMILY, FONT_SIZE) -DEFAULT_PLUGIN_PATHS = [os.path.dirname(__file__) + "/plugins"] +RESOURCE_PATH = __config.get('paths.resources') +if not os.path.isabs(RESOURCE_PATH): + RESOURCE_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), RESOURCE_PATH)) -LOGGER_FORMAT = "%(levelname)-9s %(module)-10s %(message)s" -LOGGER_LEVEL = "WARNING" +CONFIG_PATH = __config.get('paths.config') +if not os.path.isabs(CONFIG_PATH): + CONFIG_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), CONFIG_PATH)) -EMAIL_SUBJECT_PREFIX = "cuemail: please check " -EMAIL_BODY_PREFIX = "Your PSTs request that you check " -EMAIL_BODY_SUFFIX = "\n\n" -EMAIL_DOMAIN = "" +DEFAULT_INI_PATH = os.getenv('CUEGUI_DEFAULT_INI_PATH', __config.get('paths.default_ini_path')) +if not os.path.isabs(DEFAULT_INI_PATH): + DEFAULT_INI_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), DEFAULT_INI_PATH)) -GITHUB_CREATE_ISSUE_URL = 'https://github.com/AcademySoftwareFoundation/OpenCue/issues/new' -URL_USERGUIDE = "https://www.opencue.io/docs/" -URL_SUGGESTION = "%s?labels=enhancement&template=enhancement.md" % GITHUB_CREATE_ISSUE_URL -URL_BUG = "%s?labels=bug&template=bug_report.md" % GITHUB_CREATE_ISSUE_URL +DEFAULT_PLUGIN_PATHS = __config.get('paths.plugins') +for i, path in enumerate(DEFAULT_PLUGIN_PATHS): + if not os.path.isabs(path): + DEFAULT_PLUGIN_PATHS[i] = os.path.abspath(os.path.join(os.path.dirname(__file__), path)) -if platform.system() == "Windows": - DEFAULT_EDITOR = "notepad" +LOGGER_FORMAT = __config.get('logger.format') +LOGGER_LEVEL = __config.get('logger.level') + +EMAIL_SUBJECT_PREFIX = __config.get('email.subject_prefix') +EMAIL_BODY_PREFIX = __config.get('email.body_prefix') +EMAIL_BODY_SUFFIX = __config.get('email.body_suffix') +EMAIL_DOMAIN = __config.get('email.domain') + +GITHUB_CREATE_ISSUE_URL = __config.get('links.issue.create') +URL_USERGUIDE = __config.get('links.user_guide') +URL_SUGGESTION = GITHUB_CREATE_ISSUE_URL + __config.get('links.issue.suggestion') +URL_BUG = GITHUB_CREATE_ISSUE_URL + __config.get('links.issue.bug') + +if platform.system() == 'Windows': + DEFAULT_EDITOR = __config.get('editor.windows') +elif platform.system() == 'Darwin': + DEFAULT_EDITOR = __config.get('editor.mac') else: - DEFAULT_EDITOR = "gview -R -m -M -U %s/gvimrc +" % DEFAULT_INI_PATH + DEFAULT_EDITOR = __config.get('editor.linux') +DEFAULT_EDITOR = DEFAULT_EDITOR.format(config_path=CONFIG_PATH) + +LOG_ROOT_OS = __config.get('render_logs.root') + +ALLOWED_TAGS = tuple(__config.get('allowed_tags')) + +DARK_STYLE_SHEET = os.path.join(CONFIG_PATH, __config.get('style.style_sheet')) +COLOR_THEME = __config.get('style.color_theme') +__bg_colors = __config.get('style.colors.background') +COLOR_USER_1 = QtGui.QColor(*__bg_colors[0]) +COLOR_USER_2 = QtGui.QColor(*__bg_colors[1]) +COLOR_USER_3 = QtGui.QColor(*__bg_colors[2]) +COLOR_USER_4 = QtGui.QColor(*__bg_colors[3]) + +__frame_colors = __config.get('style.colors.frame_state') +RGB_FRAME_STATE = { + opencue.api.job_pb2.DEAD: QtGui.QColor(*__frame_colors.get('DEAD')), + opencue.api.job_pb2.DEPEND: QtGui.QColor(*__frame_colors.get('DEPEND')), + opencue.api.job_pb2.EATEN: QtGui.QColor(*__frame_colors.get('EATEN')), + opencue.api.job_pb2.RUNNING: QtGui.QColor(*__frame_colors.get('RUNNING')), + opencue.api.job_pb2.SETUP: QtGui.QColor(*__frame_colors.get('SETUP')), + opencue.api.job_pb2.SUCCEEDED: QtGui.QColor(*__frame_colors.get('SUCCEEDED')), + opencue.api.job_pb2.WAITING: QtGui.QColor(*__frame_colors.get('WAITING')), + opencue.api.job_pb2.CHECKPOINT: QtGui.QColor(*__frame_colors.get('CHECKPOINT')), +} -EMPTY_INDEX = QtCore.QModelIndex() +MEMORY_WARNING_LEVEL = __config.get('memory_warning_level') -QVARIANT_CENTER = QtCore.Qt.AlignCenter -QVARIANT_RIGHT = QtCore.Qt.AlignRight -QVARIANT_NULL = None -QVARIANT_BLACK = QtGui.QColor(QtCore.Qt.black) -QVARIANT_GREY = QtGui.QColor(QtCore.Qt.gray) - -ALLOWED_TAGS = ("general", "desktop", "playblast", "util", "preprocess", "wan", "cuda", "splathw", - 'naiad', 'massive') - -RGB_FRAME_STATE = {opencue.api.job_pb2.DEAD: QtGui.QColor(255, 0, 0), - opencue.api.job_pb2.DEPEND: QtGui.QColor(160, 32, 240), - opencue.api.job_pb2.EATEN: QtGui.QColor(150, 0, 0), - opencue.api.job_pb2.RUNNING: QtGui.QColor(200, 200, 55), - opencue.api.job_pb2.SETUP: QtGui.QColor(160, 32, 240), - opencue.api.job_pb2.SUCCEEDED: QtGui.QColor(55, 200, 55), - opencue.api.job_pb2.WAITING: QtGui.QColor(135, 207, 235), - opencue.api.job_pb2.CHECKPOINT: QtGui.QColor(61, 98, 247)} -QVARIANT_FRAME_STATE = \ - dict((key, RGB_FRAME_STATE[key]) for key in list(RGB_FRAME_STATE.keys())) +LOG_HIGHLIGHT_ERROR = __config.get('render_logs.highlight.error') +LOG_HIGHLIGHT_WARN = __config.get('render_logs.highlight.warning') +LOG_HIGHLIGHT_INFO = __config.get('render_logs.highlight.info') + +RESOURCE_LIMITS = __config.get('resources') TYPE_JOB = QtWidgets.QTreeWidgetItem.UserType + 1 TYPE_LAYER = QtWidgets.QTreeWidgetItem.UserType + 2 @@ -120,27 +194,7 @@ TYPE_TASK = QtWidgets.QTreeWidgetItem.UserType + 15 TYPE_LIMIT = QtWidgets.QTreeWidgetItem.UserType + 16 -COLUMN_INFO_DISPLAY = 2 - -DARK_STYLE_SHEET = os.path.join(DEFAULT_INI_PATH, "darkpalette.qss") -COLOR_THEME = "plastique" -COLOR_USER_1 = QtGui.QColor(50, 50, 100) -COLOR_USER_2 = QtGui.QColor(100, 100, 50) -COLOR_USER_3 = QtGui.QColor(0, 50, 0) -COLOR_USER_4 = QtGui.QColor(50, 30, 0) - +QVARIANT_NULL = None QT_MAX_INT = 2147483647 -LOG_HIGHLIGHT_ERROR = [ - 'error', 'aborted', 'fatal', 'failed', 'killed', 'command not found', - 'no licenses could be found', 'killMessage'] -LOG_HIGHLIGHT_WARN = ['warning', 'not found'] -LOG_HIGHLIGHT_INFO = ['info:', 'rqd cmd:'] - - -LOG_ROOT_OS = { - "rhel7": "/shots", - "linux": "/shots", - "windows": "S:", - "mac": "/Users/shots" -} +COLUMN_INFO_DISPLAY = 2 diff --git a/cuegui/cuegui/Config.py b/cuegui/cuegui/Layout.py similarity index 94% rename from cuegui/cuegui/Config.py rename to cuegui/cuegui/Layout.py index 2c80f8492..65bb9c080 100644 --- a/cuegui/cuegui/Config.py +++ b/cuegui/cuegui/Layout.py @@ -13,7 +13,7 @@ # limitations under the License. -"""Functions for loading application state and settings from disk.""" +"""Functions for loading application layout and other state from disk.""" from __future__ import print_function from __future__ import division @@ -39,8 +39,7 @@ def startup(app_name): :return: settings object containing the loaded settings :rtype: QtCore.QSettings """ - # read saved config from disk - # copy default config + # E.g. ~/.config/.cuecommander/config.ini config_path = "/.%s/config" % app_name.lower() settings = QtCore.QSettings(QtCore.QSettings.IniFormat, QtCore.QSettings.UserScope, config_path) logger.info('Reading config file from %s', settings.fileName()) diff --git a/cuegui/cuegui/Main.py b/cuegui/cuegui/Main.py index 7712a4cdb..8e5299ef9 100644 --- a/cuegui/cuegui/Main.py +++ b/cuegui/cuegui/Main.py @@ -25,7 +25,7 @@ from PySide2 import QtGui import cuegui -import cuegui.Config +import cuegui.Layout import cuegui.Constants import cuegui.Logger import cuegui.MainWindow @@ -69,7 +69,7 @@ def startup(app_name, app_version, argv): app.threadpool = cuegui.ThreadPool.ThreadPool(3, parent=app) - settings = cuegui.Config.startup(app_name) + settings = cuegui.Layout.startup(app_name) app.settings = settings cuegui.Style.init() diff --git a/cuegui/cuegui/Redirect.py b/cuegui/cuegui/Redirect.py index ae46814f2..43406793c 100644 --- a/cuegui/cuegui/Redirect.py +++ b/cuegui/cuegui/Redirect.py @@ -543,7 +543,7 @@ def __isBurstSafe(self, alloc, procs, show): burst target show burst and the number of cores being redirected. If there's a number of cores that may not be possible to pick up by the target show, that number should be lower than the threshold set in the - cue_resources config. + cuegui.yaml `resources` config. @param alloc: The name of the allocation for the cores @type alloc: str diff --git a/cuegui/cuegui/Utils.py b/cuegui/cuegui/Utils.py index 07cb5c025..957f91908 100644 --- a/cuegui/cuegui/Utils.py +++ b/cuegui/cuegui/Utils.py @@ -36,8 +36,6 @@ from PySide2 import QtGui from PySide2 import QtWidgets import six -import yaml -from yaml.scanner import ScannerError import opencue import opencue.wrappers.group @@ -389,26 +387,16 @@ def memoryToString(kmem, unit=None): return "%.01fG" % (float(kmem) / pow(k, 2)) -def getResourceConfig(path=None): +def getResourceConfig(): """Reads the given yaml file and returns the entries as a dictionary. If no config path is given, the default resources config will be read If the given path does not exist, a warning will be printed and an empty dictionary will be returned - @param path: The path for the yaml file to read - @type path: str - @return: The entries in the given yaml file + @return: Resource config settings @rtype: dict """ - config = {} - if not path: - path = '{}/cue_resources.yaml'.format(cuegui.Constants.DEFAULT_INI_PATH) - try: - with open(path, 'r') as fileObject: - config = yaml.load(fileObject, Loader=yaml.SafeLoader) - except (IOError, ScannerError) as e: - print('WARNING: Could not read config file %s: %s' % (path, e)) - return config + return cuegui.Constants.RESOURCE_LIMITS ################################################################################ diff --git a/cuegui/cuegui/config/cue_resources.yaml b/cuegui/cuegui/config/cue_resources.yaml deleted file mode 100644 index 501b6aff4..000000000 --- a/cuegui/cuegui/config/cue_resources.yaml +++ /dev/null @@ -1,28 +0,0 @@ - - -# Host Specs: -# Use this section to set the max cores and max memory based on the available -# hardware. -# These values are used by: -# - layer-properties -# - redirect plugin -# - service properties -max_cores: 32 -max_memory: 128 - -max_gpus: 8 -max_gpu_memory: 128 - - -# Redirect Plugin maximum allowed core-hour cutoff. -# Users will not be able to search for procs with frames that have been -# already used more than this many core-hours: -max_proc_hour_cutoff: 30 - -# Redirect plugin wasted cores threshold: -# When redirecting, and the target show is at or very close to subscription -# burst, killing frames will free up cores that may not be picked up by the -# target job. The plugin will warn the user if the number of potentially lost -# cores is higher that this threshold. To disable this warning, set the -# threshold to -1 -redirect_wasted_cores_threshold: 100 diff --git a/cuegui/cuegui/config/cuegui.yaml b/cuegui/cuegui/config/cuegui.yaml new file mode 100644 index 000000000..38f492cb7 --- /dev/null +++ b/cuegui/cuegui/config/cuegui.yaml @@ -0,0 +1,112 @@ +# Default CueGUI config file + +logger.format: '%(levelname)-9s %(module)-10s %(message)s' +logger.level: 'WARNING' + +# Path for static resources like images/icons. +paths.resources: './images' +# Path for various config files. +paths.config: './config' +# Path for the default application layout .ini file. If users do not have a layout stored +# in their local filesystem, the layout stored here will be copied. This value can also +# be set via the CUEGUI_DEFAULT_INI_PATH environment variable. +paths.default_ini_path: './config' +# Paths for CueGUI plugins. +paths.plugins: ['./plugins'] + +# How often the UI will refresh its contents. All values in milliseconds. +refresh.job_update_delay: 10000 +refresh.layer_update_delay: 10000 +refresh.frame_update_delay: 10000 +refresh.host_update_delay: 20000 +refresh.after_action_update_delay: 1000 +refresh.min_update_interval: 5000 + +# Log roots used by various operating systems. Used for remapping paths so logs produced on +# one platform will be accessible locally. +render_logs.root: + windows: 'S:' + mac: '/Users/shots' + darwin: '/Users/shots' + linux: '/shots' + rhel7: '/shots' +# Substrings which, when found in render logs, will cause that line to be highlighted. +render_logs.highlight.error: [ + 'error', 'aborted', 'fatal', 'failed', 'killed', 'command not found', + 'no licenses could be found', 'killMessage'] +render_logs.highlight.warning: ['warning', 'not found'] +render_logs.highlight.info: ['info:', 'rqd cmd:'] + +# File should be stored in paths.config. +style.style_sheet: 'darkpalette.qss' +style.font.family: 'Luxi Sans' +style.font.size: 10 +style.color_theme: 'plastique' +# RGB values. +style.colors.background: [ + [50, 50, 100], + [100, 100, 50], + [0, 50, 0], + [50, 30, 0], +] +style.colors.frame_state: + DEAD: [255, 0, 0] + DEPEND: [160, 32, 240] + EATEN: [150, 0, 0] + RUNNING: [200, 200, 55] + SETUP: [160, 32, 240] + SUCCEEDED: [55, 200, 55] + WAITING: [135, 207, 235] + CHECKPOINT: [61, 98, 247] + +# Default editor to use for viewing log files. +editor.windows: 'notepad' +editor.mac: 'open -t' +editor.linux: 'gview -R -m -M -U {config_path}/gvimrc +' + +resources: + # The max cores and max memory based on the available hardware. + # These values are used by: + # - layer-properties + # - redirect plugin + # - service properties + max_cores: 32 + max_memory: 128 + max_gpus: 8 + max_gpu_memory: 128 + # Redirect Plugin maximum allowed core-hour cutoff. + # Users will not be able to search for procs with frames that have been + # already used more than this many core-hours: + max_proc_hour_cutoff: 30 + # Redirect plugin wasted cores threshold: + # When redirecting, and the target show is at or very close to subscription + # burst, killing frames will free up cores that may not be picked up by the + # target job. The plugin will warn the user if the number of potentially lost + # cores is higher that this threshold. To disable this warning, set the + # threshold to -1. + redirect_wasted_cores_threshold: 100 + +links.user_guide: 'https://www.opencue.io/docs/' +links.issue.create: 'https://github.com/AcademySoftwareFoundation/OpenCue/issues/new' +# Appended to `links.issue.create`. +links.issue.suggestion: '?labels=enhancement&template=enhancement.md' +# Appended to `links.issue.create`. +links.issue.bug: '?labels=bug&template=bug_report.md' + +# List of tags to be used when viewing or editing tags. +allowed_tags: ['general', 'desktop', 'playblast', 'util', 'preprocess', 'wan', 'cuda', 'splathw', + 'naiad', 'massive'] + +email.subject_prefix: 'cuemail: please check ' +email.body_prefix: 'Your PSTs request that you check ' +email.body_suffix: "\n\n" +email.domain: '' + +# Unix epoch timestamp. If the user last viewed the startup notice before this time, the +# notice will be shown. +startup_notice.date: 0 +# Notice message. +startup_notice.msg: '' + +# Memory usage above this level will be displayed in a different color. +memory_warning_level: 5242880 diff --git a/cuegui/tests/Constants_tests.py b/cuegui/tests/Constants_tests.py new file mode 100644 index 000000000..9466dcece --- /dev/null +++ b/cuegui/tests/Constants_tests.py @@ -0,0 +1,166 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Tests for cuegui.Constants""" + + +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import + +import importlib +import os + +import mock +import pyfakefs.fake_filesystem_unittest +from PySide2 import QtGui + +import opencue +import cuegui.Constants + + +CONFIG_YAML = ''' +unused_setting: some value +version: 98.707.68 +refresh.job_update_delay: 30000 + +logger.level: INFO +''' + + +class ConstantsTests(pyfakefs.fake_filesystem_unittest.TestCase): + def setUp(self): + self.setUpPyfakefs() + self.fs.add_real_file( + os.path.join(os.path.dirname(cuegui.__file__), 'config', 'cuegui.yaml'), read_only=True) + if 'CUEGUI_CONFIG_FILE' in os.environ: + del os.environ['CUEGUI_CONFIG_FILE'] + + def test__should_load_user_config_from_env_var(self): + config_file_path = '/path/to/config.yaml' + self.fs.create_file(config_file_path, contents=CONFIG_YAML) + os.environ['CUEGUI_CONFIG_FILE'] = config_file_path + + result = importlib.reload(cuegui.Constants) + + self.assertEqual('98.707.68', result.VERSION) + self.assertEqual(30000, result.JOB_UPDATE_DELAY) + self.assertEqual(10000, result.LAYER_UPDATE_DELAY) + + @mock.patch('platform.system', new=mock.Mock(return_value='Linux')) + @mock.patch('os.path.expanduser', new=mock.Mock(return_value='/home/username')) + def test__should_load_user_config_from_user_profile(self): + config_file_path = '/home/username/.config/opencue/cuegui.yaml' + self.fs.create_file(config_file_path, contents=CONFIG_YAML) + + result = importlib.reload(cuegui.Constants) + + self.assertEqual('98.707.68', result.VERSION) + self.assertEqual(30000, result.JOB_UPDATE_DELAY) + self.assertEqual(10000, result.LAYER_UPDATE_DELAY) + + @mock.patch('platform.system', new=mock.Mock(return_value='Linux')) + def test__should_use_default_values(self): + result = importlib.reload(cuegui.Constants) + + self.assertNotEqual('98.707.68', result.VERSION) + self.assertEqual(0, result.STARTUP_NOTICE_DATE) + self.assertEqual('', result.STARTUP_NOTICE_MSG) + self.assertEqual(10000, result.JOB_UPDATE_DELAY) + self.assertEqual(10000, result.LAYER_UPDATE_DELAY) + self.assertEqual(10000, result.FRAME_UPDATE_DELAY) + self.assertEqual(20000, result.HOST_UPDATE_DELAY) + self.assertEqual(1000, result.AFTER_ACTION_UPDATE_DELAY) + self.assertEqual(5, result.MINIMUM_UPDATE_INTERVAL) + self.assertEqual('Luxi Sans', result.FONT_FAMILY) + self.assertEqual(10, result.FONT_SIZE) + self.assertEqual( + os.path.join(os.path.dirname(cuegui.__file__), 'images'), result.RESOURCE_PATH) + self.assertEqual( + os.path.join(os.path.dirname(cuegui.__file__), 'config'), result.CONFIG_PATH) + self.assertEqual( + os.path.join(os.path.dirname(cuegui.__file__), 'config'), result.DEFAULT_INI_PATH) + self.assertEqual( + [os.path.join(os.path.dirname(cuegui.__file__), 'plugins')], + result.DEFAULT_PLUGIN_PATHS) + self.assertEqual('%(levelname)-9s %(module)-10s %(message)s', result.LOGGER_FORMAT) + self.assertEqual('WARNING', result.LOGGER_LEVEL) + self.assertEqual('cuemail: please check ', result.EMAIL_SUBJECT_PREFIX) + self.assertEqual('Your PSTs request that you check ', result.EMAIL_BODY_PREFIX) + self.assertEqual('\n\n', result.EMAIL_BODY_SUFFIX) + self.assertEqual('', result.EMAIL_DOMAIN) + self.assertEqual( + 'https://github.com/AcademySoftwareFoundation/OpenCue/issues/new', + result.GITHUB_CREATE_ISSUE_URL) + self.assertEqual('https://www.opencue.io/docs/', result.URL_USERGUIDE) + self.assertEqual( + 'https://github.com/AcademySoftwareFoundation/OpenCue/issues/new' + '?labels=enhancement&template=enhancement.md', result.URL_SUGGESTION) + self.assertEqual( + 'https://github.com/AcademySoftwareFoundation/OpenCue/issues/new' + '?labels=bug&template=bug_report.md', result.URL_BUG) + self.assertEqual( + 'gview -R -m -M -U %s +' % os.path.join( + os.path.dirname(cuegui.__file__), 'config', 'gvimrc'), + result.DEFAULT_EDITOR) + self.assertEqual({ + 'rhel7': '/shots', + 'linux': '/shots', + 'windows': 'S:', + 'mac': '/Users/shots', + 'darwin': '/Users/shots', + }, result.LOG_ROOT_OS) + self.assertEqual(( + 'general', 'desktop', 'playblast', 'util', 'preprocess', 'wan', 'cuda', 'splathw', + 'naiad', 'massive'), result.ALLOWED_TAGS) + self.assertEqual( + os.path.join(os.path.dirname(cuegui.__file__), 'config', 'darkpalette.qss'), + result.DARK_STYLE_SHEET) + self.assertEqual('plastique', result.COLOR_THEME) + self.assertEqual(QtGui.QColor(50, 50, 100), result.COLOR_USER_1) + self.assertEqual(QtGui.QColor(100, 100, 50), result.COLOR_USER_2) + self.assertEqual(QtGui.QColor(0, 50, 0), result.COLOR_USER_3) + self.assertEqual(QtGui.QColor(50, 30, 00), result.COLOR_USER_4) + self.assertEqual({ + opencue.api.job_pb2.DEAD: QtGui.QColor(255, 0, 0), + opencue.api.job_pb2.DEPEND: QtGui.QColor(160, 32, 240), + opencue.api.job_pb2.EATEN: QtGui.QColor(150, 0, 0), + opencue.api.job_pb2.RUNNING: QtGui.QColor(200, 200, 55), + opencue.api.job_pb2.SETUP: QtGui.QColor(160, 32, 240), + opencue.api.job_pb2.SUCCEEDED: QtGui.QColor(55, 200, 55), + opencue.api.job_pb2.WAITING: QtGui.QColor(135, 207, 235), + opencue.api.job_pb2.CHECKPOINT: QtGui.QColor(61, 98, 247), + }, result.RGB_FRAME_STATE) + self.assertEqual(5242880, result.MEMORY_WARNING_LEVEL) + self.assertEqual( + ['error', 'aborted', 'fatal', 'failed', 'killed', 'command not found', + 'no licenses could be found', 'killMessage'], result.LOG_HIGHLIGHT_ERROR) + self.assertEqual(['warning', 'not found'], result.LOG_HIGHLIGHT_WARN) + self.assertEqual(['info:', 'rqd cmd:'], result.LOG_HIGHLIGHT_INFO) + self.assertEqual(2147483647, result.QT_MAX_INT) + self.assertEqual({ + 'max_cores': 32, + 'max_gpu_memory': 128, + 'max_gpus': 8, + 'max_memory': 128, + 'max_proc_hour_cutoff': 30, + 'redirect_wasted_cores_threshold': 100, + }, result.RESOURCE_LIMITS) + + @mock.patch('platform.system', new=mock.Mock(return_value='Darwin')) + def test__should_use_mac_editor(self): + result = importlib.reload(cuegui.Constants) + + self.assertEqual('open -t', result.DEFAULT_EDITOR) diff --git a/cuegui/tests/Config_tests.py b/cuegui/tests/Layout_tests.py similarity index 86% rename from cuegui/tests/Config_tests.py rename to cuegui/tests/Layout_tests.py index 61ab2c193..c291c3b1e 100644 --- a/cuegui/tests/Config_tests.py +++ b/cuegui/tests/Layout_tests.py @@ -13,7 +13,7 @@ # limitations under the License. -"""Tests for cuegui.Config""" +"""Tests for cuegui.Layout""" from __future__ import print_function @@ -27,7 +27,7 @@ from PySide2 import QtCore -import cuegui.Config +import cuegui.Layout CONFIG_INI = ''' @@ -50,7 +50,7 @@ ''' -class ConfigTests(unittest.TestCase): +class LayoutTests(unittest.TestCase): def setUp(self): self.config_dir = tempfile.mkdtemp() QtCore.QSettings.setPath( @@ -59,34 +59,34 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.config_dir) - def test__should_load_user_config(self): + def test__should_load_user_layout(self): app_name = 'arbitraryapp' config_file_path = os.path.join(self.config_dir, '.%s' % app_name, 'config.ini') os.mkdir(os.path.dirname(config_file_path)) with open(config_file_path, 'w') as fp: fp.write(CONFIG_INI) - settings = cuegui.Config.startup(app_name) + settings = cuegui.Layout.startup(app_name) self.assertEqual('0.14', settings.value('Version')) self.assertEqual('true', settings.value('CueCommander/Open')) self.assertEqual('CustomWindowTitle', settings.value('CueCommander/Title')) self.assertEqual('arbitrary-value', settings.value('CueCommander/OtherAttr')) - def test__should_load_default_config(self): - settings = cuegui.Config.startup('CueCommander') + def test__should_load_default_layout(self): + settings = cuegui.Layout.startup('CueCommander') self.assertEqual('false', settings.value('CueCommander/Open')) self.assertEqual('CueCommander', settings.value('CueCommander/Title')) self.assertFalse(settings.value('CueCommander/OtherAttr', False)) - def test__should_restore_default_config(self): + def test__should_restore_default_layout(self): config_file_path = os.path.join(self.config_dir, '.cuecommander', 'config.ini') os.mkdir(os.path.dirname(config_file_path)) with open(config_file_path, 'w') as fp: fp.write(CONFIG_WITH_RESTORE_FLAG) - settings = cuegui.Config.startup('CueCommander') + settings = cuegui.Layout.startup('CueCommander') self.assertEqual('false', settings.value('CueCommander/Open')) self.assertEqual('CueCommander', settings.value('CueCommander/Title')) diff --git a/cuegui/tests/Utils_tests.py b/cuegui/tests/Utils_tests.py index dddf46423..e2f4a69b0 100644 --- a/cuegui/tests/Utils_tests.py +++ b/cuegui/tests/Utils_tests.py @@ -69,6 +69,18 @@ def test_shouldSwallowExceptionAndReturnNone(self): self.assertIsNone(cuegui.Utils.findJob(jobName)) + def test_shouldReturnResourceLimitsFromYaml(self): + result = cuegui.Utils.getResourceConfig() + + self.assertEqual({ + 'max_cores': 32, + 'max_gpu_memory': 128, + 'max_gpus': 8, + 'max_memory': 128, + 'max_proc_hour_cutoff': 30, + 'redirect_wasted_cores_threshold': 100, + }, result) + if __name__ == '__main__': unittest.main() From bb745588f5d010d30a11aaa0c3a99be531f12a40 Mon Sep 17 00:00:00 2001 From: romainf-ubi <117918548+romainf-ubi@users.noreply.github.com> Date: Thu, 26 Jan 2023 19:08:46 -0500 Subject: [PATCH 245/277] [rqd] Add new config option RQD_USE_PATH_ENV_VAR. (#1241) --- rqd/rqd/__main__.py | 21 ++++++++++++--------- rqd/rqd/rqconstants.py | 8 +++++++- rqd/rqd/rqmachine.py | 2 ++ 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/rqd/rqd/__main__.py b/rqd/rqd/__main__.py index 32e6dd38b..605f72f22 100755 --- a/rqd/rqd/__main__.py +++ b/rqd/rqd/__main__.py @@ -28,7 +28,7 @@ Optional configuration file: ---------------------------- -in /etc/rqd3/rqd3.conf: +In /etc/opencue/rqd.conf (on Linux) or %LOCALAPPDATA%/OpenCue/rqd.conf (on Windows): [Override] OVERRIDE_CORES = 2 OVERRIDE_PROCS = 3 @@ -89,14 +89,17 @@ def setupLogging(): def usage(): """Prints command line syntax""" - s = sys.stderr - print("SYNOPSIS", file=s) - print(" ", sys.argv[0], "[options]\n", file=s) - print(" -d | --daemon => Run as daemon", file=s) - print(" --nimbyoff => Disables nimby activation", file=s) - print(" -c => Provide an alternate config file", file=s) - print(" Defaults to /etc/rqd3/rqd3.conf", file=s) - print(" Config file is optional", file=s) + usage_msg = f"""SYNOPSIS + {sys.argv[0]} [options] + + -d | --daemon => Run as daemon + --nimbyoff => Disables nimby activation + -c => Provide an alternate config file + On Linux: defaults to /etc/opencue/rqd.conf + On Windows: Defaults to %LOCALAPPDATA%/OpenCue/rqd.conf + Config file is optional +""" + print(usage_msg, file=sys.stderr) def main(): diff --git a/rqd/rqd/rqconstants.py b/rqd/rqd/rqconstants.py index 0d6968d27..ad82b5daf 100644 --- a/rqd/rqd/rqconstants.py +++ b/rqd/rqd/rqconstants.py @@ -66,6 +66,10 @@ RQD_RETRY_CRITICAL_REPORT_DELAY = 30 RQD_USE_IP_AS_HOSTNAME = True RQD_USE_IPV6_AS_HOSTNAME = False + +# Use the PATH environment variable from the RQD host. +RQD_USE_PATH_ENV_VAR = False + RQD_BECOME_JOB_USER = True RQD_CREATE_USER_IF_NOT_EXISTS = True RQD_TAGS = '' @@ -111,7 +115,7 @@ SYS_HERTZ = os.sysconf('SC_CLK_TCK') if platform.system() == 'Windows': - CONFIG_FILE = os.path.expandvars('$LOCALAPPDATA/OpenCue/rqd.conf') + CONFIG_FILE = os.path.expandvars('%LOCALAPPDATA%/OpenCue/rqd.conf') else: CONFIG_FILE = '/etc/opencue/rqd.conf' @@ -177,6 +181,8 @@ RQD_USE_IP_AS_HOSTNAME = config.getboolean(__section, "RQD_USE_IP_AS_HOSTNAME") if config.has_option(__section, "RQD_USE_IPV6_AS_HOSTNAME"): RQD_USE_IPV6_AS_HOSTNAME = config.getboolean(__section, "RQD_USE_IPV6_AS_HOSTNAME") + if config.has_option(__section, "RQD_USE_PATH_ENV_VAR"): + RQD_USE_PATH_ENV_VAR = config.getboolean(__section, "RQD_USE_PATH_ENV_VAR") if config.has_option(__section, "RQD_BECOME_JOB_USER"): RQD_BECOME_JOB_USER = config.getboolean(__section, "RQD_BECOME_JOB_USER") if config.has_option(__section, "RQD_TAGS"): diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 27295ed92..2fc5a32ce 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -507,6 +507,8 @@ def getHostname(self): @rqd.rqutil.Memoize def getPathEnv(self): """Returns the correct path environment for the given machine""" + if rqd.rqconstants.RQD_USE_PATH_ENV_VAR: + return os.getenv('PATH') if platform.system() == 'Linux': return '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' if platform.system() == 'Windows': From ce2253118b6808469c89aa54c227dd0e2df3171a Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 6 Feb 2023 13:10:45 -0500 Subject: [PATCH 246/277] TSC meeting notes from Feb 1 meeting. (#1256) --- tsc/meetings/2023-02-01.md | 42 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tsc/meetings/2023-02-01.md diff --git a/tsc/meetings/2023-02-01.md b/tsc/meetings/2023-02-01.md new file mode 100644 index 000000000..88ef82f5b --- /dev/null +++ b/tsc/meetings/2023-02-01.md @@ -0,0 +1,42 @@ +# OpenCue TSC Meeting Notes 1 Feb 2023 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Move outline.cfg into outline module + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1252 + * What to do with bin/ and wrappers/ code? E.g. pycuerun. Packaged with rqd? + * What do these tools do? + * pycuerun is a tool for sending jobs to Cuebot. This should be packaged with pyoutline, + ideally as a console script / entrypoint. + * wrappers are used on the RQD side. These should probably be packaged with RQD somehow. + * Conclusion: we'll need different approaches for packaging these, needs some more research. +* Integration tests + * New PR to run a job and verify it + finishes: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1253 + * Uses a pyoutline script to launch a job and a pycue script to wait for it to finish. + * Discovered outline.cfg issue above! + * Almost ready for review, waiting on pyoutline fix to be merged. +* Config guide + * CueGUI YAML config merged. + * PR with new config guide: https://github.com/AcademySoftwareFoundation/opencue.io/pull/274 + * Preview: + https://deploy-preview-274--elated-haibt-1b47ff.netlify.app/docs/other-guides/configuring-opencue/ +* PyPI + * Brian now doing some tests, cleaning up the pycue setup.py. + * Need to clean up dependency list in setup.py. This duplicates some work from requirements.txt, + but serves a different purpose, and the version restrictions should be a little looser. Have + to go one at a time to make a decision. +* RQD systemd changes + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1251 + * To be reviewed soon. + * Needs to be built into Docker image, CI pipelines. +* PySide6 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1238 + * Review complete, merging soon. +* Migrate stats columns + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1228 + * Good way to roll out destructive changes? + * Any input from PG admins? + * Let's check with Diego next time, but current change looks fine. From 1c5dce48642c5c05f00e361ae9f947c221f21815 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Fri, 10 Feb 2023 12:46:50 -0500 Subject: [PATCH 247/277] [cuegui] Use the QtPy library instead of PySide directly. (#1238) --- .github/workflows/testing-pipeline.yml | 9 ++ VERSION.in | 2 +- ci/run_gui_test.sh | 8 +- ci/test_pyside6.sh | 36 ++++++++ cuegui/cuegui/AbstractDialog.py | 4 +- cuegui/cuegui/AbstractDockWidget.py | 4 +- cuegui/cuegui/AbstractTreeWidget.py | 6 +- cuegui/cuegui/AbstractWidgetItem.py | 4 +- cuegui/cuegui/Action.py | 4 +- cuegui/cuegui/App.py | 4 +- cuegui/cuegui/Comments.py | 4 +- cuegui/cuegui/ConfirmationDialog.py | 6 +- cuegui/cuegui/Constants.py | 4 +- cuegui/cuegui/CreateShowDialog.py | 4 +- cuegui/cuegui/CreatorDialog.py | 2 +- cuegui/cuegui/CueJobMonitorTree.py | 6 +- cuegui/cuegui/CueStateBarWidget.py | 6 +- cuegui/cuegui/DarkPalette.py | 4 +- cuegui/cuegui/DependDialog.py | 4 +- cuegui/cuegui/DependMonitorTree.py | 2 +- cuegui/cuegui/DependWizard.py | 4 +- cuegui/cuegui/EmailDialog.py | 6 +- cuegui/cuegui/FilterDialog.py | 8 +- cuegui/cuegui/FrameMonitor.py | 6 +- cuegui/cuegui/FrameMonitorTree.py | 6 +- cuegui/cuegui/FrameRangeSelection.py | 18 ++-- cuegui/cuegui/GarbageCollector.py | 2 +- cuegui/cuegui/GroupDialog.py | 4 +- cuegui/cuegui/HostMonitor.py | 4 +- cuegui/cuegui/HostMonitorTree.py | 6 +- cuegui/cuegui/ItemDelegate.py | 6 +- cuegui/cuegui/JobMonitorTree.py | 6 +- cuegui/cuegui/LayerDialog.py | 4 +- cuegui/cuegui/LayerMonitorTree.py | 4 +- cuegui/cuegui/Layout.py | 2 +- cuegui/cuegui/LimitSelectionWidget.py | 2 +- cuegui/cuegui/LimitsWidget.py | 4 +- cuegui/cuegui/LocalBooking.py | 4 +- cuegui/cuegui/Main.py | 2 +- cuegui/cuegui/MainWindow.py | 6 +- cuegui/cuegui/MenuActions.py | 4 +- cuegui/cuegui/MiscDialog.py | 2 +- cuegui/cuegui/Plugins.py | 4 +- cuegui/cuegui/PreviewWidget.py | 6 +- cuegui/cuegui/ProcChildren.py | 4 +- cuegui/cuegui/ProcMonitor.py | 4 +- cuegui/cuegui/ProcMonitorTree.py | 4 +- cuegui/cuegui/ProgressDialog.py | 4 +- cuegui/cuegui/Redirect.py | 6 +- cuegui/cuegui/ServiceDialog.py | 4 +- cuegui/cuegui/ShowDialog.py | 4 +- cuegui/cuegui/ShowsWidget.py | 4 +- cuegui/cuegui/SplashWindow.py | 6 +- cuegui/cuegui/Style.py | 2 +- cuegui/cuegui/SubscriptionGraphWidget.py | 4 +- cuegui/cuegui/SubscriptionsWidget.py | 4 +- cuegui/cuegui/TagsWidget.py | 2 +- cuegui/cuegui/TasksDialog.py | 4 +- cuegui/cuegui/TextEditDialog.py | 4 +- cuegui/cuegui/ThreadPool.py | 2 +- cuegui/cuegui/UnbookDialog.py | 4 +- cuegui/cuegui/Utils.py | 6 +- cuegui/cuegui/images/crystal/icons_rcc.py | 2 +- cuegui/cuegui/images/icons_rcc.py | 2 +- cuegui/cuegui/plugins/AllocationsPlugin.py | 2 +- cuegui/cuegui/plugins/AttributesPlugin.py | 4 +- cuegui/cuegui/plugins/LogViewPlugin.py | 8 +- cuegui/cuegui/plugins/MonitorCuePlugin.py | 6 +- cuegui/cuegui/plugins/MonitorHostsPlugin.py | 4 +- .../cuegui/plugins/MonitorJobDetailsPlugin.py | 4 +- cuegui/cuegui/plugins/MonitorJobsPlugin.py | 6 +- cuegui/cuegui/plugins/ShowsPlugin.py | 2 +- cuegui/cuegui/plugins/StuckFramePlugin.py | 6 +- .../plugins/SubscriptionsGraphPlugin.py | 4 +- cuegui/cuegui/plugins/SubscriptionsPlugin.py | 4 +- cuegui/setup.py | 1 + cuegui/tests/CueJobMonitorTree_tests.py | 10 +-- cuegui/tests/DependWizard_tests.py | 12 +-- cuegui/tests/FilterDialog_tests.py | 62 ++++++------- cuegui/tests/FrameMonitorTree_tests.py | 44 ++++----- cuegui/tests/LayerDialog_tests.py | 10 +-- cuegui/tests/Layout_tests.py | 2 +- cuegui/tests/MenuActions_tests.py | 90 +++++++++---------- cuegui/tests/Redirect_tests.py | 6 +- cuegui/tests/UnbookDialog_tests.py | 24 ++--- cuegui/tests/plugins/LogViewPlugin_tests.py | 28 +++--- requirements_gui.txt | 2 + 87 files changed, 357 insertions(+), 303 deletions(-) create mode 100755 ci/test_pyside6.sh diff --git a/.github/workflows/testing-pipeline.yml b/.github/workflows/testing-pipeline.yml index 98e5ca166..7d2b59af9 100644 --- a/.github/workflows/testing-pipeline.yml +++ b/.github/workflows/testing-pipeline.yml @@ -91,6 +91,15 @@ jobs: chown -R aswfuser:aswfgroup . su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser + test_pyside6: + name: Run CueGUI Tests using PySide6 + runs-on: ubuntu-latest + container: almalinux:9 + steps: + - uses: actions/checkout@v2 + - name: Run CueGUI Tests + run: ci/test_pyside6.sh + lint_python: name: Lint Python Code runs-on: ubuntu-latest diff --git a/VERSION.in b/VERSION.in index 5320adc1c..e34629406 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.21 +0.22 diff --git a/ci/run_gui_test.sh b/ci/run_gui_test.sh index b80505bdb..3c7d92a6d 100755 --- a/ci/run_gui_test.sh +++ b/ci/run_gui_test.sh @@ -14,8 +14,14 @@ # > OK # +py="$(command -v python3)" +if [[ -z "$py" ]]; then + py="$(command -v python)" +fi +echo "Using Python binary ${py}" + test_log="/tmp/cuegui_result.log" -PYTHONPATH=pycue xvfb-run -d python cuegui/setup.py test | tee ${test_log} +PYTHONPATH=pycue xvfb-run -d "${py}" cuegui/setup.py test | tee ${test_log} grep -Pz 'Ran \d+ tests in [0-9\.]+s\n\nOK' ${test_log} if [ $? -eq 0 ]; then diff --git a/ci/test_pyside6.sh b/ci/test_pyside6.sh new file mode 100755 index 000000000..05bd4c173 --- /dev/null +++ b/ci/test_pyside6.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Script for testing CueGUI with PySide6. +# +# This script is written to be run within an almalinux environment in the OpenCue +# GitHub Actions environment. See .github/workflows/testing-pipeline.yml. + +set -e + +# Install needed packages. +yum -y install \ + dbus-libs \ + fontconfig \ + gcc \ + libxkbcommon-x11 \ + mesa-libEGL-devel \ + python-devel \ + which \ + xcb-util-keysyms \ + xcb-util-image \ + xcb-util-renderutil \ + xcb-util-wm \ + Xvfb + +# Install Python requirements. +python3 -m pip install --user -r requirements.txt -r requirements_gui.txt +# Replace PySide2 with PySide6. +python3 -m pip uninstall -y PySide2 +python3 -m pip install --user PySide6==6.3.2 + +# Fix compiled proto code for Python 3. +python3 -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto +2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py + +# Run tests. +ci/run_gui_test.sh diff --git a/cuegui/cuegui/AbstractDialog.py b/cuegui/cuegui/AbstractDialog.py index f6d701fe4..2474cf14b 100644 --- a/cuegui/cuegui/AbstractDialog.py +++ b/cuegui/cuegui/AbstractDialog.py @@ -21,8 +21,8 @@ from __future__ import absolute_import from builtins import str -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets class AbstractDialog(QtWidgets.QDialog): diff --git a/cuegui/cuegui/AbstractDockWidget.py b/cuegui/cuegui/AbstractDockWidget.py index 15857d5a2..0697662e3 100644 --- a/cuegui/cuegui/AbstractDockWidget.py +++ b/cuegui/cuegui/AbstractDockWidget.py @@ -22,8 +22,8 @@ from __future__ import print_function from __future__ import division -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.Plugins diff --git a/cuegui/cuegui/AbstractTreeWidget.py b/cuegui/cuegui/AbstractTreeWidget.py index 88e98ad4f..0f4e9500a 100644 --- a/cuegui/cuegui/AbstractTreeWidget.py +++ b/cuegui/cuegui/AbstractTreeWidget.py @@ -26,9 +26,9 @@ from builtins import range import time -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import cuegui.AbstractWidgetItem import cuegui.Constants diff --git a/cuegui/cuegui/AbstractWidgetItem.py b/cuegui/cuegui/AbstractWidgetItem.py index 301a3aa59..8a05699e8 100644 --- a/cuegui/cuegui/AbstractWidgetItem.py +++ b/cuegui/cuegui/AbstractWidgetItem.py @@ -24,8 +24,8 @@ from builtins import str -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.Constants import cuegui.Logger diff --git a/cuegui/cuegui/Action.py b/cuegui/cuegui/Action.py index f9a083e09..0f24c4957 100644 --- a/cuegui/cuegui/Action.py +++ b/cuegui/cuegui/Action.py @@ -20,8 +20,8 @@ from __future__ import print_function from __future__ import division -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtGui +from qtpy import QtWidgets import cuegui.Constants diff --git a/cuegui/cuegui/App.py b/cuegui/cuegui/App.py index e161420fe..8e5409520 100644 --- a/cuegui/cuegui/App.py +++ b/cuegui/cuegui/App.py @@ -14,8 +14,8 @@ """Module for CueGUI's custom QApplication and associated helper functions.""" -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.Exception diff --git a/cuegui/cuegui/Comments.py b/cuegui/cuegui/Comments.py index 628c1b740..c58c7bd79 100644 --- a/cuegui/cuegui/Comments.py +++ b/cuegui/cuegui/Comments.py @@ -23,8 +23,8 @@ from builtins import str import pickle -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.Utils diff --git a/cuegui/cuegui/ConfirmationDialog.py b/cuegui/cuegui/ConfirmationDialog.py index 68f0f7acd..919a19563 100644 --- a/cuegui/cuegui/ConfirmationDialog.py +++ b/cuegui/cuegui/ConfirmationDialog.py @@ -20,9 +20,9 @@ from __future__ import division from __future__ import absolute_import -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets class ConfirmationDialog(QtWidgets.QDialog): diff --git a/cuegui/cuegui/Constants.py b/cuegui/cuegui/Constants.py index 4dcef529d..38f5dbafe 100644 --- a/cuegui/cuegui/Constants.py +++ b/cuegui/cuegui/Constants.py @@ -25,8 +25,8 @@ import os import platform -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtGui +from qtpy import QtWidgets import yaml import opencue diff --git a/cuegui/cuegui/CreateShowDialog.py b/cuegui/cuegui/CreateShowDialog.py index 148bd52d2..36e66fb49 100644 --- a/cuegui/cuegui/CreateShowDialog.py +++ b/cuegui/cuegui/CreateShowDialog.py @@ -21,8 +21,8 @@ from __future__ import division -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/CreatorDialog.py b/cuegui/cuegui/CreatorDialog.py index a28d2f00b..5d2e5e450 100644 --- a/cuegui/cuegui/CreatorDialog.py +++ b/cuegui/cuegui/CreatorDialog.py @@ -23,7 +23,7 @@ from builtins import str from builtins import zip -from PySide2 import QtWidgets +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/CueJobMonitorTree.py b/cuegui/cuegui/CueJobMonitorTree.py index b7f32ebec..2305835b1 100644 --- a/cuegui/cuegui/CueJobMonitorTree.py +++ b/cuegui/cuegui/CueJobMonitorTree.py @@ -25,9 +25,9 @@ from collections import namedtuple import time -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import opencue import opencue.compiled_proto.job_pb2 diff --git a/cuegui/cuegui/CueStateBarWidget.py b/cuegui/cuegui/CueStateBarWidget.py index c35622f17..26b7feb6f 100644 --- a/cuegui/cuegui/CueStateBarWidget.py +++ b/cuegui/cuegui/CueStateBarWidget.py @@ -24,9 +24,9 @@ import time import weakref -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import cuegui.Logger diff --git a/cuegui/cuegui/DarkPalette.py b/cuegui/cuegui/DarkPalette.py index 22b89bf68..fed62996b 100644 --- a/cuegui/cuegui/DarkPalette.py +++ b/cuegui/cuegui/DarkPalette.py @@ -22,8 +22,8 @@ import platform -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtGui +from qtpy import QtWidgets import cuegui.Constants diff --git a/cuegui/cuegui/DependDialog.py b/cuegui/cuegui/DependDialog.py index ce94f526f..613e3da39 100644 --- a/cuegui/cuegui/DependDialog.py +++ b/cuegui/cuegui/DependDialog.py @@ -20,8 +20,8 @@ from __future__ import print_function from __future__ import division -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.DependMonitorTree import cuegui.Logger diff --git a/cuegui/cuegui/DependMonitorTree.py b/cuegui/cuegui/DependMonitorTree.py index fed246a36..8f778280b 100644 --- a/cuegui/cuegui/DependMonitorTree.py +++ b/cuegui/cuegui/DependMonitorTree.py @@ -22,7 +22,7 @@ from builtins import map -from PySide2 import QtWidgets +from qtpy import QtWidgets from opencue.compiled_proto import depend_pb2 import opencue.exception diff --git a/cuegui/cuegui/DependWizard.py b/cuegui/cuegui/DependWizard.py index f328f76d7..b07a5ee46 100644 --- a/cuegui/cuegui/DependWizard.py +++ b/cuegui/cuegui/DependWizard.py @@ -25,8 +25,8 @@ from builtins import range import re -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import FileSequence import opencue diff --git a/cuegui/cuegui/EmailDialog.py b/cuegui/cuegui/EmailDialog.py index f4f83387e..227b12627 100644 --- a/cuegui/cuegui/EmailDialog.py +++ b/cuegui/cuegui/EmailDialog.py @@ -38,9 +38,9 @@ pass import smtplib -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/FilterDialog.py b/cuegui/cuegui/FilterDialog.py index 46126050b..479f1c6e5 100644 --- a/cuegui/cuegui/FilterDialog.py +++ b/cuegui/cuegui/FilterDialog.py @@ -23,9 +23,9 @@ from builtins import map from builtins import str -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import opencue import opencue.compiled_proto.filter_pb2 @@ -63,7 +63,7 @@ def __init__(self, show, parent=None): :type show: opencue.wrappers.show.Show :param show: the show to manage filters for - :type parent: PySide2.QtWidgets.QWidget.QWidget + :type parent: qtpy.QtWidgets.QWidget.QWidget :param parent: the parent widget """ QtWidgets.QDialog.__init__(self, parent) diff --git a/cuegui/cuegui/FrameMonitor.py b/cuegui/cuegui/FrameMonitor.py index a854d574f..56c586cd9 100644 --- a/cuegui/cuegui/FrameMonitor.py +++ b/cuegui/cuegui/FrameMonitor.py @@ -24,9 +24,9 @@ from copy import deepcopy import math -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import FileSequence from opencue.compiled_proto import job_pb2 diff --git a/cuegui/cuegui/FrameMonitorTree.py b/cuegui/cuegui/FrameMonitorTree.py index b546be2d5..e40be0f4d 100644 --- a/cuegui/cuegui/FrameMonitorTree.py +++ b/cuegui/cuegui/FrameMonitorTree.py @@ -29,9 +29,9 @@ import re import time -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import opencue from opencue.compiled_proto import job_pb2 diff --git a/cuegui/cuegui/FrameRangeSelection.py b/cuegui/cuegui/FrameRangeSelection.py index 851e2aab2..003c4bc22 100644 --- a/cuegui/cuegui/FrameRangeSelection.py +++ b/cuegui/cuegui/FrameRangeSelection.py @@ -25,9 +25,9 @@ from builtins import range import math -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets class FrameRangeSelectionWidget(QtWidgets.QWidget): @@ -303,7 +303,7 @@ def __paintLabels(self, painter): oldPen = painter.pen() # draw hatches for labelled frames - painter.setPen(self.palette().color(QtGui.QPalette.Foreground)) + painter.setPen(self.palette().color(QtGui.QPalette.WindowText)) for frame in frames: xPos = self.__getTickArea(frame).left() painter.drawLine(xPos, -labelHeight, xPos, 0) @@ -313,7 +313,7 @@ def __paintLabels(self, painter): metric = QtGui.QFontMetrics(painter.font()) yPos = metric.ascent() + 1 rightEdge = -10000 - width = metric.width(str(frames[-1])) + width = metric.horizontalAdvance(str(frames[-1])) farEdge = self.__getTickArea(frames[-1]).right() - width // 2 farEdge -= 4 @@ -321,7 +321,7 @@ def __paintLabels(self, painter): for frame in frames: xPos = self.__getTickArea(frame).left() frameString = str(frame) - width = metric.width(frameString) + width = metric.horizontalAdvance(frameString) xPos = xPos - width // 2 if (xPos > rightEdge and xPos + width < farEdge) or frame is frames[-1]: painter.drawText(xPos, yPos, frameString) @@ -337,7 +337,7 @@ def __paintStartTime(self, painter): metric = QtGui.QFontMetrics(painter.font()) frameString = str(int(startFrame)) - xPos = timeExtent.left() - metric.width(frameString) // 2 + xPos = timeExtent.left() - metric.horizontalAdvance(frameString) // 2 yPos = metric.ascent() + 1 painter.drawText(xPos, yPos, frameString) painter.setPen(oldPen) @@ -351,7 +351,7 @@ def __paintEndTime(self, painter): metric = QtGui.QFontMetrics(painter.font()) frameString = str(int(endFrame)) - xPos = timeExtent.left() - metric.width(frameString) // 2 + xPos = timeExtent.left() - metric.horizontalAdvance(frameString) // 2 yPos = metric.ascent() + 1 painter.drawText(xPos, yPos, frameString) painter.setPen(oldPen) @@ -372,7 +372,7 @@ def __paintFloatTime(self, painter): painter.setPen(QtGui.QColor(128, 128, 128)) metric = QtGui.QFontMetrics(painter.font()) frameString = str(self.__floatTime) - xPos = timeExtent.left() - metric.width(frameString) // 2 + xPos = timeExtent.left() - metric.horizontalAdvance(frameString) // 2 yPos = timeExtent.top() + metric.ascent() painter.drawText(xPos, yPos, frameString) painter.setPen(oldPen) diff --git a/cuegui/cuegui/GarbageCollector.py b/cuegui/cuegui/GarbageCollector.py index 83cc06bc7..7f999d98f 100644 --- a/cuegui/cuegui/GarbageCollector.py +++ b/cuegui/cuegui/GarbageCollector.py @@ -23,7 +23,7 @@ import gc -from PySide2 import QtCore +from qtpy import QtCore class GarbageCollector(QtCore.QObject): diff --git a/cuegui/cuegui/GroupDialog.py b/cuegui/cuegui/GroupDialog.py index 0f949b2f7..d5f26ac18 100644 --- a/cuegui/cuegui/GroupDialog.py +++ b/cuegui/cuegui/GroupDialog.py @@ -23,8 +23,8 @@ from builtins import str -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/HostMonitor.py b/cuegui/cuegui/HostMonitor.py index cd7ffee99..66ee94287 100644 --- a/cuegui/cuegui/HostMonitor.py +++ b/cuegui/cuegui/HostMonitor.py @@ -22,8 +22,8 @@ from builtins import str -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/HostMonitorTree.py b/cuegui/cuegui/HostMonitorTree.py index 1c247e2c6..bff3d6ab2 100644 --- a/cuegui/cuegui/HostMonitorTree.py +++ b/cuegui/cuegui/HostMonitorTree.py @@ -23,9 +23,9 @@ from builtins import map import time -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import opencue from opencue.compiled_proto.host_pb2 import HardwareState diff --git a/cuegui/cuegui/ItemDelegate.py b/cuegui/cuegui/ItemDelegate.py index 27aff67ec..e4de3df02 100644 --- a/cuegui/cuegui/ItemDelegate.py +++ b/cuegui/cuegui/ItemDelegate.py @@ -24,9 +24,9 @@ from builtins import range from math import ceil -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/JobMonitorTree.py b/cuegui/cuegui/JobMonitorTree.py index 3bf15373b..d92c44afc 100644 --- a/cuegui/cuegui/JobMonitorTree.py +++ b/cuegui/cuegui/JobMonitorTree.py @@ -24,9 +24,9 @@ from builtins import map import time -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/LayerDialog.py b/cuegui/cuegui/LayerDialog.py index 5f25be5f6..8c9f0e57f 100644 --- a/cuegui/cuegui/LayerDialog.py +++ b/cuegui/cuegui/LayerDialog.py @@ -20,8 +20,8 @@ from __future__ import division from __future__ import print_function -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/LayerMonitorTree.py b/cuegui/cuegui/LayerMonitorTree.py index 7663d3276..25916e16f 100644 --- a/cuegui/cuegui/LayerMonitorTree.py +++ b/cuegui/cuegui/LayerMonitorTree.py @@ -20,8 +20,8 @@ from __future__ import print_function from __future__ import division -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets from opencue.exception import EntityNotFoundException diff --git a/cuegui/cuegui/Layout.py b/cuegui/cuegui/Layout.py index 65bb9c080..9b26d23c8 100644 --- a/cuegui/cuegui/Layout.py +++ b/cuegui/cuegui/Layout.py @@ -22,7 +22,7 @@ import os import shutil -from PySide2 import QtCore +from qtpy import QtCore import cuegui.Constants import cuegui.Logger diff --git a/cuegui/cuegui/LimitSelectionWidget.py b/cuegui/cuegui/LimitSelectionWidget.py index 8a37e370e..a760d1b79 100644 --- a/cuegui/cuegui/LimitSelectionWidget.py +++ b/cuegui/cuegui/LimitSelectionWidget.py @@ -22,7 +22,7 @@ from builtins import str -from PySide2 import QtWidgets +from qtpy import QtWidgets import cuegui.AbstractDialog diff --git a/cuegui/cuegui/LimitsWidget.py b/cuegui/cuegui/LimitsWidget.py index 26f871de1..25c962492 100644 --- a/cuegui/cuegui/LimitsWidget.py +++ b/cuegui/cuegui/LimitsWidget.py @@ -20,8 +20,8 @@ from __future__ import print_function from __future__ import division -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/LocalBooking.py b/cuegui/cuegui/LocalBooking.py index a69270198..23220e113 100644 --- a/cuegui/cuegui/LocalBooking.py +++ b/cuegui/cuegui/LocalBooking.py @@ -27,8 +27,8 @@ import os from socket import gethostname -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/Main.py b/cuegui/cuegui/Main.py index 8e5299ef9..24f08864b 100644 --- a/cuegui/cuegui/Main.py +++ b/cuegui/cuegui/Main.py @@ -22,7 +22,7 @@ import signal -from PySide2 import QtGui +from qtpy import QtGui import cuegui import cuegui.Layout diff --git a/cuegui/cuegui/MainWindow.py b/cuegui/cuegui/MainWindow.py index 50555f647..942a9a717 100644 --- a/cuegui/cuegui/MainWindow.py +++ b/cuegui/cuegui/MainWindow.py @@ -28,9 +28,9 @@ import sys import time -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/MenuActions.py b/cuegui/cuegui/MenuActions.py index 2f6771baa..8dc89577f 100644 --- a/cuegui/cuegui/MenuActions.py +++ b/cuegui/cuegui/MenuActions.py @@ -28,8 +28,8 @@ import subprocess import time -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtGui +from qtpy import QtWidgets import six import FileSequence diff --git a/cuegui/cuegui/MiscDialog.py b/cuegui/cuegui/MiscDialog.py index c3b1185f8..d2712af93 100644 --- a/cuegui/cuegui/MiscDialog.py +++ b/cuegui/cuegui/MiscDialog.py @@ -20,7 +20,7 @@ from __future__ import print_function from __future__ import division -from PySide2 import QtWidgets +from qtpy import QtWidgets import cuegui.AbstractDialog diff --git a/cuegui/cuegui/Plugins.py b/cuegui/cuegui/Plugins.py index f4e6e3524..a4d48563c 100644 --- a/cuegui/cuegui/Plugins.py +++ b/cuegui/cuegui/Plugins.py @@ -57,8 +57,8 @@ import traceback import pickle -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.Constants import cuegui.Logger diff --git a/cuegui/cuegui/PreviewWidget.py b/cuegui/cuegui/PreviewWidget.py index 56888188e..5dfea8658 100644 --- a/cuegui/cuegui/PreviewWidget.py +++ b/cuegui/cuegui/PreviewWidget.py @@ -33,8 +33,8 @@ import urllib.request import xml.etree.ElementTree as Et -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.Logger import cuegui.Utils @@ -54,7 +54,7 @@ def __init__(self, job, frame, aovs=False, parent=None): :param frame: frame to display :type aovs: bool :param aovs: whether to display AOVs or just the main image - :type parent: PySide2.QtWidgets.QWidget + :type parent: qtpy.QtWidgets.QWidget :param parent: the parent widget """ QtWidgets.QDialog.__init__(self, parent) diff --git a/cuegui/cuegui/ProcChildren.py b/cuegui/cuegui/ProcChildren.py index 5faa0415e..c281f7970 100644 --- a/cuegui/cuegui/ProcChildren.py +++ b/cuegui/cuegui/ProcChildren.py @@ -27,8 +27,8 @@ from builtins import str -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtGui +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/ProcMonitor.py b/cuegui/cuegui/ProcMonitor.py index 4221a2c92..b4ae4f5d4 100644 --- a/cuegui/cuegui/ProcMonitor.py +++ b/cuegui/cuegui/ProcMonitor.py @@ -22,8 +22,8 @@ from builtins import str -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.Logger import cuegui.ProcMonitorTree diff --git a/cuegui/cuegui/ProcMonitorTree.py b/cuegui/cuegui/ProcMonitorTree.py index cf2658de5..0f4710b53 100644 --- a/cuegui/cuegui/ProcMonitorTree.py +++ b/cuegui/cuegui/ProcMonitorTree.py @@ -23,8 +23,8 @@ from builtins import map import time -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/ProgressDialog.py b/cuegui/cuegui/ProgressDialog.py index 2f507fda7..f9b49c453 100644 --- a/cuegui/cuegui/ProgressDialog.py +++ b/cuegui/cuegui/ProgressDialog.py @@ -23,8 +23,8 @@ from builtins import map from builtins import range -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.Logger import cuegui.Utils diff --git a/cuegui/cuegui/Redirect.py b/cuegui/cuegui/Redirect.py index 43406793c..191bfb6ad 100644 --- a/cuegui/cuegui/Redirect.py +++ b/cuegui/cuegui/Redirect.py @@ -31,9 +31,9 @@ import re import time -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/ServiceDialog.py b/cuegui/cuegui/ServiceDialog.py index b29a13c23..c54ca4a88 100644 --- a/cuegui/cuegui/ServiceDialog.py +++ b/cuegui/cuegui/ServiceDialog.py @@ -23,8 +23,8 @@ from builtins import str from builtins import range -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/ShowDialog.py b/cuegui/cuegui/ShowDialog.py index 7acdfec26..f0376fcf5 100644 --- a/cuegui/cuegui/ShowDialog.py +++ b/cuegui/cuegui/ShowDialog.py @@ -22,8 +22,8 @@ from builtins import str -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.Utils diff --git a/cuegui/cuegui/ShowsWidget.py b/cuegui/cuegui/ShowsWidget.py index f10fa08e3..20b62ae73 100644 --- a/cuegui/cuegui/ShowsWidget.py +++ b/cuegui/cuegui/ShowsWidget.py @@ -20,8 +20,8 @@ from __future__ import print_function from __future__ import division -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/SplashWindow.py b/cuegui/cuegui/SplashWindow.py index 29c268c7a..57075a261 100644 --- a/cuegui/cuegui/SplashWindow.py +++ b/cuegui/cuegui/SplashWindow.py @@ -24,9 +24,9 @@ import os import time -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets __all__ = ["SplashWindow"] diff --git a/cuegui/cuegui/Style.py b/cuegui/cuegui/Style.py index d95b80dd8..95acea64f 100644 --- a/cuegui/cuegui/Style.py +++ b/cuegui/cuegui/Style.py @@ -22,7 +22,7 @@ import importlib -from PySide2 import QtGui +from qtpy import QtGui import cuegui diff --git a/cuegui/cuegui/SubscriptionGraphWidget.py b/cuegui/cuegui/SubscriptionGraphWidget.py index e129ed3eb..b17004d11 100644 --- a/cuegui/cuegui/SubscriptionGraphWidget.py +++ b/cuegui/cuegui/SubscriptionGraphWidget.py @@ -20,8 +20,8 @@ import opencue -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.AbstractTreeWidget import cuegui.AbstractWidgetItem diff --git a/cuegui/cuegui/SubscriptionsWidget.py b/cuegui/cuegui/SubscriptionsWidget.py index 575742f4b..eb474e074 100644 --- a/cuegui/cuegui/SubscriptionsWidget.py +++ b/cuegui/cuegui/SubscriptionsWidget.py @@ -24,8 +24,8 @@ import opencue -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.AbstractTreeWidget import cuegui.AbstractWidgetItem diff --git a/cuegui/cuegui/TagsWidget.py b/cuegui/cuegui/TagsWidget.py index e9d6dc129..548ba2b53 100644 --- a/cuegui/cuegui/TagsWidget.py +++ b/cuegui/cuegui/TagsWidget.py @@ -23,7 +23,7 @@ from builtins import str import re -from PySide2 import QtWidgets +from qtpy import QtWidgets import cuegui.AbstractDialog import cuegui.Constants diff --git a/cuegui/cuegui/TasksDialog.py b/cuegui/cuegui/TasksDialog.py index 614b07c2d..2f820d76c 100644 --- a/cuegui/cuegui/TasksDialog.py +++ b/cuegui/cuegui/TasksDialog.py @@ -23,8 +23,8 @@ from builtins import map from builtins import str -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue.exception diff --git a/cuegui/cuegui/TextEditDialog.py b/cuegui/cuegui/TextEditDialog.py index ed25fe45d..a45687d92 100644 --- a/cuegui/cuegui/TextEditDialog.py +++ b/cuegui/cuegui/TextEditDialog.py @@ -20,8 +20,8 @@ from __future__ import division from __future__ import absolute_import -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets class TextEditDialog(QtWidgets.QDialog): diff --git a/cuegui/cuegui/ThreadPool.py b/cuegui/cuegui/ThreadPool.py index ba03ddc8f..fa6e6c412 100644 --- a/cuegui/cuegui/ThreadPool.py +++ b/cuegui/cuegui/ThreadPool.py @@ -49,7 +49,7 @@ def someWorkCallback(work, result): from builtins import range import os -from PySide2 import QtCore +from qtpy import QtCore import cuegui.Logger diff --git a/cuegui/cuegui/UnbookDialog.py b/cuegui/cuegui/UnbookDialog.py index e74149665..a661faa93 100644 --- a/cuegui/cuegui/UnbookDialog.py +++ b/cuegui/cuegui/UnbookDialog.py @@ -25,8 +25,8 @@ from builtins import object import re -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import six import opencue diff --git a/cuegui/cuegui/Utils.py b/cuegui/cuegui/Utils.py index 957f91908..4666797b5 100644 --- a/cuegui/cuegui/Utils.py +++ b/cuegui/cuegui/Utils.py @@ -32,9 +32,9 @@ import traceback import webbrowser -from PySide2 import QtCore -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtGui +from qtpy import QtWidgets import six import opencue diff --git a/cuegui/cuegui/images/crystal/icons_rcc.py b/cuegui/cuegui/images/crystal/icons_rcc.py index 87d4c83d9..d66df7306 100644 --- a/cuegui/cuegui/images/crystal/icons_rcc.py +++ b/cuegui/cuegui/images/crystal/icons_rcc.py @@ -23,7 +23,7 @@ # # WARNING! All changes made in this file will be lost! -from PySide2 import QtCore +from qtpy import QtCore qt_resource_data = b"\ \x00\x00\x02\xfc\ diff --git a/cuegui/cuegui/images/icons_rcc.py b/cuegui/cuegui/images/icons_rcc.py index b313c2d56..4a9664ee7 100644 --- a/cuegui/cuegui/images/icons_rcc.py +++ b/cuegui/cuegui/images/icons_rcc.py @@ -28,7 +28,7 @@ from __future__ import division from __future__ import absolute_import -from PySide2 import QtCore +from qtpy import QtCore qt_resource_data = b"\ diff --git a/cuegui/cuegui/plugins/AllocationsPlugin.py b/cuegui/cuegui/plugins/AllocationsPlugin.py index 9e7d1a647..a7d18162e 100644 --- a/cuegui/cuegui/plugins/AllocationsPlugin.py +++ b/cuegui/cuegui/plugins/AllocationsPlugin.py @@ -22,7 +22,7 @@ from builtins import map -from PySide2 import QtWidgets +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/plugins/AttributesPlugin.py b/cuegui/cuegui/plugins/AttributesPlugin.py index 26f9350de..c25480f85 100644 --- a/cuegui/cuegui/plugins/AttributesPlugin.py +++ b/cuegui/cuegui/plugins/AttributesPlugin.py @@ -24,8 +24,8 @@ from builtins import str import time -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue import opencue.compiled_proto.depend_pb2 diff --git a/cuegui/cuegui/plugins/LogViewPlugin.py b/cuegui/cuegui/plugins/LogViewPlugin.py index caeacf449..1e67ac1c4 100644 --- a/cuegui/cuegui/plugins/LogViewPlugin.py +++ b/cuegui/cuegui/plugins/LogViewPlugin.py @@ -27,9 +27,9 @@ import string import time -from PySide2 import QtGui -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtGui +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.Constants import cuegui.AbstractDockWidget @@ -183,7 +183,7 @@ def get_line_number_area_width(self): while count >= 10: count /= 10 digits += 1 - space = 3 + self.fontMetrics().width('9') * digits + space = 3 + self.fontMetrics().horizontalAdvance('9') * digits return space def update_line_number_area_width(self): diff --git a/cuegui/cuegui/plugins/MonitorCuePlugin.py b/cuegui/cuegui/plugins/MonitorCuePlugin.py index c0b3e6fac..7884bfc5a 100644 --- a/cuegui/cuegui/plugins/MonitorCuePlugin.py +++ b/cuegui/cuegui/plugins/MonitorCuePlugin.py @@ -25,9 +25,9 @@ import re import weakref -from PySide2 import QtGui -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtGui +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/plugins/MonitorHostsPlugin.py b/cuegui/cuegui/plugins/MonitorHostsPlugin.py index cee8413fe..be40d7453 100644 --- a/cuegui/cuegui/plugins/MonitorHostsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorHostsPlugin.py @@ -20,8 +20,8 @@ from __future__ import division from __future__ import absolute_import -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.AbstractDockWidget import cuegui.HostMonitor diff --git a/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py b/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py index 270a74b23..bb1b61d02 100644 --- a/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py @@ -22,8 +22,8 @@ from __future__ import division from __future__ import absolute_import -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/plugins/MonitorJobsPlugin.py b/cuegui/cuegui/plugins/MonitorJobsPlugin.py index f9ab677c2..a8ee4e034 100644 --- a/cuegui/cuegui/plugins/MonitorJobsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorJobsPlugin.py @@ -26,9 +26,9 @@ import re import weakref -from PySide2 import QtGui -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtGui +from qtpy import QtCore +from qtpy import QtWidgets import opencue diff --git a/cuegui/cuegui/plugins/ShowsPlugin.py b/cuegui/cuegui/plugins/ShowsPlugin.py index deda343f9..9d292a5a8 100644 --- a/cuegui/cuegui/plugins/ShowsPlugin.py +++ b/cuegui/cuegui/plugins/ShowsPlugin.py @@ -20,7 +20,7 @@ from __future__ import division from __future__ import absolute_import -from PySide2 import QtWidgets +from qtpy import QtWidgets import cuegui.AbstractDockWidget import cuegui.CreateShowDialog diff --git a/cuegui/cuegui/plugins/StuckFramePlugin.py b/cuegui/cuegui/plugins/StuckFramePlugin.py index f3d5e2904..a44c90196 100644 --- a/cuegui/cuegui/plugins/StuckFramePlugin.py +++ b/cuegui/cuegui/plugins/StuckFramePlugin.py @@ -31,9 +31,9 @@ import signal import yaml -from PySide2 import QtGui -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtGui +from qtpy import QtCore +from qtpy import QtWidgets import opencue.wrappers.frame diff --git a/cuegui/cuegui/plugins/SubscriptionsGraphPlugin.py b/cuegui/cuegui/plugins/SubscriptionsGraphPlugin.py index 80dde33ca..a8c0078ef 100644 --- a/cuegui/cuegui/plugins/SubscriptionsGraphPlugin.py +++ b/cuegui/cuegui/plugins/SubscriptionsGraphPlugin.py @@ -20,8 +20,8 @@ from __future__ import division from __future__ import print_function -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.AbstractDockWidget import cuegui.SubscriptionGraphWidget diff --git a/cuegui/cuegui/plugins/SubscriptionsPlugin.py b/cuegui/cuegui/plugins/SubscriptionsPlugin.py index 7c0d11731..e40081802 100644 --- a/cuegui/cuegui/plugins/SubscriptionsPlugin.py +++ b/cuegui/cuegui/plugins/SubscriptionsPlugin.py @@ -20,8 +20,8 @@ from __future__ import division from __future__ import absolute_import -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.AbstractDockWidget import cuegui.SubscriptionsWidget diff --git a/cuegui/setup.py b/cuegui/setup.py index 79882eda3..2068936e0 100644 --- a/cuegui/setup.py +++ b/cuegui/setup.py @@ -65,6 +65,7 @@ 'grpcio-tools', 'PySide2', 'PyYAML', + 'QtPy', ] ) diff --git a/cuegui/tests/CueJobMonitorTree_tests.py b/cuegui/tests/CueJobMonitorTree_tests.py index 2cf93d0d5..2698ccedc 100644 --- a/cuegui/tests/CueJobMonitorTree_tests.py +++ b/cuegui/tests/CueJobMonitorTree_tests.py @@ -19,9 +19,9 @@ import unittest import mock -import PySide2.QtCore -import PySide2.QtGui -import PySide2.QtWidgets +import qtpy.QtCore +import qtpy.QtGui +import qtpy.QtWidgets import opencue.compiled_proto.job_pb2 import opencue.compiled_proto.show_pb2 @@ -40,7 +40,7 @@ class CueJobMonitorTreeTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, get_stub_mock): app = test_utils.createApplication() - app.settings = PySide2.QtCore.QSettings() + app.settings = qtpy.QtCore.QSettings() cuegui.Style.init() self.show_name = 'arbitrary-show-name' @@ -60,7 +60,7 @@ def setUp(self, get_stub_mock): name=self.show_name, jobs=self.jobs)) - self.main_window = PySide2.QtWidgets.QMainWindow() + self.main_window = qtpy.QtWidgets.QMainWindow() self.widget = cuegui.plugins.MonitorCuePlugin.MonitorCueDockWidget(self.main_window) self.cue_job_monitor_tree = cuegui.CueJobMonitorTree.CueJobMonitorTree(self.widget) self.cue_job_monitor_tree.addShow(self.show_name) diff --git a/cuegui/tests/DependWizard_tests.py b/cuegui/tests/DependWizard_tests.py index de1632502..5efb49f74 100644 --- a/cuegui/tests/DependWizard_tests.py +++ b/cuegui/tests/DependWizard_tests.py @@ -19,10 +19,10 @@ import unittest import mock -import PySide2.QtCore -import PySide2.QtGui -import PySide2.QtWidgets -import PySide2.QtTest +import qtpy.QtCore +import qtpy.QtGui +import qtpy.QtWidgets +import qtpy.QtTest import opencue.compiled_proto.job_pb2 import opencue.wrappers.frame @@ -41,10 +41,10 @@ class DependWizardTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): app = test_utils.createApplication() - app.settings = PySide2.QtCore.QSettings() + app.settings = qtpy.QtCore.QSettings() cuegui.Style.init() - self.parentWidget = PySide2.QtWidgets.QWidget() + self.parentWidget = qtpy.QtWidgets.QWidget() @mock.patch('cuegui.Cuedepend.createJobOnLayerDepend') @mock.patch('opencue.api.findJob') diff --git a/cuegui/tests/FilterDialog_tests.py b/cuegui/tests/FilterDialog_tests.py index bfe4c5832..536d65ec4 100644 --- a/cuegui/tests/FilterDialog_tests.py +++ b/cuegui/tests/FilterDialog_tests.py @@ -20,10 +20,10 @@ import mock -import PySide2.QtCore -import PySide2.QtGui -import PySide2.QtWidgets -import PySide2.QtTest +import qtpy.QtCore +import qtpy.QtGui +import qtpy.QtWidgets +import qtpy.QtTest import opencue.compiled_proto.show_pb2 import opencue.compiled_proto.filter_pb2 @@ -42,7 +42,7 @@ class FilterDialogTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, getStubMock): app = test_utils.createApplication() - app.settings = PySide2.QtCore.QSettings() + app.settings = qtpy.QtCore.QSettings() cuegui.Style.init() self.show = opencue.wrappers.show.Show(opencue.compiled_proto.show_pb2.Show(name='fooShow')) @@ -54,7 +54,7 @@ def setUp(self, getStubMock): opencue.compiled_proto.show_pb2.ShowGetFiltersResponse( filters=opencue.compiled_proto.filter_pb2.FilterSeq(filters=[filterProto])) - self.parentWidget = PySide2.QtWidgets.QWidget() + self.parentWidget = qtpy.QtWidgets.QWidget() self.filterDialog = cuegui.FilterDialog.FilterDialog(self.show, parent=self.parentWidget) def test_shouldTriggerRefresh(self): @@ -64,7 +64,7 @@ def test_shouldTriggerRefresh(self): self.show.getFilters.assert_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_shouldAddFilter(self, getTextMock): newFilterId = 'new-filter-id' newFilterName = 'new-filter-name' @@ -78,7 +78,7 @@ def test_shouldAddFilter(self, getTextMock): self.show.createFilter.assert_called_with(newFilterName) - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_shouldCancelAddingFilter(self, getTextMock): self.show.createFilter = mock.Mock() getTextMock.return_value = (None, False) @@ -142,11 +142,11 @@ def test_shouldTriggerCreateAction(self): self.filterDialog._FilterDialog__actions.createAction.assert_called() def test_shouldCloseDialog(self): - self.assertEqual(PySide2.QtWidgets.QDialog.DialogCode.Rejected, self.filterDialog.result()) + self.assertEqual(qtpy.QtWidgets.QDialog.DialogCode.Rejected, self.filterDialog.result()) self.filterDialog._FilterDialog__btnDone.click() - self.assertEqual(PySide2.QtWidgets.QDialog.DialogCode.Accepted, self.filterDialog.result()) + self.assertEqual(qtpy.QtWidgets.QDialog.DialogCode.Accepted, self.filterDialog.result()) @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) @@ -155,7 +155,7 @@ class FilterMonitorTreeTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, getStubMock): app = test_utils.createApplication() - app.settings = PySide2.QtCore.QSettings() + app.settings = qtpy.QtCore.QSettings() cuegui.Style.init() show = opencue.wrappers.show.Show(opencue.compiled_proto.show_pb2.Show(name='fooShow')) @@ -169,7 +169,7 @@ def setUp(self, getStubMock): opencue.compiled_proto.show_pb2.ShowGetFiltersResponse( filters=opencue.compiled_proto.filter_pb2.FilterSeq(filters=filters)) - self.parentWidget = PySide2.QtWidgets.QWidget() + self.parentWidget = qtpy.QtWidgets.QWidget() self.filterDialog = cuegui.FilterDialog.FilterDialog(show, parent=self.parentWidget) self.filterMonitorTree = self.filterDialog._FilterDialog__filters @@ -182,13 +182,13 @@ def test_shouldPopulateFiltersList(self): self.assertEqual('2', secondItem.text(0)) self.assertEqual(False, self.filterMonitorTree.itemWidget(secondItem, 1).isChecked()) - @mock.patch('PySide2.QtWidgets.QMenu') + @mock.patch('qtpy.QtWidgets.QMenu') def test_shouldRaiseContextMenu(self, qMenuMock): filterBeingSelected = self.filterMonitorTree.topLevelItem(0) self.filterMonitorTree.contextMenuEvent( - PySide2.QtGui.QContextMenuEvent( - PySide2.QtGui.QContextMenuEvent.Reason.Mouse, + qtpy.QtGui.QContextMenuEvent( + qtpy.QtGui.QContextMenuEvent.Reason.Mouse, self.filterMonitorTree.visualItemRect(filterBeingSelected).center())) qMenuMock.return_value.exec_.assert_called() @@ -200,7 +200,7 @@ class MatcherMonitorTreeTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, getStubMock): app = test_utils.createApplication() - app.settings = PySide2.QtCore.QSettings() + app.settings = qtpy.QtCore.QSettings() cuegui.Style.init() self.matchers = [ @@ -219,7 +219,7 @@ def setUp(self, getStubMock): opencue.wrappers.filter.Matcher(matcher) for matcher in self.matchers] self.filter = opencue.wrappers.filter.Filter(opencue.compiled_proto.filter_pb2.Filter()) - self.parentWidget = PySide2.QtWidgets.QWidget() + self.parentWidget = qtpy.QtWidgets.QWidget() self.matcherMonitorTree = cuegui.FilterDialog.MatcherMonitorTree(None, self.parentWidget) def test_shouldPopulateMatchersList(self): @@ -238,8 +238,8 @@ def test_shouldPopulateMatchersList(self): self.assertEqual('IS', self.matcherMonitorTree.itemWidget(secondItem, 1).currentText()) self.assertEqual('showName', self.matcherMonitorTree.itemWidget(secondItem, 2).text()) - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') def test_shouldAddMatcher(self, getItemMock, getTextMock): matcherSubject = opencue.compiled_proto.filter_pb2.FACILITY matcherType = opencue.compiled_proto.filter_pb2.CONTAINS @@ -270,8 +270,8 @@ def test_shouldAddMatcher(self, getItemMock, getTextMock): 'CONTAINS', self.matcherMonitorTree.itemWidget(matcherWidget, 1).currentText()) self.assertEqual(matcherText, self.matcherMonitorTree.itemWidget(matcherWidget, 2).text()) - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') def test_shouldCancelMatcherAdditionAtFirstPrompt(self, getItemMock, getTextMock): self.filter.createMatcher = mock.Mock() getItemMock.side_effect = [ @@ -285,8 +285,8 @@ def test_shouldCancelMatcherAdditionAtFirstPrompt(self, getItemMock, getTextMock self.filter.createMatcher.assert_not_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') def test_shouldCancelMatcherAdditionAtSecondPrompt(self, getItemMock, getTextMock): self.filter.createMatcher = mock.Mock() getItemMock.side_effect = [ @@ -300,8 +300,8 @@ def test_shouldCancelMatcherAdditionAtSecondPrompt(self, getItemMock, getTextMoc self.filter.createMatcher.assert_not_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') def test_shouldCancelMatcherAdditionAtThirdrompt(self, getItemMock, getTextMock): self.filter.createMatcher = mock.Mock() getItemMock.side_effect = [ @@ -316,8 +316,8 @@ def test_shouldCancelMatcherAdditionAtThirdrompt(self, getItemMock, getTextMock) self.filter.createMatcher.assert_not_called() @mock.patch( - 'PySide2.QtWidgets.QMessageBox.question', - new=mock.Mock(return_value=PySide2.QtWidgets.QMessageBox.Yes)) + 'qtpy.QtWidgets.QMessageBox.question', + new=mock.Mock(return_value=qtpy.QtWidgets.QMessageBox.Yes)) def test_shouldDeleteAllMatchers(self): self.filter.getMatchers = mock.Mock(return_value=self.matcherWrappers) for matcher in self.matcherWrappers: @@ -330,8 +330,8 @@ def test_shouldDeleteAllMatchers(self): matcher.delete.assert_called() @mock.patch( - 'PySide2.QtWidgets.QMessageBox.question', - new=mock.Mock(return_value=PySide2.QtWidgets.QMessageBox.No)) + 'qtpy.QtWidgets.QMessageBox.question', + new=mock.Mock(return_value=qtpy.QtWidgets.QMessageBox.No)) def test_shouldNotDeleteAnyMatchers(self): self.filter.getMatchers = mock.Mock(return_value=self.matcherWrappers) for matcher in self.matcherWrappers: @@ -345,7 +345,7 @@ def test_shouldNotDeleteAnyMatchers(self): @mock.patch('cuegui.Utils.questionBoxYesNo', new=mock.Mock(return_value=True)) @mock.patch('cuegui.TextEditDialog.TextEditDialog') - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') def test_shouldAddMultipleMatchers(self, getItemMock, textEditDialogMock): matcherSubject = opencue.compiled_proto.filter_pb2.SHOT matcherType = opencue.compiled_proto.filter_pb2.IS @@ -382,7 +382,7 @@ def test_shouldAddMultipleMatchers(self, getItemMock, textEditDialogMock): @mock.patch('cuegui.Utils.questionBoxYesNo', new=mock.Mock(return_value=True)) @mock.patch('cuegui.TextEditDialog.TextEditDialog') - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') def test_shouldReplaceAllMatchers(self, getItemMock, textEditDialogMock): matcherSubject = opencue.compiled_proto.filter_pb2.SHOT matcherType = opencue.compiled_proto.filter_pb2.IS diff --git a/cuegui/tests/FrameMonitorTree_tests.py b/cuegui/tests/FrameMonitorTree_tests.py index fc77e864c..75521572f 100644 --- a/cuegui/tests/FrameMonitorTree_tests.py +++ b/cuegui/tests/FrameMonitorTree_tests.py @@ -19,10 +19,10 @@ import unittest import mock -import PySide2.QtCore -import PySide2.QtGui -import PySide2.QtTest -import PySide2.QtWidgets +import qtpy.QtCore +import qtpy.QtGui +import qtpy.QtTest +import qtpy.QtWidgets import opencue.compiled_proto.job_pb2 import opencue.wrappers.frame @@ -44,9 +44,9 @@ class FrameMonitorTreeTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): app = test_utils.createApplication() - app.settings = PySide2.QtCore.QSettings() + app.settings = qtpy.QtCore.QSettings() cuegui.Style.init() - self.parentWidget = PySide2.QtWidgets.QWidget() + self.parentWidget = qtpy.QtWidgets.QWidget() self.frameMonitorTree = cuegui.FrameMonitorTree.FrameMonitorTree(self.parentWidget) self.job = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(id='foo')) self.frameMonitorTree.setJob(self.job) @@ -126,11 +126,11 @@ def test_getCores(self): @mock.patch.object(cuegui.FrameMonitorTree.FrameContextMenu, 'exec_') def test_rightClickItem(self, execMock): - mouse_position = PySide2.QtCore.QPoint() + mouse_position = qtpy.QtCore.QPoint() self.frameMonitorTree.contextMenuEvent( - PySide2.QtGui.QContextMenuEvent( - PySide2.QtGui.QContextMenuEvent.Reason.Mouse, mouse_position, mouse_position)) + qtpy.QtGui.QContextMenuEvent( + qtpy.QtGui.QContextMenuEvent.Reason.Mouse, mouse_position, mouse_position)) execMock.assert_called_with(mouse_position) @@ -152,7 +152,7 @@ def setUp(self): checkpoint_state=opencue.compiled_proto.job_pb2.ENABLED)) # The widget needs a var, otherwise it gets garbage-collected before tests can run. - parentWidget = PySide2.QtWidgets.QWidget() + parentWidget = qtpy.QtWidgets.QWidget() self.frameWidgetItem = cuegui.FrameMonitorTree.FrameWidgetItem( self.frame, @@ -165,46 +165,46 @@ def test_data(self): self.assertEqual( self.dispatch_order, - self.frameWidgetItem.data(dispatch_order_col, PySide2.QtCore.Qt.DisplayRole)) + self.frameWidgetItem.data(dispatch_order_col, qtpy.QtCore.Qt.DisplayRole)) self.assertEqual( cuegui.Style.ColorTheme.COLOR_JOB_FOREGROUND, - self.frameWidgetItem.data(dispatch_order_col, PySide2.QtCore.Qt.ForegroundRole)) + self.frameWidgetItem.data(dispatch_order_col, qtpy.QtCore.Qt.ForegroundRole)) self.assertEqual( cuegui.FrameMonitorTree.QCOLOR_BLACK, self.frameWidgetItem.data( - cuegui.FrameMonitorTree.STATUS_COLUMN, PySide2.QtCore.Qt.ForegroundRole)) + cuegui.FrameMonitorTree.STATUS_COLUMN, qtpy.QtCore.Qt.ForegroundRole)) self.assertEqual( cuegui.FrameMonitorTree.QCOLOR_GREEN, self.frameWidgetItem.data( - cuegui.FrameMonitorTree.PROC_COLUMN, PySide2.QtCore.Qt.ForegroundRole)) + cuegui.FrameMonitorTree.PROC_COLUMN, qtpy.QtCore.Qt.ForegroundRole)) self.assertEqual( cuegui.Constants.RGB_FRAME_STATE[self.state], self.frameWidgetItem.data( - cuegui.FrameMonitorTree.STATUS_COLUMN, PySide2.QtCore.Qt.BackgroundRole)) + cuegui.FrameMonitorTree.STATUS_COLUMN, qtpy.QtCore.Qt.BackgroundRole)) self.assertEqual( - PySide2.QtGui.QIcon, + qtpy.QtGui.QIcon, self.frameWidgetItem.data( cuegui.FrameMonitorTree.CHECKPOINT_COLUMN, - PySide2.QtCore.Qt.DecorationRole).__class__) + qtpy.QtCore.Qt.DecorationRole).__class__) self.assertEqual( - PySide2.QtCore.Qt.AlignCenter, + qtpy.QtCore.Qt.AlignCenter, self.frameWidgetItem.data( - cuegui.FrameMonitorTree.STATUS_COLUMN, PySide2.QtCore.Qt.TextAlignmentRole)) + cuegui.FrameMonitorTree.STATUS_COLUMN, qtpy.QtCore.Qt.TextAlignmentRole)) self.assertEqual( - PySide2.QtCore.Qt.AlignRight, + qtpy.QtCore.Qt.AlignRight, self.frameWidgetItem.data( - cuegui.FrameMonitorTree.PROC_COLUMN, PySide2.QtCore.Qt.TextAlignmentRole)) + cuegui.FrameMonitorTree.PROC_COLUMN, qtpy.QtCore.Qt.TextAlignmentRole)) self.assertEqual( cuegui.Constants.TYPE_FRAME, - self.frameWidgetItem.data(dispatch_order_col, PySide2.QtCore.Qt.UserRole)) + self.frameWidgetItem.data(dispatch_order_col, qtpy.QtCore.Qt.UserRole)) if __name__ == '__main__': diff --git a/cuegui/tests/LayerDialog_tests.py b/cuegui/tests/LayerDialog_tests.py index 78d4d5060..f50e9cfab 100644 --- a/cuegui/tests/LayerDialog_tests.py +++ b/cuegui/tests/LayerDialog_tests.py @@ -20,9 +20,9 @@ import mock -import PySide2.QtCore -import PySide2.QtGui -import PySide2.QtWidgets +import qtpy.QtCore +import qtpy.QtGui +import qtpy.QtWidgets import opencue.compiled_proto.show_pb2 import opencue.compiled_proto.filter_pb2 @@ -48,7 +48,7 @@ class LayerPropertiesDialogTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, get_stub_mock, get_layer_mock, get_limits_mock): app = test_utils.createApplication() - app.settings = PySide2.QtCore.QSettings() + app.settings = qtpy.QtCore.QSettings() cuegui.Style.init() self.layers = { @@ -80,7 +80,7 @@ def setUp(self, get_stub_mock, get_layer_mock, get_limits_mock): opencue.compiled_proto.limit_pb2.Limit(id='limit4Id', name='limit4Name')), ] - self.parent_widget = PySide2.QtWidgets.QWidget() + self.parent_widget = qtpy.QtWidgets.QWidget() self.layer_properties_dialog = cuegui.LayerDialog.LayerPropertiesDialog( ['layer1Id', 'layer2Id'], parent=self.parent_widget) diff --git a/cuegui/tests/Layout_tests.py b/cuegui/tests/Layout_tests.py index c291c3b1e..2b6818f9c 100644 --- a/cuegui/tests/Layout_tests.py +++ b/cuegui/tests/Layout_tests.py @@ -25,7 +25,7 @@ import tempfile import unittest -from PySide2 import QtCore +from qtpy import QtCore import cuegui.Layout diff --git a/cuegui/tests/MenuActions_tests.py b/cuegui/tests/MenuActions_tests.py index 0127cc7ce..e65c590da 100644 --- a/cuegui/tests/MenuActions_tests.py +++ b/cuegui/tests/MenuActions_tests.py @@ -23,8 +23,8 @@ import unittest import mock -import PySide2.QtGui -import PySide2.QtWidgets +import qtpy.QtGui +import qtpy.QtWidgets import opencue.compiled_proto.depend_pb2 import opencue.compiled_proto.facility_pb2 @@ -101,7 +101,7 @@ def test_emailArtist(self, emailDialogMock): emailDialogMock.assert_called_with(job, self.widgetMock) emailDialogMock.return_value.show.assert_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_setMinCores(self, getDoubleMock): highest_current_core_count = 20 new_core_count = 50 @@ -123,7 +123,7 @@ def test_setMinCores(self, getDoubleMock): job1.setMinCores.assert_called_with(new_core_count) job2.setMinCores.assert_called_with(new_core_count) - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_setMinCoresCanceled(self, getDoubleMock): job1 = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(min_cores=0)) job1.setMinCores = mock.Mock() @@ -136,7 +136,7 @@ def test_setMinCoresCanceled(self, getDoubleMock): job1.setMinCores.assert_not_called() job2.setMinCores.assert_not_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_setMaxCores(self, getDoubleMock): highest_current_core_count = 20 new_core_count = 50 @@ -158,7 +158,7 @@ def test_setMaxCores(self, getDoubleMock): job1.setMaxCores.assert_called_with(new_core_count) job2.setMaxCores.assert_called_with(new_core_count) - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_setMaxCoresCanceled(self, getDoubleMock): job1 = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(max_cores=0)) job1.setMaxCores = mock.Mock() @@ -171,7 +171,7 @@ def test_setMaxCoresCanceled(self, getDoubleMock): job1.setMaxCores.assert_not_called() job2.setMaxCores.assert_not_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getInt') + @mock.patch('qtpy.QtWidgets.QInputDialog.getInt') def test_setPriority(self, getIntMock): job = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(priority=0)) job.setPriority = mock.Mock() @@ -182,7 +182,7 @@ def test_setPriority(self, getIntMock): job.setPriority.assert_called_with(new_priority) - @mock.patch('PySide2.QtWidgets.QInputDialog.getInt') + @mock.patch('qtpy.QtWidgets.QInputDialog.getInt') def test_setPriorityCanceled(self, getIntMock): job = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(priority=0)) job.setPriority = mock.Mock() @@ -192,7 +192,7 @@ def test_setPriorityCanceled(self, getIntMock): job.setPriority.assert_not_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getInt') + @mock.patch('qtpy.QtWidgets.QInputDialog.getInt') def test_setMaxRetries(self, getIntMock): job = opencue.wrappers.job.Job() job.setMaxRetries = mock.Mock() @@ -203,7 +203,7 @@ def test_setMaxRetries(self, getIntMock): job.setMaxRetries.assert_called_with(new_retries) - @mock.patch('PySide2.QtWidgets.QInputDialog.getInt') + @mock.patch('qtpy.QtWidgets.QInputDialog.getInt') def test_setMaxRetriesCanceled(self, getIntMock): job = opencue.wrappers.job.Job() job.setMaxRetries = mock.Mock() @@ -356,8 +356,8 @@ def test_dependWizard(self, dependWizardMock): dependWizardMock.assert_called_with(self.widgetMock, jobs) - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_reorder(self, getTextMock, getItemMock): original_range = '1-10' new_order = 'REVERSE' @@ -373,8 +373,8 @@ def test_reorder(self, getTextMock, getItemMock): job.reorderFrames.assert_called_with(original_range, opencue.compiled_proto.job_pb2.REVERSE) - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_reorderCanceled(self, getTextMock, getItemMock): original_range = '1-10' job = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(name='job-name')) @@ -397,8 +397,8 @@ def test_reorderCanceled(self, getTextMock, getItemMock): job.reorderFrames.assert_not_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getInt') - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getInt') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_stagger(self, getTextMock, getIntMock): original_range = '1-10' new_step = 28 @@ -414,8 +414,8 @@ def test_stagger(self, getTextMock, getIntMock): job.staggerFrames.assert_called_with(original_range, new_step) - @mock.patch('PySide2.QtWidgets.QInputDialog.getInt') - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getInt') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_staggerCanceled(self, getTextMock, getIntMock): original_range = '1-10' job = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(name='job-name')) @@ -507,7 +507,7 @@ def test_useLocalCores(self, localBookingDialogMock): localBookingDialogMock.assert_called_with(job, self.widgetMock) localBookingDialogMock.return_value.exec_.assert_called() - @mock.patch('PySide2.QtWidgets.QApplication.clipboard') + @mock.patch('qtpy.QtWidgets.QApplication.clipboard') def test_copyLogFileDir(self, clipboardMock): logDir1 = '/some/random/dir' logDir2 = '/a/different/random/dir' @@ -571,7 +571,7 @@ def test_viewDepends(self, dependDialogMock): dependDialogMock.return_value.show.assert_called() @mock.patch.object(opencue.wrappers.layer.Layer, 'setMinCores', autospec=True) - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_setMinCores(self, getDoubleMock, setMinCoresMock): highest_current_core_count = 20 new_core_count = 50 @@ -592,7 +592,7 @@ def test_setMinCores(self, getDoubleMock, setMinCoresMock): mock.call(layer1, new_core_count), mock.call(layer2, new_core_count)]) @mock.patch.object(opencue.wrappers.layer.Layer, 'setMinCores') - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_setMinCoresCanceled(self, getDoubleMock, setMinCoresMock): layer1 = opencue.wrappers.layer.Layer( opencue.compiled_proto.job_pb2.Layer(min_cores=0)) @@ -605,7 +605,7 @@ def test_setMinCoresCanceled(self, getDoubleMock, setMinCoresMock): setMinCoresMock.assert_not_called() @mock.patch.object(opencue.wrappers.layer.Layer, 'setMinMemory', autospec=True) - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_setMinMemoryKb(self, getDoubleMock, setMinMemoryMock): highest_current_mem_limit_gb = 20 new_mem_limit_gb = 50 @@ -630,7 +630,7 @@ def test_setMinMemoryKb(self, getDoubleMock, setMinMemoryMock): ]) @mock.patch.object(opencue.wrappers.layer.Layer, 'setMinMemory') - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_setMinMemoryKbCanceled(self, getDoubleMock, setMinMemoryMock): layer1 = opencue.wrappers.layer.Layer(opencue.compiled_proto.job_pb2.Layer(min_memory=0)) layer2 = opencue.wrappers.layer.Layer(opencue.compiled_proto.job_pb2.Layer(min_memory=0)) @@ -758,8 +758,8 @@ def test_dependWizard(self, dependWizardMock): dependWizardMock.assert_called_with(self.widgetMock, [self.job], layers=layers) @mock.patch.object(opencue.wrappers.layer.Layer, 'reorderFrames', autospec=True) - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_reorder(self, getTextMock, getItemMock, reorderFramesMock): original_range = '1-10' new_order = 'REVERSE' @@ -775,8 +775,8 @@ def test_reorder(self, getTextMock, getItemMock, reorderFramesMock): layer, original_range, opencue.compiled_proto.job_pb2.REVERSE) @mock.patch.object(opencue.wrappers.layer.Layer, 'staggerFrames', autospec=True) - @mock.patch('PySide2.QtWidgets.QInputDialog.getInt') - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getInt') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_stagger(self, getTextMock, getIntMock, staggerFramesMock): original_range = '1-10' new_step = 28 @@ -998,7 +998,7 @@ def test_markdone(self, yesNoMock): self.job.markdoneFrames.assert_called_with(name=[frame_name]) @mock.patch.object(opencue.wrappers.layer.Layer, 'reorderFrames', autospec=True) - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') def test_reorder(self, getItemMock, reorderFramesMock): new_order = 'REVERSE' getItemMock.return_value = (new_order, True) @@ -1015,7 +1015,7 @@ def test_reorder(self, getItemMock, reorderFramesMock): reorderFramesMock.assert_called_with( layer, str(frame_num), opencue.compiled_proto.job_pb2.REVERSE) - @mock.patch('PySide2.QtWidgets.QApplication.clipboard') + @mock.patch('qtpy.QtWidgets.QApplication.clipboard') @mock.patch('cuegui.Utils.getFrameLogFile') def test_copyLogFileName(self, getFrameLogFileMock, clipboardMock): frame_log_path = '/some/path/to/job/logs/job-name.frame-name.rqlog' @@ -1133,21 +1133,21 @@ def setUp(self): self.subscription_actions = cuegui.MenuActions.SubscriptionActions( self.widgetMock, mock.Mock(), None, None) - @mock.patch('PySide2.QtWidgets.QMessageBox') - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QMessageBox') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_editSize(self, getDoubleMock, qMessageBoxMock): sub = opencue.wrappers.subscription.Subscription( opencue.compiled_proto.subscription_pb2.Subscription(size=382)) sub.setSize = mock.MagicMock() newSize = 8479 getDoubleMock.return_value = (newSize, True) - qMessageBoxMock.return_value.exec_.return_value = PySide2.QtWidgets.QMessageBox.Yes + qMessageBoxMock.return_value.exec_.return_value = qtpy.QtWidgets.QMessageBox.Yes self.subscription_actions.editSize(rpcObjects=[sub]) sub.setSize.assert_called_with(newSize*100.0) - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_editBurst(self, getDoubleMock): sub = opencue.wrappers.subscription.Subscription( opencue.compiled_proto.subscription_pb2.Subscription(burst=922)) @@ -1249,7 +1249,7 @@ def test_rebootWhenIdle(self): host.rebootWhenIdle.assert_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_addTags(self, getTextMock): host = opencue.wrappers.host.Host( opencue.compiled_proto.host_pb2.Host(id='arbitrary-id')) @@ -1261,7 +1261,7 @@ def test_addTags(self, getTextMock): host.addTags.assert_called_with(['firstTag', 'anotherTag', 'oneMoreTag']) - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_removeTags(self, getTextMock): host = opencue.wrappers.host.Host( opencue.compiled_proto.host_pb2.Host( @@ -1273,8 +1273,8 @@ def test_removeTags(self, getTextMock): host.removeTags.assert_called_with(['firstTag', 'anotherTag', 'oneMoreTag']) - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') def test_renameTag(self, getItemMock, getTextMock): host = opencue.wrappers.host.Host( opencue.compiled_proto.host_pb2.Host(id='arbitrary-id')) @@ -1288,7 +1288,7 @@ def test_renameTag(self, getItemMock, getTextMock): host.renameTag.assert_called_with(oldTagName, newTagName) - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') @mock.patch('opencue.api.getAllocations') def test_changeAllocation(self, getAllocationsMock, getItemMock): host = opencue.wrappers.host.Host( @@ -1427,7 +1427,7 @@ def setUp(self): self.filter_actions = cuegui.MenuActions.FilterActions( self.widgetMock, mock.Mock(), None, None) - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_rename(self, getTextMock): filter_wrapper = opencue.wrappers.filter.Filter(opencue.compiled_proto.filter_pb2.Filter()) filter_wrapper.setName = mock.MagicMock() @@ -1479,7 +1479,7 @@ def test_orderLast(self): filter_wrapper.orderLast.assert_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getInt') + @mock.patch('qtpy.QtWidgets.QInputDialog.getInt') def test_setOrder(self, getTextMock): filter_wrapper = opencue.wrappers.filter.Filter(opencue.compiled_proto.filter_pb2.Filter()) filter_wrapper.setOrder = mock.MagicMock() @@ -1510,7 +1510,7 @@ def test_delete(self): matcher.delete.assert_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_setValue(self, getTextMock): matcher = opencue.wrappers.filter.Matcher(opencue.compiled_proto.filter_pb2.Matcher()) matcher.setValue = mock.MagicMock() @@ -1552,7 +1552,7 @@ def setUp(self): self.task_actions = cuegui.MenuActions.TaskActions( self.widgetMock, mock.Mock(), None, None) - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_setMinCores(self, getDoubleMock): task = opencue.wrappers.task.Task(opencue.compiled_proto.task_pb2.Task(min_cores=10)) task.setMinCores = mock.MagicMock() @@ -1592,7 +1592,7 @@ def setUp(self): self.widgetMock, mock.Mock(), None, None) @mock.patch('opencue.api.createLimit') - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_create(self, getTextMock, createLimitMock): limitName = 'newLimitName' getTextMock.return_value = ('%s \t ' % limitName, True) @@ -1610,7 +1610,7 @@ def test_delete(self): limit.delete.assert_called() - @mock.patch('PySide2.QtWidgets.QInputDialog.getDouble') + @mock.patch('qtpy.QtWidgets.QInputDialog.getDouble') def test_editMaxValue(self, getDoubleMock): limit = opencue.wrappers.limit.Limit(opencue.compiled_proto.limit_pb2.Limit(max_value=920)) limit.setMaxValue = mock.MagicMock() @@ -1622,7 +1622,7 @@ def test_editMaxValue(self, getDoubleMock): limit.setMaxValue.assert_called_with(newMaxValue) - @mock.patch('PySide2.QtWidgets.QInputDialog.getText') + @mock.patch('qtpy.QtWidgets.QInputDialog.getText') def test_rename(self, getTextMock): limit = opencue.wrappers.limit.Limit(opencue.compiled_proto.limit_pb2.Limit()) limit.rename = mock.MagicMock() diff --git a/cuegui/tests/Redirect_tests.py b/cuegui/tests/Redirect_tests.py index 5e1f6ad27..ecfffcc48 100644 --- a/cuegui/tests/Redirect_tests.py +++ b/cuegui/tests/Redirect_tests.py @@ -19,8 +19,8 @@ import unittest import mock -import PySide2.QtCore -import PySide2.QtGui +import qtpy.QtCore +import qtpy.QtGui import opencue.compiled_proto.show_pb2 import opencue.wrappers.show @@ -37,7 +37,7 @@ class RedirectTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, getStubMock): app = test_utils.createApplication() - app.settings = PySide2.QtCore.QSettings() + app.settings = qtpy.QtCore.QSettings() cuegui.Style.init() getStubMock.return_value.GetActiveShows.return_value = \ diff --git a/cuegui/tests/UnbookDialog_tests.py b/cuegui/tests/UnbookDialog_tests.py index a62b390e7..bc5c348d8 100644 --- a/cuegui/tests/UnbookDialog_tests.py +++ b/cuegui/tests/UnbookDialog_tests.py @@ -20,8 +20,8 @@ import mock -import PySide2.QtCore -import PySide2.QtGui +import qtpy.QtCore +import qtpy.QtGui import opencue.compiled_proto.criterion_pb2 import opencue.compiled_proto.host_pb2 @@ -47,7 +47,7 @@ class UnbookDialogTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, get_stub_mock, find_show_mock): app = test_utils.createApplication() - app.settings = PySide2.QtCore.QSettings() + app.settings = qtpy.QtCore.QSettings() cuegui.Style.init() show_name = 'showname' @@ -85,7 +85,7 @@ def test__should_show_all_jobs_and_subscriptions(self): self.assertEqual(self.tag_names, subscriptions_shown) self.assertEqual(self.tag_names, subscriptions_checked) - @mock.patch('PySide2.QtWidgets.QMessageBox', new=mock.Mock()) + @mock.patch('qtpy.QtWidgets.QMessageBox', new=mock.Mock()) @mock.patch('opencue.api.getProcs') def test__should_unbook_procs(self, get_procs_mock): num_procs = 17 @@ -135,9 +135,9 @@ def test__should_show_kill_confirmation_dialog(self, kill_dialog_mock): kill_dialog_mock.assert_called_with(expected_proc_search, mock.ANY) - @mock.patch('PySide2.QtWidgets.QMessageBox', new=mock.Mock()) + @mock.patch('qtpy.QtWidgets.QMessageBox', new=mock.Mock()) @mock.patch('opencue.api.getProcs') - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') @mock.patch('opencue.api.getActiveShows') def test__should_redirect_proc_to_group( self, get_active_shows_mock, get_item_mock, get_procs_mock): @@ -165,11 +165,11 @@ def test__should_redirect_proc_to_group( get_procs_mock.assert_called_with(**expected_proc_search.options) proc_to_redirect.redirectToGroup.assert_called_with(group, False) - @mock.patch('PySide2.QtWidgets.QMessageBox', new=mock.Mock()) + @mock.patch('qtpy.QtWidgets.QMessageBox', new=mock.Mock()) @mock.patch('opencue.api.getProcs') @mock.patch('cuegui.UnbookDialog.SelectItemsWithSearchDialog') @mock.patch('opencue.api.getJobs') - @mock.patch('PySide2.QtWidgets.QInputDialog.getItem') + @mock.patch('qtpy.QtWidgets.QInputDialog.getItem') @mock.patch('opencue.api.getActiveShows') def test__should_redirect_proc_to_job( self, get_active_shows_mock, get_item_mock, get_jobs_mock, select_job_mock, @@ -203,7 +203,7 @@ class SelectItemsWithSearchDialogTests(unittest.TestCase): def setUp(self): app = test_utils.createApplication() - app.settings = PySide2.QtCore.QSettings() + app.settings = qtpy.QtCore.QSettings() cuegui.Style.init() def test__should_display_all_items(self): @@ -246,10 +246,10 @@ class KillConfirmationDialogTests(unittest.TestCase): def setUp(self): app = test_utils.createApplication() - app.settings = PySide2.QtCore.QSettings() + app.settings = qtpy.QtCore.QSettings() cuegui.Style.init() - @mock.patch('PySide2.QtWidgets.QMessageBox.information', new=mock.Mock()) + @mock.patch('qtpy.QtWidgets.QMessageBox.information', new=mock.Mock()) @mock.patch('opencue.api.getProcs') def test__should_kill_procs(self, get_procs_mock): proc_search = opencue.search.ProcSearch( @@ -268,7 +268,7 @@ def test__should_kill_procs(self, get_procs_mock): proc1.kill.assert_called() proc2.kill.assert_called() - @mock.patch('PySide2.QtWidgets.QMessageBox.information', new=mock.Mock()) + @mock.patch('qtpy.QtWidgets.QMessageBox.information', new=mock.Mock()) @mock.patch('opencue.api.getProcs') def test__should_cancel_kill(self, get_procs_mock): proc_search = opencue.search.ProcSearch( diff --git a/cuegui/tests/plugins/LogViewPlugin_tests.py b/cuegui/tests/plugins/LogViewPlugin_tests.py index 8206176f8..a135747b5 100644 --- a/cuegui/tests/plugins/LogViewPlugin_tests.py +++ b/cuegui/tests/plugins/LogViewPlugin_tests.py @@ -22,10 +22,10 @@ import mock import pyfakefs.fake_filesystem_unittest -import PySide2.QtCore -import PySide2.QtGui -import PySide2.QtTest -import PySide2.QtWidgets +import qtpy.QtCore +import qtpy.QtGui +import qtpy.QtTest +import qtpy.QtWidgets import cuegui.Main import cuegui.plugins.LogViewPlugin @@ -64,9 +64,9 @@ def setUp(self): self.fs.create_file(self.logPath2, contents=_LOG_TEXT_2) test_utils.createApplication() - cuegui.app().settings = PySide2.QtCore.QSettings() + cuegui.app().settings = qtpy.QtCore.QSettings() cuegui.Style.init() - self.parentWidget = PySide2.QtWidgets.QMainWindow() + self.parentWidget = qtpy.QtWidgets.QMainWindow() self.logViewPlugin = cuegui.plugins.LogViewPlugin.LogViewPlugin(self.parentWidget) def test_shouldDisplayFirstLogFile(self): @@ -85,7 +85,7 @@ def test_shouldUpdateLogFile(self): def test_shouldHighlightAllSearchResults(self): cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( - PySide2.QtCore.Qt.CheckState.Unchecked) + qtpy.QtCore.Qt.CheckState.Unchecked) self.logViewPlugin.logview_widget._search_box.setText('lorem') self.logViewPlugin.logview_widget._search_button.click() @@ -102,7 +102,7 @@ def test_shouldHighlightAllSearchResults(self): def test_shouldMoveCursorToSecondSearchResult(self): cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( - PySide2.QtCore.Qt.CheckState.Unchecked) + qtpy.QtCore.Qt.CheckState.Unchecked) self.logViewPlugin.logview_widget._search_box.setText('lorem') self.logViewPlugin.logview_widget._search_button.click() @@ -116,7 +116,7 @@ def test_shouldMoveCursorToSecondSearchResult(self): def test_shouldMoveCursorLastSearchResult(self): cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( - PySide2.QtCore.Qt.CheckState.Unchecked) + qtpy.QtCore.Qt.CheckState.Unchecked) self.logViewPlugin.logview_widget._search_box.setText('lorem') self.logViewPlugin.logview_widget._search_button.click() @@ -130,7 +130,7 @@ def test_shouldMoveCursorLastSearchResult(self): def test_shouldPerformCaseInsensitiveSearch(self): cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( - PySide2.QtCore.Qt.CheckState.Checked) + qtpy.QtCore.Qt.CheckState.Checked) self.logViewPlugin.logview_widget._search_box.setText('lorem') self.logViewPlugin.logview_widget._search_button.click() @@ -143,12 +143,12 @@ def test_shouldPerformCaseInsensitiveSearch(self): @staticmethod def __isHighlighted(textBox, startPosition, selectionLength): - cursor = textBox.cursorForPosition(PySide2.QtCore.QPoint(0, 0)) + cursor = textBox.cursorForPosition(qtpy.QtCore.QPoint(0, 0)) cursor.setPosition(startPosition) - cursor.movePosition(PySide2.QtGui.QTextCursor.Right, - PySide2.QtGui.QTextCursor.KeepAnchor, + cursor.movePosition(qtpy.QtGui.QTextCursor.Right, + qtpy.QtGui.QTextCursor.KeepAnchor, selectionLength) - return cursor.charFormat().background() == PySide2.QtCore.Qt.red + return cursor.charFormat().background() == qtpy.QtCore.Qt.red if __name__ == '__main__': diff --git a/requirements_gui.txt b/requirements_gui.txt index c589c3517..b7ff2d6b0 100644 --- a/requirements_gui.txt +++ b/requirements_gui.txt @@ -1 +1,3 @@ PySide2==5.15.2.1 +QtPy==1.11.3;python_version<"3.7" +QtPy==2.3.0;python_version>="3.7" From a22391f518e92e54d234dce06597cca688d371a3 Mon Sep 17 00:00:00 2001 From: Nuwan Jayawardene Date: Thu, 2 Mar 2023 00:53:21 +0530 Subject: [PATCH 248/277] Update Blender version in sample Dockerfile. (#1259) --- samples/rqd/blender/Dockerfile | 34 +++++++++++++++++++ .../rqd/blender/blender2.79-docker/Dockerfile | 25 -------------- 2 files changed, 34 insertions(+), 25 deletions(-) create mode 100644 samples/rqd/blender/Dockerfile delete mode 100644 samples/rqd/blender/blender2.79-docker/Dockerfile diff --git a/samples/rqd/blender/Dockerfile b/samples/rqd/blender/Dockerfile new file mode 100644 index 000000000..208f3c7a7 --- /dev/null +++ b/samples/rqd/blender/Dockerfile @@ -0,0 +1,34 @@ +# Builds on the latest base image of RQD from Docker Hub +FROM opencue/rqd + +# Install dependencies to run Blender on the opencue/rqd image +RUN yum -y update +RUN yum -y install \ + bzip2 \ + libfreetype6 \ + libgl1-mesa-dev \ + libXi-devel \ + mesa-libGLU-devel \ + zlib-devel \ + libXinerama-devel \ + libXrandr-devel + +# Set Blender install directory +ARG BLENDER_INSTALL_DIR=/usr/local/blender + +# Set Blender download source +ARG BLENDER_DOWNLOAD_SRC=https://download.blender.org/release/Blender3.3/blender-3.3.3-linux-x64.tar.xz + +# Download and install Blender +RUN mkdir ${BLENDER_INSTALL_DIR} +RUN curl -SL ${BLENDER_DOWNLOAD_SRC} \ + -o blender.tar.xz + +RUN tar -xvf blender.tar.xz \ + -C ${BLENDER_INSTALL_DIR} \ + --strip-components=1 + +RUN rm blender.tar.xz + +# Verify Blender installation +RUN ${BLENDER_INSTALL_DIR}/blender --version diff --git a/samples/rqd/blender/blender2.79-docker/Dockerfile b/samples/rqd/blender/blender2.79-docker/Dockerfile deleted file mode 100644 index 2c1b908f7..000000000 --- a/samples/rqd/blender/blender2.79-docker/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -# Builds on the latest base image of RQD from Docker Hub -FROM opencue/rqd - -# Install dependencies to run Blender on the opencue/rqd image -RUN yum -y update -RUN yum -y install \ - bzip2 \ - libfreetype6 \ - libgl1-mesa-dev \ - libXi-devel \ - mesa-libGLU-devel \ - zlib-devel \ - libXinerama-devel \ - libXrandr-devel - -# Download and install Blender 2.79 -RUN mkdir /usr/local/blender -RUN curl -SL https://download.blender.org/release/Blender2.79/blender-2.79-linux-glibc219-x86_64.tar.bz2 \ - -o blender.tar.bz2 - -RUN tar -jxvf blender.tar.bz2 \ - -C /usr/local/blender \ - --strip-components=1 - -RUN rm blender.tar.bz2 From deef621ecae9369d62fcefb8f3d6260284fffb36 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 1 Mar 2023 16:23:09 -0500 Subject: [PATCH 249/277] [pyoutline] Move outline.cfg into the outline module. (#1252) --- pyoutline/outline/config.py | 11 ++++------- pyoutline/{etc => outline}/outline.cfg | 0 pyoutline/setup.py | 8 ++++++-- pyoutline/tests/config_test.py | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) rename pyoutline/{etc => outline}/outline.cfg (100%) diff --git a/pyoutline/outline/config.py b/pyoutline/outline/config.py index de74a8faa..8cf298f62 100644 --- a/pyoutline/outline/config.py +++ b/pyoutline/outline/config.py @@ -110,13 +110,10 @@ def read_config_from_disk(): config_file = config_from_user_profile if not config_file: - default_config_paths = [__file_path__.parent.parent.parent / 'etc' / 'outline.cfg', - __file_path__.parent.parent / 'etc' / 'outline.cfg'] - for default_config_path in default_config_paths: - logger.info('Loading default outline config from %s', default_config_path) - if default_config_path.exists(): - config_file = default_config_path - break + default_config_path = __file_path__.parent / 'outline.cfg' + logger.info('Loading default outline config from %s', default_config_path) + if default_config_path.exists(): + config_file = default_config_path if not config_file: raise FileNotFoundError('outline config file was not found') diff --git a/pyoutline/etc/outline.cfg b/pyoutline/outline/outline.cfg similarity index 100% rename from pyoutline/etc/outline.cfg rename to pyoutline/outline/outline.cfg diff --git a/pyoutline/setup.py b/pyoutline/setup.py index a18b50d4f..0ecc79e0c 100644 --- a/pyoutline/setup.py +++ b/pyoutline/setup.py @@ -52,8 +52,12 @@ packages=find_packages(exclude=['tests']), data_files=[ ('bin', ['bin/cuerunbase.py', 'bin/pycuerun', 'bin/util_qc_job_layer.py']), - ('etc', ['etc/outline.cfg']), - ('wrappers', ['wrappers/opencue_wrap_frame', 'wrappers/opencue_wrap_frame_no_ss', 'wrappers/local_wrap_frame']), + ('wrappers', [ + 'wrappers/opencue_wrap_frame', 'wrappers/opencue_wrap_frame_no_ss', + 'wrappers/local_wrap_frame']), ], + package_data={ + 'outline': ['outline.cfg'], + }, test_suite='tests', ) diff --git a/pyoutline/tests/config_test.py b/pyoutline/tests/config_test.py index 5e0132d61..a7e089e39 100644 --- a/pyoutline/tests/config_test.py +++ b/pyoutline/tests/config_test.py @@ -60,7 +60,7 @@ def test__should_load_default_values(self): self.assertIsNone(os.environ.get('OL_CONF')) self.assertIsNone(os.environ.get('OUTLINE_CONFIG_FILE')) self.fs.add_real_file( - os.path.join(os.path.dirname(os.path.dirname(outline.__file__)), 'etc', 'outline.cfg'), + os.path.join(os.path.dirname(outline.__file__), 'outline.cfg'), read_only=True) config = read_config_from_disk() From 2f7f2974c4033119b61da33fb3948a40411fe7c6 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 1 Mar 2023 16:24:00 -0500 Subject: [PATCH 250/277] Upgrade future to 0.18.3. (#1254) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 44adbe5b8..4ca2a8b42 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ 2to3==1.0 enum34==1.1.6 evdev==1.4.0;python_version<"3.0" and "linux" in sys_platform -future==0.17.1 +future==0.18.3 futures==3.2.0;python_version<"3.0" grpcio==1.26.0;python_version<"3.0" grpcio-tools==1.26.0;python_version<"3.0" From 60cbecd137146900a5dcb784b5a39d95e4c6fd85 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 1 Mar 2023 16:25:14 -0500 Subject: [PATCH 251/277] [cuebot] Configure test logging for gradle --info and --debug. (#1263) --- cuebot/build.gradle | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/cuebot/build.gradle b/cuebot/build.gradle index a9b7cfaa5..6715a0ae6 100644 --- a/cuebot/build.gradle +++ b/cuebot/build.gradle @@ -1,4 +1,7 @@ +import org.gradle.api.tasks.testing.logging.TestExceptionFormat +import org.gradle.api.tasks.testing.logging.TestLogEvent + buildscript { repositories { mavenCentral() @@ -167,3 +170,26 @@ tasks.withType(AbstractArchiveTask) { preserveFileTimestamps = false reproducibleFileOrder = true } + +tasks.withType(Test) { + // Configure logging when running Gradle with --info or --debug. + testLogging { + info { + // Don't show STANDARD_OUT messages, these clutter up the output + // and make it hard to find actual failures. + events TestLogEvent.FAILED + exceptionFormat TestExceptionFormat.FULL + showStandardStreams false + } + debug { + // Show everything. + events TestLogEvent.STARTED, + TestLogEvent.FAILED, + TestLogEvent.PASSED, + TestLogEvent.SKIPPED, + TestLogEvent.STANDARD_ERROR, + TestLogEvent.STANDARD_OUT + exceptionFormat TestExceptionFormat.FULL + } + } +} From 2b7bfdddef7b7994a0dc5800142ab6a22f88d528 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 1 Mar 2023 16:45:56 -0500 Subject: [PATCH 252/277] [cuegui] Fix Comment dialog macros load and add tests. (#1261) --- cuegui/cuegui/Comments.py | 7 +++- cuegui/tests/Comments_tests.py | 74 ++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 cuegui/tests/Comments_tests.py diff --git a/cuegui/cuegui/Comments.py b/cuegui/cuegui/Comments.py index c58c7bd79..9cccdf576 100644 --- a/cuegui/cuegui/Comments.py +++ b/cuegui/cuegui/Comments.py @@ -209,8 +209,11 @@ def refreshComments(self): def __macroLoad(self): """Loads the defined comment macros from settings""" - self.__macroList = pickle.loads( - str(self.app.settings.value("Comments", pickle.dumps({})))) + try: + self.__macroList = pickle.loads(self.app.settings.value("Comments", pickle.dumps({}))) + except TypeError: + self.__macroList = pickle.loads( + str(self.app.settings.value("Comments", pickle.dumps({})))) self.__macroRefresh() def __macroRefresh(self): diff --git a/cuegui/tests/Comments_tests.py b/cuegui/tests/Comments_tests.py new file mode 100644 index 000000000..29bdf613f --- /dev/null +++ b/cuegui/tests/Comments_tests.py @@ -0,0 +1,74 @@ +# Copyright (c) OpenCue Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Tests for cuegui.Comments.""" + + +import time +import unittest + +import mock + +from qtpy import QtCore +from qtpy import QtWidgets + +import opencue.compiled_proto.comment_pb2 +import opencue.compiled_proto.job_pb2 +import opencue.wrappers.comment +import opencue.wrappers.job + +import cuegui.Comments +import cuegui.Style + +from . import test_utils + + +@mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) +class CommentsTests(unittest.TestCase): + @mock.patch('opencue.cuebot.Cuebot.getStub') + def setUp(self, getStubMock): + app = test_utils.createApplication() + app.settings = QtCore.QSettings() + cuegui.Style.init() + + commentProto = opencue.compiled_proto.comment_pb2.Comment( + id='comment-id-1', timestamp=int(time.time()), user='user-who-made-comment', + subject='comment-subject', message='this is the comment message body') + self.comment = opencue.wrappers.comment.Comment(commentProto) + getStubMock.return_value.GetComments.return_value = \ + opencue.compiled_proto.job_pb2.JobGetCommentsResponse( + comments=opencue.compiled_proto.comment_pb2.CommentSeq(comments=[commentProto])) + + self.job = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(name='fooJob')) + self.parentWidget = QtWidgets.QWidget() + self.commentListDialog = cuegui.Comments.CommentListDialog( + self.job, parent=self.parentWidget) + + def test_shouldDisplayComment(self): + self.assertEqual( + 1, self.commentListDialog._CommentListDialog__treeSubjects.topLevelItemCount()) + gotTreeWidgetItem = self.commentListDialog._CommentListDialog__treeSubjects.topLevelItem(0) + gotComment = gotTreeWidgetItem._Comment__comment + self.assertEqual(self.comment.timestamp(), gotComment.timestamp()) + self.assertEqual(self.comment.user(), gotComment.user()) + self.assertEqual(self.comment.subject(), gotComment.subject()) + self.assertEqual(self.comment.message(), gotComment.message()) + + def test_shouldRefreshJobComments(self): + self.job.getComments = mock.Mock(return_value=[]) + + self.commentListDialog.refreshComments() + + self.job.getComments.assert_called() From f74c1acf7b23c63aa68bb870fe39ac6306a44991 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Thu, 16 Mar 2023 13:31:21 -0400 Subject: [PATCH 253/277] Notes from Mar 1 TSC meeting. (#1272) --- tsc/meetings/2023-03-01.md | 47 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 tsc/meetings/2023-03-01.md diff --git a/tsc/meetings/2023-03-01.md b/tsc/meetings/2023-03-01.md new file mode 100644 index 000000000..9c065a76c --- /dev/null +++ b/tsc/meetings/2023-03-01.md @@ -0,0 +1,47 @@ +# OpenCue TSC Meeting Notes 1 Mar 2023 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Managing database migrations + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1228 + * Sequential numbering in the repo creates problems for users with forks. Migrations in forks + need to be renumbered after the fact to not conflict, and reapplying these changes to a + production database is tricky. + * Suggestion: leave gaps in between main repo migrations, e.g. v30, v40, etc. Forks can fill + this in. + * This could also create problems if v40 conflicts with v32 for example, and cause a need to + renumber fork migrations still. + * Diego and Brian to do some further research on this. + * Another suggestion: fork migrations could use version numbers starting at a very high number + e.g. 1000. Fork migrations would always be applied on top of the full main repo schema. + * Any conflicts would need to be resolved by the user. + * Any new main repo migrations would need to be applied manually. Flyway won't apply v40 if + it thinks the database is at v1000. + * This might be the least painful option. +* Customizing frame display colors + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1246 + * Author has given more context in the PR. + * It feels wrong to store color information in the database, it's purely a visual change. + * Brian to take another look. + * Maybe we can change PR to use something like "reason code" in the database rather than color + information directly, and update cuegui.yaml to convert reason code -> color. +* Preparing for next release + * PySide6 cuegui changes + * Merged, done. + * CueSubmit PySide6 + * Not started yet. Need to include this in the same release. + * Update test script to run example job + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1253 + * Ready for review. + * Includes pycue and pyoutline example scripts which should be generally useful to users + beyond tests. + * Config guide doc + * https://github.com/AcademySoftwareFoundation/opencue.io/pull/274 + * Ready for review, to be merged/published once release is done. + * show_stats table PR + * Is there a better way to test potentially destructive changes? + * There's no easy way in pure SQL to verify changes before dropping cols/tables. + * We should expand our doc on applying database migrations to cover a db backup/restore. + * The current change seems fine, good to merge. From cd770343b9e1f6b0db9761b44e4ec95b4619f366 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 21 Mar 2023 12:50:26 -0400 Subject: [PATCH 254/277] Remove obsolete pyoutline subdir from Dockerfiles. (#1273) --- cuesubmit/Dockerfile | 1 - pyoutline/Dockerfile | 1 - 2 files changed, 2 deletions(-) diff --git a/cuesubmit/Dockerfile b/cuesubmit/Dockerfile index cc8f51178..6f97d671f 100644 --- a/cuesubmit/Dockerfile +++ b/cuesubmit/Dockerfile @@ -41,7 +41,6 @@ RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py COPY pyoutline/README.md ./pyoutline/ COPY pyoutline/setup.py ./pyoutline/ COPY pyoutline/bin ./pyoutline/bin -COPY pyoutline/etc ./pyoutline/etc COPY pyoutline/wrappers ./pyoutline/wrappers COPY pyoutline/outline ./pyoutline/outline diff --git a/pyoutline/Dockerfile b/pyoutline/Dockerfile index e898452e1..02f954c23 100644 --- a/pyoutline/Dockerfile +++ b/pyoutline/Dockerfile @@ -29,7 +29,6 @@ RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py COPY pyoutline/README.md ./pyoutline/ COPY pyoutline/setup.py ./pyoutline/ COPY pyoutline/bin ./pyoutline/bin -COPY pyoutline/etc ./pyoutline/etc COPY pyoutline/tests/ ./pyoutline/tests COPY pyoutline/wrappers ./pyoutline/wrappers COPY pyoutline/outline ./pyoutline/outline From d091d56c5a66cd5dbb4d2ca17d0a1696e1161b60 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 21 Mar 2023 13:11:45 -0400 Subject: [PATCH 255/277] [ci] Update test script to run a job and confirm it finishes. (#1253) --- .github/workflows/temp-integration-test.yml | 25 ++++++++ ci/run_integration_test.sh | 30 ++++++++- samples/outline-files/hello.outline | 6 -- samples/outline-files/hello_frame.outline | 6 -- samples/pycue/wait_for_job.py | 61 +++++++++++++++++++ .../hellomodule.py => pyoutline/basic_job.py} | 21 +++++-- .../example_module.py} | 0 7 files changed, 130 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/temp-integration-test.yml delete mode 100644 samples/outline-files/hello.outline delete mode 100644 samples/outline-files/hello_frame.outline create mode 100755 samples/pycue/wait_for_job.py rename samples/{outline-modules/hellomodule.py => pyoutline/basic_job.py} (56%) mode change 100644 => 100755 rename samples/{outline-modules/example.py => pyoutline/example_module.py} (100%) diff --git a/.github/workflows/temp-integration-test.yml b/.github/workflows/temp-integration-test.yml new file mode 100644 index 000000000..3a746276d --- /dev/null +++ b/.github/workflows/temp-integration-test.yml @@ -0,0 +1,25 @@ +name: (TEMP) OpenCue Integration Test + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + integration_test: + name: Run Integration Test + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Run test + run: ci/run_integration_test.sh + + - name: Archive log files + uses: actions/upload-artifact@v3 + if: ${{ always() }} + with: + name: test-logs + path: /tmp/opencue-test/*.log diff --git a/ci/run_integration_test.sh b/ci/run_integration_test.sh index ee0a37fe6..8bde5335d 100755 --- a/ci/run_integration_test.sh +++ b/ci/run_integration_test.sh @@ -1,4 +1,25 @@ #!/bin/bash +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# OpenCue integration test script +# +# Stands up a clean environment using Docker compose and verifies all +# components are functioning as expected. +# +# Run with: +# ./run_integration_test.sh set -e @@ -162,6 +183,13 @@ test_cueadmin() { fi } +run_job() { + samples/pyoutline/basic_job.py + job_name="testing-shot01-${USER}_basic_job" + samples/pycue/wait_for_job.py "${job_name}" --timeout 300 + log INFO "Job succeeded (PASS)" +} + cleanup() { docker compose rm --stop --force >>"${DOCKER_COMPOSE_LOG}" 2>&1 rm -rf "${RQD_ROOT}" || true @@ -224,7 +252,7 @@ main() { log INFO "Testing cueadmin..." test_cueadmin - # TODO Launch a job and verify it finishes. + run_job cleanup diff --git a/samples/outline-files/hello.outline b/samples/outline-files/hello.outline deleted file mode 100644 index 6288ca30c..000000000 --- a/samples/outline-files/hello.outline +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python - -from outline.modules.shell import Shell - -Shell("hello", - command="echo 'Hello World!'") diff --git a/samples/outline-files/hello_frame.outline b/samples/outline-files/hello_frame.outline deleted file mode 100644 index c060719e1..000000000 --- a/samples/outline-files/hello_frame.outline +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python - -from outline.modules.shell import Shell - -Shell("hello", - command="echo 'Hello frame %{FRAME}!'") diff --git a/samples/pycue/wait_for_job.py b/samples/pycue/wait_for_job.py new file mode 100755 index 000000000..d66de1a44 --- /dev/null +++ b/samples/pycue/wait_for_job.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Basic script that waits for a job to complete.""" + +import argparse +import datetime +import logging +import sys +import time + +import opencue +from opencue.wrappers.job import Job + + +def wait_for_job(job_name, timeout_sec=None): + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s %(levelname)s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S') + logging.info('Waiting for job %s...', job_name) + start_time = datetime.datetime.now() + while True: + if (datetime.datetime.now() - start_time).seconds > timeout_sec: + logging.error('Timed out') + return False + jobs = opencue.api.getJobs(job=[job_name], include_finished=True) + if not jobs: + logging.error("Job %s not found", job_name) + return False + job = jobs[0] + logging.info('Job state = %s', Job.JobState(job.state()).name) + if job.state() == Job.JobState.FINISHED: + logging.info('Job succeeded') + return True + if job.deadFrames() > 0: + logging.error('Job is failing') + return False + time.sleep(5) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("job_name", help="name of the job to wait for") + parser.add_argument("--timeout", help="number of seconds to wait before timing out", type=int) + args = parser.parse_args() + result = wait_for_job(args.job_name, timeout_sec=args.timeout) + if not result: + sys.exit(1) diff --git a/samples/outline-modules/hellomodule.py b/samples/pyoutline/basic_job.py old mode 100644 new mode 100755 similarity index 56% rename from samples/outline-modules/hellomodule.py rename to samples/pyoutline/basic_job.py index 2e8f09ae1..7cae6eb19 --- a/samples/outline-modules/hellomodule.py +++ b/samples/pyoutline/basic_job.py @@ -1,5 +1,4 @@ #!/usr/bin/env python - # Copyright Contributors to the OpenCue Project # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,11 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Basic job structure with a single layer and five frames. + +The frames just print out the current frame number.""" + + +import getpass -from outline import Outline, cuerun -from outline.modules.tutorial import HelloModule +import outline +import outline.cuerun +import outline.modules.shell -ol = Outline("my_job") -ol.add_layer(HelloModule("my_layer")) -cuerun.launch(ol, range="1-10", pause=True) +ol = outline.Outline( + 'basic_job', shot='shot01', show='testing', user=getpass.getuser()) +layer = outline.modules.shell.Shell( + 'echo_frame', command=['echo', '#IFRAME#'], chunk=1, threads=1, range='1-5') +ol.add_layer(layer) +outline.cuerun.launch(ol, use_pycuerun=False) diff --git a/samples/outline-modules/example.py b/samples/pyoutline/example_module.py similarity index 100% rename from samples/outline-modules/example.py rename to samples/pyoutline/example_module.py From 669d6c401034b560fb698b547efffa902ab6444c Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 21 Mar 2023 13:27:58 -0400 Subject: [PATCH 256/277] Remove temp workflow. (#1274) --- .github/workflows/temp-integration-test.yml | 25 --------------------- 1 file changed, 25 deletions(-) delete mode 100644 .github/workflows/temp-integration-test.yml diff --git a/.github/workflows/temp-integration-test.yml b/.github/workflows/temp-integration-test.yml deleted file mode 100644 index 3a746276d..000000000 --- a/.github/workflows/temp-integration-test.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: (TEMP) OpenCue Integration Test - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - integration_test: - name: Run Integration Test - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Run test - run: ci/run_integration_test.sh - - - name: Archive log files - uses: actions/upload-artifact@v3 - if: ${{ always() }} - with: - name: test-logs - path: /tmp/opencue-test/*.log From 6fa72ffba969510a109daf75abe3cca9b72bcf6e Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 30 May 2023 14:02:14 -0400 Subject: [PATCH 257/277] Notes from TSC meeting. (#1294) --- tsc/meetings/2023-04-12.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 tsc/meetings/2023-04-12.md diff --git a/tsc/meetings/2023-04-12.md b/tsc/meetings/2023-04-12.md new file mode 100644 index 000000000..43d001d65 --- /dev/null +++ b/tsc/meetings/2023-04-12.md @@ -0,0 +1,38 @@ +# OpenCue TSC Meeting Notes 12 Apr 2023 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Host ping time until marked as DOWN + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1265 + * This does seem long, any reason why? + * Diego: seems unusual. Should be less than a minute for host to be marked DOWN. CueGUI should + update 15-20s + * Should we make this a config setting? + * SPI to check on their code for differences + * Might need to lower default value, this is a good candidate for config flag. +* RQD config file overhaul + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1270 + * Would be good to get SPI input on this. + * Let's review in more detail. No immediate concerns. SPI has some similar configuration already +* Rez setup email thread + * https://lists.aswf.io/g/opencue-dev/topic/97805737#571 + * Diego: might make a better tutorial doc than merging into master branch. We don't want to + confuse new users with multiple packaging options. + * Look into spk, an OSS project. + * pip packages will make this setup much simpler. +* Prepending timestamps to RQD child process output + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1286 + * Doesn't modify any existing output other than prepending timestamp to each line. + * Linux-specific, configurable. +* Python 2 support + * Not ready to drop support for 2 entirely, especially python libraries. + * GUI should be fine to go 3-only. + * If we're going to do it, document which tag contains the last 2 support + * A py2 branch might be helpful if anyone wants to backport, but might have issues with our + versioning tooling. +* Blender plugin update + * Basic plugin is loading, currently navigating issues with installing pyoutline into Blender + environment. Will start to send test jobs soon + * Will continue to update email thread. From 48abfd819f0278d15d38f3f9f458a35a940cc66a Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Wed, 7 Jun 2023 14:10:29 -0700 Subject: [PATCH 258/277] Prepend every rqd log line with a timestamp (#1286) * Prepend every rqd log line with a timestamp Not all applications launched on rqd have logs with a timestamp, which makes is difficult to debug jobs that are taking more than expected on the cue. This feature prepends a timestamp for every line. * Add rqconstant to turn timestamp feature on/off * Add rqconstant to turn timestamp feature on/off --- rqd/rqd/rqconstants.py | 3 ++ rqd/rqd/rqcore.py | 108 ++++++++++++++++++++++++++++++++++++-- rqd/tests/rqcore_tests.py | 3 +- 3 files changed, 108 insertions(+), 6 deletions(-) diff --git a/rqd/rqd/rqconstants.py b/rqd/rqd/rqconstants.py index ad82b5daf..fe793970e 100644 --- a/rqd/rqd/rqconstants.py +++ b/rqd/rqd/rqconstants.py @@ -73,6 +73,7 @@ RQD_BECOME_JOB_USER = True RQD_CREATE_USER_IF_NOT_EXISTS = True RQD_TAGS = '' +RQD_PREPEND_TIMESTAMP = False KILL_SIGNAL = 9 if platform.system() == 'Linux': @@ -197,6 +198,8 @@ if config.has_option(__section, "FILE_LOG_LEVEL"): level = config.get(__section, "FILE_LOG_LEVEL") FILE_LOG_LEVEL = logging.getLevelName(level) + if config.has_option(__section, "RQD_PREPEND_TIMESTAMP"): + RQD_PREPEND_TIMESTAMP = config.getboolean(__section, "RQD_PREPEND_TIMESTAMP") # pylint: disable=broad-except except Exception as e: logging.warning( diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index 224485f2b..ce073ac45 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -315,14 +315,17 @@ def runLinux(self): else: tempCommand += [self._createCommandFile(runFrame.command)] - # Actual cwd is set by /shots/SHOW/home/perl/etc/qwrap.cuerun + if rqd.rqconstants.RQD_PREPEND_TIMESTAMP: + file_descriptor = subprocess.PIPE + else: + file_descriptor = self.rqlog # pylint: disable=subprocess-popen-preexec-fn frameInfo.forkedCommand = subprocess.Popen(tempCommand, env=self.frameEnv, cwd=self.rqCore.machine.getTempPath(), stdin=subprocess.PIPE, - stdout=self.rqlog, - stderr=self.rqlog, + stdout=file_descriptor, + stderr=file_descriptor, close_fds=True, preexec_fn=os.setsid) finally: @@ -335,6 +338,8 @@ def runLinux(self): self.rqCore.updateRss) self.rqCore.updateRssThread.start() + if rqd.rqconstants.RQD_PREPEND_TIMESTAMP: + pipe_to_file(frameInfo.forkedCommand.stdout, frameInfo.forkedCommand.stderr, self.rqlog) returncode = frameInfo.forkedCommand.wait() # Find exitStatus and exitSignal @@ -535,7 +540,7 @@ def run(self): else: raise RuntimeError(err) try: - self.rqlog = open(runFrame.log_dir_file, "w", 1) + self.rqlog = open(runFrame.log_dir_file, "w+", 1) self.waitForFile(runFrame.log_dir_file) # pylint: disable=broad-except except Exception as e: @@ -1161,3 +1166,98 @@ def sendStatusReport(self): def isWaitingForIdle(self): """Returns whether the host is waiting until idle to take some action.""" return self.__whenIdle + +def pipe_to_file(stdout, stderr, outfile): + """ + Prepend entries on stdout and stderr with a timestamp and write to outfile. + + The logic to poll stdout/stderr is inspired by the Popen.communicate implementation. + This feature is linux specific + """ + # Importing packages internally to avoid compatibility issues with Windows + + if stdout is None or stderr is None: + return + outfile.flush() + os.fsync(outfile) + + import select + import errno + + fd2file = {} + fd2output = {} + + poller = select.poll() + + def register_and_append(file_ojb, eventmask): + poller.register(file_ojb, eventmask) + fd2file[file_ojb.fileno()] = file_ojb + + def close_and_unregister_and_remove(fd, close=False): + poller.unregister(fd) + if close: + fd2file[fd].close() + fd2file.pop(fd) + + def print_and_flush_ln(fd, last_timestamp): + txt = ''.join(fd2output[fd]) + lines = txt.split('\n') + next_line_timestamp = None + + # Save the timestamp of the first break + if last_timestamp is None: + curr_line_timestamp = datetime.datetime.now().strftime("%H:%M:%S") + else: + curr_line_timestamp = last_timestamp + + # There are no line breaks + if len(lines) < 2: + return curr_line_timestamp + else: + next_line_timestamp = datetime.datetime.now().strftime("%H:%M:%S") + + remainder = lines[-1] + for line in lines[0:-1]: + print("[%s] %s" % (curr_line_timestamp, line), file=outfile) + outfile.flush() + os.fsync(outfile) + fd2output[fd] = [remainder] + + if next_line_timestamp is None: + return curr_line_timestamp + else: + return next_line_timestamp + + def translate_newlines(data): + data = data.decode("utf-8", "ignore") + return data.replace("\r\n", "\n").replace("\r", "\n") + + select_POLLIN_POLLPRI = select.POLLIN | select.POLLPRI + # stdout + register_and_append(stdout, select_POLLIN_POLLPRI) + fd2output[stdout.fileno()] = [] + + # stderr + register_and_append(stderr, select_POLLIN_POLLPRI) + fd2output[stderr.fileno()] = [] + + while fd2file: + try: + ready = poller.poll() + except select.error as e: + if e.args[0] == errno.EINTR: + continue + raise + + first_chunk_timestamp = None + for fd, mode in ready: + if mode & select_POLLIN_POLLPRI: + data = os.read(fd, 4096) + if not data: + close_and_unregister_and_remove(fd) + if not isinstance(data, str): + data = translate_newlines(data) + fd2output[fd].append(data) + first_chunk_timestamp = print_and_flush_ln(fd, first_chunk_timestamp) + else: + close_and_unregister_and_remove(fd) diff --git a/rqd/tests/rqcore_tests.py b/rqd/tests/rqcore_tests.py index aad307dac..256b96b2b 100644 --- a/rqd/tests/rqcore_tests.py +++ b/rqd/tests/rqcore_tests.py @@ -567,6 +567,7 @@ def setUp(self): @mock.patch('platform.system', new=mock.Mock(return_value='Linux')) @mock.patch('tempfile.gettempdir') + @mock.patch('rqd.rqcore.pipe_to_file', new=mock.MagicMock()) def test_runLinux(self, getTempDirMock, permsUser, timeMock, popenMock): # mkdirMock, openMock, # given currentTime = 1568070634.3 @@ -632,8 +633,6 @@ def test_runLinux(self, getTempDirMock, permsUser, timeMock, popenMock): # mkdir self.assertTrue(os.path.exists(logDir)) self.assertTrue(os.path.isfile(logFile)) _, kwargs = popenMock.call_args - self.assertEqual(logFile, kwargs['stdout'].name) - self.assertEqual(logFile, kwargs['stderr'].name) rqCore.network.reportRunningFrameCompletion.assert_called_with( rqd.compiled_proto.report_pb2.FrameCompleteReport( From 0199fb2445f03cb700c77eb11515e935479d124d Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 14 Jun 2023 17:53:52 -0400 Subject: [PATCH 259/277] Notes from June 7 TSC meeting. (#1298) --- tsc/meetings/2023-06-07.md | 44 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 tsc/meetings/2023-06-07.md diff --git a/tsc/meetings/2023-06-07.md b/tsc/meetings/2023-06-07.md new file mode 100644 index 000000000..98df98a61 --- /dev/null +++ b/tsc/meetings/2023-06-07.md @@ -0,0 +1,44 @@ +# OpenCue TSC Meeting Notes 7 Jun 2023 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Host ping time until marked as DOWN + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1265 + * Any update here? + * Needs some further verification and response. +* Appending timestamps to logs + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1286 + * LGTM, needs merge from master, looking into test failures. +* Cuesubmit batch of PRs + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1278 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1280 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1281 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1282 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1283 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1284 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1285 + * Reviews needed. + * Be careful we're not making CueSubmit too specialized, keep it generally useful for new + users. + * Let's invite the author to a TSC meeting soon. + * Improvements are good, is there something else we can offer? What would be helpful for + larger-studio users? Or is the Python library good enough? + * Best to expand pyoutline examples / docs to help developers who have already tried + CueSubmit. + * Build on basic example used in integration test script. +* Blender plugin update + * Currently testing job submission, blocked on some submission code. + * Loading python deps (opencue, filesequence) + * Can manually copy into blender plugin directory, but how to automate this? + * Does Blender offer alternatives e.g. configuring plugin path via env var? + * Look into creating additional packages, maybe as empty packages. +* Openshift Cuebot version + * Putting multiple Cuebots behind gRPC load balancer, and pointing RQD at the LB. Currently to + take a Cuebot offline all RQDs need to be restarted to move to a new Cuebot host, this solves + that problem. + * Would make a good tutorial or sample to include in the main repo. + * Prometheus export needs to be reworked. Currently using a separate client to query metrics, + which doesn't work with the LB setup as it will not redirect requests to a consistent Cuebot. + Working on a change to send metrics directly from Cuebot. From d2057d3f1c3c01d5a78fedd9a22fe8400d91ec83 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Fri, 16 Jun 2023 09:57:07 -0700 Subject: [PATCH 260/277] Migrate stats columns from show to show_stats (#1228) * Migrate stats columns from show to show_stats Frequently changing columns on the SHOW table were causing the show indexes to bloat faster than desired and impacting the system performance, specially in the Whiteboard queries for jobs. * Fix merge issues Add gpu columns * Version bump * Update cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java Signed-off-by: Diego Tavares da Silva * Creating a new show should also create a show_stats entry A show_stats row needs to be created and deleted together with a show row. This bug was introduced on https://gitlab.spimageworks.com/spi/dev/infrastructure/api/opencue/-/merge_requests/427 * Rename V21__AddShowStats.sql to V16__AddShowStats.sql --------- Signed-off-by: Diego Tavares da Silva --- .../spcue/dao/postgres/JobDaoJdbc.java | 2 +- .../spcue/dao/postgres/ShowDaoJdbc.java | 10 +++++- .../spcue/dao/postgres/WhiteboardDaoJdbc.java | 17 +++++++++- .../postgres/migrations/V16__AddShowStats.sql | 33 +++++++++++++++++++ .../resources/conf/ddl/postgres/seed_data.sql | 4 ++- .../spcue/test/dao/postgres/ShowDaoTests.java | 8 ++--- .../resources/conf/ddl/postgres/test_data.sql | 8 +++-- 7 files changed, 72 insertions(+), 10 deletions(-) create mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V16__AddShowStats.sql diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java index d32a5a259..a5f595f4e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java @@ -574,7 +574,7 @@ public void activateJob(JobInterface job, JobState jobState) { jobTotals[0] + jobTotals[1], layers.size(), job.getJobId()); getJdbcTemplate().update( - "UPDATE show SET int_frame_insert_count=int_frame_insert_count+?, int_job_insert_count=int_job_insert_count+1 WHERE pk_show=?", + "UPDATE show_stats SET int_frame_insert_count=int_frame_insert_count+?, int_job_insert_count=int_job_insert_count+1 WHERE pk_show=?", jobTotals[0] + jobTotals[1], job.getShowId()); updateState(job, jobState); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java index 8361a6c5f..86e126559 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java @@ -138,9 +138,15 @@ public ShowEntity getShowDetail(HostInterface host) { private static final String INSERT_SHOW = "INSERT INTO show (pk_show,str_name) VALUES (?,?)"; + private static final String INSERT_SHOW_STATS = + "INSERT INTO show_stats " + + "(pk_show, int_frame_insert_count, int_job_insert_count, int_frame_success_count, int_frame_fail_count) " + + "VALUES (?, 0, 0, 0, 0)"; + public void insertShow(ShowEntity show) { show.id = SqlUtil.genKeyRandom(); getJdbcTemplate().update(INSERT_SHOW, show.id, show.name); + getJdbcTemplate().update(INSERT_SHOW_STATS, show.id); } private static final String SHOW_EXISTS = @@ -169,6 +175,8 @@ public void delete(ShowInterface s) { s.getShowId()); getJdbcTemplate().update("DELETE FROM show_alias WHERE pk_show=?", s.getShowId()); + getJdbcTemplate().update("DELETE FROM show_stats WHERE pk_show=?", + s.getShowId()); getJdbcTemplate().update("DELETE FROM show WHERE pk_show=?", s.getShowId()); } @@ -262,7 +270,7 @@ public void updateFrameCounters(ShowInterface s, int exitStatus) { col = "int_frame_fail_count = int_frame_fail_count + 1"; } getJdbcTemplate().update( - "UPDATE show SET " + col + " WHERE pk_show=?", s.getShowId()); + "UPDATE show_stats SET " + col + " WHERE pk_show=?", s.getShowId()); } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index 38566470a..fe3217f96 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -2060,7 +2060,21 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { private static final String GET_SHOW = "SELECT " + - "show.*," + + "show.pk_show," + + "show.str_name," + + "show.b_paused," + + "show.int_default_min_cores," + + "show.int_default_max_cores," + + "show.int_default_min_gpus," + + "show.int_default_max_gpus," + + "show.b_booking_enabled," + + "show.b_dispatch_enabled," + + "show.b_active," + + "show.str_comment_email," + + "show_stats.int_frame_insert_count," + + "show_stats.int_job_insert_count," + + "show_stats.int_frame_success_count," + + "show_stats.int_frame_fail_count," + "COALESCE(vs_show_stat.int_pending_count,0) AS int_pending_count," + "COALESCE(vs_show_stat.int_running_count,0) AS int_running_count," + "COALESCE(vs_show_stat.int_dead_count,0) AS int_dead_count," + @@ -2069,6 +2083,7 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "COALESCE(vs_show_stat.int_job_count,0) AS int_job_count " + "FROM " + "show " + + "JOIN show_stats ON (show.pk_show = show_stats.pk_show) " + "LEFT JOIN vs_show_stat ON (vs_show_stat.pk_show = show.pk_show) " + "LEFT JOIN vs_show_resource ON (vs_show_resource.pk_show=show.pk_show) " + "WHERE " + diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V16__AddShowStats.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V16__AddShowStats.sql new file mode 100644 index 000000000..f97c14f7c --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V16__AddShowStats.sql @@ -0,0 +1,33 @@ +CREATE TABLE show_stats ( + pk_show VARCHAR(36) NOT NULL, + int_frame_insert_count BIGINT DEFAULT 0 NOT NULL, + int_job_insert_count BIGINT DEFAULT 0 NOT NULL, + int_frame_success_count BIGINT DEFAULT 0 NOT NULL, + int_frame_fail_count BIGINT DEFAULT 0 NOT NULL +); + +INSERT INTO show_stats ( + pk_show, + int_frame_insert_count, + int_job_insert_count, + int_frame_success_count, + int_frame_fail_count +) SELECT + pk_show, + int_frame_insert_count, + int_job_insert_count, + int_frame_success_count, + int_frame_fail_count + FROM show; + +CREATE UNIQUE INDEX c_show_stats_pk ON show_stats (pk_show); +ALTER TABLE show_stats ADD CONSTRAINT c_show_stats_pk PRIMARY KEY + USING INDEX c_show_stats_pk; + + +-- Destructive changes. Please test changes above prior to executing this. +ALTER TABLE show + DROP COLUMN int_frame_insert_count, + DROP COLUMN int_job_insert_count, + DROP COLUMN int_frame_success_count, + DROP COLUMN int_frame_fail_count; diff --git a/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql b/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql index ebebaa75d..f5707b62c 100644 --- a/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql +++ b/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql @@ -1,4 +1,6 @@ -Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000000', 'testing', 200000, 100, 0, 0, 0, 0, true, true, true); +Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000000', 'testing', 200000, 100, true, true, true); + +Insert into SHOW_STATS (PK_SHOW,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT) values ('00000000-0000-0000-0000-000000000000',0,0,0,0) Insert into SHOW_ALIAS (PK_SHOW_ALIAS,PK_SHOW,STR_NAME) values ('00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000000', 'test'); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java index d430ab3b0..e44393ab6 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java @@ -219,20 +219,20 @@ public void testUpdateActive() { public void testUpdateFrameCounters() { ShowEntity show = showDao.findShowDetail(SHOW_NAME); int frameSuccess = jdbcTemplate.queryForObject( - "SELECT int_frame_success_count FROM show WHERE pk_show=?", + "SELECT int_frame_success_count FROM show_stats WHERE pk_show=?", Integer.class, show.id); showDao.updateFrameCounters(show, 0); int frameSucces2 = jdbcTemplate.queryForObject( - "SELECT int_frame_success_count FROM show WHERE pk_show=?", + "SELECT int_frame_success_count FROM show_stats WHERE pk_show=?", Integer.class, show.id); assertEquals(frameSuccess + 1,frameSucces2); int frameFail= jdbcTemplate.queryForObject( - "SELECT int_frame_fail_count FROM show WHERE pk_show=?", + "SELECT int_frame_fail_count FROM show_stats WHERE pk_show=?", Integer.class, show.id); showDao.updateFrameCounters(show, 1); int frameFail2 = jdbcTemplate.queryForObject( - "SELECT int_frame_fail_count FROM show WHERE pk_show=?", + "SELECT int_frame_fail_count FROM show_stats WHERE pk_show=?", Integer.class, show.id); assertEquals(frameFail+ 1,frameFail2); } diff --git a/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql b/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql index ae7b77354..4f9c2a0a0 100644 --- a/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql +++ b/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql @@ -1,6 +1,10 @@ -Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000000','pipe',20000,100,0,0,0,0,true,true,true) +Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000000','pipe',20000,100,true,true,true) -Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000001','edu',20000,100,0,0,0,0,true,true,true) +Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000001','edu',20000,100,true,true,true) + +Insert into SHOW_STATS (PK_SHOW,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT) values ('00000000-0000-0000-0000-000000000000',0,0,0,0) + +Insert into SHOW_STATS (PK_SHOW,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT) values ('00000000-0000-0000-0000-000000000001',0,0,0,0) Insert into SHOW_ALIAS (PK_SHOW_ALIAS,PK_SHOW,STR_NAME) values ('00000000-0000-0000-0000-000000000001','00000000-0000-0000-0000-000000000000','fx') From 6703e07de6982c7372647d2aa0216b86979b320a Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Thu, 13 Jul 2023 14:03:37 -0400 Subject: [PATCH 261/277] Test pipeline fixes and refresh. (#1305) --- .github/workflows/testing-pipeline.yml | 67 +++++-------------- ci/run_python_tests.sh | 8 ++- ci/run_python_tests_pyside6.sh | 48 +++++++++++++ .../resources/conf/ddl/postgres/seed_data.sql | 2 +- cuegui/tests/Constants_tests.py | 7 +- requirements.txt | 3 +- rqd/rqd/rqcore.py | 8 +-- 7 files changed, 85 insertions(+), 58 deletions(-) create mode 100755 ci/run_python_tests_pyside6.sh diff --git a/.github/workflows/testing-pipeline.yml b/.github/workflows/testing-pipeline.yml index 7d2b59af9..15967a4a9 100644 --- a/.github/workflows/testing-pipeline.yml +++ b/.github/workflows/testing-pipeline.yml @@ -7,41 +7,20 @@ on: branches: [ master ] jobs: - test_python_2019: - name: Run Python Unit Tests (CY2019) - runs-on: ubuntu-latest - container: aswf/ci-opencue:2019 - steps: - - uses: actions/checkout@v3 - - name: Run Python Tests - run: ci/run_python_tests.sh - - test_cuebot_2019: - name: Build Cuebot and Run Unit Tests (CY2019) - runs-on: ubuntu-latest - container: - image: aswf/ci-opencue:2019 - steps: - - uses: actions/checkout@v3 - - name: Build with Gradle - run: | - chown -R aswfuser:aswfgroup . - su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser - - test_python_2020: - name: Run Python Unit Tests (CY2020) + test_python_2022: + name: Run Python Unit Tests (CY2022) runs-on: ubuntu-latest - container: aswf/ci-opencue:2020 + container: aswf/ci-opencue:2022 steps: - uses: actions/checkout@v3 - name: Run Python Tests - run: ci/run_python_tests.sh + run: ci/run_python_tests.sh --no-gui - test_cuebot_2020: - name: Build Cuebot and Run Unit Tests (CY2020) + test_cuebot_2022: + name: Build Cuebot and Run Unit Tests (CY2022) runs-on: ubuntu-latest container: - image: aswf/ci-opencue:2020 + image: aswf/ci-opencue:2022 steps: - uses: actions/checkout@v3 - name: Build with Gradle @@ -49,20 +28,20 @@ jobs: chown -R aswfuser:aswfgroup . su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser - test_python_2021: - name: Run Python Unit Tests (CY2021) + test_python_2023: + name: Run Python Unit Tests (CY2023) runs-on: ubuntu-latest - container: aswf/ci-opencue:2021 + container: aswf/ci-opencue:2023 steps: - uses: actions/checkout@v3 - name: Run Python Tests run: ci/run_python_tests.sh - test_cuebot_2021: - name: Build Cuebot and Run Unit Tests (CY2021) + test_cuebot_2023: + name: Build Cuebot and Run Unit Tests (CY2023) runs-on: ubuntu-latest container: - image: aswf/ci-opencue:2021 + image: aswf/ci-opencue:2023 steps: - uses: actions/checkout@v3 - name: Build with Gradle @@ -70,27 +49,15 @@ jobs: chown -R aswfuser:aswfgroup . su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser - test_python_2022: - name: Run Python Unit Tests (CY2022) + test_python2: + name: Run Python Unit Tests using Python2 runs-on: ubuntu-latest - container: aswf/ci-opencue:2022 + container: aswf/ci-opencue:2019 steps: - uses: actions/checkout@v3 - name: Run Python Tests run: ci/run_python_tests.sh - test_cuebot_2022: - name: Build Cuebot and Run Unit Tests (CY2022) - runs-on: ubuntu-latest - container: - image: aswf/ci-opencue:2022 - steps: - - uses: actions/checkout@v3 - - name: Build with Gradle - run: | - chown -R aswfuser:aswfgroup . - su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser - test_pyside6: name: Run CueGUI Tests using PySide6 runs-on: ubuntu-latest @@ -113,7 +80,7 @@ jobs: name: Test Documentation Build runs-on: ubuntu-latest container: - image: aswf/ci-opencue:2020 + image: aswf/ci-opencue:2023 steps: - uses: actions/checkout@v3 - name: Run Sphinx build diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index c6a51a015..5f1bfe294 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -1,7 +1,13 @@ #!/bin/bash +# Script for running OpenCue unit tests with PySide2. +# +# This script is written to be run within the OpenCue GitHub Actions environment. +# See `.github/workflows/testing-pipeline.yml`. + set -e +args=("$@") python_version=$(python -V 2>&1) echo "Will run tests using ${python_version}" @@ -23,6 +29,6 @@ PYTHONPATH=pycue:pyoutline python cuesubmit/setup.py test python rqd/setup.py test # Xvfb no longer supports Python 2. -if [[ "$python_version" =~ "Python 3" ]]; then +if [[ "$python_version" =~ "Python 3" && ${args[0]} != "--no-gui" ]]; then ci/run_gui_test.sh fi diff --git a/ci/run_python_tests_pyside6.sh b/ci/run_python_tests_pyside6.sh new file mode 100755 index 000000000..384841cfe --- /dev/null +++ b/ci/run_python_tests_pyside6.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Script for running OpenCue unit tests with PySide6. +# +# This script is written to be run within the OpenCue GitHub Actions environment. +# See `.github/workflows/testing-pipeline.yml`. + +set -e + +python_version=$(python -V 2>&1) +echo "Will run tests using ${python_version}" + +# NOTE: To run this in an almalinux environment, install these packages: +# yum -y install \ +# dbus-libs \ +# fontconfig \ +# gcc \ +# libxkbcommon-x11 \ +# mesa-libEGL-devel \ +# python-devel \ +# which \ +# xcb-util-keysyms \ +# xcb-util-image \ +# xcb-util-renderutil \ +# xcb-util-wm \ +# Xvfb + +# Install Python requirements. +python3 -m pip install --user -r requirements.txt -r requirements_gui.txt +# Replace PySide2 with PySide6. +python3 -m pip uninstall -y PySide2 +python3 -m pip install --user PySide6==6.3.2 + +# Protos need to have their Python code generated in order for tests to pass. +python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto +python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto + +# Fix compiled proto code for Python 3. +2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py + +python pycue/setup.py test +PYTHONPATH=pycue python pyoutline/setup.py test +PYTHONPATH=pycue python cueadmin/setup.py test +PYTHONPATH=pycue:pyoutline python cuesubmit/setup.py test +python rqd/setup.py test + +ci/run_gui_test.sh diff --git a/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql b/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql index f5707b62c..7b189174c 100644 --- a/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql +++ b/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql @@ -1,6 +1,6 @@ Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000000', 'testing', 200000, 100, true, true, true); -Insert into SHOW_STATS (PK_SHOW,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT) values ('00000000-0000-0000-0000-000000000000',0,0,0,0) +Insert into SHOW_STATS (PK_SHOW,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT) values ('00000000-0000-0000-0000-000000000000',0,0,0,0); Insert into SHOW_ALIAS (PK_SHOW_ALIAS,PK_SHOW,STR_NAME) values ('00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000000', 'test'); diff --git a/cuegui/tests/Constants_tests.py b/cuegui/tests/Constants_tests.py index 9466dcece..3cfe3866f 100644 --- a/cuegui/tests/Constants_tests.py +++ b/cuegui/tests/Constants_tests.py @@ -25,7 +25,7 @@ import mock import pyfakefs.fake_filesystem_unittest -from PySide2 import QtGui +from qtpy import QtGui import opencue import cuegui.Constants @@ -40,6 +40,7 @@ ''' +# pylint: disable=import-outside-toplevel,redefined-outer-name,reimported class ConstantsTests(pyfakefs.fake_filesystem_unittest.TestCase): def setUp(self): self.setUpPyfakefs() @@ -53,6 +54,7 @@ def test__should_load_user_config_from_env_var(self): self.fs.create_file(config_file_path, contents=CONFIG_YAML) os.environ['CUEGUI_CONFIG_FILE'] = config_file_path + import cuegui.Constants result = importlib.reload(cuegui.Constants) self.assertEqual('98.707.68', result.VERSION) @@ -65,6 +67,7 @@ def test__should_load_user_config_from_user_profile(self): config_file_path = '/home/username/.config/opencue/cuegui.yaml' self.fs.create_file(config_file_path, contents=CONFIG_YAML) + import cuegui.Constants result = importlib.reload(cuegui.Constants) self.assertEqual('98.707.68', result.VERSION) @@ -73,6 +76,7 @@ def test__should_load_user_config_from_user_profile(self): @mock.patch('platform.system', new=mock.Mock(return_value='Linux')) def test__should_use_default_values(self): + import cuegui.Constants result = importlib.reload(cuegui.Constants) self.assertNotEqual('98.707.68', result.VERSION) @@ -161,6 +165,7 @@ def test__should_use_default_values(self): @mock.patch('platform.system', new=mock.Mock(return_value='Darwin')) def test__should_use_mac_editor(self): + import cuegui.Constants result = importlib.reload(cuegui.Constants) self.assertEqual('open -t', result.DEFAULT_EDITOR) diff --git a/requirements.txt b/requirements.txt index 4ca2a8b42..262f681f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,8 @@ packaging==20.9 pathlib==1.0.1;python_version<"3.4" protobuf==3.17.3;python_version<"3.0" psutil==5.6.7 -pyfakefs==3.6 +pyfakefs==3.6;python_version<"3.7" +pyfakefs==5.2.3;python_version>="3.7" pylint==2.6.0;python_version>="3.7" pynput==1.7.6 PyYAML==5.1 diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index ce073ac45..842028732 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -1181,8 +1181,10 @@ def pipe_to_file(stdout, stderr, outfile): outfile.flush() os.fsync(outfile) + # pylint: disable=import-outside-toplevel import select import errno + # pylint: enable=import-outside-toplevel fd2file = {} fd2output = {} @@ -1213,8 +1215,7 @@ def print_and_flush_ln(fd, last_timestamp): # There are no line breaks if len(lines) < 2: return curr_line_timestamp - else: - next_line_timestamp = datetime.datetime.now().strftime("%H:%M:%S") + next_line_timestamp = datetime.datetime.now().strftime("%H:%M:%S") remainder = lines[-1] for line in lines[0:-1]: @@ -1225,8 +1226,7 @@ def print_and_flush_ln(fd, last_timestamp): if next_line_timestamp is None: return curr_line_timestamp - else: - return next_line_timestamp + return next_line_timestamp def translate_newlines(data): data = data.decode("utf-8", "ignore") From c1f335d22e59cdf75859aa14ecdfe43d9cb43e95 Mon Sep 17 00:00:00 2001 From: Kern Attila GERMAIN <5556461+KernAttila@users.noreply.github.com> Date: Wed, 9 Aug 2023 17:47:45 +0200 Subject: [PATCH 262/277] [cuesubmit] Add tooltip to display available command tokens. (#1278) --- .../spcue/dispatcher/DispatchSupportService.java | 4 ++++ cuesubmit/cuesubmit/Constants.py | 13 +++++++++++++ cuesubmit/cuesubmit/ui/Command.py | 8 ++++---- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java index c935024d6..887d0c29e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java @@ -404,6 +404,10 @@ public RunFrame prepareRqdRunFrame(VirtualProc proc, DispatchFrame frame) { .replaceAll("#JOB#", frame.jobName) .replaceAll("#FRAMESPEC#", frameSpec) .replaceAll("#FRAME#", frame.name)); + /* The special command tokens above (#ZFRAME# and others) are provided to the user in cuesubmit. + * see: cuesubmit/cuesubmit/Constants.py + * Update the Constant.py file when updating tokens here, they will appear in the cuesubmit tooltip popup. + */ frame.uid.ifPresent(builder::setUid); diff --git a/cuesubmit/cuesubmit/Constants.py b/cuesubmit/cuesubmit/Constants.py index 123f32723..b4f82f13f 100644 --- a/cuesubmit/cuesubmit/Constants.py +++ b/cuesubmit/cuesubmit/Constants.py @@ -39,6 +39,19 @@ BLENDER_RENDER_CMD = config.get('BLENDER_RENDER_CMD', 'blender') FRAME_TOKEN = config.get('FRAME_TOKEN', '#IFRAME#') +# Tokens are replaced by cuebot during dispatch with their computed value. +# see: cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java +# Update this file when updating tokens in cuebot, they will appear in the cuesubmit tooltip popup. +COMMAND_TOKENS = {'#ZFRAME#': 'Current frame with a padding of 4', + '#IFRAME#': 'Current frame', + '#FRAME_START#': 'First frame of chunk', + '#FRAME_END#': 'Last frame of chunk', + '#FRAME_CHUNK#': 'Chunk size', + '#FRAMESPEC#': 'Full frame range', + '#LAYER#': 'Name of the Layer', + '#JOB#': 'Name of the Job', + '#FRAME#': 'Name of the Frame' + } BLENDER_FORMATS = ['', 'AVIJPEG', 'AVIRAW', 'BMP', 'CINEON', 'DPX', 'EXR', 'HDR', 'IRIS', 'IRIZ', 'JP2', 'JPEG', 'MPEG', 'MULTILAYER', 'PNG', 'RAWTGA', 'TGA', 'TIFF'] BLENDER_OUTPUT_OPTIONS_URL = \ diff --git a/cuesubmit/cuesubmit/ui/Command.py b/cuesubmit/cuesubmit/ui/Command.py index 548e2a836..c1d144b24 100644 --- a/cuesubmit/cuesubmit/ui/Command.py +++ b/cuesubmit/cuesubmit/ui/Command.py @@ -23,6 +23,7 @@ from PySide2 import QtCore, QtWidgets from cuesubmit.ui import Widgets +from cuesubmit import Constants class CueCommandWidget(Widgets.CueHelpWidget): @@ -69,11 +70,10 @@ def __init__(self, *args, **kwargs): self.commandBox.setAccessibleName('commandBox') self.horizontalLine = Widgets.CueHLine() self.setFixedHeight(120) + tokensToolTip = '\n'.join([' {0} -- {1}'.format(token, info) + for token, info in Constants.COMMAND_TOKENS.items()]) self.commandBox.setToolTip('Enter the command to be run. Valid replacement tokens are:\n' - ' #IFRAME# -- frame number\n' - ' #LAYER# -- layer name\n' - ' #JOB# -- job name\n' - ' #FRAME# -- frame name') + + tokensToolTip) self.setupUi() def setupUi(self): From baa122a033420258d9488dacc4364da6d4c42ddb Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Wed, 27 Sep 2023 22:26:44 +0200 Subject: [PATCH 263/277] Fix reading stats from /proc/{pid}/statm (#1308) --- rqd/rqd/rqmachine.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 2fc5a32ce..65061ef82 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -216,7 +216,9 @@ def __updateGpuAndLlu(self, frame): def _getStatFields(self, pidFilePath): with open(pidFilePath, "r") as statFile: - return [None, None] + statFile.read().rsplit(")", 1)[-1].split() + stats = statFile.read().split() + stats[1] = stats[1].strip('()') + return stats def rssUpdate(self, frames): """Updates the rss and maxrss for all running frames""" From b606a593a2ab26f3b26caf806a1ce397c1491a40 Mon Sep 17 00:00:00 2001 From: Ramon Figueiredo Date: Wed, 27 Sep 2023 15:38:33 -0700 Subject: [PATCH 264/277] [cuebot] Prevent booking frames on hosts with no temp space. (#1306) --- .../com/imageworks/spcue/dao/CommentDao.java | 22 ++ .../com/imageworks/spcue/dao/HostDao.java | 8 + .../spcue/dao/postgres/CommentDaoJdbc.java | 13 + .../spcue/dao/postgres/HostDaoJdbc.java | 7 + .../spcue/dispatcher/HostReportHandler.java | 79 +++++- .../spcue/service/CommentManager.java | 22 ++ .../spcue/service/CommentManagerService.java | 12 + .../imageworks/spcue/service/HostManager.java | 8 + .../spcue/service/HostManagerService.java | 5 + cuebot/src/main/resources/opencue.properties | 6 + .../test/dao/criteria/ProcSearchTests.java | 6 +- .../test/dao/postgres/BookingDaoTests.java | 5 +- .../test/dao/postgres/CommentDaoTests.java | 5 +- .../spcue/test/dao/postgres/DeedDaoTests.java | 5 +- .../dao/postgres/DispatcherDaoFifoTests.java | 6 +- .../test/dao/postgres/DispatcherDaoTests.java | 5 +- .../test/dao/postgres/FrameDaoTests.java | 5 +- .../spcue/test/dao/postgres/HostDaoTests.java | 5 +- .../spcue/test/dao/postgres/ProcDaoTests.java | 5 +- .../spcue/test/dao/postgres/ShowDaoTests.java | 5 +- .../test/dao/postgres/WhiteboardDaoTests.java | 5 +- .../CoreUnitDispatcherGpuJobTests.java | 5 +- .../CoreUnitDispatcherGpuTests.java | 5 +- .../CoreUnitDispatcherGpusJobTests.java | 5 +- .../dispatcher/CoreUnitDispatcherTests.java | 6 +- .../test/dispatcher/DispatchSupportTests.java | 5 +- .../dispatcher/FrameCompleteHandlerTests.java | 10 +- .../test/dispatcher/HistoryControlTests.java | 5 +- .../dispatcher/HostReportHandlerGpuTests.java | 5 +- .../dispatcher/HostReportHandlerTests.java | 237 ++++++++++++++++-- .../test/dispatcher/LocalDispatcherTests.java | 5 +- .../test/dispatcher/RedirectManagerTests.java | 6 +- .../test/dispatcher/StrandedCoreTests.java | 6 +- .../test/dispatcher/TestBookingQueue.java | 5 +- .../test/service/BookingManagerTests.java | 5 +- .../spcue/test/service/HostManagerTests.java | 5 +- .../spcue/test/service/JobManagerTests.java | 5 +- .../spcue/test/service/OwnerManagerTests.java | 5 +- cuebot/src/test/resources/opencue.properties | 6 + 39 files changed, 489 insertions(+), 81 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/CommentDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/CommentDao.java index 5568bb2fd..08bea59b1 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/CommentDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/CommentDao.java @@ -23,6 +23,8 @@ import com.imageworks.spcue.HostInterface; import com.imageworks.spcue.JobInterface; +import java.util.List; + public interface CommentDao { /** @@ -32,6 +34,26 @@ public interface CommentDao { */ public void deleteComment(String id); + /** + * Deletes comments using host, user, and subject + * + * @param host + * @param user + * @param subject + * @return boolean: returns true if one or more comments where deleted + */ + public boolean deleteCommentByHostUserAndSubject(HostInterface host, String user, String subject); + + /** + * Get comments using host, user, and subject + * + * @param host + * @param user + * @param subject + * @return List + */ + public List getCommentsByHostUserAndSubject(HostInterface host, String user, String subject); + /** * Retrieves the specified comment. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java index 768bcdbd2..5ed18947e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java @@ -78,6 +78,14 @@ public interface HostDao { */ void updateHostState(HostInterface host, HardwareState state); + /** + * updates a host with the passed free temporary directory + * + * @param host + * @param freeTempDir + */ + void updateHostFreeTempDir(HostInterface host, Long freeTempDir); + /** * returns a full host detail * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/CommentDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/CommentDaoJdbc.java index 9587e41db..ea61f07bb 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/CommentDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/CommentDaoJdbc.java @@ -21,6 +21,7 @@ import java.sql.ResultSet; import java.sql.SQLException; +import java.util.List; import java.util.Map; import org.springframework.jdbc.core.RowMapper; @@ -71,6 +72,18 @@ public CommentDetail mapRow(ResultSet rs, int row) throws SQLException { } }; + public boolean deleteCommentByHostUserAndSubject(HostInterface host, String user, String subject) { + return getJdbcTemplate().update( + "DELETE FROM comments WHERE pk_host=? AND str_user=? AND str_subject=?", + host.getHostId(), user, subject) > 0; + } + + public List getCommentsByHostUserAndSubject(HostInterface host, String user, String subject) { + return getJdbcTemplate().query( + "SELECT * FROM comments WHERE pk_host=? AND str_user=? AND str_subject=?", + COMMENT_DETAIL_MAPPER, host.getHostId(), user, subject); + } + public CommentDetail getCommentDetail(String id) { return getJdbcTemplate().queryForObject( "SELECT * FROM comments WHERE pk_comment=?", diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java index 5c106335c..6fe898b44 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java @@ -523,6 +523,13 @@ public void updateHostState(HostInterface host, HardwareState state) { state.toString(), host.getHostId()); } + @Override + public void updateHostFreeTempDir(HostInterface host, Long freeTempDir) { + getJdbcTemplate().update( + "UPDATE host_stat SET int_mcp_free=? WHERE pk_host=?", + freeTempDir, host.getHostId()); + } + @Override public void updateHostSetAllocation(HostInterface host, AllocationInterface alloc) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java index d763cce53..2adef34fb 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java @@ -29,10 +29,13 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; import org.springframework.core.task.TaskRejectedException; import org.springframework.dao.DataAccessException; import org.springframework.dao.EmptyResultDataAccessException; +import com.imageworks.spcue.CommentDetail; import com.imageworks.spcue.DispatchHost; import com.imageworks.spcue.FrameInterface; import com.imageworks.spcue.JobEntity; @@ -57,6 +60,7 @@ import com.imageworks.spcue.rqd.RqdClient; import com.imageworks.spcue.rqd.RqdClientException; import com.imageworks.spcue.service.BookingManager; +import com.imageworks.spcue.service.CommentManager; import com.imageworks.spcue.service.HostManager; import com.imageworks.spcue.service.JobManager; import com.imageworks.spcue.service.JobManagerSupport; @@ -80,6 +84,14 @@ public class HostReportHandler { private JobManagerSupport jobManagerSupport; private JobDao jobDao; private LayerDao layerDao; + @Autowired + private Environment env; + @Autowired + private CommentManager commentManager; + // Comment constants + private static final String SUBJECT_COMMENT_FULL_TEMP_DIR = "Host set to REPAIR for not having enough storage " + + "space on the temporary directory (mcp)"; + private static final String CUEBOT_COMMENT_USER = "cuebot"; /** * Boolean to toggle if this class is accepting data or not. @@ -156,7 +168,7 @@ public void handleHostReport(HostReport report, boolean isBoot) { rhost.getLoad(), new Timestamp(rhost.getBootTime() * 1000l), rhost.getAttributesMap().get("SP_OS")); - changeHardwareState(host, report.getHost().getState(), isBoot); + changeHardwareState(host, report.getHost().getState(), isBoot, report.getHost().getFreeMcp()); changeNimbyState(host, report.getHost()); /** @@ -221,7 +233,14 @@ public void handleHostReport(HostReport report, boolean isBoot) { } } - if (host.idleCores < Dispatcher.CORE_POINTS_RESERVED_MIN) { + // The minimum amount of free space in the temporary directory to book a host + Long minBookableFreeTempDir = env.getRequiredProperty("dispatcher.min_bookable_free_temp_dir_kb", Long.class); + + if (minBookableFreeTempDir != -1 && report.getHost().getFreeMcp() < minBookableFreeTempDir) { + msg = String.format("%s doens't have enough free space in the temporary directory (mcp), %dMB needs %dMB", + host.name, (report.getHost().getFreeMcp()/1024), (minBookableFreeTempDir/1024)); + } + else if (host.idleCores < Dispatcher.CORE_POINTS_RESERVED_MIN) { msg = String.format("%s doesn't have enough idle cores, %d needs %d", host.name, host.idleCores, Dispatcher.CORE_POINTS_RESERVED_MIN); } @@ -231,7 +250,7 @@ else if (host.idleMemory < Dispatcher.MEM_RESERVED_MIN) { } else if (report.getHost().getFreeMem() < CueUtil.MB512) { msg = String.format("%s doens't have enough free system mem, %d needs %d", - host.name, report.getHost().getFreeMem(), Dispatcher.MEM_RESERVED_MIN); + host.name, report.getHost().getFreeMem(), Dispatcher.MEM_RESERVED_MIN); } else if(!host.hardwareState.equals(HardwareState.UP)) { msg = host + " is not in the Up state."; @@ -309,13 +328,61 @@ else if (!dispatchSupport.isCueBookable(host)) { * updated with a boot report. If the state is Repair, then state is * never updated via RQD. * + * + * Prevent cue frames from booking on hosts with full temporary directories. + * + * Change host state to REPAIR or UP according the amount of free space + * in the temporary directory: + * - Set the host state to REPAIR, when the amount of free space in the + * temporary directory is less than the minimum required. Add a comment with + * subject: SUBJECT_COMMENT_FULL_TEMP_DIR + * - Set the host state to UP, when the amount of free space in the temporary directory + * is greater or equals to the minimum required and the host has a comment with + * subject: SUBJECT_COMMENT_FULL_TEMP_DIR + * * @param host * @param reportState * @param isBoot + * @param freeTempDir */ - private void changeHardwareState(DispatchHost host, - HardwareState reportState, boolean isBoot) { + private void changeHardwareState(DispatchHost host, HardwareState reportState, boolean isBoot, long freeTempDir) { + + // The minimum amount of free space in the temporary directory to book a host + Long minBookableFreeTempDir = env.getRequiredProperty("dispatcher.min_bookable_free_temp_dir_kb", Long.class); + + // Prevent cue frames from booking on hosts with full temporary directories + if (minBookableFreeTempDir != -1) { + if (host.hardwareState == HardwareState.UP && freeTempDir < minBookableFreeTempDir) { + + // Insert a comment indicating that the Host status = Repair with reason = Full temporary directory + CommentDetail c = new CommentDetail(); + c.subject = SUBJECT_COMMENT_FULL_TEMP_DIR; + c.user = CUEBOT_COMMENT_USER; + c.timestamp = null; + c.message = "Host " + host.getName() + " marked as REPAIR. The current amount of free space in the " + + "temporary directory (mcp) is " + (freeTempDir/1024) + "MB. It must have at least " + + (minBookableFreeTempDir/1024) + "MB of free space in temporary directory"; + commentManager.addComment(host, c); + // Set the host state to REPAIR + hostManager.setHostState(host, HardwareState.REPAIR); + host.hardwareState = HardwareState.REPAIR; + + return; + } else if (host.hardwareState == HardwareState.REPAIR && freeTempDir >= minBookableFreeTempDir) { + // Check if the host with REPAIR status has comments with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and + // user=CUEBOT_COMMENT_USER and delete the comments, if they exists + boolean commentsDeleted = commentManager.deleteCommentByHostUserAndSubject(host, + CUEBOT_COMMENT_USER, SUBJECT_COMMENT_FULL_TEMP_DIR); + + if (commentsDeleted) { + // Set the host state to UP + hostManager.setHostState(host, HardwareState.UP); + host.hardwareState = HardwareState.UP; + return; + } + } + } // If the states are the same there is no reason to do this update. if (host.hardwareState.equals(reportState)) { @@ -374,7 +441,7 @@ private void changeNimbyState(DispatchHost host, RenderHost rh) { * locked if all cores are locked. * * @param host DispatchHost - * @param renderHost RenderHost + * @param coreInfo CoreDetail */ private void changeLockState(DispatchHost host, CoreDetail coreInfo) { if (host.lockState == LockState.LOCKED) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/CommentManager.java b/cuebot/src/main/java/com/imageworks/spcue/service/CommentManager.java index 10533c542..faee9dff9 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/CommentManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/CommentManager.java @@ -23,6 +23,8 @@ import com.imageworks.spcue.HostInterface; import com.imageworks.spcue.JobInterface; +import java.util.List; + public interface CommentManager { /** @@ -47,6 +49,26 @@ public interface CommentManager { */ public void deleteComment(String id); + /** + * Deletes comments using host, user, and subject + * + * @param host + * @param user + * @param subject + * @return boolean: returns true if one or more comments where deleted + */ + public boolean deleteCommentByHostUserAndSubject(HostInterface host, String user, String subject); + + /** + * Get comments using host, user, and subject + * + * @param host + * @param user + * @param subject + * @return List + */ + public List getCommentsByHostUserAndSubject(HostInterface host, String user, String subject); + /** * * @param id diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/CommentManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/CommentManagerService.java index cc9a016ef..b6d4430ec 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/CommentManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/CommentManagerService.java @@ -28,6 +28,8 @@ import com.imageworks.spcue.ShowEntity; import com.imageworks.spcue.dao.CommentDao; +import java.util.List; + @Transactional public class CommentManagerService implements CommentManager { @@ -55,6 +57,16 @@ public void deleteComment(String id) { commentDao.deleteComment(id); } + @Transactional(propagation = Propagation.REQUIRED) + public boolean deleteCommentByHostUserAndSubject(HostInterface host, String user, String subject) { + return commentDao.deleteCommentByHostUserAndSubject(host, user, subject); + } + + @Transactional(propagation = Propagation.REQUIRED) + public List getCommentsByHostUserAndSubject(HostInterface host, String user, String subject) { + return commentDao.getCommentsByHostUserAndSubject(host, user, subject); + } + @Transactional(propagation = Propagation.REQUIRED) public void setCommentSubject(String id, String subject) { commentDao.updateCommentSubject(id, subject); diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java b/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java index 8b176c77e..aaf401688 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java @@ -62,6 +62,14 @@ public interface HostManager { */ void setHostState(HostInterface host, HardwareState state); + /** + * Updates the free temporary directory (mcp) of a host. + * + * @param host HostInterface + * @param freeTempDir Long + */ + void setHostFreeTempDir(HostInterface host, Long freeTempDir); + /** * Return true if the host is swapping hard enough * that killing frames will save the entire machine. diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java index a7c5b0729..a1533d695 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java @@ -93,6 +93,11 @@ public void setHostState(HostInterface host, HardwareState state) { hostDao.updateHostState(host, state); } + @Override + public void setHostFreeTempDir(HostInterface host, Long freeTempDir) { + hostDao.updateHostFreeTempDir(host, freeTempDir); + } + @Override @Transactional(propagation = Propagation.REQUIRED, readOnly=true) public boolean isSwapping(HostInterface host) { diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index a08522eb1..6b2875899 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -110,6 +110,12 @@ dispatcher.report_queue.max_pool_size=8 # Queue capacity for handling Host Report. dispatcher.report_queue.queue_capacity=1000 +# The minimum amount of free space in the temporary directory (mcp) to book a host. +# E.g: 1G = 1048576 kB => dispatcher.min_bookable_free_temp_dir_kb=1048576 +# Default = -1 (deactivated) +# If equals to -1, it means the feature is turned off +dispatcher.min_bookable_free_temp_dir_kb=-1 + # Number of threads to keep in the pool for kill frame operation. dispatcher.kill_queue.core_pool_size=6 # Maximum number of threads to allow in the pool for kill frame operation. diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/criteria/ProcSearchTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/criteria/ProcSearchTests.java index 59f00df9a..78a13b321 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/criteria/ProcSearchTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/criteria/ProcSearchTests.java @@ -50,6 +50,7 @@ import com.imageworks.spcue.service.HostManager; import com.imageworks.spcue.service.JobLauncher; import com.imageworks.spcue.service.JobManager; +import com.imageworks.spcue.util.CueUtil; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; @@ -209,11 +210,12 @@ private void launchJobs() { private RenderHost.Builder buildRenderHost() { return RenderHost.newBuilder() .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/BookingDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/BookingDaoTests.java index c6c03d604..577b53eac 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/BookingDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/BookingDaoTests.java @@ -96,11 +96,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName("test_host") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap((int) CueUtil.GB16) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/CommentDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/CommentDaoTests.java index 668e666e9..9282d7b79 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/CommentDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/CommentDaoTests.java @@ -140,11 +140,12 @@ public void testInsertCommentOnHost() { RenderHost host = RenderHost.newBuilder() .setName("boo") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(15290520) .setFreeSwap(2076) .setLoad(1) - .setTotalMcp(19543) + .setTotalMcp(CueUtil.GB4) .setTotalMem(15290520) .setTotalSwap(2096) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DeedDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DeedDaoTests.java index a04e7e5e6..962b669bb 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DeedDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DeedDaoTests.java @@ -73,11 +73,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName("test_host") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(15290520) .setFreeSwap(2076) .setLoad(1) - .setTotalMcp(19543) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap((int) CueUtil.GB16) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java index c34396709..4f6db1072 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoFifoTests.java @@ -56,6 +56,7 @@ import com.imageworks.spcue.service.JobLauncher; import com.imageworks.spcue.service.JobManager; import com.imageworks.spcue.test.AssumingPostgresEngine; +import com.imageworks.spcue.util.CueUtil; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -147,11 +148,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java index 900f50afe..99fe2543a 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/DispatcherDaoTests.java @@ -146,11 +146,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/FrameDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/FrameDaoTests.java index 8d64d918e..6312e6502 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/FrameDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/FrameDaoTests.java @@ -114,11 +114,12 @@ public void create() { RenderHost host = RenderHost.newBuilder() .setName(HOST) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java index df965893b..a6261e464 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java @@ -86,12 +86,13 @@ public static RenderHost buildRenderHost(String name) { RenderHost host = RenderHost.newBuilder() .setName(name) .setBootTime(1192369572) - .setFreeMcp(7602) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(15290520) .setFreeSwap((int) CueUtil.MB512) .setLoad(1) .setNimbyEnabled(false) - .setTotalMcp(19543) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap((int) CueUtil.GB2) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java index ab95e7e1a..f6cabc23a 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java @@ -118,11 +118,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName("beta") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB32) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java index e44393ab6..ea0ed67b8 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ShowDaoTests.java @@ -71,11 +71,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName("test_host") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap((int) CueUtil.GB16) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java index d419b6ce9..293359f8d 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java @@ -266,11 +266,12 @@ public RenderHost getRenderHost() { RenderHost host = RenderHost.newBuilder() .setName(HOST) .setBootTime(1192369572) - .setFreeMcp(7602) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) Dispatcher.MEM_RESERVED_MIN * 4) .setFreeSwap(2076) .setLoad(1) - .setTotalMcp(19543) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) Dispatcher.MEM_RESERVED_MIN * 4) .setTotalSwap(2096) .setNimbyEnabled(true) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuJobTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuJobTests.java index 4cc1c1f03..55bd44463 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuJobTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuJobTests.java @@ -100,11 +100,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) CueUtil.GB8) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB8) .setTotalSwap((int) CueUtil.GB2) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuTests.java index 0a4f6b74a..c61c9553f 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpuTests.java @@ -100,11 +100,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) CueUtil.GB8) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB8) .setTotalSwap((int) CueUtil.GB2) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpusJobTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpusJobTests.java index 4972b8f9b..e2d1cb564 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpusJobTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherGpusJobTests.java @@ -112,11 +112,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) CueUtil.GB8) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(CueUtil.GB8) .setTotalSwap(CueUtil.GB2) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherTests.java index adb6c404d..89112dd69 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/CoreUnitDispatcherTests.java @@ -45,6 +45,7 @@ import com.imageworks.spcue.service.JobLauncher; import com.imageworks.spcue.service.JobManager; import com.imageworks.spcue.test.TransactionalTest; +import com.imageworks.spcue.util.CueUtil; import static org.junit.Assert.assertEquals; @@ -99,11 +100,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/DispatchSupportTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/DispatchSupportTests.java index 98c60fd9c..55a7806c0 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/DispatchSupportTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/DispatchSupportTests.java @@ -96,11 +96,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java index f022fc687..d313c5293 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java @@ -115,11 +115,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) CueUtil.GB8) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(CueUtil.GB8) .setTotalSwap(CueUtil.GB2) .setNimbyEnabled(false) @@ -139,11 +140,12 @@ public void createHost() { RenderHost host2 = RenderHost.newBuilder() .setName(HOSTNAME2) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) CueUtil.GB4) .setFreeSwap((int) CueUtil.GB4) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB8) .setTotalSwap((int) CueUtil.GB8) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java index 138a3f33c..de67ff26a 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java @@ -102,11 +102,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) CueUtil.GB8) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(CueUtil.GB8) .setTotalSwap(CueUtil.GB2) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java index dee9d0792..ce1e98ae1 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java @@ -81,11 +81,12 @@ private static RenderHost getRenderHost() { return RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(1048576L * 4096) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java index d27f76c32..970b97a95 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java @@ -22,6 +22,7 @@ import java.io.File; import java.sql.Timestamp; import java.util.List; +import java.util.concurrent.ThreadPoolExecutor; import javax.annotation.Resource; import org.junit.Before; @@ -31,6 +32,7 @@ import org.springframework.transaction.annotation.Transactional; import com.imageworks.spcue.AllocationEntity; +import com.imageworks.spcue.CommentDetail; import com.imageworks.spcue.DispatchHost; import com.imageworks.spcue.dispatcher.Dispatcher; import com.imageworks.spcue.dispatcher.HostReportHandler; @@ -43,6 +45,7 @@ import com.imageworks.spcue.grpc.report.RenderHost; import com.imageworks.spcue.grpc.report.RunningFrameInfo; import com.imageworks.spcue.service.AdminManager; +import com.imageworks.spcue.service.CommentManager; import com.imageworks.spcue.service.HostManager; import com.imageworks.spcue.service.JobLauncher; import com.imageworks.spcue.service.JobManager; @@ -50,7 +53,10 @@ import com.imageworks.spcue.util.CueUtil; import com.imageworks.spcue.VirtualProc; +import java.util.UUID; + import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; @ContextConfiguration public class HostReportHandlerTests extends TransactionalTest { @@ -73,8 +79,16 @@ public class HostReportHandlerTests extends TransactionalTest { @Resource JobManager jobManager; + @Resource + CommentManager commentManager; + private static final String HOSTNAME = "beta"; private static final String NEW_HOSTNAME = "gamma"; + private String hostname; + private String hostname2; + private static final String SUBJECT_COMMENT_FULL_TEMP_DIR = "Host set to REPAIR for not having enough storage " + + "space on the temporary directory (mcp)"; + private static final String CUEBOT_COMMENT_USER = "cuebot"; @Before public void setTestMode() { @@ -83,7 +97,11 @@ public void setTestMode() { @Before public void createHost() { - hostManager.createHost(getRenderHost(), + hostname = UUID.randomUUID().toString().substring(0, 8); + hostname2 = UUID.randomUUID().toString().substring(0, 8); + hostManager.createHost(getRenderHost(hostname, HardwareState.UP), + adminManager.findAllocationDetail("spi","general")); + hostManager.createHost(getRenderHost(hostname2, HardwareState.UP), adminManager.findAllocationDetail("spi","general")); } @@ -96,26 +114,51 @@ private static CoreDetail getCoreDetail(int total, int idle, int booked, int loc .build(); } - private DispatchHost getHost() { - return hostManager.findDispatchHost(HOSTNAME); + private DispatchHost getHost(String hostname) { + return hostManager.findDispatchHost(hostname); } - private static RenderHost getRenderHost() { + private static RenderHost getRenderHost(String hostname, HardwareState state) { return RenderHost.newBuilder() - .setName(HOSTNAME) + .setName(hostname) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem((int) CueUtil.GB8) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(CueUtil.GB8) .setTotalSwap(CueUtil.GB2) .setNimbyEnabled(false) .setNumProcs(2) .setCoresPerProc(100) .addTags("test") - .setState(HardwareState.UP) + .setState(state) + .setFacility("spi") + .putAttributes("SP_OS", "Linux") + .setFreeGpuMem((int) CueUtil.MB512) + .setTotalGpuMem((int) CueUtil.MB512) + .build(); + } + + private static RenderHost getRenderHost(String hostname, HardwareState state, Long freeTempDir) { + return RenderHost.newBuilder() + .setName(hostname) + .setBootTime(1192369572) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(freeTempDir) + .setFreeMem((int) CueUtil.GB8) + .setFreeSwap(20760) + .setLoad(0) + .setTotalMcp(freeTempDir * 4) + .setTotalMem(CueUtil.GB8) + .setTotalSwap(CueUtil.GB2) + .setNimbyEnabled(false) + .setNumProcs(2) + .setCoresPerProc(100) + .addTags("test") + .setState(state) .setFacility("spi") .putAttributes("SP_OS", "Linux") .setFreeGpuMem((int) CueUtil.MB512) @@ -127,11 +170,12 @@ private static RenderHost getNewRenderHost(String tags) { return RenderHost.newBuilder() .setName(NEW_HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) @@ -149,17 +193,40 @@ private static RenderHost getNewRenderHost(String tags) { @Test @Transactional @Rollback(true) - public void testHandleHostReport() { - boolean isBoot = false; + public void testHandleHostReport() throws InterruptedException { CoreDetail cores = getCoreDetail(200, 200, 0, 0); - HostReport report = HostReport.newBuilder() - .setHost(getRenderHost()) + HostReport report1 = HostReport.newBuilder() + .setHost(getRenderHost(hostname, HardwareState.UP)) .setCoreInfo(cores) .build(); + HostReport report2 = HostReport.newBuilder() + .setHost(getRenderHost(hostname2, HardwareState.UP)) + .setCoreInfo(cores) + .build(); + HostReport report1_2 = HostReport.newBuilder() + .setHost(getRenderHost(hostname, HardwareState.UP)) + .setCoreInfo(getCoreDetail(200, 200, 100, 0)) + .build(); - hostReportHandler.handleHostReport(report, isBoot); - DispatchHost host = getHost(); - assertEquals(host.lockState, LockState.OPEN); + hostReportHandler.handleHostReport(report1, false); + DispatchHost host = getHost(hostname); + assertEquals(LockState.OPEN, host.lockState); + assertEquals(HardwareState.UP, host.hardwareState); + hostReportHandler.handleHostReport(report1_2, false); + host = getHost(hostname); + assertEquals(HardwareState.UP, host.hardwareState); + + // Test Queue thread handling + ThreadPoolExecutor queue = hostReportHandler.getReportQueue(); + // Make sure jobs flow normally without any nullpointer exception + // Expecting results from a ThreadPool based class on JUnit is tricky + // A future test will be developed in the future to better address the behavior of + // this feature + hostReportHandler.queueHostReport(report1); // HOSTNAME + hostReportHandler.queueHostReport(report2); // HOSTNAME2 + hostReportHandler.queueHostReport(report1); // HOSTNAME + hostReportHandler.queueHostReport(report1); // HOSTNAME + hostReportHandler.queueHostReport(report1_2); // HOSTNAME } @Test @@ -228,6 +295,138 @@ public void testHandleHostReportWithNonExistentTags() { assertEquals(host.getAllocationId(), alloc.id); } + @Test + @Transactional + @Rollback(true) + public void testHandleHostReportWithFullTemporaryDirectories() { + // Create CoreDetail + CoreDetail cores = getCoreDetail(200, 200, 0, 0); + + /* + * Test 1: + * Precondition: + * - HardwareState=UP + * Action: + * - Receives a HostReport with freeTempDir < dispatcher.min_bookable_free_temp_dir_kb (opencue.properties) + * Postcondition: + * - Host hardwareState changes to REPAIR + * - A comment is created with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and user=CUEBOT_COMMENT_USER + * */ + // Create HostReport + HostReport report1 = HostReport.newBuilder() + .setHost(getRenderHost(hostname, HardwareState.UP, 1024L)) + .setCoreInfo(cores) + .build(); + // Call handleHostReport() => Create the comment with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and change the + // host's hardwareState to REPAIR + hostReportHandler.handleHostReport(report1, false); + // Get host + DispatchHost host = getHost(hostname); + // Get list of comments by host, user, and subject + List comments = commentManager.getCommentsByHostUserAndSubject(host, CUEBOT_COMMENT_USER, + SUBJECT_COMMENT_FULL_TEMP_DIR); + // Check if there is 1 comment + assertEquals(comments.size(), 1); + // Get host comment + CommentDetail comment = comments.get(0); + // Check if the comment has the user = CUEBOT_COMMENT_USER + assertEquals(comment.user, CUEBOT_COMMENT_USER); + // Check if the comment has the subject = SUBJECT_COMMENT_FULL_TEMP_DIR + assertEquals(comment.subject, SUBJECT_COMMENT_FULL_TEMP_DIR); + // Check host lock state + assertEquals(LockState.OPEN, host.lockState); + // Check if host hardware state is REPAIR + assertEquals(HardwareState.REPAIR, host.hardwareState); + // Test Queue thread handling + ThreadPoolExecutor queue = hostReportHandler.getReportQueue(); + // Make sure jobs flow normally without any nullpointer exception + hostReportHandler.queueHostReport(report1); // HOSTNAME + hostReportHandler.queueHostReport(report1); // HOSTNAME + + /* + * Test 2: + * Precondition: + * - HardwareState=REPAIR + * - There is a comment for the host with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and user=CUEBOT_COMMENT_USER + * Action: + * - Receives a HostReport with freeTempDir >= dispatcher.min_bookable_free_temp_dir_kb (opencue.properties) + * Postcondition: + * - Host hardwareState changes to UP + * - Comment with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and user=CUEBOT_COMMENT_USER gets deleted + * */ + // Set the host freeTempDir to the minimum size required = 1GB (1048576 KB) + HostReport report2 = HostReport.newBuilder() + .setHost(getRenderHost(hostname, HardwareState.UP, 1048576L)) + .setCoreInfo(cores) + .build(); + // Call handleHostReport() => Delete the comment with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and change the + // host's hardwareState to UP + hostReportHandler.handleHostReport(report2, false); + // Get host + host = getHost(hostname); + // Get list of comments by host, user, and subject + comments = commentManager.getCommentsByHostUserAndSubject(host, CUEBOT_COMMENT_USER, + SUBJECT_COMMENT_FULL_TEMP_DIR); + // Check if there is no comment associated with the host + assertEquals(comments.size(), 0); + // Check host lock state + assertEquals(LockState.OPEN, host.lockState); + // Check if host hardware state is UP + assertEquals(HardwareState.UP, host.hardwareState); + // Test Queue thread handling + queue = hostReportHandler.getReportQueue(); + // Make sure jobs flow normally without any nullpointer exception + hostReportHandler.queueHostReport(report1); // HOSTNAME + hostReportHandler.queueHostReport(report1); // HOSTNAME + } + + @Test + @Transactional + @Rollback(true) + public void testHandleHostReportWithHardwareStateRepairNotRelatedToFullTempDir() { + // Create CoreDetail + CoreDetail cores = getCoreDetail(200, 200, 0, 0); + + /* + * Test if host.hardwareState == HardwareState.REPAIR + * (Not related to freeMcp < dispatcher.min_bookable_free_mcp_kb (opencue.properties)) + * + * - There is no comment with subject=SUBJECT_COMMENT_FULL_MCP_DIR and user=CUEBOT_COMMENT_USER associated with + * the host + * The host.hardwareState continue as HardwareState.REPAIR + * */ + // Create HostReport + HostReport report = HostReport.newBuilder() + .setHost(getRenderHost(hostname, HardwareState.UP, 1048576L)) + .setCoreInfo(cores) + .build(); + // Get host + DispatchHost host = getHost(hostname); + // Host's HardwareState set to REPAIR + hostManager.setHostState(host, HardwareState.REPAIR); + host.hardwareState = HardwareState.REPAIR; + // Get list of comments by host, user, and subject + List hostComments = commentManager.getCommentsByHostUserAndSubject(host, CUEBOT_COMMENT_USER, + SUBJECT_COMMENT_FULL_TEMP_DIR); + // Check if there is no comment + assertEquals(hostComments.size(), 0); + // There is no comment to delete + boolean commentsDeleted = commentManager.deleteCommentByHostUserAndSubject(host, + CUEBOT_COMMENT_USER, SUBJECT_COMMENT_FULL_TEMP_DIR); + assertFalse(commentsDeleted); + // Call handleHostReport() + hostReportHandler.handleHostReport(report, false); + // Check host lock state + assertEquals(LockState.OPEN, host.lockState); + // Check if host hardware state is REPAIR + assertEquals(HardwareState.REPAIR, host.hardwareState); + // Test Queue thread handling + ThreadPoolExecutor queueThread = hostReportHandler.getReportQueue(); + // Make sure jobs flow normally without any nullpointer exception + hostReportHandler.queueHostReport(report); // HOSTNAME + hostReportHandler.queueHostReport(report); // HOSTNAME + } + @Test @Transactional @Rollback(true) @@ -235,7 +434,7 @@ public void testMemoryAndLlu() { jobLauncher.testMode = true; jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec_simple.xml")); - DispatchHost host = getHost(); + DispatchHost host = getHost(hostname); List procs = dispatcher.dispatchHost(host); assertEquals(1, procs.size()); VirtualProc proc = procs.get(0); @@ -252,7 +451,7 @@ public void testMemoryAndLlu() { .setMaxRss(420000) .build(); HostReport report = HostReport.newBuilder() - .setHost(getRenderHost()) + .setHost(getRenderHost(hostname, HardwareState.UP)) .setCoreInfo(cores) .addFrames(info) .build(); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/LocalDispatcherTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/LocalDispatcherTests.java index 97a270085..a7218b47a 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/LocalDispatcherTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/LocalDispatcherTests.java @@ -95,11 +95,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(0) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/RedirectManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/RedirectManagerTests.java index 25ccf69c5..70e3db4af 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/RedirectManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/RedirectManagerTests.java @@ -60,6 +60,7 @@ import com.imageworks.spcue.service.RedirectService; import com.imageworks.spcue.service.Whiteboard; import com.imageworks.spcue.util.Convert; +import com.imageworks.spcue.util.CueUtil; import static org.hamcrest.Matchers.emptyString; import static org.hamcrest.Matchers.is; @@ -137,11 +138,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/StrandedCoreTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/StrandedCoreTests.java index 4211c9866..7d02d44e8 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/StrandedCoreTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/StrandedCoreTests.java @@ -43,6 +43,7 @@ import com.imageworks.spcue.service.JobLauncher; import com.imageworks.spcue.service.JobManager; import com.imageworks.spcue.test.TransactionalTest; +import com.imageworks.spcue.util.CueUtil; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -98,11 +99,12 @@ public void createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java index 74b21c102..7654570a0 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/TestBookingQueue.java @@ -64,11 +64,12 @@ public void create() { RenderHost host = RenderHost.newBuilder() .setName(HOSTNAME) .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem(8173264) .setTotalSwap(20960) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/BookingManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/BookingManagerTests.java index 9b6813c33..1e894eb1c 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/BookingManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/BookingManagerTests.java @@ -112,11 +112,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName("test_host") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap((int) CueUtil.GB16) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/HostManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/HostManagerTests.java index cf86e5362..29970441d 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/HostManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/HostManagerTests.java @@ -101,11 +101,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName(HOST_NAME) .setBootTime(1192369572) - .setFreeMcp(7602) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(15290520) .setFreeSwap(2076) .setLoad(1) - .setTotalMcp(19543) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap(2076) .setNimbyEnabled(true) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/JobManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/JobManagerTests.java index 3be56bf06..2ea9b5dde 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/JobManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/JobManagerTests.java @@ -129,11 +129,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName("test_host") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap((int) CueUtil.GB16) .setNimbyEnabled(false) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/OwnerManagerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/OwnerManagerTests.java index 224dcac75..51dcafec4 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/OwnerManagerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/OwnerManagerTests.java @@ -69,11 +69,12 @@ public DispatchHost createHost() { RenderHost host = RenderHost.newBuilder() .setName("test_host") .setBootTime(1192369572) - .setFreeMcp(76020) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB) .setFreeMem(53500) .setFreeSwap(20760) .setLoad(1) - .setTotalMcp(195430) + .setTotalMcp(CueUtil.GB4) .setTotalMem((int) CueUtil.GB16) .setTotalSwap((int) CueUtil.GB16) .setNimbyEnabled(true) diff --git a/cuebot/src/test/resources/opencue.properties b/cuebot/src/test/resources/opencue.properties index 334408470..00d0c4463 100644 --- a/cuebot/src/test/resources/opencue.properties +++ b/cuebot/src/test/resources/opencue.properties @@ -64,3 +64,9 @@ dispatcher.kill_queue.queue_capacity=1000 dispatcher.booking_queue.core_pool_size=6 dispatcher.booking_queue.max_pool_size=6 dispatcher.booking_queue.queue_capacity=1000 + +# The minimum amount of free space in the temporary directory (mcp) to book a host. +# E.g: 1G = 1048576 kB => dispatcher.min_bookable_free_temp_dir_kb=1048576 +# Default = 1G = 1048576 kB +# If equals to -1, it means the feature is turned off +dispatcher.min_bookable_free_temp_dir_kb=1048576 \ No newline at end of file From a94393aa3d408b38894403ce461ca349da41093d Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 27 Sep 2023 18:39:15 -0400 Subject: [PATCH 265/277] TSC meeting notes. (#1318) --- tsc/meetings/2023-07-19.md | 45 ++++++++++++++++++++++++++++++++++++++ tsc/meetings/2023-09-27.md | 24 ++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 tsc/meetings/2023-07-19.md create mode 100644 tsc/meetings/2023-09-27.md diff --git a/tsc/meetings/2023-07-19.md b/tsc/meetings/2023-07-19.md new file mode 100644 index 000000000..61051f9c9 --- /dev/null +++ b/tsc/meetings/2023-07-19.md @@ -0,0 +1,45 @@ +# OpenCue TSC Meeting Notes 19 July 2023 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* CI pipeline updates + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1305 + * Added new VFX ref platform years, retired old ones. + * Added CY2023. + * Keet CY2022. + * Drop CY2021, CY2020. + * Keet CY2019, but repurposed it as an explicitly-named Python 2 test. + * Disabled GUI tests on older platforms due to flakes, we'll keep running them on CY2023. + * A few other minor dependency upgrades and fixes. +* New release v0.22.14. + * We needed to get the latest database query fixes into an official release, newer versions of + Postgres that trigger those issues are more common now. + * Release includes: + * PySide6 in CueGUI, still needed for CueSubmit. + * Config / env var cleanup. Published a new doc page covering + this: https://www.opencue.io/docs/other-guides/configuring-opencue/ +* Enable GPU booking + * https://lists.aswf.io/g/opencue-user/topic/local_deployment_issues/100008713 + * Any ideas for this user? + * You need to have the nvidia-smi tool on RQD to detect GPU hardware. + * Once we figure this out, we should write up a doc page on how to enable GPU. + * If the user is on Docker, they may need to use the nvidia base image. +* Minimum bookable free mcp + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1306 + * Enforce minimum mcp (scratch space) for all hosts, take host offline if low on space. + * Brian to review. + * Ideally we should avoid spreading the "mcp" terminology, but this is a much larger project, + let's just avoid it where we can. +* Siggraph + * Nothing official planned, some folks attending virtually. +* SPI updates + * Finally up-to-date with “current” version of github. + * Performance issues on DispatchQuery. + * Using database migrations starting at v1000, this works because migrations are all applied + manually anyway, not via e.g. Flyway. + * When we create migrations, if you rename a field, you need to copy the value as well. +* Blender plugin update + * Added the ability to refresh/update opencue code from the addon. + * Brian to follow up on email thread. diff --git a/tsc/meetings/2023-09-27.md b/tsc/meetings/2023-09-27.md new file mode 100644 index 000000000..562f31f28 --- /dev/null +++ b/tsc/meetings/2023-09-27.md @@ -0,0 +1,24 @@ +# OpenCue TSC Meeting Notes 27 Sep 2023 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* OOM protection logic: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1317 + * Reworks OOM protection using percentages instead of hardcoded values, helps for larger hosts. + * Draft status for now, working on some more testing. + * Cuebot only for now. Solves 99% of cases but 1% still have a race condition because RQD does + not send an exit code indicating OOM. RQD fixes coming next. +* Reserve all cores / negative cores: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1313 + * Diego / SPI to discuss. + * Definitely needs to be wrapped in a flag. +* Nimby override: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1311 + * Let's double check the fallback behavior is working and there's not some other error. + * PR could be a good idea anyway, we know pynput isn't installed in certain environments and the + warning in the logs can be confusing. +* SPI update + * Merging CueGUI updates. Config file change and qtpy change. +* Blender plugin + * Running in a container working now. + * Docker run flag to mount the host network worked. Let's update the linux contributing page. + * Job submitted but didn't show up in the job list. RQD can't find blender command, debugging. From 226a27dbd9c3bdbc9effdddd03a4904bfa02824d Mon Sep 17 00:00:00 2001 From: Ramon Figueiredo Date: Wed, 27 Sep 2023 16:36:49 -0700 Subject: [PATCH 266/277] [cuegui] Bug fix missing jobs on MonitorJobs (#1312) * [cuegui] Bug fix missing jobs on MonitorJobs When the option group dependent is checked, jobs without dependency are being omitted. * [cuegui] Bug fix missing jobs on MonitorJobs - Remove dependent if it has the same name as the job. This avoids missing jobs on MonitorJobs. Remove the parent job is necessary to avoid remove the parent job and all the dependents when del self.__load[j] is called - Use list comprehension to remove dependent if it has the same name as the job - List comprehension avoid many nested blocks and follow the project code style * [cuegui] Bug fix when unmonitor finished jobs with dependents - AbstractTreeWidget.py > _removeItem(): Check if object ID exists before delete it * Merge master into fix_missing_jobs_on_monitor_jobs * [cuegui] Bug fix missing jobs on MonitorJobs - When the option group dependent is checked, jobs without dependency are omitted. - Code refactoring --- cuegui/cuegui/JobMonitorTree.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cuegui/cuegui/JobMonitorTree.py b/cuegui/cuegui/JobMonitorTree.py index d92c44afc..7b829e959 100644 --- a/cuegui/cuegui/JobMonitorTree.py +++ b/cuegui/cuegui/JobMonitorTree.py @@ -285,6 +285,14 @@ def addJob(self, job, timestamp=None, loading_from_config=False): dep = self.__menuActions.jobs( ).getRecursiveDependentJobs([newJobObj], active_only=active_only) + + # Remove dependent if it has the same name as the job + # - This avoids missing jobs on MonitorJobs + # - Remove the parent job is necessary to avoid remove + # the parent job and all the dependents + # in the step 2 below + dep = [j for j in dep if j.data.name != newJobObj.data.name] + self.__dependentJobs[jobKey] = dep # we'll also store a reversed dictionary for # dependencies with the dependent as key and the main From 287dc1ab53607f50b79ef70cb9b1f2121ce588f2 Mon Sep 17 00:00:00 2001 From: Nuwan Jayawardene Date: Tue, 3 Oct 2023 20:44:25 +0530 Subject: [PATCH 267/277] [rqd] Disable pynput in the Docker image to prevent confusing warning logs. (#1311) --- rqd/Dockerfile | 4 ++++ rqd/rqd/rqconstants.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/rqd/Dockerfile b/rqd/Dockerfile index 15f5ac21c..93b222aa7 100644 --- a/rqd/Dockerfile +++ b/rqd/Dockerfile @@ -52,6 +52,10 @@ RUN versioned_name="rqd-$(cat ./VERSION)-all" \ && tar -cvzf $versioned_name.tar.gz $versioned_name/* \ && ln -s $versioned_name rqd +RUN mkdir -p /etc/opencue +RUN echo "[Override]" > /etc/opencue/rqd.conf +RUN echo "USE_NIMBY_PYNPUT=false" >> /etc/opencue/rqd.conf + # RQD gRPC server EXPOSE 8444 diff --git a/rqd/rqd/rqconstants.py b/rqd/rqd/rqconstants.py index fe793970e..1daf2f993 100644 --- a/rqd/rqd/rqconstants.py +++ b/rqd/rqd/rqconstants.py @@ -172,6 +172,8 @@ CUEBOT_HOSTNAME = config.get(__section, "OVERRIDE_CUEBOT") if config.has_option(__section, "OVERRIDE_NIMBY"): OVERRIDE_NIMBY = config.getboolean(__section, "OVERRIDE_NIMBY") + if config.has_option(__section, "USE_NIMBY_PYNPUT"): + USE_NIMBY_PYNPUT = config.getboolean(__section, "USE_NIMBY_PYNPUT") if config.has_option(__section, "OVERRIDE_HOSTNAME"): OVERRIDE_HOSTNAME = config.get(__section, "OVERRIDE_HOSTNAME") if config.has_option(__section, "GPU"): From b203380ba4361e887692ae89f9c5a371e4b7177c Mon Sep 17 00:00:00 2001 From: Nuwan Jayawardene Date: Tue, 3 Oct 2023 20:46:07 +0530 Subject: [PATCH 268/277] [rqd] Add Blender to PATH and set RQD_USE_PATH_ENV_VAR in sample Docker image. (#1319) --- samples/rqd/blender/Dockerfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/samples/rqd/blender/Dockerfile b/samples/rqd/blender/Dockerfile index 208f3c7a7..bb2235613 100644 --- a/samples/rqd/blender/Dockerfile +++ b/samples/rqd/blender/Dockerfile @@ -30,5 +30,11 @@ RUN tar -xvf blender.tar.xz \ RUN rm blender.tar.xz +# Add Blender path as environment variable +ENV PATH=$PATH:${BLENDER_INSTALL_DIR} + +# Allows RQD to read Blender install directory in PATH env. variable +RUN echo "RQD_USE_PATH_ENV_VAR=true" >> /etc/opencue/rqd.conf + # Verify Blender installation -RUN ${BLENDER_INSTALL_DIR}/blender --version +RUN blender --version From e3136f4a0b571698043c807b85c2a909b84a1c58 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Wed, 8 Nov 2023 09:05:14 -0800 Subject: [PATCH 269/277] Oom protection (#1321) * The current logic relies on hardcoded values which are not suitable for large hosts. The new logic takes into account the size of hosts and also tries to be more aggressive with misbehaving frames. Prevent host from entering an OOM state where oom-killer might start killing important OS processes. The kill logic will kick in one of the following conditions is met: Host has less than OOM_MEMORY_LEFT_THRESHOLD_PERCENT memory available A frame is taking more than OOM_FRAME_OVERBOARD_PERCENT of what it had reserved For frames that are using more than they had reserved but not above the threshold, negotiate expanding the reservations with other frames on the same host (cherry picked from commit e88a5295f23bd927614de6d5af6a09d496d3e6ac) * Frames killed for OOM should be retried (cherry picked from commit b88f7bcb1ad43f83fb8357576c33483dc2bf4952) * OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD can be deactivated with -1 (cherry picked from commit 647e75e2254c7a7ff68c544e438080f412bf04c1) * Limit the number of kill retries There's an error condition on rqd where a frame that cannot be killed will end up preventing the host from picking up new jobs. This logic limits the number of repeated killRequests to give host a chance to pick up new jobs. At the same time, blocked frames are logged to spcue.log to be handled manually. (cherry picked from commit aea4864ef66aca494fb455a7c103e4a832b63d41) * Fix merge conflicts * Handle MR comments * Minor improvements to the logic Signed-off-by: Diego Tavares --------- Signed-off-by: Diego Tavares --- .../com/imageworks/spcue/FrameInterface.java | 1 - .../com/imageworks/spcue/dao/FrameDao.java | 7 + .../com/imageworks/spcue/dao/HostDao.java | 9 - .../spcue/dao/postgres/FrameDaoJdbc.java | 18 + .../spcue/dao/postgres/HostDaoJdbc.java | 9 - .../spcue/dao/postgres/ProcDaoJdbc.java | 2 +- .../spcue/dispatcher/DispatchSupport.java | 8 + .../dispatcher/DispatchSupportService.java | 13 +- .../spcue/dispatcher/Dispatcher.java | 10 +- .../dispatcher/FrameCompleteHandler.java | 73 +-- .../spcue/dispatcher/HostReportHandler.java | 519 +++++++++++------- .../commands/DispatchRqdKillFrame.java | 20 +- .../commands/DispatchRqdKillFrameMemory.java | 78 +++ .../imageworks/spcue/service/HostManager.java | 9 - .../spcue/service/HostManagerService.java | 6 - .../spring/applicationContext-service.xml | 1 - cuebot/src/main/resources/opencue.properties | 16 +- .../spcue/test/dao/postgres/HostDaoTests.java | 18 - .../dispatcher/FrameCompleteHandlerTests.java | 6 +- .../dispatcher/HostReportHandlerGpuTests.java | 10 +- .../dispatcher/HostReportHandlerTests.java | 243 ++++++-- .../conf/jobspec/jobspec_multiple_frames.xml | 48 ++ cuebot/src/test/resources/opencue.properties | 13 +- rqd/rqd/rqdservicers.py | 2 + rqd/rqd/rqnetwork.py | 1 + 25 files changed, 764 insertions(+), 376 deletions(-) create mode 100644 cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrameMemory.java create mode 100644 cuebot/src/test/resources/conf/jobspec/jobspec_multiple_frames.xml diff --git a/cuebot/src/main/java/com/imageworks/spcue/FrameInterface.java b/cuebot/src/main/java/com/imageworks/spcue/FrameInterface.java index eff22768f..945685444 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/FrameInterface.java +++ b/cuebot/src/main/java/com/imageworks/spcue/FrameInterface.java @@ -1,4 +1,3 @@ - /* * Copyright Contributors to the OpenCue Project * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java index 7c2e3c050..4dbb0e987 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/FrameDao.java @@ -202,6 +202,13 @@ boolean updateFrameStopped(FrameInterface frame, FrameState state, int exitStatu * @return */ boolean updateFrameCleared(FrameInterface frame); + /** + * Sets a frame exitStatus to EXIT_STATUS_MEMORY_FAILURE + * + * @param frame + * @return whether the frame has been updated + */ + boolean updateFrameMemoryError(FrameInterface frame); /** * Sets a frame to an unreserved waiting state. diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java index 5ed18947e..94ba316b1 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java @@ -252,15 +252,6 @@ public interface HostDao { */ void updateThreadMode(HostInterface host, ThreadMode mode); - /** - * When a host is in kill mode that means its 256MB+ into the swap and the - * the worst memory offender is killed. - * - * @param h HostInterface - * @return boolean - */ - boolean isKillMode(HostInterface h); - /** * Update the specified host's hardware information. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java index 3c752ad96..21c197a3b 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java @@ -155,6 +155,24 @@ public boolean updateFrameCleared(FrameInterface frame) { return updateFrame(frame, Dispatcher.EXIT_STATUS_FRAME_CLEARED) > 0; } + private static final String UPDATE_FRAME_MEMORY_ERROR = + "UPDATE "+ + "frame "+ + "SET " + + "int_exit_status = ?, " + + "int_version = int_version + 1 " + + "WHERE " + + "frame.pk_frame = ? "; + @Override + public boolean updateFrameMemoryError(FrameInterface frame) { + int result = getJdbcTemplate().update( + UPDATE_FRAME_MEMORY_ERROR, + Dispatcher.EXIT_STATUS_MEMORY_FAILURE, + frame.getFrameId()); + + return result > 0; + } + private static final String UPDATE_FRAME_STARTED = "UPDATE " + "frame " + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java index 6fe898b44..f703416b2 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java @@ -612,15 +612,6 @@ public void updateHostOs(HostInterface host, String os) { os, host.getHostId()); } - @Override - public boolean isKillMode(HostInterface h) { - return getJdbcTemplate().queryForObject( - "SELECT COUNT(1) FROM host_stat WHERE pk_host = ? " + - "AND int_swap_total - int_swap_free > ? AND int_mem_free < ?", - Integer.class, h.getHostId(), Dispatcher.KILL_MODE_SWAP_THRESHOLD, - Dispatcher.KILL_MODE_MEM_THRESHOLD) > 0; - } - @Override public int getStrandedCoreUnits(HostInterface h) { try { diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java index 5af292fb3..8f6322690 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java @@ -564,7 +564,7 @@ public boolean increaseReservedMemory(ProcInterface p, long value) { value, p.getProcId(), value) == 1; } catch (Exception e) { // check by trigger erify_host_resources - throw new ResourceReservationFailureException("failed to increase memory reserveration for proc " + throw new ResourceReservationFailureException("failed to increase memory reservation for proc " + p.getProcId() + " to " + value + ", proc does not have that much memory to spare."); } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java index 4accd0f8d..aa20e6266 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java @@ -415,6 +415,14 @@ List findNextDispatchFrames(LayerInterface layer, VirtualProc pro */ void clearFrame(DispatchFrame frame); + /** + * Sets the frame state exitStatus to EXIT_STATUS_MEMORY_FAILURE + * + * @param frame + * @return whether the frame has been updated + */ + boolean updateFrameMemoryError(FrameInterface frame); + /** * Update Memory usage data and LLU time for the given frame. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java index 887d0c29e..713f6c86c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java @@ -42,6 +42,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.springframework.dao.EmptyResultDataAccessException; +import org.springframework.dao.DataAccessException; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -184,7 +185,11 @@ public boolean increaseReservedMemory(ProcInterface p, long value) { @Override public boolean clearVirtualProcAssignement(ProcInterface proc) { - return procDao.clearVirtualProcAssignment(proc); + try { + return procDao.clearVirtualProcAssignment(proc); + } catch (DataAccessException e) { + return false; + } } @Transactional(propagation = Propagation.REQUIRED) @@ -343,6 +348,12 @@ public void clearFrame(DispatchFrame frame) { frameDao.updateFrameCleared(frame); } + @Override + @Transactional(propagation = Propagation.REQUIRED) + public boolean updateFrameMemoryError(FrameInterface frame) { + return frameDao.updateFrameMemoryError(frame); + } + @Transactional(propagation = Propagation.SUPPORTS) public RunFrame prepareRqdRunFrame(VirtualProc proc, DispatchFrame frame) { int threads = proc.coresReserved / 100; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java index 6ac703a41..072b04113 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java @@ -1,4 +1,3 @@ - /* * Copyright Contributors to the OpenCue Project * @@ -108,13 +107,8 @@ public interface Dispatcher { // without being penalized for it. public static final long VIRTUAL_MEM_THRESHHOLD = CueUtil.GB2; - // The amount of swap that must be used before a host can go - // into kill mode. - public static final long KILL_MODE_SWAP_THRESHOLD = CueUtil.MB128; - - // When the amount of free memory drops below this point, the - // host can go into kill mode. - public static final long KILL_MODE_MEM_THRESHOLD = CueUtil.MB512; + // How long to keep track of a frame kill request + public static final int FRAME_KILL_CACHE_EXPIRE_AFTER_WRITE_MINUTES = 3; // A higher number gets more deep booking but less spread on the cue. public static final int DEFAULT_MAX_FRAMES_PER_PASS = 4; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java index 55336aaf4..c405a9e31 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java @@ -33,6 +33,7 @@ import com.imageworks.spcue.DispatchFrame; import com.imageworks.spcue.DispatchHost; import com.imageworks.spcue.DispatchJob; +import com.imageworks.spcue.FrameDetail; import com.imageworks.spcue.JobDetail; import com.imageworks.spcue.LayerDetail; import com.imageworks.spcue.LayerInterface; @@ -143,49 +144,35 @@ public void handleFrameCompleteReport(final FrameCompleteReport report) { } try { - - final VirtualProc proc; - - try { - - proc = hostManager.getVirtualProc( - report.getFrame().getResourceId()); - } - catch (EmptyResultDataAccessException e) { - /* - * Do not propagate this exception to RQD. This - * usually means the cue lost connectivity to - * the host and cleared out the record of the proc. - * If this is propagated back to RQD, RQD will - * keep retrying the operation forever. - */ - logger.info("failed to acquire data needed to " + - "process completed frame: " + - report.getFrame().getFrameName() + " in job " + - report.getFrame().getJobName() + "," + e); - return; - } - + final VirtualProc proc = hostManager.getVirtualProc(report.getFrame().getResourceId()); final DispatchJob job = jobManager.getDispatchJob(proc.getJobId()); final LayerDetail layer = jobManager.getLayerDetail(report.getFrame().getLayerId()); + final FrameDetail frameDetail = jobManager.getFrameDetail(report.getFrame().getFrameId()); final DispatchFrame frame = jobManager.getDispatchFrame(report.getFrame().getFrameId()); final FrameState newFrameState = determineFrameState(job, layer, frame, report); final String key = proc.getJobId() + "_" + report.getFrame().getLayerId() + "_" + report.getFrame().getFrameId(); + if (dispatchSupport.stopFrame(frame, newFrameState, report.getExitStatus(), report.getFrame().getMaxRss())) { - dispatchQueue.execute(new KeyRunnable(key) { - @Override - public void run() { - try { - handlePostFrameCompleteOperations(proc, report, job, frame, - newFrameState); - } catch (Exception e) { - logger.warn("Exception during handlePostFrameCompleteOperations " + - "in handleFrameCompleteReport" + CueExceptionUtil.getStackTrace(e)); + if (dispatcher.isTestMode()) { + // Database modifications on a threadpool cannot be captured by the test thread + handlePostFrameCompleteOperations(proc, report, job, frame, + newFrameState, frameDetail); + } else { + dispatchQueue.execute(new KeyRunnable(key) { + @Override + public void run() { + try { + handlePostFrameCompleteOperations(proc, report, job, frame, + newFrameState, frameDetail); + } catch (Exception e) { + logger.warn("Exception during handlePostFrameCompleteOperations " + + "in handleFrameCompleteReport" + CueExceptionUtil.getStackTrace(e)); + } } - } - }); + }); + } } else { /* @@ -222,6 +209,19 @@ public void run() { } } } + catch (EmptyResultDataAccessException e) { + /* + * Do not propagate this exception to RQD. This + * usually means the cue lost connectivity to + * the host and cleared out the record of the proc. + * If this is propagated back to RQD, RQD will + * keep retrying the operation forever. + */ + logger.info("failed to acquire data needed to " + + "process completed frame: " + + report.getFrame().getFrameName() + " in job " + + report.getFrame().getJobName() + "," + e); + } catch (Exception e) { /* @@ -259,7 +259,7 @@ public void run() { */ public void handlePostFrameCompleteOperations(VirtualProc proc, FrameCompleteReport report, DispatchJob job, DispatchFrame frame, - FrameState newFrameState) { + FrameState newFrameState, FrameDetail frameDetail) { try { /* @@ -313,7 +313,8 @@ public void handlePostFrameCompleteOperations(VirtualProc proc, * specified in the show's service override, service or 2GB. */ if (report.getExitStatus() == Dispatcher.EXIT_STATUS_MEMORY_FAILURE - || report.getExitSignal() == Dispatcher.EXIT_STATUS_MEMORY_FAILURE) { + || report.getExitSignal() == Dispatcher.EXIT_STATUS_MEMORY_FAILURE + || frameDetail.exitStatus == Dispatcher.EXIT_STATUS_MEMORY_FAILURE) { long increase = CueUtil.GB2; // since there can be multiple services, just going for the diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java index 2adef34fb..997e32fd4 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java @@ -21,12 +21,16 @@ import java.sql.Timestamp; import java.util.ArrayList; -import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.imageworks.spcue.JobInterface; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.springframework.beans.factory.annotation.Autowired; @@ -50,6 +54,7 @@ import com.imageworks.spcue.dispatcher.commands.DispatchBookHostLocal; import com.imageworks.spcue.dispatcher.commands.DispatchHandleHostReport; import com.imageworks.spcue.dispatcher.commands.DispatchRqdKillFrame; +import com.imageworks.spcue.dispatcher.commands.DispatchRqdKillFrameMemory; import com.imageworks.spcue.grpc.host.HardwareState; import com.imageworks.spcue.grpc.host.LockState; import com.imageworks.spcue.grpc.report.BootReport; @@ -63,10 +68,11 @@ import com.imageworks.spcue.service.CommentManager; import com.imageworks.spcue.service.HostManager; import com.imageworks.spcue.service.JobManager; -import com.imageworks.spcue.service.JobManagerSupport; import com.imageworks.spcue.util.CueExceptionUtil; import com.imageworks.spcue.util.CueUtil; +import static com.imageworks.spcue.dispatcher.Dispatcher.*; + public class HostReportHandler { private static final Logger logger = LogManager.getLogger(HostReportHandler.class); @@ -81,7 +87,6 @@ public class HostReportHandler { private Dispatcher localDispatcher; private RqdClient rqdClient; private JobManager jobManager; - private JobManagerSupport jobManagerSupport; private JobDao jobDao; private LayerDao layerDao; @Autowired @@ -93,6 +98,13 @@ public class HostReportHandler { "space on the temporary directory (mcp)"; private static final String CUEBOT_COMMENT_USER = "cuebot"; + // A cache to store kill requests and count the number of occurrences. + // The cache expires after write to avoid growing unbounded. If a request for a host-frame doesn't appear + // for a period of time, the entry will be removed. + Cache killRequestCounterCache = CacheBuilder.newBuilder() + .expireAfterWrite(FRAME_KILL_CACHE_EXPIRE_AFTER_WRITE_MINUTES, TimeUnit.MINUTES) + .build(); + /** * Boolean to toggle if this class is accepting data or not. */ @@ -143,7 +155,6 @@ public void queueHostReport(HostReport report) { reportQueue.execute(new DispatchHandleHostReport(report, this)); } - public void handleHostReport(HostReport report, boolean isBoot) { long startTime = System.currentTimeMillis(); try { @@ -211,9 +222,9 @@ public void handleHostReport(HostReport report, boolean isBoot) { killTimedOutFrames(report); /* - * Increase/decreased reserved memory. + * Prevent OOM (Out-Of-Memory) issues on the host and manage frame reserved memory */ - handleMemoryReservations(host, report); + handleMemoryUsage(host, report); /* * The checks are done in order of least CPU intensive to @@ -456,103 +467,279 @@ private void changeLockState(DispatchHost host, CoreDetail coreInfo) { } /** - * Handle memory reservations for the given host. This will re-balance memory - * reservations on the machine and kill and frames that are out of control. - * + * Prevent host from entering an OOM state where oom-killer might start killing important OS processes. + * The kill logic will kick in one of the following conditions is met: + * - Host has less than OOM_MEMORY_LEFT_THRESHOLD_PERCENT memory available + * - A frame is taking more than OOM_FRAME_OVERBOARD_PERCENT of what it had reserved + * For frames that are using more than they had reserved but not above the threshold, negotiate expanding + * the reservations with other frames on the same host * @param host * @param report */ - private void handleMemoryReservations(final DispatchHost host, final HostReport report) { + private void handleMemoryUsage(final DispatchHost host, final HostReport report) { + // Don't keep memory balances on nimby hosts + if (host.isNimby) { + return; + } - // TODO: GPU: Need to keep frames from growing into space reserved for GPU frames - // However all this is done in the database without a chance to edit the values here + final double OOM_MAX_SAFE_USED_MEMORY_THRESHOLD = + env.getRequiredProperty("dispatcher.oom_max_safe_used_memory_threshold", Double.class); + final double OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD = + env.getRequiredProperty("dispatcher.oom_frame_overboard_allowed_threshold", Double.class); + RenderHost renderHost = report.getHost(); + List runningFrames = report.getFramesList(); + + boolean memoryWarning = renderHost.getTotalMem() > 0 && + ((double)renderHost.getFreeMem()/renderHost.getTotalMem() < + (1.0 - OOM_MAX_SAFE_USED_MEMORY_THRESHOLD)); + + if (memoryWarning) { + long memoryAvailable = renderHost.getFreeMem(); + long minSafeMemoryAvailable = (long)(renderHost.getTotalMem() * (1.0 - OOM_MAX_SAFE_USED_MEMORY_THRESHOLD)); + // Only allow killing up to 10 frames at a time + int killAttemptsRemaining = 10; + VirtualProc killedProc = null; + do { + killedProc = killWorstMemoryOffender(host); + killAttemptsRemaining -= 1; + if (killedProc != null) { + memoryAvailable = memoryAvailable + killedProc.memoryUsed; + } + } while (killAttemptsRemaining > 0 && + memoryAvailable < minSafeMemoryAvailable && + killedProc != null); + } else { + // When no mass cleaning was required, check for frames going overboard + // if frames didn't go overboard, manage its reservations trying to increase + // them accordingly + for (final RunningFrameInfo frame : runningFrames) { + if (OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD > 0 && isFrameOverboard(frame)) { + if (!killFrameOverusingMemory(frame, host.getName())) { + logger.warn("Frame " + frame.getJobName() + "." + frame.getFrameName() + + " is overboard but could not be killed"); + } + } else { + handleMemoryReservations(frame); + } + } + } + } - /* - * Check to see if we enable kill mode to free up memory. - */ - boolean killMode = hostManager.isSwapping(host); + public enum KillCause { + FrameOverboard("This frame is using more memory than it had reserved."), + HostUnderOom("Frame killed by host under OOM pressure"), + FrameTimedOut("Frame timed out"), + FrameLluTimedOut("Frame LLU timed out"), + FrameVerificationFailure("Frame failed to be verified on the database"); + private final String message; - for (final RunningFrameInfo f: report.getFramesList()) { + private KillCause(String message) { + this.message = message; + } + @Override + public String toString() { + return message; + } + } - VirtualProc proc = null; + private boolean killFrameOverusingMemory(RunningFrameInfo frame, String hostname) { + try { + VirtualProc proc = hostManager.getVirtualProc(frame.getResourceId()); + + // Don't mess with localDispatch procs + if (proc.isLocalDispatch) { + return false; + } + + logger.info("Killing frame on " + frame.getJobName() + "." + frame.getFrameName() + + ", using too much memory."); + return killProcForMemory(proc, hostname, KillCause.FrameOverboard); + } catch (EmptyResultDataAccessException e) { + return false; + } + } + + private boolean getKillClearance(String hostname, String frameId) { + String cacheKey = hostname + "-" + frameId; + final int FRAME_KILL_RETRY_LIMIT = + env.getRequiredProperty("dispatcher.frame_kill_retry_limit", Integer.class); + + // Cache frame+host receiving a killRequest and count how many times the request is being retried + // meaning rqd is probably failing at attempting to kill the related proc + long cachedCount; + try { + cachedCount = 1 + killRequestCounterCache.get(cacheKey, () -> 0L); + } catch (ExecutionException e) { + return false; + } + killRequestCounterCache.put(cacheKey, cachedCount); + if (cachedCount > FRAME_KILL_RETRY_LIMIT) { + FrameInterface frame = jobManager.getFrame(frameId); + JobInterface job = jobManager.getJob(frame.getJobId()); + + logger.warn("KillRequest blocked for " + job.getName() + "." + frame.getName() + + " blocked for host " + hostname + ". The kill retry limit has been reached."); + return false; + } + return true; + } + + private boolean killProcForMemory(VirtualProc proc, String hostname, KillCause killCause) { + if (!getKillClearance(hostname, proc.frameId)) { + return false; + } + + FrameInterface frame = jobManager.getFrame(proc.frameId); + if (dispatcher.isTestMode()) { + // Different threads don't share the same database state on the test environment + (new DispatchRqdKillFrameMemory(hostname, frame, killCause.toString(), rqdClient, + dispatchSupport, dispatcher.isTestMode())).run(); + } else { try { - proc = hostManager.getVirtualProc(f.getResourceId()); + killQueue.execute(new DispatchRqdKillFrameMemory(hostname, frame, killCause.toString(), rqdClient, + dispatchSupport, dispatcher.isTestMode())); + } catch (TaskRejectedException e) { + logger.warn("Unable to add a DispatchRqdKillFrame request, task rejected, " + e); + return false; + } + } + DispatchSupport.killedOffenderProcs.incrementAndGet(); + return true; + } - // TODO: handle memory management for local dispatches - // Skip local dispatches for now. - if (proc.isLocalDispatch) { - continue; - } + private boolean killFrame(String frameId, String hostname, KillCause killCause) { + if (!getKillClearance(hostname, frameId)) { + return false; + } + if (dispatcher.isTestMode()) { + // Different threads don't share the same database state on the test environment + (new DispatchRqdKillFrame(hostname, frameId, killCause.toString(), rqdClient)).run(); + } else { + try { + killQueue.execute(new DispatchRqdKillFrame(hostname, + frameId, + killCause.toString(), + rqdClient)); + } catch (TaskRejectedException e) { + logger.warn("Unable to add a DispatchRqdKillFrame request, task rejected, " + e); + } + } + DispatchSupport.killedOffenderProcs.incrementAndGet(); + return true; + } - if (f.getRss() > host.memory) { - try{ - logger.info("Killing frame " + f.getJobName() + "/" + f.getFrameName() + ", " - + proc.getName() + " was OOM"); - try { - killQueue.execute(new DispatchRqdKillFrame(proc, "The frame required " + - CueUtil.KbToMb(f.getRss()) + " but the machine only has " + - CueUtil.KbToMb(host.memory), rqdClient)); - } catch (TaskRejectedException e) { - logger.warn("Unable to queue RQD kill, task rejected, " + e); - } - DispatchSupport.killedOomProcs.incrementAndGet(); - } catch (Exception e) { - logger.info("failed to kill frame on " + proc.getName() + - "," + e); - } - } + /** + * Kill proc with the worst user/reserved memory ratio. + * + * @param host + * @return killed proc, or null if none could be found or failed to be killed + */ + private VirtualProc killWorstMemoryOffender(final DispatchHost host) { + try { + VirtualProc proc = hostManager.getWorstMemoryOffender(host); + logger.info("Killing frame on " + proc.getName() + ", host is under stress."); - if (dispatchSupport.increaseReservedMemory(proc, f.getRss())) { - proc.memoryReserved = f.getRss(); - logger.info("frame " + f.getFrameName() + " on job " + f.getJobName() - + " increased its reserved memory to " + - CueUtil.KbToMb(f.getRss())); - } + if (!killProcForMemory(proc, host.getName(), KillCause.HostUnderOom)) { + proc = null; + } + return proc; + } + catch (EmptyResultDataAccessException e) { + logger.error(host.name + " is under OOM and no proc is running on it."); + return null; + } + } + + /** + * Check frame memory usage comparing the amount used with the amount it had reserved + * @param frame + * @return + */ + private boolean isFrameOverboard(final RunningFrameInfo frame) { + final double OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD = + env.getRequiredProperty("dispatcher.oom_frame_overboard_allowed_threshold", Double.class); + + if (OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD < 0) { + return false; + } + + double rss = (double)frame.getRss(); + double maxRss = (double)frame.getMaxRss(); + final double MAX_RSS_OVERBOARD_THRESHOLD = OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD * 2; + final double RSS_AVAILABLE_FOR_MAX_RSS_TRIGGER = 0.1; + + try { + VirtualProc proc = hostManager.getVirtualProc(frame.getResourceId()); + double reserved = (double)proc.memoryReserved; + + // Last memory report is higher than the threshold + if (isOverboard(rss, reserved, OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD)) { + return true; + } + // If rss is not overboard, handle the situation where the frame might be going overboard from + // time to time but the last report wasn't during a spike. For this case, consider a combination + // of rss and maxRss. maxRss > 2 * threshold and rss > 0.9 + else { + return isOverboard(maxRss, reserved, MAX_RSS_OVERBOARD_THRESHOLD) && + isOverboard(rss, reserved, -RSS_AVAILABLE_FOR_MAX_RSS_TRIGGER); + } + } catch (EmptyResultDataAccessException e) { + logger.info("HostReportHandler(isFrameOverboard): Virtual proc for frame " + + frame.getFrameName() + " on job " + frame.getJobName() + " doesn't exist on the database"); + // Not able to mark the frame overboard is it couldn't be found on the db. + // Proc accounting (verifyRunningProc) should take care of it + return false; + } + } - } catch (ResourceReservationFailureException e) { + private boolean isOverboard(double value, double total, double threshold) { + return value/total >= (1 + threshold); + } - long memNeeded = f.getRss() - proc.memoryReserved; + /** + * Handle memory reservations for the given frame + * + * @param frame + */ + private void handleMemoryReservations(final RunningFrameInfo frame) { + VirtualProc proc = null; + try { + proc = hostManager.getVirtualProc(frame.getResourceId()); - logger.info("frame " + f.getFrameName() + " on job " + f.getJobName() + if (proc.isLocalDispatch) { + return; + } + + if (dispatchSupport.increaseReservedMemory(proc, frame.getRss())) { + proc.memoryReserved = frame.getRss(); + logger.info("frame " + frame.getFrameName() + " on job " + frame.getJobName() + + " increased its reserved memory to " + + CueUtil.KbToMb(frame.getRss())); + } + } catch (ResourceReservationFailureException e) { + if (proc != null) { + long memNeeded = frame.getRss() - proc.memoryReserved; + logger.info("frame " + frame.getFrameName() + " on job " + frame.getJobName() + "was unable to reserve an additional " + CueUtil.KbToMb(memNeeded) + "on proc " + proc.getName() + ", " + e); - try { if (dispatchSupport.balanceReservedMemory(proc, memNeeded)) { - proc.memoryReserved = f.getRss(); + proc.memoryReserved = frame.getRss(); logger.info("was able to balance host: " + proc.getName()); - } - else { + } else { logger.info("failed to balance host: " + proc.getName()); } } catch (Exception ex) { logger.warn("failed to balance host: " + proc.getName() + ", " + e); } - } catch (EmptyResultDataAccessException e) { - logger.info("HostReportHandler: frame " + f.getFrameName() + - " on job " + f.getJobName() + - " was unable be processed" + - " because the proc could not be found"); - } - } - - if (killMode) { - VirtualProc proc; - try { - proc = hostManager.getWorstMemoryOffender(host); - } - catch (EmptyResultDataAccessException e) { - logger.info(host.name + " is swapping and no proc is running on it."); - return; + } else { + logger.info("frame " + frame.getFrameName() + " on job " + frame.getJobName() + + "was unable to reserve an additional memory. Proc could not be found"); } - - logger.info("Killing frame on " + - proc.getName() + ", host is distressed."); - - DispatchSupport.killedOffenderProcs.incrementAndGet(); - jobManagerSupport.kill(proc, new Source( - "The host was dangerously low on memory and swapping.")); + } catch (EmptyResultDataAccessException e) { + logger.info("HostReportHandler: Memory reservations for frame " + frame.getFrameName() + + " on job " + frame.getJobName() + " proc could not be found"); } } @@ -562,7 +749,6 @@ private void handleMemoryReservations(final DispatchHost host, final HostReport * @param rFrames */ private void killTimedOutFrames(HostReport report) { - final Map layers = new HashMap(5); for (RunningFrameInfo frame: report.getFramesList()) { @@ -570,36 +756,16 @@ private void killTimedOutFrames(HostReport report) { LayerDetail layer = layerDao.getLayerDetail(layerId); long runtimeMinutes = ((System.currentTimeMillis() - frame.getStartTime()) / 1000l) / 60; - if (layer.timeout != 0 && runtimeMinutes > layer.timeout){ - try { - killQueue.execute(new DispatchRqdKillFrame(report.getHost().getName(), - frame.getFrameId(), - "This frame has reached it timeout.", - rqdClient)); - } catch (TaskRejectedException e) { - logger.warn("Unable to queue RQD kill, task rejected, " + e); - } - } + String hostname = report.getHost().getName(); - if (layer.timeout_llu == 0){ - continue; - } - - if (frame.getLluTime() == 0){ - continue; - } - - long r = System.currentTimeMillis() / 1000; - long lastUpdate = (r - frame.getLluTime()) / 60; + if (layer.timeout != 0 && runtimeMinutes > layer.timeout){ + killFrame(frame.getFrameId(), hostname, KillCause.FrameTimedOut); + } else if (layer.timeout_llu != 0 && frame.getLluTime() != 0) { + long r = System.currentTimeMillis() / 1000; + long lastUpdate = (r - frame.getLluTime()) / 60; - if (layer.timeout_llu != 0 && lastUpdate > (layer.timeout_llu -1)){ - try { - killQueue.execute(new DispatchRqdKillFrame(report.getHost().getName(), - frame.getFrameId(), - "This frame has reached it LLU timeout.", - rqdClient)); - } catch (TaskRejectedException e) { - logger.warn("Unable to queue RQD kill, task rejected, " + e); + if (layer.timeout_llu != 0 && lastUpdate > (layer.timeout_llu - 1)){ + killFrame(frame.getFrameId(), hostname, KillCause.FrameLluTimedOut); } } } @@ -725,98 +891,59 @@ public void verifyRunningFrameInfo(HostReport report) { continue; } + if (hostManager.verifyRunningProc(runningFrame.getResourceId(), runningFrame.getFrameId())) { + runningFrames.add(runningFrame); + continue; + } - if (!hostManager.verifyRunningProc(runningFrame.getResourceId(), - runningFrame.getFrameId())) { - - /* - * The frame this proc is running is no longer - * assigned to this proc. Don't ever touch - * the frame record. If we make it here that means - * the proc has been running for over 2 min. - */ - - String msg; - VirtualProc proc = null; + /* + * The frame this proc is running is no longer + * assigned to this proc. Don't ever touch + * the frame record. If we make it here that means + * the proc has been running for over 2 min. + */ + String msg; + VirtualProc proc = null; - try { - proc = hostManager.getVirtualProc(runningFrame.getResourceId()); - msg = "Virutal proc " + proc.getProcId() + + try { + proc = hostManager.getVirtualProc(runningFrame.getResourceId()); + msg = "Virtual proc " + proc.getProcId() + "is assigned to " + proc.getFrameId() + " not " + runningFrame.getFrameId(); - } - catch (Exception e) { - /* - * This will happen if the host goes off line and then - * comes back. In this case, we don't touch the frame - * since it might already be running somewhere else. We - * do however kill the proc. - */ - msg = "Virtual proc did not exist."; - } - - logger.info("warning, the proc " + - runningFrame.getResourceId() + " on host " + - report.getHost().getName() + " was running for " + - (runtimeSeconds / 60.0f) + " minutes " + - runningFrame.getJobName() + "/" + runningFrame.getFrameName() + - "but the DB did not " + - "reflect this " + - msg); - - DispatchSupport.accountingErrors.incrementAndGet(); - - try { - /* - * If the proc did exist unbook it if we can't - * verify its running something. - */ - boolean rqd_kill = false; - if (proc != null) { - - /* - * Check to see if the proc is an orphan. - */ - if (hostManager.isOprhan(proc)) { - dispatchSupport.clearVirtualProcAssignement(proc); - dispatchSupport.unbookProc(proc); - rqd_kill = true; - } - } - else { - /* Proc doesn't exist so a kill won't hurt */ - rqd_kill = true; - } - - if (rqd_kill) { - try { - killQueue.execute(new DispatchRqdKillFrame(report.getHost().getName(), - runningFrame.getFrameId(), - "OpenCue could not verify this frame.", - rqdClient)); - } catch (TaskRejectedException e) { - logger.warn("Unable to queue RQD kill, task rejected, " + e); - } - } + } + catch (Exception e) { + /* + * This will happen if the host goes offline and then + * comes back. In this case, we don't touch the frame + * since it might already be running somewhere else. We + * do however kill the proc. + */ + msg = "Virtual proc did not exist."; + } - } catch (RqdClientException rqde) { - logger.warn("failed to kill " + - runningFrame.getJobName() + "/" + - runningFrame.getFrameName() + - " when trying to clear a failed " + - " frame verification, " + rqde); - - } catch (Exception e) { - CueExceptionUtil.logStackTrace("failed", e); - logger.warn("failed to verify " + - runningFrame.getJobName() + "/" + - runningFrame.getFrameName() + - " was running but the frame was " + - " unable to be killed, " + e); - } + DispatchSupport.accountingErrors.incrementAndGet(); + if (proc != null && hostManager.isOprhan(proc)) { + dispatchSupport.clearVirtualProcAssignement(proc); + dispatchSupport.unbookProc(proc); + proc = null; } - else { - runningFrames.add(runningFrame); + if (proc == null) { + if (killFrame(runningFrame.getFrameId(), + report.getHost().getName(), + KillCause.FrameVerificationFailure)) { + logger.info("FrameVerificationError, the proc " + + runningFrame.getResourceId() + " on host " + + report.getHost().getName() + " was running for " + + (runtimeSeconds / 60.0f) + " minutes " + + runningFrame.getJobName() + "/" + runningFrame.getFrameName() + + "but the DB did not " + + "reflect this. " + + msg); + } else { + logger.warn("FrameStuckWarning: frameId=" + runningFrame.getFrameId() + + " render_node=" + report.getHost().getName() + " - " + + runningFrame.getJobName() + "/" + runningFrame.getFrameName()); + } } } } @@ -877,14 +1004,6 @@ public void setJobManager(JobManager jobManager) { this.jobManager = jobManager; } - public JobManagerSupport getJobManagerSupport() { - return jobManagerSupport; - } - - public void setJobManagerSupport(JobManagerSupport jobManagerSupport) { - this.jobManagerSupport = jobManagerSupport; - } - public JobDao getJobDao() { return jobDao; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrame.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrame.java index 61258a824..fe9bde60e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrame.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrame.java @@ -31,9 +31,7 @@ public class DispatchRqdKillFrame extends KeyRunnable { private static final Logger logger = LogManager.getLogger(DispatchRqdKillFrame.class); - private VirtualProc proc = null; private String message; - private String hostname; private String frameId; @@ -47,28 +45,14 @@ public DispatchRqdKillFrame(String hostname, String frameId, String message, Rqd this.rqdClient = rqdClient; } - public DispatchRqdKillFrame(VirtualProc proc, String message, RqdClient rqdClient) { - super("disp_rqd_kill_frame_" + proc.getProcId() + "_" + rqdClient.toString()); - this.proc = proc; - this.hostname = proc.hostName; - this.message = message; - this.rqdClient = rqdClient; - } - @Override public void run() { long startTime = System.currentTimeMillis(); try { - if (proc != null) { - rqdClient.killFrame(proc, message); - } - else { - rqdClient.killFrame(hostname, frameId, message); - } + rqdClient.killFrame(hostname, frameId, message); } catch (RqdClientException e) { logger.info("Failed to contact host " + hostname + ", " + e); - } - finally { + } finally { long elapsedTime = System.currentTimeMillis() - startTime; logger.info("RQD communication with " + hostname + " took " + elapsedTime + "ms"); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrameMemory.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrameMemory.java new file mode 100644 index 000000000..301f77479 --- /dev/null +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchRqdKillFrameMemory.java @@ -0,0 +1,78 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.imageworks.spcue.dispatcher.commands; + +import com.imageworks.spcue.FrameInterface; +import com.imageworks.spcue.VirtualProc; +import com.imageworks.spcue.dispatcher.DispatchSupport; +import com.imageworks.spcue.rqd.RqdClient; +import com.imageworks.spcue.rqd.RqdClientException; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; + +/** + * A runnable to communicate with rqd requesting for a frame to be killed due to memory issues. + *

+ * Before killing a frame, the database is updated to mark the frame status as EXIT_STATUS_MEMORY_FAILURE, + * this allows the FrameCompleteHandler to possibly retry the frame after increasing its memory requirements + */ +public class DispatchRqdKillFrameMemory extends KeyRunnable { + + private static final Logger logger = LogManager.getLogger(DispatchRqdKillFrameMemory.class); + + private String message; + private String hostname; + private DispatchSupport dispatchSupport; + private final RqdClient rqdClient; + private final boolean isTestMode; + + private FrameInterface frame; + + public DispatchRqdKillFrameMemory(String hostname, FrameInterface frame, String message, RqdClient rqdClient, + DispatchSupport dispatchSupport, boolean isTestMode) { + super("disp_rqd_kill_frame_" + frame.getFrameId() + "_" + rqdClient.toString()); + this.frame = frame; + this.hostname = hostname; + this.message = message; + this.rqdClient = rqdClient; + this.dispatchSupport = dispatchSupport; + this.isTestMode = isTestMode; + } + + @Override + public void run() { + long startTime = System.currentTimeMillis(); + try { + if (dispatchSupport.updateFrameMemoryError(frame) && !isTestMode) { + rqdClient.killFrame(hostname, frame.getFrameId(), message); + } else { + logger.warn("Could not update frame " + frame.getFrameId() + + " status to EXIT_STATUS_MEMORY_FAILURE. Canceling kill request!"); + } + } catch (RqdClientException e) { + logger.warn("Failed to contact host " + hostname + ", " + e); + } finally { + long elapsedTime = System.currentTimeMillis() - startTime; + logger.info("RQD communication with " + hostname + + " took " + elapsedTime + "ms"); + } + } +} + diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java b/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java index aaf401688..e62d8647b 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/HostManager.java @@ -70,15 +70,6 @@ public interface HostManager { */ void setHostFreeTempDir(HostInterface host, Long freeTempDir); - /** - * Return true if the host is swapping hard enough - * that killing frames will save the entire machine. - * - * @param host - * @return - */ - boolean isSwapping(HostInterface host); - DispatchHost createHost(HostReport report); DispatchHost createHost(RenderHost host); diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java index a1533d695..36de34a1c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/HostManagerService.java @@ -98,12 +98,6 @@ public void setHostFreeTempDir(HostInterface host, Long freeTempDir) { hostDao.updateHostFreeTempDir(host, freeTempDir); } - @Override - @Transactional(propagation = Propagation.REQUIRED, readOnly=true) - public boolean isSwapping(HostInterface host) { - return hostDao.isKillMode(host); - } - public void rebootWhenIdle(HostInterface host) { try { hostDao.updateHostState(host, HardwareState.REBOOT_WHEN_IDLE); diff --git a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml index 35bd5cbb8..5aedc91b3 100644 --- a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml +++ b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml @@ -384,7 +384,6 @@ - diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index 6b2875899..b7f2a23ff 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -130,7 +130,21 @@ dispatcher.booking_queue.max_pool_size=6 # Queue capacity for booking. dispatcher.booking_queue.queue_capacity=1000 -# Whether or not to satisfy dependents (*_ON_FRAME and *_ON_LAYER) only on Frame success +# Percentage of used memory to consider a risk for triggering oom-killer +dispatcher.oom_max_safe_used_memory_threshold=0.95 + +# How much can a frame exceed its reserved memory. +# - 0.5 means 50% above reserve +# - -1.0 makes the feature inactive +# This feature is being kept inactive for now as we work on improving the +# frame retry logic (See commit comment for more details). +dispatcher.oom_frame_overboard_allowed_threshold=-1.0 + +# How many times should cuebot send a kill request for the same frame-host before reporting +# the frame as stuck +dispatcher.frame_kill_retry_limit=3 + +# Whether to satisfy dependents (*_ON_FRAME and *_ON_LAYER) only on Frame success depend.satisfy_only_on_frame_success=true # Jobs will be archived to the history tables after being completed for this long. diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java index a6261e464..918b43679 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/HostDaoTests.java @@ -331,24 +331,6 @@ public void testIsHostLocked() { assertEquals(hostDao.isHostLocked(host),true); } - @Test - @Transactional - @Rollback(true) - public void testIsKillMode() { - hostDao.insertRenderHost(buildRenderHost(TEST_HOST), - hostManager.getDefaultAllocationDetail(), - false); - - HostEntity host = hostDao.findHostDetail(TEST_HOST); - assertFalse(hostDao.isKillMode(host)); - - jdbcTemplate.update( - "UPDATE host_stat SET int_swap_free = ?, int_mem_free = ? WHERE pk_host = ?", - CueUtil.MB256, CueUtil.MB256, host.getHostId()); - - assertTrue(hostDao.isKillMode(host)); - } - @Test @Transactional @Rollback(true) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java index d313c5293..1f452e92a 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java @@ -310,10 +310,11 @@ private void executeDepend( DispatchJob dispatchJob = jobManager.getDispatchJob(proc.getJobId()); DispatchFrame dispatchFrame = jobManager.getDispatchFrame(report.getFrame().getFrameId()); + FrameDetail frameDetail = jobManager.getFrameDetail(report.getFrame().getFrameId()); dispatchSupport.stopFrame(dispatchFrame, frameState, report.getExitStatus(), report.getFrame().getMaxRss()); frameCompleteHandler.handlePostFrameCompleteOperations(proc, - report, dispatchJob, dispatchFrame, frameState); + report, dispatchJob, dispatchFrame, frameState, frameDetail); assertTrue(jobManager.isLayerComplete(layerFirst)); assertFalse(jobManager.isLayerComplete(layerSecond)); @@ -401,10 +402,11 @@ private void executeMinMemIncrease(int expected, boolean override) { DispatchJob dispatchJob = jobManager.getDispatchJob(proc.getJobId()); DispatchFrame dispatchFrame = jobManager.getDispatchFrame(report.getFrame().getFrameId()); + FrameDetail frameDetail = jobManager.getFrameDetail(report.getFrame().getFrameId()); dispatchSupport.stopFrame(dispatchFrame, FrameState.DEAD, report.getExitStatus(), report.getFrame().getMaxRss()); frameCompleteHandler.handlePostFrameCompleteOperations(proc, - report, dispatchJob, dispatchFrame, FrameState.WAITING); + report, dispatchJob, dispatchFrame, FrameState.WAITING, frameDetail); assertFalse(jobManager.isLayerComplete(layer)); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java index ce1e98ae1..120c620a1 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerGpuTests.java @@ -83,12 +83,12 @@ private static RenderHost getRenderHost() { .setBootTime(1192369572) // The minimum amount of free space in the temporary directory to book a host. .setFreeMcp(CueUtil.GB) - .setFreeMem(53500) - .setFreeSwap(20760) + .setFreeMem(CueUtil.GB8) + .setFreeSwap(CueUtil.GB2) .setLoad(0) .setTotalMcp(CueUtil.GB4) - .setTotalMem(1048576L * 4096) - .setTotalSwap(20960) + .setTotalMem(CueUtil.GB8) + .setTotalSwap(CueUtil.GB2) .setNimbyEnabled(false) .setNumProcs(2) .setCoresPerProc(100) @@ -115,7 +115,7 @@ public void testHandleHostReport() { hostReportHandler.handleHostReport(report, true); DispatchHost host = getHost(); assertEquals(host.lockState, LockState.OPEN); - assertEquals(host.memory, 4294443008L); + assertEquals(host.memory, CueUtil.GB8 - 524288); assertEquals(host.gpus, 64); assertEquals(host.idleGpus, 64); assertEquals(host.gpuMemory, 1048576L * 2048); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java index 970b97a95..971df8d14 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java @@ -21,10 +21,15 @@ import java.io.File; import java.sql.Timestamp; +import java.util.Arrays; import java.util.List; import java.util.concurrent.ThreadPoolExecutor; import javax.annotation.Resource; +import com.imageworks.spcue.dispatcher.DispatchSupport; +import com.imageworks.spcue.dispatcher.HostReportQueue; +import com.imageworks.spcue.dispatcher.FrameCompleteHandler; +import com.imageworks.spcue.grpc.job.FrameState; import org.junit.Before; import org.junit.Test; import org.springframework.test.annotation.Rollback; @@ -44,6 +49,7 @@ import com.imageworks.spcue.grpc.report.HostReport; import com.imageworks.spcue.grpc.report.RenderHost; import com.imageworks.spcue.grpc.report.RunningFrameInfo; +import com.imageworks.spcue.grpc.report.FrameCompleteReport; import com.imageworks.spcue.service.AdminManager; import com.imageworks.spcue.service.CommentManager; import com.imageworks.spcue.service.HostManager; @@ -52,6 +58,7 @@ import com.imageworks.spcue.test.TransactionalTest; import com.imageworks.spcue.util.CueUtil; import com.imageworks.spcue.VirtualProc; +import com.imageworks.spcue.LayerDetail; import java.util.UUID; @@ -70,6 +77,9 @@ public class HostReportHandlerTests extends TransactionalTest { @Resource HostReportHandler hostReportHandler; + @Resource + FrameCompleteHandler frameCompleteHandler; + @Resource Dispatcher dispatcher; @@ -99,9 +109,9 @@ public void setTestMode() { public void createHost() { hostname = UUID.randomUUID().toString().substring(0, 8); hostname2 = UUID.randomUUID().toString().substring(0, 8); - hostManager.createHost(getRenderHost(hostname, HardwareState.UP), + hostManager.createHost(getRenderHost(hostname), adminManager.findAllocationDetail("spi","general")); - hostManager.createHost(getRenderHost(hostname2, HardwareState.UP), + hostManager.createHost(getRenderHost(hostname2), adminManager.findAllocationDetail("spi","general")); } @@ -118,52 +128,32 @@ private DispatchHost getHost(String hostname) { return hostManager.findDispatchHost(hostname); } - private static RenderHost getRenderHost(String hostname, HardwareState state) { + private static RenderHost.Builder getRenderHostBuilder(String hostname) { return RenderHost.newBuilder() .setName(hostname) .setBootTime(1192369572) // The minimum amount of free space in the temporary directory to book a host. .setFreeMcp(CueUtil.GB) - .setFreeMem((int) CueUtil.GB8) - .setFreeSwap(20760) + .setFreeMem(CueUtil.GB8) + .setFreeSwap(CueUtil.GB2) .setLoad(0) .setTotalMcp(CueUtil.GB4) .setTotalMem(CueUtil.GB8) .setTotalSwap(CueUtil.GB2) .setNimbyEnabled(false) - .setNumProcs(2) + .setNumProcs(16) .setCoresPerProc(100) .addTags("test") - .setState(state) + .setState(HardwareState.UP) .setFacility("spi") .putAttributes("SP_OS", "Linux") - .setFreeGpuMem((int) CueUtil.MB512) - .setTotalGpuMem((int) CueUtil.MB512) - .build(); + .setNumGpus(0) + .setFreeGpuMem(0) + .setTotalGpuMem(0); } - private static RenderHost getRenderHost(String hostname, HardwareState state, Long freeTempDir) { - return RenderHost.newBuilder() - .setName(hostname) - .setBootTime(1192369572) - // The minimum amount of free space in the temporary directory to book a host. - .setFreeMcp(freeTempDir) - .setFreeMem((int) CueUtil.GB8) - .setFreeSwap(20760) - .setLoad(0) - .setTotalMcp(freeTempDir * 4) - .setTotalMem(CueUtil.GB8) - .setTotalSwap(CueUtil.GB2) - .setNimbyEnabled(false) - .setNumProcs(2) - .setCoresPerProc(100) - .addTags("test") - .setState(state) - .setFacility("spi") - .putAttributes("SP_OS", "Linux") - .setFreeGpuMem((int) CueUtil.MB512) - .setTotalGpuMem((int) CueUtil.MB512) - .build(); + private static RenderHost getRenderHost(String hostname) { + return getRenderHostBuilder(hostname).build(); } private static RenderHost getNewRenderHost(String tags) { @@ -172,12 +162,12 @@ private static RenderHost getNewRenderHost(String tags) { .setBootTime(1192369572) // The minimum amount of free space in the temporary directory to book a host. .setFreeMcp(CueUtil.GB) - .setFreeMem(53500) - .setFreeSwap(20760) + .setFreeMem(CueUtil.GB8) + .setFreeSwap(CueUtil.GB2) .setLoad(0) - .setTotalMcp(CueUtil.GB4) - .setTotalMem(8173264) - .setTotalSwap(20960) + .setTotalMcp(195430) + .setTotalMem(CueUtil.GB8) + .setTotalSwap(CueUtil.GB2) .setNimbyEnabled(false) .setNumProcs(2) .setCoresPerProc(100) @@ -196,15 +186,15 @@ private static RenderHost getNewRenderHost(String tags) { public void testHandleHostReport() throws InterruptedException { CoreDetail cores = getCoreDetail(200, 200, 0, 0); HostReport report1 = HostReport.newBuilder() - .setHost(getRenderHost(hostname, HardwareState.UP)) + .setHost(getRenderHost(hostname)) .setCoreInfo(cores) .build(); HostReport report2 = HostReport.newBuilder() - .setHost(getRenderHost(hostname2, HardwareState.UP)) + .setHost(getRenderHost(hostname2)) .setCoreInfo(cores) .build(); HostReport report1_2 = HostReport.newBuilder() - .setHost(getRenderHost(hostname, HardwareState.UP)) + .setHost(getRenderHost(hostname)) .setCoreInfo(getCoreDetail(200, 200, 100, 0)) .build(); @@ -314,7 +304,7 @@ public void testHandleHostReportWithFullTemporaryDirectories() { * */ // Create HostReport HostReport report1 = HostReport.newBuilder() - .setHost(getRenderHost(hostname, HardwareState.UP, 1024L)) + .setHost(getRenderHostBuilder(hostname).setFreeMcp(1024L).build()) .setCoreInfo(cores) .build(); // Call handleHostReport() => Create the comment with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and change the @@ -356,7 +346,7 @@ public void testHandleHostReportWithFullTemporaryDirectories() { * */ // Set the host freeTempDir to the minimum size required = 1GB (1048576 KB) HostReport report2 = HostReport.newBuilder() - .setHost(getRenderHost(hostname, HardwareState.UP, 1048576L)) + .setHost(getRenderHostBuilder(hostname).setFreeMcp(CueUtil.GB).build()) .setCoreInfo(cores) .build(); // Call handleHostReport() => Delete the comment with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and change the @@ -397,7 +387,7 @@ public void testHandleHostReportWithHardwareStateRepairNotRelatedToFullTempDir() * */ // Create HostReport HostReport report = HostReport.newBuilder() - .setHost(getRenderHost(hostname, HardwareState.UP, 1048576L)) + .setHost(getRenderHostBuilder(hostname).setFreeMcp(CueUtil.GB).build()) .setCoreInfo(cores) .build(); // Get host @@ -451,7 +441,7 @@ public void testMemoryAndLlu() { .setMaxRss(420000) .build(); HostReport report = HostReport.newBuilder() - .setHost(getRenderHost(hostname, HardwareState.UP)) + .setHost(getRenderHost(hostname)) .setCoreInfo(cores) .addFrames(info) .build(); @@ -462,5 +452,170 @@ public void testMemoryAndLlu() { assertEquals(frame.dateLLU, new Timestamp(now / 1000 * 1000)); assertEquals(420000, frame.maxRss); } + + @Test + @Transactional + @Rollback(true) + public void testMemoryAggressionRss() { + jobLauncher.testMode = true; + dispatcher.setTestMode(true); + + jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec_simple.xml")); + + DispatchHost host = getHost(hostname); + List procs = dispatcher.dispatchHost(host); + assertEquals(1, procs.size()); + VirtualProc proc = procs.get(0); + + // 1.6 = 1 + dispatcher.oom_frame_overboard_allowed_threshold + long memoryOverboard = (long) Math.ceil((double) proc.memoryReserved * 1.6); + + // Test rss overboard + RunningFrameInfo info = RunningFrameInfo.newBuilder() + .setJobId(proc.getJobId()) + .setLayerId(proc.getLayerId()) + .setFrameId(proc.getFrameId()) + .setResourceId(proc.getProcId()) + .setRss(memoryOverboard) + .setMaxRss(memoryOverboard) + .build(); + HostReport report = HostReport.newBuilder() + .setHost(getRenderHost(hostname)) + .setCoreInfo(getCoreDetail(200, 200, 0, 0)) + .addFrames(info) + .build(); + + long killCount = DispatchSupport.killedOffenderProcs.get(); + hostReportHandler.handleHostReport(report, false); + assertEquals(killCount + 1, DispatchSupport.killedOffenderProcs.get()); + } + + @Test + @Transactional + @Rollback(true) + public void testMemoryAggressionMaxRss() { + jobLauncher.testMode = true; + dispatcher.setTestMode(true); + jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec_simple.xml")); + + DispatchHost host = getHost(hostname); + List procs = dispatcher.dispatchHost(host); + assertEquals(1, procs.size()); + VirtualProc proc = procs.get(0); + + // 0.6 = dispatcher.oom_frame_overboard_allowed_threshold + long memoryOverboard = (long) Math.ceil((double) proc.memoryReserved * + (1.0 + (2 * 0.6))); + + // Test rss>90% and maxRss overboard + RunningFrameInfo info = RunningFrameInfo.newBuilder() + .setJobId(proc.getJobId()) + .setLayerId(proc.getLayerId()) + .setFrameId(proc.getFrameId()) + .setResourceId(proc.getProcId()) + .setRss((long)Math.ceil(0.95 * proc.memoryReserved)) + .setMaxRss(memoryOverboard) + .build(); + HostReport report = HostReport.newBuilder() + .setHost(getRenderHost(hostname)) + .setCoreInfo(getCoreDetail(200, 200, 0, 0)) + .addFrames(info) + .build(); + + long killCount = DispatchSupport.killedOffenderProcs.get(); + hostReportHandler.handleHostReport(report, false); + assertEquals(killCount + 1, DispatchSupport.killedOffenderProcs.get()); + } + + @Test + @Transactional + @Rollback(true) + public void testMemoryAggressionMemoryWarning() { + jobLauncher.testMode = true; + dispatcher.setTestMode(true); + jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec_multiple_frames.xml")); + + DispatchHost host = getHost(hostname); + List procs = dispatcher.dispatchHost(host); + assertEquals(3, procs.size()); + VirtualProc proc1 = procs.get(0); + VirtualProc proc2 = procs.get(1); + VirtualProc proc3 = procs.get(2); + + // Ok + RunningFrameInfo info1 = RunningFrameInfo.newBuilder() + .setJobId(proc1.getJobId()) + .setLayerId(proc1.getLayerId()) + .setFrameId(proc1.getFrameId()) + .setResourceId(proc1.getProcId()) + .setRss(CueUtil.GB2) + .setMaxRss(CueUtil.GB2) + .build(); + + // Overboard Rss + RunningFrameInfo info2 = RunningFrameInfo.newBuilder() + .setJobId(proc2.getJobId()) + .setLayerId(proc2.getLayerId()) + .setFrameId(proc2.getFrameId()) + .setResourceId(proc2.getProcId()) + .setRss(CueUtil.GB4) + .setMaxRss(CueUtil.GB4) + .build(); + + // Overboard Rss + long memoryUsedProc3 = CueUtil.GB8; + RunningFrameInfo info3 = RunningFrameInfo.newBuilder() + .setJobId(proc3.getJobId()) + .setLayerId(proc3.getLayerId()) + .setFrameId(proc3.getFrameId()) + .setResourceId(proc3.getProcId()) + .setRss(memoryUsedProc3) + .setMaxRss(memoryUsedProc3) + .build(); + + RenderHost hostAfterUpdate = getRenderHostBuilder(hostname).setFreeMem(0).build(); + + HostReport report = HostReport.newBuilder() + .setHost(hostAfterUpdate) + .setCoreInfo(getCoreDetail(200, 200, 0, 0)) + .addAllFrames(Arrays.asList(info1, info2, info3)) + .build(); + + // Get layer state before report gets sent + LayerDetail layerBeforeIncrease = jobManager.getLayerDetail(proc3.getLayerId()); + + // In this case, killing one job should be enough to ge the machine to a safe state + long killCount = DispatchSupport.killedOffenderProcs.get(); + hostReportHandler.handleHostReport(report, false); + assertEquals(killCount + 1, DispatchSupport.killedOffenderProcs.get()); + + // Confirm the frame will be set to retry after it's completion has been processed + + RunningFrameInfo runningFrame = RunningFrameInfo.newBuilder() + .setFrameId(proc3.getFrameId()) + .setFrameName("frame_name") + .setLayerId(proc3.getLayerId()) + .setRss(memoryUsedProc3) + .setMaxRss(memoryUsedProc3) + .setResourceId(proc3.id) + .build(); + FrameCompleteReport completeReport = FrameCompleteReport.newBuilder() + .setHost(hostAfterUpdate) + .setFrame(runningFrame) + .setExitSignal(9) + .setRunTime(1) + .setExitStatus(1) + .build(); + + frameCompleteHandler.handleFrameCompleteReport(completeReport); + FrameDetail killedFrame = jobManager.getFrameDetail(proc3.getFrameId()); + LayerDetail layer = jobManager.getLayerDetail(proc3.getLayerId()); + assertEquals(FrameState.WAITING, killedFrame.state); + // Memory increases are processed in two different places one will set the new value to proc.reserved + 2GB + // and the other will set to the maximum reported proc.maxRss the end value will be whoever is higher. + // In this case, proc.maxRss + assertEquals(Math.max(memoryUsedProc3, layerBeforeIncrease.getMinimumMemory() + CueUtil.GB2), + layer.getMinimumMemory()); + } } diff --git a/cuebot/src/test/resources/conf/jobspec/jobspec_multiple_frames.xml b/cuebot/src/test/resources/conf/jobspec/jobspec_multiple_frames.xml new file mode 100644 index 000000000..3baa0b22b --- /dev/null +++ b/cuebot/src/test/resources/conf/jobspec/jobspec_multiple_frames.xml @@ -0,0 +1,48 @@ + + + + + + + + + spi + pipe + default + testuser + 9860 + + + False + 2 + False + + + + echo hello + 1-3 + 1 + 2gb + + + shell + + + + + + diff --git a/cuebot/src/test/resources/opencue.properties b/cuebot/src/test/resources/opencue.properties index 00d0c4463..10516f881 100644 --- a/cuebot/src/test/resources/opencue.properties +++ b/cuebot/src/test/resources/opencue.properties @@ -38,7 +38,7 @@ dispatcher.job_query_max=20 dispatcher.job_lock_expire_seconds=2 dispatcher.job_lock_concurrency_level=3 dispatcher.frame_query_max=10 -dispatcher.job_frame_dispatch_max=2 +dispatcher.job_frame_dispatch_max=3 dispatcher.host_frame_dispatch_max=12 dispatcher.launch_queue.core_pool_size=1 @@ -64,9 +64,8 @@ dispatcher.kill_queue.queue_capacity=1000 dispatcher.booking_queue.core_pool_size=6 dispatcher.booking_queue.max_pool_size=6 dispatcher.booking_queue.queue_capacity=1000 - -# The minimum amount of free space in the temporary directory (mcp) to book a host. -# E.g: 1G = 1048576 kB => dispatcher.min_bookable_free_temp_dir_kb=1048576 -# Default = 1G = 1048576 kB -# If equals to -1, it means the feature is turned off -dispatcher.min_bookable_free_temp_dir_kb=1048576 \ No newline at end of file +dispatcher.min_bookable_free_temp_dir_kb=1048576 +dispatcher.min_bookable_free_mcp_kb=1048576 +dispatcher.oom_max_safe_used_memory_threshold=0.95 +dispatcher.oom_frame_overboard_allowed_threshold=0.6 +dispatcher.frame_kill_retry_limit=3 \ No newline at end of file diff --git a/rqd/rqd/rqdservicers.py b/rqd/rqd/rqdservicers.py index 98ab358ac..b736ef43b 100644 --- a/rqd/rqd/rqdservicers.py +++ b/rqd/rqd/rqdservicers.py @@ -67,6 +67,8 @@ def KillRunningFrame(self, request, context): frame = self.rqCore.getRunningFrame(request.frame_id) if frame: frame.kill(message=request.message) + else: + log.warning("Wasn't able to find frame(%s) to kill", request.frame_id) return rqd.compiled_proto.rqd_pb2.RqdStaticKillRunningFrameResponse() def ShutdownRqdNow(self, request, context): diff --git a/rqd/rqd/rqnetwork.py b/rqd/rqd/rqnetwork.py index 9f33e64a2..9d3591fdd 100644 --- a/rqd/rqd/rqnetwork.py +++ b/rqd/rqd/rqnetwork.py @@ -162,6 +162,7 @@ def kill(self, message=""): else: os.killpg(self.pid, rqd.rqconstants.KILL_SIGNAL) finally: + log.warning("kill() successfully killed frameId=%s pid=%s", self.frameId, self.pid) rqd.rqutil.permissionsLow() except OSError as e: log.warning( From 4e87140ed592e0689ef8597782268916706f17b9 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 8 Nov 2023 19:00:20 -0500 Subject: [PATCH 270/277] Fix lint warning. (#1329) --- rqd/rqd/rqnetwork.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rqd/rqd/rqnetwork.py b/rqd/rqd/rqnetwork.py index 9d3591fdd..6bb26cfce 100644 --- a/rqd/rqd/rqnetwork.py +++ b/rqd/rqd/rqnetwork.py @@ -162,7 +162,8 @@ def kill(self, message=""): else: os.killpg(self.pid, rqd.rqconstants.KILL_SIGNAL) finally: - log.warning("kill() successfully killed frameId=%s pid=%s", self.frameId, self.pid) + log.warning( + "kill() successfully killed frameId=%s pid=%s", self.frameId, self.pid) rqd.rqutil.permissionsLow() except OSError as e: log.warning( From fe519e3c377b0d10477bb20740aae087aaf0da35 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 8 Nov 2023 20:24:10 -0500 Subject: [PATCH 271/277] TSC meeting notes. (#1328) --- tsc/meetings/2023-11-08.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tsc/meetings/2023-11-08.md diff --git a/tsc/meetings/2023-11-08.md b/tsc/meetings/2023-11-08.md new file mode 100644 index 000000000..07f18796c --- /dev/null +++ b/tsc/meetings/2023-11-08.md @@ -0,0 +1,37 @@ +# OpenCue TSC Meeting Notes 8 Nov 2023 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* OOM protection logic: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1317 + * PR is reviewed and merged. + * Has been running in production for a while now, working well in 90% of cases. + * Other 10% are hitting a race condition with RQD. RQD sends fresh host report after OOM + decision has been made but not actuated yet, clearing the OOM state in the database. Frame is + rescheduled with same memory requirements instead of increased. + * Followup change to RQD coming soon, kill frame with correct OOM code so database state isn't + required. +* RQD env var expansion: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1323 + * Expanding any env vars in the command itself in RQD, before command is executed on host. + * This works already on Linux, Windows doesn't expand vars in command as command written to a + temp file batch script. + * Env vars should be expanded as late as possible, host/RQD env might differ slightly from frame + env. + * Let's move the change into the Windows section. +* RQD copy env vars from host: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1324 + * Needed to preserve the host PYTHONPATH in the frame env. + * Reviewed, change is too broad, left comments on the PR. +* DB indexes: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1304 + * Adding new database indexes to resolve some performance issues. + * Has some commented out pieces, why? + * Diego to check and update PR. +* CUDA RQD image: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1327 + * Let's try to copy RQD package from base image to reduce duplication. + * Should revisit the RQD base image at some point to see if we can upgrade past Python 3.6. +* Blender plugin: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1309 + * Plugin working e2e on Nuwan's machine, getting renders back. + * Starting review of draft PR. + * Would like to see a doc on how to install/use the plugin, will help to understand the code. +* CueGUI PySide compatibility change is rolling out to production soon, working in initial tests but + will get more feedback as usage expands. From b69156fa0ff522aee473c8e4dd2feac2978a7ad9 Mon Sep 17 00:00:00 2001 From: Ramon Figueiredo Date: Mon, 22 Jan 2024 09:08:08 -0800 Subject: [PATCH 272/277] Update Sphinx version (#1336) --- docs/conf.py | 2 +- docs/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 2c8322352..230855ca1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -67,7 +67,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. diff --git a/docs/requirements.txt b/docs/requirements.txt index 9d324374d..1f3307af5 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ -sphinx==4.3.1 +sphinx==5.0.0 sphinx-rtd-theme==1.0.0 From 5b9defd5f5fa87b49270bf7b5851f84066952ed6 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 22 Jan 2024 12:20:02 -0500 Subject: [PATCH 273/277] TSC meeting notes. (#1338) --- tsc/meetings/2024-01-17.md | 39 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 tsc/meetings/2024-01-17.md diff --git a/tsc/meetings/2024-01-17.md b/tsc/meetings/2024-01-17.md new file mode 100644 index 000000000..55e355faf --- /dev/null +++ b/tsc/meetings/2024-01-17.md @@ -0,0 +1,39 @@ +# OpenCue TSC Meeting Notes 17 Jan 2024 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Upgrade Cuebot to Java 21 + * Code changes in progress, PR coming. Let's discuss this next meeting once Diego is back, + interested to get more detail on any non-backwards-compatible changes. +* PR to fix crash in RQD with non-ascii chars + * Short term, fixing a crash is a priority. + * Longer term, there's a potential future project to improve non-ascii compatibility through the + system. +* Refactoring Cuebot host report handler + * PR coming soon. +* RQD changes for OOM protection + * PR coming soon. +* CueGUI web UI project launching at SPI + * Next.js on top of React. + * REST wrapper for gRPC. Many folks will find this useful outside of the GUI. + * Prototypes provide readonly functionality, for now. + * We should discuss the authorization system more, things are currently wide open if you have + network access. +* Blender plugin update + * Working on plugin bug fixes. + * Docs update coming soon. Packaging/distribution/install/upgrade process still a big open + question, we should look at the updated docs to see what the process currently looks like then + formulate a plan for this. + * RQD GPU image needs another look. +* M2/M3 compatibility + * We have been working on M1 compatibility for OpenCue. Are M2/M3 also supported? We're not + sure, let's check. + * Still need to finish the embedded postgres fix to complete M1 compatibility. +* OpenJobDescription + * https://github.com/OpenJobDescription/openjd-specifications/wiki + * Effort to standardize job descriptions between work management systems. + * OpenCue open to this effort? In theory yes, we will need to look into more detail to see what + would be required. Could add a layer for converting between the new job description and + OpenCue's internal format. From 9b4f90fd61df2f83d49e81a5e14388b67d0cc10f Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Mon, 29 Jan 2024 12:05:15 -0800 Subject: [PATCH 274/277] Fix GUI not updating bugs (#1180) * Fix GUI not updating bugs - Check if regex expression is empty before querying - Reduce timeout to a workable value. According to (gRPC Python docs)[https://grpc.github.io/grpc/python/grpc.html#grpc.UnaryUnaryMultiCallable.__call__], timeout is measured in seconds, meaning the current configuration is 2.7h which doesn't seem useful. Reducing to 10s. - Reducing JobMonitor tick to 10s - Check if regex expression is empty before querying - Fix JobMonitorTree getJobs call * Apply suggestions from code review Signed-off-by: Diego Tavares da Silva * Update MonitorJobsPlugin.py --------- Signed-off-by: Diego Tavares da Silva --- cuegui/cuegui/JobMonitorTree.py | 2 +- cuegui/cuegui/plugins/MonitorJobsPlugin.py | 20 ++++++++++---------- pycue/opencue/api.py | 1 + 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/cuegui/cuegui/JobMonitorTree.py b/cuegui/cuegui/JobMonitorTree.py index 7b829e959..5cb49f3d0 100644 --- a/cuegui/cuegui/JobMonitorTree.py +++ b/cuegui/cuegui/JobMonitorTree.py @@ -494,7 +494,7 @@ def _getUpdate(self): # an empty list for the id argument! if not ids: continue - tmp = opencue.api.getJobs(id=ids, all=True) + tmp = opencue.api.getJobs(id=ids, include_finished=True) self.__dependentJobs[job] = tmp if self.__loadMine: diff --git a/cuegui/cuegui/plugins/MonitorJobsPlugin.py b/cuegui/cuegui/plugins/MonitorJobsPlugin.py index a8ee4e034..875da70db 100644 --- a/cuegui/cuegui/plugins/MonitorJobsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorJobsPlugin.py @@ -197,17 +197,17 @@ def _regexLoadJobsHandle(self): self.jobMonitor.removeAllItems() - if cuegui.Utils.isStringId(substring): - # If a uuid is provided, load it - self.jobMonitor.addJob(substring) - elif load_finished_jobs or re.search( + if substring: + if cuegui.Utils.isStringId(substring): + # If a uuid is provided, load it + self.jobMonitor.addJob(substring) + elif load_finished_jobs or re.search( r"^([a-z0-9_]+)\-([a-z0-9\.]+)\-", substring, re.IGNORECASE): - # If show and shot is provided, or if "load finished" checkbox is checked, load all jobs - for job in opencue.api.getJobs(regex=[substring], include_finished=True): - self.jobMonitor.addJob(job) - else: - # Otherwise, just load current matching jobs (except for the empty string) - if substring: + # Load all ff show and shot is provided or if "load finished" checkbox is checked + for job in opencue.api.getJobs(regex=[substring], include_finished=True): + self.jobMonitor.addJob(job) + else: + # Otherwise, just load current matching jobs (except for the empty string) for job in opencue.api.getJobs(regex=[substring]): self.jobMonitor.addJob(job) diff --git a/pycue/opencue/api.py b/pycue/opencue/api.py index 075d5c374..d43575496 100644 --- a/pycue/opencue/api.py +++ b/pycue/opencue/api.py @@ -325,6 +325,7 @@ def getJobs(**options): - show: show names - list - shot: shot names - list - user: user names - list + - include_finished - bool :rtype: list :return: a list of Job objects From 42542c1ad253e66a9c0f9dc81ba1d556883144ca Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Tue, 30 Jan 2024 09:44:07 -0800 Subject: [PATCH 275/277] Add new indexes to improve booking performance (#1304) * Add new indexes to improve booking performance After the changes on the gpu PR https://github.com/AcademySoftwareFoundation/OpenCue/pull/924 the performance of the booking query degraded up to 4 times the previous throughput. Creating some indexes for columns that changed names seems to have fixed the problem. Signed-off-by: Diego Tavares * Update cuebot/src/main/resources/conf/ddl/postgres/migrations/V18_Add_New_Indexes Signed-off-by: Diego Tavares da Silva * Update cuebot/src/main/resources/conf/ddl/postgres/migrations/V18_Add_New_Indexes Signed-off-by: Diego Tavares da Silva * Update cuebot/src/main/resources/conf/ddl/postgres/migrations/V18_Add_New_Indexes Signed-off-by: Diego Tavares da Silva --------- Signed-off-by: Diego Tavares Signed-off-by: Diego Tavares da Silva --- .../postgres/migrations/V18_Add_New_Indexes | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V18_Add_New_Indexes diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V18_Add_New_Indexes b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V18_Add_New_Indexes new file mode 100644 index 000000000..107f426a0 --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V18_Add_New_Indexes @@ -0,0 +1,29 @@ + +--Performance issue, Created new index on column int_gpus_min + + +CREATE INDEX IF NOT EXISTS i_layer_int_gpu_mem_min + ON public.layer USING btree + (int_gpus_min ASC NULLS LAST) + TABLESPACE pg_default; + + +CREATE INDEX IF NOT EXISTS i_layer_int_gpu_mem_min_1 + ON public.layer USING btree + (int_gpu_min ASC NULLS LAST) + TABLESPACE pg_default; + + +create index concurrently i_layer_int_cores_max on layer(int_cores_max); + +create index concurrently i_job_resource_int_priority on job_resource(int_priority); + +create index concurrently i_job_int_min_cores on job(int_min_cores); + +create index concurrently i_layer_limit_pk_layer on layer_limit(pk_layer); + +create index concurrently i_folder_resource_int_cores on folder_resource(int_cores); + +create index concurrently i_job_ts_updated on job(ts_updated); + +create index concurrently i_layer_str_tags on layer(str_tags); From c1dcad9e4f627fddc88ce69cce9d1c56900c4ee6 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Tue, 30 Jan 2024 09:44:24 -0800 Subject: [PATCH 276/277] Logview threadpool (#861) * Update dispatchQuery to use min_cores Sorting jobs only by priority causes a situation where low priority jobs can get starved by a constant flow of high priority jobs. The new formula adds a modifier to the sorting rank to take into account the number of cores the job is requesting and also the number of days the job is waiting on the queue. Priorities numbers over 200 will mostly override the formula and work as a priority only based scheduling. sort = priority + (100 * (1 - (job.cores/job.int_min_cores))) + (age in days) Besides that, also take layer_int_cores_min into account when filtering folder_resourse limitations to avoid allocating more cores than the folder limits. (cherry picked from commit 566411aeeddc60983a30eabe121fd03263d05525) * Revert "Update dispatchQuery to use min_cores" This reverts commit 2eb4936c * Add threadpool logic for handling incoming log update requests (cherry picked from commit 351e210a9f0d49bfe61e9a57bba349bf1951af4e) * Suppress known ui warning messages For some ui process that especially involves multi threading or concurrent update of widget data, some warnings can happen due to obsolete function calls. The code has been updated to suppress a couple of known warnings (cherry picked from commit dd0c4f900fb75d25d63897b6aacd2d0b02e230f5) * Fix logview unit tests to work around the threading logic (cherry picked from commit 09446bbe5f0c631c74525cbc78b0e60ce114a2e2) * FIx unit tests * Fix python lint * Fix python lint * Fix lint Signed-off-by: Diego Tavares da Silva --------- Signed-off-by: Diego Tavares da Silva Co-authored-by: Fermi Perumal --- cuegui/cuegui/Main.py | 1 - cuegui/cuegui/plugins/LogViewPlugin.py | 141 ++++++++++++++------ cuegui/tests/plugins/LogViewPlugin_tests.py | 16 +-- 3 files changed, 105 insertions(+), 53 deletions(-) diff --git a/cuegui/cuegui/Main.py b/cuegui/cuegui/Main.py index 24f08864b..e46102b56 100644 --- a/cuegui/cuegui/Main.py +++ b/cuegui/cuegui/Main.py @@ -92,7 +92,6 @@ def startup(app_name, app_version, argv): app.aboutToQuit.connect(closingTime) # pylint: disable=no-member app.exec_() - def closingTime(): """Window close callback.""" logger.info("Closing all threads...") diff --git a/cuegui/cuegui/plugins/LogViewPlugin.py b/cuegui/cuegui/plugins/LogViewPlugin.py index 1e67ac1c4..5c36ec717 100644 --- a/cuegui/cuegui/plugins/LogViewPlugin.py +++ b/cuegui/cuegui/plugins/LogViewPlugin.py @@ -25,7 +25,9 @@ import os import re import string +import sys import time +import traceback from qtpy import QtGui from qtpy import QtCore @@ -292,10 +294,40 @@ def line_number_area_paint_event(self, event): bottom = top + self.blockBoundingRect(block).height() block_number += 1 +class LogLoadSignals(QtCore.QObject): + """Signals for the LoadLog action""" + SIG_LOG_LOAD_ERROR = QtCore.Signal(tuple) + SIG_LOG_LOAD_RESULT = QtCore.Signal(str, str) + SIG_LOG_LOAD_FINISHED = QtCore.Signal() + +class LogLoader(QtCore.QRunnable): + """A thread to load logs""" + def __init__(self, fn, *args, **kwargs): + super(LogLoader, self).__init__() + self.fn = fn + self.args = args + self.kwargs = kwargs + self.signals = LogLoadSignals() + + @QtCore.Slot() + def run(self): + # pylint: disable=bare-except + try: + content, log_mtime = self.fn(*self.args, **self.kwargs) + except: + exctype, value = sys.exc_info()[:2] + self.signals.SIG_LOG_LOAD_ERROR.emit( + (exctype, value, traceback.format_exc())) + else: + self.signals.SIG_LOG_LOAD_RESULT.emit(content, log_mtime) + finally: + self.signals.SIG_LOG_LOAD_FINISHED.emit() class LogViewWidget(QtWidgets.QWidget): - """Displays the log file for the selected frame.""" - + """ + Displays the log file for the selected frame + """ + SIG_CONTENT_UPDATED = QtCore.Signal(str, str) def __init__(self, parent=None): """ Create the UI elements @@ -453,6 +485,9 @@ def __init__(self, parent=None): self._current_match = 0 self._content_box.mousePressedSignal.connect(self._on_mouse_pressed) + self.SIG_CONTENT_UPDATED.connect(self._update_log_content) + self.log_thread_pool = QtCore.QThreadPool() + def _on_mouse_pressed(self, pos): """ Mouse press event, to be called when the user scrolls by hand or moves @@ -788,12 +823,56 @@ def _display_log_content(self): """ try: - self._update_log() - self._new_log = False + if not os.path.exists(self._log_file): + self._log_file_exists = False + content = 'Log file does not exist: %s' % self._log_file + self._content_timestamp = time.time() + self._update_log_content(content, self._log_mtime) + else: + # Creating the load logs process as qrunnables so + # that they don't block the ui while loading + log_loader = LogLoader(self._load_log, self._log_file, + self._new_log, self._log_mtime) + log_loader.signals.SIG_LOG_LOAD_RESULT.connect( + self._receive_log_results) + log_loader.setAutoDelete(True) + self.log_thread_pool.start(log_loader) + self.log_thread_pool.waitForDone() + self._new_log = False finally: QtCore.QTimer.singleShot(5000, self._display_log_content) - def _update_log(self): + # pylint: disable=no-self-use + @QtCore.Slot() + def _load_log(self, log_file, new_log, curr_log_mtime): + content = None + log_size = int(os.stat(log_file).st_size) + if log_size > 1 * 1e6: + content = ('Log file size (%0.1f MB) exceeds the size ' + 'threshold (1.0 MB).' + % float(log_size / (1024 * 1024))) + elif not new_log and os.path.exists(log_file): + log_mtime = os.path.getmtime(log_file) + if log_mtime > curr_log_mtime: + curr_log_mtime = log_mtime # no new updates + content = '' + + if content is None: + content = '' + try: + with open(log_file, 'r') as f: + content = f.read() + except IOError: + content = 'Can not access log file: %s' % log_file + + return content, curr_log_mtime + + @QtCore.Slot() + def _receive_log_results(self, content, log_mtime): + self.SIG_CONTENT_UPDATED.emit(content, log_mtime) + + @QtCore.Slot(str, str) + def _update_log_content(self, content, log_mtime): """ Updates the content of the content box with the content of the log file, if necessary. The full path to the log file will be populated in @@ -813,49 +892,23 @@ def _update_log(self): (if necessary) """ - # Get the content of the log file - if not self._log_file: - return # There's no log file, nothing to do here! - self._path.setText(self._log_file) - content = None - if not os.path.exists(self._log_file): - self._log_file_exists = False - content = 'Log file does not exist: %s' % self._log_file - self._content_timestamp = time.time() - else: - log_size = int(os.stat(self._log_file).st_size) - if log_size > 5 * 1e6: - content = ('Log file size (%0.1f MB) exceeds the size ' - 'threshold (5.0 MB).' - % float(log_size / (1024 * 1024))) - elif not self._new_log and os.path.exists(self._log_file): - log_mtime = os.path.getmtime(self._log_file) - if log_mtime > self._log_mtime: - self._log_mtime = log_mtime # no new updates - content = '' - - if content is None: - content = '' - try: - with open(self._log_file, 'r') as f: - content = f.read() - except IOError: - content = 'Can not access log file: %s' % self._log_file + self._log_mtime = log_mtime - # Do we need to scroll to the end? - scroll_to_end = (self._scrollbar_max == self._scrollbar_value - or self._new_log) + self.app.processEvents() # Update the content in the gui (if necessary) - current_text = (self._content_box.toPlainText() or '') - new_text = content.lstrip(str(current_text)) - if new_text: - if self._new_log: - self._content_box.setPlainText(content) - else: + if self._new_log: + self._content_box.setPlainText(content) + else: + current_text = (self._content_box.toPlainText() or '') + new_text = content.lstrip(str(current_text)) + if new_text: self._content_box.appendPlainText(new_text) - self._content_timestamp = time.time() - self.app.processEvents() + self._content_timestamp = time.time() + self._path.setText(self._log_file) + + scroll_to_end = (self._scrollbar_max == self._scrollbar_value + or self._new_log) # Adjust scrollbar value (if necessary) self._scrollbar_max = self._log_scrollbar.maximum() diff --git a/cuegui/tests/plugins/LogViewPlugin_tests.py b/cuegui/tests/plugins/LogViewPlugin_tests.py index a135747b5..62752c2f3 100644 --- a/cuegui/tests/plugins/LogViewPlugin_tests.py +++ b/cuegui/tests/plugins/LogViewPlugin_tests.py @@ -71,21 +71,22 @@ def setUp(self): def test_shouldDisplayFirstLogFile(self): cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) - + self.logViewPlugin.logview_widget._receive_log_results(_LOG_TEXT_1, 0) self.assertEqual(_LOG_TEXT_1, self.logViewPlugin.logview_widget._content_box.toPlainText()) def test_shouldUpdateLogFile(self): cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) + self.logViewPlugin.logview_widget._receive_log_results(_LOG_TEXT_1, 0) new_contents = _LOG_TEXT_1 + '\nanother line at the end' self.log1.set_contents(new_contents) cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) - + self.logViewPlugin.logview_widget._receive_log_results(new_contents, 0) self.assertEqual(new_contents, self.logViewPlugin.logview_widget._content_box.toPlainText()) def test_shouldHighlightAllSearchResults(self): - cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) + self.logViewPlugin.logview_widget._receive_log_results(_LOG_TEXT_1, 0) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( - qtpy.QtCore.Qt.CheckState.Unchecked) + qtpy.QtCore.Qt.CheckState.Unchecked) self.logViewPlugin.logview_widget._search_box.setText('lorem') self.logViewPlugin.logview_widget._search_button.click() @@ -100,7 +101,7 @@ def test_shouldHighlightAllSearchResults(self): self.logViewPlugin.logview_widget._content_box, matches[1][0], matches[1][1])) def test_shouldMoveCursorToSecondSearchResult(self): - cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) + self.logViewPlugin.logview_widget._receive_log_results(_LOG_TEXT_1, 0) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( qtpy.QtCore.Qt.CheckState.Unchecked) @@ -114,7 +115,7 @@ def test_shouldMoveCursorToSecondSearchResult(self): self.assertEqual(132, self.logViewPlugin.logview_widget._cursor.position()) def test_shouldMoveCursorLastSearchResult(self): - cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) + self.logViewPlugin.logview_widget._receive_log_results(_LOG_TEXT_1, 0) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( qtpy.QtCore.Qt.CheckState.Unchecked) @@ -128,10 +129,9 @@ def test_shouldMoveCursorLastSearchResult(self): self.assertEqual(132, self.logViewPlugin.logview_widget._cursor.position()) def test_shouldPerformCaseInsensitiveSearch(self): - cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) + self.logViewPlugin.logview_widget._receive_log_results(_LOG_TEXT_1, 0) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( qtpy.QtCore.Qt.CheckState.Checked) - self.logViewPlugin.logview_widget._search_box.setText('lorem') self.logViewPlugin.logview_widget._search_button.click() matches = self.logViewPlugin.logview_widget._matches From a694b8dce53e6adf804abfe6a758316946e87124 Mon Sep 17 00:00:00 2001 From: Angela Li <144162000+angelali-ms@users.noreply.github.com> Date: Wed, 7 Feb 2024 08:17:44 -0800 Subject: [PATCH 277/277] [rqd] Add SYSTEMDRIVE to env var list for Windows machines (#1341) --- rqd/rqd/rqcore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index 842028732..9391a2126 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -101,7 +101,7 @@ def __createEnvVariables(self): self.frameEnv["MAIL"] = "/usr/mail/%s" % self.runFrame.user_name self.frameEnv["HOME"] = "/net/homedirs/%s" % self.runFrame.user_name elif platform.system() == "Windows": - for variable in ["SYSTEMROOT", "APPDATA", "TMP", "COMMONPROGRAMFILES"]: + for variable in ["SYSTEMROOT", "APPDATA", "TMP", "COMMONPROGRAMFILES", "SYSTEMDRIVE"]: if variable in os.environ: self.frameEnv[variable] = os.environ[variable]