From 18ae9df769c8b03e302fff55ff04a3251241a57b Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Tue, 23 Jul 2024 20:20:29 +0200 Subject: [PATCH] Cy2024 (#1416) * Remove python < 3.7 requirements * Update pylint to 2.17.7 * Adjust pylintrc's and ignore consider-using-f-string * Ignore the proto modules globally for linting * Change pylint version to 2.15.10 * Fix pylint errors on pycue, pyoutline and cueadmin * Pylint fixed for cuegui part 1 * Fix some of the items conversions * Pylint fixed for cuegui part 2 * Remove python2 unit tests * Pylint fixed for cuegui part 3 * Pylint fixed for cuegui part 4 * Pylint fixed for cuesubmit * Add more ignored proto modules * Pylint fixed for rqd * Add more ignored proto modules * Pylint fixes for rqd * Update actions/checkout to v3 to keep in line with other tests * Revert change for FrameMonitorTree.py * Revert some of the "consider-using-dict-items" changes that somehow seems to crash under tests (just to be safe, for now) * Revert some of the "consider-using-dict-items" changes that somehow seems to crash under tests (just to be safe, for now) * Remove some redundant conditions * Update psutil * Add CY2024 tests * Remove dependency for PySide2 specifically in setup.py and rely on qtpy and requirements_gui.txt * Fix check for search case sensitivity in log viewer * Disable the LogViewerWidget._set_scrollbar_value since it's causing the search buttons to only search within visible text. Also it doesn't really seem to do anything. * Fix logging to file, it shouldn't open the file since it's already open. * Ignore non-ascii characters and don't add the extra newline. * Disabled connecting the _set_scrollbar_value since it's causing the search buttons to only search within visible text. Also it doesn't really seem to do anything. --- .github/workflows/testing-pipeline.yml | 22 ++++-- ci/pylintrc_main | 14 ++-- ci/pylintrc_test | 14 ++-- cueadmin/cueadmin/common.py | 2 + cuegui/cuegui/AbstractTreeWidget.py | 7 +- cuegui/cuegui/Comments.py | 1 + cuegui/cuegui/Constants.py | 6 +- cuegui/cuegui/CreateShowDialog.py | 1 + cuegui/cuegui/CueStateBarWidget.py | 4 +- cuegui/cuegui/DarkPalette.py | 3 +- cuegui/cuegui/DependWizard.py | 2 + cuegui/cuegui/EmailDialog.py | 1 + cuegui/cuegui/FilterDialog.py | 2 + cuegui/cuegui/FrameMonitorTree.py | 8 ++- cuegui/cuegui/HostMonitor.py | 2 +- cuegui/cuegui/ItemDelegate.py | 1 + cuegui/cuegui/JobMonitorTree.py | 4 +- cuegui/cuegui/LayerDialog.py | 3 + cuegui/cuegui/LimitsWidget.py | 1 - cuegui/cuegui/LocalBooking.py | 2 +- cuegui/cuegui/MainWindow.py | 7 +- cuegui/cuegui/MenuActions.py | 3 +- cuegui/cuegui/Plugins.py | 4 +- cuegui/cuegui/PreviewWidget.py | 34 ++++----- cuegui/cuegui/ProcMonitor.py | 1 + cuegui/cuegui/ProcMonitorTree.py | 1 - cuegui/cuegui/Redirect.py | 1 + cuegui/cuegui/ServiceDialog.py | 1 + cuegui/cuegui/ShowsWidget.py | 1 - cuegui/cuegui/TasksDialog.py | 1 + cuegui/cuegui/ThreadPool.py | 2 +- cuegui/cuegui/UnbookDialog.py | 1 + cuegui/cuegui/Utils.py | 19 ++--- cuegui/cuegui/eta.py | 52 +++++++------- cuegui/cuegui/plugins/AllocationsPlugin.py | 2 +- cuegui/cuegui/plugins/AttributesPlugin.py | 3 - cuegui/cuegui/plugins/LogViewPlugin.py | 7 +- cuegui/cuegui/plugins/StuckFramePlugin.py | 64 ++++++++--------- cuegui/setup.py | 1 - cuegui/tests/Layout_tests.py | 4 +- cuesubmit/cuesubmit/Config.py | 2 +- cuesubmit/cuesubmit/ui/Submit.py | 2 +- pycue/FileSequence/FrameSet.py | 2 +- pycue/opencue/config.py | 4 +- pycue/opencue/cuebot.py | 1 + pyoutline/outline/backend/local.py | 2 +- pyoutline/outline/event.py | 3 +- pyoutline/outline/executor.py | 2 +- pyoutline/outline/io.py | 12 ++-- pyoutline/outline/loader.py | 4 +- pyoutline/outline/modules/shell.py | 8 +-- pyoutline/outline/plugins/local.py | 12 ++-- pyoutline/outline/plugins/manager.py | 4 +- pyoutline/outline/session.py | 14 ++-- pyoutline/outline/versions/session.py | 1 + pyoutline/tests/json_test.py | 4 +- pyoutline/tests/layer_test.py | 13 ++-- pyoutline/tests/modules/shell_test.py | 4 +- requirements.txt | 17 ++--- requirements_gui.txt | 3 +- rqd/rqd/rqcore.py | 82 ++++++++-------------- rqd/rqd/rqmachine.py | 41 +++++------ rqd/rqd/rqnetwork.py | 2 + rqd/rqd/rqnimby.py | 4 +- rqd/rqd/rqswap.py | 4 +- rqd/rqd/rqutil.py | 38 +++++----- rqd/tests/rqconstants_tests.py | 2 +- 67 files changed, 307 insertions(+), 289 deletions(-) diff --git a/.github/workflows/testing-pipeline.yml b/.github/workflows/testing-pipeline.yml index c3816db57..bcbbb7706 100644 --- a/.github/workflows/testing-pipeline.yml +++ b/.github/workflows/testing-pipeline.yml @@ -53,23 +53,33 @@ jobs: chown -R aswfuser:aswfgroup . su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser - test_python2: - name: Run Python Unit Tests using Python2 + test_python_2024: + name: Run Python Unit Tests (CY2024) runs-on: ubuntu-latest - container: aswf/ci-opencue:2019 - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + container: aswf/ci-opencue:2024 steps: - uses: actions/checkout@v3 - name: Run Python Tests run: ci/run_python_tests.sh + test_cuebot_2024: + name: Build Cuebot and Run Unit Tests (CY2024) + runs-on: ubuntu-latest + container: + image: aswf/ci-opencue:2024 + steps: + - uses: actions/checkout@v3 + - name: Build with Gradle + run: | + chown -R aswfuser:aswfgroup . + su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser + test_pyside6: name: Run CueGUI Tests using PySide6 runs-on: ubuntu-latest container: almalinux:9 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Run CueGUI Tests run: ci/test_pyside6.sh diff --git a/ci/pylintrc_main b/ci/pylintrc_main index 85edaf4b4..cdd7f1ab6 100644 --- a/ci/pylintrc_main +++ b/ci/pylintrc_main @@ -77,7 +77,8 @@ disable=c-extension-no-member, too-many-public-methods, too-many-return-statements, too-many-statements, - useless-object-inheritance + useless-object-inheritance, + consider-using-f-string # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option @@ -202,7 +203,12 @@ ignored-classes=optparse.Values,thread._local,_thread._local # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis). It # supports qualified module names, as well as Unix pattern matching. -ignored-modules= +ignored-modules=opencue.compiled_proto, + opencue.compiled_proto.filter_pb2, + opencue.compiled_proto.host_pb2, + rqd.compiled_proto.rqd_pb2, + rqd.compiled_proto.host_pb2, + rqd.compiled_proto.report_pb2 # Show a hint with possible names when a member name was not found. The aspect # of finding the hint is based on edit distance. @@ -523,5 +529,5 @@ min-public-methods=2 # Exceptions that will emit a warning when being caught. Defaults to # "BaseException, Exception". -overgeneral-exceptions=BaseException, - Exception +overgeneral-exceptions=builtins.BaseException, + builtins.Exception diff --git a/ci/pylintrc_test b/ci/pylintrc_test index 4116f7e5f..434a1d754 100644 --- a/ci/pylintrc_test +++ b/ci/pylintrc_test @@ -77,7 +77,8 @@ disable=arguments-differ, too-many-statements, unused-argument, unused-variable, - useless-object-inheritance + useless-object-inheritance, + consider-using-f-string # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option @@ -202,7 +203,12 @@ ignored-classes=optparse.Values,thread._local,_thread._local # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis). It # supports qualified module names, as well as Unix pattern matching. -ignored-modules= +ignored-modules=opencue.compiled_proto, + opencue.compiled_proto.filter_pb2, + opencue.compiled_proto.host_pb2, + rqd.compiled_proto.rqd_pb2, + rqd.compiled_proto.host_pb2, + rqd.compiled_proto.report_pb2 # Show a hint with possible names when a member name was not found. The aspect # of finding the hint is based on edit distance. @@ -523,5 +529,5 @@ min-public-methods=2 # Exceptions that will emit a warning when being caught. Defaults to # "BaseException, Exception". -overgeneral-exceptions=BaseException, - Exception +overgeneral-exceptions=builtins.BaseException, + builtins.Exception diff --git a/cueadmin/cueadmin/common.py b/cueadmin/cueadmin/common.py index a48292846..b80a76e12 100644 --- a/cueadmin/cueadmin/common.py +++ b/cueadmin/cueadmin/common.py @@ -289,6 +289,7 @@ def _convert(val): else: result = opencue.api.criterion_pb2.GreaterThanFloatSearchCriterion( value=_convert(mixed)) + # pylint: disable=use-a-generator elif any([isinstance(mixed.__class__, crit_cls) for crit_cls in criterions]): result = mixed elif not mixed: @@ -336,6 +337,7 @@ def _convert(val): else: result = opencue.api.criterion_pb2.GreaterThanIntegerSearchCriterion( value=_convert(mixed)) + # pylint: disable=use-a-generator elif any([isinstance(mixed.__class__, crit_cls) for crit_cls in criterions]): result = mixed elif not mixed: diff --git a/cuegui/cuegui/AbstractTreeWidget.py b/cuegui/cuegui/AbstractTreeWidget.py index 0f4e9500a..eb748fa8a 100644 --- a/cuegui/cuegui/AbstractTreeWidget.py +++ b/cuegui/cuegui/AbstractTreeWidget.py @@ -48,11 +48,16 @@ COLUMN_TOOLTIP = 5 COLUMN_INFO_LENGTH = 6 -DEFAULT_LAMBDA = lambda s: "" DEFAULT_NAME = "" DEFAULT_WIDTH = 0 +# pylint: disable=unused-argument +def DEFAULT_LAMBDA(s): + """Dummy function to return something""" + return "" + + class AbstractTreeWidget(QtWidgets.QTreeWidget): """Base class for CueGUI tree widgets. diff --git a/cuegui/cuegui/Comments.py b/cuegui/cuegui/Comments.py index d1fdbc12d..27eed68bc 100644 --- a/cuegui/cuegui/Comments.py +++ b/cuegui/cuegui/Comments.py @@ -215,6 +215,7 @@ def refreshComments(self): comments[source.data.name] = source.getComments() self.__treeSubjects.clear() comments_length = 0 + # pylint: disable=consider-using-dict-items for source in comments: heading = CommentSource(source) heading.setSizeHint(0, QtCore.QSize(500, 1)) diff --git a/cuegui/cuegui/Constants.py b/cuegui/cuegui/Constants.py index 140a08093..575d20918 100644 --- a/cuegui/cuegui/Constants.py +++ b/cuegui/cuegui/Constants.py @@ -54,7 +54,7 @@ def __getLogger(): def __loadConfigFromFile(): logger = __getLogger() - with open(__DEFAULT_CONFIG_FILE) as fp: + with open(__DEFAULT_CONFIG_FILE, encoding='utf-8') as fp: config = yaml.load(fp, Loader=yaml.SafeLoader) user_config_file = None @@ -73,7 +73,7 @@ def __loadConfigFromFile(): if user_config_file: logger.info('Loading cuegui config from %s', user_config_file) - with open(user_config_file, 'r') as fp: + with open(user_config_file, 'r', encoding='utf-8') as fp: config.update(yaml.load(fp, Loader=yaml.SafeLoader)) return config @@ -83,7 +83,7 @@ def __packaged_version(): possible_version_path = os.path.join( os.path.abspath(os.path.join(__file__, "../../..")), 'VERSION.in') if os.path.exists(possible_version_path): - with open(possible_version_path) as fp: + with open(possible_version_path, encoding='utf-8') as fp: default_version = fp.read().strip() return default_version return "1.3.0" diff --git a/cuegui/cuegui/CreateShowDialog.py b/cuegui/cuegui/CreateShowDialog.py index 36e66fb49..efbf00e15 100644 --- a/cuegui/cuegui/CreateShowDialog.py +++ b/cuegui/cuegui/CreateShowDialog.py @@ -165,6 +165,7 @@ def __validateNoDuplicateShow(self): return False + # pylint: disable=inconsistent-return-statements def tryCreateShow(self): """Try to create the show in OpenCue diff --git a/cuegui/cuegui/CueStateBarWidget.py b/cuegui/cuegui/CueStateBarWidget.py index 26b7feb6f..90fd7d652 100644 --- a/cuegui/cuegui/CueStateBarWidget.py +++ b/cuegui/cuegui/CueStateBarWidget.py @@ -94,7 +94,7 @@ def paintEvent(self, event): @type event: QEvent @param event: The draw event""" del event - assert threading.currentThread().getName() == "MainThread" + assert threading.current_thread().name == "MainThread" self.__colorsLock.lockForWrite() try: if not self.__colors: @@ -136,7 +136,7 @@ def __updateBackgroundPixmap(self, colors): @param colors: List of job background colors""" # Could draw it the max size and allow the resize on drawPixmap # that way the same buffer is always used - assert threading.currentThread().getName() == "MainThread" + assert threading.current_thread().name == "MainThread" buffer = QtGui.QPixmap(self.contentsRect().size()) buffer.fill(self.__baseColor) diff --git a/cuegui/cuegui/DarkPalette.py b/cuegui/cuegui/DarkPalette.py index 373624e18..a71702a16 100644 --- a/cuegui/cuegui/DarkPalette.py +++ b/cuegui/cuegui/DarkPalette.py @@ -44,7 +44,8 @@ def init(): def setDarkStyleSheet(): """Sets the stylesheet.""" - cuegui.app().setStyleSheet(open(cuegui.Constants.DARK_STYLE_SHEET).read()) + with open(cuegui.Constants.DARK_STYLE_SHEET, encoding='utf-8') as fp: + cuegui.app().setStyleSheet(fp.read()) def DarkPalette(): diff --git a/cuegui/cuegui/DependWizard.py b/cuegui/cuegui/DependWizard.py index b07a5ee46..25de89944 100644 --- a/cuegui/cuegui/DependWizard.py +++ b/cuegui/cuegui/DependWizard.py @@ -122,6 +122,7 @@ def __init__(self, parent, jobs, layers=None, frames=None): self.__pages[PAGE_CONFIRMATION] = PageConfirmation(self, jobs, layers, frames) # Add the pages to the wizard + # pylint: disable=consider-using-dict-items for key in self.__pages : self.setPage(key, self.__pages[key]) @@ -389,6 +390,7 @@ def initializePage(self): self.wizard().setMinimumSize(500, 500) def validatePage(self): + # pylint: disable=consider-using-dict-items for option in self.__options: if self.__options[option].isChecked(): self.wizard().dependType = option diff --git a/cuegui/cuegui/EmailDialog.py b/cuegui/cuegui/EmailDialog.py index c5ef35799..dfc3f7eaa 100644 --- a/cuegui/cuegui/EmailDialog.py +++ b/cuegui/cuegui/EmailDialog.py @@ -83,6 +83,7 @@ class EmailWidget(QtWidgets.QWidget): def __init__(self, jobs, parent=None): QtWidgets.QWidget.__init__(self, parent=parent) + # pylint: disable=unused-private-member self.__jobs = jobs # Temporary workaround when pwd library is not available (i.e. Windows). diff --git a/cuegui/cuegui/FilterDialog.py b/cuegui/cuegui/FilterDialog.py index 479f1c6e5..0d4ca69e3 100644 --- a/cuegui/cuegui/FilterDialog.py +++ b/cuegui/cuegui/FilterDialog.py @@ -176,6 +176,7 @@ def _createItem(self, filter_object): def _processUpdate(self, work, rpcObjects): """Adds the feature of forcing the items to be sorted by the first column""" + # pylint: disable=protected-access cuegui.AbstractTreeWidget.AbstractTreeWidget._processUpdate(self, work, rpcObjects) def _getUpdate(self): @@ -224,6 +225,7 @@ def __init__(self, parent_filter, parent): cuegui.AbstractTreeWidget.AbstractTreeWidget.__init__(self, parent) # Used to build right click context menus + # pylint: disable=unused-private-member self.__menuActions = cuegui.MenuActions.MenuActions( self, self.updateSoon, self.selectedObjects) self._timer.stop() diff --git a/cuegui/cuegui/FrameMonitorTree.py b/cuegui/cuegui/FrameMonitorTree.py index abd80ada1..f7ead676e 100644 --- a/cuegui/cuegui/FrameMonitorTree.py +++ b/cuegui/cuegui/FrameMonitorTree.py @@ -218,6 +218,7 @@ def getFrameStateOverride(frame): cuegui.AbstractTreeWidget.AbstractTreeWidget.__init__(self, parent) # Used to build right click context menus + # pylint: disable=unused-private-member self.__menuActions = cuegui.MenuActions.MenuActions( self, self.updateSoon, self.selectedObjects, self.getJob) self.__sortByColumnCache = {} @@ -619,11 +620,13 @@ def __init__(self, rpcObject, parent, job): self.__class__.__alignCenter = QtCore.Qt.AlignCenter self.__class__.__alignRight = QtCore.Qt.AlignRight self.__class__.__rgbFrameState = {} + # pylint: disable=consider-using-dict-items for key in cuegui.Constants.RGB_FRAME_STATE: self.__class__.__rgbFrameState[key] = cuegui.Constants.RGB_FRAME_STATE[key] self.__class__.__type = cuegui.Constants.TYPE_FRAME cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( self, cuegui.Constants.TYPE_FRAME, rpcObject, parent, job) + # pylint: disable=unused-private-member self.__show = job.data.show def data(self, col, role): @@ -712,6 +715,7 @@ def __init__(self): self.__LINE = 2 self.__LLU = 3 + # pylint: disable=inconsistent-return-statements def getLastLineData(self, job, frame): """Returns the last line and LLU of the log file or queues a request to update it""" @@ -784,7 +788,7 @@ def __saveWork(self, work, results): __cached[self.__TIME] = time.time() __cached[self.__LINE] = results[1] __cached[self.__LLU] = results[2] - except KeyError as e: + except KeyError: # Could happen while switching jobs with work in the queue pass except Exception as e: @@ -871,7 +875,7 @@ def __saveWork(self, work, results): __cached = self.__cache[results[0]] __cached[self.__TIME] = time.time() __cached[self.__ETA] = results[1] - except KeyError as e: + except KeyError: # Could happen while switching jobs with work in the queue pass except Exception as e: diff --git a/cuegui/cuegui/HostMonitor.py b/cuegui/cuegui/HostMonitor.py index 66ee94287..c5b84491a 100644 --- a/cuegui/cuegui/HostMonitor.py +++ b/cuegui/cuegui/HostMonitor.py @@ -116,7 +116,7 @@ def __filterByHostNameSetup(self, layout): QtCore.QObject.connect(btn, QtCore.SIGNAL('clicked()'), self.__filterByHostNameClear) - + # pylint: disable=unused-private-member self.__filterByHostNameClearBtn = btn def __filterByHostNameHandle(self): diff --git a/cuegui/cuegui/ItemDelegate.py b/cuegui/cuegui/ItemDelegate.py index e4de3df02..a91f16b4f 100644 --- a/cuegui/cuegui/ItemDelegate.py +++ b/cuegui/cuegui/ItemDelegate.py @@ -419,6 +419,7 @@ def paint(self, painter, option, index): points = QtGui.QPolygon(len(hostItem.coresHistory) + 2) points.setPoint(0, option.rect.bottomLeft()) num = 1 + # pylint: disable=consider-using-enumerate for i in range(len(hostItem.coresHistory)): points.setPoint( num, option.rect.x() + stepWidth * i, diff --git a/cuegui/cuegui/JobMonitorTree.py b/cuegui/cuegui/JobMonitorTree.py index 2938e799b..8647d9df9 100644 --- a/cuegui/cuegui/JobMonitorTree.py +++ b/cuegui/cuegui/JobMonitorTree.py @@ -268,7 +268,7 @@ def addJob(self, job, timestamp=None, loading_from_config=False): else: # We'll only add the new job if it's not already listed # as a dependent on another job - if jobKey not in self.__reverseDependents.keys(): + if jobKey not in self.__reverseDependents: self.__load[jobKey] = newJobObj # when we are adding jobs manually, we want to calculate @@ -337,12 +337,14 @@ def _removeItem(self, item): @param item: A tree widget item @type item: AbstractTreeWidgetItem""" self.app.unmonitor.emit(item.rpcObject) + # pylint: disable=protected-access cuegui.AbstractTreeWidget.AbstractTreeWidget._removeItem(self, item) self.__jobTimeLoaded.pop(item.rpcObject, "") try: jobKey = cuegui.Utils.getObjectKey(item) # Remove the item from the main _items dictionary as well as the # __dependentJobs and the reverseDependent dictionaries + # pylint: disable=protected-access cuegui.AbstractTreeWidget.AbstractTreeWidget._removeItem(self, item) dependent_jobs = self.__dependentJobs.get(jobKey, []) for djob in dependent_jobs: diff --git a/cuegui/cuegui/LayerDialog.py b/cuegui/cuegui/LayerDialog.py index 8c9f0e57f..c0d1722b8 100644 --- a/cuegui/cuegui/LayerDialog.py +++ b/cuegui/cuegui/LayerDialog.py @@ -71,6 +71,7 @@ class LayerPropertiesItem(QtWidgets.QWidget): """An key/value widget for populating a dialog box.""" def __init__(self, label, widget, stretch=True, help_widget=None, parent=None): QtWidgets.QWidget.__init__(self, parent) + # pylint: disable=unused-private-member self.__label = label self.__widget = widget @@ -357,6 +358,7 @@ def getMaxMemory(self): def getMaxGpuMemory(self): """Gets the layer max GPU memory.""" + # pylint: disable=consider-using-generator return max([layer.data.min_gpu_memory // self.gpu_mem_tick_kb for layer in self.__layers]) def getMinCores(self): @@ -524,6 +526,7 @@ def __init__(self, layers, parent=None): QtWidgets.QDialog.__init__(self, parent) self._tags_widget = LayerTagsWidget(layers=layers, parent=parent) + # pylint: disable=unused-private-member self.__warning = QtWidgets.QLabel( 'Warning: Changing these tags may cause your job to not run any frames') self.__buttons = QtWidgets.QDialogButtonBox( diff --git a/cuegui/cuegui/LimitsWidget.py b/cuegui/cuegui/LimitsWidget.py index 25c962492..82f236342 100644 --- a/cuegui/cuegui/LimitsWidget.py +++ b/cuegui/cuegui/LimitsWidget.py @@ -133,7 +133,6 @@ def _createItem(self, rpcObject): item = LimitWidgetItem(rpcObject, self) return item - # pylint: disable=no-self-use def _getUpdate(self): """Returns the proper data from the cuebot""" try: diff --git a/cuegui/cuegui/LocalBooking.py b/cuegui/cuegui/LocalBooking.py index 23220e113..d4233208c 100644 --- a/cuegui/cuegui/LocalBooking.py +++ b/cuegui/cuegui/LocalBooking.py @@ -242,7 +242,7 @@ def deedLocalhost(self): user = os.environ["USER"] try: owner = opencue.api.getOwner(user) - except opencue.EntityNotFoundException as e: + except opencue.EntityNotFoundException: # Owner does not exist owner = _show.createOwner(user) diff --git a/cuegui/cuegui/MainWindow.py b/cuegui/cuegui/MainWindow.py index 5800ba2e2..787458ea2 100644 --- a/cuegui/cuegui/MainWindow.py +++ b/cuegui/cuegui/MainWindow.py @@ -172,9 +172,9 @@ def __facilityMenuHandle(self, action): self.__actions_facility[self.facility_default].setChecked(True) # Uncheck all other facilities if one is checked else: - for facility in self.__actions_facility: + for facility, facvalue in self.__actions_facility.items(): if facility != action.text(): - self.__actions_facility[facility].setChecked(False) + facvalue.setChecked(False) for facility in list(self.__actions_facility.values()): if facility.isChecked(): @@ -300,12 +300,13 @@ def __windowMenuHandle(self, action): action_title = str(action.text()) if action_title.startswith("Open Window: "): window_title = action_title.replace("Open Window: ","") + # pylint: disable=consider-using-dict-items for name in self.windows_titles: if self.windows_titles[name] == window_title: self.windowMenuOpenWindow(name) elif action_title.endswith("Add new window") and len(action_title) == 18: - number = int(action_title[1:].split(")")[0]) - 1 + number = int(action_title[1:].split(")", maxsplit=1)[0]) - 1 self.windowMenuOpenWindow(self.windows_names[number]) elif action_title.startswith("Raise Window: "): diff --git a/cuegui/cuegui/MenuActions.py b/cuegui/cuegui/MenuActions.py index 42be94cca..4a1b44249 100644 --- a/cuegui/cuegui/MenuActions.py +++ b/cuegui/cuegui/MenuActions.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=consider-using-generator """Provides actions and functions for right click menu items.""" @@ -70,7 +71,7 @@ DEFAULT_JOB_KILL_REASON = "Manual Job Kill Request in Cuegui by " + getpass.getuser() DEFAULT_FRAME_KILL_REASON = "Manual Frame(s) Kill Request in Cuegui by " + getpass.getuser() -# pylint: disable=missing-function-docstring,no-self-use,unused-argument +# pylint: disable=missing-function-docstring,unused-argument class AbstractActions(object): diff --git a/cuegui/cuegui/Plugins.py b/cuegui/cuegui/Plugins.py index db0e1b335..c4826c2b8 100644 --- a/cuegui/cuegui/Plugins.py +++ b/cuegui/cuegui/Plugins.py @@ -321,11 +321,13 @@ def setupPluginMenu(self, menu): menu_locations[category] = [] # Store the plugin name in the proper menu_locations category + # pylint: disable=consider-using-dict-items for plugin in self.__plugins: category = self.__plugins[plugin].get(CATEGORY, "root") menu_locations[category].append(plugin) # Create the QAction and add it to the correct menu (sorted) + # pylint: disable=consider-using-dict-items for category in menu_locations: for plugin in sorted(menu_locations[category]): action = QtWidgets.QAction("{}".format(plugin), menu) @@ -340,7 +342,7 @@ def _handlePluginMenu(self, action): """Handles what happens when a plugin menu item is clicked on @param action: The action that was selected from the menu @type action: QAction""" - plugin_name = str(action.text()).split("%s" % self.__menu_separator)[0] + plugin_name = str(action.text()).split("%s" % self.__menu_separator, maxsplit=1)[0] self.launchPlugin(plugin_name, "") diff --git a/cuegui/cuegui/PreviewWidget.py b/cuegui/cuegui/PreviewWidget.py index 5dfea8658..56365e177 100644 --- a/cuegui/cuegui/PreviewWidget.py +++ b/cuegui/cuegui/PreviewWidget.py @@ -65,6 +65,7 @@ def __init__(self, job, frame, aovs=False, parent=None): self.__aovs = aovs self.__previewThread = None + # pylint: disable=unused-private-member self.__itvFile = None layout = QtWidgets.QVBoxLayout(self) @@ -92,6 +93,7 @@ def process(self): if not items: return + # pylint: disable=unused-private-member self.__itvFile = self.__writePlaylist(playlist) self.__previewThread = PreviewProcessorWatchThread(items, self) self.app.threads.append(self.__previewThread) @@ -120,26 +122,26 @@ def processTimedOut(self): def __writePlaylist(data): (fh, name) = tempfile.mkstemp(suffix=".itv", prefix="playlist") os.close(fh) - fp = open(name, "w") - try: - fp.write(data) - finally: - fp.close() + with open(name, "w", encoding='utf-8') as fp: + try: + fp.write(data) + finally: + fp.close() return name def __findHttpPort(self): log = cuegui.Utils.getFrameLogFile(self.__job, self.__frame) - fp = open(log, "r") - try: - counter = 0 - for line in fp: - counter += 1 - if counter >= 5000: - break - if line.startswith("Preview Server"): - return int(line.split(":")[1].strip()) - finally: - fp.close() + with open(log, "r", encoding='utf-8') as fp: + try: + counter = 0 + for line in fp: + counter += 1 + if counter >= 5000: + break + if line.startswith("Preview Server"): + return int(line.split(":")[1].strip()) + finally: + fp.close() raise Exception("Katana 2.7.19 and above is required for preview feature.") diff --git a/cuegui/cuegui/ProcMonitor.py b/cuegui/cuegui/ProcMonitor.py index b4ae4f5d4..422a89a4f 100644 --- a/cuegui/cuegui/ProcMonitor.py +++ b/cuegui/cuegui/ProcMonitor.py @@ -111,6 +111,7 @@ def __filterByHostNameSetup(self, layout): btn.setFixedWidth(24) layout.addWidget(btn) btn.clicked.connect(self.__filterByHostNameClear) # pylint: disable=no-member + # pylint: disable=unused-private-member self.__filterByHostNameClearBtn = btn self.__filterByHostNameLastInput = None diff --git a/cuegui/cuegui/ProcMonitorTree.py b/cuegui/cuegui/ProcMonitorTree.py index 0f4710b53..d549e35f3 100644 --- a/cuegui/cuegui/ProcMonitorTree.py +++ b/cuegui/cuegui/ProcMonitorTree.py @@ -126,7 +126,6 @@ def __itemSingleClickedCopy(self, item, col): if selected: QtWidgets.QApplication.clipboard().setText(",".join(selected)) - # pylint: disable=no-self-use def __itemDoubleClickedViewLog(self, item, col): """Called when a proc is double clicked @type item: QTreeWidgetItem diff --git a/cuegui/cuegui/Redirect.py b/cuegui/cuegui/Redirect.py index 191bfb6ad..1476b1ad2 100644 --- a/cuegui/cuegui/Redirect.py +++ b/cuegui/cuegui/Redirect.py @@ -575,6 +575,7 @@ def __isBurstSafe(self, alloc, procs, show): try: procsBurst = (showSubs.get(alloc).data.burst - showSubs.get(alloc).data.reserved_cores) + # pylint: disable=consider-using-generator procsRedirect = int(sum([p.data.reserved_cores for p in procs])) wastedCores = int(procsRedirect - procsBurst) diff --git a/cuegui/cuegui/ServiceDialog.py b/cuegui/cuegui/ServiceDialog.py index 8c6ebec2a..f83dbb5c3 100644 --- a/cuegui/cuegui/ServiceDialog.py +++ b/cuegui/cuegui/ServiceDialog.py @@ -345,6 +345,7 @@ class ServiceDialog(QtWidgets.QDialog): def __init__(self, show, parent=None): QtWidgets.QDialog.__init__(self, parent) + # pylint: disable=unused-private-member self.__srv_manager = ServiceManager(show, self) self.setWindowTitle("Services") diff --git a/cuegui/cuegui/ShowsWidget.py b/cuegui/cuegui/ShowsWidget.py index 20b62ae73..8d24b7ae7 100644 --- a/cuegui/cuegui/ShowsWidget.py +++ b/cuegui/cuegui/ShowsWidget.py @@ -85,7 +85,6 @@ def _createItem(self, rpcObject): item = ShowWidgetItem(rpcObject, self) return item - # pylint: disable=no-self-use def _getUpdate(self): """Returns the proper data from the cuebot""" try: diff --git a/cuegui/cuegui/TasksDialog.py b/cuegui/cuegui/TasksDialog.py index 2f820d76c..79e2cdb1a 100644 --- a/cuegui/cuegui/TasksDialog.py +++ b/cuegui/cuegui/TasksDialog.py @@ -199,6 +199,7 @@ def _createItem(self, rpcObject): def _update(self): """Adds the feature of forcing the items to be sorted by the first column""" + # pylint: disable=protected-access cuegui.AbstractTreeWidget.AbstractTreeWidget._update(self) self.sortByColumn(0, QtCore.Qt.AscendingOrder) diff --git a/cuegui/cuegui/ThreadPool.py b/cuegui/cuegui/ThreadPool.py index fa6e6c412..8a615ee6c 100644 --- a/cuegui/cuegui/ThreadPool.py +++ b/cuegui/cuegui/ThreadPool.py @@ -118,7 +118,6 @@ def local(self, callable_to_queue, callback, comment, *args): if work[1]: self.runCallback(work, result) - # pylint: disable=no-self-use def runCallback(self, work, result): """Runs the callback function.""" if work[1]: @@ -136,6 +135,7 @@ class WorkerThread(QtCore.QThread): def __init__(self, name, parent): QtCore.QThread.__init__(self, parent) self.__parent = parent + # pylint: disable=unused-private-member self.__name = name self.__running = False diff --git a/cuegui/cuegui/UnbookDialog.py b/cuegui/cuegui/UnbookDialog.py index a661faa93..334c43909 100644 --- a/cuegui/cuegui/UnbookDialog.py +++ b/cuegui/cuegui/UnbookDialog.py @@ -405,6 +405,7 @@ def __init__(self, procSearch, parent=None): self.setFixedWidth(500) self.setWindowTitle("Unbook and kill frames?") + # pylint: disable=unused-private-member self.__procSearch = procSearch self.__procs = opencue.api.getProcs(**procSearch.options) self.__amount = len(self.__procs) diff --git a/cuegui/cuegui/Utils.py b/cuegui/cuegui/Utils.py index 4666797b5..5b88345fd 100644 --- a/cuegui/cuegui/Utils.py +++ b/cuegui/cuegui/Utils.py @@ -199,8 +199,9 @@ def getCuewho(show): @return: The username who is cuewho for the show @rtype: string""" try: - file = open("/shots/%s/home/cue/cuewho.who" % show, "r") - return file.read() + + with open("/shots/%s/home/cue/cuewho.who" % show, "r", encoding='utf-8') as file: + return file.read() except Exception as e: logger.warning("Failed to update cuewho: %s\n%s", show, e) return "Unknown" @@ -315,6 +316,7 @@ def exceptionOutput(e): def handleExceptions(function): """Custom exception handler.""" + # pylint: disable=inconsistent-return-statements def new(*args): try: return function(*args) @@ -447,15 +449,14 @@ def getLastLine(path): ansiEscape = r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]' try: - fp = open(path, 'rb') - fp.seek(0, 2) + with open(path, 'rb') as fp: + fp.seek(0, 2) - backseek = min(4096, fp.tell()) - fp.seek(-backseek, 1) - buf = fp.read(4096) + backseek = min(4096, fp.tell()) + fp.seek(-backseek, 1) + buf = fp.read(4096) - newline_pos = buf.rfind(b'\n', 0, len(buf)-1) - fp.close() + newline_pos = buf.rfind(b'\n', 0, len(buf)-1) line = buf[newline_pos+1:].strip().decode("utf-8") diff --git a/cuegui/cuegui/eta.py b/cuegui/cuegui/eta.py index 3a91a77e8..b67bdf73c 100644 --- a/cuegui/cuegui/eta.py +++ b/cuegui/cuegui/eta.py @@ -55,7 +55,8 @@ def GetFrameEta(self, job, frame): """Gets ETA for the given frame.""" self.log = opencue.util.logPath(job, frame) if os.path.isfile(self.log): - self.log_lines = len(open(self.log).readlines()) + with open(self.log, encoding='utf-8') as fp: + self.log_lines = len(fp.readlines()) self.GetFrameBuildTime(frame) try: layer = opencue.api.findLayer(job.data.name, frame.data.layer_name) @@ -120,21 +121,22 @@ def Svea(self, frame): del frame if os.path.isfile(self.log): line = '' - for line in reversed(open(self.log).readlines()): - # Checks log directory for a percentage complete in reverse to limit time in log. - if 'Running generator batch' in line: - # pylint: disable=bare-except - try: - time_on_log = self.GetSeconds(line) - line = line.split(' ') - current = float(line[16]) - total = float(line[18].split('\n')[0]) - percent = float(current / total) * 100 - self.percents.append((percent, time_on_log)) - if len(self.percents) > 1: - break - except: - pass + with open(self.log, encoding='utf-8') as fp: + for line in reversed(fp.readlines()): + # Checks log directory for a percentage complete in reverse to limit time in log + if 'Running generator batch' in line: + # pylint: disable=bare-except + try: + time_on_log = self.GetSeconds(line) + line = line.split(' ') + current = float(line[16]) + total = float(line[18].split('\n')[0]) + percent = float(current / total) * 100 + self.percents.append((percent, time_on_log)) + if len(self.percents) > 1: + break + except: + pass if len(self.percents) > 1: self.percents = sorted(self.percents, reverse=True) self.total_completion = ( @@ -186,10 +188,11 @@ def GetFrameStartTime(self, frame): return self.startTimeCache[key] # Read the logFile here for time. result = '' - for line in open(self.log): - if '% done' in line: - result = line - break + with open(self.log, encoding='utf-8') as fp: + for line in fp: + if '% done' in line: + result = line + break if not result: return result self.startTimeCache[key] = result @@ -202,10 +205,11 @@ def GetFrameBuildTime(self, frame): return self.buildTimeCache[key] # Read the logFile here for time. result_line = None - for line in open(self.log): - if 'Building scene done' in line: - result_line = line - break + with open(self.log, encoding='utf-8') as fp: + for line in fp: + if 'Building scene done' in line: + result_line = line + break if result_line is not None: result = { 'scene_build_seconds': self.GetSeconds(result_line), diff --git a/cuegui/cuegui/plugins/AllocationsPlugin.py b/cuegui/cuegui/plugins/AllocationsPlugin.py index a7d18162e..d594466f5 100644 --- a/cuegui/cuegui/plugins/AllocationsPlugin.py +++ b/cuegui/cuegui/plugins/AllocationsPlugin.py @@ -96,6 +96,7 @@ def __init__(self, parent): cuegui.AbstractTreeWidget.AbstractTreeWidget.__init__(self, parent) # Used to build right click context menus + # pylint: disable=unused-private-member self.__menuActions = cuegui.MenuActions.MenuActions( self, self.updateSoon, self.selectedObjects) @@ -112,7 +113,6 @@ def _createItem(self, rpcObject): """Creates and returns the proper item""" return AllocationWidgetItem(rpcObject, self) - # pylint: disable=no-self-use def _getUpdate(self): """Returns the proper data from the cuebot""" try: diff --git a/cuegui/cuegui/plugins/AttributesPlugin.py b/cuegui/cuegui/plugins/AttributesPlugin.py index c25480f85..b978ce804 100644 --- a/cuegui/cuegui/plugins/AttributesPlugin.py +++ b/cuegui/cuegui/plugins/AttributesPlugin.py @@ -240,7 +240,6 @@ def preload(layer): """Prepopulates needed layer information.""" return {"depends": layer.getWhatThisDependsOn()} - # pylint: disable=no-self-use def dataSource(self, layer, preload): """Returns layer information structured as needed for the attributes list.""" d = { @@ -314,7 +313,6 @@ def preload(jobObject): """Prepopulates needed job information.""" return {"depends": jobObject.getWhatThisDependsOn()} - # pylint: disable=no-self-use def dataSource(self, job, preload): """Returns job information structured as needed for the attributes list.""" if isinstance(job, opencue.wrappers.job.NestedJob): @@ -398,7 +396,6 @@ class HostAttributes(AbstractAttributes): NAME = "Host" - # pylint: disable=no-self-use def dataSource(self, host, preload): """Returns host information structured as needed for the attributes list.""" del preload diff --git a/cuegui/cuegui/plugins/LogViewPlugin.py b/cuegui/cuegui/plugins/LogViewPlugin.py index 9d86e712f..c2a7837a2 100644 --- a/cuegui/cuegui/plugins/LogViewPlugin.py +++ b/cuegui/cuegui/plugins/LogViewPlugin.py @@ -455,7 +455,6 @@ def __init__(self, parent=None): # Signals are defined in code, so pylint thinks they don't exist. self.app.display_log_file_content.connect(self._set_log_files) self._log_scrollbar = self._content_box.verticalScrollBar() - self._log_scrollbar.valueChanged.connect(self._set_scrollbar_value) self._new_log = False self._current_log_index = 0 @@ -494,6 +493,7 @@ def _on_mouse_pressed(self, pos): self._update_visible_indices() cursor_for_pos = self._content_box.cursorForPosition(pos) index = cursor_for_pos.position() + # pylint: disable=consider-using-enumerate for i in range(0, len(self._matches)): if index < self._matches[i][0]: self._current_match = i @@ -667,7 +667,7 @@ def _find_text(self): self._clear_search_data() return - search_case_stv = self._case_stv_checkbox.checkState() + search_case_stv = self._case_stv_checkbox.isChecked() if self._content_timestamp <= self._search_timestamp: if prev_search == self._search_text: # Same content & pattern if self._last_search_case_stv == search_case_stv: @@ -834,7 +834,6 @@ def _display_log_content(self): finally: QtCore.QTimer.singleShot(5000, self._display_log_content) - # pylint: disable=no-self-use @QtCore.Slot() def _load_log(self, log_file, new_log, curr_log_mtime): content = None @@ -852,7 +851,7 @@ def _load_log(self, log_file, new_log, curr_log_mtime): if content is None: content = '' try: - with open(log_file, 'r') as f: + with open(log_file, 'r', encoding='utf-8') as f: content = f.read() except IOError: content = 'Can not access log file: %s' % log_file diff --git a/cuegui/cuegui/plugins/StuckFramePlugin.py b/cuegui/cuegui/plugins/StuckFramePlugin.py index 3696bd97f..5920a373a 100644 --- a/cuegui/cuegui/plugins/StuckFramePlugin.py +++ b/cuegui/cuegui/plugins/StuckFramePlugin.py @@ -814,7 +814,7 @@ def logIt(self): def logResult(self, work, rpcObjects): self.frames = {} - # pylint: disable=redefined-builtin,inconsistent-return-statements,no-self-use + # pylint: disable=redefined-builtin,inconsistent-return-statements def numFormat(self, num, type): """Returns string formatting based on the number""" if num == "" or num < .001 or num is None: @@ -906,7 +906,6 @@ def updateRequest(self): self.ticksWithoutUpdate = 999 self.completeRefresh = True - # pylint: disable=no-self-use def get_frame_run_time(self, item): """Returns frame run time.""" if cuegui.Utils.isProc(item): @@ -919,7 +918,6 @@ def get_frame_run_time(self, item): run_time = current_time - start_time return run_time - # pylint: disable=no-self-use def get_llu_time(self, item): """Returns LLU time.""" if cuegui.Utils.isProc(item): @@ -957,6 +955,7 @@ def confirm(self, update): """Confirm frame filter.""" currentHostsNew = [] nextIndex = 2 + # pylint: disable=consider-using-enumerate for index in range(len(self.currentHosts)): if index == nextIndex: frame = self.currentHosts[index] @@ -1072,8 +1071,8 @@ def _getUpdate(self): if percentStuck * 100 > self.time_filter and percentStuck < 1.1: please_exclude = False for exclude in self.excludes: - if (layerName.__contains__(exclude) or - jobName.__contains__(exclude)): + if (exclude in layerName or + exclude in jobName): please_exclude = True continue @@ -1301,6 +1300,7 @@ def topMachine(self): def remove(self): currentHostsNew = [] nextIndex = 2 + # pylint: disable=consider-using-enumerate for index in range(len(self.currentHosts)): if index == nextIndex: @@ -1434,6 +1434,7 @@ def RemoveJob(self): jobName = self.selectedObjects()[0].data.name currentHostsNew = [] nextIndex = 2 + # pylint: disable=consider-using-enumerate for index in range(len(self.currentHosts)): if index == nextIndex: @@ -1500,28 +1501,27 @@ def finalize(self, frames): yaml_path = "/shots/" + self.show + "/home/etc/stuck_frames_db.yaml" if not os.path.exists(yaml_path): - yaml_ob = open(yaml_path, 'w') - yaml.dump(dict, yaml_ob) - yaml_ob.close() + + with open(yaml_path, 'w', encoding='utf-8') as yaml_ob: + yaml.dump(dict, yaml_ob) else: - yaml_ob = open(yaml_path, 'r') - old_dict = yaml.load(yaml_ob) - yaml_ob.close() + with open(yaml_path, 'r', encoding='utf-8') as yaml_ob: + old_dict = yaml.load(yaml_ob) - yaml_ob = open(yaml_path, 'w') + with open(yaml_path, 'w', encoding='utf-8') as yaml_ob: - for key in dict: # updates old dict - old_dict[key] = dict[key] + for key in dict: # updates old dict + old_dict[key] = dict[key] - yaml.dump(old_dict, yaml_ob) - yaml_ob.close() + yaml.dump(old_dict, yaml_ob) class CommentWidget(QtWidgets.QWidget): """Represents a comment.""" def __init__(self, subject, message, parent=None): QtWidgets.QWidget.__init__(self, parent) + # pylint: disable=unused-private-member self.__textSubject = subject self.__textMessage = message @@ -1576,28 +1576,23 @@ def data(self, col, role): class LogFinal(): """Utility class for logging to yaml.""" - # pylint: disable=no-self-use def finalize(self, frames, show): """Saves logs to yaml. If file not created, will create one.""" frames_dict = frames yaml_path = "/shots/" + show + "/home/etc/stuck_frames_db.yaml" if not os.path.exists(yaml_path): - yaml_ob = open(yaml_path, 'w') - yaml.dump(frames_dict, yaml_ob) - yaml_ob.close() - else: - yaml_ob = open(yaml_path, 'r') - old_dict = yaml.load(yaml_ob) - yaml_ob.close() - - yaml_ob = open(yaml_path, 'w') + with open(yaml_path, 'w', encoding='utf-8') as yaml_ob: + yaml.dump(frames_dict, yaml_ob) - for key in frames_dict: # updates old dict - old_dict[key] = frames_dict[key] + else: + with open(yaml_path, 'r', encoding='utf-8') as yaml_ob: + old_dict = yaml.load(yaml_ob) + with open(yaml_path, 'w', encoding='utf-8') as yaml_ob: + for key in frames_dict: # updates old dict + old_dict[key] = frames_dict[key] - yaml.dump(old_dict, yaml_ob) - yaml_ob.close() + yaml.dump(old_dict, yaml_ob) class HostWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): @@ -1765,7 +1760,6 @@ def __init__(self, show=None): show = os.environ.get('SHOW') self.show = show - # pylint: disable=no-self-use def getLog(self, job, frame): """Return the contents of a log given a job and a frame.""" log_dir = job.logDir() @@ -1773,10 +1767,10 @@ def getLog(self, job, frame): log_file = os.path.join(log_dir, log_name) if not os.path.exists(log_file): return [] - f = open(log_file, 'r') - log_lines = [line.strip() for line in f.readlines() if line.strip()] - f.close() - return log_lines + with open(log_file, 'r', encoding='utf-8') as f: + log_lines = [line.strip() for line in f.readlines() if line.strip()] + + return log_lines def getBuildTimes(self, job, layers=None): """Return a dictionary with layer names as keys, and build tiems as diff --git a/cuegui/setup.py b/cuegui/setup.py index 2068936e0..1c7c6455b 100644 --- a/cuegui/setup.py +++ b/cuegui/setup.py @@ -63,7 +63,6 @@ 'future', 'grpcio', 'grpcio-tools', - 'PySide2', 'PyYAML', 'QtPy', ] diff --git a/cuegui/tests/Layout_tests.py b/cuegui/tests/Layout_tests.py index 2b6818f9c..70b2c186d 100644 --- a/cuegui/tests/Layout_tests.py +++ b/cuegui/tests/Layout_tests.py @@ -63,7 +63,7 @@ def test__should_load_user_layout(self): app_name = 'arbitraryapp' config_file_path = os.path.join(self.config_dir, '.%s' % app_name, 'config.ini') os.mkdir(os.path.dirname(config_file_path)) - with open(config_file_path, 'w') as fp: + with open(config_file_path, 'w', encoding='utf-8') as fp: fp.write(CONFIG_INI) settings = cuegui.Layout.startup(app_name) @@ -83,7 +83,7 @@ def test__should_load_default_layout(self): def test__should_restore_default_layout(self): config_file_path = os.path.join(self.config_dir, '.cuecommander', 'config.ini') os.mkdir(os.path.dirname(config_file_path)) - with open(config_file_path, 'w') as fp: + with open(config_file_path, 'w', encoding='utf-8') as fp: fp.write(CONFIG_WITH_RESTORE_FLAG) settings = cuegui.Layout.startup('CueCommander') diff --git a/cuesubmit/cuesubmit/Config.py b/cuesubmit/cuesubmit/Config.py index d657f5402..3bfb443a7 100644 --- a/cuesubmit/cuesubmit/Config.py +++ b/cuesubmit/cuesubmit/Config.py @@ -40,7 +40,7 @@ def getConfigValues(): if not configFile: configFile = os.path.join(opencue.config.config_base_directory(), 'cuesubmit.yaml') if os.path.exists(configFile): - with open(configFile, 'r') as data: + with open(configFile, 'r', encoding='utf-8') as data: try: configData = yaml.load(data, Loader=yaml.SafeLoader) except yaml.YAMLError: diff --git a/cuesubmit/cuesubmit/ui/Submit.py b/cuesubmit/cuesubmit/ui/Submit.py index 36a6a43ba..5f0b42f5b 100644 --- a/cuesubmit/cuesubmit/ui/Submit.py +++ b/cuesubmit/cuesubmit/ui/Submit.py @@ -81,7 +81,7 @@ class CueSubmitWidget(QtWidgets.QWidget): def __init__( self, settingsWidgetType, jobTypes=JobTypes.JobTypes, parent=None, *args, **kwargs): super(CueSubmitWidget, self).__init__(parent) - self.startupErrors = list() + self.startupErrors = [] self.skipDataChangedEvent = False self.settings = QtCore.QSettings('opencue', 'cuesubmit') self.clearMessageShown = False diff --git a/pycue/FileSequence/FrameSet.py b/pycue/FileSequence/FrameSet.py index 5ff1629e4..7ee41267d 100644 --- a/pycue/FileSequence/FrameSet.py +++ b/pycue/FileSequence/FrameSet.py @@ -82,7 +82,7 @@ def parseFrameRange(frameRange): :rtype: FrameSet :return: The FrameSet representing the same sequence. """ - frameList = list() + frameList = [] for frameRangeSection in frameRange.split(','): frameList.extend(FrameRange.parseFrameRange(frameRangeSection)) return frameList diff --git a/pycue/opencue/config.py b/pycue/opencue/config.py index 7d1040a87..d1cfcf2c0 100644 --- a/pycue/opencue/config.py +++ b/pycue/opencue/config.py @@ -70,7 +70,7 @@ def load_config_from_file(): :rtype: dict :return: config settings """ - with open(__DEFAULT_CONFIG_FILE) as file_object: + with open(__DEFAULT_CONFIG_FILE, encoding="utf-8") as file_object: config = yaml.load(file_object, Loader=yaml.SafeLoader) user_config_file = None @@ -90,7 +90,7 @@ def load_config_from_file(): if user_config_file: logger.info('Loading opencue config from %s', user_config_file) - with open(user_config_file) as file_object: + with open(user_config_file, encoding="utf-8") as file_object: config.update(yaml.load(file_object, Loader=yaml.SafeLoader)) return config diff --git a/pycue/opencue/cuebot.py b/pycue/opencue/cuebot.py index 6090e2403..daafa4bfb 100644 --- a/pycue/opencue/cuebot.py +++ b/pycue/opencue/cuebot.py @@ -357,6 +357,7 @@ def __init__(self, self._sleeping_policy = sleeping_policy self._retry_statuses = status_for_retry + # pylint: disable=inconsistent-return-statements def _intercept_call(self, continuation, client_call_details, request_or_iterator): for attempt in range(self._max_attempts): diff --git a/pyoutline/outline/backend/local.py b/pyoutline/outline/backend/local.py index 7aa90595a..750c5d447 100644 --- a/pyoutline/outline/backend/local.py +++ b/pyoutline/outline/backend/local.py @@ -93,7 +93,7 @@ def build_frame_range(frame_range, chunk_size): frames = [] if chunk_size > 1: if chunk_size >= len(frame_set): - frames.append(frame_set.__getitem__(0)) + frames.append(frame_set[0]) else: unique_frames = list(set(frame_set)) for i, unique_frame in enumerate(unique_frames): diff --git a/pyoutline/outline/event.py b/pyoutline/outline/event.py index 4ba5edf84..98a82b75a 100644 --- a/pyoutline/outline/event.py +++ b/pyoutline/outline/event.py @@ -53,6 +53,7 @@ class EventHandler(object): EventHandler keeps track of who is listening for which events. """ def __init__(self, component): + # pylint: disable=unused-private-member self.__component = component self.__listeners = {} @@ -87,7 +88,7 @@ def get_event_listeners(self, event_type): try: return self.__listeners[event_type] except KeyError: - return list() + return [] class LaunchEvent(object): diff --git a/pyoutline/outline/executor.py b/pyoutline/outline/executor.py index bd7c6bff7..347dc4f1b 100644 --- a/pyoutline/outline/executor.py +++ b/pyoutline/outline/executor.py @@ -45,7 +45,7 @@ def __init__(self, threads): for i in range(0, threads): logger.debug("executor creating thread #%d", i) t = threading.Thread(target=self.worker) - t.setDaemon(True) + t.daemon = True t.start() def execute(self, callable_, *args): diff --git a/pyoutline/outline/io.py b/pyoutline/outline/io.py index 67d4b1138..5ce0bac96 100644 --- a/pyoutline/outline/io.py +++ b/pyoutline/outline/io.py @@ -258,7 +258,7 @@ def exists(path): for f in frame_set: path = self.get_frame_path(f) if not exists(path): - for ext in self.get_attribute("checkExt", list()): + for ext in self.get_attribute("checkExt", []): n = path[0:path.rfind(self.get_ext())] n = "%s%s" % (n, ext) if exists(n): @@ -267,7 +267,7 @@ def exists(path): else: for path in self.__fs: if not exists(path): - for ext in self.get_attribute("checkExt", list()): + for ext in self.get_attribute("checkExt", []): n = path[0:path.rfind(self.get_ext())] n = "%s%s" % (n, ext) if exists(n): @@ -436,9 +436,9 @@ def file_spec_serializer(dumper, data): Serialize a FileSpec object. This is required for Yaml to serialize a FileSpec properly. """ - return dumper.represent_scalar(u'!FileSpec', - u'%s' % yaml.dump([data.get_path(), - data.get_attributes()])) + return dumper.represent_scalar('!FileSpec', + '%s' % yaml.dump([data.get_path(), + data.get_attributes()])) def file_spec_constructor(loader, node): @@ -451,4 +451,4 @@ def file_spec_constructor(loader, node): # Register the yaml serialize/unserialize callbacks. yaml.add_representer(FileSpec, file_spec_serializer) -yaml.add_constructor(u'!FileSpec', file_spec_constructor) +yaml.add_constructor('!FileSpec', file_spec_constructor) diff --git a/pyoutline/outline/loader.py b/pyoutline/outline/loader.py index 0b0d43a18..907914936 100644 --- a/pyoutline/outline/loader.py +++ b/pyoutline/outline/loader.py @@ -77,7 +77,7 @@ def load_outline(path): ext = os.path.splitext(path) if ext[1] == ".yaml" or path.find("cue_archive") != -1: - with open(path) as file_object: + with open(path, encoding='utf-8') as file_object: ol = yaml.load(file_object, Loader=yaml.FullLoader) Outline.current = ol if not isinstance(ol, Outline): @@ -172,7 +172,7 @@ def parse_outline_script(path): """ try: logger.info("parsing outline file %s", path) - with open(path) as fp: + with open(path, encoding='utf-8') as fp: code = compile(fp.read(), path, 'exec') exec(code) # pylint: disable=exec-used except Exception as exp: diff --git a/pyoutline/outline/modules/shell.py b/pyoutline/outline/modules/shell.py index 482fcda4a..f86397023 100644 --- a/pyoutline/outline/modules/shell.py +++ b/pyoutline/outline/modules/shell.py @@ -47,16 +47,14 @@ def __init__(self, name, code, **args): self.__code = code def _setup(self): - fp = open("%s/script" % self.get_path(), "w") - try: + with open(f"{self.get_path()}/script", "w", encoding="utf-8") as fp: fp.write(self.__code) - finally: - fp.close() + self.__code = None def _execute(self, frames): path = self.get_file("script") - with open(path) as fp: + with open(path, encoding="utf-8") as fp: code = compile(fp.read(), path, 'exec') exec(code) # pylint: disable=exec-used diff --git a/pyoutline/outline/plugins/local.py b/pyoutline/outline/plugins/local.py index c0bc1fd17..d3fe38467 100644 --- a/pyoutline/outline/plugins/local.py +++ b/pyoutline/outline/plugins/local.py @@ -118,9 +118,9 @@ def setup_local_cores(e): def get_half_host_memory(): """Returns half of the amount of RAM the local machine has.""" - pipe = subprocess.Popen("vmstat -s", - shell=True, bufsize=1000, stdout=subprocess.PIPE).stdout - data = pipe.read().strip() - data = int(data[0:data.find(" ")]) - data = data / 2 - return str(data) + with subprocess.Popen("vmstat -s", shell=True, + bufsize=1000, stdout=subprocess.PIPE).stdout as pipe: + data = pipe.read().strip() + data = int(data[0:data.find(" ")]) + data = data / 2 + return str(data) diff --git a/pyoutline/outline/plugins/manager.py b/pyoutline/outline/plugins/manager.py index 40b0c4e78..56373e9fe 100644 --- a/pyoutline/outline/plugins/manager.py +++ b/pyoutline/outline/plugins/manager.py @@ -57,7 +57,7 @@ def load_plugin(cls, module_name): [module_name]) try: module.loaded() - except AttributeError as e: + except AttributeError: pass cls.registered_plugins.append(module) @@ -72,7 +72,7 @@ def init_plugin(cls, module_name, layer): plugin = __import__(module_name, globals(), locals(), [module_name]) try: plugin.init(layer) - except AttributeError as e: + except AttributeError: pass except ImportError as e: sys.stderr.write("Warning: plugin load failed: %s\n" % e) diff --git a/pyoutline/outline/session.py b/pyoutline/outline/session.py index ad8df3f64..30becf344 100644 --- a/pyoutline/outline/session.py +++ b/pyoutline/outline/session.py @@ -140,7 +140,7 @@ def __load_session(self): raise outline.exception.SessionException(msg % session) logger.info("loading session: %s", session) - with open(session) as file_object: + with open(session, encoding='utf-8') as file_object: try: data = yaml.load(file_object, Loader=yaml.FullLoader) except Exception as exp: @@ -163,11 +163,8 @@ def get_name(self): def save(self): """Save the current session file.""" - fp = open("%s/session" % self.get_path(), "w") - try: + with open(f"{self.get_path()}/session", "w", encoding="utf-8") as fp: fp.write(self.__name) - finally: - fp.close() @staticmethod def __layer_name(layer): @@ -323,11 +320,8 @@ def put_data(self, name, data, layer=None, force=False): raise outline.exception.SessionException("There is already data being \ stored under this name.") - fp = open(path, "w") - try: + with open(path, "w", encoding="utf-8") as fp: fp.write(yaml.dump(data)) - finally: - fp.close() def get_data(self, name, layer=None): """ @@ -349,7 +343,7 @@ def get_data(self, name, layer=None): stored under that name.") logger.debug("opening data path for %s : %s", name, path) - with open(path) as file_object: + with open(path, encoding='utf-8') as file_object: try: return yaml.load(file_object, Loader=yaml.FullLoader) except Exception as exp: diff --git a/pyoutline/outline/versions/session.py b/pyoutline/outline/versions/session.py index 2d68c4bed..4f41e2ffb 100644 --- a/pyoutline/outline/versions/session.py +++ b/pyoutline/outline/versions/session.py @@ -181,6 +181,7 @@ def __link_version(src, dst): # pylint: disable=broad-except,import-outside-toplevel @staticmethod def __run_manifest(path): + # pylint: disable=deprecated-module import imp try: fob, path, desc = imp.find_module('manifest', [path]) diff --git a/pyoutline/tests/json_test.py b/pyoutline/tests/json_test.py index 551beec8a..afd197bfa 100644 --- a/pyoutline/tests/json_test.py +++ b/pyoutline/tests/json_test.py @@ -66,7 +66,7 @@ def testJson(self): @mock.patch.dict(os.environ, {}, clear=True) def testJsonFile(self, systemMock): """Load JSON from a file""" - with open(os.path.join(JSON_DIR, 'shell.outline')) as fp: + with open(os.path.join(JSON_DIR, 'shell.outline'), encoding='utf-8') as fp: ol = outline.load_json(fp.read()) with test_utils.TemporarySessionDirectory(): ol.setup() @@ -79,7 +79,7 @@ def testJsonFile(self, systemMock): def testFacility(self): """Test facility from JSON""" - with open(os.path.join(JSON_DIR, 'facility.json')) as fp: + with open(os.path.join(JSON_DIR, 'facility.json'), encoding='utf-8') as fp: ol = outline.load_json(fp.read()) self.assertEqual('test_facility', ol.get_facility()) diff --git a/pyoutline/tests/layer_test.py b/pyoutline/tests/layer_test.py index 686233cbc..df4bf1f53 100644 --- a/pyoutline/tests/layer_test.py +++ b/pyoutline/tests/layer_test.py @@ -202,23 +202,23 @@ def test_invalid_type_args(self): strArgName = 'some-str-arg' self.layer.require_arg(strArgName, str) self.assertRaises( - outline.LayerException, self.layer.set_arg, strArgName, dict()) + outline.LayerException, self.layer.set_arg, strArgName, {}) self.layer.set_arg(strArgName, 'py3-string') else: strArgName = 'some-str-arg' self.layer.require_arg(strArgName, str) self.assertRaises( - outline.LayerException, self.layer.set_arg, strArgName, dict()) + outline.LayerException, self.layer.set_arg, strArgName, {}) self.layer.set_arg(strArgName, 'standard-py2-string') - self.layer.set_arg(strArgName, u'py2-unicode') + self.layer.set_arg(strArgName, 'py2-unicode') self.layer.set_arg(strArgName, future.types.newstr('py3-string-backport')) newstrArgName = 'some-newstr-arg' self.layer.require_arg(newstrArgName, future.types.newstr) self.assertRaises( - outline.LayerException, self.layer.set_arg, newstrArgName, dict()) + outline.LayerException, self.layer.set_arg, newstrArgName, {}) self.layer.set_arg(newstrArgName, 'standard-py2-string') - self.layer.set_arg(newstrArgName, u'py2-unicode') + self.layer.set_arg(newstrArgName, 'py2-unicode') self.layer.set_arg(newstrArgName, future.types.newstr('py3-string-backport')) def test_require_arg(self): @@ -366,7 +366,8 @@ def test_should_fail_on_invalid_child(self): def test_should_add_event_listener(self): event_type = 'arbitrary-event-type' - callback = lambda x: x + def callback(x): + return x self.layer.add_event_listener(event_type, callback) diff --git a/pyoutline/tests/modules/shell_test.py b/pyoutline/tests/modules/shell_test.py index 0efea1680..acaed5822 100644 --- a/pyoutline/tests/modules/shell_test.py +++ b/pyoutline/tests/modules/shell_test.py @@ -84,7 +84,7 @@ def testShellScript(self, systemMock): scriptContents = '# !/bin/sh\necho zoom zoom zoom' - with open(scriptFile.name, 'w') as fp: + with open(scriptFile.name, 'w', encoding='utf-8') as fp: fp.write(scriptContents) outln = outline.Outline() @@ -97,7 +97,7 @@ def testShellScript(self, systemMock): shellScript._setup() shellScript._execute(FrameSet('5-6')) - with open(expectedSessionPath) as fp: + with open(expectedSessionPath, encoding='utf-8') as fp: sessionScriptContents = fp.read() self.assertEqual(scriptContents, sessionScriptContents) diff --git a/requirements.txt b/requirements.txt index 4363e51c1..7d21fa252 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,13 @@ 2to3==1.0 enum34==1.1.6 -evdev==1.4.0;python_version<"3.0" and "linux" in sys_platform future==1.0.0 -futures==3.2.0;python_version<"3.0" -grpcio==1.26.0;python_version<"3.0" -grpcio-tools==1.26.0;python_version<"3.0" -grpcio==1.53.2;python_version>="3.0" -grpcio-tools==1.47.0;python_version>="3.0" +grpcio==1.53.2 +grpcio-tools==1.53.2 mock==2.0.0 packaging==20.9 -pathlib==1.0.1;python_version<"3.4" -protobuf==3.17.3;python_version<"3.0" -psutil==5.6.7 -pyfakefs==3.6;python_version<"3.7" -pyfakefs==5.2.3;python_version>="3.7" -pylint==2.6.0;python_version>="3.7" +psutil==5.9.8 +pyfakefs==5.2.3 +pylint==2.15.10 pynput==1.7.6 PyYAML==5.1 six==1.16.0 diff --git a/requirements_gui.txt b/requirements_gui.txt index 1f5b19637..eb5e544d4 100644 --- a/requirements_gui.txt +++ b/requirements_gui.txt @@ -1,5 +1,4 @@ PySide6==6.7.1;python_version>"3.11" PySide6==6.5.3;python_version=="3.11" PySide2==5.15.2.1;python_version<="3.10" -QtPy==1.11.3;python_version<"3.7" -QtPy==2.4.1;python_version>="3.7" +QtPy==2.4.1 diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index 6f93765b3..07c10f379 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -125,6 +125,7 @@ def __createEnvVariables(self): if 'GPU_LIST' in self.runFrame.attributes: self.frameEnv['CUE_GPU_CORES'] = self.runFrame.attributes['GPU_LIST'] + # pylint: disable=inconsistent-return-statements def _createCommandFile(self, command): """Creates a file that subprocess. Popen then executes. @type command: string @@ -151,10 +152,10 @@ def _createCommandFile(self, command): else: commandFile = os.path.join(tempfile.gettempdir(), 'rqd-cmd-%s-%s' % (self.runFrame.frame_id, time.time())) - rqexe = open(commandFile, "w") - self._tempLocations.append(commandFile) - rqexe.write(command) - rqexe.close() + with open(commandFile, "w", encoding='utf-8') as rqexe: + self._tempLocations.append(commandFile) + rqexe.write(command) + rqexe.close() os.chmod(commandFile, 0o777) return commandFile # pylint: disable=broad-except @@ -318,7 +319,7 @@ def runLinux(self): else: tempCommand += [self._createCommandFile(runFrame.command)] - # pylint: disable=subprocess-popen-preexec-fn + # pylint: disable=subprocess-popen-preexec-fn,consider-using-with frameInfo.forkedCommand = subprocess.Popen(tempCommand, env=self.frameEnv, cwd=self.rqCore.machine.getTempPath(), @@ -340,14 +341,11 @@ def runLinux(self): if rqd.rqconstants.RQD_PREPEND_TIMESTAMP: pipe_to_file(frameInfo.forkedCommand.stdout, frameInfo.forkedCommand.stderr, self.rqlog) else: - with open(self.rqlog.name, 'a') as f: - # Convert to ASCII while discarding characters that can not be encoded - for line in frameInfo.forkedCommand.stdout: - line = line.encode('ascii', 'ignore') - f.write(line.decode('ascii') + '\n') - for line in frameInfo.forkedCommand.stderr: - line = line.encode('ascii', 'ignore') - f.write(line.decode('ascii') + '\n') + # Convert to ASCII while discarding characters that can not be encoded + for line in frameInfo.forkedCommand.stdout: + self.rqlog.write(line.decode('ascii', errors='ignore')) + for line in frameInfo.forkedCommand.stderr: + self.rqlog.write(line.decode('ascii', errors='ignore')) returncode = frameInfo.forkedCommand.wait() @@ -361,11 +359,11 @@ def runLinux(self): frameInfo.exitSignal = 0 try: - statFile = open(tempStatFile,"r") - frameInfo.realtime = statFile.readline().split()[1] - frameInfo.utime = statFile.readline().split()[1] - frameInfo.stime = statFile.readline().split()[1] - statFile.close() + with open(tempStatFile, "r", encoding='utf-8') as statFile: + frameInfo.realtime = statFile.readline().split()[1] + frameInfo.utime = statFile.readline().split()[1] + frameInfo.stime = statFile.readline().split()[1] + statFile.close() # pylint: disable=broad-except except Exception: pass # This happens when frames are killed @@ -385,6 +383,7 @@ def runWindows(self): runFrame.command = runFrame.command.replace('%{frame}', self.frameEnv['CUE_IFRAME']) tempCommand = [self._createCommandFile(runFrame.command)] + # pylint: disable=consider-using-with frameInfo.forkedCommand = subprocess.Popen(tempCommand, env=self.frameEnv, stdin=subprocess.PIPE, @@ -433,7 +432,7 @@ def runDarwin(self): tempCommand = ["/usr/bin/su", frameInfo.runFrame.user_name, "-c", '"' + self._createCommandFile(frameInfo.runFrame.command) + '"'] - # pylint: disable=subprocess-popen-preexec-fn + # pylint: disable=subprocess-popen-preexec-fn,consider-using-with frameInfo.forkedCommand = subprocess.Popen(tempCommand, env=self.frameEnv, cwd=self.rqCore.machine.getTempPath(), @@ -549,7 +548,8 @@ def run(self): else: raise RuntimeError(err) try: - self.rqlog = open(runFrame.log_dir_file, "w+", 1) + # pylint: disable=consider-using-with + self.rqlog = open(runFrame.log_dir_file, "w+", 1, encoding='utf-8') self.waitForFile(runFrame.log_dir_file) # pylint: disable=broad-except except Exception as e: @@ -638,6 +638,7 @@ def __init__(self, optNimbyoff=False): self.intervalStartTime = None self.intervalSleepTime = rqd.rqconstants.RQD_MIN_PING_INTERVAL_SEC + # pylint: disable=unused-private-member self.__cluster = None self.__session = None self.__stmt = None @@ -745,21 +746,17 @@ def storeFrame(self, frameId, runningFrame): @param frameId: A frame's unique Id @type runningFrame: rqd.rqnetwork.RunningFrame @param runningFrame: rqd.rqnetwork.RunningFrame object""" - self.__threadLock.acquire() - try: + with self.__threadLock: if frameId in self.__cache: raise rqd.rqexceptions.RqdException( "frameId " + frameId + " is already running on this machine") self.__cache[frameId] = runningFrame - finally: - self.__threadLock.release() def deleteFrame(self, frameId): """Deletes a frame from the cache @type frameId: string @param frameId: A frame's unique Id""" - self.__threadLock.acquire() - try: + with self.__threadLock: if frameId in self.__cache: del self.__cache[frameId] # pylint: disable=no-member @@ -770,8 +767,6 @@ def deleteFrame(self, frameId): self.cores.reserved_cores) # pylint: disable=no-member self.cores.reserved_cores.clear() - finally: - self.__threadLock.release() def killAllFrame(self, reason): """Will execute .kill() on every frame in cache until no frames remain @@ -805,8 +800,7 @@ def releaseCores(self, reqRelease, releaseHT=None, releaseGpus=None): """The requested number of cores are released @type reqRelease: int @param reqRelease: Number of cores to release, 100 = 1 physical core""" - self.__threadLock.acquire() - try: + with self.__threadLock: # pylint: disable=no-member self.cores.booked_cores -= reqRelease maxRelease = (self.cores.total_cores - @@ -824,9 +818,6 @@ def releaseCores(self, reqRelease, releaseHT=None, releaseGpus=None): if releaseGpus: self.machine.releaseGpus(releaseGpus) - finally: - self.__threadLock.release() - # pylint: disable=no-member if self.cores.idle_cores > self.cores.total_cores: log.critical( @@ -916,8 +907,7 @@ def launchFrame(self, runFrame): raise rqd.rqexceptions.CoreReservationFailureException(err) # See if all requested cores are available - self.__threadLock.acquire() - try: + with self.__threadLock: # pylint: disable=no-member if self.cores.idle_cores < runFrame.num_cores: err = "Not launching, insufficient idle cores" @@ -940,8 +930,6 @@ def launchFrame(self, runFrame): self.cores.idle_cores -= runFrame.num_cores self.cores.booked_cores += runFrame.num_cores # pylint: enable=no-member - finally: - self.__threadLock.release() runningFrame = rqd.rqnetwork.RunningFrame(self, runFrame) runningFrame.frameAttendantThread = FrameAttendantThread(self, runFrame, runningFrame) @@ -1062,8 +1050,7 @@ def lock(self, reqLock): @type reqLock: int @param reqLock: Number of cores to lock, 100 = 1 physical core""" sendUpdate = False - self.__threadLock.acquire() - try: + with self.__threadLock: # pylint: disable=no-member numLock = min(self.cores.total_cores - self.cores.locked_cores, reqLock) @@ -1072,8 +1059,6 @@ def lock(self, reqLock): self.cores.idle_cores -= min(numLock, self.cores.idle_cores) sendUpdate = True # pylint: enable=no-member - finally: - self.__threadLock.release() log.debug(self.cores) @@ -1084,16 +1069,13 @@ def lockAll(self): """"Locks all cores on the machine. If a locked status changes, a status report is sent.""" sendUpdate = False - self.__threadLock.acquire() - try: + with self.__threadLock: # pylint: disable=no-member if self.cores.locked_cores < self.cores.total_cores: self.cores.locked_cores = self.cores.total_cores self.cores.idle_cores = 0 sendUpdate = True # pylint: enable=no-member - finally: - self.__threadLock.release() log.debug(self.cores) @@ -1118,8 +1100,7 @@ def unlock(self, reqUnlock): self.__respawn = False self.machine.state = rqd.compiled_proto.host_pb2.UP - self.__threadLock.acquire() - try: + with self.__threadLock: # pylint: disable=no-member numUnlock = min(self.cores.locked_cores, reqUnlock) if numUnlock > 0: @@ -1127,8 +1108,6 @@ def unlock(self, reqUnlock): self.cores.idle_cores += numUnlock sendUpdate = True # pylint: enable=no-member - finally: - self.__threadLock.release() log.debug(self.cores) @@ -1151,8 +1130,7 @@ def unlockAll(self): self.__respawn = False self.machine.state = rqd.compiled_proto.host_pb2.UP - self.__threadLock.acquire() - try: + with self.__threadLock: # pylint: disable=no-member if self.cores.locked_cores > 0: if not self.nimby.locked: @@ -1160,8 +1138,6 @@ def unlockAll(self): self.cores.locked_cores = 0 sendUpdate = True # pylint: enable=no-member - finally: - self.__threadLock.release() log.debug(self.cores) diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 65061ef82..87c7a5538 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -133,7 +133,6 @@ def isNimbySafeToUnlock(self): return False return True - # pylint: disable=no-self-use @rqd.rqutil.Memoize def isDesktop(self): """Returns True if machine starts in run level 5 (X11) @@ -141,10 +140,10 @@ def isDesktop(self): if rqd.rqconstants.OVERRIDE_IS_DESKTOP: return True if platform.system() == "Linux" and os.path.exists(rqd.rqconstants.PATH_INITTAB): - inittabFile = open(rqd.rqconstants.PATH_INITTAB, "r") - for line in inittabFile: - if line.startswith("id:5:initdefault:"): - return True + with open(rqd.rqconstants.PATH_INITTAB, "r", encoding='utf-8') as inittabFile: + for line in inittabFile: + if line.startswith("id:5:initdefault:"): + return True if os.path.islink(rqd.rqconstants.PATH_INIT_TARGET): if os.path.realpath(rqd.rqconstants.PATH_INIT_TARGET).endswith('graphical.target'): return True @@ -215,7 +214,7 @@ def __updateGpuAndLlu(self, frame): frame.lluTime = int(stat) def _getStatFields(self, pidFilePath): - with open(pidFilePath, "r") as statFile: + with open(pidFilePath, "r", encoding='utf-8') as statFile: stats = statFile.read().split() stats[1] = stats[1].strip('()') return stats @@ -399,23 +398,23 @@ def getLoadAvg(self): """Returns average number of processes waiting to be served for the last 1 minute multiplied by 100.""" if platform.system() == "Linux": - loadAvgFile = open(rqd.rqconstants.PATH_LOADAVG, "r") - loadAvg = int(float(loadAvgFile.read().split()[0]) * 100) - if self.__enabledHT(): - loadAvg = loadAvg // self.__getHyperthreadingMultiplier() - loadAvg = loadAvg + rqd.rqconstants.LOAD_MODIFIER - loadAvg = max(loadAvg, 0) - return loadAvg + with open(rqd.rqconstants.PATH_LOADAVG, "r", encoding='utf-8') as loadAvgFile: + loadAvg = int(float(loadAvgFile.read().split()[0]) * 100) + if self.__enabledHT(): + loadAvg = loadAvg // self.__getHyperthreadingMultiplier() + loadAvg = loadAvg + rqd.rqconstants.LOAD_MODIFIER + loadAvg = max(loadAvg, 0) + return loadAvg return 0 @rqd.rqutil.Memoize def getBootTime(self): """Returns epoch when the system last booted""" if platform.system() == "Linux": - statFile = open(rqd.rqconstants.PATH_STAT, "r") - for line in statFile: - if line.startswith("btime"): - return int(line.split()[1]) + with open(rqd.rqconstants.PATH_STAT, "r", encoding='utf-8') as statFile: + for line in statFile: + if line.startswith("btime"): + return int(line.split()[1]) return 0 @rqd.rqutil.Memoize @@ -528,7 +527,8 @@ def reboot(self): """Reboots the machine immediately""" if platform.system() == "Linux": log.warning("Rebooting machine") - subprocess.Popen(['/usr/bin/sudo','/sbin/reboot', '-f']) + # pylint: disable=consider-using-with + subprocess.Popen(['/usr/bin/sudo', '/sbin/reboot', '-f']) # pylint: disable=no-member def __initMachineTags(self): @@ -578,7 +578,8 @@ def __initMachineStats(self, pathCpuInfo=None): self.__physid_and_coreid_by_proc = {} # Reads static information from /proc/cpuinfo - with open(pathCpuInfo or rqd.rqconstants.PATH_CPUINFO, "r") as cpuinfoFile: + with open(pathCpuInfo or rqd.rqconstants.PATH_CPUINFO, "r", + encoding='utf-8') as cpuinfoFile: currCore = {} procsFound = [] for line in cpuinfoFile: @@ -728,7 +729,7 @@ def updateMachineStats(self): self.__renderHost.free_mcp = (mcpStat.f_bavail * mcpStat.f_bsize) // KILOBYTE # Reads dynamic information from /proc/meminfo - with open(rqd.rqconstants.PATH_MEMINFO, "r") as fp: + with open(rqd.rqconstants.PATH_MEMINFO, "r", encoding='utf-8') as fp: for line in fp: if line.startswith("MemFree"): freeMem = int(line.split()[1]) diff --git a/rqd/rqd/rqnetwork.py b/rqd/rqd/rqnetwork.py index 6bb26cfce..fa1fb8060 100644 --- a/rqd/rqd/rqnetwork.py +++ b/rqd/rqd/rqnetwork.py @@ -158,6 +158,7 @@ def kill(self, message=""): rqd.rqutil.permissionsHigh() try: if platform.system() == "Windows": + # pylint: disable=consider-using-with subprocess.Popen('taskkill /F /T /PID %i' % self.pid, shell=True) else: os.killpg(self.pid, rqd.rqconstants.KILL_SIGNAL) @@ -373,6 +374,7 @@ def __init__(self, self._sleeping_policy = sleeping_policy self._retry_statuses = status_for_retry + # pylint: disable=inconsistent-return-statements def _intercept_call(self, continuation, client_call_details, request_or_iterator): for attempt in range(self._max_attempts): diff --git a/rqd/rqd/rqnimby.py b/rqd/rqd/rqnimby.py index 4784ba90f..04f1698da 100644 --- a/rqd/rqd/rqnimby.py +++ b/rqd/rqd/rqnimby.py @@ -241,7 +241,9 @@ def openEvents(self): for device in os.listdir("/dev/input/"): if device.startswith("event") or device.startswith("mice"): try: - self.fileObjList.append(open("/dev/input/%s" % device, "rb")) + # pylint: disable=consider-using-with + self.fileObjList.append(open("/dev/input/%s" % device, "rb", + encoding='utf-8')) except IOError: # Bad device found log.exception("IOError: Failed to open /dev/input/%s", device) diff --git a/rqd/rqd/rqswap.py b/rqd/rqd/rqswap.py index 733750702..ae5021e8a 100644 --- a/rqd/rqd/rqswap.py +++ b/rqd/rqd/rqswap.py @@ -115,7 +115,7 @@ def __init__(self): self.__repeatedTimer = RepeatedTimer( self.__interval, self.__getPgoutNum) - self.__repeatedTimer.setDaemon(True) + self.__repeatedTimer.daemon = True self.__repeatedTimer.start() def __getSampleDataCopy(self): @@ -130,7 +130,7 @@ def __getPgoutNum(self): foundPgpgout = False pgpgoutNum = 0 try: - with open("/proc/vmstat") as vmStatFile: + with open("/proc/vmstat", encoding='utf-8') as vmStatFile: for line in vmStatFile.readlines(): matchObj = PGPGOUT_RE.match(line) if matchObj: diff --git a/rqd/rqd/rqutil.py b/rqd/rqd/rqutil.py index da627a2d5..e9e95fe2a 100644 --- a/rqd/rqd/rqutil.py +++ b/rqd/rqd/rqutil.py @@ -79,14 +79,14 @@ def permissionsHigh(): """Sets the effective gid/uid to processes original values (root)""" if platform.system() == "Windows" or not rqd.rqconstants.RQD_BECOME_JOB_USER: return - PERMISSIONS.acquire() - os.setegid(os.getgid()) - os.seteuid(os.getuid()) - try: - os.setgroups(HIGH_PERMISSION_GROUPS) - # pylint: disable=broad-except - except Exception: - pass + with PERMISSIONS: + os.setegid(os.getgid()) + os.seteuid(os.getuid()) + try: + os.setgroups(HIGH_PERMISSION_GROUPS) + # pylint: disable=broad-except + except Exception: + pass def permissionsLow(): @@ -107,17 +107,17 @@ def permissionsUser(uid, gid): """Sets the effective gid/uid to supplied values""" if platform.system() in ('Windows', 'Darwin') or not rqd.rqconstants.RQD_BECOME_JOB_USER: return - PERMISSIONS.acquire() - __becomeRoot() - try: - username = pwd.getpwuid(uid).pw_name - groups = [20] + [g.gr_gid for g in grp.getgrall() if username in g.gr_mem] - os.setgroups(groups) - # pylint: disable=broad-except - except Exception: - pass - os.setegid(gid) - os.seteuid(uid) + with PERMISSIONS: + __becomeRoot() + try: + username = pwd.getpwuid(uid).pw_name + groups = [20] + [g.gr_gid for g in grp.getgrall() if username in g.gr_mem] + os.setgroups(groups) + # pylint: disable=broad-except + except Exception: + pass + os.setegid(gid) + os.seteuid(uid) def __becomeRoot(): diff --git a/rqd/tests/rqconstants_tests.py b/rqd/tests/rqconstants_tests.py index 46da55f39..45e52c0b1 100644 --- a/rqd/tests/rqconstants_tests.py +++ b/rqd/tests/rqconstants_tests.py @@ -58,7 +58,7 @@ def __init__(self, tempdir, content): config = os.path.join(tempdir, str(uuid.uuid4())) self.patcher = mock.patch("sys.argv", ["rqd", "-c", config]) - with open(config, "w") as f: + with open(config, "w", encoding='utf-8') as f: print(content, file=f) def __enter__(self):