From 6fafd62947f4d63951353e48478882ef18b6a619 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Wed, 13 Jul 2022 12:22:46 -0700 Subject: [PATCH 01/21] [rqd] Core affinity for cache optimization (#1171) When possible, try to book frames from the same Layer on the same core to leverage shared cache --- VERSION.in | 2 +- .../spcue/dao/postgres/ProcDaoJdbc.java | 2 + cuegui/cuegui/ProcChildren.py | 2 +- proto/report.proto | 7 + pycue/opencue/wrappers/host.py | 2 +- rqd/rqd/rqcore.py | 9 + rqd/rqd/rqmachine.py | 191 +++++++++++------- rqd/tests/rqmachine_tests.py | 96 ++++++++- 8 files changed, 228 insertions(+), 83 deletions(-) diff --git a/VERSION.in b/VERSION.in index a4d2aceeb..caa4836d8 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.18 +0.19 diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java index b68f8a74c..32c454e0f 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java @@ -29,6 +29,8 @@ import java.util.List; import java.util.Map; +import com.imageworks.spcue.dispatcher.AbstractDispatcher; +import org.apache.log4j.Logger; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.RowMapper; import org.springframework.jdbc.core.support.JdbcDaoSupport; diff --git a/cuegui/cuegui/ProcChildren.py b/cuegui/cuegui/ProcChildren.py index 7a4190e88..5faa0415e 100644 --- a/cuegui/cuegui/ProcChildren.py +++ b/cuegui/cuegui/ProcChildren.py @@ -75,7 +75,7 @@ def update(self): try: procs = opencue.api.getProcs(job=[self._job.name()], - layer=[self._layer.name()], + layer=[x.name() for x in self._job.getLayers()], host=self._hosts) for proc in procs: data['children_processes'] =\ diff --git a/proto/report.proto b/proto/report.proto index 78c8e4da6..441f810c3 100644 --- a/proto/report.proto +++ b/proto/report.proto @@ -36,6 +36,13 @@ message CoreDetail { int32 idle_cores = 2; int32 locked_cores = 3; int32 booked_cores = 4; + //map + map reserved_cores = 5; +} + +message CoreId { + repeated int64 coreid = 1; + } message FrameCompleteReport { diff --git a/pycue/opencue/wrappers/host.py b/pycue/opencue/wrappers/host.py index 83dac252f..aa54d44d0 100644 --- a/pycue/opencue/wrappers/host.py +++ b/pycue/opencue/wrappers/host.py @@ -279,7 +279,7 @@ def coresReserved(self): :rtype: float :return: number of cores reserved """ - return self.data.cores - self.data.idle_ores + return self.data.cores - self.data.idle_cores def coresIdle(self): """Returns the number of cores the host currently has idel. diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index 48ef7ccc9..cb74f9d34 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -593,6 +593,7 @@ def __init__(self, optNimbyoff=False): idle_cores=0, locked_cores=0, booked_cores=0, + reserved_cores=[], ) self.nimby = rqd.rqnimby.NimbyFactory.getNimby(self) @@ -732,6 +733,14 @@ def deleteFrame(self, frameId): try: if frameId in self.__cache: del self.__cache[frameId] + # pylint: disable=no-member + if not self.__cache and self.cores.reserved_cores: + # pylint: disable=no-member + log.error( + 'No running frames but reserved_cores is not empty: %s', + self.cores.reserved_cores) + # pylint: disable=no-member + self.cores.reserved_cores.clear() finally: self.__threadLock.release() diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index bed78fdef..37e1f5771 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -79,9 +79,16 @@ def __init__(self, rqCore, coreInfo): """ self.__rqCore = rqCore self.__coreInfo = coreInfo - self.__tasksets = set() self.__gpusets = set() + # A dictionary built from /proc/cpuinfo containing + # { : { : set([, , ...]), ... }, ... } + self.__procs_by_physid_and_coreid = {} + + # A reverse mapping of the above. + # { : (, ), ... } + self.__physid_and_coreid_by_proc = {} + if platform.system() == 'Linux': self.__vmstat = rqd.rqswap.VmStat() @@ -103,8 +110,8 @@ def __init__(self, rqCore, coreInfo): self.__pidHistory = {} - self.setupHT() self.setupGpu() + self.setupTaskset() def isNimbySafeToRunJobs(self): """Returns False if nimby should be triggered due to resource limits""" @@ -207,17 +214,9 @@ def __updateGpuAndLlu(self, frame): stat = os.stat(frame.runFrame.log_dir_file).st_mtime frame.lluTime = int(stat) - def _getFields(self, pidFilePath): - fields = [] - - try: - with open(pidFilePath, "r") as statFile: - fields = statFile.read().split() - # pylint: disable=broad-except - except Exception: - log.warning("Not able to read pidFilePath: %s", pidFilePath) - - return fields + def _getStatFields(self, pidFilePath): + with open(pidFilePath, "r") as statFile: + return [None, None] + statFile.read().rsplit(")", 1)[-1].split() def rssUpdate(self, frames): """Updates the rss and maxrss for all running frames""" @@ -247,10 +246,8 @@ def rssUpdate(self, frames): for pid in os.listdir("/proc"): if pid.isdigit(): try: - with open(rqd.rqconstants.PATH_PROC_PID_STAT - .format(pid), "r") as statFile: - statFields = [None, None] + statFile.read().rsplit(")", 1)[-1].split() - + statFields = self._getStatFields(rqd.rqconstants.PATH_PROC_PID_STAT + .format(pid)) pids[pid] = { "name": statFields[1], "state": statFields[2], @@ -272,24 +269,21 @@ def rssUpdate(self, frames): p = psutil.Process(int(pid)) pids[pid]["cmd_line"] = p.cmdline() - try: - # 2. Collect Statm file: /proc/[pid]/statm (same as status vsize in kb) - # - size: "total program size" - # - rss: inaccurate, similar to VmRss in /proc/[pid]/status - child_statm_fields = self._getFields( - rqd.rqconstants.PATH_PROC_PID_STATM.format(pid)) - pids[pid]['statm_size'] = \ - int(re.search("\\d+", child_statm_fields[0]).group()) \ - if re.search("\\d+", child_statm_fields[0]) else -1 - pids[pid]['statm_rss'] = \ - int(re.search("\\d+", child_statm_fields[1]).group()) \ - if re.search("\\d+", child_statm_fields[1]) else -1 - except rqd.rqexceptions.RqdException as e: - log.warning("Failed to read statm file: %s", e) + # 2. Collect Statm file: /proc/[pid]/statm (same as status vsize in kb) + # - size: "total program size" + # - rss: inaccurate, similar to VmRss in /proc/[pid]/status + child_statm_fields = self._getStatFields( + rqd.rqconstants.PATH_PROC_PID_STATM.format(pid)) + pids[pid]['statm_size'] = \ + int(re.search(r"\d+", child_statm_fields[0]).group()) \ + if re.search(r"\d+", child_statm_fields[0]) else -1 + pids[pid]['statm_rss'] = \ + int(re.search(r"\d+", child_statm_fields[1]).group()) \ + if re.search(r"\d+", child_statm_fields[1]) else -1 # pylint: disable=broad-except - except rqd.rqexceptions.RqdException: - log.exception('Failed to read stat file for pid %s', pid) + except (OSError, IOError): + log.exception('Failed to read stat/statm file for pid %s', pid) # pylint: disable=too-many-nested-blocks try: @@ -298,10 +292,8 @@ def rssUpdate(self, frames): bootTime = self.getBootTime() values = list(frames.values()) - for frame in values: if frame.pid > 0: - session = str(frame.pid) rss = 0 vsize = 0 @@ -573,33 +565,50 @@ def __initMachineStats(self, pathCpuInfo=None): mcpStat = os.statvfs(self.getTempPath()) self.__renderHost.total_mcp = mcpStat.f_blocks * mcpStat.f_frsize // KILOBYTE + # Reset mappings + self.__procs_by_physid_and_coreid = {} + self.__physid_and_coreid_by_proc = {} + # Reads static information from /proc/cpuinfo with open(pathCpuInfo or rqd.rqconstants.PATH_CPUINFO, "r") as cpuinfoFile: - singleCore = {} + currCore = {} procsFound = [] for line in cpuinfoFile: - lineList = line.strip().replace("\t","").split(": ") + lineList = line.strip().replace("\t", "").split(": ") # A normal entry added to the singleCore dictionary if len(lineList) >= 2: - singleCore[lineList[0]] = lineList[1] + currCore[lineList[0]] = lineList[1] # The end of a processor block elif lineList == ['']: # Check for hyper-threading - hyperthreadingMultiplier = (int(singleCore.get('siblings', '1')) - // int(singleCore.get('cpu cores', '1'))) + hyperthreadingMultiplier = (int(currCore.get('siblings', '1')) + // int(currCore.get('cpu cores', '1'))) __totalCores += rqd.rqconstants.CORE_VALUE - if "core id" in singleCore \ - and "physical id" in singleCore \ - and not singleCore["physical id"] in procsFound: - procsFound.append(singleCore["physical id"]) + if "core id" in currCore \ + and "physical id" in currCore \ + and not currCore["physical id"] in procsFound: + procsFound.append(currCore["physical id"]) __numProcs += 1 - elif "core id" not in singleCore: + elif "core id" not in currCore: __numProcs += 1 - singleCore = {} + + if 'physical id' in currCore and 'core id' in currCore: + # Keep track of what processors are on which core on + # which physical socket. + procid, physid, coreid = ( + currCore['processor'], + currCore['physical id'], + currCore['core id']) + self.__procs_by_physid_and_coreid \ + .setdefault(physid, {}) \ + .setdefault(coreid, set()).add(procid) + self.__physid_and_coreid_by_proc[procid] = physid, coreid + currCore = {} + # An entry without data elif len(lineList) == 1: - singleCore[lineList[0]] = "" + currCore[lineList[0]] = "" else: hyperthreadingMultiplier = 1 @@ -782,50 +791,76 @@ def __enabledHT(self): def __getHyperthreadingMultiplier(self): return int(self.__renderHost.attributes['hyperthreadingMultiplier']) - def setupHT(self): + def setupTaskset(self): """ Setup rqd for hyper-threading """ - - if self.__enabledHT(): - self.__tasksets = set(range(self.__coreInfo.total_cores // 100)) + self.__coreInfo.reserved_cores.clear() def setupGpu(self): """ Setup rqd for Gpus """ self.__gpusets = set(range(self.getGpuCount())) - def reserveHT(self, reservedCores): + def reserveHT(self, frameCores): """ Reserve cores for use by taskset taskset -c 0,1,8,9 COMMAND Not thread save, use with locking. - @type reservedCores: int - @param reservedCores: The total physical cores reserved by the frame. + @type frameCores: int + @param frameCores: The total physical cores reserved by the frame. @rtype: string @return: The cpu-list for taskset -c """ - if not self.__enabledHT(): - return None - - if reservedCores % 100: - log.debug('Taskset: Can not reserveHT with fractional cores') + if frameCores % 100: + log.warning('Taskset: Can not reserveHT with fractional cores') return None + log.warning('Taskset: Requesting reserve of %d', (frameCores // 100)) + + # Look for the most idle physical cpu. + # Prefer to assign cores from the same physical cpu. + # Spread different frames around on different physical cpus. + avail_cores = {} + avail_cores_count = 0 + reserved_cores = self.__coreInfo.reserved_cores + + for physid, cores in self.__procs_by_physid_and_coreid.items(): + for coreid in cores.keys(): + if int(physid) in reserved_cores and \ + int(coreid) in reserved_cores[int(physid)].coreid: + continue + avail_cores.setdefault(physid, set()).add(coreid) + avail_cores_count += 1 - log.debug('Taskset: Requesting reserve of %d', (reservedCores // 100)) + remaining_cores = frameCores / 100 - if len(self.__tasksets) < reservedCores // 100: + if avail_cores_count < remaining_cores: err = ('Not launching, insufficient hyperthreading cores to reserve ' - 'based on reservedCores') + 'based on frameCores (%s < %s)') \ + % (avail_cores_count, remaining_cores) log.critical(err) raise rqd.rqexceptions.CoreReservationFailureException(err) - hyperthreadingMultiplier = self.__getHyperthreadingMultiplier() tasksets = [] - for _ in range(reservedCores // 100): - core = self.__tasksets.pop() - tasksets.append(str(core)) - if hyperthreadingMultiplier > 1: - tasksets.append(str(core + self.__coreInfo.total_cores // 100)) - log.debug('Taskset: Reserving cores - %s', ','.join(tasksets)) + for physid, cores in sorted( + avail_cores.items(), + # Return the physical socket that has + # the most idle cores first. + key=lambda tup: len(tup[1]), + reverse=True): + + while remaining_cores > 0 and len(cores) > 0: + coreid = cores.pop() + # Give all the hyperthreads on this core. + # This counts as one core. + reserved_cores[int(physid)].coreid.extend([int(coreid)]) + remaining_cores -= 1 + + for procid in self.__procs_by_physid_and_coreid[physid][coreid]: + tasksets.append(procid) + + if remaining_cores == 0: + break + + log.warning('Taskset: Reserving procs - %s', ','.join(tasksets)) return ','.join(tasksets) @@ -838,13 +873,21 @@ def releaseHT(self, reservedHT): @param: The cpu-list used for taskset to release. ex: '0,8,1,9' """ - if not self.__enabledHT(): - return None - log.debug('Taskset: Releasing cores - %s', reservedHT) + + # Remove these cores from the reserved set. + # Silently ignore any that weren't really reserved or + # aren't valid core identities. + reserved_cores = self.__coreInfo.reserved_cores for core in reservedHT.split(','): - if int(core) < self.__coreInfo.total_cores // 100: - self.__tasksets.add(int(core)) + physical_id_str, core_id_str = self.__physid_and_coreid_by_proc.get(core) + physical_id = int(physical_id_str) + core_id = int(core_id_str) + + if physical_id in reserved_cores and core_id in reserved_cores[physical_id].coreid: + reserved_cores[physical_id].coreid.remove(core_id) + if len(reserved_cores[physical_id].coreid) == 0: + del reserved_cores[physical_id] def reserveGpus(self, reservedGpus): """ Reserve gpus diff --git a/rqd/tests/rqmachine_tests.py b/rqd/tests/rqmachine_tests.py index 489cc35c8..1b1bdaf4a 100644 --- a/rqd/tests/rqmachine_tests.py +++ b/rqd/tests/rqmachine_tests.py @@ -160,7 +160,6 @@ PROC_PID_CMDLINE = ' sleep 20' - @mock.patch.object(rqd.rqutil.Memoize, 'isCached', new=mock.MagicMock(return_value=False)) @mock.patch('platform.system', new=mock.MagicMock(return_value='Linux')) @mock.patch('os.statvfs', new=mock.MagicMock()) @@ -447,18 +446,103 @@ def test_getBootReport(self): self.assertEqual(25699176, bootReport.host.free_mem) def test_reserveHT(self): + """ + Total 2 physical(ph) processors with 4 cores each with 2 threads each + step1 - taskset1: Reserve 3 cores (ph1) + step2 - taskset0: Reserve 4 cores (ph0) + step3 - Release cores on taskset0 + step4 - taskset3: Reserve 2 cores (ph0) + step5 - taskset4: 3 remaining, Reserve 3 cores (ph0+ph1) + step5 - taskset5: No more cores + """ cpuInfo = os.path.join(os.path.dirname(__file__), 'cpuinfo', '_cpuinfo_shark_ht_8-4-2-2') self.fs.add_real_file(cpuInfo) self.machine.testInitMachineStats(cpuInfo) - self.machine.setupHT() - tasksets = self.machine.reserveHT(300) + self.machine.setupTaskset() + + # ------------------------step1------------------------- + # phys_id 1 + # - core_id 0 + # - process_id 4 + # - process_id 12 + # - core_id 1 + # - process_id 5 + # - process_id 13 + # - core_id 3 + # - process_id 7 + # - process_id 15 + tasksets1 = self.machine.reserveHT(300) + # pylint: disable=no-member + self.assertItemsEqual(['4', '5', '7', '12', '13', '15'], sorted(tasksets1.split(','))) + + # ------------------------step2------------------------- + # phys_id 0 + # - core_id 0 + # - process_id 0 + # - process_id 8 + # - core_id 1 + # - process_id 1 + # - process_id 9 + # - core_id 2 + # - process_id 2 + # - process_id 10 + # - core_id 3 + # - process_id 3 + # - process_id 11 + tasksets0 = self.machine.reserveHT(400) + # pylint: disable=no-member + self.assertItemsEqual(['0', '1', '2', '3', '8', '9', '10', '11'], + sorted(tasksets0.split(','))) + + # reserved cores got updated properly + # pylint: disable=no-member + self.assertItemsEqual([0, 1, 2, 3], self.coreDetail.reserved_cores[0].coreid) + + # Make sure tastsets don't overlap + self.assertTrue(set(tasksets0.split(',')).isdisjoint(tasksets1.split(','))) - self.assertEqual('0,8,1,9,2,10', tasksets) + # ------------------------step3------------------------- + # Releasing a physcore shouldn't impact other physcores + self.machine.releaseHT(tasksets0) + # pylint: disable=no-member + self.assertTrue(1 in self.coreDetail.reserved_cores) + # pylint: disable=no-member + self.assertItemsEqual([0, 1, 3], self.coreDetail.reserved_cores[1].coreid) + + # ------------------------step4------------------------- + # phys_id 0 + # - core_id 0 + # - process_id 0 + # - process_id 8 + # - core_id 1 + # - process_id 1 + # - process_id 9 + tasksets3 = self.machine.reserveHT(200) + # pylint: disable=no-member + self.assertItemsEqual(['0', '1', '8', '9'], sorted(tasksets3.split(','))) + + # ------------------------step5------------------------- + # phys_id 0 + # - core_id 2 + # - process_id 2 + # - process_id 10 + # - core_id 3 + # - process_id 3 + # - process_id 11 + # phys_id 1 + # - core_id 2 + # - process_id 6 + # - process_id 14 + tasksets4 = self.machine.reserveHT(300) + # pylint: disable=no-member + self.assertItemsEqual(['2', '10', '3', '11', '6', '14'], sorted(tasksets4.split(','))) - self.machine.releaseHT(tasksets) + # ------------------------step6------------------------- + # No cores available + with self.assertRaises(rqd.rqexceptions.CoreReservationFailureException): + self.machine.reserveHT(300) - self.assertEqual({0, 1, 2, 3, 4, 5, 6, 7}, self.machine._Machine__tasksets) def test_tags(self): tags = ["test1", "test2", "test3"] From d9d24f0c9aa4a9303d45354ba40fd484b1c7f0a6 Mon Sep 17 00:00:00 2001 From: Rosa Behrens Camp Date: Wed, 13 Jul 2022 14:27:01 -0700 Subject: [PATCH 02/21] Min mem increase (#1157) Added the ability for admins to set the minimum memory increase value on a service. OpenCue increases the minimum memory requirements every time a job is retried when it fails from running out of memory but the old memory increase (hardcoded to 2G) was causing some jobs to be retried many times before it finally succeeded. --- VERSION.in | 2 +- cuebot/{build.gradle => build.gradle.orig} | 0 .../com/imageworks/spcue/ServiceEntity.java | 2 + .../spcue/dao/postgres/ServiceDaoJdbc.java | 33 ++++-- .../spcue/dao/postgres/WhiteboardDaoJdbc.java | 8 +- .../spcue/dispatcher/Dispatcher.java | 4 + .../dispatcher/FrameCompleteHandler.java | 64 +++++++++- .../spcue/servant/ManageService.java | 2 + .../spcue/servant/ManageServiceOverride.java | 1 + .../imageworks/spcue/servant/ManageShow.java | 1 + .../V15__Add_min_memory_increase_service.sql | 5 + .../spring/applicationContext-service.xml | 3 + .../test/dao/postgres/ServiceDaoTests.java | 11 ++ .../test/dao/postgres/WhiteboardDaoTests.java | 1 + .../dispatcher/FrameCompleteHandlerTests.java | 110 ++++++++++++++++-- .../test/dispatcher/HistoryControlTests.java | 2 +- .../resources/conf/ddl/postgres/test_data.sql | 2 + .../conf/jobspec/jobspec_gpus_test.xml | 17 +++ proto/service.proto | 1 + pycue/opencue/wrappers/service.py | 19 +++ pycue/opencue/wrappers/show.py | 5 + pycue/tests/api_test.py | 4 +- pycue/tests/wrappers/service_test.py | 31 ++++- pycue/tests/wrappers/show_test.py | 6 + 24 files changed, 305 insertions(+), 29 deletions(-) rename cuebot/{build.gradle => build.gradle.orig} (100%) create mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V15__Add_min_memory_increase_service.sql diff --git a/VERSION.in b/VERSION.in index caa4836d8..9f4eca259 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.19 +0.20 diff --git a/cuebot/build.gradle b/cuebot/build.gradle.orig similarity index 100% rename from cuebot/build.gradle rename to cuebot/build.gradle.orig diff --git a/cuebot/src/main/java/com/imageworks/spcue/ServiceEntity.java b/cuebot/src/main/java/com/imageworks/spcue/ServiceEntity.java index 16d03c5c5..0bb47c02c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/ServiceEntity.java +++ b/cuebot/src/main/java/com/imageworks/spcue/ServiceEntity.java @@ -70,5 +70,7 @@ public class ServiceEntity extends Entity { public int timeout_llu = 0; + public long minMemoryIncrease = Dispatcher.MINIMUM_MEMORY_INCREASE; + } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ServiceDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ServiceDaoJdbc.java index 6330cc8cb..a637b34d6 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ServiceDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ServiceDaoJdbc.java @@ -67,6 +67,7 @@ public ServiceEntity mapRow(ResultSet rs, int rowNum) throws SQLException { s.tags = splitTags(rs.getString("str_tags")); s.timeout = rs.getInt("int_timeout"); s.timeout_llu = rs.getInt("int_timeout_llu"); + s.minMemoryIncrease = rs.getLong("int_min_memory_increase"); return s; } }; @@ -89,6 +90,7 @@ public ServiceOverrideEntity mapRow(ResultSet rs, int rowNum) s.showId = rs.getString("pk_show"); s.timeout = rs.getInt("int_timeout"); s.timeout_llu = rs.getInt("int_timeout_llu"); + s.minMemoryIncrease = rs.getLong("int_min_memory_increase"); return s; } }; @@ -106,7 +108,8 @@ public ServiceOverrideEntity mapRow(ResultSet rs, int rowNum) "service.int_gpu_mem_min," + "service.str_tags, " + "service.int_timeout, " + - "service.int_timeout_llu " + + "service.int_timeout_llu, " + + "service.int_min_memory_increase " + "FROM " + "service "; @@ -131,6 +134,7 @@ public ServiceEntity get(String id) { "show_service.str_tags," + "show_service.int_timeout," + "show_service.int_timeout_llu," + + "show_service.int_min_memory_increase," + "show.pk_show " + "FROM " + "show_service," + @@ -180,8 +184,9 @@ public boolean isOverridden(String service, String show) { "int_gpu_mem_min," + "str_tags," + "int_timeout," + - "int_timeout_llu " + - ") VALUES (?,?,?,?,?,?,?,?,?,?,?,?)"; + "int_timeout_llu, " + + "int_min_memory_increase " + + ") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)"; @Override public void insert(ServiceEntity service) { @@ -191,7 +196,8 @@ public void insert(ServiceEntity service) { service.maxCores, service.minMemory, service.minGpus, service.maxGpus, service.minGpuMemory, StringUtils.join(service.tags.toArray(), " | "), - service.timeout, service.timeout_llu); + service.timeout, service.timeout_llu, + service.minMemoryIncrease); } private static final String INSERT_SERVICE_WITH_SHOW = @@ -210,8 +216,9 @@ public void insert(ServiceEntity service) { "int_gpu_mem_min," + "str_tags," + "int_timeout," + - "int_timeout_llu " + - ") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)"; + "int_timeout_llu, " + + "int_min_memory_increase " + + ") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; @Override public void insert(ServiceOverrideEntity service) { @@ -220,7 +227,7 @@ public void insert(ServiceOverrideEntity service) { service.showId, service.name, service.threadable, service.minCores, service.maxCores, service.minMemory, service.minGpus, service.maxGpus, service.minGpuMemory, joinTags(service.tags), - service.timeout, service.timeout_llu); + service.timeout, service.timeout_llu, service.minMemoryIncrease); } private static final String UPDATE_SERVICE = @@ -237,7 +244,8 @@ service.minGpus, service.maxGpus, service.minGpuMemory, joinTags(service.tags), "int_gpu_mem_min=?," + "str_tags=?," + "int_timeout=?," + - "int_timeout_llu=? " + + "int_timeout_llu=?, " + + "int_min_memory_increase=? " + "WHERE " + "pk_service = ?"; @@ -246,7 +254,8 @@ public void update(ServiceEntity service) { getJdbcTemplate().update(UPDATE_SERVICE, service.name, service.threadable, service.minCores, service.maxCores, service.minMemory, service.minGpus, service.maxGpus, service.minGpuMemory, joinTags(service.tags), - service.timeout, service.timeout_llu, service.getId()); + service.timeout, service.timeout_llu, service.minMemoryIncrease, + service.getId()); } private static final String UPDATE_SERVICE_WITH_SHOW = @@ -263,7 +272,8 @@ service.minMemory, service.minGpus, service.maxGpus, service.minGpuMemory, joinT "int_gpu_mem_min=?," + "str_tags=?," + "int_timeout=?," + - "int_timeout_llu=? " + + "int_timeout_llu=?, " + + "int_min_memory_increase=? " + "WHERE " + "pk_show_service = ?"; @@ -272,7 +282,8 @@ public void update(ServiceOverrideEntity service) { getJdbcTemplate().update(UPDATE_SERVICE_WITH_SHOW, service.name, service.threadable, service.minCores, service.maxCores, service.minMemory, service.minGpus, service.maxGpus, service.minGpuMemory, joinTags(service.tags), - service.timeout, service.timeout_llu, service.getId()); + service.timeout, service.timeout_llu, service.minMemoryIncrease, + service.getId()); } @Override diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index aec665423..7a1a85351 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -1461,6 +1461,7 @@ public Service mapRow(ResultSet rs, int rowNum) throws SQLException { SqlUtil.getString(rs,"str_tags")))) .setTimeout(rs.getInt("int_timeout")) .setTimeoutLlu(rs.getInt("int_timeout_llu")) + .setMinMemoryIncrease(rs.getInt("int_min_memory_increase")) .build(); } }; @@ -1482,6 +1483,7 @@ public ServiceOverride mapRow(ResultSet rs, int rowNum) throws SQLException { SqlUtil.getString(rs,"str_tags")))) .setTimeout(rs.getInt("int_timeout")) .setTimeoutLlu(rs.getInt("int_timeout_llu")) + .setMinMemoryIncrease(rs.getInt("int_min_memory_increase")) .build(); return ServiceOverride.newBuilder() .setId(SqlUtil.getString(rs,"pk_show_service")) @@ -2087,7 +2089,8 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "service.int_gpu_mem_min," + "service.str_tags," + "service.int_timeout," + - "service.int_timeout_llu " + + "service.int_timeout_llu," + + "service.int_min_memory_increase " + "FROM "+ "service "; @@ -2104,7 +2107,8 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { "show_service.int_gpu_mem_min," + "show_service.str_tags," + "show_service.int_timeout," + - "show_service.int_timeout_llu " + + "show_service.int_timeout_llu," + + "show_service.int_min_memory_increase " + "FROM "+ "show_service, " + "show " + diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java index 6e06bdc44..6ac703a41 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java @@ -132,6 +132,10 @@ public interface Dispatcher { // The default operating system assigned to host that don't report one. public static final String OS_DEFAULT = "rhel40"; + // The default minimum memory increase for when jobs fail due to not enough + // memory + public static final long MINIMUM_MEMORY_INCREASE = CueUtil.GB2; + /** * Dispatch a host to the facility. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java index 7d6fad4fb..d4e619894 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java @@ -24,6 +24,7 @@ import java.util.Random; import java.util.concurrent.atomic.AtomicLong; +import com.imageworks.spcue.*; import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; @@ -53,6 +54,12 @@ import com.imageworks.spcue.util.CueExceptionUtil; import com.imageworks.spcue.util.CueUtil; +import com.imageworks.spcue.dao.WhiteboardDao; +import com.imageworks.spcue.dao.ShowDao; +import com.imageworks.spcue.dao.ServiceDao; +import com.imageworks.spcue.grpc.service.Service; +import com.imageworks.spcue.grpc.service.ServiceOverride; + /** * The FrameCompleteHandler encapsulates all logic necessary for processing * FrameComplete reports from RQD. @@ -75,6 +82,10 @@ public class FrameCompleteHandler { private DispatchSupport dispatchSupport; private JmsMover jsmMover; + private WhiteboardDao whiteboardDao; + private ServiceDao serviceDao; + private ShowDao showDao; + /* * The last time a proc was unbooked for subscription or job balancing. * Since there are so many more dispatch threads than booking threads, the @@ -298,14 +309,45 @@ public void handlePostFrameCompleteOperations(VirtualProc proc, /* * An exit status of 33 indicates that the frame was killed by the * application due to a memory issue and should be retried. In this - * case, disable the optimizer and raise the memory by 2GB. + * case, disable the optimizer and raise the memory by what is + * specified in the show's service override, service or 2GB. */ if (report.getExitStatus() == Dispatcher.EXIT_STATUS_MEMORY_FAILURE || report.getExitSignal() == Dispatcher.EXIT_STATUS_MEMORY_FAILURE) { + long increase = CueUtil.GB2; + + // since there can be multiple services, just going for the + // first service (primary) + String serviceName = ""; + try { + serviceName = frame.services.split(",")[0]; + ServiceOverride showService = whiteboardDao.getServiceOverride( + showDao.findShowDetail(frame.show), serviceName); + // increase override is stored in Kb format so convert to Mb + // for easier reading. Note: Kb->Mb conversion uses 1024 blocks + increase = showService.getData().getMinMemoryIncrease(); + logger.info("Using " + serviceName + " service show " + + "override for memory increase: " + + Math.floor(increase / 1024) + "Mb."); + } + catch (NullPointerException e) { + logger.info("Frame has no associated services"); + } + catch (EmptyResultDataAccessException e) { + logger.info(frame.show + " has no service override for " + + serviceName + "."); + Service service = whiteboardDao.findService(serviceName); + increase = service.getMinMemoryIncrease(); + logger.info("Using service default for mem increase: " + + Math.floor(increase / 1024) + "Mb."); + } + unbookProc = true; jobManager.enableMemoryOptimizer(frame, false); jobManager.increaseLayerMemoryRequirement(frame, - proc.memoryReserved + CueUtil.GB2); + proc.memoryReserved + increase); + logger.info("Increased mem usage to: " + + (proc.memoryReserved + increase)); } /* @@ -501,7 +543,6 @@ else if (report.getHost().getNimbyLocked()) { dispatchSupport.unbookProc(proc, "frame state was " + newFrameState.toString()); } - } catch (Exception e) { /* * At this point, the proc has no place to go. Since we've run into @@ -692,5 +733,22 @@ public JmsMover getJmsMover() { public void setJmsMover(JmsMover jsmMover) { this.jsmMover = jsmMover; } + + public WhiteboardDao getWhiteboardDao() { return whiteboardDao; } + + public void setWhiteboardDao(WhiteboardDao whiteboardDao) { + this.whiteboardDao = whiteboardDao; } + + public ServiceDao getServiceDao() { return serviceDao; } + + public void setServiceDao(ServiceDao serviceDao) { + this.serviceDao = serviceDao; } + + public ShowDao getShowDao() { return showDao; } + + public void setShowDao(ShowDao showDao) { + this.showDao = showDao; } + } + diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageService.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageService.java index 70a15f3bf..8cd9029c8 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageService.java @@ -62,6 +62,7 @@ public void createService(ServiceCreateServiceRequest request, service.threadable = request.getData().getThreadable(); service.timeout = request.getData().getTimeout(); service.timeout_llu = request.getData().getTimeoutLlu(); + service.minMemoryIncrease = request.getData().getMinMemoryIncrease(); serviceManager.createService(service); responseObserver.onNext(ServiceCreateServiceResponse.newBuilder() .setService(whiteboard.getService(service.getId())) @@ -138,6 +139,7 @@ private ServiceEntity toServiceEntity(Service service) { entity.threadable = service.getThreadable(); entity.timeout = service.getTimeout(); entity.timeout_llu = service.getTimeoutLlu(); + entity.minMemoryIncrease = service.getMinMemoryIncrease(); return entity; } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageServiceOverride.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageServiceOverride.java index ed3d46107..16afb3bf2 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageServiceOverride.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageServiceOverride.java @@ -74,6 +74,7 @@ private ServiceEntity toServiceEntity(Service service) { entity.threadable = service.getThreadable(); entity.timeout = service.getTimeout(); entity.timeout_llu = service.getTimeoutLlu(); + entity.minMemoryIncrease = service.getMinMemoryIncrease(); return entity; } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageShow.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageShow.java index 6e5fbcbe8..0c496b25d 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageShow.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageShow.java @@ -388,6 +388,7 @@ public void createServiceOverride(ShowCreateServiceOverrideRequest request, service.minGpuMemory = requestService.getMinGpuMemory(); service.tags = Sets.newLinkedHashSet(requestService.getTagsList()); service.threadable = requestService.getThreadable(); + service.minMemoryIncrease = requestService.getMinMemoryIncrease(); serviceManager.createService(service); ServiceOverride serviceOverride = whiteboard.getServiceOverride(show, service.name); responseObserver.onNext(ShowCreateServiceOverrideResponse.newBuilder() diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V15__Add_min_memory_increase_service.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V15__Add_min_memory_increase_service.sql new file mode 100644 index 000000000..0b0655521 --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V15__Add_min_memory_increase_service.sql @@ -0,0 +1,5 @@ + +-- Add minimum memory increase - default 2G + +ALTER TABLE show_service ADD COLUMN int_min_memory_increase INT DEFAULT 2097152 NOT NULL; +ALTER TABLE service ADD COLUMN int_min_memory_increase INT DEFAULT 2097152 NOT NULL; diff --git a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml index 6c3a9f2bd..35bd5cbb8 100644 --- a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml +++ b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml @@ -369,6 +369,9 @@ + + + diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ServiceDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ServiceDaoTests.java index 16168f245..811cb129d 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ServiceDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ServiceDaoTests.java @@ -73,6 +73,7 @@ public void testInsertService() { s.minGpuMemory = CueUtil.GB; s.threadable = false; s.tags.addAll(Sets.newHashSet(new String[] { "general"})); + s.minMemoryIncrease = CueUtil.GB4; serviceDao.insert(s); assertEquals(s, serviceDao.get("dillweed")); @@ -91,6 +92,7 @@ public void testUpdateService() { s.minGpuMemory = CueUtil.GB; s.threadable = false; s.tags.addAll(Sets.newHashSet(new String[] { "general"})); + s.minMemoryIncrease = CueUtil.GB; serviceDao.insert(s); assertEquals(s, serviceDao.get("dillweed")); @@ -104,6 +106,7 @@ public void testUpdateService() { s.threadable = true; s.tags = Sets.newLinkedHashSet(); s.tags.add("linux"); + s.minMemoryIncrease = CueUtil.GB4 + CueUtil.GB2; serviceDao.update(s); ServiceEntity s1 = serviceDao.get(s.getId()); @@ -113,6 +116,7 @@ public void testUpdateService() { assertEquals(s.minMemory, s1.minMemory); assertEquals(s.threadable, s1.threadable); assertEquals(s.tags.toArray()[0], s1.tags.toArray()[0]); + assertEquals(s.minMemoryIncrease, s1.minMemoryIncrease); } @Test @@ -128,6 +132,7 @@ public void testDeleteService() { s.minGpuMemory = CueUtil.GB; s.threadable = false; s.tags.addAll(Sets.newHashSet(new String[] { "general"})); + s.minMemoryIncrease = CueUtil.GB2; serviceDao.insert(s); assertEquals(s, serviceDao.get("dillweed")); @@ -153,6 +158,7 @@ public void testInsertServiceOverride() { s.threadable = false; s.tags.addAll(Sets.newHashSet(new String[] { "general"})); s.showId = "00000000-0000-0000-0000-000000000000"; + s.minMemoryIncrease = CueUtil.GB2; serviceDao.insert(s); assertEquals(s, serviceDao.getOverride("dillweed")); @@ -172,6 +178,7 @@ public void testUpdateServiceOverride() { s.threadable = false; s.tags.addAll(Sets.newHashSet(new String[] { "general"})); s.showId = "00000000-0000-0000-0000-000000000000"; + s.minMemoryIncrease = CueUtil.GB2; serviceDao.insert(s); assertEquals(s, serviceDao.getOverride("dillweed")); @@ -186,6 +193,7 @@ public void testUpdateServiceOverride() { s.threadable = true; s.tags = Sets.newLinkedHashSet(); s.tags.add("linux"); + s.minMemoryIncrease = CueUtil.GB4; serviceDao.update(s); ServiceEntity s1 = serviceDao.getOverride(s.getId()); @@ -198,6 +206,8 @@ public void testUpdateServiceOverride() { assertEquals(s.minGpuMemory, s1.minGpuMemory); assertEquals(s.threadable, s1.threadable); assertEquals(s.tags.toArray()[0], s1.tags.toArray()[0]); + assertEquals(s.minMemoryIncrease, s1.minMemoryIncrease); + assertEquals(s1.minMemoryIncrease, CueUtil.GB4); } @Test @@ -214,6 +224,7 @@ public void testDeleteServiceOverride() { s.threadable = false; s.tags.addAll(Sets.newHashSet(new String[] { "general"})); s.showId = "00000000-0000-0000-0000-000000000000"; + s.minMemoryIncrease = CueUtil.GB2; serviceDao.insert(s); assertEquals(s, serviceDao.getOverride("dillweed")); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java index 8807514d4..d419b6ce9 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/WhiteboardDaoTests.java @@ -330,6 +330,7 @@ public void getServiceOverride() { s.tags.add("general"); s.threadable = false; s.showId = show.getId(); + s.minMemoryIncrease = CueUtil.GB4; serviceManager.createService(s); whiteboardDao.getServiceOverride(getShow(), "test"); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java index 9228da36d..f022fc687 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java @@ -22,6 +22,7 @@ import java.io.File; import java.util.List; import javax.annotation.Resource; +import java.util.LinkedHashSet; import org.junit.Before; import org.junit.Test; @@ -36,6 +37,7 @@ import com.imageworks.spcue.FrameDetail; import com.imageworks.spcue.JobDetail; import com.imageworks.spcue.LayerDetail; +import com.imageworks.spcue.ServiceOverrideEntity; import com.imageworks.spcue.VirtualProc; import com.imageworks.spcue.dao.FrameDao; import com.imageworks.spcue.dao.LayerDao; @@ -51,6 +53,7 @@ import com.imageworks.spcue.service.HostManager; import com.imageworks.spcue.service.JobLauncher; import com.imageworks.spcue.service.JobManager; +import com.imageworks.spcue.service.ServiceManager; import com.imageworks.spcue.test.TransactionalTest; import com.imageworks.spcue.util.CueUtil; @@ -88,10 +91,15 @@ public class FrameCompleteHandlerTests extends TransactionalTest { @Resource DispatchSupport dispatchSupport; + @Resource + ServiceManager serviceManager; + private static final String HOSTNAME = "beta"; + private static final String HOSTNAME2 = "zeta"; @Before public void setTestMode() { + dispatcher.setTestMode(true); } @@ -127,10 +135,31 @@ public void createHost() { hostManager.createHost(host, adminManager.findAllocationDetail("spi", "general")); + + RenderHost host2 = RenderHost.newBuilder() + .setName(HOSTNAME2) + .setBootTime(1192369572) + .setFreeMcp(76020) + .setFreeMem((int) CueUtil.GB4) + .setFreeSwap((int) CueUtil.GB4) + .setLoad(0) + .setTotalMcp(195430) + .setTotalMem((int) CueUtil.GB8) + .setTotalSwap((int) CueUtil.GB8) + .setNimbyEnabled(false) + .setNumProcs(8) + .setCoresPerProc(100) + .setState(HardwareState.UP) + .setFacility("spi") + .putAttributes("SP_OS", "Linux") + .build(); + + hostManager.createHost(host2, + adminManager.findAllocationDetail("spi", "general")); } - public DispatchHost getHost() { - return hostManager.findDispatchHost(HOSTNAME); + public DispatchHost getHost(String hostname) { + return hostManager.findDispatchHost(hostname); } @Test @@ -141,7 +170,7 @@ public void testGpuReport() { LayerDetail layer = layerDao.findLayerDetail(job, "layer0"); jobManager.setJobPaused(job, false); - DispatchHost host = getHost(); + DispatchHost host = getHost(HOSTNAME); List procs = dispatcher.dispatchHost(host); assertEquals(1, procs.size()); VirtualProc proc = procs.get(0); @@ -177,7 +206,7 @@ public void testGpuReportMultiple() { LayerDetail layer1_0 = layerDao.findLayerDetail(job1, "layer0"); jobManager.setJobPaused(job1, false); - DispatchHost host = getHost(); + DispatchHost host = getHost(HOSTNAME); List procs = dispatcher.dispatchHost(host); assertEquals(2, procs.size()); @@ -216,7 +245,7 @@ public void testGpuReportOver() { LayerDetail layer2_0 = layerDao.findLayerDetail(job2, "layer0"); jobManager.setJobPaused(job2, false); - DispatchHost host = getHost(); + DispatchHost host = getHost(HOSTNAME); List procs = dispatcher.dispatchHost(host); assertEquals(1, procs.size()); @@ -258,7 +287,7 @@ private void executeDepend( jobManager.setJobPaused(job, false); - DispatchHost host = getHost(); + DispatchHost host = getHost(HOSTNAME); List procs = dispatcher.dispatchHost(host); assertEquals(1, procs.size()); VirtualProc proc = procs.get(0); @@ -327,5 +356,72 @@ public void testDependOnFailureSatisfyOnAny() { executeDepend(FrameState.EATEN, -1, 0, FrameState.WAITING); frameCompleteHandler.setSatisfyDependOnlyOnFrameSuccess(true); } -} + private void executeMinMemIncrease(int expected, boolean override) { + if (override) { + ServiceOverrideEntity soe = new ServiceOverrideEntity(); + soe.showId = "00000000-0000-0000-0000-000000000000"; + soe.name = "apitest"; + soe.threadable = false; + soe.minCores = 10; + soe.minMemory = (int) CueUtil.GB2; + soe.tags = new LinkedHashSet<>(); + soe.tags.add("general"); + soe.minMemoryIncrease = (int) CueUtil.GB8; + + serviceManager.createService(soe); + } + + String jobName = "pipe-default-testuser_min_mem_test"; + JobDetail job = jobManager.findJobDetail(jobName); + LayerDetail layer = layerDao.findLayerDetail(job, "test_layer"); + FrameDetail frame = frameDao.findFrameDetail(job, "0000-test_layer"); + jobManager.setJobPaused(job, false); + + DispatchHost host = getHost(HOSTNAME2); + List procs = dispatcher.dispatchHost(host); + assertEquals(1, procs.size()); + VirtualProc proc = procs.get(0); + assertEquals(job.getId(), proc.getJobId()); + assertEquals(layer.getId(), proc.getLayerId()); + assertEquals(frame.getId(), proc.getFrameId()); + + RunningFrameInfo info = RunningFrameInfo.newBuilder() + .setJobId(proc.getJobId()) + .setLayerId(proc.getLayerId()) + .setFrameId(proc.getFrameId()) + .setResourceId(proc.getProcId()) + .build(); + FrameCompleteReport report = FrameCompleteReport.newBuilder() + .setFrame(info) + .setExitStatus(Dispatcher.EXIT_STATUS_MEMORY_FAILURE) + .build(); + + DispatchJob dispatchJob = jobManager.getDispatchJob(proc.getJobId()); + DispatchFrame dispatchFrame = jobManager.getDispatchFrame(report.getFrame().getFrameId()); + dispatchSupport.stopFrame(dispatchFrame, FrameState.DEAD, report.getExitStatus(), + report.getFrame().getMaxRss()); + frameCompleteHandler.handlePostFrameCompleteOperations(proc, + report, dispatchJob, dispatchFrame, FrameState.WAITING); + + assertFalse(jobManager.isLayerComplete(layer)); + + JobDetail ujob = jobManager.findJobDetail(jobName); + LayerDetail ulayer = layerDao.findLayerDetail(ujob, "test_layer"); + assertEquals(expected, ulayer.getMinimumMemory()); + } + + @Test + @Transactional + @Rollback(true) + public void testMinMemIncrease() { + executeMinMemIncrease(6291456, false); + } + + @Test + @Transactional + @Rollback(true) + public void testMinMemIncreaseShowOverride() { + executeMinMemIncrease(10485760, true); + } +} diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java index a9787566b..138a3f33c 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HistoryControlTests.java @@ -169,7 +169,7 @@ public void testEnabled() { launchAndDeleteJob(); - assertEquals(Integer.valueOf(4), jdbcTemplate.queryForObject( + assertEquals(Integer.valueOf(5), jdbcTemplate.queryForObject( "SELECT COUNT(*) FROM job_history", Integer.class)); assertEquals(Integer.valueOf(1), jdbcTemplate.queryForObject( "SELECT COUNT(*) FROM frame_history", Integer.class)); diff --git a/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql b/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql index 14c56afcc..ae7b77354 100644 --- a/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql +++ b/cuebot/src/test/resources/conf/ddl/postgres/test_data.sql @@ -104,6 +104,8 @@ Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN, Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS,INT_CORES_MAX,INT_GPU_MEM_MIN) values ('488c75f0-eae4-4dd0-83e0-29b982adbbff','cuda',true,100,3354624,'cuda',0,262144) +Insert into SERVICE (PK_SERVICE,STR_NAME,B_THREADABLE,INT_CORES_MIN,INT_MEM_MIN,STR_TAGS,INT_MIN_MEMORY_INCREASE) values ('123c75f0-eie4-4cc0-84e0-46b982abcdef','apitest',false,10,2097152,'general',4194304) + Insert into CONFIG (PK_CONFIG,STR_KEY,INT_VALUE,LONG_VALUE,STR_VALUE,B_VALUE) values ('00000000-0000-0000-0000-000000000005','MAX_FRAME_RETRIES',16,0,null,false) diff --git a/cuebot/src/test/resources/conf/jobspec/jobspec_gpus_test.xml b/cuebot/src/test/resources/conf/jobspec/jobspec_gpus_test.xml index 9d9aa8245..cfa6e694c 100644 --- a/cuebot/src/test/resources/conf/jobspec/jobspec_gpus_test.xml +++ b/cuebot/src/test/resources/conf/jobspec/jobspec_gpus_test.xml @@ -74,6 +74,22 @@ + + true + 3 + + + /shots/pipe/usr_testuser/logs/help.py + 0 + 1 + false + + apitest + + + + + True @@ -108,4 +124,5 @@ layer_first + diff --git a/proto/service.proto b/proto/service.proto index 23633bc40..57110bdaf 100644 --- a/proto/service.proto +++ b/proto/service.proto @@ -53,6 +53,7 @@ message Service { int32 timeout_llu = 10; int32 min_gpus = 11; int32 max_gpus = 12; + int32 min_memory_increase = 13; } message ServiceSeq { diff --git a/pycue/opencue/wrappers/service.py b/pycue/opencue/wrappers/service.py index 3878d9b3d..b3f6563d2 100644 --- a/pycue/opencue/wrappers/service.py +++ b/pycue/opencue/wrappers/service.py @@ -29,6 +29,10 @@ def __init__(self, service=None): def create(self): """Creates a service in the database using the current instance data.""" + # min_memory_increase has to be greater than 0. + if self.data.min_memory_increase <= 0: + raise ValueError("Minimum memory increase must be > 0") + response = self.stub.CreateService( service_pb2.ServiceCreateServiceRequest(data=self.data), timeout=Cuebot.Timeout) @@ -69,6 +73,10 @@ def getService(name): def update(self): """Updates the service database record with the current instance data.""" + # min_memory_increase has to be greater than 0. + if self.data.min_memory_increase <= 0: + raise ValueError("Minimum memory increase must be > 0") + return self.stub.Update( service_pb2.ServiceUpdateRequest(service=self.data), timeout=Cuebot.Timeout) @@ -240,3 +248,14 @@ def timeoutLLU(self): def setTimeoutLLU(self, timeout_llu): """Sets the default service LLU timeout.""" self.data.timeout_llu = timeout_llu + + def minMemoryIncrease(self): + """Gets the default service minimum memory increment""" + return self.data.min_memory_increase + + def setMinMemoryIncrease(self, min_memory_increase): + """Sets the default service minimum memory increment""" + if min_memory_increase > 0: + self.data.min_memory_increase = min_memory_increase + else: + raise ValueError("Minimum memory increase must be > 0") diff --git a/pycue/opencue/wrappers/show.py b/pycue/opencue/wrappers/show.py index 90094e757..5dad1a000 100644 --- a/pycue/opencue/wrappers/show.py +++ b/pycue/opencue/wrappers/show.py @@ -70,6 +70,11 @@ def createServiceOverride(self, data): :type data: service_pb2.Service :param data: service data, typically from opencue.wrappers.service.Service.data """ + + # min_memory_increase has to be greater than 0. + if data.min_memory_increase <= 0: + raise ValueError("Minimum memory increase must be > 0") + self.stub.CreateServiceOverride( show_pb2.ShowCreateServiceOverrideRequest(show=self.data, service=data), timeout=Cuebot.Timeout) diff --git a/pycue/tests/api_test.py b/pycue/tests/api_test.py index b20acfff9..759c09da1 100644 --- a/pycue/tests/api_test.py +++ b/pycue/tests/api_test.py @@ -357,11 +357,13 @@ class ServiceTests(unittest.TestCase): testThreadable = False testMinCores = 1000 testMaxCores = 2000 + minMemoryIncrease = 2068378 def setUp(self): self.service = service_pb2.Service( name=self.testName, threadable=self.testThreadable, min_cores=self.testMinCores, - max_cores=self.testMaxCores, tags=self.testTags) + max_cores=self.testMaxCores, tags=self.testTags, + min_memory_increase=self.minMemoryIncrease) @mock.patch('opencue.cuebot.Cuebot.getStub') def testCreate(self, getStubMock): diff --git a/pycue/tests/wrappers/service_test.py b/pycue/tests/wrappers/service_test.py index e46b29bdf..3500eec5c 100644 --- a/pycue/tests/wrappers/service_test.py +++ b/pycue/tests/wrappers/service_test.py @@ -51,16 +51,23 @@ def testDelete(self, getStubMock): def testCreateService(self, getStubMock): stubMock = mock.Mock() stubMock.CreateService.return_value = service_pb2.ServiceCreateServiceResponse( - service=service_pb2.Service(name=TEST_SERVICE_NAME)) + service=service_pb2.Service(name=TEST_SERVICE_NAME, min_memory_increase=2097152)) getStubMock.return_value = stubMock wrapper = opencue.wrappers.service.Service( - service_pb2.Service(name=TEST_SERVICE_NAME)) + service_pb2.Service(name=TEST_SERVICE_NAME, min_memory_increase=2097152)) service = wrapper.create() stubMock.CreateService.assert_called_with( service_pb2.ServiceCreateServiceRequest(data=wrapper.data), timeout=mock.ANY) self.assertEqual(wrapper.name(), service.name()) + self.assertEqual(wrapper.minMemoryIncrease(), service.minMemoryIncrease()) + + def testCreateServiceMemError(self, getStubMock): + service = opencue.wrappers.service.Service(service_pb2.Service( + name=TEST_SERVICE_NAME)) + + self.assertRaises(ValueError, service.create) def testGetDefaultServices(self, getStubMock): stubMock = mock.Mock() @@ -96,7 +103,7 @@ def testUpdate(self, getStubMock): getStubMock.return_value = stubMock wrapper = opencue.wrappers.service.Service(service=service_pb2.Service( - name=TEST_SERVICE_NAME)) + name=TEST_SERVICE_NAME, min_memory_increase=302)) wrapper.update() stubMock.Update.assert_called_with( @@ -124,5 +131,23 @@ def testGpus(self, getStubMock): service_pb2.ServiceGetServiceRequest(name=TEST_SERVICE_NAME), timeout=mock.ANY) self.assertEqual(service.name(), TEST_SERVICE_NAME) + def testUpdateMemError(self, getStubMock): + service = opencue.wrappers.service.Service(service=service_pb2.Service( + name=TEST_SERVICE_NAME)) + + self.assertRaises(ValueError, service.update) + + def testSetMinMemIncrease(self, getStubMock): + service = opencue.wrappers.service.Service( + service_pb2.Service(name=TEST_SERVICE_NAME, + min_memory_increase=2097152)) + + self.assertRaises(ValueError, service.setMinMemoryIncrease, -1) + self.assertRaises(ValueError, service.setMinMemoryIncrease, 0) + + service.setMinMemoryIncrease(12345678) + self.assertEqual(service.minMemoryIncrease(), 12345678) + + if __name__ == '__main__': unittest.main() diff --git a/pycue/tests/wrappers/show_test.py b/pycue/tests/wrappers/show_test.py index b8fb26285..cb73a4b5d 100644 --- a/pycue/tests/wrappers/show_test.py +++ b/pycue/tests/wrappers/show_test.py @@ -324,6 +324,12 @@ def testEnableDispatching(self, getStubMock): show_pb2.ShowEnableDispatchingRequest(show=show.data, enabled=TEST_ENABLE_VALUE), timeout=mock.ANY) + def testCreateServiceOverrideMemError(self, getStubMock): + service = service_pb2.Service(name=TEST_SERVICE_NAME) + show = opencue.wrappers.show.Show(show_pb2.Show(name=TEST_SHOW_NAME)) + + self.assertRaises(ValueError, show.createServiceOverride, service) + if __name__ == '__main__': unittest.main() From 3aca86f8752c6ac7d90e6e575e3fc674b4b07d28 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Wed, 20 Jul 2022 14:16:19 -0400 Subject: [PATCH 03/21] Notes from July 6 TSC meeting. (#1176) --- tsc/meetings/2022-07-06.md | 125 +++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 tsc/meetings/2022-07-06.md diff --git a/tsc/meetings/2022-07-06.md b/tsc/meetings/2022-07-06.md new file mode 100644 index 000000000..9b52bfb9b --- /dev/null +++ b/tsc/meetings/2022-07-06.md @@ -0,0 +1,125 @@ +# OpenCue TSC Meeting Notes 6 July 2022 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* New business + * API changes to fail on incorrect parameters + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1165/files + * no issues, good to merge when code/tests are ready + * Make frames and layers readonly after job finishes + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1164 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1163 + * solution seems ok, but should be a cuebot config flag for now + * Siggraph/Open Source Days + * Few folks attending Siggraph likely. Brian will be OOO and unable to attend. + * Will hybrid in-person/virtual sessions be supported? + * How to design the session? + * Brian to do some research and start email thread. + * Integration tests + * Integration test script being developed at SPI. + * We will want to look at turning it into a github action. + * This will help speed up the release process a lot. Most of the time spent making a release is QAing it. + * SPI to update soon. +* Ongoing work + * CueGUI memory leak + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1025 + * What we know: bug in job monitor tree creates unnecessary refreshing. Probably present in all tree plugins, + but most apparent when monitoring 100s/1000s of jobs. + * SPI: researching and testing their fix. + * RQD change, adding child proc info into log, storing in database + * For understanding memory usage of complex jobs. + * Linux-only. + * Testing ongoing, feedback has been positive so far. + * Change was deployed to production, found some issues, fixed. + * Work is done. Has it been merged to github yet? + * Avoiding complex merges from upstream. + * Problem: users will maintain their own OpenCue forks with many preferred customizations. When it comes time to + merge in changes from upstream, a complex merge is needed. This is time consuming and results in long gaps + between upstream merges. + * How do others solve this? + * Suggestion: let's identify the various areas needing customization, and discuss how we can generalize a + solution by creating options/plugins/hooks so customization can live outside the main codebase. + * Most spots identified, PRs have been sent + * Upstream merge almost complete, will resume that soon. + * OS-dependent log root + * Issue: https://github.com/AcademySoftwareFoundation/OpenCue/issues/1096 + * PR: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1102 + * PR merged. + * New PR to fix log paths on the CueGUI side: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1138 + * All PRs merged. Done now. + * Nimby in new thread: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1063 + * Not reproducible? We need input from SPI. + * Reproduced now. PR coming with several rqd/nimby fixes. + * pynput will become the default for detecting system activity. + * Sentry logging in CueGUI: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1127 + * Change looks ok, looking for more detail. + * Let's follow up on that PR again, review, figure out if we need to clone to new PR. +* 2022 Goals + * New user UX + * Publish PyPI packages + * Design doc: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/ + * Proposal for config file + standardization: https://docs.google.com/document/d/1pWjMNGzGmBvuHoBzXewQEzdwTKtrY6VOtaPi7ALwWg0/edit#heading=h.v85uo9klwqft + * Config cleanup for pycue, pyoutline, cuesubmit done. + * CueGUI in progress. + * RQD still todo. + * Fix cuebot:latest tag for sandbox setup + * Proposal: publish Docker images for every new patch version (i.e. every new commit to master) + * Proposal accepted. + * No progress implementing this yet. + * Related: for PyPI design we may want to also publish packages for every new patch version. + * Docs refresh + * Tech writing help + * Discussing with John Mertic at LF and OCIO reps who are also interested. + * User guide refresh + * Split into "install from latest release" and "install from master" + * Needed to assist transition to publish Docker images on all commits + * No progress. + * API reference + * We have the ability to generate HTML from any commit, but we need a new CI pipeline to build and publish + this on opencue.io for new releases. + * Proposal: for now, highlight existing docs for building reference locally + * Proposal accepted. + * Brian: cleaned up local build process, sent email with doc + link: https://www.opencue.io/contributing/opencue/build-docs/ + * pyoutline examples + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/177 + * No progress yet. + * User guide for CueGUI + * No progress. + * Configuration guides for Python/Cuebot/RQD + * In progress, will be done as part of PyPI work. + * Improve scheduler logic + * Diego has volunteered to own this. + * Email reply from Matt, looped in others. + * Other PRs with improvements — see ongoing work above. + * Prototype in progress. + * High-level prototype details using Redis: https://github.com/AcademySoftwareFoundation/OpenCue/issues/1001 + * Expanded Cloud functionality + * Finalize GSoC plugin + * Test Azure support + * Add AWS support + * Design expanded functionality e.g. autoscaling for work in the queue. + * Last update from Greg: started to test / clean up azure support. Looking good so far. + * Terraform scripts? + * Let's look at the Azure version of this + * Generic k8s setup and cuebot pool size suggestions + * Existing Azure work on this: https://github.com/Azure/Avere/tree/main/src/terraform/examples/vfxt/opencue + * Issue for discussion: https://github.com/AcademySoftwareFoundation/OpenCue/issues/1097 + * Use Kubernetes for rqd? Kubernetes adds some overhead. A very small amount but some people care. + * A Helm chart could be a good option. Easy to use and get started. + * Let's review the Github issue, leave notes. Short, rough draft of design doc for auto-scaling would be good + for further discussion. + * User permissions + * Need a github issue for discussion. + * Added initial + thoughts: https://github.com/AcademySoftwareFoundation/OpenCue/issues/218#issuecomment-1016904933 + * Need more research into integration with Active Directory. + * Permissions model: proposal to be written up. + * Expand DCC plugins + * High priority: Blender, Houdini + * Important for wider user adoption. + * Worth writing to the user group to see what folks have already. + * No further progress. From b2d4fc220c9440d958c3141d56b06473bad1fe40 Mon Sep 17 00:00:00 2001 From: Yaash Jain Date: Thu, 21 Jul 2022 11:13:01 -0700 Subject: [PATCH 04/21] Add OOM Increase field to Service Properties (#1160) * Add OOM Increase field to Service Properties Allow setting minimum memory increase for services through Cuecommander * Make range for OOM increase field more accurate Previously, the min. value for the input was 512. This caused confusion when db value for min_memory_increase was 0. Also includes check to prevent users from setting the field to 0. --- cuegui/cuegui/ServiceDialog.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/cuegui/cuegui/ServiceDialog.py b/cuegui/cuegui/ServiceDialog.py index bdeec4dd6..b29a13c23 100644 --- a/cuegui/cuegui/ServiceDialog.py +++ b/cuegui/cuegui/ServiceDialog.py @@ -72,6 +72,9 @@ def __init__(self, parent=None): self.timeout_llu = QtWidgets.QSpinBox(self) self.timeout_llu.setRange(0, 4320) self.timeout_llu.setValue(0) + self.min_memory_increase = QtWidgets.QSpinBox(self) + self.min_memory_increase.setRange(0, int(self._cfg().get('max_memory', 48)) * 1024) + self.min_memory_increase.setValue(0) layout = QtWidgets.QGridLayout(self) layout.addWidget(QtWidgets.QLabel("Name:", self), 0, 0) layout.addWidget(self.name, 0, 1) @@ -89,15 +92,17 @@ def __init__(self, parent=None): layout.addWidget(self.timeout, 6, 1) layout.addWidget(QtWidgets.QLabel("Timeout LLU (in minutes):", self), 7, 0) layout.addWidget(self.timeout_llu, 7, 1) + layout.addWidget(QtWidgets.QLabel("OOM Increase MB:", self), 8, 0) + layout.addWidget(self.min_memory_increase, 8, 1) self._tags_w = cuegui.TagsWidget.TagsWidget(allowed_tags=cuegui.Constants.ALLOWED_TAGS) - layout.addWidget(self._tags_w, 8, 0, 1, 2) + layout.addWidget(self._tags_w, 9, 0, 1, 2) self.__buttons = QtWidgets.QDialogButtonBox(QtWidgets.QDialogButtonBox.Save, QtCore.Qt.Horizontal, self) self.__buttons.setDisabled(True) - layout.addWidget(self.__buttons, 9, 1) + layout.addWidget(self.__buttons, 10, 1) self.__buttons.accepted.connect(self.save) # pylint: disable=no-member @@ -128,6 +133,8 @@ def setService(self, service): self._tags_w.set_tags(service.data.tags) self.timeout.setValue(service.data.timeout) self.timeout_llu.setValue(service.data.timeout_llu) + self.min_memory_increase.setValue(service.data.min_memory_increase // 1024) + self.__service = service.data def new(self): """ @@ -144,6 +151,7 @@ def new(self): self.min_gpu_memory.setValue(self.gpu_min_mb) self.timeout.setValue(0) self.timeout_llu.setValue(0) + self.min_memory_increase.setValue(2048) self._tags_w.set_tags(['general']) def save(self): @@ -160,6 +168,11 @@ def save(self): QtWidgets.QMessageBox.critical(self, "Error", "The service name must alphanumeric.") return + if self.min_memory_increase.value() <= 0: + QtWidgets.QMessageBox.critical(self, "Error", + "The minimum memory increase must be more than 0 MB") + return + service = opencue.wrappers.service.Service() if self.__service: service.data.id = self.__service.data.id @@ -171,6 +184,7 @@ def save(self): service.setMinGpuMemory(self.min_gpu_memory.value() * 1024) service.setTimeout(self.timeout.value()) service.setTimeoutLLU(self.timeout_llu.value()) + service.setMinMemoryIncrease(self.min_memory_increase.value() * 1024) service.setTags(self._tags_w.get_tags()) self.saved.emit(service) @@ -330,4 +344,4 @@ def __init__(self, show, parent=None): self.setWindowTitle("Services") self.setAttribute(QtCore.Qt.WA_DeleteOnClose) self.setSizeGripEnabled(True) - self.resize(620, 420) + self.resize(700, 700) From 1d59497ac6bc5b8a8756839f59e74fa9b7172658 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 1 Aug 2022 15:47:20 -0400 Subject: [PATCH 05/21] Notes from July 20 TSC meeting. (#1179) --- tsc/meetings/2022-07-20.md | 45 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tsc/meetings/2022-07-20.md diff --git a/tsc/meetings/2022-07-20.md b/tsc/meetings/2022-07-20.md new file mode 100644 index 000000000..99037e579 --- /dev/null +++ b/tsc/meetings/2022-07-20.md @@ -0,0 +1,45 @@ +# OpenCue TSC Meeting Notes 20 July 2022 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Open Source Days + * Sections + * New stuff + * Improved Windows support, including hybrid farms with multiple OSes (multiple log roots, path conversion + in CueGUI) + * Improved GPU support + * New scheduler modes + * Scitech investigations + * Github stats + * Last year 34 -> 41 unique contributors from 10+ organizations + * Grown committer pool to 8 committers + * 102 PRs merged = 2/week + * Issues + * Over past year, 27 unique people have submitted issues. + * Would be nice to count people who have contributed to issues as well, many folks will chime in but not + necessarily open their own issues. + * Any way to do this? + * Contributor pool significantly more diverse + * In the past year 50% of commits came from "founding" organizations. + * In past years / all time this was ~70%. + * Good to highlight project is not dependent on the original creators. + * SPI update + * Fixing things from migration + * Working on upstream merge issues + * When was oracle removed? Apr 2021 + * Potential good story: migration cue3 -> opencue -> opencue master branch + * Stats on shows that have used opencue + * Anything usable from scitech presentation? + * Q&A + * Let's collect potential warmup questions + * We could also maybe use the LF poll tool. For example to vote on feature requests. + * Brian to create new doc to polish this outline. + * Presenters + * Diego: in-person, run through the project update then open up Q&A + * Ben: virtual + * Greg: in-person + * Brian to email LF organizers with contact info. +* Record video on using opencue from wrangler perspective? From Noel? Let's check in after OSD event. +* Yaash: introductions. Waiting on a code review. From e8660d99c08ea0ca17e7fc9b6bb1eb351a879c59 Mon Sep 17 00:00:00 2001 From: Akim Ruslanov Date: Thu, 11 Aug 2022 17:03:41 -0700 Subject: [PATCH 06/21] Open log files in binary mode for Python 3 comp (#1182) --- cuegui/cuegui/Utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cuegui/cuegui/Utils.py b/cuegui/cuegui/Utils.py index 17df4e559..8d41df3cf 100644 --- a/cuegui/cuegui/Utils.py +++ b/cuegui/cuegui/Utils.py @@ -459,17 +459,17 @@ def getLastLine(path): ansiEscape = r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]' try: - fp = open(path, 'r') + fp = open(path, 'rb') fp.seek(0, 2) backseek = min(4096, fp.tell()) fp.seek(-backseek, 1) buf = fp.read(4096) - newline_pos = buf.rfind("\n",0,len(buf)-1) + newline_pos = buf.rfind(b'\n', 0, len(buf)-1) fp.close() - line = buf[newline_pos+1:].strip() + line = buf[newline_pos+1:].strip().decode("utf-8") return re.sub(ansiEscape, "", line) except IOError: From d530cc74a7fcb9ebaea636a570d6d44ce6c31011 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 15 Aug 2022 16:25:51 -0400 Subject: [PATCH 07/21] Restore build.gradle original filename. (#1184) --- cuebot/{build.gradle.orig => build.gradle} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cuebot/{build.gradle.orig => build.gradle} (100%) diff --git a/cuebot/build.gradle.orig b/cuebot/build.gradle similarity index 100% rename from cuebot/build.gradle.orig rename to cuebot/build.gradle From e7c38c6bf5a39eec650a1f7a3851dabcc10d6f48 Mon Sep 17 00:00:00 2001 From: Akim Ruslanov Date: Tue, 23 Aug 2022 09:27:45 -0700 Subject: [PATCH 08/21] cuebot: Make frames and layers readonly after job is done (#1164) * Make frames and layers readonly after job is done * Add prop to toggle viewing done jobs as readonly * Encapsulate editing response observer * Refactor attemptChange method into utils --- .../imageworks/spcue/servant/ManageJob.java | 249 +++++++++------- .../imageworks/spcue/servant/ManageLayer.java | 269 +++++++++++------- .../imageworks/spcue/servant/ServantUtil.java | 28 ++ cuebot/src/main/resources/opencue.properties | 6 + 4 files changed, 343 insertions(+), 209 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java index f7c765636..01601998a 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java @@ -25,6 +25,8 @@ import io.grpc.Status; import io.grpc.stub.StreamObserver; import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; import org.springframework.dao.EmptyResultDataAccessException; import com.imageworks.spcue.BuildableJob; @@ -156,6 +158,8 @@ import com.imageworks.spcue.util.Convert; import com.imageworks.spcue.util.FrameSet; +import static com.imageworks.spcue.servant.ServantUtil.attemptChange; + public class ManageJob extends JobInterfaceGrpc.JobInterfaceImplBase { private static final Logger logger = Logger.getLogger(ManageJob.class); private Whiteboard whiteboard; @@ -173,6 +177,9 @@ public class ManageJob extends JobInterfaceGrpc.JobInterfaceImplBase { private JobInterface job; private FrameSearchFactory frameSearchFactory; private JobSearchFactory jobSearchFactory; + private final String property = "frame.finished_jobs_readonly"; + @Autowired + private Environment env; @Override public void findJob(JobFindJobRequest request, StreamObserver responseObserver) { @@ -357,9 +364,11 @@ public void resume(JobResumeRequest request, StreamObserver r public void setMaxCores(JobSetMaxCoresRequest request, StreamObserver responseObserver) { try{ setupJobData(request.getJob()); - jobDao.updateMaxCores(job, Convert.coresToWholeCoreUnits(request.getVal())); - responseObserver.onNext(JobSetMaxCoresResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + jobDao.updateMaxCores(job, Convert.coresToWholeCoreUnits(request.getVal())); + responseObserver.onNext(JobSetMaxCoresResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -417,9 +426,11 @@ public void setMinGpus(JobSetMinGpusRequest request, StreamObserver responseObserver) { try{ setupJobData(request.getJob()); - jobDao.updatePriority(job, request.getVal()); - responseObserver.onNext(JobSetPriorityResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + jobDao.updatePriority(job, request.getVal()); + responseObserver.onNext(JobSetPriorityResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -449,13 +460,15 @@ public void getCurrent(JobGetCurrentRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - manageQueue.execute( - new DispatchEatFrames( - frameSearchFactory.create(job, request.getReq()), - new Source(request.toString()), - jobManagerSupport)); - responseObserver.onNext(JobEatFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + manageQueue.execute( + new DispatchEatFrames( + frameSearchFactory.create(job, request.getReq()), + new Source(request.toString()), + jobManagerSupport)); + responseObserver.onNext(JobEatFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -468,13 +481,15 @@ public void eatFrames(JobEatFramesRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - manageQueue.execute( - new DispatchKillFrames( - frameSearchFactory.create(job, request.getReq()), - new Source(request.toString()), - jobManagerSupport)); - responseObserver.onNext(JobKillFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + manageQueue.execute( + new DispatchKillFrames( + frameSearchFactory.create(job, request.getReq()), + new Source(request.toString()), + jobManagerSupport)); + responseObserver.onNext(JobKillFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -488,11 +503,13 @@ public void markDoneFrames(JobMarkDoneFramesRequest request, StreamObserver responseObserver) { try{ setupJobData(request.getJob()); - manageQueue.execute( - new DispatchSatisfyDepends( - frameSearchFactory.create(job, request.getReq()), jobManagerSupport)); - responseObserver.onNext(JobMarkDoneFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + manageQueue.execute( + new DispatchSatisfyDepends( + frameSearchFactory.create(job, request.getReq()), jobManagerSupport)); + responseObserver.onNext(JobMarkDoneFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -505,13 +522,15 @@ public void markDoneFrames(JobMarkDoneFramesRequest request, public void retryFrames(JobRetryFramesRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - manageQueue.execute( - new DispatchRetryFrames( - frameSearchFactory.create(job, request.getReq()), - new Source(request.toString()), - jobManagerSupport)); - responseObserver.onNext(JobRetryFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + manageQueue.execute( + new DispatchRetryFrames( + frameSearchFactory.create(job, request.getReq()), + new Source(request.toString()), + jobManagerSupport)); + responseObserver.onNext(JobRetryFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -524,9 +543,11 @@ public void retryFrames(JobRetryFramesRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - jobDao.updateAutoEat(job, request.getValue()); - responseObserver.onNext(JobSetAutoEatResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + jobDao.updateAutoEat(job, request.getValue()); + responseObserver.onNext(JobSetAutoEatResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -540,13 +561,15 @@ public void createDependencyOnFrame(JobCreateDependencyOnFrameRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - JobOnFrame depend = new JobOnFrame(job, - jobManager.getFrameDetail(request.getFrame().getId())); - dependManager.createDepend(depend); - responseObserver.onNext(JobCreateDependencyOnFrameResponse.newBuilder() - .setDepend(whiteboard.getDepend(depend)) - .build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + JobOnFrame depend = new JobOnFrame(job, + jobManager.getFrameDetail(request.getFrame().getId())); + dependManager.createDepend(depend); + responseObserver.onNext(JobCreateDependencyOnFrameResponse.newBuilder() + .setDepend(whiteboard.getDepend(depend)) + .build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -560,13 +583,15 @@ public void createDependencyOnJob(JobCreateDependencyOnJobRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - JobOnJob depend = new JobOnJob(job, - jobManager.getJobDetail(request.getOnJob().getId())); - dependManager.createDepend(depend); - responseObserver.onNext(JobCreateDependencyOnJobResponse.newBuilder() - .setDepend(whiteboard.getDepend(depend)) - .build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + JobOnJob depend = new JobOnJob(job, + jobManager.getJobDetail(request.getOnJob().getId())); + dependManager.createDepend(depend); + responseObserver.onNext(JobCreateDependencyOnJobResponse.newBuilder() + .setDepend(whiteboard.getDepend(depend)) + .build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -580,13 +605,15 @@ public void createDependencyOnLayer(JobCreateDependencyOnLayerRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - JobOnLayer depend = new JobOnLayer(job, - jobManager.getLayerDetail(request.getLayer().getId())); - dependManager.createDepend(depend); - responseObserver.onNext(JobCreateDependencyOnLayerResponse.newBuilder() - .setDepend(whiteboard.getDepend(depend)) - .build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + JobOnLayer depend = new JobOnLayer(job, + jobManager.getLayerDetail(request.getLayer().getId())); + dependManager.createDepend(depend); + responseObserver.onNext(JobCreateDependencyOnLayerResponse.newBuilder() + .setDepend(whiteboard.getDepend(depend)) + .build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -671,9 +698,11 @@ public void getUpdatedFrames(JobGetUpdatedFramesRequest request, StreamObserver< public void setMaxRetries(JobSetMaxRetriesRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - jobDao.updateMaxFrameRetries(job, request.getMaxRetries()); - responseObserver.onNext(JobSetMaxRetriesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + jobDao.updateMaxFrameRetries(job, request.getMaxRetries()); + responseObserver.onNext(JobSetMaxRetriesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -723,9 +752,11 @@ public void getComments(JobGetCommentsRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - manageQueue.execute(new DispatchDropDepends(job, request.getTarget(), dependManager)); - responseObserver.onNext(JobDropDependsResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + manageQueue.execute(new DispatchDropDepends(job, request.getTarget(), dependManager)); + responseObserver.onNext(JobDropDependsResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -738,9 +769,11 @@ public void dropDepends(JobDropDependsRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - jobDao.updateParent(job, groupManager.getGroupDetail(request.getGroupId())); - responseObserver.onNext(JobSetGroupResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + jobDao.updateParent(job, groupManager.getGroupDetail(request.getGroupId())); + responseObserver.onNext(JobSetGroupResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -754,10 +787,12 @@ public void markAsWaiting(JobMarkAsWaitingRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - jobManagerSupport.markFramesAsWaiting( - frameSearchFactory.create(job, request.getReq()), new Source(request.toString())); - responseObserver.onNext(JobMarkAsWaitingResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + jobManagerSupport.markFramesAsWaiting( + frameSearchFactory.create(job, request.getReq()), new Source(request.toString())); + responseObserver.onNext(JobMarkAsWaitingResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -771,10 +806,12 @@ public void reorderFrames(JobReorderFramesRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - manageQueue.execute(new DispatchReorderFrames(job, - new FrameSet(request.getRange()), request.getOrder(), jobManagerSupport)); - responseObserver.onNext(JobReorderFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + manageQueue.execute(new DispatchReorderFrames(job, + new FrameSet(request.getRange()), request.getOrder(), jobManagerSupport)); + responseObserver.onNext(JobReorderFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -804,10 +841,12 @@ public void staggerFrames(JobStaggerFramesRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - manageQueue.execute( - new DispatchStaggerFrames(job, request.getRange(), request.getStagger(), jobManagerSupport)); - responseObserver.onNext(JobStaggerFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + manageQueue.execute( + new DispatchStaggerFrames(job, request.getRange(), request.getStagger(), jobManagerSupport)); + responseObserver.onNext(JobStaggerFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL @@ -820,31 +859,33 @@ public void staggerFrames(JobStaggerFramesRequest request, public void addRenderPartition(JobAddRenderPartRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - LocalHostAssignment lha = new LocalHostAssignment(); - lha.setJobId(job.getId()); - lha.setThreads(request.getThreads()); - lha.setMaxCoreUnits(request.getMaxCores() * 100); - lha.setMaxMemory(request.getMaxMemory()); - lha.setMaxGpuUnits(request.getMaxGpus()); - lha.setMaxGpuMemory(request.getMaxGpuMemory()); - lha.setType(RenderPartitionType.JOB_PARTITION); - - if (localBookingSupport.bookLocal(job, request.getHost(), request.getUsername(), lha)) { - try { - RenderPartition renderPart = whiteboard.getRenderPartition(lha); - responseObserver.onNext(JobAddRenderPartResponse.newBuilder() - .setRenderPartition(renderPart) - .build()); - responseObserver.onCompleted(); - } catch (EmptyResultDataAccessException e) { + if (attemptChange(env, property, jobManager, job, responseObserver)) { + LocalHostAssignment lha = new LocalHostAssignment(); + lha.setJobId(job.getId()); + lha.setThreads(request.getThreads()); + lha.setMaxCoreUnits(request.getMaxCores() * 100); + lha.setMaxMemory(request.getMaxMemory()); + lha.setMaxGpuUnits(request.getMaxGpus()); + lha.setMaxGpuMemory(request.getMaxGpuMemory()); + lha.setType(RenderPartitionType.JOB_PARTITION); + + if (localBookingSupport.bookLocal(job, request.getHost(), request.getUsername(), lha)) { + try { + RenderPartition renderPart = whiteboard.getRenderPartition(lha); + responseObserver.onNext(JobAddRenderPartResponse.newBuilder() + .setRenderPartition(renderPart) + .build()); + responseObserver.onCompleted(); + } catch (EmptyResultDataAccessException e) { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to allocate render partition to host.") + .asRuntimeException()); + } + } else { responseObserver.onError(Status.INTERNAL - .withDescription("Failed to allocate render partition to host.") + .withDescription("Failed to find suitable frames.") .asRuntimeException()); } - } else { - responseObserver.onError(Status.INTERNAL - .withDescription("Failed to find suitable frames.") - .asRuntimeException()); } } catch (EmptyResultDataAccessException e) { @@ -858,10 +899,12 @@ public void addRenderPartition(JobAddRenderPartRequest request, StreamObserver responseObserver) { try { setupJobData(request.getJob()); - JobDetail jobDetail = jobManager.getJobDetail(job.getJobId()); - filterManager.runFiltersOnJob(jobDetail); - responseObserver.onNext(JobRunFiltersResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + JobDetail jobDetail = jobManager.getJobDetail(job.getJobId()); + filterManager.runFiltersOnJob(jobDetail); + responseObserver.onNext(JobRunFiltersResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } catch (EmptyResultDataAccessException e) { responseObserver.onError(Status.INTERNAL diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageLayer.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageLayer.java index 5bb5a022f..7f61bc287 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageLayer.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageLayer.java @@ -22,6 +22,8 @@ import com.google.protobuf.Descriptors; import io.grpc.Status; import io.grpc.stub.StreamObserver; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; import org.springframework.dao.EmptyResultDataAccessException; import com.imageworks.spcue.LayerDetail; @@ -123,6 +125,8 @@ import com.imageworks.spcue.util.Convert; import com.imageworks.spcue.util.FrameSet; +import static com.imageworks.spcue.servant.ServantUtil.attemptChange; + public class ManageLayer extends LayerInterfaceGrpc.LayerInterfaceImplBase { private LayerDetail layer; @@ -135,6 +139,9 @@ public class ManageLayer extends LayerInterfaceGrpc.LayerInterfaceImplBase { private Whiteboard whiteboard; private LocalBookingSupport localBookingSupport; private FrameSearchFactory frameSearchFactory; + private final String property = "layer.finished_jobs_readonly"; + @Autowired + private Environment env; @Override public void findLayer(LayerFindLayerRequest request, StreamObserver responseObserver) { @@ -169,10 +176,12 @@ public void getLayer(LayerGetLayerRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - manageQueue.execute(new DispatchEatFrames(frameSearch, - new Source(request.toString()), jobManagerSupport)); - responseObserver.onNext(LayerEatFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + manageQueue.execute(new DispatchEatFrames(frameSearch, + new Source(request.toString()), jobManagerSupport)); + responseObserver.onNext(LayerEatFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override @@ -188,118 +197,142 @@ public void getFrames(LayerGetFramesRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - manageQueue.execute(new DispatchKillFrames(frameSearch, - new Source(request.toString()), jobManagerSupport)); - responseObserver.onNext(LayerKillFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + manageQueue.execute(new DispatchKillFrames(frameSearch, + new Source(request.toString()), jobManagerSupport)); + responseObserver.onNext(LayerKillFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void markdoneFrames(LayerMarkdoneFramesRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - manageQueue.execute(new DispatchSatisfyDepends(layer, jobManagerSupport)); - responseObserver.onNext(LayerMarkdoneFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + manageQueue.execute(new DispatchSatisfyDepends(layer, jobManagerSupport)); + responseObserver.onNext(LayerMarkdoneFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void retryFrames(LayerRetryFramesRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - manageQueue.execute(new DispatchRetryFrames(frameSearch, - new Source(request.toString()), jobManagerSupport)); - responseObserver.onNext(LayerRetryFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + manageQueue.execute(new DispatchRetryFrames(frameSearch, + new Source(request.toString()), jobManagerSupport)); + responseObserver.onNext(LayerRetryFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void setTags(LayerSetTagsRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - layerDao.updateLayerTags(layer, new HashSet<>(request.getTagsList())); - responseObserver.onNext(LayerSetTagsResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + layerDao.updateLayerTags(layer, new HashSet<>(request.getTagsList())); + responseObserver.onNext(LayerSetTagsResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void setMinCores(LayerSetMinCoresRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - jobManager.setLayerMinCores(layer, Convert.coresToCoreUnits(request.getCores())); - responseObserver.onNext(LayerSetMinCoresResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + jobManager.setLayerMinCores(layer, Convert.coresToCoreUnits(request.getCores())); + responseObserver.onNext(LayerSetMinCoresResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void setMinGpus(LayerSetMinGpusRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - jobManager.setLayerMinGpus(layer, request.getMinGpus()); - responseObserver.onNext(LayerSetMinGpusResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + jobManager.setLayerMinGpus(layer, request.getMinGpus()); + responseObserver.onNext(LayerSetMinGpusResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void setMinMemory(LayerSetMinMemoryRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - layerDao.updateLayerMinMemory(layer, request.getMemory()); - responseObserver.onNext(LayerSetMinMemoryResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + layerDao.updateLayerMinMemory(layer, request.getMemory()); + responseObserver.onNext(LayerSetMinMemoryResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void setMinGpuMemory(LayerSetMinGpuMemoryRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - layerDao.updateLayerMinGpuMemory(layer, request.getGpuMemory()); - responseObserver.onNext(LayerSetMinGpuMemoryResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + layerDao.updateLayerMinGpuMemory(layer, request.getGpuMemory()); + responseObserver.onNext(LayerSetMinGpuMemoryResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void createDependencyOnFrame(LayerCreateDependOnFrameRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - LayerOnFrame depend = new LayerOnFrame(layer, jobManager.getFrameDetail(request.getFrame().getId())); - dependManager.createDepend(depend); - responseObserver.onNext(LayerCreateDependOnFrameResponse.newBuilder() - .setDepend(whiteboard.getDepend(depend)) - .build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + LayerOnFrame depend = new LayerOnFrame(layer, jobManager.getFrameDetail(request.getFrame().getId())); + dependManager.createDepend(depend); + responseObserver.onNext(LayerCreateDependOnFrameResponse.newBuilder() + .setDepend(whiteboard.getDepend(depend)) + .build()); + responseObserver.onCompleted(); + } } @Override public void createDependencyOnJob(LayerCreateDependOnJobRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - LayerOnJob depend = new LayerOnJob(layer, jobManager.getJobDetail(request.getJob().getId())); - dependManager.createDepend(depend); - responseObserver.onNext(LayerCreateDependOnJobResponse.newBuilder() - .setDepend(whiteboard.getDepend(depend)) - .build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + LayerOnJob depend = new LayerOnJob(layer, jobManager.getJobDetail(request.getJob().getId())); + dependManager.createDepend(depend); + responseObserver.onNext(LayerCreateDependOnJobResponse.newBuilder() + .setDepend(whiteboard.getDepend(depend)) + .build()); + responseObserver.onCompleted(); + } } @Override public void createDependencyOnLayer(LayerCreateDependOnLayerRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - LayerOnLayer depend = new LayerOnLayer(layer, jobManager.getLayerDetail(request.getDependOnLayer().getId())); - dependManager.createDepend(depend); - responseObserver.onNext(LayerCreateDependOnLayerResponse.newBuilder() - .setDepend(whiteboard.getDepend(depend)) - .build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + LayerOnLayer depend = new LayerOnLayer(layer, jobManager.getLayerDetail(request.getDependOnLayer().getId())); + dependManager.createDepend(depend); + responseObserver.onNext(LayerCreateDependOnLayerResponse.newBuilder() + .setDepend(whiteboard.getDepend(depend)) + .build()); + responseObserver.onCompleted(); + } } @Override public void createFrameByFrameDependency(LayerCreateFrameByFrameDependRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - FrameByFrame depend = new FrameByFrame(layer, jobManager.getLayerDetail(request.getDependLayer().getId())); - dependManager.createDepend(depend); - responseObserver.onNext(LayerCreateFrameByFrameDependResponse.newBuilder() - .setDepend(whiteboard.getDepend(depend)) - .build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + FrameByFrame depend = new FrameByFrame(layer, jobManager.getLayerDetail(request.getDependLayer().getId())); + dependManager.createDepend(depend); + responseObserver.onNext(LayerCreateFrameByFrameDependResponse.newBuilder() + .setDepend(whiteboard.getDepend(depend)) + .build()); + responseObserver.onCompleted(); + } } @Override @@ -326,94 +359,112 @@ public void getWhatThisDependsOn(LayerGetWhatThisDependsOnRequest request, public void dropDepends(LayerDropDependsRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - manageQueue.execute(new DispatchDropDepends(layer, request.getTarget(), dependManager)); - responseObserver.onNext(LayerDropDependsResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + manageQueue.execute(new DispatchDropDepends(layer, request.getTarget(), dependManager)); + responseObserver.onNext(LayerDropDependsResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void dropLimit(LayerDropLimitRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - layerDao.dropLimit(layer, request.getLimitId()); - responseObserver.onNext(LayerDropLimitResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + layerDao.dropLimit(layer, request.getLimitId()); + responseObserver.onNext(LayerDropLimitResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void reorderFrames(LayerReorderFramesRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - manageQueue.execute(new DispatchReorderFrames(layer, new FrameSet(request.getRange()), request.getOrder(), - jobManagerSupport)); - responseObserver.onNext(LayerReorderFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + manageQueue.execute(new DispatchReorderFrames(layer, new FrameSet(request.getRange()), request.getOrder(), + jobManagerSupport)); + responseObserver.onNext(LayerReorderFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void staggerFrames(LayerStaggerFramesRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - manageQueue.execute(new DispatchStaggerFrames(layer, request.getRange(), request.getStagger(), - jobManagerSupport)); - responseObserver.onNext(LayerStaggerFramesResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + manageQueue.execute(new DispatchStaggerFrames(layer, request.getRange(), request.getStagger(), + jobManagerSupport)); + responseObserver.onNext(LayerStaggerFramesResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void setThreadable(LayerSetThreadableRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - layerDao.updateThreadable(layer, request.getThreadable()); - responseObserver.onNext(LayerSetThreadableResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + layerDao.updateThreadable(layer, request.getThreadable()); + responseObserver.onNext(LayerSetThreadableResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void setTimeout(LayerSetTimeoutRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - layerDao.updateTimeout(layer, request.getTimeout()); - responseObserver.onNext(LayerSetTimeoutResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + layerDao.updateTimeout(layer, request.getTimeout()); + responseObserver.onNext(LayerSetTimeoutResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void setTimeoutLLU(LayerSetTimeoutLLURequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - layerDao.updateTimeoutLLU(layer, request.getTimeoutLlu()); - responseObserver.onNext(LayerSetTimeoutLLUResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + layerDao.updateTimeoutLLU(layer, request.getTimeoutLlu()); + responseObserver.onNext(LayerSetTimeoutLLUResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void addLimit(LayerAddLimitRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - layerDao.addLimit(layer, request.getLimitId()); - responseObserver.onNext(LayerAddLimitResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + layerDao.addLimit(layer, request.getLimitId()); + responseObserver.onNext(LayerAddLimitResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void addRenderPartition(LayerAddRenderPartitionRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - LocalHostAssignment lha = new LocalHostAssignment(); - lha.setThreads(request.getThreads()); - lha.setMaxCoreUnits(request.getMaxCores() * 100); - lha.setMaxMemory(request.getMaxMemory()); - lha.setMaxGpuUnits(request.getMaxGpus()); - lha.setMaxGpuMemory(request.getMaxGpuMemory()); - lha.setType(RenderPartitionType.LAYER_PARTITION); - if (localBookingSupport.bookLocal(layer, request.getHost(), request.getUsername(), lha)) { - RenderPartition partition = whiteboard.getRenderPartition(lha); - responseObserver.onNext(LayerAddRenderPartitionResponse.newBuilder() - .setRenderPartition(partition) - .build()); - responseObserver.onCompleted(); - } else { - responseObserver.onError(Status.INTERNAL - .withDescription("Failed to find suitable frames.") - .asRuntimeException()); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + LocalHostAssignment lha = new LocalHostAssignment(); + lha.setThreads(request.getThreads()); + lha.setMaxCoreUnits(request.getMaxCores() * 100); + lha.setMaxMemory(request.getMaxMemory()); + lha.setMaxGpuUnits(request.getMaxGpus()); + lha.setMaxGpuMemory(request.getMaxGpuMemory()); + lha.setType(RenderPartitionType.LAYER_PARTITION); + if (localBookingSupport.bookLocal(layer, request.getHost(), request.getUsername(), lha)) { + RenderPartition partition = whiteboard.getRenderPartition(lha); + responseObserver.onNext(LayerAddRenderPartitionResponse.newBuilder() + .setRenderPartition(partition) + .build()); + responseObserver.onCompleted(); + } else { + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to find suitable frames.") + .asRuntimeException()); + } } } @@ -422,9 +473,11 @@ public void addRenderPartition(LayerAddRenderPartitionRequest request, public void registerOutputPath(LayerRegisterOutputPathRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - jobManager.registerLayerOutput(layer, request.getSpec()); - responseObserver.onNext(LayerRegisterOutputPathResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + jobManager.registerLayerOutput(layer, request.getSpec()); + responseObserver.onNext(LayerRegisterOutputPathResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override @@ -450,17 +503,21 @@ public void getOutputPaths(LayerGetOutputPathsRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - jobManager.enableMemoryOptimizer(layer, request.getValue()); - responseObserver.onNext(LayerEnableMemoryOptimizerResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + jobManager.enableMemoryOptimizer(layer, request.getValue()); + responseObserver.onNext(LayerEnableMemoryOptimizerResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override public void setMaxCores(LayerSetMaxCoresRequest request, StreamObserver responseObserver) { updateLayer(request.getLayer()); - jobManager.setLayerMaxCores(layer, Convert.coresToWholeCoreUnits(request.getCores())); - responseObserver.onNext(LayerSetMaxCoresResponse.newBuilder().build()); - responseObserver.onCompleted(); + if (attemptChange(env, property, jobManager, layer, responseObserver)) { + jobManager.setLayerMaxCores(layer, Convert.coresToWholeCoreUnits(request.getCores())); + responseObserver.onNext(LayerSetMaxCoresResponse.newBuilder().build()); + responseObserver.onCompleted(); + } } @Override diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ServantUtil.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ServantUtil.java index a5146947e..a88c8e87d 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ServantUtil.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ServantUtil.java @@ -21,10 +21,19 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; +import com.imageworks.spcue.JobDetail; +import com.imageworks.spcue.JobInterface; import com.imageworks.spcue.LayerInterface; +import com.imageworks.spcue.grpc.job.JobState; import com.imageworks.spcue.grpc.job.Layer; import com.imageworks.spcue.grpc.job.LayerSeq; +import com.imageworks.spcue.service.JobManager; + +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import org.springframework.core.env.Environment; public class ServantUtil { @@ -44,5 +53,24 @@ public static List convertLayerFilterList(LayerSeq layers) { } return result; } + + private static boolean isJobFinished(Environment env, String property, JobManager jobManager, JobInterface job) { + if (env.getProperty(property, String.class) != null && + Objects.equals(env.getProperty(property, String.class), "true")) { + JobDetail jobDetail = jobManager.getJobDetail(job.getJobId()); + return jobDetail.state == JobState.FINISHED; + } + return false; + } + + public static boolean attemptChange(Environment env, String property, JobManager jobManager, JobInterface job, StreamObserver responseObserver) { + if (ServantUtil.isJobFinished(env, property, jobManager, job)) { + responseObserver.onError(Status.FAILED_PRECONDITION + .withDescription("Finished jobs are readonly") + .asRuntimeException()); + return false; + } + return true; + } } diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index 6d541bd00..7f35eb90c 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -138,3 +138,9 @@ smtp_host=smtp # These shows won't be deactivated by the scheduled tasks protected_shows=pipe,swtest,edu + +# These flags determine whether or not layers/frames will be readonly when job is finished. +# If flags are set as true, layers/frames cannot be retried, eaten, edited dependency on, etc. +# In order to toggle the same functionility on cuegui side, set flags in cue_resources.yaml +layer.finished_jobs_readonly=false +frame.finished_jobs_readonly=false \ No newline at end of file From cd0fc3d9c32c1af30efe02409c765b26e9947ce4 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Sat, 17 Sep 2022 23:29:19 -0400 Subject: [PATCH 09/21] TSC notes from Aug 17 and Sep 14. (#1200) --- tsc/meetings/2022-08-17.md | 39 +++++++++++++++++++++++++++ tsc/meetings/2022-09-14.md | 54 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 tsc/meetings/2022-08-17.md create mode 100644 tsc/meetings/2022-09-14.md diff --git a/tsc/meetings/2022-08-17.md b/tsc/meetings/2022-08-17.md new file mode 100644 index 000000000..af88fcbe5 --- /dev/null +++ b/tsc/meetings/2022-08-17.md @@ -0,0 +1,39 @@ +# OpenCue TSC Meeting Notes 17 Aug 2022 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Siggraph/OSD debrief + * Event attendance ~20 in room, 50-60 total + * Q: who is using opencue? + * Should we change the TSC meeting time? To account for EU participants. + * OpenCue on GKE + * Some peculiarities there as both OpenCue and GKE act as schedulers, in different ways but with some + overlapping features. + * How can we help decide which pods can be killed? + * Could maybe use metrics/prometheus data as a source. + * We should add a helm chart for the repo. + * Both kubernetes and OpenCue want to do bin packing, maybe we could find a way to help them play better + together. + * PyPI packages still a need + * Look at "poetry"? + * Protobuf compiling is an obstacle as it breaks modularity. + * Can we make compiled proto its own package that other components depend on? + * Not getting consistent responses to issues / threads. + * Currently this is best effort. + * TSC should do an issue/PR cleanout. + * Let's try to make a standard process to keep up with it. + * Finish setting up stale issue closer to help with cleanout. + * Bring up with TAC to see if other projects have similar issues or thoughts. +* M1 issues + * Having some difficulties running OpenCue on a local M1 macbook, which is primary development machine for an + increasing number of people. + * Need to upgrade some dependencies like protoc. + * Docker build issues due to architecture mismatch. + * PySide2 might not be supported anymore. + * Has VFX reference platform dealt with this? +* Unit test failures + * Coming from code that hasn't changed + * Possibly due to changes in ref platform docker images? + * More investigation needed. diff --git a/tsc/meetings/2022-09-14.md b/tsc/meetings/2022-09-14.md new file mode 100644 index 000000000..5b4df0279 --- /dev/null +++ b/tsc/meetings/2022-09-14.md @@ -0,0 +1,54 @@ +# OpenCue TSC Meeting Notes 14 Sep 2022 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Upgrade Cuebot dependencies + * M1 has issues with grpc and protoc. + * PR to upgrade, needs review: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1185 + * Will review soon. +* PySide qApp problem + * This is causing unit test / CI failures. + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1192 + * Fix in progress https://github.com/AcademySoftwareFoundation/OpenCue/pull/1193 +* PySide6 + * PySide2 is deprecated, wheels are no longer published for newer architectures like M1. + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1194 + * Fix in progress https://github.com/AcademySoftwareFoundation/OpenCue/pull/1195 + * Will need some help testing. + * VFX reference platform still uses Qt 5 and will until at least 2024. This may create an issue if we drop support + for PySide2 which depends on Qt 5. PySide6 uses Qt 6. + * Can the pyside2/6 versions coexist? + * We could also create a wrapper / compatibility layer. This will likely be useful in the future as PySide will + change again. + * OpenTimelineIO has dealt with the same issue. + * See Github issues: + * https://github.com/AcademySoftwareFoundation/OpenTimelineIO/issues/1215 + * https://github.com/AcademySoftwareFoundation/OpenTimelineIO/issues/1220 + * https://github.com/AcademySoftwareFoundation/OpenTimelineIO/issues/1261 + * Appears they've fixed it, fix looks very similar to our in-progress fix. + * Has the VFX reference platform dealt with architectures at all? arm is getting more popular, and the switch to arm + created/exposed these issues. +* Redis + * Considering adding Redis into the mix to help with Cuebot/database load and Cuebot/RQD communication issues. + * Draft PR: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1187 + * Brian has left some initial comments on that PR. + * This will break the connection between RQD and a single Cuebot. Currently RQD must communicate with a single + Cuebot via gRPC and must be restarted to switch to a different one. + * Using Redis is highly experimental, not production tested. + * The PR must be updated to be completely optional, coexist with the existing code. We need to collect performance + data and decide whether we want to proceed with this project. + * Redis should probably be optional in perpetuity. Default install should not require it, but it can be used to + increase performance for larger deployments. +* Docker image issues + * Can't build on M1 mac. + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1196 + * Also slowing down development. +* Need to do a release + * Blocked on qApp issue as this causes CI failures, we can't use our release pipeline. + * Any update on integration test script? + * This would help speed up the release by automating the QA procedure. + * There was a basic script that had some issues, not much progress. + * Work may resume soon. + * If the in-progress script is sharable, that would save some time. From bce47b590fa097c577ba694fb69f985d54251561 Mon Sep 17 00:00:00 2001 From: Mark Sisson <5761292+marksisson@users.noreply.github.com> Date: Tue, 20 Sep 2022 14:51:11 -0500 Subject: [PATCH 10/21] Add .envrc to .gitignore (#1207) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9de17bdf1..12d65dce4 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ venv*/ coverage.xml htmlcov/ /.env +.envrc From 3bb4528472287ad802de478c0f734d51c5d3c3ca Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Thu, 29 Sep 2022 13:36:15 -0400 Subject: [PATCH 11/21] Upgrade gRPC and protobuf dependencies. (#1185) --- VERSION.in | 2 +- cuebot/build.gradle | 8 ++++---- requirements.txt | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/VERSION.in b/VERSION.in index 9f4eca259..5320adc1c 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.20 +0.21 diff --git a/cuebot/build.gradle b/cuebot/build.gradle index 403cc893e..fe5abf3e2 100644 --- a/cuebot/build.gradle +++ b/cuebot/build.gradle @@ -44,7 +44,7 @@ dependencies { compile group: 'com.google.guava', name: 'guava', version: '26.0-android' compile group: 'com.sun.mail', name: 'mailapi', version: '1.5.4' compile group: 'commons-lang', name: 'commons-lang', version: '2.6' - compile group: 'io.grpc', name: 'grpc-all', version: '1.36.2' + compile group: 'io.grpc', name: 'grpc-all', version: '1.47.0' compile group: 'org.apache.activemq', name: 'activemq-pool', version: activemqVersion compile group: 'org.apache.velocity', name: 'velocity', version: '1.7' compile group: 'org.jdom', name: 'jdom', version: '1.1.3' @@ -54,7 +54,7 @@ dependencies { compile group: 'org.springframework', name: 'spring-jms' compile group: 'org.quartz-scheduler', name: 'quartz', version: '2.2.1', { exclude group: 'c3p0', module: 'c3p0' } compile group: 'org.postgresql', name: 'postgresql', version: '42.2.2' - compile group: 'com.google.protobuf', name: 'protobuf-java', version: '3.13.0' + compile group: 'com.google.protobuf', name: 'protobuf-java', version: '3.21.2' compile group: 'log4j', name: 'log4j', version: '1.2.17' compile group: 'org.slf4j', name: 'slf4j-log4j12', version: '1.7.26' @@ -81,12 +81,12 @@ compileTestJava { protobuf { protoc { // The protoc compiler - artifact = 'com.google.protobuf:protoc:3.13.0' + artifact = 'com.google.protobuf:protoc:3.21.2' } plugins { grpc { // Generate gRPC stubs. - artifact = 'io.grpc:protoc-gen-grpc-java:1.7.0' + artifact = 'io.grpc:protoc-gen-grpc-java:1.47.0' } } generateProtoTasks { diff --git a/requirements.txt b/requirements.txt index ff5a63761..f465c5910 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,8 +4,8 @@ future==0.17.1 futures==3.2.0;python_version<"3.0" grpcio==1.26.0;python_version<"3.0" grpcio-tools==1.26.0;python_version<"3.0" -grpcio==1.39.0;python_version>="3.0" -grpcio-tools==1.39.0;python_version>="3.0" +grpcio==1.47.0;python_version>="3.0" +grpcio-tools==1.47.0;python_version>="3.0" mock==2.0.0 packaging==20.9 pathlib==1.0.1;python_version<"3.4" From 3859a5da29f98ed426d0e5b3ca312a4be02fec61 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Thu, 29 Sep 2022 13:37:21 -0400 Subject: [PATCH 12/21] [rqd] Reduce the log level when failing to read a stat file. (#1191) --- rqd/rqd/rqmachine.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index 37e1f5771..d4cf918ca 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -283,7 +283,9 @@ def rssUpdate(self, frames): # pylint: disable=broad-except except (OSError, IOError): - log.exception('Failed to read stat/statm file for pid %s', pid) + # Many Linux processes are ephemeral and will disappear before we're able + # to read them. This is not typically indicative of a problem. + log.debug('Failed to read stat/statm file for pid %s', pid) # pylint: disable=too-many-nested-blocks try: From a6ceb2fe3b66aaf7e84fb1af77b994264da95830 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Thu, 29 Sep 2022 13:53:27 -0400 Subject: [PATCH 13/21] Update Dockerfile base images to fix compatibility issues. (#1198) --- cueadmin/Dockerfile | 62 ++++++++------------------------------------ cuebot/Dockerfile | 16 +++--------- cuegui/Dockerfile | 30 ++++----------------- cuesubmit/Dockerfile | 45 +++++--------------------------- pycue/Dockerfile | 56 +++++++-------------------------------- pyoutline/Dockerfile | 58 ++++++++--------------------------------- rqd/Dockerfile | 15 +++-------- 7 files changed, 49 insertions(+), 233 deletions(-) diff --git a/cueadmin/Dockerfile b/cueadmin/Dockerfile index 37caf1d5f..74b318d46 100644 --- a/cueadmin/Dockerfile +++ b/cueadmin/Dockerfile @@ -1,35 +1,14 @@ -# ----------------- -# BUILD -# ----------------- -FROM centos:7 as build - -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "CueAdmin build stage" +FROM python:3.9.14 WORKDIR /src -RUN yum -y install \ - epel-release \ - gcc \ - python-devel - -RUN yum -y install \ - python-pip \ - python36 \ - python36-devel \ - python36-pip - -RUN python -m pip install --upgrade 'pip<21' -RUN python3.6 -m pip install --upgrade pip - -RUN python -m pip install --upgrade 'setuptools<45' -RUN python3.6 -m pip install --upgrade setuptools +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools COPY LICENSE ./ COPY requirements.txt ./ -RUN python -m pip install -r requirements.txt -RUN python3.6 -m pip install -r requirements.txt +RUN python3 -m pip install -r requirements.txt COPY proto/ ./proto COPY pycue/README.md ./pycue/ @@ -37,7 +16,7 @@ COPY pycue/setup.py ./pycue/ COPY pycue/opencue ./pycue/opencue COPY pycue/FileSequence ./pycue/FileSequence -RUN python -m grpc_tools.protoc \ +RUN python3 -m grpc_tools.protoc \ -I=./proto \ --python_out=./pycue/opencue/compiled_proto \ --grpc_python_out=./pycue/opencue/compiled_proto \ @@ -55,32 +34,13 @@ COPY cueadmin/cueadmin ./cueadmin/cueadmin COPY VERSION.in VERSIO[N] ./ RUN test -e VERSION || echo "$(cat VERSION.in)-custom" | tee VERSION -RUN cd pycue && python setup.py install - -RUN cd pycue && python3.6 setup.py install - -# TODO(bcipriano) Lint the code here. (Issue #78) - -RUN cd cueadmin && python setup.py test - -RUN cd cueadmin && python3.6 setup.py test +RUN cd pycue && python3 setup.py install +RUN cd cueadmin && python3 setup.py test RUN cp LICENSE requirements.txt VERSION cueadmin/ RUN versioned_name="cueadmin-$(cat ./VERSION)-all" \ - && mv cueadmin $versioned_name \ - && tar -cvzf $versioned_name.tar.gz $versioned_name/* - - -# ----------------- -# RUN -# ----------------- -FROM centos:7 - -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "CueAdmin runtime stage" - -WORKDIR /opt/opencue - -COPY --from=build /src/cueadmin-*-all.tar.gz ./ - + && mv cueadmin "${versioned_name}" \ + && tar -cvzf "${versioned_name}.tar.gz" ${versioned_name}/* \ + && mkdir -p /opt/opencue \ + && cp "${versioned_name}.tar.gz" /opt/opencue/ diff --git a/cuebot/Dockerfile b/cuebot/Dockerfile index 2dd5ae960..d82b861cc 100644 --- a/cuebot/Dockerfile +++ b/cuebot/Dockerfile @@ -1,19 +1,16 @@ # ----------------- # BUILD # ----------------- -FROM gradle:6.0.1-jdk11 AS build +FROM gradle:6.0.1-jdk13 AS build USER gradle -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "Cuebot build stage" - COPY --chown=gradle:gradle ./cuebot /home/gradle/cuebot/ COPY --chown=gradle:gradle ./proto /home/gradle/proto/ WORKDIR /home/gradle/cuebot -RUN gradle build --stacktrace +RUN gradle build --info --stacktrace COPY --chown=gradle:gradle VERSION.in VERSIO[N] ./ RUN test -e VERSION || echo "$(cat VERSION.in)-custom" | tee VERSION @@ -25,12 +22,8 @@ RUN mv ./build/libs/cuebot.jar ./build/libs/cuebot-$(cat ./VERSION)-all.jar # ----------------- FROM jc21/rpmbuild-centos7:latest AS rpm -# Random first line after from - USER rpmbuilder -RUN echo "Cuebot RPM Stage" - COPY --chown=rpmbuilder:rpmbuilder LICENSE ./ COPY --from=build \ @@ -49,10 +42,7 @@ RUN chmod +x create_rpm.sh && ./create_rpm.sh cuebot "$(cat VERSION)" # ----------------- # RUN # ----------------- -FROM openjdk:11-jre-slim-buster - -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "Cuebot runtime stage" +FROM openjdk:18-slim-buster ARG CUEBOT_GRPC_CUE_PORT=8443 ARG CUEBOT_GRPC_RQD_PORT=8444 diff --git a/cuegui/Dockerfile b/cuegui/Dockerfile index f3cd788ce..8b0b189f9 100644 --- a/cuegui/Dockerfile +++ b/cuegui/Dockerfile @@ -1,10 +1,4 @@ -# ----------------- -# BUILD -# ----------------- -FROM centos:7 as build - -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "CueGUI build stage" +FROM --platform=linux/x86_64 centos:7 WORKDIR /src @@ -26,13 +20,11 @@ RUN yum -y install \ xcb-util-wm.x86_64 RUN yum -y install \ - python-pip \ python36 \ python36-devel \ python36-pip RUN python3.6 -m pip install --upgrade pip - RUN python3.6 -m pip install --upgrade setuptools RUN dbus-uuidgen > /etc/machine-id @@ -74,19 +66,7 @@ RUN cd cuegui && xvfb-run -d python3.6 setup.py test RUN cp LICENSE requirements.txt requirements_gui.txt VERSION cuegui/ RUN versioned_name="cuegui-$(cat ./VERSION)-all" \ - && mv cuegui $versioned_name \ - && tar -cvzf $versioned_name.tar.gz $versioned_name/* - - -# ----------------- -# RUN -# ----------------- -FROM centos:7 - -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "CueGUI runtime stage" - -WORKDIR /opt/opencue - -COPY --from=build /src/cuegui-*-all.tar.gz ./ - + && mv cuegui "${versioned_name}" \ + && tar -cvzf "${versioned_name}.tar.gz" ${versioned_name}/* \ + && mkdir -p /opt/opencue \ + && cp "${versioned_name}.tar.gz" /opt/opencue/ diff --git a/cuesubmit/Dockerfile b/cuesubmit/Dockerfile index 252936fa9..cc8f51178 100644 --- a/cuesubmit/Dockerfile +++ b/cuesubmit/Dockerfile @@ -1,10 +1,4 @@ -# ----------------- -# BUILD -# ----------------- -FROM centos:7 as build - -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "CueSubmit build stage" +FROM --platform=linux/x86_64 centos:7 WORKDIR /src @@ -15,22 +9,17 @@ RUN yum -y install \ python-devel RUN yum -y install \ - python-pip \ python36 \ python36-devel \ python36-pip -RUN python -m pip install --upgrade 'pip<21' RUN python3.6 -m pip install --upgrade pip - -RUN python -m pip install --upgrade 'setuptools<45' RUN python3.6 -m pip install --upgrade setuptools COPY LICENSE ./ COPY requirements.txt ./ COPY requirements_gui.txt ./ -RUN python -m pip install -r requirements.txt -r requirements_gui.txt RUN python3.6 -m pip install -r requirements.txt -r requirements_gui.txt COPY proto/ ./proto @@ -39,7 +28,7 @@ COPY pycue/setup.py ./pycue/ COPY pycue/opencue ./pycue/opencue COPY pycue/FileSequence ./pycue/FileSequence -RUN python -m grpc_tools.protoc \ +RUN python3.6 -m grpc_tools.protoc \ -I=./proto \ --python_out=./pycue/opencue/compiled_proto \ --grpc_python_out=./pycue/opencue/compiled_proto \ @@ -65,36 +54,14 @@ COPY cuesubmit/cuesubmit ./cuesubmit/cuesubmit COPY VERSION.in VERSIO[N] ./ RUN test -e VERSION || echo "$(cat VERSION.in)-custom" | tee VERSION -RUN cd pycue && python setup.py install - RUN cd pycue && python3.6 setup.py install - -RUN cd pyoutline && python setup.py install - RUN cd pyoutline && python3.6 setup.py install - -# TODO(bcipriano) Lint the code here. (Issue #78) - -RUN cd cuesubmit && python setup.py test - RUN cd cuesubmit && python3.6 setup.py test RUN cp LICENSE requirements.txt requirements_gui.txt VERSION cuesubmit/ RUN versioned_name="cuesubmit-$(cat ./VERSION)-all" \ - && mv cuesubmit $versioned_name \ - && tar -cvzf $versioned_name.tar.gz $versioned_name/* - - -# ----------------- -# RUN -# ----------------- -FROM centos:7 - -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "CueSubmit runtime stage" - -WORKDIR /opt/opencue - -COPY --from=build /src/cuesubmit-*-all.tar.gz ./ - + && mv cuesubmit "${versioned_name}" \ + && tar -cvzf "${versioned_name}.tar.gz" ${versioned_name}/* \ + && mkdir -p /opt/opencue \ + && cp "${versioned_name}.tar.gz" /opt/opencue/ diff --git a/pycue/Dockerfile b/pycue/Dockerfile index 54f8a2dc2..e93f9526a 100644 --- a/pycue/Dockerfile +++ b/pycue/Dockerfile @@ -1,35 +1,14 @@ -# ----------------- -# BUILD -# ----------------- -FROM centos:7 as build - -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "PyCue build stage" +FROM python:3.9.14 WORKDIR /src -RUN yum -y install \ - epel-release \ - gcc \ - python-devel - -RUN yum -y install \ - python-pip \ - python36 \ - python36-devel \ - python36-pip - -RUN python -m pip install --upgrade 'pip<21' -RUN python3.6 -m pip install --upgrade pip - -RUN python -m pip install --upgrade 'setuptools<45' -RUN python3.6 -m pip install --upgrade setuptools +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools COPY LICENSE ./ COPY requirements.txt ./ -RUN python -m pip install -r requirements.txt -RUN python3.6 -m pip install -r requirements.txt +RUN python3 -m pip install -r requirements.txt COPY proto/ ./proto COPY pycue/README.md ./pycue/ @@ -38,7 +17,7 @@ COPY pycue/FileSequence ./pycue/FileSequence COPY pycue/tests/ ./pycue/tests COPY pycue/opencue ./pycue/opencue -RUN python -m grpc_tools.protoc \ +RUN python3 -m grpc_tools.protoc \ -I=./proto \ --python_out=./pycue/opencue/compiled_proto \ --grpc_python_out=./pycue/opencue/compiled_proto \ @@ -48,30 +27,15 @@ RUN python -m grpc_tools.protoc \ # for more info. RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py -# TODO(bcipriano) Lint the code here. (Issue #78) - COPY VERSION.in VERSIO[N] ./ RUN test -e VERSION || echo "$(cat VERSION.in)-custom" | tee VERSION -RUN cd pycue && python setup.py test - -RUN cd pycue && python3.6 setup.py test +RUN cd pycue && python3 setup.py test RUN cp LICENSE requirements.txt VERSION pycue/ RUN versioned_name="pycue-$(cat ./VERSION)-all" \ - && mv pycue $versioned_name \ - && tar -cvzf $versioned_name.tar.gz $versioned_name/* - - -# ----------------- -# RUN -# ----------------- -FROM centos:7 - -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "PyCue runtime stage" - -WORKDIR /opt/opencue - -COPY --from=build /src/pycue-*-all.tar.gz ./ + && mv pycue "${versioned_name}" \ + && tar -cvzf "${versioned_name}.tar.gz" ${versioned_name}/* \ + && mkdir /opt/opencue \ + && cp "${versioned_name}.tar.gz" /opt/opencue/ diff --git a/pyoutline/Dockerfile b/pyoutline/Dockerfile index fb24d1136..e898452e1 100644 --- a/pyoutline/Dockerfile +++ b/pyoutline/Dockerfile @@ -1,35 +1,14 @@ -# ----------------- -# BUILD -# ----------------- -FROM centos:7 as build - -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "PyCue build stage" +FROM python:3.9.14 WORKDIR /src -RUN yum -y install \ - epel-release \ - gcc \ - python-devel - -RUN yum -y install \ - python-pip \ - python36 \ - python36-devel \ - python36-pip - -RUN python -m pip install --upgrade 'pip<21' -RUN python3.6 -m pip install --upgrade pip - -RUN python -m pip install --upgrade 'setuptools<45' -RUN python3.6 -m pip install --upgrade setuptools +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools COPY LICENSE ./ COPY requirements.txt ./ -RUN python -m pip install -r requirements.txt -RUN python3.6 -m pip install -r requirements.txt +RUN python3 -m pip install -r requirements.txt COPY proto/ ./proto COPY pycue/README.md ./pycue/ @@ -37,7 +16,7 @@ COPY pycue/setup.py ./pycue/ COPY pycue/opencue ./pycue/opencue COPY pycue/FileSequence ./pycue/FileSequence -RUN python -m grpc_tools.protoc \ +RUN python3 -m grpc_tools.protoc \ -I=./proto \ --python_out=./pycue/opencue/compiled_proto \ --grpc_python_out=./pycue/opencue/compiled_proto \ @@ -58,28 +37,13 @@ COPY pyoutline/outline ./pyoutline/outline COPY VERSION.in VERSIO[N] ./ RUN test -e VERSION || echo "$(cat VERSION.in)-custom" | tee VERSION -RUN cd pycue && python setup.py install -RUN cd pycue && python3.6 setup.py install - -RUN cd pyoutline && python setup.py test -RUN cd pyoutline && python3.6 setup.py test +RUN cd pycue && python3 setup.py install +RUN cd pyoutline && python3 setup.py test RUN cp LICENSE requirements.txt VERSION pyoutline/ RUN versioned_name="pyoutline-$(cat ./VERSION)-all" \ - && mv pyoutline $versioned_name \ - && tar -cvzf $versioned_name.tar.gz $versioned_name/* - - -# ----------------- -# RUN -# ----------------- -FROM centos:7 - -# First line after FROM should be unique to avoid --cache-from weirdness. -RUN echo "PyCue runtime stage" - -WORKDIR /opt/opencue - -COPY --from=build /src/pyoutline-*-all.tar.gz ./ - + && mv pyoutline "${versioned_name}" \ + && tar -cvzf "${versioned_name}.tar.gz" ${versioned_name}/* \ + && mkdir -p /opt/opencue \ + && cp "${versioned_name}.tar.gz" /opt/opencue/ diff --git a/rqd/Dockerfile b/rqd/Dockerfile index fd772c59e..15f5ac21c 100644 --- a/rqd/Dockerfile +++ b/rqd/Dockerfile @@ -1,4 +1,6 @@ -FROM centos:7 +FROM --platform=linux/x86_64 centos:7 + +WORKDIR /opt/opencue RUN yum -y install \ epel-release \ @@ -7,23 +9,16 @@ RUN yum -y install \ time RUN yum -y install \ - python-pip \ python36 \ python36-devel \ python36-pip -RUN python -m pip install --upgrade 'pip<21' RUN python3.6 -m pip install --upgrade pip - -RUN python -m pip install --upgrade 'setuptools<45' RUN python3.6 -m pip install --upgrade setuptools -WORKDIR /opt/opencue - COPY LICENSE ./ COPY requirements.txt ./ -RUN python -m pip install -r requirements.txt RUN python3.6 -m pip install -r requirements.txt COPY proto/ ./proto @@ -43,13 +38,9 @@ RUN python3.6 -m grpc_tools.protoc \ # for more info. RUN 2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py -# TODO(bcipriano) Lint the code here. (Issue #78) - COPY VERSION.in VERSIO[N] ./ RUN test -e VERSION || echo "$(cat VERSION.in)-custom" | tee VERSION -# Test in Python 2 and 3, but only install in the Python 3 environment. -RUN cd rqd && python setup.py test RUN cd rqd && python3.6 setup.py test RUN cd rqd && python3.6 setup.py install From d458bc2d2f2f3dc8867547e5d4240a8c0f9ea537 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Thu, 29 Sep 2022 14:27:30 -0400 Subject: [PATCH 14/21] [pyoutline] Replace use of execfile() with exec(). (#1201) --- pyoutline/outline/loader.py | 5 +++-- pyoutline/outline/modules/shell.py | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pyoutline/outline/loader.py b/pyoutline/outline/loader.py index 3db6b2c4f..0b0d43a18 100644 --- a/pyoutline/outline/loader.py +++ b/pyoutline/outline/loader.py @@ -25,7 +25,6 @@ import os import logging import json -from past.builtins import execfile import time import uuid import yaml @@ -173,7 +172,9 @@ def parse_outline_script(path): """ try: logger.info("parsing outline file %s", path) - execfile(path, {}) + with open(path) as fp: + code = compile(fp.read(), path, 'exec') + exec(code) # pylint: disable=exec-used except Exception as exp: logger.warning("failed to parse as python file, %s", exp) raise outline.exception.OutlineException( diff --git a/pyoutline/outline/modules/shell.py b/pyoutline/outline/modules/shell.py index e564e2162..482fcda4a 100644 --- a/pyoutline/outline/modules/shell.py +++ b/pyoutline/outline/modules/shell.py @@ -22,7 +22,6 @@ import logging import os -from past.builtins import execfile import outline.layer import outline.util @@ -56,7 +55,10 @@ def _setup(self): self.__code = None def _execute(self, frames): - execfile(self.get_file("script")) + path = self.get_file("script") + with open(path) as fp: + code = compile(fp.read(), path, 'exec') + exec(code) # pylint: disable=exec-used class Shell(outline.layer.Layer): From 962b22701c4ff76a503fe60639d7478629c032a4 Mon Sep 17 00:00:00 2001 From: Diego Tavares da Silva Date: Thu, 29 Sep 2022 11:49:31 -0700 Subject: [PATCH 15/21] [cuebot] Add a new property max_show_stale_days to control show expiration rule. (#1208) --- .../spcue/dao/postgres/ShowDaoJdbc.java | 38 +++++++------------ cuebot/src/main/resources/opencue.properties | 9 ++++- 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java index 2c4182122..5d674ed82 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ShowDaoJdbc.java @@ -237,32 +237,22 @@ public void updateShowCommentEmail(ShowInterface s, String[] email) { @Override public void updateShowsStatus() { - Stream protectedShowsRaw = Arrays.stream(env.getProperty("protected_shows", String.class).split(",")); + Stream protectedShowsRaw = Arrays + .stream(env.getProperty("protected_shows", String.class, "").split(",")); String protectedShows = protectedShowsRaw.map(show -> "'" + show + "'") .collect(Collectors.joining(",")); - getJdbcTemplate().update("UPDATE " + - "show " + - "SET " + - "b_active=false " + - "WHERE " + - "pk_show NOT IN (" + - "SELECT " + - "pk_show " + - "FROM (" + - "SELECT " + - "pk_show, count(pk_job) " + - "FROM " + - "job_history " + - "WHERE " + - "(DATE_PART('days', NOW()) - DATE_PART('days', dt_last_modified)) < 30 " + - "GROUP BY " + - "pk_show " + - "HAVING COUNT(pk_job)>0 " + - ") pk_show" + - ") " + - "AND " + - "str_name NOT IN (?)", - protectedShows); + int maxShowStaleDays = env.getProperty("max_show_stale_days", Integer.class, -1); + + if (maxShowStaleDays > 0) { + getJdbcTemplate().update("UPDATE show SET b_active=false " + + "WHERE pk_show NOT IN (SELECT pk_show " + + " FROM (SELECT pk_show, count(pk_job) FROM job_history " + + " WHERE " + + " (DATE_PART('days', NOW()) - DATE_PART('days', dt_last_modified)) < ? " + + "GROUP BY pk_show HAVING COUNT(pk_job) > 0) pk_show) " + + " AND str_name NOT IN (?)", + maxShowStaleDays, protectedShows); + } } @Override diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index 7f35eb90c..145bbb352 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -136,8 +136,13 @@ maintenance.auto_delete_down_hosts=false # Set hostname/IP of the smtp host. Will be used for mailing smtp_host=smtp -# These shows won't be deactivated by the scheduled tasks -protected_shows=pipe,swtest,edu +# Flags related to a job that runs periodically to deactivate shows that haven't been +# receiving jobs. +# A comma separated list of shows that won't be deactivated by the scheduled tasks +protected_shows=testing +# Number of days a show needs to be stale before it gets deactivated. +# -1 means shows should not get deactivated at all. +max_show_stale_days=-1 # These flags determine whether or not layers/frames will be readonly when job is finished. # If flags are set as true, layers/frames cannot be retried, eaten, edited dependency on, etc. From 234a9cef483bbf35599841ce7771609adc7d6da2 Mon Sep 17 00:00:00 2001 From: Akim Ruslanov Date: Tue, 25 Oct 2022 11:30:32 -0700 Subject: [PATCH 16/21] [pycue] Add missing exception parsing to setAllocation. (#1172) --- pycue/opencue/wrappers/host.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pycue/opencue/wrappers/host.py b/pycue/opencue/wrappers/host.py index aa54d44d0..860847bb0 100644 --- a/pycue/opencue/wrappers/host.py +++ b/pycue/opencue/wrappers/host.py @@ -19,6 +19,7 @@ import time from opencue import Cuebot +from opencue import util from opencue.compiled_proto import comment_pb2 from opencue.compiled_proto import host_pb2 import opencue.wrappers.comment @@ -149,6 +150,7 @@ def renameTag(self, oldTag, newTag): host_pb2.HostRenameTagRequest(host=self.data, old_tag=oldTag, new_tag=newTag), timeout=Cuebot.Timeout) + @util.grpcExceptionParser def setAllocation(self, allocation): """Sets the host to the given allocation. From 3742ff0281d9ff61a411a018bde8ab3843c7b007 Mon Sep 17 00:00:00 2001 From: Olivier Evers Date: Thu, 27 Oct 2022 18:03:25 +0200 Subject: [PATCH 17/21] [rqd] On Windows, pass environment and properly escape command chars. (#1215) --- rqd/rqd/rqcore.py | 22 ++++++++++++++++++---- rqd/rqd/rqmachine.py | 2 ++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index cb74f9d34..646c4f73d 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -83,8 +83,6 @@ def __createEnvVariables(self): self.frameEnv["TZ"] = self.rqCore.machine.getTimezone() self.frameEnv["USER"] = self.runFrame.user_name self.frameEnv["LOGNAME"] = self.runFrame.user_name - self.frameEnv["MAIL"] = "/usr/mail/%s" % self.runFrame.user_name - self.frameEnv["HOME"] = "/net/homedirs/%s" % self.runFrame.user_name self.frameEnv["mcp"] = "1" self.frameEnv["show"] = self.runFrame.show self.frameEnv["shot"] = self.runFrame.shot @@ -99,8 +97,18 @@ def __createEnvVariables(self): self.frameEnv["CUE_GPU_MEMORY"] = str(self.rqCore.machine.getGpuMemoryFree()) self.frameEnv["SP_NOMYCSHRC"] = "1" - for key in self.runFrame.environment: - self.frameEnv[key] = self.runFrame.environment[key] + if platform.system() in ("Linux", "Darwin"): + self.frameEnv["MAIL"] = "/usr/mail/%s" % self.runFrame.user_name + self.frameEnv["HOME"] = "/net/homedirs/%s" % self.runFrame.user_name + elif platform.system() == "Windows": + self.frameEnv["APPDATA"] = os.environ["APPDATA"] + self.frameEnv["SYSTEMROOT"] = os.environ["SYSTEMROOT"] + + for key, value in self.runFrame.environment.items(): + if key == 'PATH': + self.frameEnv[key] += os.pathsep + value + else: + self.frameEnv[key] = value # Add threads to use all assigned hyper-threading cores if 'CPU_LIST' in self.runFrame.attributes and 'CUE_THREADS' in self.frameEnv: @@ -128,6 +136,11 @@ def _createCommandFile(self, command): except OSError: pass # okay, already exists + # Windows Batch needs some characters escaped: + command = command.replace('%', '%%') + for char in '^&<>|': + command = command.replace(char, '^' + char) + commandFile = os.path.join( rqd_tmp_dir, 'cmd-%s-%s.bat' % (self.runFrame.frame_id, time.time())) @@ -358,6 +371,7 @@ def runWindows(self): tempCommand = [self._createCommandFile(runFrame.command)] frameInfo.forkedCommand = subprocess.Popen(tempCommand, + env=self.frameEnv, stdin=subprocess.PIPE, stdout=self.rqlog, stderr=self.rqlog) diff --git a/rqd/rqd/rqmachine.py b/rqd/rqd/rqmachine.py index d4cf918ca..27295ed92 100644 --- a/rqd/rqd/rqmachine.py +++ b/rqd/rqd/rqmachine.py @@ -509,6 +509,8 @@ def getPathEnv(self): """Returns the correct path environment for the given machine""" if platform.system() == 'Linux': return '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + if platform.system() == 'Windows': + return 'C:/Windows/system32;C:/Windows;C:/Windows/System32/Wbem' return '' @rqd.rqutil.Memoize From 6500fe2859a52641774499eda820c154b557f696 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Thu, 27 Oct 2022 12:08:28 -0400 Subject: [PATCH 18/21] [cuegui] Replace qApp with a new cuegui.App library. (#1193) --- cuegui/cuegui/AbstractTreeWidget.py | 19 ++--- cuegui/cuegui/AbstractWidgetItem.py | 1 + cuegui/cuegui/App.py | 78 +++++++++++++++++++ cuegui/cuegui/Comments.py | 11 +-- cuegui/cuegui/CueJobMonitorTree.py | 6 +- cuegui/cuegui/CueStateBarWidget.py | 5 +- cuegui/cuegui/DarkPalette.py | 11 ++- cuegui/cuegui/Exception.py | 35 +++++++++ cuegui/cuegui/FrameMonitorTree.py | 27 +++---- cuegui/cuegui/HostMonitor.py | 14 +--- cuegui/cuegui/HostMonitorTree.py | 12 +-- cuegui/cuegui/JobMonitorTree.py | 14 +--- cuegui/cuegui/LayerDialog.py | 6 +- cuegui/cuegui/LayerMonitorTree.py | 3 +- cuegui/cuegui/LimitsWidget.py | 7 +- cuegui/cuegui/Main.py | 32 ++------ cuegui/cuegui/MainWindow.py | 39 +++------- cuegui/cuegui/MenuActions.py | 27 +++---- cuegui/cuegui/Plugins.py | 23 ++---- cuegui/cuegui/PreviewWidget.py | 7 +- cuegui/cuegui/ProcMonitor.py | 14 ++-- cuegui/cuegui/ProcMonitorTree.py | 13 +--- cuegui/cuegui/ProgressDialog.py | 11 +-- cuegui/cuegui/Redirect.py | 3 +- cuegui/cuegui/ShowsWidget.py | 7 +- cuegui/cuegui/Style.py | 10 +-- cuegui/cuegui/SubscriptionsWidget.py | 6 +- cuegui/cuegui/ThreadPool.py | 6 +- cuegui/cuegui/Utils.py | 7 +- cuegui/cuegui/__init__.py | 8 ++ cuegui/cuegui/plugins/AllocationsPlugin.py | 7 +- cuegui/cuegui/plugins/AttributesPlugin.py | 16 ++-- cuegui/cuegui/plugins/LogViewPlugin.py | 11 +-- cuegui/cuegui/plugins/MonitorCuePlugin.py | 14 +--- .../cuegui/plugins/MonitorJobDetailsPlugin.py | 9 +-- cuegui/cuegui/plugins/MonitorJobsPlugin.py | 6 +- cuegui/cuegui/plugins/ServicePlugin.py | 6 +- cuegui/cuegui/plugins/StuckFramePlugin.py | 19 +++-- cuegui/tests/CueJobMonitorTree_tests.py | 4 +- cuegui/tests/DependWizard_tests.py | 4 +- cuegui/tests/FilterDialog_tests.py | 12 +-- cuegui/tests/FrameMonitorTree_tests.py | 5 +- cuegui/tests/LayerDialog_tests.py | 4 +- cuegui/tests/MenuActions_tests.py | 44 +++++++---- cuegui/tests/Redirect_tests.py | 4 +- cuegui/tests/UnbookDialog_tests.py | 12 +-- cuegui/tests/plugins/LogViewPlugin_tests.py | 16 ++-- cuegui/tests/test_utils.py | 13 +--- 48 files changed, 328 insertions(+), 340 deletions(-) create mode 100644 cuegui/cuegui/App.py create mode 100644 cuegui/cuegui/Exception.py diff --git a/cuegui/cuegui/AbstractTreeWidget.py b/cuegui/cuegui/AbstractTreeWidget.py index 96250453e..88e98ad4f 100644 --- a/cuegui/cuegui/AbstractTreeWidget.py +++ b/cuegui/cuegui/AbstractTreeWidget.py @@ -75,6 +75,7 @@ def __init__(self, parent): @type parent: QWidget @param parent: The widget to set as the parent""" QtWidgets.QTreeWidget.__init__(self, parent) + self.app = cuegui.app() self._items = {} self._lastUpdate = 0 @@ -104,8 +105,8 @@ def __init__(self, parent): self.itemClicked.connect(self.__itemSingleClickedEmitToApp) self.itemDoubleClicked.connect(self.__itemDoubleClickedEmitToApp) self._timer.timeout.connect(self.updateRequest) - QtGui.qApp.request_update.connect(self.updateRequest) # pylint: enable=no-member + self.app.request_update.connect(self.updateRequest) self.updateRequest() self.setUpdateInterval(10) @@ -279,9 +280,7 @@ def __itemSingleClickedEmitToApp(item, col): @type col: int @param col: Column number single clicked on""" del col - # pylint: disable=no-member - QtGui.qApp.single_click.emit(item.rpcObject) - # pylint: enable=no-member + cuegui.app().single_click.emit(item.rpcObject) @staticmethod def __itemDoubleClickedEmitToApp(item, col): @@ -293,10 +292,8 @@ def __itemDoubleClickedEmitToApp(item, col): @type col: int @param col: Column number double clicked on""" del col - # pylint: disable=no-member - QtGui.qApp.view_object.emit(item.rpcObject) - QtGui.qApp.double_click.emit(item.rpcObject) - # pylint: enable=no-member + cuegui.app().view_object.emit(item.rpcObject) + cuegui.app().double_click.emit(item.rpcObject) def addObject(self, rpcObject): """Adds or updates an rpcObject in the list using the _createItem function @@ -385,11 +382,9 @@ def _update(self): """Updates the items in the TreeWidget without checking when it was last updated""" self._lastUpdate = time.time() - if hasattr(QtGui.qApp, "threadpool"): - # pylint: disable=no-member - QtGui.qApp.threadpool.queue( + if self.app.threadpool is not None: + self.app.threadpool.queue( self._getUpdate, self._processUpdate, "getting data for %s" % self.__class__) - # pylint: enable=no-member else: logger.warning("threadpool not found, doing work in gui thread") self._processUpdate(None, self._getUpdate()) diff --git a/cuegui/cuegui/AbstractWidgetItem.py b/cuegui/cuegui/AbstractWidgetItem.py index 548dde72a..301a3aa59 100644 --- a/cuegui/cuegui/AbstractWidgetItem.py +++ b/cuegui/cuegui/AbstractWidgetItem.py @@ -47,6 +47,7 @@ class AbstractWidgetItem(QtWidgets.QTreeWidgetItem): def __init__(self, itemType, rpcObject, parent, source=None): QtWidgets.QTreeWidgetItem.__init__(self, parent, itemType) + self.app = cuegui.app() self.column_info = self.treeWidget().getColumnInfo(itemType) self._cache = {} self._source = source diff --git a/cuegui/cuegui/App.py b/cuegui/cuegui/App.py new file mode 100644 index 000000000..e161420fe --- /dev/null +++ b/cuegui/cuegui/App.py @@ -0,0 +1,78 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module for CueGUI's custom QApplication and associated helper functions.""" + +from PySide2 import QtCore +from PySide2 import QtWidgets + +import cuegui.Exception + +__QAPPLICATION_SINGLETON = None + + +class CueGuiApplication(QtWidgets.QApplication): + """The CueGUI application.""" + + # Settings + settings = None + + # Global signals + display_log_file_content = QtCore.Signal(object) + double_click = QtCore.Signal(object) + facility_changed = QtCore.Signal() + single_click = QtCore.Signal(object) + unmonitor = QtCore.Signal(object) + view_hosts = QtCore.Signal(object) + view_object = QtCore.Signal(object) + view_procs = QtCore.Signal(object) + request_update = QtCore.Signal() + status = QtCore.Signal() + quit = QtCore.Signal() + + # Thread pool + threadpool = None + threads = [] + + # Shutdown signal + closingApp = False + + +def create_app(argv): + """ + Create an instance of the CueGUI application. + + :param argv: user-provided commandline arguments + :type argv: list + :return: the application instance + :rtype: CueGuiApplication + """ + # pylint: disable=global-statement + global __QAPPLICATION_SINGLETON + if __QAPPLICATION_SINGLETON is None: + __QAPPLICATION_SINGLETON = CueGuiApplication(argv) + return __QAPPLICATION_SINGLETON + + +def app(): + """Returns the current application instance. + + :return: the current application instance + :rtype: CueGuiApplication + :raises: opencue.exception.ApplicationNotRunningException: the application has not been + initialized yet + """ + if __QAPPLICATION_SINGLETON is None: + raise cuegui.Exception.ApplicationNotRunningException() + return __QAPPLICATION_SINGLETON diff --git a/cuegui/cuegui/Comments.py b/cuegui/cuegui/Comments.py index c24a9802e..628c1b740 100644 --- a/cuegui/cuegui/Comments.py +++ b/cuegui/cuegui/Comments.py @@ -24,7 +24,6 @@ import pickle from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import cuegui.Utils @@ -50,6 +49,8 @@ def __init__(self, source, parent=None): @type parent: QWidget @param parent: The dialog's parent""" QtWidgets.QDialog.__init__(self, parent) + self.app = cuegui.app() + self.__source = source self.__labelTitle = QtWidgets.QLabel(self.__source.data.name, self) @@ -208,10 +209,8 @@ def refreshComments(self): def __macroLoad(self): """Loads the defined comment macros from settings""" - # pylint: disable=no-member self.__macroList = pickle.loads( - str(QtGui.qApp.settings.value("Comments", pickle.dumps({})))) - # pylint: enable=no-member + str(self.app.settings.value("Comments", pickle.dumps({})))) self.__macroRefresh() def __macroRefresh(self): @@ -225,9 +224,7 @@ def __macroRefresh(self): def __macroSave(self): """Saves the current comment macros to settings""" - # pylint: disable=no-member - QtGui.qApp.settings.setValue("Comments", pickle.dumps(self.__macroList)) - # pylint: enable=no-member + self.app.settings.setValue("Comments", pickle.dumps(self.__macroList)) def __macroHandle(self, selection): """Called when the comment macro combo box is selected diff --git a/cuegui/cuegui/CueJobMonitorTree.py b/cuegui/cuegui/CueJobMonitorTree.py index 45a9fb07e..b7f32ebec 100644 --- a/cuegui/cuegui/CueJobMonitorTree.py +++ b/cuegui/cuegui/CueJobMonitorTree.py @@ -223,8 +223,8 @@ def __init__(self, parent): self.__menuActions = cuegui.MenuActions.MenuActions( self, self.updateSoon, self.selectedObjects) + self.app.facility_changed.connect(self.removeAllShows) # pylint: disable=no-member - QtGui.qApp.facility_changed.connect(self.removeAllShows) self.itemClicked.connect(self.__itemSingleClickedCopy) self.itemClicked.connect(self.__itemSingleClickedComment) # pylint: enable=no-member @@ -726,9 +726,7 @@ def __init__(self, rpcObject, parent): self.__class__.__initialized = True self.__class__.__commentIcon = QtGui.QIcon(":comment.png") self.__class__.__eatIcon = QtGui.QIcon(":eat.png") - # pylint: disable=no-member - self.__class__.__backgroundColor = QtGui.qApp.palette().color(QtGui.QPalette.Base) - # pylint: enable=no-member + self.__class__.__backgroundColor = cuegui.app().palette().color(QtGui.QPalette.Base) self.__class__.__foregroundColor = cuegui.Style.ColorTheme.COLOR_JOB_FOREGROUND self.__class__.__pausedColor = cuegui.Style.ColorTheme.COLOR_JOB_PAUSED_BACKGROUND self.__class__.__finishedColor = cuegui.Style.ColorTheme.COLOR_JOB_FINISHED_BACKGROUND diff --git a/cuegui/cuegui/CueStateBarWidget.py b/cuegui/cuegui/CueStateBarWidget.py index 780ea319d..c35622f17 100644 --- a/cuegui/cuegui/CueStateBarWidget.py +++ b/cuegui/cuegui/CueStateBarWidget.py @@ -47,6 +47,7 @@ def __init__(self, sourceTree, parent=None): @type parent: QWidget @param parent: The parent widget""" QtWidgets.QWidget.__init__(self, parent) + self.app = cuegui.app() self.__background = None @@ -55,9 +56,7 @@ def __init__(self, sourceTree, parent=None): self.__sourceTree = weakref.proxy(sourceTree) self.__colors = [] - # pylint: disable=no-member - self.__baseColor = QtGui.qApp.palette().color(QtGui.QPalette.Base) - # pylint: enable=no-member + self.__baseColor = self.app.palette().color(QtGui.QPalette.Base) self.__colorsLock = QtCore.QReadWriteLock() self.__timer = QtCore.QTimer(self) self.__lastUpdate = 0 diff --git a/cuegui/cuegui/DarkPalette.py b/cuegui/cuegui/DarkPalette.py index 2b15bd66a..22b89bf68 100644 --- a/cuegui/cuegui/DarkPalette.py +++ b/cuegui/cuegui/DarkPalette.py @@ -32,20 +32,19 @@ def init(): """Convenience function that takes the QApplication object for the application and configures the palette and style for the Plastique color scheme""" - # pylint: disable=no-member - QtGui.qApp.setPalette(DarkPalette()) + app = cuegui.app() + app.setPalette(DarkPalette()) if platform.system() in ['Darwin', 'Linux']: setDarkStyleSheet() elif platform.system() == 'Windows': - QtGui.qApp.setStyle('Fusion') + app.setStyle('Fusion') else: - QtGui.qApp.setStyle(QtWidgets.QStyleFactory.create(cuegui.Constants.COLOR_THEME)) + app.setStyle(QtWidgets.QStyleFactory.create(cuegui.Constants.COLOR_THEME)) def setDarkStyleSheet(): """Sets the stylesheet.""" - # pylint: disable=no-member - QtGui.qApp.setStyleSheet(open(cuegui.Constants.DARK_STYLE_SHEET).read()) + cuegui.app().setStyleSheet(open(cuegui.Constants.DARK_STYLE_SHEET).read()) def DarkPalette(): diff --git a/cuegui/cuegui/Exception.py b/cuegui/cuegui/Exception.py new file mode 100644 index 000000000..151386b32 --- /dev/null +++ b/cuegui/cuegui/Exception.py @@ -0,0 +1,35 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Custom exception classes for CueGUI application errors.""" + + +class CueGuiException(Exception): + """Base class for all CueGUI exceptions. + + Note that this class does NOT inherit from opencue.exception.CueException, so that error + handling code can easily distinguish between API errors and CueGUI errors. + """ + + +class ApplicationNotRunningException(CueGuiException): + """Raised when the CueGUI application has not been instantiated but is required to be.""" + + default_message = ( + 'attempted to access the CueGUI application before cuegui.create_app() was called') + + def __init__(self, message=None): + if message is None: + message = self.default_message + super().__init__(message) diff --git a/cuegui/cuegui/FrameMonitorTree.py b/cuegui/cuegui/FrameMonitorTree.py index e7ee6119c..b546be2d5 100644 --- a/cuegui/cuegui/FrameMonitorTree.py +++ b/cuegui/cuegui/FrameMonitorTree.py @@ -351,9 +351,7 @@ def __itemSingleClickedViewLog(self, item, col): except ValueError: old_log_files = [] - # pylint: disable=no-member - QtGui.qApp.display_log_file_content.emit([current_log_file] + old_log_files) - # pylint: enable=no-member + self.app.display_log_file_content.emit([current_log_file] + old_log_files) def __itemDoubleClickedViewLog(self, item, col): """Called when a frame is double clicked, views the frame log in a popup @@ -447,11 +445,9 @@ def _update(self): updated""" logger.info("_update") self._lastUpdate = time.time() - if hasattr(QtGui.qApp, "threadpool"): - # pylint: disable=no-member - QtGui.qApp.threadpool.queue( + if self.app.threadpool is not None: + self.app.threadpool.queue( self._getUpdate, self._processUpdate, "getting data for %s" % self.__class__) - # pylint: enable=no-member else: logger.warning("threadpool not found, doing work in gui thread") self._processUpdate(None, self._getUpdate()) @@ -461,12 +457,10 @@ def _updateChanged(self): updated""" logger.info("_updateChanged") self._lastUpdate = time.time() - if hasattr(QtGui.qApp, "threadpool"): - # pylint: disable=no-member - QtGui.qApp.threadpool.queue( + if self.app.threadpool is not None: + self.app.threadpool.queue( self._getUpdateChanged, self._processUpdateChanged, "getting data for %s" % self.__class__) - # pylint: enable=no-member else: logger.warning("threadpool not found, doing work in gui thread") self._processUpdateChanged(None, self._getUpdateChanged()) @@ -596,9 +590,7 @@ class FrameWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): def __init__(self, rpcObject, parent, job): if not self.__initialized: self.__class__.__initialized = True - # pylint: disable=no-member - self.__class__.__backgroundColor = QtGui.qApp.palette().color(QtGui.QPalette.Base) - # pylint: enable=no-member + self.__class__.__backgroundColor = cuegui.app().palette().color(QtGui.QPalette.Base) self.__class__.__foregroundColor = cuegui.Style.ColorTheme.COLOR_JOB_FOREGROUND self.__class__.__foregroundColorBlack = QCOLOR_BLACK self.__class__.__foregroundColorGreen = QCOLOR_GREEN @@ -862,6 +854,7 @@ class FrameContextMenu(QtWidgets.QMenu): def __init__(self, widget, filterSelectedLayersCallback): super(FrameContextMenu, self).__init__() + self.app = cuegui.app() self.__menuActions = cuegui.MenuActions.MenuActions( widget, widget.updateSoon, widget.selectedObjects, widget.getJob) @@ -880,13 +873,11 @@ def __init__(self, widget, filterSelectedLayersCallback): elif count == 2: self.__menuActions.frames().addAction(self, "xdiff2") - if bool(int(QtGui.qApp.settings.value("AllowDeeding", 0))): + if bool(int(self.app.settings.value("AllowDeeding", 0))): self.__menuActions.frames().addAction(self, "useLocalCores") - # pylint: disable=no-member - if QtGui.qApp.applicationName() == "CueCommander": + if self.app.applicationName() == "CueCommander": self.__menuActions.frames().addAction(self, "viewHost") - # pylint: enable=no-member depend_menu = QtWidgets.QMenu("&Dependencies", self) self.__menuActions.frames().addAction(depend_menu, "viewDepends") diff --git a/cuegui/cuegui/HostMonitor.py b/cuegui/cuegui/HostMonitor.py index ad6c16a5e..cd7ffee99 100644 --- a/cuegui/cuegui/HostMonitor.py +++ b/cuegui/cuegui/HostMonitor.py @@ -23,7 +23,6 @@ from builtins import str from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import opencue @@ -43,6 +42,7 @@ class HostMonitor(QtWidgets.QWidget): def __init__(self, parent): QtWidgets.QWidget.__init__(self, parent) + self.app = cuegui.app() self.__filterByHostNameLastInput = None self.hostMonitorTree = cuegui.HostMonitorTree.HostMonitorTree(self) @@ -70,10 +70,8 @@ def __init__(self, parent): self.__viewHostsSetup() - # pylint: disable=no-member - if bool(int(QtGui.qApp.settings.value("AutoRefreshMonitorHost", 1))): + if bool(int(self.app.settings.value("AutoRefreshMonitorHost", 1))): self.updateRequest() - # pylint: enable=no-member def updateRequest(self): """Requests an update of the displayed information.""" @@ -279,9 +277,7 @@ def __refreshToggleCheckBoxSetup(self, layout): def __refreshToggleCheckBoxHandle(self, state): self.hostMonitorTree.enableRefresh = bool(state) - # pylint: disable=no-member - QtGui.qApp.settings.setValue("AutoRefreshMonitorHost", int(bool(state))) - # pylint: enable=no-member + self.app.settings.setValue("AutoRefreshMonitorHost", int(bool(state))) # ============================================================================== # Button to refresh @@ -332,9 +328,7 @@ def __clearButtonHandle(self): # Monitors and handles the view_hosts signal # ============================================================================== def __viewHostsSetup(self): - # pylint: disable=no-member - QtGui.qApp.view_hosts.connect(self.__viewHostsHandle) - # pylint: enable=no-member + self.app.view_hosts.connect(self.__viewHostsHandle) def __viewHostsHandle(self, hosts): self.__clearButtonHandle() diff --git a/cuegui/cuegui/HostMonitorTree.py b/cuegui/cuegui/HostMonitorTree.py index 06143769d..1c247e2c6 100644 --- a/cuegui/cuegui/HostMonitorTree.py +++ b/cuegui/cuegui/HostMonitorTree.py @@ -169,17 +169,13 @@ def __init__(self, parent): # pylint: enable=no-member # Don't use the standard space bar to refresh - # pylint: disable=no-member - QtGui.qApp.request_update.connect(self.updateRequest) - # pylint: enable=no-member + self.app.request_update.connect(self.updateRequest) self.startTicksUpdate(40) # Don't start refreshing until the user sets a filter or hits refresh self.ticksWithoutUpdate = -1 - # pylint: disable=no-member - self.enableRefresh = bool(int(QtGui.qApp.settings.value("AutoRefreshMonitorHost", 1))) - # pylint: enable=no-member + self.enableRefresh = bool(int(self.app.settings.value("AutoRefreshMonitorHost", 1))) def tick(self): if self.ticksWithoutUpdate >= self.updateInterval and \ @@ -301,9 +297,7 @@ def __init__(self, rpcObject, parent): cuegui.Style.init() self.__class__.__initialized = True self.__class__.__commentIcon = QtGui.QIcon(":comment.png") - # pylint: disable=no-member - self.__class__.__backgroundColor = QtGui.qApp.palette().color(QtGui.QPalette.Base) - # pylint: enable=no-member + self.__class__.__backgroundColor = cuegui.app().palette().color(QtGui.QPalette.Base) self.__class__.__foregroundColor = cuegui.Style.ColorTheme.COLOR_JOB_FOREGROUND self.__class__.__pausedColor = cuegui.Style.ColorTheme.COLOR_JOB_PAUSED_BACKGROUND self.__class__.__dyingColor = cuegui.Style.ColorTheme.COLOR_JOB_DYING_BACKGROUND diff --git a/cuegui/cuegui/JobMonitorTree.py b/cuegui/cuegui/JobMonitorTree.py index a35d02897..3bf15373b 100644 --- a/cuegui/cuegui/JobMonitorTree.py +++ b/cuegui/cuegui/JobMonitorTree.py @@ -328,9 +328,7 @@ def _removeItem(self, item): """Removes an item from the TreeWidget without locking @param item: A tree widget item @type item: AbstractTreeWidgetItem""" - # pylint: disable=no-member - QtGui.qApp.unmonitor.emit(item.rpcObject) - # pylint: enable=no-member + self.app.unmonitor.emit(item.rpcObject) cuegui.AbstractTreeWidget.AbstractTreeWidget._removeItem(self, item) self.__jobTimeLoaded.pop(item.rpcObject, "") try: @@ -352,9 +350,7 @@ def removeAllItems(self): """Notifies the other widgets of each item being unmonitored, then calls the the AbstractTreeWidget.removeAllItems like normal""" for proxy in list(self._items.keys()): - # pylint: disable=no-member - QtGui.qApp.unmonitor.emit(proxy) - # pylint: enable=no-member + self.app.unmonitor.emit(proxy) if proxy in self.__jobTimeLoaded: del self.__jobTimeLoaded[proxy] self.__dependentJobs.clear() @@ -382,7 +378,7 @@ def contextMenuEvent(self, e): self.__menuActions.jobs().addAction(menu, "emailArtist") self.__menuActions.jobs().addAction(menu, "viewComments") - if bool(int(QtGui.qApp.settings.value("AllowDeeding", 0))): + if bool(int(self.app.settings.value("AllowDeeding", 0))): self.__menuActions.jobs().addAction(menu, "useLocalCores") depend_menu = QtWidgets.QMenu("&Dependencies",self) @@ -597,9 +593,7 @@ def __init__(self, rpcObject, parent, created): self.__class__.__initialized = True self.__class__.__commentIcon = QtGui.QIcon(":comment.png") self.__class__.__eatIcon = QtGui.QIcon(":eat.png") - # pylint: disable=no-member - self.__class__.__backgroundColor = QtGui.qApp.palette().color(QtGui.QPalette.Base) - # pylint: enable=no-member + self.__class__.__backgroundColor = cuegui.app().palette().color(QtGui.QPalette.Base) self.__class__.__foregroundColor = cuegui.Style.ColorTheme.COLOR_JOB_FOREGROUND self.__class__.__pausedColor = cuegui.Style.ColorTheme.COLOR_JOB_PAUSED_BACKGROUND self.__class__.__dyingColor = cuegui.Style.ColorTheme.COLOR_JOB_DYING_BACKGROUND diff --git a/cuegui/cuegui/LayerDialog.py b/cuegui/cuegui/LayerDialog.py index 3efd36de3..5f25be5f6 100644 --- a/cuegui/cuegui/LayerDialog.py +++ b/cuegui/cuegui/LayerDialog.py @@ -21,7 +21,6 @@ from __future__ import print_function from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import opencue @@ -106,6 +105,7 @@ class LayerPropertiesDialog(QtWidgets.QDialog): def __init__(self, layers, parent=None): QtWidgets.QDialog.__init__(self, parent) + self.app = cuegui.app() self.__layers = [opencue.api.getLayer(opencue.id(layer)) for layer in layers] self.setWindowTitle("Layer Properties") @@ -147,10 +147,8 @@ def __init__(self, layers, parent=None): self.__max_cores.setSingleStep(1) # Disable this for everything except commander. - # pylint: disable=no-member - if QtGui.qApp.applicationName() != "CueCommander": + if self.app.applicationName() != "CueCommander": self.__core.setDisabled(True) - # pylint: enable=no-member # Threads self.__thread = QtWidgets.QCheckBox(self) diff --git a/cuegui/cuegui/LayerMonitorTree.py b/cuegui/cuegui/LayerMonitorTree.py index a8665c1a0..7663d3276 100644 --- a/cuegui/cuegui/LayerMonitorTree.py +++ b/cuegui/cuegui/LayerMonitorTree.py @@ -21,7 +21,6 @@ from __future__ import division from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets from opencue.exception import EntityNotFoundException @@ -232,7 +231,7 @@ def contextMenuEvent(self, e): if len(__selectedObjects) == 1: menu.addSeparator() - if bool(int(QtGui.qApp.settings.value("AllowDeeding", 0))): + if bool(int(self.app.settings.value("AllowDeeding", 0))): self.__menuActions.layers().addAction(menu, "useLocalCores") if len({layer.data.range for layer in __selectedObjects}) == 1: self.__menuActions.layers().addAction(menu, "reorder") diff --git a/cuegui/cuegui/LimitsWidget.py b/cuegui/cuegui/LimitsWidget.py index ec0753f2e..26f871de1 100644 --- a/cuegui/cuegui/LimitsWidget.py +++ b/cuegui/cuegui/LimitsWidget.py @@ -21,7 +21,6 @@ from __future__ import division from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import opencue @@ -110,10 +109,8 @@ def __init__(self, parent): self.__menuActions = cuegui.MenuActions.MenuActions( self, self.updateSoon, self.selectedObjects) - # pylint: disable=no-member - self.itemClicked.connect(self.__itemSingleClickedToDouble) - QtGui.qApp.facility_changed.connect(self.__facilityChanged) - # pylint: enable=no-member + self.itemClicked.connect(self.__itemSingleClickedToDouble) # pylint: disable=no-member + self.app.facility_changed.connect(self.__facilityChanged) self.setUpdateInterval(60) diff --git a/cuegui/cuegui/Main.py b/cuegui/cuegui/Main.py index 58bb3073d..7712a4cdb 100644 --- a/cuegui/cuegui/Main.py +++ b/cuegui/cuegui/Main.py @@ -22,10 +22,9 @@ import signal -from PySide2 import QtCore from PySide2 import QtGui -from PySide2 import QtWidgets +import cuegui import cuegui.Config import cuegui.Constants import cuegui.Logger @@ -40,23 +39,6 @@ logger = cuegui.Logger.getLogger(__file__) -class CueGuiApplication(QtWidgets.QApplication): - """The CueGUI application.""" - - # Global signals - display_log_file_content = QtCore.Signal(object) - double_click = QtCore.Signal(object) - facility_changed = QtCore.Signal() - single_click = QtCore.Signal(object) - unmonitor = QtCore.Signal(object) - view_hosts = QtCore.Signal(object) - view_object = QtCore.Signal(object) - view_procs = QtCore.Signal(object) - request_update = QtCore.Signal() - status = QtCore.Signal() - quit = QtCore.Signal() - - def cuetopia(argv): """Starts the Cuetopia window.""" startup("Cuetopia", cuegui.Constants.VERSION, argv) @@ -70,8 +52,7 @@ def cuecommander(argv): def startup(app_name, app_version, argv): """Starts an application window.""" - app = CueGuiApplication(argv) - QtGui.qApp = app + app = cuegui.create_app(argv) # Start splash screen splash = cuegui.SplashWindow.SplashWindow( @@ -86,13 +67,10 @@ def startup(app_name, app_version, argv): app.setApplicationName(app_name) app.lastWindowClosed.connect(app.quit) # pylint: disable=no-member - # pylint: disable=attribute-defined-outside-init - QtGui.qApp.threadpool = cuegui.ThreadPool.ThreadPool(3, parent=app) - QtGui.qApp.threads = [] - # pylint: enable=attribute-defined-outside-init + app.threadpool = cuegui.ThreadPool.ThreadPool(3, parent=app) settings = cuegui.Config.startup(app_name) - QtGui.qApp.settings = settings # pylint: disable=attribute-defined-outside-init + app.settings = settings cuegui.Style.init() @@ -118,6 +96,6 @@ def startup(app_name, app_version, argv): def closingTime(): """Window close callback.""" logger.info("Closing all threads...") - threads = QtGui.qApp.threads # pylint: disable=no-member + threads = cuegui.app().threads for thread in threads: cuegui.Utils.shutdownThread(thread) diff --git a/cuegui/cuegui/MainWindow.py b/cuegui/cuegui/MainWindow.py index 9fd5c5492..50555f647 100644 --- a/cuegui/cuegui/MainWindow.py +++ b/cuegui/cuegui/MainWindow.py @@ -53,16 +53,14 @@ class MainWindow(QtWidgets.QMainWindow): def __init__(self, app_name, app_version, window_name, parent = None): QtWidgets.QMainWindow.__init__(self, parent) + self.app = cuegui.app() self.__actions_facility = {} self.facility_default = None self.facility_dict = None self.windowMenu = None - self.qApp = QtGui.qApp - # pylint: disable=no-member - self.settings = QtGui.qApp.settings - # pylint: enable=no-member + self.settings = self.app.settings self.windows_names = [app_name] + ["%s_%s" % (app_name, num) for num in range(2, 5)] self.app_name = app_name self.app_version = app_version @@ -96,10 +94,7 @@ def __init__(self, app_name, app_version, window_name, parent = None): # Restore saved settings self.__restoreSettings() - # pylint: disable=no-member - QtGui.qApp.status.connect(self.showStatusBarMessage) - # pylint: enable=no-member - + self.app.status.connect(self.showStatusBarMessage) self.showStatusBarMessage("Ready") def displayStartupNotice(self): @@ -184,9 +179,7 @@ def __facilityMenuHandle(self, action): for facility in list(self.__actions_facility.values()): if facility.isChecked(): opencue.Cuebot.setFacility(str(facility.text())) - # pylint: disable=no-member - QtGui.qApp.facility_changed.emit() - # pylint: enable=no-member + self.app.facility_changed.emit() return ################################################################################ @@ -359,24 +352,18 @@ def windowMenuOpenWindow(self, name): def __windowOpened(self): """Called from __init__ on window creation""" - # pylint: disable=no-member - self.qApp.quit.connect(self.close) + self.app.quit.connect(self.close) self.windows.append(self) - self.qApp.closingApp = False - # pylint: enable=no-member + self.app.closingApp = False def __windowClosed(self): """Called from closeEvent on window close""" # Disconnect to avoid multiple attempts to close a window - # pylint: disable=no-member - self.qApp.quit.connect(self.close) - # pylint: enable=no-member + self.app.quit.connect(self.close) # Save the fact that this window is open or not when the app closed - # pylint: disable=no-member - self.settings.setValue("%s/Open" % self.name, self.qApp.closingApp) - # pylint: enable=no-member + self.settings.setValue("%s/Open" % self.name, self.app.closingApp) # pylint: disable=bare-except try: @@ -392,10 +379,8 @@ def __windowCloseWindow(self): def __windowCloseApplication(self): """Called when the entire application should exit. Signals other windows to exit.""" - # pylint: disable=no-member - self.qApp.closingApp = True - self.qApp.quit.emit() - # pylint: enable=no-member + self.app.closingApp = True + self.app.quit.emit() ################################################################################ @@ -420,9 +405,7 @@ def __toggleFullscreen(self): def keyPressEvent(self, event): if event.key() == QtCore.Qt.Key_Space: - # pylint: disable=no-member - QtGui.qApp.request_update.emit() - # pylint: enable=no-member + self.app.request_update.emit() event.accept() def closeEvent(self, event): diff --git a/cuegui/cuegui/MenuActions.py b/cuegui/cuegui/MenuActions.py index 2cd284f6d..2f6771baa 100644 --- a/cuegui/cuegui/MenuActions.py +++ b/cuegui/cuegui/MenuActions.py @@ -35,12 +35,14 @@ import FileSequence import opencue import opencue.compiled_proto.job_pb2 +import opencue.wrappers.depend # pylint: disable=cyclic-import import cuegui.Action import cuegui.Comments import cuegui.Constants import cuegui.CreatorDialog +import cuegui.CueJobMonitorTree import cuegui.DependDialog import cuegui.DependWizard import cuegui.EmailDialog @@ -78,6 +80,7 @@ def __init__(self, caller, updateCallable, selectedRpcObjectsCallable, sourceCal self.__selectedRpcObjects = selectedRpcObjectsCallable self._getSource = sourceCallable self._update = updateCallable + self.app = cuegui.app() self.__actionCache = {} @@ -218,9 +221,7 @@ def unmonitor(self, rpcObjects=None): def view(self, rpcObjects=None): for job in self._getOnlyJobObjects(rpcObjects): - # pylint: disable=no-member - QtGui.qApp.view_object.emit(job) - # pylint: enable=no-member + self.app.view_object.emit(job) viewDepends_info = ["&View Dependencies...", None, "log"] @@ -427,10 +428,8 @@ def getJobByName(self, job_name): def dropJobsDependingOnThis(self, job): for dep in job.getWhatDependsOnThis(): if not dep.isInternal(): - # pylint: disable=no-member - job = self.getJobByName(self, dep.dependOnJob()) - job.dropDepends(opencue.wrappers.depend.DependTarget.EXTERNAL) - # pylint: enable=no-member + job = self.getJobByName(dep.dependOnJob()) + job.dropDepends(opencue.wrappers.depend.Depend.DependTarget.EXTERNAL) eatDead_info = ["Eat dead frames", None, "eat"] @@ -995,10 +994,8 @@ def viewHost(self, rpcObjects=None): hosts = list({frame.data.last_resource.split("/")[0] for frame in frames if frame.data.last_resource}) if hosts: - # pylint: disable=no-member - QtGui.qApp.view_hosts.emit(hosts) - QtGui.qApp.single_click.emit(opencue.api.findHost(hosts[0])) - # pylint: enable=no-member + self.app.view_hosts.emit(hosts) + self.app.single_click.emit(opencue.api.findHost(hosts[0])) getWhatThisDependsOn_info = ["print getWhatThisDependsOn", None, "log"] @@ -1452,9 +1449,7 @@ def viewProc(self, rpcObjects=None): hosts = self._getOnlyHostObjects(rpcObjects) hosts = list({host.data.name for host in hosts}) if hosts: - # pylint: disable=no-member - QtGui.qApp.view_procs.emit(hosts) - # pylint: enable=no-member + self.app.view_procs.emit(hosts) lock_info = ["Lock Host", None, "lock"] @@ -1650,9 +1645,7 @@ def __init__(self, *args): def view(self, rpcObjects=None): for job in list({proc.data.job_name for proc in self._getOnlyProcObjects(rpcObjects)}): try: - # pylint: disable=no-member - QtGui.qApp.view_object.emit(opencue.api.findJob(job)) - # pylint: enable=no-member + self.app.view_object.emit(opencue.api.findJob(job)) except opencue.exception.CueException: logger.warning("Unable to load: %s", job) diff --git a/cuegui/cuegui/Plugins.py b/cuegui/cuegui/Plugins.py index 2b67ebd7e..f4e6e3524 100644 --- a/cuegui/cuegui/Plugins.py +++ b/cuegui/cuegui/Plugins.py @@ -58,7 +58,6 @@ import pickle from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import cuegui.Constants @@ -98,13 +97,12 @@ def __init__(self, mainWindow, name): self.__running = [] self.name = name self.mainWindow = mainWindow + self.app = cuegui.app() self.__menu_separator = " \t-> " # Load plugin paths from the config file - # pylint: disable=no-member - __pluginPaths = QtGui.qApp.settings.value("Plugin_Paths", []) - # pylint: enable=no-member + __pluginPaths = self.app.settings.value("Plugin_Paths", []) for path in cuegui.Constants.DEFAULT_PLUGIN_PATHS + __pluginPaths: self.loadPluginPath(str(path)) @@ -123,9 +121,7 @@ def loadConfigFilePlugins(self, configGroup): The imported module must have an init function and a QMainWindow will be passed to it. """ - # pylint: disable=no-member - __plugins = QtGui.qApp.settings.value("%s/Plugins" % configGroup, []) - # pylint: enable=no-member + __plugins = self.app.settings.value("%s/Plugins" % configGroup, []) for plugin in __plugins: path = os.path.dirname(str(plugin)) @@ -181,26 +177,20 @@ def saveState(self): opened.append("%s::%s" % (plugin[0], json.dumps(plugin[1].pluginSaveState()))) except Exception as e: logger.warning("Error saving plugin state for: %s\n%s", plugin[0], e) - # pylint: disable=no-member - QtGui.qApp.settings.setValue("%s/Plugins_Opened" % self.name, opened) - # pylint: enable=no-member + self.app.settings.setValue("%s/Plugins_Opened" % self.name, opened) def restoreState(self): """Loads any user defined plugin directories and restores all open plugins. Calls .restoreSettings (if available) on all plugins.""" # Loads any user defined plugin directories - # pylint: disable=no-member - pluginPaths = QtGui.qApp.settings.value("Plugins/Paths", []) - # pylint: enable=no-member + pluginPaths = self.app.settings.value("Plugins/Paths", []) for path in pluginPaths: self.loadPluginPath(str(path)) # Runs any plugins that were saved to the settings - # pylint: disable=no-member - openPlugins = QtGui.qApp.settings.value("%s/Plugins_Opened" % self.name) or [] - # pylint: enable=no-member + openPlugins = self.app.settings.value("%s/Plugins_Opened" % self.name) or [] for plugin in openPlugins: if '::' in plugin: plugin_name, plugin_state = str(plugin).split("::") @@ -357,6 +347,7 @@ class Plugin(object): def __init__(self): self.__settings = [] + self.app = cuegui.app() def pluginRestoreState(self, saved_settings): """Called on plugin start with any previously saved state. diff --git a/cuegui/cuegui/PreviewWidget.py b/cuegui/cuegui/PreviewWidget.py index 6e60a0a6a..56888188e 100644 --- a/cuegui/cuegui/PreviewWidget.py +++ b/cuegui/cuegui/PreviewWidget.py @@ -34,7 +34,6 @@ import xml.etree.ElementTree as Et from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import cuegui.Logger @@ -59,6 +58,8 @@ def __init__(self, job, frame, aovs=False, parent=None): :param parent: the parent widget """ QtWidgets.QDialog.__init__(self, parent) + self.app = cuegui.app() + self.__job = job self.__frame = frame self.__aovs = aovs @@ -93,9 +94,7 @@ def process(self): self.__itvFile = self.__writePlaylist(playlist) self.__previewThread = PreviewProcessorWatchThread(items, self) - # pylint: disable=no-member - QtGui.qApp.threads.append(self.__previewThread) - # pylint: enable=no-member + self.app.threads.append(self.__previewThread) self.__previewThread.start() self.__progbar.setRange(0, len(items)) diff --git a/cuegui/cuegui/ProcMonitor.py b/cuegui/cuegui/ProcMonitor.py index 84e01fd4b..4221a2c92 100644 --- a/cuegui/cuegui/ProcMonitor.py +++ b/cuegui/cuegui/ProcMonitor.py @@ -23,7 +23,6 @@ from builtins import str from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import cuegui.Logger @@ -42,6 +41,7 @@ class ProcMonitor(QtWidgets.QWidget): def __init__(self, parent): QtWidgets.QWidget.__init__(self, parent) + self.app = cuegui.app() self.__filterByHostNameLastInput = None @@ -71,7 +71,7 @@ def __init__(self, parent): self.__viewHostsSetup() - if bool(int(QtGui.qApp.settings.value("AutoRefreshMonitorProc", 1))): # pylint: disable=no-member + if bool(int(self.app.settings.value("AutoRefreshMonitorProc", 1))): self.updateRequest() def updateRequest(self): @@ -141,9 +141,7 @@ def __refreshToggleCheckBoxSetup(self, layout): def __refreshToggleCheckBoxHandle(self, state): self.procMonitorTree.enableRefresh = bool(state) - # pylint: disable=no-member - QtGui.qApp.settings.setValue("AutoRefreshMonitorProc", int(bool(state))) - # pylint: enable=no-member + self.app.settings.setValue("AutoRefreshMonitorProc", int(bool(state))) # ============================================================================== # Button to refresh @@ -192,7 +190,7 @@ def __clearButtonHandle(self): # Monitors and handles the view_procs signal # ============================================================================== def __viewProcsSetup(self): - QtGui.qApp.view_procs.connect(self.__viewProcsHandle) # pylint: disable=no-member + self.app.view_procs.connect(self.__viewProcsHandle) def __viewProcsHandle(self, hosts): self.procMonitorTree.procSearch.options['host'] = hosts @@ -202,7 +200,7 @@ def __viewProcsHandle(self, hosts): # Views procs when a host is double clicked # ============================================================================== def __hostDoubleClickedSetup(self): - QtGui.qApp.view_object.connect(self.__hostDoubleClickedHandle) # pylint: disable=no-member + self.app.view_object.connect(self.__hostDoubleClickedHandle) def __hostDoubleClickedHandle(self, rpcObject): if cuegui.Utils.isHost(rpcObject): @@ -213,7 +211,7 @@ def __hostDoubleClickedHandle(self, rpcObject): # Monitors and handles the view_hosts signal # ============================================================================== def __viewHostsSetup(self): - QtGui.qApp.view_hosts.connect(self.__viewHostsHandle) # pylint: disable=no-member + self.app.view_hosts.connect(self.__viewHostsHandle) def __viewHostsHandle(self, hosts): if hosts: diff --git a/cuegui/cuegui/ProcMonitorTree.py b/cuegui/cuegui/ProcMonitorTree.py index 97171e08f..cf2658de5 100644 --- a/cuegui/cuegui/ProcMonitorTree.py +++ b/cuegui/cuegui/ProcMonitorTree.py @@ -24,7 +24,6 @@ import time from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import opencue @@ -89,17 +88,13 @@ def __init__(self, parent): self.itemDoubleClicked.connect(self.__itemDoubleClickedViewLog) # Don't use the standard space bar to refresh - # pylint: disable=no-member - QtGui.qApp.request_update.connect(self.updateRequest) - # pylint: enable=no-member + self.app.request_update.connect(self.updateRequest) self.startTicksUpdate(40) # Don't start refreshing until the user sets a filter or hits refresh self.ticksWithoutUpdate = -1 - # pylint: disable=no-member - self.enableRefresh = bool(int(QtGui.qApp.settings.value("AutoRefreshMonitorProc", 1))) - # pylint: enable=no-member + self.enableRefresh = bool(int(self.app.settings.value("AutoRefreshMonitorProc", 1))) def tick(self): if self.ticksWithoutUpdate >= self.updateInterval and \ @@ -140,9 +135,7 @@ def __itemDoubleClickedViewLog(self, item, col): @param col: Column number double clicked on""" del col job_name = item.rpcObject.data.job_name - # pylint: disable=no-member - QtGui.qApp.view_object.emit(opencue.api.findJob(job_name)) - # pylint: enable=no-member + self.app.view_object.emit(opencue.api.findJob(job_name)) def clearFilters(self): """Removes all sorting and filtering to restore default state.""" diff --git a/cuegui/cuegui/ProgressDialog.py b/cuegui/cuegui/ProgressDialog.py index 375e2a189..2f507fda7 100644 --- a/cuegui/cuegui/ProgressDialog.py +++ b/cuegui/cuegui/ProgressDialog.py @@ -24,7 +24,6 @@ from builtins import range from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import cuegui.Logger @@ -57,6 +56,7 @@ def __init__(self, title, function, work, concurrent, cancelTitle, @type parent: QObject @param parent: The parent for this object""" QtWidgets.QDialog.__init__(self, parent) + self.app = cuegui.app() self.__work = work self.__function = function @@ -161,12 +161,9 @@ def _submitWork(self): """Submits a new unit of work to threadpool""" self.__count += 1 - if hasattr(QtGui.qApp, "threadpool"): - # pylint: disable=no-member - QtGui.qApp.threadpool.queue(self.__doWork, - self.__doneWork, - "getting data for %s" % self.__class__) - # pylint: enable=no-member + if self.app.threadpool is not None: + self.app.threadpool.queue( + self.__doWork, self.__doneWork, "getting data for %s" % self.__class__) else: logger.warning("threadpool not found, doing work in gui thread") self.__doneWork(None, self.__doWork()) diff --git a/cuegui/cuegui/Redirect.py b/cuegui/cuegui/Redirect.py index 49fb6b7a5..ae46814f2 100644 --- a/cuegui/cuegui/Redirect.py +++ b/cuegui/cuegui/Redirect.py @@ -415,6 +415,7 @@ class RedirectWidget(QtWidgets.QWidget): def __init__(self, parent=None): QtWidgets.QWidget.__init__(self, parent) + self.app = cuegui.app() self.__hosts = {} self.__controls = RedirectControls(self) @@ -464,7 +465,7 @@ def mouseDoubleClickEvent(self, index): jobObject = opencue.api.getJobs(job=[index.data()]) if jobObject: if cuegui.Utils.isJob(jobObject[0]): - QtGui.qApp.view_object.emit(jobObject[0]) + self.app.view_object.emit(jobObject[0]) except opencue.exception.CueException as e: text = ('Not able to add job to Job Monitor Tree. ' 'Error Message:\n %s' % e) diff --git a/cuegui/cuegui/ShowsWidget.py b/cuegui/cuegui/ShowsWidget.py index a91685cd1..f10fa08e3 100644 --- a/cuegui/cuegui/ShowsWidget.py +++ b/cuegui/cuegui/ShowsWidget.py @@ -21,7 +21,6 @@ from __future__ import division from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import opencue @@ -62,10 +61,8 @@ def __init__(self, parent): self.__menuActions = cuegui.MenuActions.MenuActions( self, self.updateSoon, self.selectedObjects) - # pylint: disable=no-member - self.itemClicked.connect(self.__itemSingleClickedToDouble) - QtGui.qApp.facility_changed.connect(self.__facilityChanged) - # pylint: enable=no-member + self.itemClicked.connect(self.__itemSingleClickedToDouble) # pylint: disable=no-member + self.app.facility_changed.connect(self.__facilityChanged) self.setUpdateInterval(60) diff --git a/cuegui/cuegui/Style.py b/cuegui/cuegui/Style.py index 3f25bbed0..d95b80dd8 100644 --- a/cuegui/cuegui/Style.py +++ b/cuegui/cuegui/Style.py @@ -24,6 +24,8 @@ from PySide2 import QtGui +import cuegui + DEFAULT_COLOR = "DarkPalette" DEFAULT_ICON = "crystal" @@ -56,16 +58,12 @@ def setFont(font): """Sets the application font.""" global Font Font = font - # pylint: disable=no-member - QtGui.qApp.setFont(font) - # pylint: enable=no-member + cuegui.app().setFont(font) def init(): """Initializes the global style settings.""" - # pylint: disable=no-member - settings = QtGui.qApp.settings - # pylint: enable=no-member + settings = cuegui.app().settings loadColorTheme(settings.value("Style/colorTheme", DEFAULT_COLOR)) setIconTheme(settings.value("Style/iconTheme", DEFAULT_ICON)) diff --git a/cuegui/cuegui/SubscriptionsWidget.py b/cuegui/cuegui/SubscriptionsWidget.py index 8be93ad00..575742f4b 100644 --- a/cuegui/cuegui/SubscriptionsWidget.py +++ b/cuegui/cuegui/SubscriptionsWidget.py @@ -25,7 +25,6 @@ import opencue from PySide2 import QtCore -from PySide2 import QtGui from PySide2 import QtWidgets import cuegui.AbstractTreeWidget @@ -41,6 +40,7 @@ class SubscriptionsWidget(QtWidgets.QWidget): def __init__(self, parent): QtWidgets.QWidget.__init__(self, parent) + self.app = cuegui.app() self.__show = None self.__shows = None @@ -68,9 +68,9 @@ def __init__(self, parent): self.__btnShowProperties.clicked.connect(self.__showProperties) self.__btnAddSubscription.clicked.connect(self.__addSubscription) self.__comboShows.currentIndexChanged.connect(self.setShow) - QtGui.qApp.view_object.connect(self.setShow) - QtGui.qApp.facility_changed.connect(self.changeFacility) # pylint: enable=no-member + self.app.view_object.connect(self.setShow) + self.app.facility_changed.connect(self.changeFacility) self.__menuActions = cuegui.MenuActions.MenuActions( self, self.updateSoon, self.selectedObjects) diff --git a/cuegui/cuegui/ThreadPool.py b/cuegui/cuegui/ThreadPool.py index 73d791d50..ba03ddc8f 100644 --- a/cuegui/cuegui/ThreadPool.py +++ b/cuegui/cuegui/ThreadPool.py @@ -50,7 +50,6 @@ def someWorkCallback(work, result): import os from PySide2 import QtCore -from PySide2 import QtGui import cuegui.Logger @@ -74,6 +73,7 @@ class ThreadPool(QtCore.QObject): def __init__(self, num_threads, max_queue=20, parent=None): QtCore.QObject.__init__(self, parent=parent) + self.app = cuegui.app() self.__threads = [] self.__started = False self.__max_queue = max_queue @@ -90,9 +90,7 @@ def start(self): self.__started = True for i in range(0, self.__num_threads): thread = ThreadPool.WorkerThread(i, self) - # pylint: disable=no-member - QtGui.qApp.threads.append(thread) - # pylint: enable=no-member + self.app.threads.append(thread) self.__threads.append(thread) self.__threads[i].start() self.__threads[i].workComplete.connect(self.runCallback, diff --git a/cuegui/cuegui/Utils.py b/cuegui/cuegui/Utils.py index 8d41df3cf..07cb5c025 100644 --- a/cuegui/cuegui/Utils.py +++ b/cuegui/cuegui/Utils.py @@ -489,14 +489,13 @@ def popupView(file, facility=None): """Opens the given file in your editor.""" if file and not popupWeb(file, facility): editor_from_env = os.getenv('EDITOR') - # pylint: disable=no-member + app = cuegui.app() if editor_from_env: job_log_cmd = editor_from_env.split() - elif QtGui.qApp.settings.contains('LogEditor'): - job_log_cmd = QtGui.qApp.settings.value("LogEditor") + elif app.settings.contains('LogEditor'): + job_log_cmd = app.settings.value("LogEditor") else: job_log_cmd = cuegui.Constants.DEFAULT_EDITOR.split() - # pylint: enable=no-member job_log_cmd.append(str(file)) checkShellOut(job_log_cmd) diff --git a/cuegui/cuegui/__init__.py b/cuegui/cuegui/__init__.py index d9c80f13d..ce0dbd72e 100644 --- a/cuegui/cuegui/__init__.py +++ b/cuegui/cuegui/__init__.py @@ -11,3 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +"""Top level module for the CueGUI application. + +See __main__.py and Main.py for the application entrypoint. +""" + +from .App import create_app +from .App import app diff --git a/cuegui/cuegui/plugins/AllocationsPlugin.py b/cuegui/cuegui/plugins/AllocationsPlugin.py index b181a350b..9e7d1a647 100644 --- a/cuegui/cuegui/plugins/AllocationsPlugin.py +++ b/cuegui/cuegui/plugins/AllocationsPlugin.py @@ -22,7 +22,7 @@ from builtins import map -from PySide2 import QtGui, QtWidgets +from PySide2 import QtWidgets import opencue @@ -104,10 +104,7 @@ def __init__(self, parent): self.setDragEnabled(True) self.setDragDropMode(QtWidgets.QAbstractItemView.DragDrop) - # Signals are defined in code, so pylint thinks they don't exist. - # pylint: disable=no-member - QtGui.qApp.facility_changed.connect(self._update) - # pylint: enable=no-member + self.app.facility_changed.connect(self._update) self.setUpdateInterval(60) diff --git a/cuegui/cuegui/plugins/AttributesPlugin.py b/cuegui/cuegui/plugins/AttributesPlugin.py index b20825204..26f9350de 100644 --- a/cuegui/cuegui/plugins/AttributesPlugin.py +++ b/cuegui/cuegui/plugins/AttributesPlugin.py @@ -24,7 +24,6 @@ from builtins import str import time -from PySide2 import QtGui from PySide2 import QtCore from PySide2 import QtWidgets @@ -89,6 +88,8 @@ class Attributes(QtWidgets.QWidget): """ def __init__(self, parent=None): QtWidgets.QWidget.__init__(self, parent) + self.app = cuegui.app() + layout = QtWidgets.QVBoxLayout(self) layout.setContentsMargins(0, 0, 0, 0) @@ -106,9 +107,7 @@ def __init__(self, parent=None): self.__scrollWidget.layout().addWidget(self.__stack) self.__scrollArea.setWidget(self.__scrollWidget) layout.addWidget(self.__scrollArea) - # pylint: disable=no-member - QtGui.qApp.single_click.connect(self.setWidget) - # pylint: enable=no-member + self.app.single_click.connect(self.setWidget) self.__load = None @@ -138,13 +137,10 @@ def setWidget(self, item): # called in a worker thread prior to the creation of the widget. # Otherwise the widget will just be created now. if hasattr(function, "preload"): - if hasattr(QtGui.qApp, "threadpool"): + if self.app.threadpool is not None: self.__load = {"item": item, "function": function} - # pylint: disable=no-member - QtGui.qApp.threadpool.queue(self.__getUpdate, - self.__processResults, - "getting data for %s" % self.__class__) - # pylint: enable=no-member + self.app.threadpool.queue( + self.__getUpdate, self.__processResults, "getting data for %s" % self.__class__) else: logger.warning("threadpool not found, doing work in gui thread") return self.__createItemAttribute(item, function, function.preload(item)) diff --git a/cuegui/cuegui/plugins/LogViewPlugin.py b/cuegui/cuegui/plugins/LogViewPlugin.py index 8e8486a1b..caeacf449 100644 --- a/cuegui/cuegui/plugins/LogViewPlugin.py +++ b/cuegui/cuegui/plugins/LogViewPlugin.py @@ -300,9 +300,10 @@ def __init__(self, parent=None): """ Create the UI elements """ + QtWidgets.QWidget.__init__(self, parent) + self.app = cuegui.app() # Main Widget - QtWidgets.QWidget.__init__(self, parent) layout = QtWidgets.QVBoxLayout(self) layout.setContentsMargins(0, 0, 0, 0) self._scrollArea = QtWidgets.QScrollArea() @@ -428,9 +429,7 @@ def __init__(self, parent=None): pos = QtCore.QPoint(0, 0) self._highlight_cursor = self._content_box.cursorForPosition(pos) # Signals are defined in code, so pylint thinks they don't exist. - # pylint: disable=no-member - QtGui.qApp.display_log_file_content.connect(self._set_log_files) - # pylint: enable=no-member + self.app.display_log_file_content.connect(self._set_log_files) self._log_scrollbar = self._content_box.verticalScrollBar() self._log_scrollbar.valueChanged.connect(self._set_scrollbar_value) @@ -856,9 +855,7 @@ def _update_log(self): else: self._content_box.appendPlainText(new_text) self._content_timestamp = time.time() - # pylint: disable=no-member - QtGui.qApp.processEvents() - # pylint: enable=no-member + self.app.processEvents() # Adjust scrollbar value (if necessary) self._scrollbar_max = self._log_scrollbar.maximum() diff --git a/cuegui/cuegui/plugins/MonitorCuePlugin.py b/cuegui/cuegui/plugins/MonitorCuePlugin.py index 8edbda25b..c0b3e6fac 100644 --- a/cuegui/cuegui/plugins/MonitorCuePlugin.py +++ b/cuegui/cuegui/plugins/MonitorCuePlugin.py @@ -78,9 +78,7 @@ def __init__(self, parent): self.layout().addLayout(self.__hlayout) - # pylint: disable=no-member - self.__monitorCue.view_object.connect(QtGui.qApp.view_object.emit) - # pylint: enable=no-member + self.__monitorCue.view_object.connect(self.app.view_object.emit) self.pluginRegisterSettings([("shows", self.__monitorCue.getShowNames, @@ -98,9 +96,7 @@ def __init__(self, parent): self.addShows([os.getenv('SHOW')]) def __cueStateBarSetup(self, layout): - # pylint: disable=no-member - cueStateBarEnabled = QtGui.qApp.settings.value("CueStateBar", False) - # pylint: enable=no-member + cueStateBarEnabled = self.app.settings.value("CueStateBar", False) if cueStateBarEnabled: self.__cueStateBar = cuegui.CueStateBarWidget.CueStateBarWidget(self.__monitorCue, self) layout.addWidget(self.__cueStateBar) @@ -175,10 +171,8 @@ def __showMenuSetup(self): self.__showMenuBtn.setMenu(self.__showMenu) self.__showMenuBtn.setFocusPolicy(QtCore.Qt.NoFocus) self.__showMenu.setFont(cuegui.Constants.STANDARD_FONT) - # pylint: disable=no-member - self.__showMenu.triggered.connect(self.__showMenuHandle) - QtGui.qApp.facility_changed.connect(self.__showMenuUpdate) - # pylint: enable=no-member + self.__showMenu.triggered.connect(self.__showMenuHandle) # pylint: disable=no-member + self.app.facility_changed.connect(self.__showMenuUpdate) self.__showMenuUpdate() diff --git a/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py b/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py index 809cefbfc..270a74b23 100644 --- a/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py @@ -22,7 +22,6 @@ from __future__ import division from __future__ import absolute_import -from PySide2 import QtGui from PySide2 import QtCore from PySide2 import QtWidgets @@ -64,11 +63,9 @@ def __init__(self, parent): self.__splitter.addWidget(self.__monitorLayers) self.__splitter.addWidget(self.__monitorFrames) - # pylint: disable=no-member - QtGui.qApp.view_object.connect(self.__setJob) - QtGui.qApp.unmonitor.connect(self.__unmonitor) - QtGui.qApp.facility_changed.connect(self.__setJob) - # pylint: enable=no-member + self.app.view_object.connect(self.__setJob) + self.app.unmonitor.connect(self.__unmonitor) + self.app.facility_changed.connect(self.__setJob) self.__monitorLayers.handle_filter_layers_byLayer.connect(self.handleLayerFilter) self.__splitter.splitterMoved.connect(self.__splitterMoved) # pylint: disable=no-member diff --git a/cuegui/cuegui/plugins/MonitorJobsPlugin.py b/cuegui/cuegui/plugins/MonitorJobsPlugin.py index 94b081b13..f9ab677c2 100644 --- a/cuegui/cuegui/plugins/MonitorJobsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorJobsPlugin.py @@ -69,10 +69,8 @@ def __init__(self, parent): self.layout().addWidget(self.jobMonitor) # Signals in - # pylint: disable=no-member - QtGui.qApp.view_object.connect(self.addJob) - QtGui.qApp.facility_changed.connect(self.jobMonitor.removeAllItems) - # pylint: enable=no-member + self.app.view_object.connect(self.addJob) + self.app.facility_changed.connect(self.jobMonitor.removeAllItems) # Signals out self.jobMonitor.view_object.connect(self.view_object.emit) diff --git a/cuegui/cuegui/plugins/ServicePlugin.py b/cuegui/cuegui/plugins/ServicePlugin.py index cc2a99bef..009ed450b 100644 --- a/cuegui/cuegui/plugins/ServicePlugin.py +++ b/cuegui/cuegui/plugins/ServicePlugin.py @@ -20,8 +20,6 @@ from __future__ import division from __future__ import absolute_import -from PySide2 import QtGui - import cuegui.AbstractDockWidget import cuegui.ServiceDialog @@ -41,6 +39,4 @@ def __init__(self, parent): self.setWindowTitle("Facility Service Defaults") self.__serviceManager = cuegui.ServiceDialog.ServiceManager(None, self) self.layout().addWidget(self.__serviceManager) - # pylint: disable=no-member - QtGui.qApp.facility_changed.connect(self.__serviceManager.refresh) - # pylint: enable=no-member + self.app.facility_changed.connect(self.__serviceManager.refresh) diff --git a/cuegui/cuegui/plugins/StuckFramePlugin.py b/cuegui/cuegui/plugins/StuckFramePlugin.py index 9a289b49b..f3d5e2904 100644 --- a/cuegui/cuegui/plugins/StuckFramePlugin.py +++ b/cuegui/cuegui/plugins/StuckFramePlugin.py @@ -35,12 +35,17 @@ from PySide2 import QtCore from PySide2 import QtWidgets -import opencue +import opencue.wrappers.frame + import cuegui.AbstractDockWidget +import cuegui.AbstractTreeWidget +import cuegui.AbstractWidgetItem import cuegui.Action import cuegui.Constants import cuegui.JobMonitorTree import cuegui.Logger +import cuegui.MenuActions +import cuegui.Style import cuegui.Utils logger = cuegui.Logger.getLogger(__file__) @@ -772,7 +777,7 @@ def __init__(self, parent): self._updateProgressMax.connect(self.updateProgressMax) # Don't use the standard space bar to refres - self.disconnect(QtGui.qApp, + self.disconnect(self.app, QtCore.SIGNAL('request_update()'), self.updateRequest) @@ -798,10 +803,10 @@ def __init__(self, parent): def logIt(self): """Logs cache to a file.""" - if hasattr(QtGui.qApp, "threadpool"): + if self.app.threadpool is not None: print("Stuck Frame Log cache is being written to file.") - QtGui.qApp.threadpool.queue(self.run_log.finalize, self.logResult, - "Writing out log", self.frames, self.show) + self.app.threadpool.queue( + self.run_log.finalize, self.logResult, "Writing out log", self.frames, self.show) else: logger.warning("threadpool not found, doing work in gui thread") @@ -1214,7 +1219,7 @@ def contextMenuEvent(self, e): menu.addSeparator() menu.addAction(cuegui.Action.create(self, "Top Machine", "Top Machine", self.topMachine, "up")) - if QtGui.qApp.applicationName() == "CueCommander3": + if self.app.applicationName() == "CueCommander3": self.__menuActions.frames().addAction(menu, "viewHost") menu.addSeparator() @@ -1603,7 +1608,7 @@ def __init__(self, object, parent): if not self.__initialized: self.__class__.__initialized = True self.__class__.__commentIcon = QtGui.QIcon(":comment.png") - self.__class__.__backgroundColor = QtGui.qApp.palette().color(QtGui.QPalette.Base) + self.__class__.__backgroundColor = cuegui.app().palette().color(QtGui.QPalette.Base) self.__class__.__foregroundColor = cuegui.Style.ColorTheme.COLOR_JOB_FOREGROUND cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__(self, cuegui.Constants.TYPE_FRAME, diff --git a/cuegui/tests/CueJobMonitorTree_tests.py b/cuegui/tests/CueJobMonitorTree_tests.py index 87dd0a635..2cf93d0d5 100644 --- a/cuegui/tests/CueJobMonitorTree_tests.py +++ b/cuegui/tests/CueJobMonitorTree_tests.py @@ -39,8 +39,8 @@ class CueJobMonitorTreeTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, get_stub_mock): - test_utils.createApplication() - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + app = test_utils.createApplication() + app.settings = PySide2.QtCore.QSettings() cuegui.Style.init() self.show_name = 'arbitrary-show-name' diff --git a/cuegui/tests/DependWizard_tests.py b/cuegui/tests/DependWizard_tests.py index 0e8667460..de1632502 100644 --- a/cuegui/tests/DependWizard_tests.py +++ b/cuegui/tests/DependWizard_tests.py @@ -40,8 +40,8 @@ class DependWizardTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): - test_utils.createApplication() - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + app = test_utils.createApplication() + app.settings = PySide2.QtCore.QSettings() cuegui.Style.init() self.parentWidget = PySide2.QtWidgets.QWidget() diff --git a/cuegui/tests/FilterDialog_tests.py b/cuegui/tests/FilterDialog_tests.py index 071a2dab9..bfe4c5832 100644 --- a/cuegui/tests/FilterDialog_tests.py +++ b/cuegui/tests/FilterDialog_tests.py @@ -41,8 +41,8 @@ class FilterDialogTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, getStubMock): - test_utils.createApplication() - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + app = test_utils.createApplication() + app.settings = PySide2.QtCore.QSettings() cuegui.Style.init() self.show = opencue.wrappers.show.Show(opencue.compiled_proto.show_pb2.Show(name='fooShow')) @@ -154,8 +154,8 @@ class FilterMonitorTreeTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, getStubMock): - test_utils.createApplication() - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + app = test_utils.createApplication() + app.settings = PySide2.QtCore.QSettings() cuegui.Style.init() show = opencue.wrappers.show.Show(opencue.compiled_proto.show_pb2.Show(name='fooShow')) @@ -199,8 +199,8 @@ class MatcherMonitorTreeTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, getStubMock): - test_utils.createApplication() - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + app = test_utils.createApplication() + app.settings = PySide2.QtCore.QSettings() cuegui.Style.init() self.matchers = [ diff --git a/cuegui/tests/FrameMonitorTree_tests.py b/cuegui/tests/FrameMonitorTree_tests.py index c3c2b4963..fc77e864c 100644 --- a/cuegui/tests/FrameMonitorTree_tests.py +++ b/cuegui/tests/FrameMonitorTree_tests.py @@ -43,9 +43,8 @@ class FrameMonitorTreeTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): - test_utils.createApplication() - - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + app = test_utils.createApplication() + app.settings = PySide2.QtCore.QSettings() cuegui.Style.init() self.parentWidget = PySide2.QtWidgets.QWidget() self.frameMonitorTree = cuegui.FrameMonitorTree.FrameMonitorTree(self.parentWidget) diff --git a/cuegui/tests/LayerDialog_tests.py b/cuegui/tests/LayerDialog_tests.py index 5e515775d..78d4d5060 100644 --- a/cuegui/tests/LayerDialog_tests.py +++ b/cuegui/tests/LayerDialog_tests.py @@ -47,8 +47,8 @@ class LayerPropertiesDialogTests(unittest.TestCase): @mock.patch('opencue.api.getLayer') @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, get_stub_mock, get_layer_mock, get_limits_mock): - test_utils.createApplication() - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + app = test_utils.createApplication() + app.settings = PySide2.QtCore.QSettings() cuegui.Style.init() self.layers = { diff --git a/cuegui/tests/MenuActions_tests.py b/cuegui/tests/MenuActions_tests.py index b15a2cc9a..25033c79b 100644 --- a/cuegui/tests/MenuActions_tests.py +++ b/cuegui/tests/MenuActions_tests.py @@ -52,7 +52,7 @@ import cuegui.CueJobMonitorTree import cuegui.Main import cuegui.MenuActions - +from . import test_utils _GB_TO_KB = 1024 * 1024 @@ -60,6 +60,7 @@ @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) class JobActionsTests(unittest.TestCase): def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.job_actions = cuegui.MenuActions.JobActions(self.widgetMock, mock.Mock(), None, None) @@ -71,14 +72,14 @@ def test_unmonitor(self): self.widgetMock.actionRemoveSelectedItems.assert_called_with() - @mock.patch('PySide2.QtGui.qApp') - def test_view(self, qAppMock): + def test_view(self): + self.app.view_object = mock.Mock() job_name = 'arbitrary-name' job = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(name=job_name)) self.job_actions.view(rpcObjects=[job, opencue.wrappers.frame.Frame()]) - qAppMock.view_object.emit.assert_called_once_with(job) + self.app.view_object.emit.assert_called_once_with(job) @mock.patch('cuegui.DependDialog.DependDialog') def test_viewDepends(self, dependDialogMock): @@ -545,6 +546,7 @@ class LayerActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.job = mock.create_autospec(opencue.wrappers.job.Job()) self.layer_actions = cuegui.MenuActions.LayerActions( @@ -793,6 +795,7 @@ class FrameActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.job = mock.create_autospec(opencue.wrappers.job.Job()) self.frame_actions = cuegui.MenuActions.FrameActions( @@ -870,8 +873,9 @@ def test_xdiff3(self, popupFrameXdiffMock): popupFrameXdiffMock.assert_called_with(self.job, frame1, frame2, frame3) @mock.patch('opencue.api.findHost') - @mock.patch('PySide2.QtGui.qApp') - def test_viewHost(self, qAppMock, findHostMock): + def test_viewHost(self, findHostMock): + self.app.view_hosts = mock.Mock() + self.app.single_click = mock.Mock() host_name = 'arbitrary-host-name' host = opencue.wrappers.host.Host( opencue.compiled_proto.host_pb2.Host(id='arbitrary-id', name=host_name)) @@ -881,8 +885,8 @@ def test_viewHost(self, qAppMock, findHostMock): self.frame_actions.viewHost(rpcObjects=[frame]) - qAppMock.view_hosts.emit.assert_called_with([host_name]) - qAppMock.single_click.emit.assert_called_with(host) + self.app.view_hosts.emit.assert_called_with([host_name]) + self.app.single_click.emit.assert_called_with(host) def test_getWhatThisDependsOn(self): frame = opencue.wrappers.frame.Frame() @@ -1049,6 +1053,7 @@ class ShowActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.show_actions = cuegui.MenuActions.ShowActions( self.widgetMock, mock.Mock(), None, None) @@ -1085,6 +1090,7 @@ class GroupActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.group_actions = cuegui.MenuActions.GroupActions( self.widgetMock, mock.Mock(), None, None) @@ -1122,6 +1128,7 @@ class SubscriptionActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.subscription_actions = cuegui.MenuActions.SubscriptionActions( self.widgetMock, mock.Mock(), None, None) @@ -1175,6 +1182,7 @@ class HostActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.host_actions = cuegui.MenuActions.HostActions( self.widgetMock, mock.Mock(), None, None) @@ -1188,15 +1196,15 @@ def test_viewComments(self, commentListDialogMock): commentListDialogMock.assert_called_with(host, mock.ANY) commentListDialogMock.return_value.show.assert_called() - @mock.patch('PySide2.QtGui.qApp') - def test_viewProc(self, qAppMock): + def test_viewProc(self): + self.app.view_procs = mock.Mock() hostName = 'arbitrary-name' host = opencue.wrappers.host.Host( opencue.compiled_proto.host_pb2.Host(id='arbitrary-id', name=hostName)) self.host_actions.viewProc(rpcObjects=[opencue.wrappers.layer.Layer, host, host]) - qAppMock.view_procs.emit.assert_called_with([hostName]) + self.app.view_procs.emit.assert_called_with([hostName]) def test_lock(self): host = opencue.wrappers.host.Host( @@ -1337,13 +1345,14 @@ class ProcActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.proc_actions = cuegui.MenuActions.ProcActions( self.widgetMock, mock.Mock(), None, None) - @mock.patch('PySide2.QtGui.qApp') @mock.patch('opencue.api.findJob') - def test_view(self, findJobMock, qAppMock): + def test_view(self, findJobMock): + self.app.view_object = mock.Mock() jobName = 'arbitraryJobName' job = opencue.wrappers.job.Job(opencue.compiled_proto.job_pb2.Job(name=jobName)) proc = opencue.wrappers.proc.Proc(opencue.compiled_proto.host_pb2.Proc(job_name=jobName)) @@ -1351,7 +1360,7 @@ def test_view(self, findJobMock, qAppMock): self.proc_actions.view(rpcObjects=[opencue.wrappers.layer.Layer, proc]) - qAppMock.view_object.emit.assert_called_once_with(job) + self.app.view_object.emit.assert_called_once_with(job) @mock.patch('cuegui.Utils.questionBoxYesNo', new=mock.Mock(return_value=True)) def test_kill(self): @@ -1386,6 +1395,7 @@ class DependenciesActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.dep_actions = cuegui.MenuActions.DependenciesActions( self.widgetMock, mock.Mock(), None, None) @@ -1412,6 +1422,7 @@ class FilterActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.filter_actions = cuegui.MenuActions.FilterActions( self.widgetMock, mock.Mock(), None, None) @@ -1485,6 +1496,7 @@ class MatcherActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.matcher_actions = cuegui.MenuActions.MatcherActions( self.widgetMock, mock.Mock(), None, None) @@ -1515,6 +1527,7 @@ class ActionActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.action_actions = cuegui.MenuActions.ActionActions( self.widgetMock, mock.Mock(), None, None) @@ -1534,6 +1547,7 @@ class TaskActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.task_actions = cuegui.MenuActions.TaskActions( self.widgetMock, mock.Mock(), None, None) @@ -1572,6 +1586,7 @@ class LimitActionsTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.limit_actions = cuegui.MenuActions.LimitActions( self.widgetMock, mock.Mock(), None, None) @@ -1622,6 +1637,7 @@ def test_rename(self, getTextMock): @mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) class MenuActionsTests(unittest.TestCase): def setUp(self): + self.app = test_utils.createApplication() self.widgetMock = mock.Mock() self.args = [self.widgetMock, lambda: None, lambda: None, lambda: None] self.menuActions = cuegui.MenuActions.MenuActions(*self.args) diff --git a/cuegui/tests/Redirect_tests.py b/cuegui/tests/Redirect_tests.py index d1e7ce5dd..5e1f6ad27 100644 --- a/cuegui/tests/Redirect_tests.py +++ b/cuegui/tests/Redirect_tests.py @@ -36,8 +36,8 @@ class RedirectTests(unittest.TestCase): @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, getStubMock): - test_utils.createApplication() - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + app = test_utils.createApplication() + app.settings = PySide2.QtCore.QSettings() cuegui.Style.init() getStubMock.return_value.GetActiveShows.return_value = \ diff --git a/cuegui/tests/UnbookDialog_tests.py b/cuegui/tests/UnbookDialog_tests.py index 2482aa201..a62b390e7 100644 --- a/cuegui/tests/UnbookDialog_tests.py +++ b/cuegui/tests/UnbookDialog_tests.py @@ -46,8 +46,8 @@ class UnbookDialogTests(unittest.TestCase): @mock.patch('opencue.api.findShow') @mock.patch('opencue.cuebot.Cuebot.getStub') def setUp(self, get_stub_mock, find_show_mock): - test_utils.createApplication() - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + app = test_utils.createApplication() + app.settings = PySide2.QtCore.QSettings() cuegui.Style.init() show_name = 'showname' @@ -202,8 +202,8 @@ def test__should_redirect_proc_to_job( class SelectItemsWithSearchDialogTests(unittest.TestCase): def setUp(self): - test_utils.createApplication() - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + app = test_utils.createApplication() + app.settings = PySide2.QtCore.QSettings() cuegui.Style.init() def test__should_display_all_items(self): @@ -245,8 +245,8 @@ def test__should_return_selected_items(self): class KillConfirmationDialogTests(unittest.TestCase): def setUp(self): - test_utils.createApplication() - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + app = test_utils.createApplication() + app.settings = PySide2.QtCore.QSettings() cuegui.Style.init() @mock.patch('PySide2.QtWidgets.QMessageBox.information', new=mock.Mock()) diff --git a/cuegui/tests/plugins/LogViewPlugin_tests.py b/cuegui/tests/plugins/LogViewPlugin_tests.py index 9ceb4a5b6..8206176f8 100644 --- a/cuegui/tests/plugins/LogViewPlugin_tests.py +++ b/cuegui/tests/plugins/LogViewPlugin_tests.py @@ -64,26 +64,26 @@ def setUp(self): self.fs.create_file(self.logPath2, contents=_LOG_TEXT_2) test_utils.createApplication() - PySide2.QtGui.qApp.settings = PySide2.QtCore.QSettings() + cuegui.app().settings = PySide2.QtCore.QSettings() cuegui.Style.init() self.parentWidget = PySide2.QtWidgets.QMainWindow() self.logViewPlugin = cuegui.plugins.LogViewPlugin.LogViewPlugin(self.parentWidget) def test_shouldDisplayFirstLogFile(self): - PySide2.QtGui.qApp.display_log_file_content.emit([self.logPath1, self.logPath2]) + cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) self.assertEqual(_LOG_TEXT_1, self.logViewPlugin.logview_widget._content_box.toPlainText()) def test_shouldUpdateLogFile(self): - PySide2.QtGui.qApp.display_log_file_content.emit([self.logPath1, self.logPath2]) + cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) new_contents = _LOG_TEXT_1 + '\nanother line at the end' self.log1.set_contents(new_contents) - PySide2.QtGui.qApp.display_log_file_content.emit([self.logPath1, self.logPath2]) + cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) self.assertEqual(new_contents, self.logViewPlugin.logview_widget._content_box.toPlainText()) def test_shouldHighlightAllSearchResults(self): - PySide2.QtGui.qApp.display_log_file_content.emit([self.logPath1, self.logPath2]) + cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( PySide2.QtCore.Qt.CheckState.Unchecked) @@ -100,7 +100,7 @@ def test_shouldHighlightAllSearchResults(self): self.logViewPlugin.logview_widget._content_box, matches[1][0], matches[1][1])) def test_shouldMoveCursorToSecondSearchResult(self): - PySide2.QtGui.qApp.display_log_file_content.emit([self.logPath1, self.logPath2]) + cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( PySide2.QtCore.Qt.CheckState.Unchecked) @@ -114,7 +114,7 @@ def test_shouldMoveCursorToSecondSearchResult(self): self.assertEqual(132, self.logViewPlugin.logview_widget._cursor.position()) def test_shouldMoveCursorLastSearchResult(self): - PySide2.QtGui.qApp.display_log_file_content.emit([self.logPath1, self.logPath2]) + cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( PySide2.QtCore.Qt.CheckState.Unchecked) @@ -128,7 +128,7 @@ def test_shouldMoveCursorLastSearchResult(self): self.assertEqual(132, self.logViewPlugin.logview_widget._cursor.position()) def test_shouldPerformCaseInsensitiveSearch(self): - PySide2.QtGui.qApp.display_log_file_content.emit([self.logPath1, self.logPath2]) + cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2]) self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState( PySide2.QtCore.Qt.CheckState.Checked) diff --git a/cuegui/tests/test_utils.py b/cuegui/tests/test_utils.py index dfcb446db..dd07840d4 100644 --- a/cuegui/tests/test_utils.py +++ b/cuegui/tests/test_utils.py @@ -16,17 +16,8 @@ """Common utility functions for CueGUI test code.""" -import PySide2.QtGui +import cuegui -import cuegui.Main - -__QAPPLICATION_SINGLETON = None - - -# pylint: disable=global-statement def createApplication(): - global __QAPPLICATION_SINGLETON - if __QAPPLICATION_SINGLETON is None: - __QAPPLICATION_SINGLETON = cuegui.Main.CueGuiApplication() - PySide2.QtGui.qApp = __QAPPLICATION_SINGLETON + return cuegui.create_app([]) From c8eb6cad0cc1ce61d863afe2ea0ee7af10306751 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Mon, 7 Nov 2022 12:24:57 -0500 Subject: [PATCH 19/21] Notes from last two TSC meetings. (#1216) --- tsc/meetings/2022-09-28.md | 31 +++++++++++++++++++++++++++++++ tsc/meetings/2022-10-26.md | 24 ++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 tsc/meetings/2022-09-28.md create mode 100644 tsc/meetings/2022-10-26.md diff --git a/tsc/meetings/2022-09-28.md b/tsc/meetings/2022-09-28.md new file mode 100644 index 000000000..f22833839 --- /dev/null +++ b/tsc/meetings/2022-09-28.md @@ -0,0 +1,31 @@ +# OpenCue TSC Meeting Notes 28 Sep 2022 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Update on M1 issues / CI failures + * All issues now have proposed fixes. Working our way through review. +* Preparing for next release + * qApp PR needs review/merge. This is the highest priority to unbreak our CI pipelines. + * Logging flags. Dropping --log.frame-log-root is a breaking change. Continue supporting it for + now? Add a deprecation warning? + * Possible solution: https://github.com/AcademySoftwareFoundation/OpenCue/pull/1203 + * Could also just rename the new flags. + * Will follow up on the PR. + * Integration test script + * Existing script is very basic and SPI specific. Won't really save us time vs writing from + scratch. + * CueGUI is the challenge here. Automating API calls is trivial, automating GUI tests much + more involved. + * Script to test our sandbox setup is a good place to start. This is critical for ensuring + new users have a good experience. + * "testing" show deactivated? + * Reported in https://github.com/AcademySoftwareFoundation/OpenCue/issues/1205 + * Culprit PR: ​​https://github.com/AcademySoftwareFoundation/OpenCue/pull/1151 + * Does automated show cleanup, testing show is not in the allowlist. + * Quick fix: allowlist testing show in properties file. + * Permanent fix: change 30 day hardcoded limit to come from properties files. -1 means + feature is disabled. Disable this by default. +* Redis + * No update yet, work to resume later. diff --git a/tsc/meetings/2022-10-26.md b/tsc/meetings/2022-10-26.md new file mode 100644 index 000000000..d4c21d39f --- /dev/null +++ b/tsc/meetings/2022-10-26.md @@ -0,0 +1,24 @@ +# OpenCue TSC Meeting Notes 26 Oct 2022 + +Secretary: Brian Cipriano + +Agenda/Notes: + +* Intros + * Holovisn.com + * Intro from team, discussion of OpenCue use case, demo. +* RQD completely ignores env on Windows + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/1211 + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1215 + * LGTM +* RQD doesn't Nimby with Remote users on machine + * https://github.com/AcademySoftwareFoundation/OpenCue/issues/564 + * Diego: this is fixed by using pynput. PR for this has been merged, release is needed. +* Release blockers + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1193 + * Testing/review will commence shortly. + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1203 + * Let's merge the PR. Not ideal but lesson is learned for the future. + * https://github.com/AcademySoftwareFoundation/OpenCue/pull/1199 + * Brian to try again to see if problem persists. + * Diego to ask DBA about the issue and potential performance issues created by the fix. From 35aa4612d70fc15c1b2b6a97bde2f0f3aed7a29a Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 6 Dec 2022 12:29:29 -0500 Subject: [PATCH 20/21] [cuebot] Automatically replace --log.frame-log-root with the new flag. (#1203) --- .../imageworks/spcue/CuebotApplication.java | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/CuebotApplication.java b/cuebot/src/main/java/com/imageworks/spcue/CuebotApplication.java index a5747e558..12087f93e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/CuebotApplication.java +++ b/cuebot/src/main/java/com/imageworks/spcue/CuebotApplication.java @@ -16,18 +16,48 @@ */ - package com.imageworks.spcue; +import java.util.Arrays; +import java.util.Optional; +import java.util.stream.Stream; + +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @SpringBootApplication public class CuebotApplication extends SpringApplication { + private static String[] checkArgs(String[] args) { + Optional deprecatedFlag = Arrays.stream(args) + .filter(arg -> arg.startsWith("--log.frame-log-root=")).findFirst(); + if (deprecatedFlag.isPresent()) { + // Log a deprecation warning. + Logger warning_logger = Logger.getLogger(CuebotApplication.class); + warning_logger.warn("`--log.frame-log-root` is deprecated and will be removed in an " + + "upcoming release. It has been replaced with `--log.frame-log-root.default_os`. " + + "See opencue.properties for details on OpenCue's new OS-dependent root directories."); + // If new flags are not present, swap in the value provided using the new flag. + // If the new flags are already present, don't do anything. + Optional newFlags = Arrays.stream(args) + .filter(arg -> arg.startsWith("--log.frame-log-root.")).findAny(); + if (!newFlags.isPresent()) { + String fixedFlag = "--log.frame-log-root.default_os=" + + StringUtils.substringAfter(deprecatedFlag.get(), "="); + args = Stream.concat( + Arrays.stream(args).filter(arg -> !arg.startsWith("--log.frame-log-root=")), + Stream.of(fixedFlag)) + .toArray(String[]::new); + } + } + return args; + } public static void main(String[] args) { // Cuebot startup - SpringApplication.run(CuebotApplication.class, args); + String[] filteredArgs = checkArgs(args); + SpringApplication.run(CuebotApplication.class, filteredArgs); } } From 529d606c67012a6630ac34838eb626fe03b69871 Mon Sep 17 00:00:00 2001 From: Brian Cipriano Date: Tue, 6 Dec 2022 12:36:37 -0500 Subject: [PATCH 21/21] [cuebot] Remove some stray debugging log messages. (#1209) --- .../com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index 7a1a85351..9ffe56beb 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -548,7 +548,6 @@ public ProcSeq getProcs(HostInterface host) { r.filterByHost(host); r.sortByHostName(); r.sortByDispatchedTime(); - logger.info("!!!! INSIDE getProcs Whiteboard!!! called getProcs !!! line 551"); return ProcSeq.newBuilder().addAllProcs(getProcs(r).getProcsList()).build(); } @@ -556,7 +555,6 @@ public ProcSeq getProcs(HostInterface host) { public ProcSeq getProcs(ProcSearchInterface p) { p.sortByHostName(); p.sortByDispatchedTime(); - logger.info("!!!! Inside getPROCS!!!!! line 559"); List procs = getJdbcTemplate().query(p.getFilteredQuery(GET_PROC), PROC_MAPPER, p.getValuesArray()); return ProcSeq.newBuilder().addAllProcs(procs).build(); @@ -976,7 +974,6 @@ public Proc mapRow(ResultSet rs, int row) throws SQLException { .addAllServices(Arrays.asList(SqlUtil.getString(rs,"str_services").split(","))) .build(); } -// logger.info("called ROW MAPPER!!! setChildProcesses!!!"); }; public static final RowMapper TASK_MAPPER =