diff --git a/.github/workflows/testing-pipeline.yml b/.github/workflows/testing-pipeline.yml index 15967a4a9..c3816db57 100644 --- a/.github/workflows/testing-pipeline.yml +++ b/.github/workflows/testing-pipeline.yml @@ -11,6 +11,8 @@ jobs: name: Run Python Unit Tests (CY2022) runs-on: ubuntu-latest container: aswf/ci-opencue:2022 + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: - uses: actions/checkout@v3 - name: Run Python Tests @@ -21,6 +23,8 @@ jobs: runs-on: ubuntu-latest container: image: aswf/ci-opencue:2022 + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: - uses: actions/checkout@v3 - name: Build with Gradle @@ -53,6 +57,8 @@ jobs: name: Run Python Unit Tests using Python2 runs-on: ubuntu-latest container: aswf/ci-opencue:2019 + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: - uses: actions/checkout@v3 - name: Run Python Tests @@ -71,6 +77,8 @@ jobs: name: Lint Python Code runs-on: ubuntu-latest container: aswf/ci-opencue:2022 + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: - uses: actions/checkout@v3 - name: Lint Python Code @@ -93,7 +101,7 @@ jobs: - uses: actions/checkout@v3 - name: Get Changed Files id: get_changed_files - uses: tj-actions/changed-files@v35 + uses: tj-actions/changed-files@v41 - name: Check for Version Change run: ci/check_changed_files.py ${{ steps.get_changed_files.outputs.modified_files }} ${{ steps.get_changed_files.outputs.deleted_files }} @@ -112,6 +120,6 @@ jobs: - uses: actions/checkout@v3 - name: Get Changed Files id: get_changed_files - uses: tj-actions/changed-files@v35 + uses: tj-actions/changed-files@v41 - name: Check for Version Change run: ci/check_version_bump.py ${{ steps.get_changed_files.outputs.all_changed_and_modified_files }} diff --git a/VERSION.in b/VERSION.in index eec15f902..f7c6c31b6 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -0.29 +0.30 diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/LayerDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/LayerDaoJdbc.java index 15941a196..f555bef6e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/LayerDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/LayerDaoJdbc.java @@ -77,7 +77,9 @@ public void insertLayerOutput(LayerInterface layer, String filespec) { "FROM " + "layer_output " + "WHERE " + - "pk_layer = ?"; + "pk_layer = ?" + + "ORDER BY " + + "ser_order"; private static final RowMapper OUTPUT_MAPPER = new RowMapper() { diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java index 1c042f110..c31edc53c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java @@ -100,6 +100,7 @@ public class HostReportHandler { private static final String SUBJECT_COMMENT_FULL_TEMP_DIR = "Host set to REPAIR for not having enough storage " + "space on the temporary directory (mcp)"; private static final String CUEBOT_COMMENT_USER = "cuebot"; + private static final String WINDOWS_OS = "Windows"; // A cache to store kill requests and count the number of occurrences. // The cache expires after write to avoid growing unbounded. If a request for a host-frame doesn't appear @@ -182,7 +183,11 @@ public void handleHostReport(HostReport report, boolean isBoot) { rhost.getLoad(), new Timestamp(rhost.getBootTime() * 1000l), rhost.getAttributesMap().get("SP_OS")); - changeHardwareState(host, report.getHost().getState(), isBoot, report.getHost().getFreeMcp()); + // Both logics are conflicting, only change hardware state if + // there was no need for a tempDirStorage state change + if (!changeStateForTempDirStorage(host, report.getHost())) { + changeHardwareState(host, report.getHost().getState(), isBoot); + } changeNimbyState(host, report.getHost()); /** @@ -247,12 +252,10 @@ public void handleHostReport(HostReport report, boolean isBoot) { } } - // The minimum amount of free space in the temporary directory to book a host - Long minBookableFreeTempDir = env.getRequiredProperty("dispatcher.min_bookable_free_temp_dir_kb", Long.class); - - if (minBookableFreeTempDir != -1 && report.getHost().getFreeMcp() < minBookableFreeTempDir) { - msg = String.format("%s doens't have enough free space in the temporary directory (mcp), %dMB needs %dMB", - host.name, (report.getHost().getFreeMcp()/1024), (minBookableFreeTempDir/1024)); + if (!isTempDirStorageEnough(report.getHost().getTotalMcp(), report.getHost().getFreeMcp(), host.os)) { + msg = String.format( + "%s doens't have enough free space in the temporary directory (mcp), %dMB", + host.name, (report.getHost().getFreeMcp()/1024)); } else if (host.idleCores < Dispatcher.CORE_POINTS_RESERVED_MIN) { msg = String.format("%s doesn't have enough idle cores, %d needs %d", @@ -333,6 +336,27 @@ else if (!dispatchSupport.isCueBookable(host)) { } } + /** + * Check if a reported temp storage size and availability is enough for running a job + * + * Use dispatcher.min_available_temp_storage_percentage (opencue.properties) to + * define what's the accepted threshold. Providing hostOs is necessary as this feature + * is currently not available on Windows hosts + * + * @param tempTotalStorage Total storage on the temp directory + * @param tempFreeStorage Free storage on the temp directory + * @param hostOs Reported os + * @return + */ + private boolean isTempDirStorageEnough(Long tempTotalStorage, Long tempFreeStorage, String hostOs) { + // The minimum amount of free space in the temporary directory to book a host + int minAvailableTempPercentage = env.getRequiredProperty( + "dispatcher.min_available_temp_storage_percentage", Integer.class); + + return minAvailableTempPercentage == -1 || hostOs.equalsIgnoreCase(WINDOWS_OS) || + (((tempFreeStorage * 100.0) / tempTotalStorage) >= minAvailableTempPercentage); + } + /** * Update the hardware state property. * @@ -342,62 +366,11 @@ else if (!dispatchSupport.isCueBookable(host)) { * updated with a boot report. If the state is Repair, then state is * never updated via RQD. * - * - * Prevent cue frames from booking on hosts with full temporary directories. - * - * Change host state to REPAIR or UP according the amount of free space - * in the temporary directory: - * - Set the host state to REPAIR, when the amount of free space in the - * temporary directory is less than the minimum required. Add a comment with - * subject: SUBJECT_COMMENT_FULL_TEMP_DIR - * - Set the host state to UP, when the amount of free space in the temporary directory - * is greater or equals to the minimum required and the host has a comment with - * subject: SUBJECT_COMMENT_FULL_TEMP_DIR - * * @param host * @param reportState * @param isBoot - * @param freeTempDir */ - private void changeHardwareState(DispatchHost host, HardwareState reportState, boolean isBoot, long freeTempDir) { - - // The minimum amount of free space in the temporary directory to book a host - Long minBookableFreeTempDir = env.getRequiredProperty("dispatcher.min_bookable_free_temp_dir_kb", Long.class); - - // Prevent cue frames from booking on hosts with full temporary directories - if (minBookableFreeTempDir != -1 && !host.os.equalsIgnoreCase("Windows")) { - if (host.hardwareState == HardwareState.UP && freeTempDir < minBookableFreeTempDir) { - - // Insert a comment indicating that the Host status = Repair with reason = Full temporary directory - CommentDetail c = new CommentDetail(); - c.subject = SUBJECT_COMMENT_FULL_TEMP_DIR; - c.user = CUEBOT_COMMENT_USER; - c.timestamp = null; - c.message = "Host " + host.getName() + " marked as REPAIR. The current amount of free space in the " + - "temporary directory (mcp) is " + (freeTempDir/1024) + "MB. It must have at least " - + (minBookableFreeTempDir/1024) + "MB of free space in temporary directory"; - commentManager.addComment(host, c); - - // Set the host state to REPAIR - hostManager.setHostState(host, HardwareState.REPAIR); - host.hardwareState = HardwareState.REPAIR; - - return; - } else if (host.hardwareState == HardwareState.REPAIR && freeTempDir >= minBookableFreeTempDir) { - // Check if the host with REPAIR status has comments with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and - // user=CUEBOT_COMMENT_USER and delete the comments, if they exists - boolean commentsDeleted = commentManager.deleteCommentByHostUserAndSubject(host, - CUEBOT_COMMENT_USER, SUBJECT_COMMENT_FULL_TEMP_DIR); - - if (commentsDeleted) { - // Set the host state to UP - hostManager.setHostState(host, HardwareState.UP); - host.hardwareState = HardwareState.UP; - return; - } - } - } - + private void changeHardwareState(DispatchHost host, HardwareState reportState, boolean isBoot) { // If the states are the same there is no reason to do this update. if (host.hardwareState.equals(reportState)) { return; @@ -427,6 +400,61 @@ private void changeHardwareState(DispatchHost host, HardwareState reportState, b } } + /** + * Prevent cue frames from booking on hosts with full temporary directories. + * + * Change host state to REPAIR or UP according to the amount of free space + * in the temporary directory: + * - Set the host state to REPAIR, when the amount of free space in the + * temporary directory is less than the minimum required. + * - Set the host state to UP, when the amount of free space in the temporary directory + * is greater or equal to the minimum required and the host has a comment with + * subject: SUBJECT_COMMENT_FULL_TEMP_DIR + * + * @param host + * @param reportHost + * @return + */ + private boolean changeStateForTempDirStorage(DispatchHost host, RenderHost reportHost) { + // The minimum amount of free space in the temporary directory to book a host + int minAvailableTempPercentage = env.getRequiredProperty( + "dispatcher.min_available_temp_storage_percentage", Integer.class); + + // Prevent cue frames from booking on hosts with full temporary directories + boolean hasEnoughTempStorage = isTempDirStorageEnough(reportHost.getTotalMcp(), reportHost.getFreeMcp(), host.os); + if (!hasEnoughTempStorage && host.hardwareState == HardwareState.UP) { + // Insert a comment indicating that the Host status = Repair with reason = Full temporary directory + CommentDetail c = new CommentDetail(); + c.subject = SUBJECT_COMMENT_FULL_TEMP_DIR; + c.user = CUEBOT_COMMENT_USER; + c.timestamp = null; + long requiredTempMb = (long)((minAvailableTempPercentage / 100.0) * reportHost.getTotalMcp()/ 1024); + c.message = "Host " + host.getName() + " marked as REPAIR. The current amount of free space in the " + + "temporary directory (mcp) is " + (reportHost.getFreeMcp()/1024) + "MB. It must have at least " + + ((requiredTempMb)) + "MB of free space in temporary directory"; + commentManager.addComment(host, c); + + // Set the host state to REPAIR + hostManager.setHostState(host, HardwareState.REPAIR); + host.hardwareState = HardwareState.REPAIR; + + return true; + } else if (hasEnoughTempStorage && host.hardwareState == HardwareState.REPAIR) { + // Check if the host with REPAIR status has comments with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and + // user=CUEBOT_COMMENT_USER and delete the comments, if they exist + boolean commentsDeleted = commentManager.deleteCommentByHostUserAndSubject(host, + CUEBOT_COMMENT_USER, SUBJECT_COMMENT_FULL_TEMP_DIR); + + if (commentsDeleted) { + // Set the host state to UP + hostManager.setHostState(host, HardwareState.UP); + host.hardwareState = HardwareState.UP; + return true; + } + } + return false; + } + /** * Changes the NIMBY lock state. If the DB indicates a NIMBY lock * but RQD does not, then the host is unlocked. If the DB indicates diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/EmailSupport.java b/cuebot/src/main/java/com/imageworks/spcue/service/EmailSupport.java index b25e7d520..23e51037d 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/EmailSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/EmailSupport.java @@ -15,23 +15,16 @@ * limitations under the License. */ - - package com.imageworks.spcue.service; -import java.io.ByteArrayOutputStream; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.StringWriter; +import java.io.*; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Properties; +import java.util.stream.Collectors; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -40,6 +33,8 @@ import org.apache.velocity.app.VelocityEngine; import org.jdom.output.Format; import org.jdom.output.XMLOutputter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; import org.springframework.mail.MailException; import org.springframework.mail.MailSender; import org.springframework.mail.SimpleMailMessage; @@ -61,49 +56,19 @@ public class EmailSupport { private MailSender mailSender; private JobManager jobManager; + private String emailDomain; + private String emailFromAddress; + private String[] emailCcAddresses; - private Properties opencueProperties; - - private final Map imageMap; + private Map imageMap; private static final Logger logger = LogManager.getLogger(EmailSupport.class); - public EmailSupport() { - - /* - * The OpenCue configuration file which we need to find the email template. - */ - opencueProperties = getOpenCueProperties(); - - Map map = new HashMap(); - - loadImage(map, "bar.png"); - loadImage(map, "opencue.png"); - loadImage(map, "fail.png"); - loadImage(map, "frame.png"); - loadImage(map, "graph_bar.png"); - loadImage(map, "header.png"); - loadImage(map, "html_bg.png"); - loadImage(map, "logo.png"); - loadImage(map, "memory.png"); - loadImage(map, "play.png"); - loadImage(map, "success.png"); - loadImage(map, "services/comp.png"); - loadImage(map, "services/default.png"); - loadImage(map, "services/ginsu.png"); - loadImage(map, "services/houdini.png"); - loadImage(map, "services/katana.png"); - loadImage(map, "services/maya.png"); - loadImage(map, "services/mentalray.png"); - loadImage(map, "services/nuke.png"); - loadImage(map, "services/playblast.png"); - loadImage(map, "services/prman.png"); - loadImage(map, "services/shell.png"); - loadImage(map, "services/simulation.png"); - loadImage(map, "services/svea.png"); - loadImage(map, "services/trinity.png"); - - imageMap = Collections.unmodifiableMap(map); + @Autowired + public EmailSupport(Environment env) { + this.emailDomain = env.getProperty("email.domain", "opencue.io"); + this.emailFromAddress = env.getProperty("email.from.address", "opencue-noreply@opencue.io"); + this.emailCcAddresses = env.getProperty("email.cc.addresses", "").split(","); } private static void loadImage(Map map, String path) { @@ -172,9 +137,9 @@ private static void loadImage(Map map, String path) { public void reportLaunchError(JobSpec spec, Throwable t) { SimpleMailMessage msg = new SimpleMailMessage(); - msg.setTo(String.format("%s@imageworks.com", spec.getUser())); - msg.setFrom("middle-tier@imageworks.com"); - msg.setCc("middle-tier@imageworks.com"); + msg.setTo(String.format("%s@%s", spec.getUser(), this.emailDomain)); + msg.setFrom(this.emailFromAddress); + msg.setCc(this.emailCcAddresses); msg.setSubject("Failed to launch OpenCue job."); StringBuilder sb = new StringBuilder(131072); @@ -187,7 +152,7 @@ public void reportLaunchError(JobSpec spec, Throwable t) { sb.append(" provided below.\n\n"); sb.append("Failed to launch jobs:\n"); - for (BuildableJob job: spec.getJobs()) { + for (BuildableJob job : spec.getJobs()) { sb.append(job.detail.name); sb.append("\n"); } @@ -206,7 +171,7 @@ public void reportJobComment(JobInterface job, CommentDetail c, String[] emails) SimpleMailMessage msg = new SimpleMailMessage(); msg.setTo(emails); - msg.setFrom("opencue-noreply@imageworks.com"); + msg.setFrom(this.emailFromAddress); msg.setSubject("New comment on " + job.getName()); StringBuilder sb = new StringBuilder(8096); @@ -228,52 +193,49 @@ public void sendMessage(SimpleMailMessage message) { } } - public Properties getOpenCueProperties() { - - // get the input stream of the properties file - InputStream in = EmailSupport.class.getClassLoader() - .getResourceAsStream("opencue.properties"); - - Properties props = new java.util.Properties(); - try { - props.load(in); - } catch (IOException e) { - props = new Properties(); - props.setProperty( "resource.loader", "file" ); - props.setProperty("class.resource.loader.class", - "org.apache.velocity.runtime.resource.loader.FileResourceLoader" ); - props.setProperty("file.resource.loader.path", "/opt/opencue/webapps/spcue"); - } - return props; - } - public void sendShutdownEmail(JobInterface job) { JobDetail d = jobManager.getJobDetail(job.getJobId()); - if (d.email == null ) { return; } + if (d.email == null) { + return; + } try { VelocityEngine ve = new VelocityEngine(); ve.setProperty("resource.loader", "class"); - ve.setProperty("class.resource.loader.class", "org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader"); + ve.setProperty("class.resource.loader.class", + "org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader"); ve.init(); VelocityContext context = new VelocityContext(); ExecutionSummary exj = jobManager.getExecutionSummary(job); FrameStateTotals jts = jobManager.getFrameStateTotals(job); + String status = ""; + if (jts.total != jts.succeeded) { + status = "Failed "; + } else { + status = "Succeeded "; + } + context.put("jobName", d.name); + context.put("jobStatus", status.toUpperCase()); context.put("deptName", d.deptName.toUpperCase()); context.put("showName", d.showName.toUpperCase()); + context.put("totalLayers", d.totalLayers); context.put("shotName", d.shot.toUpperCase()); - context.put("totalFrames", String.format("%04d", jts.total)); - context.put("succeededFrames", String.format("%04d", jts.succeeded)); - context.put("failedFrames", String.format("%04d", jts.dead + jts.eaten + jts.waiting)); - context.put("checkpointFrames", String.format("%04d", jts.checkpoint)); + context.put("succeededFrames", jts.succeeded); + context.put("totalFrames", jts.total); + context.put("dependFrames", jts.depend); + context.put("deadFrames", jts.dead); + context.put("waitingFrames", jts.waiting); + context.put("eatenFrames", jts.eaten); + context.put("failedFrames", jts.dead + jts.eaten + jts.waiting); + context.put("checkpointFrames", jts.checkpoint); context.put("maxRSS", String.format(Locale.ROOT, "%.1fGB", exj.highMemoryKb / 1024.0 / 1024.0)); - context.put("coreTime", String.format(Locale.ROOT, "%.1f", + context.put("coreTime", String.format(Locale.ROOT, "%.1f", exj.coreTime / 3600.0)); Template t = ve.getTemplate("/conf/webapp/html/email_template.html"); @@ -281,43 +243,78 @@ public void sendShutdownEmail(JobInterface job) { List layers = jobManager.getLayerDetails(job); List layerStats = new ArrayList(layers.size()); - for (LayerDetail layer: layers) { + boolean shouldCreateFile = false; + + Map map = new HashMap(); + loadImage(map, "opencue_logo.png"); + + for (LayerDetail layer : layers) { if (layer.type.equals(LayerType.RENDER)) { LayerStats stats = new LayerStats(); stats.setDetail(layer); stats.setExecutionSummary(jobManager.getExecutionSummary(layer)); stats.setFrameStateTotals(jobManager.getFrameStateTotals(layer)); stats.setThreadStats(jobManager.getThreadStats(layer)); - stats.setOutputs(jobManager.getLayerOutputs(layer)); + stats.setOutputs(jobManager.getLayerOutputs(layer).stream().sorted().collect(Collectors.toList())); layerStats.add(stats); + if (stats.getOutputs().size() > 3) + shouldCreateFile = true; + if (!layer.services.isEmpty()) + loadImage(map, "services/" + layer.services.toArray()[0] + ".png"); } } + imageMap = Collections.unmodifiableMap(map); + context.put("layers", layerStats); StringWriter w = new StringWriter(); t.merge(context, w); String subject = "OpenCue Job " + d.getName(); - if (jts.total != jts.succeeded) { - subject = "Failed " + subject; - } - else { - subject = "Succeeded " + subject; + + subject = status + subject; + + BufferedWriter output = null; + File file = null; + if (shouldCreateFile) { + try { + file = new File("my_outputs.txt"); + output = new BufferedWriter(new FileWriter(file)); + for (LayerDetail layer : layers) { + if (layer.type.equals(LayerType.RENDER)) { + List sortedNames = jobManager + .getLayerOutputs(layer) + .stream() + .sorted() + .collect(Collectors.toList()); + output.write(layer.name + "\n" + String.join("\n", sortedNames) + "\n"); + } + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + if (output != null) { + try { + output.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } } - String from = "middle-tier@imageworks.com"; for (String email : d.email.split(",")) { try { - CueUtil.sendmail(email, from, subject, new StringBuilder(w.toString()), imageMap); + CueUtil.sendmail(email, this.emailFromAddress, subject, new StringBuilder(w.toString()), imageMap, + file); } catch (Exception e) { // just log and eat if the mail server is down or something // of that nature. logger.info("Failed to send job complete mail, reason: " + e); } } - } - catch (Exception e) { + } catch (Exception e) { e.printStackTrace(); throw new SpcueRuntimeException("Failed " + e, e); } @@ -335,4 +332,3 @@ public void setMailSender(MailSender mailSender) { this.mailSender = mailSender; } } - diff --git a/cuebot/src/main/java/com/imageworks/spcue/servlet/HealthCheckServlet.java b/cuebot/src/main/java/com/imageworks/spcue/servlet/HealthCheckServlet.java index d8af5ab07..9da011ade 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servlet/HealthCheckServlet.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servlet/HealthCheckServlet.java @@ -26,7 +26,6 @@ import com.imageworks.spcue.servant.CueStatic; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.web.servlet.FrameworkServlet; @@ -45,8 +44,6 @@ public class HealthCheckServlet extends FrameworkServlet { private static final Logger logger = LogManager.getLogger(HealthCheckServlet.class); private CueStatic cueStatic; - - @Autowired private Environment env; private enum HealthStatus { @@ -62,6 +59,8 @@ private enum HealthStatus { public void initFrameworkServlet() throws ServletException { this.cueStatic = (CueStatic) Objects.requireNonNull(this.getWebApplicationContext()).getBean("cueStaticServant"); + this.env = (Environment) + Objects.requireNonNull(this.getWebApplicationContext()).getBean("environment"); } private ArrayList getHealthStatus() { @@ -96,7 +95,7 @@ private ArrayList getHealthStatus() { } private void getJobs() { - if (this.cueStatic != null) { + if (this.cueStatic != null && this.env != null) { // Defaults to testing show, which is added as part of the seeding data script String defaultShow = env.getProperty("protected_shows", String.class, "testing").split(",")[0]; diff --git a/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java b/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java index 88b325483..67d4d17e1 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java +++ b/cuebot/src/main/java/com/imageworks/spcue/util/CueUtil.java @@ -19,6 +19,7 @@ package com.imageworks.spcue.util; +import java.io.File; import java.lang.management.ManagementFactory; import java.lang.management.ThreadMXBean; import java.util.ArrayList; @@ -168,7 +169,7 @@ public static int findChunk(List dependOnFrames, int dependErFrame) { * @param body * @param images */ - public static void sendmail(String to, String from, String subject, StringBuilder body, Map images) { + public static void sendmail(String to, String from, String subject, StringBuilder body, Map images, File attachment) { try { Properties props = System.getProperties(); props.put("mail.smtp.host", CueUtil.smtpHost); @@ -199,6 +200,11 @@ public static void sendmail(String to, String from, String subject, StringBuilde imageBodyPart.setHeader("Content-ID", '<' + name + '>'); mimeMultipart.addBodyPart(imageBodyPart); } + if (attachment != null && attachment.length() != 0){ + MimeBodyPart attachmentPart = new MimeBodyPart(); + attachmentPart.attachFile(attachment); + mimeMultipart.addBodyPart(attachmentPart); + } msg.setContent(mimeMultipart); msg.setHeader("X-Mailer", "OpenCueMailer"); diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V30__Add_order_column_to_outputs.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V30__Add_order_column_to_outputs.sql new file mode 100644 index 000000000..cf6244bba --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V30__Add_order_column_to_outputs.sql @@ -0,0 +1,3 @@ +-- Add a serial column to layer_output for getting correct order of outputs + +alter table layer_output add ser_order SERIAL not null; diff --git a/cuebot/src/main/resources/conf/webapp/html/email_template.html b/cuebot/src/main/resources/conf/webapp/html/email_template.html index ff1524d39..4d5ea1be3 100644 --- a/cuebot/src/main/resources/conf/webapp/html/email_template.html +++ b/cuebot/src/main/resources/conf/webapp/html/email_template.html @@ -2,143 +2,259 @@ Job Execution Summary - -
- -
-
- $showName/$shotName -
+ + #if( $jobStatus == "FAILED " ) + #set( $statusColor = "#FF0000") + #else + #set( $statusColor = "#37C837") + #end +
+
+ + + + + + + + + + + + + + + + + + + +
Show: $showName
Shot: $shotName
Job: $jobName
Status: $jobStatus
+ Logo +
+ #set( $successToOverall = $succeededFrames*1.0 / $totalFrames) + #set( $successPercentage = $successToOverall * 100) + #set( $successPercentageRounded = $successPercentage.intValue()) + #set( $waitingToOverall = $waitingFrames*1.0 / $totalFrames) + #set( $waitingPercentage = $waitingToOverall * 100) + #set( $dependToOverall = $dependFrames*1.0 / $totalFrames) + #set( $dependPercentage = $dependToOverall * 100) + #set( $eatenToOverall = $eatenFrames*1.0 / $totalFrames) + #set( $eatenPercentage = $eatenToOverall * 100) + #set( $deadToOverall = $deadFrames*1.0 / $totalFrames) + #set( $deadPercentage = $deadToOverall * 100) +
+ + + + + +
+
+
+ + + + + + + #if( $checkpointFrames > 0) + + + + + #end + #if( $checkpointFrames == "0.0") + + + + + #end + + + + + #if( $eatenFrames > 0) + + + + + #end + #if( $dependFrames > 0) + + + + + #end + #if( $deadFrames > 0) + + + + + #end + #if( $waitingFrames > 0) + + + + + #end + #if( $succeededFrames > 0) + + + + + #end + #if( $failedFrames > 0) + + + + + #end + + + + + +
Peak Memory:$maxRSS
Checkpoints:$checkpointFrames
Proc Hours:$coreTime
Layers:$totalLayers
Eaten Frames:$eatenFrames
Depend Frames:$dependFrames
Dead Frames:$deadFrames
Incomplete Frames:$waitingFrames
Succeeded Frames:$succeededFrames
Failed Frames:$failedFrames
Total Frames:$totalFrames
+
+
+
-
-
-
- $jobName -
-
$deptName
-
-
-
-
- - - - - - - - - - - - - - + #foreach( $layer in $layers ) + + #set( $service = $layer.detail.services) + #set($serviceCommaSeparated ="") + #set($separator="") + #foreach($item in $service) + #set($serviceCommaSeparated = $serviceCommaSeparated + $separator + $item) + #set($separator = ",") + #end + + #set( $range = $layer.detail.range) + #set( $coreTime = $layer.getFormattedProcHours()) + #set( $failedFrames = $layer.getFailedFrames()) + #set( $highMemory = $layer.getFormattedHighMemory()) + #set( $graphLegend = $layer.getGraphLegend()) + #set( $graphData = $layer.getGraphData()) + #set( $graphScale = $layer.getGraphScale()) + #set( $graphUnits = $layer.getGraphUnits()) + #set( $waitingFrames = $layer.frameStateTotals.waiting) + #set( $dependFrames = $layer.frameStateTotals.depend) + #set( $succeededFrames = $layer.frameStateTotals.succeeded) + #set( $totalFrames = $layer.frameStateTotals.total) + #set( $checkpoints = $layer.frameStateTotals.checkpoint) + #set( $deadFrames = $layer.frameStateTotals.dead) + #set( $eatenFrames = $layer.frameStateTotals.eaten) + #set( $incompleteFrames = $layer.frameStateTotals.waiting + $layer.frameStateTotals.depend) + #set( $outputCount = $layer.getOutputs().size()) + #set( $outputs = $layer.outputs) + + #set( $successToOverall = $succeededFrames*1.0 / $totalFrames) + #set( $successPercentage = $successToOverall * 100) + #set( $successPercentageRounded = $successPercentage.intValue()) + #set( $waitingToOverall = $waitingFrames*1.0 / $totalFrames) + #set( $waitingPercentage = $waitingToOverall * 100) + #set( $dependToOverall = $dependFrames*1.0 / $totalFrames) + #set( $dependPercentage = $dependToOverall * 100) + #set( $eatenToOverall = $eatenFrames*1.0 / $totalFrames) + #set( $eatenPercentage = $eatenToOverall * 100) + #set( $deadToOverall = $deadFrames*1.0 / $totalFrames) + #set( $deadPercentage = $deadToOverall * 100) + + #if( "$!service.get(0)" == "" ) + #set( $serviceIconName = "default") + #else + #set( $serviceIconName = $service.get(0)) + #end + +
+
+
$totalFramesTOTAL FRAMES
$succeededFramesSUCCEEDED
$failedFramesFAILED
+ + + + + + +
Layer:$layer.detail.name
+ serviceIcon +
+ + + + + +
+ + - - + + - - ----> + + -
$eatenFramesEATENPeak Memory:$highMemory
$waitingFramesINCOMPLETECheckpoints:$checkpoints
-
-
- - - + + - - + + - - + + -
$maxRSSPEAK MEMORYProc Hours:$coreTime
$checkpointFramesCHECKPOINTSRange:$range
$coreTimePROC HOURSServices:$serviceCommaSeparated
-
-
-
- - - #foreach( $layer in $layers ) - - #set( $service = $layer.detail.services ) - #set( $coreTime = $layer.getFormattedProcHours() ) - #set( $failedFrames = $layer.getFailedFrames() ) - #set( $highMemory = $layer.getFormattedHighMemory() ) - #set( $graphLegend = $layer.getGraphLegend() ) - #set( $graphData = $layer.getGraphData() ) - #set( $graphScale = $layer.getGraphScale() ) - #set( $graphUnits = $layer.getGraphUnits() ) - #set( $outputCount = $layer.getOutputs().size() ) - #set( $waitingFrames = $layer.frameStateTotals.waiting ) - #set( $deadFrames = $layer.frameStateTotals.dead ) - #set( $eatenFrames = $layer.frameStateTotals.eaten ) - #set( $incompleteFrames = $layer.frameStateTotals.waiting + $layer.frameStateTotals.depend ) - -
-
- $layer.detail.name -
-
-
-
- DEAD -
$deadFrames -
-
- EATEN -
$eatenFrames -
-
- INCOMPLETE -
$incompleteFrames -
-
-
-
- PEAK MEMORY -
${highMemory} -
-
- PROC HOURS -
$coreTime -
-
- OUTPUTS -
$outputCount -
-
-
-
- Avg Frame Time

${graphData} + #if( $outputCount > 0) + + Outputs: + + + #foreach($i in [0..2]) + #if( $i < $outputCount) + + + + #end + #end + #if( $outputs.size() > 3) + + + + #end +
$outputs.get($i)
... [$outputCount outputs attached]
+ + + #end + + +
+ #end
-
-

- Outputs -
- - - - #foreach( $output in $layer.outputs ) - - - - - #end -
${output}
-
-#end -
diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index 4dd691e07..21f8d0602 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -110,11 +110,10 @@ dispatcher.report_queue.max_pool_size=8 # Queue capacity for handling Host Report. dispatcher.report_queue.queue_capacity=1000 -# The minimum amount of free space in the temporary directory (mcp) to book a host. -# E.g: 1G = 1048576 kB => dispatcher.min_bookable_free_temp_dir_kb=1048576 -# Default = -1 (deactivated) +# The minimum amount of available space on the temporary directory (mcp) to book a host. +# Hosts with less free space than the limit will be marked as REPAIR # If equals to -1, it means the feature is turned off -dispatcher.min_bookable_free_temp_dir_kb=-1 +dispatcher.min_available_temp_storage_percentage=20 # Number of threads to keep in the pool for kill frame operation. dispatcher.kill_queue.core_pool_size=6 diff --git a/cuebot/src/main/resources/public/opencue_logo.png b/cuebot/src/main/resources/public/opencue_logo.png new file mode 100644 index 000000000..1827d958b Binary files /dev/null and b/cuebot/src/main/resources/public/opencue_logo.png differ diff --git a/cuebot/src/main/resources/public/services/arnold.png b/cuebot/src/main/resources/public/services/arnold.png new file mode 100755 index 000000000..190dfb016 Binary files /dev/null and b/cuebot/src/main/resources/public/services/arnold.png differ diff --git a/cuebot/src/main/resources/public/services/arnoldlive.png b/cuebot/src/main/resources/public/services/arnoldlive.png new file mode 100755 index 000000000..9376e1438 Binary files /dev/null and b/cuebot/src/main/resources/public/services/arnoldlive.png differ diff --git a/cuebot/src/main/resources/public/services/arnoldpreview.png b/cuebot/src/main/resources/public/services/arnoldpreview.png new file mode 100755 index 000000000..0be8a5824 Binary files /dev/null and b/cuebot/src/main/resources/public/services/arnoldpreview.png differ diff --git a/cuebot/src/main/resources/public/services/cnode.png b/cuebot/src/main/resources/public/services/cnode.png new file mode 100755 index 000000000..abb8fb6ca Binary files /dev/null and b/cuebot/src/main/resources/public/services/cnode.png differ diff --git a/cuebot/src/main/resources/public/services/comp.png b/cuebot/src/main/resources/public/services/comp.png old mode 100644 new mode 100755 index 78ec52e9a..09796c17c Binary files a/cuebot/src/main/resources/public/services/comp.png and b/cuebot/src/main/resources/public/services/comp.png differ diff --git a/cuebot/src/main/resources/public/services/default.png b/cuebot/src/main/resources/public/services/default.png old mode 100644 new mode 100755 index b990a034d..49886829d Binary files a/cuebot/src/main/resources/public/services/default.png and b/cuebot/src/main/resources/public/services/default.png differ diff --git a/cuebot/src/main/resources/public/services/distsim16.png b/cuebot/src/main/resources/public/services/distsim16.png new file mode 100755 index 000000000..04ab6beb0 Binary files /dev/null and b/cuebot/src/main/resources/public/services/distsim16.png differ diff --git a/cuebot/src/main/resources/public/services/distsim32.png b/cuebot/src/main/resources/public/services/distsim32.png new file mode 100755 index 000000000..98ee5979c Binary files /dev/null and b/cuebot/src/main/resources/public/services/distsim32.png differ diff --git a/cuebot/src/main/resources/public/services/distsim4.png b/cuebot/src/main/resources/public/services/distsim4.png new file mode 100755 index 000000000..aef5aa883 Binary files /dev/null and b/cuebot/src/main/resources/public/services/distsim4.png differ diff --git a/cuebot/src/main/resources/public/services/distsim8.png b/cuebot/src/main/resources/public/services/distsim8.png new file mode 100755 index 000000000..298d5a742 Binary files /dev/null and b/cuebot/src/main/resources/public/services/distsim8.png differ diff --git a/cuebot/src/main/resources/public/services/eddy.png b/cuebot/src/main/resources/public/services/eddy.png new file mode 100755 index 000000000..872c57135 Binary files /dev/null and b/cuebot/src/main/resources/public/services/eddy.png differ diff --git a/cuebot/src/main/resources/public/services/finalizer.png b/cuebot/src/main/resources/public/services/finalizer.png new file mode 100755 index 000000000..03d3a2e25 Binary files /dev/null and b/cuebot/src/main/resources/public/services/finalizer.png differ diff --git a/cuebot/src/main/resources/public/services/ginsu.png b/cuebot/src/main/resources/public/services/ginsu.png deleted file mode 100644 index 28d41e9fd..000000000 Binary files a/cuebot/src/main/resources/public/services/ginsu.png and /dev/null differ diff --git a/cuebot/src/main/resources/public/services/houdini.png b/cuebot/src/main/resources/public/services/houdini.png old mode 100644 new mode 100755 index e4418a8ad..0ea84fffa Binary files a/cuebot/src/main/resources/public/services/houdini.png and b/cuebot/src/main/resources/public/services/houdini.png differ diff --git a/cuebot/src/main/resources/public/services/katana.png b/cuebot/src/main/resources/public/services/katana.png old mode 100644 new mode 100755 index e2c103e66..91b7d6a41 Binary files a/cuebot/src/main/resources/public/services/katana.png and b/cuebot/src/main/resources/public/services/katana.png differ diff --git a/cuebot/src/main/resources/public/services/limboreaper.png b/cuebot/src/main/resources/public/services/limboreaper.png new file mode 100755 index 000000000..2aa089ae9 Binary files /dev/null and b/cuebot/src/main/resources/public/services/limboreaper.png differ diff --git a/cuebot/src/main/resources/public/services/massive.png b/cuebot/src/main/resources/public/services/massive.png new file mode 100755 index 000000000..390d8d3c3 Binary files /dev/null and b/cuebot/src/main/resources/public/services/massive.png differ diff --git a/cuebot/src/main/resources/public/services/maya.png b/cuebot/src/main/resources/public/services/maya.png old mode 100644 new mode 100755 index 90e30a403..8e541a758 Binary files a/cuebot/src/main/resources/public/services/maya.png and b/cuebot/src/main/resources/public/services/maya.png differ diff --git a/cuebot/src/main/resources/public/services/mayapub.png b/cuebot/src/main/resources/public/services/mayapub.png new file mode 100755 index 000000000..d190aaa7f Binary files /dev/null and b/cuebot/src/main/resources/public/services/mayapub.png differ diff --git a/cuebot/src/main/resources/public/services/mayaziva.png b/cuebot/src/main/resources/public/services/mayaziva.png new file mode 100755 index 000000000..ff0d57672 Binary files /dev/null and b/cuebot/src/main/resources/public/services/mayaziva.png differ diff --git a/cuebot/src/main/resources/public/services/mentalray.png b/cuebot/src/main/resources/public/services/mentalray.png deleted file mode 100644 index f54b3d96a..000000000 Binary files a/cuebot/src/main/resources/public/services/mentalray.png and /dev/null differ diff --git a/cuebot/src/main/resources/public/services/neat.png b/cuebot/src/main/resources/public/services/neat.png new file mode 100755 index 000000000..44a570421 Binary files /dev/null and b/cuebot/src/main/resources/public/services/neat.png differ diff --git a/cuebot/src/main/resources/public/services/nuke.png b/cuebot/src/main/resources/public/services/nuke.png old mode 100644 new mode 100755 index ea993a6b0..086c8709c Binary files a/cuebot/src/main/resources/public/services/nuke.png and b/cuebot/src/main/resources/public/services/nuke.png differ diff --git a/cuebot/src/main/resources/public/services/ofx.png b/cuebot/src/main/resources/public/services/ofx.png new file mode 100755 index 000000000..b940cb954 Binary files /dev/null and b/cuebot/src/main/resources/public/services/ofx.png differ diff --git a/cuebot/src/main/resources/public/services/opflares.png b/cuebot/src/main/resources/public/services/opflares.png new file mode 100755 index 000000000..041be7fb1 Binary files /dev/null and b/cuebot/src/main/resources/public/services/opflares.png differ diff --git a/cuebot/src/main/resources/public/services/playblast.png b/cuebot/src/main/resources/public/services/playblast.png old mode 100644 new mode 100755 index eca5dcf43..d0b89c1d7 Binary files a/cuebot/src/main/resources/public/services/playblast.png and b/cuebot/src/main/resources/public/services/playblast.png differ diff --git a/cuebot/src/main/resources/public/services/posteddy.png b/cuebot/src/main/resources/public/services/posteddy.png new file mode 100755 index 000000000..89df396e9 Binary files /dev/null and b/cuebot/src/main/resources/public/services/posteddy.png differ diff --git a/cuebot/src/main/resources/public/services/postprocess.png b/cuebot/src/main/resources/public/services/postprocess.png new file mode 100755 index 000000000..5f386676b Binary files /dev/null and b/cuebot/src/main/resources/public/services/postprocess.png differ diff --git a/cuebot/src/main/resources/public/services/preeddy.png b/cuebot/src/main/resources/public/services/preeddy.png new file mode 100755 index 000000000..cb6c2d9b3 Binary files /dev/null and b/cuebot/src/main/resources/public/services/preeddy.png differ diff --git a/cuebot/src/main/resources/public/services/preprocess.png b/cuebot/src/main/resources/public/services/preprocess.png new file mode 100755 index 000000000..bfce5d707 Binary files /dev/null and b/cuebot/src/main/resources/public/services/preprocess.png differ diff --git a/cuebot/src/main/resources/public/services/prman.png b/cuebot/src/main/resources/public/services/prman.png deleted file mode 100644 index ee038d863..000000000 Binary files a/cuebot/src/main/resources/public/services/prman.png and /dev/null differ diff --git a/cuebot/src/main/resources/public/services/proxy.png b/cuebot/src/main/resources/public/services/proxy.png new file mode 100755 index 000000000..928abb123 Binary files /dev/null and b/cuebot/src/main/resources/public/services/proxy.png differ diff --git a/cuebot/src/main/resources/public/services/reaper.png b/cuebot/src/main/resources/public/services/reaper.png new file mode 100755 index 000000000..31fd69733 Binary files /dev/null and b/cuebot/src/main/resources/public/services/reaper.png differ diff --git a/cuebot/src/main/resources/public/services/rebelle.png b/cuebot/src/main/resources/public/services/rebelle.png new file mode 100755 index 000000000..db15ffc26 Binary files /dev/null and b/cuebot/src/main/resources/public/services/rebelle.png differ diff --git a/cuebot/src/main/resources/public/services/rsmb.png b/cuebot/src/main/resources/public/services/rsmb.png new file mode 100755 index 000000000..c662aa42f Binary files /dev/null and b/cuebot/src/main/resources/public/services/rsmb.png differ diff --git a/cuebot/src/main/resources/public/services/sapphire.png b/cuebot/src/main/resources/public/services/sapphire.png new file mode 100755 index 000000000..2c1ad37a2 Binary files /dev/null and b/cuebot/src/main/resources/public/services/sapphire.png differ diff --git a/cuebot/src/main/resources/public/services/shell.png b/cuebot/src/main/resources/public/services/shell.png old mode 100644 new mode 100755 index 10cf381b5..63d8d0b23 Binary files a/cuebot/src/main/resources/public/services/shell.png and b/cuebot/src/main/resources/public/services/shell.png differ diff --git a/cuebot/src/main/resources/public/services/simulation.png b/cuebot/src/main/resources/public/services/simulation.png old mode 100644 new mode 100755 index 87be4dd17..a12bb2b1d Binary files a/cuebot/src/main/resources/public/services/simulation.png and b/cuebot/src/main/resources/public/services/simulation.png differ diff --git a/cuebot/src/main/resources/public/services/simulation16.png b/cuebot/src/main/resources/public/services/simulation16.png new file mode 100755 index 000000000..f568a5de0 Binary files /dev/null and b/cuebot/src/main/resources/public/services/simulation16.png differ diff --git a/cuebot/src/main/resources/public/services/simulation32.png b/cuebot/src/main/resources/public/services/simulation32.png new file mode 100755 index 000000000..b7e4e0e7f Binary files /dev/null and b/cuebot/src/main/resources/public/services/simulation32.png differ diff --git a/cuebot/src/main/resources/public/services/simulation4.png b/cuebot/src/main/resources/public/services/simulation4.png new file mode 100755 index 000000000..b6ec1d9ba Binary files /dev/null and b/cuebot/src/main/resources/public/services/simulation4.png differ diff --git a/cuebot/src/main/resources/public/services/simulation8.png b/cuebot/src/main/resources/public/services/simulation8.png new file mode 100755 index 000000000..09f28e793 Binary files /dev/null and b/cuebot/src/main/resources/public/services/simulation8.png differ diff --git a/cuebot/src/main/resources/public/services/simulationhi.png b/cuebot/src/main/resources/public/services/simulationhi.png new file mode 100755 index 000000000..176ca34ef Binary files /dev/null and b/cuebot/src/main/resources/public/services/simulationhi.png differ diff --git a/cuebot/src/main/resources/public/services/spotless.png b/cuebot/src/main/resources/public/services/spotless.png new file mode 100755 index 000000000..cde291db4 Binary files /dev/null and b/cuebot/src/main/resources/public/services/spotless.png differ diff --git a/cuebot/src/main/resources/public/services/svea.png b/cuebot/src/main/resources/public/services/svea.png deleted file mode 100644 index f028a8d55..000000000 Binary files a/cuebot/src/main/resources/public/services/svea.png and /dev/null differ diff --git a/cuebot/src/main/resources/public/services/trinity.png b/cuebot/src/main/resources/public/services/trinity.png deleted file mode 100644 index c9ddeb64d..000000000 Binary files a/cuebot/src/main/resources/public/services/trinity.png and /dev/null differ diff --git a/cuebot/src/main/resources/public/services/unreal.png b/cuebot/src/main/resources/public/services/unreal.png new file mode 100755 index 000000000..94d601761 Binary files /dev/null and b/cuebot/src/main/resources/public/services/unreal.png differ diff --git a/cuebot/src/main/resources/public/success.png b/cuebot/src/main/resources/public/success.png deleted file mode 100644 index 17c8a5a01..000000000 Binary files a/cuebot/src/main/resources/public/success.png and /dev/null differ diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java index b610ff11c..b39b97ad0 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/HostReportHandlerTests.java @@ -297,14 +297,16 @@ public void testHandleHostReportWithFullTemporaryDirectories() { * Precondition: * - HardwareState=UP * Action: - * - Receives a HostReport with freeTempDir < dispatcher.min_bookable_free_temp_dir_kb (opencue.properties) + * - Receives a HostReport with less freeTempDir than the threshold + * (opencue.properties: min_available_temp_storage_percentage) * Postcondition: * - Host hardwareState changes to REPAIR - * - A comment is created with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and user=CUEBOT_COMMENT_USER + * - A comment is created with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and + * user=CUEBOT_COMMENT_USER * */ - // Create HostReport + // Create HostReport with totalMcp=4GB and freeMcp=128MB HostReport report1 = HostReport.newBuilder() - .setHost(getRenderHostBuilder(hostname).setFreeMcp(1024L).build()) + .setHost(getRenderHostBuilder(hostname).setFreeMcp(CueUtil.MB128).build()) .setCoreInfo(cores) .build(); // Call handleHostReport() => Create the comment with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and change the @@ -337,9 +339,11 @@ public void testHandleHostReportWithFullTemporaryDirectories() { * Test 2: * Precondition: * - HardwareState=REPAIR - * - There is a comment for the host with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and user=CUEBOT_COMMENT_USER + * - There is a comment for the host with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and + * user=CUEBOT_COMMENT_USER * Action: - * - Receives a HostReport with freeTempDir >= dispatcher.min_bookable_free_temp_dir_kb (opencue.properties) + * Receives a HostReport with more freeTempDir than the threshold + * (opencue.properties: min_available_temp_storage_percentage) * Postcondition: * - Host hardwareState changes to UP * - Comment with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and user=CUEBOT_COMMENT_USER gets deleted diff --git a/cuebot/src/test/resources/opencue.properties b/cuebot/src/test/resources/opencue.properties index 1547b9d5e..7edcfe445 100644 --- a/cuebot/src/test/resources/opencue.properties +++ b/cuebot/src/test/resources/opencue.properties @@ -1,6 +1,14 @@ cue.proxy = tcp -h cuetest01-vm -p 9019 -t 10000:tcp -h cuetest02-vm -p 9019 -t 10000:tcp -h cuetest03-vm -p 9019 -t 10000 spring.velocity.checkTemplateLocation=false +# A domain to attach to usernames to send job completition emails +# eg: job_user=jon -> email-to: jon@opencue.io +email.domain=opencue.io +# An email address to use as From for cuebot emails +email.from.address=opencue-noreply@opencue.io +# A comma-separated list of emails to be cc'ed on maintenance communications +email.cc.addresses=dev-team@opencue.io + grpc.cue_port=8453 grpc.rqd_server_port=${CUEBOT_GRPC_RQD_SERVER_PORT:50051} grpc.max_message_bytes=104857600 @@ -64,7 +72,7 @@ dispatcher.kill_queue.queue_capacity=1000 dispatcher.booking_queue.core_pool_size=6 dispatcher.booking_queue.max_pool_size=6 dispatcher.booking_queue.queue_capacity=1000 -dispatcher.min_bookable_free_temp_dir_kb=1048576 +dispatcher.min_available_temp_storage_percentage=20 dispatcher.min_bookable_free_mcp_kb=1048576 dispatcher.oom_max_safe_used_memory_threshold=0.95 dispatcher.oom_frame_overboard_allowed_threshold=0.6 diff --git a/cuegui/cuegui/DarkPalette.py b/cuegui/cuegui/DarkPalette.py index fed62996b..373624e18 100644 --- a/cuegui/cuegui/DarkPalette.py +++ b/cuegui/cuegui/DarkPalette.py @@ -52,50 +52,50 @@ def DarkPalette(): p = QtGui.QPalette() c = GreyF(0.175) - p.setColor(p.Window, c) - p.setColor(p.Button, c) + p.setColor(QtGui.QPalette.Window, c) + p.setColor(QtGui.QPalette.Button, c) c = GreyF(0.79) - p.setColor(p.WindowText, c) - p.setColor(p.Text, c) - p.setColor(p.ButtonText, c) - p.setColor(p.BrightText, c) + p.setColor(QtGui.QPalette.WindowText, c) + p.setColor(QtGui.QPalette.Text, c) + p.setColor(QtGui.QPalette.ButtonText, c) + p.setColor(QtGui.QPalette.BrightText, c) c = ColorF(0.6, 0.6, 0.8) - p.setColor(p.Link, c) + p.setColor(QtGui.QPalette.Link, c) c = ColorF(0.8, 0.6, 0.8) - p.setColor(p.LinkVisited, c) + p.setColor(QtGui.QPalette.LinkVisited, c) c = GreyF(0.215) - p.setColor(p.Base, c) + p.setColor(QtGui.QPalette.Base, c) c = GreyF(0.25) - p.setColor(p.AlternateBase, c) + p.setColor(QtGui.QPalette.AlternateBase, c) c = GreyF(0.0) - p.setColor(p.Shadow, c) + p.setColor(QtGui.QPalette.Shadow, c) c = GreyF(0.13) - p.setColor(p.Dark, c) + p.setColor(QtGui.QPalette.Dark, c) c = GreyF(0.21) - p.setColor(p.Mid, c) + p.setColor(QtGui.QPalette.Mid, c) c = GreyF(0.25) - p.setColor(p.Midlight, c) + p.setColor(QtGui.QPalette.Midlight, c) c = GreyF(0.40) - p.setColor(p.Light, c) + p.setColor(QtGui.QPalette.Light, c) c = ColorF(0.31, 0.31, 0.25) - p.setColor(p.Highlight, c) + p.setColor(QtGui.QPalette.Highlight, c) c = GreyF(0.46) - p.setColor(QtGui.QPalette.Disabled, p.WindowText, c) - p.setColor(QtGui.QPalette.Disabled, p.Text, c) - p.setColor(QtGui.QPalette.Disabled, p.ButtonText, c) + p.setColor(QtGui.QPalette.Disabled, QtGui.QPalette.WindowText, c) + p.setColor(QtGui.QPalette.Disabled, QtGui.QPalette.Text, c) + p.setColor(QtGui.QPalette.Disabled, QtGui.QPalette.ButtonText, c) c = GreyF(0.55) - p.setColor(QtGui.QPalette.Disabled, p.BrightText, c) + p.setColor(QtGui.QPalette.Disabled, QtGui.QPalette.BrightText, c) return p diff --git a/cuegui/cuegui/FrameMonitor.py b/cuegui/cuegui/FrameMonitor.py index 56c586cd9..9a76e861e 100644 --- a/cuegui/cuegui/FrameMonitor.py +++ b/cuegui/cuegui/FrameMonitor.py @@ -428,14 +428,14 @@ def _filterStatusSetup(self, layout): btn.setMenu(menu) menu.triggered.connect(self._filterStatusHandle) # pylint: disable=no-member - for item in [("Clear", QtCore.Qt.ALT + QtCore.Qt.Key_QuoteLeft), + for item in [("Clear", QtCore.Qt.ALT | QtCore.Qt.Key_QuoteLeft), None, - ("Succeeded", QtCore.Qt.ALT + QtCore.Qt.Key_1), - ("Running", QtCore.Qt.ALT + QtCore.Qt.Key_2), - ("Waiting", QtCore.Qt.ALT + QtCore.Qt.Key_3), - ("Depend", QtCore.Qt.ALT + QtCore.Qt.Key_4), - ("Dead", QtCore.Qt.ALT + QtCore.Qt.Key_5), - ("Eaten", QtCore.Qt.ALT + QtCore.Qt.Key_6)]: + ("Succeeded", QtCore.Qt.ALT | QtCore.Qt.Key_1), + ("Running", QtCore.Qt.ALT | QtCore.Qt.Key_2), + ("Waiting", QtCore.Qt.ALT | QtCore.Qt.Key_3), + ("Depend", QtCore.Qt.ALT | QtCore.Qt.Key_4), + ("Dead", QtCore.Qt.ALT | QtCore.Qt.Key_5), + ("Eaten", QtCore.Qt.ALT | QtCore.Qt.Key_6)]: if item: a = QtWidgets.QAction(item[0], menu) if item[0] != "Clear": diff --git a/cuegui/cuegui/SubscribeToJobDialog.py b/cuegui/cuegui/SubscribeToJobDialog.py index 758497b90..fa332a8b2 100644 --- a/cuegui/cuegui/SubscribeToJobDialog.py +++ b/cuegui/cuegui/SubscribeToJobDialog.py @@ -24,8 +24,8 @@ from builtins import map -from PySide2 import QtCore -from PySide2 import QtWidgets +from qtpy import QtCore +from qtpy import QtWidgets import cuegui.Constants import cuegui.Logger import cuegui.Utils diff --git a/cuegui/cuegui/plugins/LogViewPlugin.py b/cuegui/cuegui/plugins/LogViewPlugin.py index 5c36ec717..9d86e712f 100644 --- a/cuegui/cuegui/plugins/LogViewPlugin.py +++ b/cuegui/cuegui/plugins/LogViewPlugin.py @@ -149,14 +149,6 @@ def mouseReleaseEvent(self, event): self.mousePressedSignal.emit(pos) self.copy_selection(QtGui.QClipboard.Selection) - def scrollContentsBy(self, *args, **kwargs): - """ - Overriding to make sure the line numbers area is updated when scrolling - """ - - self._line_num_area.repaint() - return QtWidgets.QPlainTextEdit.scrollContentsBy(self, *args, **kwargs) - def copy_selection(self, mode): """ Copy (Ctrl + C) action. Stores the currently selected text in the diff --git a/cuesubmit/cuesubmit/Submission.py b/cuesubmit/cuesubmit/Submission.py index 59799663f..f1083b23d 100644 --- a/cuesubmit/cuesubmit/Submission.py +++ b/cuesubmit/cuesubmit/Submission.py @@ -21,6 +21,7 @@ from __future__ import absolute_import from builtins import str +import re import outline import outline.cuerun @@ -73,7 +74,7 @@ def buildBlenderCmd(layerData): renderCommand += ' -o {}'.format(outputPath) if outputFormat: renderCommand += ' -F {}'.format(outputFormat) - if frameRange: + if re.match(r"^\d+-\d+$", frameRange): # Render frames from start to end (inclusive) via '-a' command argument renderCommand += (' -s {startFrame} -e {endFrame} -a' .format(startFrame=Constants.FRAME_START_TOKEN, diff --git a/cuesubmit/cuesubmit/__main__.py b/cuesubmit/cuesubmit/__main__.py index 3adecc770..d712f30a7 100644 --- a/cuesubmit/cuesubmit/__main__.py +++ b/cuesubmit/cuesubmit/__main__.py @@ -23,8 +23,8 @@ from __future__ import absolute_import import sys -from PySide2 import QtGui -from PySide2 import QtWidgets +from qtpy import QtGui +from qtpy import QtWidgets from cuesubmit import Constants from cuesubmit import JobTypes diff --git a/cuesubmit/cuesubmit/ui/Command.py b/cuesubmit/cuesubmit/ui/Command.py index c1d144b24..ac0afb1a6 100644 --- a/cuesubmit/cuesubmit/ui/Command.py +++ b/cuesubmit/cuesubmit/ui/Command.py @@ -20,7 +20,7 @@ from __future__ import division from __future__ import absolute_import -from PySide2 import QtCore, QtWidgets +from qtpy import QtCore, QtWidgets from cuesubmit.ui import Widgets from cuesubmit import Constants diff --git a/cuesubmit/cuesubmit/ui/Job.py b/cuesubmit/cuesubmit/ui/Job.py index e8fe51d0a..fc15d5536 100644 --- a/cuesubmit/cuesubmit/ui/Job.py +++ b/cuesubmit/cuesubmit/ui/Job.py @@ -21,7 +21,7 @@ from __future__ import absolute_import from builtins import range -from PySide2 import QtCore, QtGui, QtWidgets +from qtpy import QtCore, QtGui, QtWidgets from cuesubmit import Layer from cuesubmit.ui import Style diff --git a/cuesubmit/cuesubmit/ui/SettingsWidgets.py b/cuesubmit/cuesubmit/ui/SettingsWidgets.py index 01601265a..612e67193 100644 --- a/cuesubmit/cuesubmit/ui/SettingsWidgets.py +++ b/cuesubmit/cuesubmit/ui/SettingsWidgets.py @@ -20,7 +20,7 @@ from __future__ import division from __future__ import absolute_import -from PySide2 import QtCore, QtWidgets +from qtpy import QtCore, QtWidgets from cuesubmit import Constants from cuesubmit.ui import Command diff --git a/cuesubmit/cuesubmit/ui/Style.py b/cuesubmit/cuesubmit/ui/Style.py index 4730bf706..0679e6aa4 100644 --- a/cuesubmit/cuesubmit/ui/Style.py +++ b/cuesubmit/cuesubmit/ui/Style.py @@ -20,7 +20,7 @@ from __future__ import division from __future__ import absolute_import -from PySide2 import QtGui +from qtpy import QtGui MAIN_WINDOW = """ diff --git a/cuesubmit/cuesubmit/ui/Submit.py b/cuesubmit/cuesubmit/ui/Submit.py index 1e0246d17..36a6a43ba 100644 --- a/cuesubmit/cuesubmit/ui/Submit.py +++ b/cuesubmit/cuesubmit/ui/Submit.py @@ -24,7 +24,7 @@ from builtins import range import getpass -from PySide2 import QtCore, QtGui, QtWidgets +from qtpy import QtCore, QtGui, QtWidgets import opencue from cuesubmit import Constants diff --git a/cuesubmit/cuesubmit/ui/Widgets.py b/cuesubmit/cuesubmit/ui/Widgets.py index bc658a3a7..0be3ca359 100644 --- a/cuesubmit/cuesubmit/ui/Widgets.py +++ b/cuesubmit/cuesubmit/ui/Widgets.py @@ -21,7 +21,7 @@ from __future__ import absolute_import from builtins import object -from PySide2 import QtCore, QtGui, QtWidgets +from qtpy import QtCore, QtGui, QtWidgets from cuesubmit import Constants from cuesubmit.ui import Style diff --git a/pycue/FileSequence/FileSequence.py b/pycue/FileSequence/FileSequence.py new file mode 100644 index 000000000..61d86a55e --- /dev/null +++ b/pycue/FileSequence/FileSequence.py @@ -0,0 +1,130 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Helper class for representing a frame seqeunce path. + +It supports a complex syntax implementing features such as comma-separated frame ranges, +stepped frame ranges and more. See the FileSequence class for more detail. +""" + +import re +from .FrameSet import FrameSet + + +class FileSequence: + """Represents a file path to a frame sequence""" + __filepath = None + frameSet = None + __basename = None + __prefix = None + __suffix = None + __dirname = "" + __padSize = 1 + __iter_index = 0 + + def __init__(self, filepath): + """ + Construct a FileSequence object by parsing a filepath. + Details on how to specify a frame range can be seen in the FrameRange class + """ + + filePathMatch = re.match(r"^(?P.*\.)(?P[\d#,\-@x]+)(?P\.[.\w]+)$", filepath) + if filePathMatch is not None: + self.__filepath = filepath + self.__prefix = filePathMatch.group('pf') + self.__suffix = filePathMatch.group('sf') + dirmatch = re.match(r"^(?P.*/)(?P.*)$", self.__prefix) + if dirmatch is not None: + self.__dirname = dirmatch.group("dirname") + self.__basename = dirmatch.group("basename") + else: + self.__basename = self.__prefix + framerangematch = re.match(r"^([\d\-x,]+)", filePathMatch.group("fspec")) + if framerangematch is not None: + self.frameSet = FrameSet(framerangematch.group(1)) + if self.frameSet.get(0) > self.frameSet.get(-1): + raise ValueError('invalid filesequence range : ' + framerangematch.group(1)) + firstFrameMatch = re.findall(r"^[-0]\d+", framerangematch.group(1)) + if len(firstFrameMatch) > 0: + self.__padSize = len(firstFrameMatch[0]) + + padmatch = re.findall(r"#+$", filePathMatch.group("fspec")) + if len(padmatch) > 0: + self.__padSize = len(padmatch[0]) + else: + raise ValueError('invalid filesequence path : ' + filepath) + + def getPrefix(self): + """Returns the prefix of the file sequence""" + return self.__prefix + + def getSuffix(self): + """Returns the suffix of the file sequence""" + return self.__suffix + + def getDirname(self): + """Returns the dirname of the file sequence, if given otherwise returns empty an string""" + return self.__dirname + + def getBasename(self): + """Returns the base name of the file sequence""" + return self.__basename.rstrip(".") + + def getPadSize(self): + """Returns the size of the frame padding. It defaults to 1 if none is detected""" + return self.__padSize + + def getFileList(self, frameSet=None): + """ Returns the file list of the sequence """ + filelist = [] + paddingString = "%%0%dd" % self.getPadSize() + for frame in self.frameSet.getAll(): + if frameSet is None or (isinstance(frameSet, FrameSet) and frame in frameSet.getAll()): + framepath = self.getPrefix() + paddingString % frame + self.getSuffix() + filelist.append(framepath) + return filelist + + def getOpenRVPath(self, frameSet=None): + """ Returns a string specific for the OpenRV player""" + frameRange = "" + curFrameSet = frameSet or self.frameSet + if isinstance(curFrameSet, FrameSet): + frameRange = "%d-%d" % (curFrameSet.get(0), curFrameSet.get(-1)) + framepath = self.getPrefix() + frameRange + "@"*self.__padSize + self.getSuffix() + return framepath + + def __getitem__(self, index): + return self.getFileList()[index] + + def __next__(self): + self.__iter_index += 1 + if self.__iter_index <= len(self): + return self.getFileList()[self.__iter_index - 1] + raise StopIteration + + def __iter__(self): + self.__iter_index = 0 + return self + + def __len__(self): + return len(self.getFileList()) + + def __call__(self, frame): + paddingString = "%%0%dd" % self.getPadSize() + framepath = self.getPrefix() + paddingString % frame + self.getSuffix() + return framepath + + def __str__(self): + return self.__filepath diff --git a/pycue/FileSequence/__init__.py b/pycue/FileSequence/__init__.py index 711e1dc83..8010a19bc 100644 --- a/pycue/FileSequence/__init__.py +++ b/pycue/FileSequence/__init__.py @@ -23,3 +23,4 @@ from __future__ import division from .FrameRange import FrameRange from .FrameSet import FrameSet +from .FileSequence import FileSequence diff --git a/pycue/tests/file_sequence.py b/pycue/tests/file_sequence.py index f03438f9c..2adc6907d 100644 --- a/pycue/tests/file_sequence.py +++ b/pycue/tests/file_sequence.py @@ -24,6 +24,7 @@ from FileSequence import FrameRange from FileSequence import FrameSet +from FileSequence import FileSequence class FrameRangeTests(unittest.TestCase): @@ -189,5 +190,114 @@ def testNormalize(self): self.assertEqual([1, 2, 3], duplicates.getAll()) +class FileSequenceTests(unittest.TestCase): + def __testFileSequence(self, filespec, **kwargs): + fs = FileSequence(filespec) + + tests = {'prefix': fs.getPrefix(), + 'frameSet': fs.frameSet, + 'suffix': fs.getSuffix(), + 'padSize': fs.getPadSize(), + 'dirname': fs.getDirname(), + 'basename': fs.getBasename() + } + + for arg, member in tests.items(): + if arg in kwargs: + if isinstance(member, FrameSet): + self.assertEqual(member.getAll(), kwargs[arg].getAll(), + "Comparing '%s', got '%s', expected '%s'" % (arg, str(member), + str(kwargs[arg]))) + else: + self.assertEqual(member, kwargs[arg], + "Comparing '%s', got '%s', expected '%s'" % (arg, str(member), + str(kwargs[arg]))) + + def testVariousFileSequences(self): + """Test various file sequences are correctly parsed.""" + self.__testFileSequence('foo.1-1####.bar', prefix='foo.', frameSet=FrameSet('1-1'), + suffix='.bar', padSize=4) + self.__testFileSequence('foo.####.bar', prefix='foo.', frameSet=None, suffix='.bar', + padSize=4) + # Not sure why this becomes padSize of 10 + # self.__testFileSequence('foo.1-15x2#@#@.bar', prefix='foo.', frameSet=FrameSet('1-15x2'), + # suffix='.bar', + # padSize=10) + self.__testFileSequence('foo.1-15x2.bar', prefix='foo.', frameSet=FrameSet('1-15x2'), + suffix='.bar', padSize=1) + self.__testFileSequence('someImage.1,3,5####.rla', prefix='someImage.', + frameSet=FrameSet('1,3,5'), suffix='.rla', padSize=4) + self.__testFileSequence('foo.####.exr.tx', prefix='foo.', frameSet=None, suffix='.exr.tx', + padSize=4) + self.__testFileSequence('foo.1-10#.bar.1-9####.bar', prefix='foo.1-10#.bar.', + frameSet=FrameSet('1-9'), suffix='.bar', padSize=4) + self.__testFileSequence('foo.1-9.bar', prefix='foo.', frameSet=FrameSet('1-9'), + suffix='.bar', padSize=1) + self.__testFileSequence('foo.1-10.bar', prefix='foo.', frameSet=FrameSet('1-10'), + suffix='.bar', padSize=1) + self.__testFileSequence('foo.9.bar', prefix='foo.', frameSet=FrameSet('9-9'), suffix='.bar', + padSize=1) + + self.__testFileSequence('foo.1-10#.bar', prefix='foo.', dirname='', basename='foo') + self.__testFileSequence('/foo.1-10#.bar', prefix='/foo.', dirname='/', basename='foo') + self.__testFileSequence('baz/foo.1-10#.bar', prefix='baz/foo.', dirname='baz/', + basename='foo') + self.__testFileSequence('/baz/foo.1-10#.bar', prefix='/baz/foo.', dirname='/baz/', + basename='foo') + self.__testFileSequence('/bar/baz/foo.1-10#.bar', prefix='/bar/baz/foo.', + dirname='/bar/baz/', basename='foo') + + self.__testFileSequence('foo.-15-15####.bar', prefix='foo.', frameSet=FrameSet('-15-15'), + suffix='.bar', padSize=4) + self.__testFileSequence('foo.-15--1####.bar', prefix='foo.', frameSet=FrameSet('-15--1'), + suffix='.bar', padSize=4) + + def testPadSizeWithoutPadTokens(self): + """Test the pad size is correctly guessed when no padding tokens are given.""" + self.__testFileSequence('foo.0009.bar', padSize=4) + self.__testFileSequence('foo.1-9x0002.bar', padSize=1) + # This test contradicts another test for negative steps + # self.__testFileSequence('foo.9-1x-0002.bar', padSize=1) + self.__testFileSequence('foo.9-09x0002.bar', padSize=1) + self.__testFileSequence('foo.9,10.bar', padSize=1) + self.__testFileSequence('foo.009,10.bar', padSize=3) + self.__testFileSequence('foo.-011.bar', padSize=4) + + # sequence padded to 4 but frame count goes above 9999 + self.__testFileSequence('foo.0001-10000.bar', padSize=4) + + def testInvalidSequences(self): + """Test invalid file sequences throw expected exception.""" + self.assertRaises(ValueError, FileSequence, 'asdasdasda') + self.assertRaises(ValueError, FileSequence, 'foo.fred#.bar') + self.assertRaises(ValueError, FileSequence, 'foo..bar') + self.assertRaises(ValueError, FileSequence, 'foo.-,x#.bar') + self.assertRaises(ValueError, FileSequence, 'foo.x2.bar') + self.assertRaises(ValueError, FileSequence, 'foo.-20---10.bar') + # order reversed + self.assertRaises(ValueError, FileSequence, 'foo.10-1.bar') + self.assertRaises(ValueError, FileSequence, 'foo.-10--20.bar') + # require a prefix + self.assertRaises(ValueError, FileSequence, '.1') + self.assertRaises(ValueError, FileSequence, '0.1') + + def __testStringify(self, filespec, index, expected): + fs = FileSequence(filespec) + self.assertEqual(expected, fs[index]) + + def testStringify(self): + self.__testStringify('foo.011.bar', 0, 'foo.011.bar') + self.__testStringify('foo.-011.bar', 0, 'foo.-011.bar') + + def __testFrameList(self, filespec, frame, expected): + fs = FileSequence(filespec) + self.assertEqual(expected, fs(frame)) + + def testFrameList(self): + self.__testFrameList('foo.1-10.bar', 4, 'foo.4.bar') + self.__testFrameList('foo.1-10####.bar', 4, 'foo.0004.bar') + self.__testFrameList('foo.####.bar', 4, 'foo.0004.bar') + + if __name__ == '__main__': unittest.main() diff --git a/pyoutline/tests/layer_test.py b/pyoutline/tests/layer_test.py index f1fd61b9f..686233cbc 100644 --- a/pyoutline/tests/layer_test.py +++ b/pyoutline/tests/layer_test.py @@ -592,9 +592,7 @@ class OutputRegistrationTest(unittest.TestCase): def setUp(self): outline.Outline.current = None - # TODO(bcipriano) Re-enable this test once FileSequence has a Python - # implementation. (Issue #242) - def disabled__test_output_passing(self): + def test_output_passing(self): """ Test that output registered in a pre-process is serialized to a ol:outputs file in the render layer. @@ -608,7 +606,7 @@ def disabled__test_output_passing(self): # the preprocess prelayer = outline.LayerPreProcess(layer1) prelayer._execute = lambda frames: prelayer.add_output( - "test", outline.io.FileSpec("/tmp/foo.#.exr")) + "test", outline.io.Path("/tmp/foo.#.exr")) # Add both to the outline ol.add_layer(layer1) diff --git a/requirements.txt b/requirements.txt index 262f681f9..4363e51c1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ 2to3==1.0 enum34==1.1.6 evdev==1.4.0;python_version<"3.0" and "linux" in sys_platform -future==0.18.3 +future==1.0.0 futures==3.2.0;python_version<"3.0" grpcio==1.26.0;python_version<"3.0" grpcio-tools==1.26.0;python_version<"3.0" -grpcio==1.47.0;python_version>="3.0" +grpcio==1.53.2;python_version>="3.0" grpcio-tools==1.47.0;python_version>="3.0" mock==2.0.0 packaging==20.9 @@ -17,4 +17,4 @@ pyfakefs==5.2.3;python_version>="3.7" pylint==2.6.0;python_version>="3.7" pynput==1.7.6 PyYAML==5.1 -six==1.11.0 +six==1.16.0 diff --git a/requirements_gui.txt b/requirements_gui.txt index b7ff2d6b0..1f5b19637 100644 --- a/requirements_gui.txt +++ b/requirements_gui.txt @@ -1,3 +1,5 @@ -PySide2==5.15.2.1 +PySide6==6.7.1;python_version>"3.11" +PySide6==6.5.3;python_version=="3.11" +PySide2==5.15.2.1;python_version<="3.10" QtPy==1.11.3;python_version<"3.7" -QtPy==2.3.0;python_version>="3.7" +QtPy==2.4.1;python_version>="3.7" diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index 40d073cc0..271ac6122 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -318,17 +318,13 @@ def runLinux(self): else: tempCommand += [self._createCommandFile(runFrame.command)] - if rqd.rqconstants.RQD_PREPEND_TIMESTAMP: - file_descriptor = subprocess.PIPE - else: - file_descriptor = self.rqlog # pylint: disable=subprocess-popen-preexec-fn frameInfo.forkedCommand = subprocess.Popen(tempCommand, env=self.frameEnv, cwd=self.rqCore.machine.getTempPath(), stdin=subprocess.PIPE, - stdout=file_descriptor, - stderr=file_descriptor, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, close_fds=True, preexec_fn=os.setsid) finally: @@ -343,6 +339,16 @@ def runLinux(self): if rqd.rqconstants.RQD_PREPEND_TIMESTAMP: pipe_to_file(frameInfo.forkedCommand.stdout, frameInfo.forkedCommand.stderr, self.rqlog) + else: + with open(self.rqlog, 'a') as f: + # Convert to ASCII while discarding characters that can not be encoded + for line in frameInfo.forkedCommand.stdout: + line = line.encode('ascii', 'ignore') + f.write(line.decode('ascii') + '\n') + for line in frameInfo.forkedCommand.stderr: + line = line.encode('ascii', 'ignore') + f.write(line.decode('ascii') + '\n') + returncode = frameInfo.forkedCommand.wait() # Find exitStatus and exitSignal @@ -1222,6 +1228,8 @@ def print_and_flush_ln(fd, last_timestamp): remainder = lines[-1] for line in lines[0:-1]: + # Convert to ASCII while discarding characters that can not be encoded + line = line.encode('ascii', 'ignore') print("[%s] %s" % (curr_line_timestamp, line), file=outfile) outfile.flush() os.fsync(outfile)