Add outputs (AcademySoftwareFoundation#1361)

* Add new xml spec with support for layer outputs * Add outputs from xml spec to database * Update VERSION.in Bumped minor version due to spec change * Add python code for adding outputs to layers and bump spec version to 1.14 * Remove debug code and add newline to new spec file
n-jay · Jun 11, 2024 · c29e445 · c29e445
1 parent 6c0991e
commit c29e445
Show file tree

Hide file tree

Showing 7 changed files with 143 additions and 2 deletions.
diff --git a/VERSION.in b/VERSION.in
@@ -1 +1 @@
-0.22
+0.23
diff --git a/cuebot/src/main/java/com/imageworks/spcue/LayerDetail.java b/cuebot/src/main/java/com/imageworks/spcue/LayerDetail.java
@@ -46,6 +46,7 @@ public class LayerDetail extends LayerEntity implements LayerInterface {
     public Set<String> tags = new LinkedHashSet<String>();
     public Set<String> services = new LinkedHashSet<String>();
     public Set<String> limits = new LinkedHashSet<String>();
+    public Set<String> outputs = new LinkedHashSet<String>();
 
     /*
      *

diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java
@@ -283,6 +283,7 @@ public JobDetail createJob(BuildableJob buildableJob) {
                 layerDao.insertLayerEnvironment(layer, buildableLayer.env);
                 layer.limits.stream()
                         .forEach(ln -> addLayerLimit(layer, limitDao.findLimit(ln).getLimitId()));
+                layer.outputs.stream().forEach(ln -> registerLayerOutput(layer, ln));
                 frameDao.insertFrames(layer, frames);
             }
 

diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java
@@ -752,6 +752,33 @@ private void determineResourceDefaults(Element layerTag,
         layer.timeout_llu = primaryService.timeout_llu;
     }
 
+    private void determineOutputs(Element layerTag,
+            BuildableJob job, LayerDetail layer) {
+
+        Element t_outputs = layerTag.getChild("outputs");
+        List<String> outputs = new ArrayList<String>();
+        /*
+         * Build a list of outputs from the XML.  Filter
+         * out duplicates and empty outputs.
+         */
+        if (t_outputs != null) {
+            for (Object tmp : t_outputs.getChildren()) {
+                Element t_output = (Element) tmp;
+                String output_path = t_output.getTextTrim();
+
+                if (output_path.length() == 0) {
+                    continue;
+                }
+
+                if (outputs.contains(output_path)) {
+                    continue;
+                }
+                outputs.add(output_path);
+            }
+        }
+        layer.outputs.addAll(outputs);
+    }
+
     /**
      * Converts the job space tagging format into a set of strings. Also
      * verifies each tag.

diff --git a/cuebot/src/main/resources/public/dtd/cjsl-1.14.dtd b/cuebot/src/main/resources/public/dtd/cjsl-1.14.dtd
@@ -0,0 +1,104 @@
+<!-- ================================================================= -->
+<!-- SpiCue Job Spec DTD ver 1.14                                      -->
+<!-- [email protected]                                        -->
+<!-- ================================================================= -->
+
+<!ELEMENT spec              (facility?,dept?,show,shot,user,email?,uid?,job*,depends*)>
+<!ELEMENT facility          (#PCDATA)*>
+<!ELEMENT dept              (#PCDATA)*>
+<!ELEMENT show              (#PCDATA)*>
+<!ELEMENT shot              (#PCDATA)*>
+<!ELEMENT user              (#PCDATA)*>
+<!ELEMENT email             (#PCDATA)*>
+<!ELEMENT uid               (#PCDATA)*>
+
+<!-- ================================================================= -->
+<!-- Jobs                                                              -->
+<!-- ================================================================= -->
+<!ELEMENT job               (paused?,priority?,maxretries?,maxcores?,maxgpus?,autoeat?,localbook?,os?,env*,layers?)>
+<!ATTLIST job
+    name                    NMTOKEN                 	#REQUIRED
+>
+<!ELEMENT paused            (#PCDATA)*>
+<!ELEMENT priority          (#PCDATA)*>
+<!ELEMENT maxretries        (#PCDATA)*>
+<!ELEMENT maxcores          (#PCDATA)*>
+<!ELEMENT maxgpus           (#PCDATA)*>
+<!ELEMENT autoeat           (#PCDATA)*>
+<!ELEMENT os                (#PCDATA)*>
+<!ELEMENT localbook         (#PCDATA)*>
+<!ATTLIST localbook
+    host                    NMTOKEN                     #REQUIRED
+    cores                   NMTOKEN                     #REQUIRED
+    memory                  NMTOKEN                     #REQUIRED
+    threads                 NMTOKEN                     #REQUIRED
+    gpu                     NMTOKEN                     #REQUIRED
+>
+
+<!-- ================================================================= -->
+<!-- Layers                                                            -->
+<!-- ================================================================= -->
+
+<!ELEMENT layers            (layer+)>
+<!ELEMENT layer             (cmd,range,chunk,cores?,threadable?,memory?,gpus?,gpu?,gpu_memory?,timeout?,timeout_llu?,tags?,limits?,env*,services?,outputs*)>
+<!ATTLIST layer
+    name                    NMTOKEN                #REQUIRED
+    type                    (Render | Util | Post) #REQUIRED
+>
+<!ELEMENT cmd               (#PCDATA)*>
+<!ELEMENT range             (#PCDATA)*>
+<!ELEMENT chunk             (#PCDATA)*>
+<!ELEMENT cores             (#PCDATA)*>
+<!ELEMENT threadable        (#PCDATA)*>
+<!ELEMENT memory            (#PCDATA)*>
+<!ELEMENT gpus              (#PCDATA)*>
+<!ELEMENT gpu               (#PCDATA)*>  <!-- deprecated: use gpu_memory -->
+<!ELEMENT gpu_memory        (#PCDATA)*>
+<!ELEMENT timeout           (#PCDATA)*>
+<!ELEMENT timeout_llu       (#PCDATA)*>
+<!ELEMENT tags              (#PCDATA)*>
+<!ELEMENT limits            (limit+)>
+<!ELEMENT services          (service+)>
+<!ELEMENT outputs           (output*)>
+<!ELEMENT env               (key*)>
+<!-- ================================================================= -->
+<!-- Layer Services                                                    -->
+<!-- ================================================================= -->
+<!ELEMENT service           (#PCDATA)*>
+
+<!-- ================================================================= -->
+<!-- Layer Ouuputs                                                     -->
+<!-- ================================================================= -->
+<!ELEMENT output            (#PCDATA)*>
+<!ATTLIST output
+    name                    NMTOKEN                 #REQUIRED
+>
+<!-- ================================================================= -->
+<!-- Layer Limits                                                    -->
+<!-- ================================================================= -->
+<!ELEMENT limit           (#PCDATA)*>
+
+<!-- ================================================================= -->
+<!-- Environment Variables -->
+<!-- ================================================================= -->
+
+<!ELEMENT key              (#PCDATA)*>
+<!ATTLIST key
+    name                    NMTOKEN                 #REQUIRED
+>
+
+<!-- ================================================================= -->
+<!-- Dependencies                                                      -->
+<!-- ================================================================= -->
+
+<!ELEMENT depends           (depend*)>
+<!ELEMENT depend            (depjob,deplayer?,depframe?,onjob,onlayer?,onframe?)>
+<!ATTLIST depend
+    anyframe               NMTOKEN                  #IMPLIED
+    type                   (LAYER_ON_SIM_FRAME|PREVIOUS_FRAME|JOB_ON_JOB|JOB_ON_LAYER|JOB_ON_FRAME|LAYER_ON_JOB|LAYER_ON_FRAME|LAYER_ON_LAYER|FRAME_ON_JOB|FRAME_ON_LAYER|FRAME_ON_FRAME|FRAME_BY_FRAME)   #REQUIRED
+>
+<!ELEMENT depjob           (#PCDATA)*>
+<!ELEMENT onjob            (#PCDATA)*>
+<!ELEMENT deplayer         (#PCDATA)*>
+<!ELEMENT onlayer          (#PCDATA)*>
+<!ELEMENT depframe         (#PCDATA)*>
diff --git a/pyoutline/outline/backend/cue.py b/pyoutline/outline/backend/cue.py
@@ -394,6 +394,14 @@ def _serialize(launcher, use_pycuerun):
         except (AttributeError, IndexError):
             service.text = "default"
 
+        if spec_version >= Version("1.14"):
+            layer_outputs = Et.SubElement(spec_layer, "outputs")
+            outputs = layer.get_outputs()
+            for output_name in outputs:
+                output_path = outputs[output_name]
+                output = Et.SubElement(layer_outputs, "output", {"name": output_name})
+                output.text = output_path.get_path()
+
         build_dependencies(ol, layer, depends)
 
     if not layers:

diff --git a/pyoutline/outline/outline.cfg b/pyoutline/outline/outline.cfg
@@ -5,7 +5,7 @@ wrapper_dir = %(home)s/wrappers
 user_dir =
 bin_dir = %(home)s/bin
 backend = cue
-spec_version = 1.13
+spec_version = 1.14
 facility = local
 domain = example.com
 maxretries = 2