feat(rosbags): adds support in apply packages and deployment, and add…

…s rosbag job update and trigger upload.

* Update package-schema.yaml

* implements the model

* separates out device rosbag spec and cloud rosbags spec

* adds rosbag to deployment schema

* Update model.py

* adds rosbag patch command

* rosbag: support only minutes for split by duration

* rosbag: support float values for max split duration

* Revert "rosbag: support float values for max split duration"

This reverts commit 17d231c.

* Update model.py

* Update validation.py

* minor changes

* fixes version in pipfile

* minor changes to deployment schema

* improves error handling

* adds rosbags examples

* reverts auth environment changes

* reverts auth environment changes

* fixes some review comments

* separated job trigger and update commands

* reverts build schema changes

* updates rosbags in package schema

* updates schema validation

Pipfile

@@ -27,6 +27,7 @@ munch = ">=2.4.0"
 pyyaml = ">=5.4.1"
 rapyuta-io = ">=1.5.0"
 tabulate = ">=0.8.0"
+pyrfc3339 = ">=1.1"
 
 [requires]
 python_version = "3"

jsonschema/deployment-schema.yaml

@@ -156,6 +156,11 @@ definitions:
                 items:
                   "$ref": "#/definitions/deviceNetworkAttachSpec"
 
+              rosBagJobs:
+                type: array
+                items:
+                  "$ref": "#/definitions/deviceROSBagJobSpec"
+
           - properties:
               runtime:
                 type: string
@@ -187,6 +192,148 @@ definitions:
                 items:
                   "$ref": "#/definitions/managedServiceSpec"
 
+              rosBagJobs:
+                type: array
+                items:
+                  "$ref": "#/definitions/cloudROSBagJobSpec"
+
+  cloudROSBagJobSpec:
+    type: object
+    properties:
+      name:
+        type: string
+      recordOptions:
+        "$ref": "#/definitions/rosbagRecordOptionsSpec"
+      overrideOptions:
+        "$ref": "#/definitions/rosbagOverrideOptionsSpec"
+    required:
+      - name
+      - recordOptions
+
+  deviceROSBagJobSpec:
+    type: object
+    properties:
+      name:
+        type: string
+      recordOptions:
+        "$ref": "#/definitions/rosbagRecordOptionsSpec"
+      uploadOptions:
+        "$ref": "#/definitions/rosbagUploadOptionsSpec"
+      overrideOptions:
+        "$ref": "#/definitions/rosbagOverrideOptionsSpec"
+    required:
+      - name
+      - recordOptions
+
+  rosbagRecordOptionsSpec:
+    type: object
+    oneOf:
+      - required:
+          - allTopics
+      - anyOf:
+          - required:
+              - topics
+          - required:
+              - topicIncludeRegex
+    properties:
+      allTopics:
+        type: boolean
+      topics:
+        type: array
+        items:
+          type: string
+      topicIncludeRegex:
+        type: array
+        items:
+          type: string
+      topicExcludeRegex:
+        type: string
+      maxMessageCount:
+        type: integer
+      node:
+        type: string
+      compression:
+        type: string
+        enum:
+          - BZ2
+          - LZ4
+      maxSplits:
+        type: integer
+      maxSplitSize:
+        type: integer
+      chunkSize:
+        type: integer
+      prefix:
+        type: string
+      maxSplitDuration:
+        type: integer
+
+  rosbagUploadOptionsSpec:
+    type: object
+    properties:
+      maxUploadRate:
+        type: integer
+        default: 1048576
+      purgeAfter:
+        type: boolean
+      uploadType:
+        type: string
+        enum:
+          - OnStop
+          - Continuous
+          - OnDemand
+        default: OnDemand
+      onDemandOpts:
+        type: object
+        "$ref": "#/definitions/rosbagOnDemandUploadOptionsSpec"
+
+  rosbagOnDemandUploadOptionsSpec:
+    type: object
+    properties:
+      timeRange:
+        type: object
+        properties:
+          from:
+            type: integer
+            default: 0
+          to:
+            type: integer
+            default: 0
+        required:
+          - from
+          - to
+    required:
+      - timeRange
+
+  rosbagOverrideOptionsSpec:
+    type: object
+    properties:
+      topicOverrideInfo:
+        type: array
+        items:
+          "$ref": "#/definitions/rosbagTopicOverrideInfoSpec"
+      excludeTopics:
+        type: array
+        items:
+          type: string
+
+  rosbagTopicOverrideInfoSpec:
+    type: object
+    oneOf:
+      - required:
+          - topicName
+          - recordFrequency
+      - required:
+          - topicName
+          - latched
+    properties:
+      topicName:
+        type: string
+      recordFrequency:
+        type: integer
+      latched:
+        type: boolean
+
   stringMap:
     type: object
     additionalProperties:

jsonschema/package-schema.yaml

@@ -60,6 +60,7 @@ definitions:
       ros:
         type: object
         "$ref": "#/definitions/rosComponentSpec"
+
     dependencies:
       runtime:
         oneOf:
@@ -74,11 +75,14 @@ definitions:
                 type: array
                 items:
                   "$ref": "#/definitions/deviceExecutableSpec"
-
               environmentArgs:
                 type: array
                 items:
                   "$ref": "#/definitions/environmentSpec"
+              rosBagJobs:
+                type: array
+                items:
+                  "$ref": "#/definitions/deviceROSBagJobSpec"
 
           - properties:
               runtime:
@@ -103,6 +107,11 @@ definitions:
                 items:
                   "$ref": "#/definitions/endpointSpec"
 
+              rosBagJobs:
+                type: array
+                items:
+                  "$ref": "#/definitions/cloudROSBagJobSpec"
+
   deviceExecutableSpec:
     type: object
     properties:
@@ -215,7 +224,7 @@ definitions:
                 type: object
                 properties:
                   depends:
-                    "$ref": "#/definitions/secretDepends"
+                    "$ref": "#/definitions/buildDepends"
                 required:
                   - depends
           - properties:
@@ -473,6 +482,141 @@ definitions:
                 enum:
                   - action
 
+  cloudROSBagJobSpec:
+    type: object
+    properties:
+      name:
+        type: string
+      recordOptions:
+        "$ref": "#/definitions/rosbagRecordOptionsSpec"
+      overrideOptions:
+        "$ref": "#/definitions/rosbagOverrideOptionsSpec"
+    required:
+      - name
+      - recordOptions
+
+  deviceROSBagJobSpec:
+    type: object
+    properties:
+      name:
+        type: string
+      recordOptions:
+        "$ref": "#/definitions/rosbagRecordOptionsSpec"
+      uploadOptions:
+        "$ref": "#/definitions/rosbagUploadOptionsSpec"
+      overrideOptions:
+        "$ref": "#/definitions/rosbagOverrideOptionsSpec"
+    required:
+      - name
+      - recordOptions
+
+  rosbagRecordOptionsSpec:
+    type: object
+    oneOf:
+      - required:
+          - allTopics
+      - anyOf:
+          - required:
+              - topics
+          - required:
+              - topicIncludeRegex
+    properties:
+      allTopics:
+        type: boolean
+      topics:
+        type: array
+        items:
+          type: string
+      topicIncludeRegex:
+        type: array
+        items:
+          type: string
+      topicExcludeRegex:
+        type: string
+      maxMessageCount:
+        type: integer
+      node:
+        type: string
+      compression:
+        type: string
+        enum:
+          - BZ2
+          - LZ4
+      maxSplits:
+        type: integer
+      maxSplitSize:
+        type: integer
+      chunkSize:
+        type: integer
+      prefix:
+        type: string
+      maxSplitDuration:
+        type: integer
+
+  rosbagUploadOptionsSpec:
+    type: object
+    properties:
+      maxUploadRate:
+        type: integer
+        default: 1048576
+      purgeAfter:
+        type: boolean
+      uploadType:
+        type: string
+        enum:
+          - OnStop
+          - Continuous
+          - OnDemand
+        default: OnDemand
+      onDemandOpts:
+        type: object
+        "$ref": "#/definitions/rosbagOnDemandUploadOptionsSpec"
+
+  rosbagOnDemandUploadOptionsSpec:
+    type: object
+    properties:
+      timeRange:
+        type: object
+        properties:
+          from:
+            type: integer
+          to:
+            type: integer
+        required:
+          - from
+          - to
+    required:
+      - timeRange
+
+  rosbagOverrideOptionsSpec:
+    type: object
+    properties:
+      topicOverrideInfo:
+        type: array
+        items:
+          "$ref": "#/definitions/rosbagTopicOverrideInfoSpec"
+      excludeTopics:
+        type: array
+        items:
+          type: string
+
+  rosbagTopicOverrideInfoSpec:
+    type: object
+    oneOf:
+      - required:
+          - topicName
+          - recordFrequency
+      - required:
+          - topicName
+          - latched
+    properties:
+      topicName:
+        type: string
+      recordFrequency:
+        type: integer
+      latched:
+        type: boolean
+
   stringMap:
     type: object
     additionalProperties:

riocli/apply/manifests/13-device-package-with-rosbag.yaml

@@ -0,0 +1,32 @@
+apiVersion: "apiextensions.rapyuta.io/v1"
+kind: "Package" #We will create a package
+metadata:
+  name: "rosbag-dev"
+  version: "1.0.0"
+  labels:
+    app: test
+spec:
+  runtime: "device"
+  device:
+          arch: "amd64"
+          restart: "always"
+  ros:
+          enabled: True
+  rosBagJobs:
+        -   name: "testbag"
+            recordOptions:
+                allTopics: True
+                maxSplits: 5
+                compression: LZ4
+                maxSplitDuration: 1
+            uploadOptions:
+                uploadType: "OnDemand"
+  executables:
+    - name: "exec"
+      type: build
+      command: "roslaunch talker talker.launch"
+      runAsBash: False
+      build:
+          depends:
+                  kind: build
+                  nameOrGUID: "build-jtmnspmxafziwbrfjknnvyaf"