feat(device): supports onboarding hwil devices via device manifest (#323

)

Wrike Ticket: https://www.wrike.com/open.htm?id=1382479736

Pipfile

@@ -24,7 +24,7 @@ graphlib-backport = ">=1.0.3"
 jinja2 = ">=3.0.1"
 munch = ">=2.4.0"
 pyyaml = ">=5.4.1"
-rapyuta-io = ">=1.15.1"
+rapyuta-io = ">=1.16.0"
 tabulate = ">=0.8.0"
 pyrfc3339 = ">=1.1"
 directory-tree = ">=0.0.3.1"

riocli/apply/manifests/device.yaml

@@ -4,13 +4,20 @@ metadata:
   name: "device-docker" # Required
   project: "project-guid"
   labels:
+    custom_label_1: label1
+    custom_label_2: label2
+    custom_label_3: label3
     app: test
 spec:
   python: "3"
   rosDistro: "melodic" # Options: ["kinetic", "melodic" (default), "noetic"]
   docker:
     enabled: True # Required
     rosbagMountPath: "/opt/rapyuta/volumes/rosbag"
+  configVariables:
+    custom_config_variable_1: value1
+    custom_config_variable_2: value2
+
 ---
 apiVersion: "apiextensions.rapyuta.io/v1"
 kind: "Device"
@@ -42,4 +49,33 @@ spec:
     rosbagMountPath: "/opt/rapyuta/volumes/rosbag"
   preinstalled:
     enabled: True
-    catkinWorkspace: "/home/rapyuta/catkin_ws"
+    catkinWorkspace: "/home/rapyuta/catkin_ws"
+
+---
+apiVersion: "apiextensions.rapyuta.io/v1"
+kind: "Device"
+metadata:
+  name: "virtual-device-docker" # Required
+  project: "project-guid"
+  labels:
+    custom_label_1: label1
+    custom_label_2: label2
+    custom_label_3: label3
+    app: test
+spec:
+  python: "3"
+  rosDistro: "melodic" # Options: ["kinetic", "melodic" (default), "noetic"]
+  virtual:
+    enabled: True # Required
+    product: "sootballs" # Required Options: ["sootballs", "flaptter", "oks"]
+    arch: "amd64" # Options: ["amd64" (default), "arm64" ]
+    os: "ubuntu" # Options: ["ubuntu" (default), "debian" ]
+    codename: "focal" # Options: ["bionic", "focal" (default), "jammy", "bullseye"]
+    highperf: False # Optional [True, False (default)]
+  docker:
+    enabled: True # Required
+    rosbagMountPath: "/opt/rapyuta/volumes/rosbag"
+  configVariables:
+    custom_config_variable_1: value1
+    custom_config_variable_2: value2
+

riocli/device/model.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Rapyuta Robotics
+# Copyright 2024 Rapyuta Robotics
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,6 +16,13 @@
 from rapyuta_io import Client
 from rapyuta_io.clients.device import Device as v1Device, DevicePythonVersion
 
+from riocli.device.util import (
+    create_hwil_device,
+    execute_onboard_command,
+    find_device_guid,
+    update_device_labels
+)
+from riocli.exceptions import DeviceNotFound
 from riocli.jsonschema.validate import load_schema
 from riocli.model import Model
 
@@ -25,6 +32,10 @@ def __init__(self, *args, **kwargs):
         self.update(*args, **kwargs)
 
     def find_object(self, client: Client) -> bool:
+        # For virtual devices, always return False to ensure re-onboarding is done even if the device already exists
+        if self.spec.get('virtual', {}).get('enabled', False):
+            return False
+
         guid, obj = self.rc.find_depends({
             "kind": "device",
             "nameOrGUID": self.metadata.name,
@@ -36,7 +47,27 @@ def find_object(self, client: Client) -> bool:
         return obj
 
     def create_object(self, client: Client, **kwargs) -> v1Device:
-        device = client.create_device(self.to_v1())
+        if not self.spec.get('virtual', {}).get('enabled', False):
+            device = client.create_device(self.to_v1())
+            return device
+
+        hwil_response = create_hwil_device(self.spec, self.metadata)
+        try:
+            device_uuid = find_device_guid(client, self.metadata.name)
+            device = client.get_device(device_uuid)
+        except DeviceNotFound:
+            update_device_labels(self.metadata, hwil_response)
+            device = client.create_device(self.to_v1())
+        except Exception as e:
+            raise e
+
+        onboard_script = device.onboard_script()
+        onboard_command = onboard_script.full_command()
+        try:
+            execute_onboard_command(hwil_response.id, onboard_command)
+        except Exception as e:
+            raise e
+
         return device
 
     def update_object(self, client: Client, obj: typing.Any) -> typing.Any:
@@ -58,11 +89,15 @@ def to_v1(self) -> v1Device:
         if preinstalled_enabled and self.spec.preinstalled.get('catkinWorkspace'):
             ros_workspace = self.spec.preinstalled.catkinWorkspace
 
+        config_variables = self.spec.get('configVariables', {})
+        labels = self.metadata.get('labels', {})
+
         return v1Device(
             name=self.metadata.name, description=self.spec.get('description'),
             runtime_docker=docker_enabled, runtime_preinstalled=preinstalled_enabled,
             ros_distro=self.spec.rosDistro, python_version=python_version,
-            rosbag_mount_path=rosbag_mount_path, ros_workspace=ros_workspace
+            rosbag_mount_path=rosbag_mount_path, ros_workspace=ros_workspace,
+            config_variables=config_variables, labels=labels,
         )
 
     @classmethod

riocli/device/util.py

@@ -12,21 +12,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import functools
+import json
 import re
 import typing
 from pathlib import Path
-import json
 
 import click
+from munch import Munch
 from rapyuta_io import Client
 from rapyuta_io.clients import LogUploads
 from rapyuta_io.clients.device import Device
 from rapyuta_io.utils import RestClient
 from rapyuta_io.utils.rest_client import HttpMethod
 
-from riocli.config import get_config_from_context, new_client
+from riocli.config import get_config_from_context, new_client, new_hwil_client
+from riocli.config.config import Configuration
 from riocli.constants import Colors
-from riocli.utils import is_valid_uuid
+from riocli.exceptions import DeviceNotFound
+from riocli.hwil.util import execute_command, find_device_id
+from riocli.utils import is_valid_uuid, trim_suffix, trim_prefix
 
 
 def name_to_guid(f: typing.Callable) -> typing.Callable:
@@ -123,6 +127,7 @@ def fetch_devices(
 
     return result
 
+
 def migrate_device_to_project(ctx: click.Context, device_id: str, dest_project_id: str) -> None:
     config = get_config_from_context(ctx)
     host = config.data.get('core_api_host', 'https://gaapiserver.apps.okd4v2.prod.rapyuta.io')
@@ -171,7 +176,81 @@ def is_remote_path(src, devices=[]):
     return None, src
 
 
-class DeviceNotFound(Exception):
-    def __init__(self, message='device not found'):
-        self.message = message
-        super().__init__(self.message)
+def create_hwil_device(spec: dict, metadata: dict) -> Munch:
+    """Create a new hardware-in-the-loop device."""
+    os = spec['virtual']['os']
+    codename = spec['virtual']['codename']
+    arch = spec['virtual']['arch']
+    labels = hwil_device_labels(spec.virtual.product, metadata.name)
+    device_name = f"{metadata['name']}-{spec['virtual']['product']}-{labels['user']}"
+    device_name = sanitize_hwil_device_name(device_name)
+    client = new_hwil_client()
+
+    try:
+        device_id = find_device_id(client, device_name)
+        return client.get_device(device_id)
+    except DeviceNotFound:
+        pass
+
+    try:
+        response = client.create_device(device_name, arch, os, codename, labels)
+        client.poll_till_device_ready(response.id, sleep_interval=5, retry_limit=3)
+        return response
+    except Exception as e:
+        raise e
+
+
+def execute_onboard_command(device_id: str, onboard_command: str) -> None:
+    """Execute the onboard command on a hardware-in-the-loop device."""
+    client = new_hwil_client()
+    try:
+        code, _, stderr = execute_command(client, device_id, onboard_command)
+        if code != 0:
+            raise Exception(f"Failed with exit code {code}: {stderr}")
+    except Exception as e:
+        raise e
+
+
+def hwil_device_labels(product_name, device_name) -> typing.Dict:
+    data = Configuration().data
+    user_email = data['email_id']
+    project_id = data['project_id']
+    organization_id = data['organization_id']
+    user_email = user_email.split('@')[0]
+
+    return {
+        "user": user_email,
+        "organization": organization_id,
+        "project": project_id,
+        "product": product_name,
+        "rapyuta_device_name": device_name,
+    }
+
+
+def update_device_labels(metadata, response) -> None:
+    device_labels = {
+        "hwil_device_id": str(response.id),
+        "hwil_device_name": response.name,
+        "arch": response.architecture,
+        "flavor": response.flavor,
+        "hwil_device_username": response.username,
+    }
+
+    existing_labels = metadata.get('labels', {})
+    existing_labels.update(device_labels)
+
+
+def sanitize_hwil_device_name(name):
+    if len(name) == 0:
+        return name
+
+    name = name[0:50]
+    name = trim_suffix(name)
+    name = trim_prefix(name)
+
+    r = ''
+    for c in name:
+        if c.isalnum() or c in ['-', '_']:
+            r = r + c
+
+    return r

riocli/exceptions/__init__.py

@@ -42,3 +42,9 @@ def __str__(self):
         return """Not logged in to HWIL. Please login first
         $ rio hwil login 
         """
+
+
+class DeviceNotFound(Exception):
+    def __init__(self, message='device not found'):
+        self.message = message
+        super().__init__(self.message)

riocli/hwil/util.py

@@ -20,7 +20,7 @@
 
 from riocli.config import new_hwil_client
 from riocli.constants import Colors
-from riocli.device.util import DeviceNotFound
+from riocli.exceptions import DeviceNotFound
 from riocli.hwilclient import Client