diff --git a/Pipfile b/Pipfile index 64e7b46f..ffcb8e82 100644 --- a/Pipfile +++ b/Pipfile @@ -24,7 +24,7 @@ graphlib-backport = ">=1.0.3" jinja2 = ">=3.0.1" munch = ">=2.4.0" pyyaml = ">=5.4.1" -rapyuta-io = ">=1.15.1" +rapyuta-io = ">=1.16.0" tabulate = ">=0.8.0" pyrfc3339 = ">=1.1" directory-tree = ">=0.0.3.1" diff --git a/Pipfile.lock b/Pipfile.lock index 8b763441..459d03bb 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "7bacb695d20153554c293689753f9eaeb88cef9fb75728ed5ca01d28c216c629" + "sha256": "9789d391580f5c4225143207da1a7a187880296fc33cb44f6f08c7b633490a95" }, "pipfile-spec": 6, "requires": { @@ -688,11 +688,11 @@ }, "rapyuta-io": { "hashes": [ - "sha256:0bb7073a06c5552510906f54236f1110a401ba73d3836ce239d7e24627192bc0", - "sha256:8ef3f8b54ca83a6e8a7d5ac63380103f3d511de32b758af5b6c69ef1637920ba" + "sha256:8c946e6561c88d176013ab197bbb46797653a4e4e5ed223972071a110e36e682", + "sha256:e77f5e42a9ad6892c6de516896e914d7f147e6aa904857bff5627bf83effbe6c" ], "index": "pypi", - "version": "==1.15.1" + "version": "==1.16.0" }, "rapyuta-io-cli": { "path": ".", @@ -716,11 +716,11 @@ }, "setuptools": { "hashes": [ - "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4", - "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0" + "sha256:937a48c7cdb7a21eb53cd7f9b59e525503aa8abaf3584c730dc5f7a5bec3a650", + "sha256:a58a8fde0541dab0419750bcc521fbdf8585f6e5cb41909df3a472ef7b81ca95" ], "markers": "python_version >= '3.8'", - "version": "==70.0.0" + "version": "==70.1.1" }, "shellingham": { "hashes": [ @@ -763,11 +763,11 @@ }, "urllib3": { "hashes": [ - "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d", - "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19" + "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472", + "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168" ], "markers": "python_version >= '3.8'", - "version": "==2.2.1" + "version": "==2.2.2" }, "waiting": { "hashes": [ @@ -1120,11 +1120,11 @@ }, "pip": { "hashes": [ - "sha256:ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc", - "sha256:ea9bd1a847e8c5774a5777bb398c19e80bcd4e2aa16a4b301b718fe6f593aba2" + "sha256:5aa64f65e1952733ee0a9a9b1f52496ebdb3f3077cc46f80a16d983b58d1180a", + "sha256:efca15145a95e95c00608afeab66311d40bfb73bb2266a855befd705e6bb15a0" ], - "markers": "python_version >= '3.7'", - "version": "==24.0" + "markers": "python_version >= '3.8'", + "version": "==24.1.1" }, "pip-shims": { "hashes": [ @@ -1294,11 +1294,11 @@ }, "setuptools": { "hashes": [ - "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4", - "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0" + "sha256:937a48c7cdb7a21eb53cd7f9b59e525503aa8abaf3584c730dc5f7a5bec3a650", + "sha256:a58a8fde0541dab0419750bcc521fbdf8585f6e5cb41909df3a472ef7b81ca95" ], "markers": "python_version >= '3.8'", - "version": "==70.0.0" + "version": "==70.1.1" }, "six": { "hashes": [ @@ -1468,11 +1468,11 @@ }, "urllib3": { "hashes": [ - "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d", - "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19" + "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472", + "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168" ], "markers": "python_version >= '3.8'", - "version": "==2.2.1" + "version": "==2.2.2" }, "vistir": { "hashes": [ diff --git a/riocli/apply/manifests/device.yaml b/riocli/apply/manifests/device.yaml index a0059221..bbd70b11 100644 --- a/riocli/apply/manifests/device.yaml +++ b/riocli/apply/manifests/device.yaml @@ -4,6 +4,9 @@ metadata: name: "device-docker" # Required project: "project-guid" labels: + custom_label_1: label1 + custom_label_2: label2 + custom_label_3: label3 app: test spec: python: "3" @@ -11,6 +14,10 @@ spec: docker: enabled: True # Required rosbagMountPath: "/opt/rapyuta/volumes/rosbag" + configVariables: + custom_config_variable_1: value1 + custom_config_variable_2: value2 + --- apiVersion: "apiextensions.rapyuta.io/v1" kind: "Device" @@ -42,4 +49,33 @@ spec: rosbagMountPath: "/opt/rapyuta/volumes/rosbag" preinstalled: enabled: True - catkinWorkspace: "/home/rapyuta/catkin_ws" \ No newline at end of file + catkinWorkspace: "/home/rapyuta/catkin_ws" + +--- +apiVersion: "apiextensions.rapyuta.io/v1" +kind: "Device" +metadata: + name: "virtual-device-docker" # Required + project: "project-guid" + labels: + custom_label_1: label1 + custom_label_2: label2 + custom_label_3: label3 + app: test +spec: + python: "3" + rosDistro: "melodic" # Options: ["kinetic", "melodic" (default), "noetic"] + virtual: + enabled: True # Required + product: "sootballs" # Required Options: ["sootballs", "flaptter", "oks"] + arch: "amd64" # Options: ["amd64" (default), "arm64" ] + os: "ubuntu" # Options: ["ubuntu" (default), "debian" ] + codename: "focal" # Options: ["bionic", "focal" (default), "jammy", "bullseye"] + highperf: False # Optional [True, False (default)] + docker: + enabled: True # Required + rosbagMountPath: "/opt/rapyuta/volumes/rosbag" + configVariables: + custom_config_variable_1: value1 + custom_config_variable_2: value2 + diff --git a/riocli/device/model.py b/riocli/device/model.py index bd781338..6b5d3c5e 100644 --- a/riocli/device/model.py +++ b/riocli/device/model.py @@ -1,4 +1,4 @@ -# Copyright 2023 Rapyuta Robotics +# Copyright 2024 Rapyuta Robotics # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,6 +16,13 @@ from rapyuta_io import Client from rapyuta_io.clients.device import Device as v1Device, DevicePythonVersion +from riocli.device.util import ( + create_hwil_device, + execute_onboard_command, + find_device_guid, + update_device_labels +) +from riocli.exceptions import DeviceNotFound from riocli.jsonschema.validate import load_schema from riocli.model import Model @@ -25,6 +32,10 @@ def __init__(self, *args, **kwargs): self.update(*args, **kwargs) def find_object(self, client: Client) -> bool: + # For virtual devices, always return False to ensure re-onboarding is done even if the device already exists + if self.spec.get('virtual', {}).get('enabled', False): + return False + guid, obj = self.rc.find_depends({ "kind": "device", "nameOrGUID": self.metadata.name, @@ -36,7 +47,27 @@ def find_object(self, client: Client) -> bool: return obj def create_object(self, client: Client, **kwargs) -> v1Device: - device = client.create_device(self.to_v1()) + if not self.spec.get('virtual', {}).get('enabled', False): + device = client.create_device(self.to_v1()) + return device + + hwil_response = create_hwil_device(self.spec, self.metadata) + try: + device_uuid = find_device_guid(client, self.metadata.name) + device = client.get_device(device_uuid) + except DeviceNotFound: + update_device_labels(self.metadata, hwil_response) + device = client.create_device(self.to_v1()) + except Exception as e: + raise e + + onboard_script = device.onboard_script() + onboard_command = onboard_script.full_command() + try: + execute_onboard_command(hwil_response.id, onboard_command) + except Exception as e: + raise e + return device def update_object(self, client: Client, obj: typing.Any) -> typing.Any: @@ -58,11 +89,15 @@ def to_v1(self) -> v1Device: if preinstalled_enabled and self.spec.preinstalled.get('catkinWorkspace'): ros_workspace = self.spec.preinstalled.catkinWorkspace + config_variables = self.spec.get('configVariables', {}) + labels = self.metadata.get('labels', {}) + return v1Device( name=self.metadata.name, description=self.spec.get('description'), runtime_docker=docker_enabled, runtime_preinstalled=preinstalled_enabled, ros_distro=self.spec.rosDistro, python_version=python_version, - rosbag_mount_path=rosbag_mount_path, ros_workspace=ros_workspace + rosbag_mount_path=rosbag_mount_path, ros_workspace=ros_workspace, + config_variables=config_variables, labels=labels, ) @classmethod diff --git a/riocli/device/util.py b/riocli/device/util.py index 28ce2b1e..0d5400f8 100644 --- a/riocli/device/util.py +++ b/riocli/device/util.py @@ -12,21 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. import functools +import json import re import typing from pathlib import Path -import json import click +from munch import Munch from rapyuta_io import Client from rapyuta_io.clients import LogUploads from rapyuta_io.clients.device import Device from rapyuta_io.utils import RestClient from rapyuta_io.utils.rest_client import HttpMethod -from riocli.config import get_config_from_context, new_client +from riocli.config import get_config_from_context, new_client, new_hwil_client +from riocli.config.config import Configuration from riocli.constants import Colors -from riocli.utils import is_valid_uuid +from riocli.exceptions import DeviceNotFound +from riocli.hwil.util import execute_command, find_device_id +from riocli.utils import is_valid_uuid, trim_suffix, trim_prefix def name_to_guid(f: typing.Callable) -> typing.Callable: @@ -123,6 +127,7 @@ def fetch_devices( return result + def migrate_device_to_project(ctx: click.Context, device_id: str, dest_project_id: str) -> None: config = get_config_from_context(ctx) host = config.data.get('core_api_host', 'https://gaapiserver.apps.okd4v2.prod.rapyuta.io') @@ -171,7 +176,81 @@ def is_remote_path(src, devices=[]): return None, src -class DeviceNotFound(Exception): - def __init__(self, message='device not found'): - self.message = message - super().__init__(self.message) +def create_hwil_device(spec: dict, metadata: dict) -> Munch: + """Create a new hardware-in-the-loop device.""" + os = spec['virtual']['os'] + codename = spec['virtual']['codename'] + arch = spec['virtual']['arch'] + labels = hwil_device_labels(spec.virtual.product, metadata.name) + device_name = f"{metadata['name']}-{spec['virtual']['product']}-{labels['user']}" + device_name = sanitize_hwil_device_name(device_name) + client = new_hwil_client() + + try: + device_id = find_device_id(client, device_name) + return client.get_device(device_id) + except DeviceNotFound: + pass + + try: + response = client.create_device(device_name, arch, os, codename, labels) + client.poll_till_device_ready(response.id, sleep_interval=5, retry_limit=3) + return response + except Exception as e: + raise e + + +def execute_onboard_command(device_id: str, onboard_command: str) -> None: + """Execute the onboard command on a hardware-in-the-loop device.""" + client = new_hwil_client() + try: + code, _, stderr = execute_command(client, device_id, onboard_command) + if code != 0: + raise Exception(f"Failed with exit code {code}: {stderr}") + except Exception as e: + raise e + + +def hwil_device_labels(product_name, device_name) -> typing.Dict: + data = Configuration().data + user_email = data['email_id'] + project_id = data['project_id'] + organization_id = data['organization_id'] + user_email = user_email.split('@')[0] + + return { + "user": user_email, + "organization": organization_id, + "project": project_id, + "product": product_name, + "rapyuta_device_name": device_name, + } + + +def update_device_labels(metadata, response) -> None: + device_labels = { + "hwil_device_id": str(response.id), + "hwil_device_name": response.name, + "arch": response.architecture, + "flavor": response.flavor, + "hwil_device_username": response.username, + } + + existing_labels = metadata.get('labels', {}) + existing_labels.update(device_labels) + + +def sanitize_hwil_device_name(name): + if len(name) == 0: + return name + + name = name[0:50] + name = trim_suffix(name) + name = trim_prefix(name) + + r = '' + for c in name: + if c.isalnum() or c in ['-', '_']: + r = r + c + + return r diff --git a/riocli/exceptions/__init__.py b/riocli/exceptions/__init__.py index 57950ddd..677eb6b7 100644 --- a/riocli/exceptions/__init__.py +++ b/riocli/exceptions/__init__.py @@ -42,3 +42,9 @@ def __str__(self): return """Not logged in to HWIL. Please login first $ rio hwil login """ + + +class DeviceNotFound(Exception): + def __init__(self, message='device not found'): + self.message = message + super().__init__(self.message) diff --git a/riocli/hwil/util.py b/riocli/hwil/util.py index 610b4841..57c886a9 100644 --- a/riocli/hwil/util.py +++ b/riocli/hwil/util.py @@ -20,7 +20,7 @@ from riocli.config import new_hwil_client from riocli.constants import Colors -from riocli.device.util import DeviceNotFound +from riocli.exceptions import DeviceNotFound from riocli.hwilclient import Client diff --git a/riocli/hwilclient/client.py b/riocli/hwilclient/client.py index 277678fb..3ec1b308 100644 --- a/riocli/hwilclient/client.py +++ b/riocli/hwilclient/client.py @@ -22,37 +22,9 @@ from rapyuta_io.utils import ConflictError, RetriesExhausted, UnauthorizedError from rapyuta_io.utils.rest_client import HttpMethod, RestClient -from riocli.utils import generate_short_guid +from riocli.utils import generate_short_guid, sanitize_label -def trim_suffix(name): - if len(name) == 0 or name[0].isalnum(): - return name - - return trim_suffix(name[1:]) - - -def trim_prefix(name): - if len(name) == 0 or name[len(name) - 1].isalnum(): - return name - - return trim_prefix(name[:len(name) - 1]) - -def sanitize_label(name): - if len(name) == 0: - return name - - name = name[0:63] - name = trim_suffix(name) - name = trim_prefix(name) - - r = '' - for c in name: - if c.isalnum() or c in ['-', '_', '.']: - r = r + c - - return r - def handle_server_errors(response: requests.Response): status_code = response.status_code diff --git a/riocli/jsonschema/schemas/device-schema.yaml b/riocli/jsonschema/schemas/device-schema.yaml index c0545a70..945d4b96 100644 --- a/riocli/jsonschema/schemas/device-schema.yaml +++ b/riocli/jsonschema/schemas/device-schema.yaml @@ -53,6 +53,52 @@ definitions: - "2" - "3" default: "3" + configVariables: + $ref: "#/definitions/stringMap" + virtual: + oneOf: + - properties: + enabled: + enum: + - False + required: + - enabled + - properties: + enabled: + enum: + - True + product: + type: string + enum: + - sootballs + - flaptter + - oks + arch: + type: string + enum: + - amd64 + - arm64 + default: amd64 + os: + type: string + enum: + - ubuntu + - debian + default: ubuntu + codename: + type: string + enum: + - bionic + - focal + - jammy + - bullseye + default: focal + highperf: + type: boolean + default: False + required: + - enabled + - product dependencies: docker: oneOf: diff --git a/riocli/utils/__init__.py b/riocli/utils/__init__.py index 0189c243..9158729b 100644 --- a/riocli/utils/__init__.py +++ b/riocli/utils/__init__.py @@ -266,3 +266,33 @@ def update_appimage(version: str): def generate_short_guid() -> str: return uuid.uuid4().hex[:8] + + +def trim_suffix(name): + if len(name) == 0 or name[0].isalnum(): + return name + + return trim_suffix(name[1:]) + + +def trim_prefix(name): + if len(name) == 0 or name[len(name) - 1].isalnum(): + return name + + return trim_prefix(name[:len(name) - 1]) + + +def sanitize_label(name): + if len(name) == 0: + return name + + name = name[0:63] + name = trim_suffix(name) + name = trim_prefix(name) + + r = '' + for c in name: + if c.isalnum() or c in ['-', '_', '.']: + r = r + c + + return r diff --git a/setup.py b/setup.py index 968c72c2..c912683d 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ "python-dateutil>=2.8.2", "pytz", "pyyaml>=5.4.1", - "rapyuta-io>=1.15.1", + "rapyuta-io>=1.16.0", "requests>=2.20.0", "setuptools", "six>=1.13.0",