Merge remote-tracking branch 'origin/pr/627'

* origin/pr/627: (75 commits)
  .gitlab-ci: update tags for tests
  tests/app: fix test name
  vm stats: nearest int instead of truncation
  storage: fix deprecated UTC naive syntax
  qubes/ext/utils: provide Port for device_class
  Make mypy happy and add CI job
  q-dev: update tests and make pylint happy
  q-dev: pylint
  q-dev: fix assignment.devices
  q-dev: less scary device category names
  q-dev: update docs
  q-dev: fix type hint
  q-dev: fix block device removing
  q-dev: call attach-confirm socket directly
  q-dev: fix deny list
  q-dev: pylint + black
  q-dev: wait for attaching devices during startup and update tests
  q-dev: fix conflicted attachments
  q-dev: keep consistency in fire_event_for_permission
  q-dev: rename attach-confirm -> qubes-device-attach-confirm
  ...

.gitlab-ci.yml

@@ -6,6 +6,7 @@ checks:pylint:
   script:
   - PYTHONPATH=test-packages:~/qubes-core-qrexec python3 -m pylint qubes
   stage: checks
+
 checks:tests:
   after_script:
   - ci/codecov-wrapper -F unittests
@@ -36,7 +37,22 @@ checks:tests:
   - PYTHONPATH=test-packages:~/qubes-core-qrexec ./run-tests
   stage: checks
   tags:
-  - vm-kvm
+  - vm
+
+mypy:
+  stage: checks
+  image: fedora:40
+  tags:
+    - docker
+  before_script:
+    - sudo dnf install -y python3-mypy python3-pip
+    - sudo python3 -m pip install lxml-stubs types-docutils types-pywin32
+  script:
+    - mypy --install-types --non-interactive --ignore-missing-imports --exclude tests/ --junit-xml mypy.xml qubes
+  artifacts:
+    reports:
+      junit: mypy.xml
+
 include:
 - file: /r4.3/gitlab-base.yml
   project: QubesOS/qubes-continuous-integration

Makefile

@@ -66,31 +66,31 @@ ADMIN_API_METHODS_SIMPLE = \
 	admin.vm.device.pci.Attached \
 	admin.vm.device.pci.Available \
 	admin.vm.device.pci.Detach \
-	admin.vm.device.pci.Set.required \
+	admin.vm.device.pci.Set.assignment \
 	admin.vm.device.pci.Unassign \
 	admin.vm.device.block.Assign \
 	admin.vm.device.block.Assigned \
 	admin.vm.device.block.Attach \
 	admin.vm.device.block.Attached \
 	admin.vm.device.block.Available \
 	admin.vm.device.block.Detach \
-	admin.vm.device.block.Set.required \
+	admin.vm.device.block.Set.assignment \
 	admin.vm.device.block.Unassign \
 	admin.vm.device.usb.Assign \
 	admin.vm.device.usb.Assigned \
 	admin.vm.device.usb.Attach \
 	admin.vm.device.usb.Attached \
 	admin.vm.device.usb.Available \
 	admin.vm.device.usb.Detach \
-	admin.vm.device.usb.Set.required \
+	admin.vm.device.usb.Set.assignment \
 	admin.vm.device.usb.Unassign \
 	admin.vm.device.mic.Assign \
 	admin.vm.device.mic.Assigned \
 	admin.vm.device.mic.Attach \
 	admin.vm.device.mic.Attached \
 	admin.vm.device.mic.Available \
 	admin.vm.device.mic.Detach \
-	admin.vm.device.mic.Set.required \
+	admin.vm.device.mic.Set.assignment \
 	admin.vm.device.mic.Unassign \
 	admin.vm.feature.CheckWithNetvm \
 	admin.vm.feature.CheckWithTemplate \
@@ -227,7 +227,7 @@ endif
 				  admin.vm.device.testclass.Unassign \
 				  admin.vm.device.testclass.Attached \
 				  admin.vm.device.testclass.Assigned \
-				  admin.vm.device.testclass.Set.required \
+				  admin.vm.device.testclass.Set.assignment \
 				  admin.vm.device.testclass.Available
 	install -d $(DESTDIR)/etc/qubes/policy.d/include
 	install -m 0644 qubes-rpc-policy/admin-local-ro \

doc/qubes-devices.rst

@@ -1,25 +1,80 @@
 :py:mod:`qubes.devices` -- Devices
-===================================
+==================================
 
-Main concept is that some domain (backend) may expose (potentially multiple)
-devices, which can be attached to other domains (frontend). Devices can be of
-different buses (like 'pci', 'usb', etc.). Each device bus is implemented by
-an extension (see :py:mod:`qubes.ext`).
+The main concept is that a domain (backend) can expose (potentially multiple)
+devices, each through a port, where only one device can be in one port at
+any given time. Such devices can be connected to other domains (frontends).
+Devices can be of different buses (like 'pci', 'usb', etc.). Each device bus
+is implemented by an extension (see :py:mod:`qubes.ext`).
 
-Devices are identified by pair of (backend domain, `ident`), where `ident` is
-:py:class:`str` and can contain only characters from `[a-zA-Z0-9._-]` set.
+Devices are identified by a pair (`port`, `device_id`), where `port` is a pair
+(backend domain, `port_id`). Both `port_id` and `device_id` are :py:class:`str`,
+and in addition, port_id is unique per backend. More about the requirements
+for `port_id` and `device_id` can be found in the sections below.
 
+Classes
+-------
+
+:py:class:`qubes.device_protocol.Port`: a pair `<backend_domain>:<port_id>`
+with `devclass` (e.g., `pci`, `usb`). In the previous version (before
+QubesOS 4.3), this was referred to as `Device`, and `port_id` was named `ident`.
+
+:py:class:`qubes.device_protocol.AnyPort`: A class used to handle cases where
+any port is accepted.
+
+:py:class:`qubes.device_protocol.VirtualDevice`: A pair `<port>:<device_id>`.
+This class links a device identified by `device_id` to a specific port.
+If both values are specified, the instance represents a device connected to
+that particular port. If the port is of type `AnyPort`, it represents a device
+identified by `device_id` that can be connected to any port. This is used by
+:py:class:`qubes.device_protocol.DeviceInfo`, which describes what to do with
+a device identified by `device_id` when connected anywhere. Similarly,
+when `device_id` is `*`, the instance represents any potential device
+connected to the given port. As a result, the device is considered "virtual"
+meaning it may or may not represent an actual device in the system.
+A device with `*:*` (any port and any device) is not permitted.
+
+:py:class:`qubes.device_protocol.DeviceInfo`: Derived from `VirtualDevice`.
+Extensions should assume that `Port` is provided, and based on that,
+`device_id` should return the same string for the same device, regardless of
+which port it is connected to. The `device_id` acts as a device hash
+and *should* be "human-readable". It must contain only digits, ASCII letters,
+spaces, and the following characters: `!#$%&()*+,-./:;<>?@[\]^_{|}~`.
+It cannot be empty or equal to `*`.
+
+:py:class:`qubes.device_protocol.DeviceAssignment`: Represents the relationship
+between a `VirtualDevice` and a `frontend_domain`. There are four modes:
+#. `manual` (attachment): The device is manually attached to `frontend_domain`.
+This type of assignment does not persist across domain restarts.
+#. `auto-attach`: Any device that matches a `VirtualDevice` will be
+automatically attached to the `frontend_domain` when discovered
+or during domain startup.
+#. `ask-to-attach`: Functions like `auto-attach`, but prompts the user for
+confirmation before attaching. If no GUI is available, the prompt is ignored.
+#. `required`: The device must be available during `frontend_domain` startup and
+will be attached before the domain is started.
+
+:py:class:`qubes.device_protocol.DeviceInterface`: Represents device interfaces
+as a 7-character code in the format `BCCSSII`, where `B` indicates the devclass
+(e.g., `p` for PCI, `u` for USB, `?` for unknown), `CC` is the class code,
+`SS` is the subclass code, and `II` represents the interface code.
+
+:py:class:`qubes.device_protocol.DeviceCategory`: Provides an easy-to-use,
+arbitrary subset of interfaces with names assigned to categories considered as
+most relevant to users. When needed, the class should be extended with new
+categories. This structure allows for quick identification of the device type
+and can be useful when displaying devices to the end user.
 
 Device Assignment vs Attachment
 -------------------------------
 
-:py:class:`qubes.device_protocol.DeviceAssignment` describes the assignment of a device
-to a frontend VM. For clarity let's us introduce two types of assignments:
+For clarity let's us introduce two types of assignments:
 *potential* and *real* (attachment). Attachment indicates that the device
 has been attached by the Qubes backend to its frontend VM and is visible
 from its perspective. Potential assignment, on the other hand,
-has two additional options: `automatically_attach` and `required`.
-For detailed descriptions, refer to the `DeviceAssignment` documentation.
+has tree modes: `auto-attach`, `ask-to-attach` and `required`.
+For detailed descriptions, take a look at
+:py:class:`qubes.device_protocol.DeviceAssignment` documentation.
 In general we refer to potential assignment as assignment
 and real assignment as attachment. To check whether the device is currently
 attached, we check :py:meth:`qubes.device_protocol.DeviceAssignment.attached`,
@@ -28,17 +83,35 @@ we check :py:meth:`qubes.device_protocol.DeviceAssignment.attach_automatically`.
 Potential and real connections may coexist at the same time,
 in which case both values will be true.
 
+Understanding Device Identity
+-----------------------------
+
+It is important to understand that :py:class:`qubes.device_protocol.Port` does not
+correspond to the device itself, but rather to the *port* to which the device
+is connected. Therefore, when assigning a device to a VM, such as
+`sys-usb:1-1.1`, the port `1-1.1` is actually assigned, and thus
+*every* devices connected to it will be automatically attached.
+Similarly, when assigning `vm:sda`, every block device with the name `sda`
+will be automatically attached. We can limit this using
+:py:meth:`qubes.device_protocol.DeviceInfo.device_id`, which returns a string
+containing information presented by the device, such as for example
+`vendor_id`, `product_id`, `serial_number`, and encoded interfaces.
+In the case of block devices, `device_id` consists of the parent's `device_id`
+to which the device is connected (if any) and the interface/partition number.
+In practice, this means that, a partition on a USB drive will only
+be automatically attached to a frontend domain if the parent presents
+the correct serial number etc.
 
 Actions
 -------
 
-The `assign` action signifies that a device will be assigned to the frontend VM
+The `assign` action means that a device will be assigned to the frontend VM
 in a potential form (this does not change the current system state).
 This will result in an attempt to automatically attach the device
-upon the next VM startup. If `required=True`, and the device cannot be attached,
+upon the next VM startup. If `mode=required`, and the device cannot be attached,
 the VM startup will fail. Additionally, upon device detection (`device-added`),
 an attempt will be made to attach the device. However, at any time
-(unless `required=True`), the user can manually modify this state by performing
+(unless `mode=required`), the user can manually modify this state by performing
 `attach` or `detach` on the device, changing the current system state.
 This will not alter the assignment, and automatic attachment attempts
 will still be made in the future. To remove the assignment the user
@@ -48,23 +121,23 @@ Assignment Management
 ---------------------
 
 Assignments can be edited at any time: regardless of whether the VM is running
-or the device is currently attached. An exception is `required=True`,
-in which case the VM must be shut down. Removing the assignment does not change the real system state, so if the device is currently attached
-and the user remove the assignment, it will not be detached,
-but it will not be automatically attached in the future.
-Similarly, it works the other way around with `assign`.
+or the device is currently attached. Removing the assignment does not change
+the real system state, so if the device is currently attached and the user
+remove the assignment, it will not be detached, but it will not be
+automatically attached in the future. Similarly, it works the other way
+around with `assign`.
 
 Proper Assignment States
 ------------------------
 
 In short, we can think of device assignment in terms of three flags:
 #. `attached` - indicating whether the device is currently assigned,
-#.  `attach_automatically` - indicating whether the device will be
+#. `attach_automatically` - indicating whether the device will be
 automatically attached by the system daemon,
 #. `required` - determining whether the failure of automatic attachment should
 result in the domain startup being interrupted.
 
-Then the proper states of assignment
+Then the possible states of assignment
 (`attached`, `automatically_attached`, `required`) are as follow:
 #. `(True, False, False)` -> domain is running, device is manually attached
 and could be manually detach any time.
@@ -79,51 +152,52 @@ because either (i) domain is halted, device (ii) manually detached or
 #. `(False, True, True)`  -> domain is halted, device assigned to domain
 and required to start domain.
 
+Note that if `required=True` then `automatically_attached=True`.
+
+Conflicted Assignments
+----------------------
+
+If a connected device has multiple assignments to different `frontend_domain`
+instances, the user will be asked to choose which domain connect the device to.
+If no GUI client is available, the device will not be connected to any domain.
+If multiple assignments exist for a connected device with different options but
+to the same `frontend_domain`, the most specific assignment will take
+precedence, according to the following order (from highest to lowest priority):
+#. Assignment specifies both `port` and `device_id`.
+#. Assignment specifies only the `port`.
+#. Assignment specifies only the `device_id`.
+
+It is important to note that only one matching assignment can exist within
+each of the categories listed above.
+
+Port Assignment
+---------------
+
+It is possible to not assign a specific device but rather a port,
+(e.g., we can use the `--port` flag in the client). In this case,
+the value `*` will appear in the `identity` field of the `qubes.xml` file.
+This indicates that the identity presented by the devices will be ignored,
+and all connected devices will be automatically attached.
+
 
 PCI Devices
 -----------
 
 PCI devices cannot be manually attached to a VM at any time.
 We must first create an assignment (`assign`) as required
-(in client we can use `--required` flag) while the VM is turned off.
-Then, it will be automatically attached upon each VM startup.
-However, if a PCI device is currently in use by another VM,
-the startup of the second VM will fail.
-PCI devices can only be assigned with the `required=True`, which does not
-allow for manual modification of the state during VM operation (attach/detach).
+(in client we can use `--required` flag). Then, it will be automatically
+attached upon each VM startup. However, if a PCI device is currently in use
+by another VM, the startup of the second VM will fail.
 
 Microphone
 ----------
 
-The microphone cannot be assigned (potentially) to any VM (attempting to attach the microphone during VM startup fails).
-
-Understanding Device Self Identity
-----------------------------------
-
-It is important to understand that :py:class:`qubes.device_protocol.Device` does not
-correspond to the device itself, but rather to the *port* to which the device
-is connected. Therefore, when assigning a device to a VM, such as
-`sys-usb:1-1.1`, the port `1-1.1` is actually assigned, and thus
-*every* devices connected to it will be automatically attached.
-Similarly, when assigning `vm:sda`, every block device with the name `sda`
-will be automatically attached. We can limit this using :py:meth:`qubes.device_protocol.DeviceInfo.self_identity`, which returns a string containing information
-presented by the device, such as, `vendor_id`, `product_id`, `serial_number`,
-and encoded interfaces. In the case of block devices, `self_identity`
-consists of the parent port to which the device is connected (if any),
-the parent's `self_identity`, and the interface/partition number.
-In practice, this means that, a partition on a USB drive will only be
-automatically attached to a frontend domain if the parent presents
-the correct serial number etc., and is connected to a specific port.
+The microphone cannot be assigned with the `mode=required` to any VM.
 
-Port Assignment
----------------
+USB Devices
+-----------
 
-It is possible to not assign a specific device but rather a port,
-(e.g., we can use the `--port` flag in the client). In this case,
-the value `any` will appear in the `identity` field of the `qubes.xml` file.
-This indicates that the identity presented by the devices will be ignored,
-and all connected devices will be automatically attached. Note that to create
-an assignment, *any* device must currently be connected to the port.
+The USB devices cannot be assigned with the `mode=required` to any VM.
 
 
 .. automodule:: qubes.devices

qubes-rpc-policy/90-admin-default.policy.header

@@ -26,14 +26,14 @@
 !include-service admin.vm.device.mic.Attached * include/admin-local-ro
 !include-service admin.vm.device.mic.Available * include/admin-local-ro
 !include-service admin.vm.device.mic.Detach * include/admin-local-rwx
-!include-service admin.vm.device.mic.Set.required * include/admin-local-rwx
+!include-service admin.vm.device.mic.Set.assignment * include/admin-local-rwx
 !include-service admin.vm.device.mic.Unassign * include/admin-local-rwx
 !include-service admin.vm.device.usb.Assign * include/admin-local-rwx
 !include-service admin.vm.device.usb.Assigned * include/admin-local-ro
 !include-service admin.vm.device.usb.Attach * include/admin-local-rwx
 !include-service admin.vm.device.usb.Attached * include/admin-local-ro
 !include-service admin.vm.device.usb.Available * include/admin-local-ro
 !include-service admin.vm.device.usb.Detach * include/admin-local-rwx
-!include-service admin.vm.device.usb.Set.required * include/admin-local-rwx
+!include-service admin.vm.device.usb.Set.assignment * include/admin-local-rwx
 !include-service admin.vm.device.usb.Unassign * include/admin-local-rwx
 

qubes/api/__init__.py

@@ -135,7 +135,7 @@ class AbstractQubesAPI:
     '''
 
     #: the preferred socket location (to be overridden in child's class)
-    SOCKNAME = None
+    SOCKNAME = ""
 
     app: qubes.Qubes
     src: qubes.vm.qubesvm.QubesVM
@@ -144,7 +144,7 @@ def __init__(self,
                  src: bytes,
                  method_name: bytes,
                  dest: bytes,
-                 arg: qubes.Qubes,
+                 arg: bytes,
                  send_event: Any = None) -> None:
         #: :py:class:`qubes.Qubes` object
         self.app = app