dynamic endpoints: add automatic attr storage, instantiation from ZAP templates

src/app/util/BUILD.gn

@@ -29,7 +29,6 @@ source_set("nullable-primitives") {
   public_configs = [ "${chip_root}/src:includes" ]
 }
 
-# These headers/cpp only depend on core/common
 source_set("types") {
   sources = [
     "att-storage.h",
@@ -52,14 +51,14 @@ source_set("types") {
   public_configs = [ "${chip_root}/src:includes" ]
 }
 
-# This source set also depends on data-model
 source_set("af-types") {
   sources = [ "af-types.h" ]
   deps = [
     ":types",
     "${chip_root}/src/app:paths",
     "${chip_root}/src/app/data-model",
     "${chip_root}/src/messaging",
+    "${chip_root}/src/platform:platform_config_header",
     "${chip_root}/src/protocols/interaction_model",
   ]
 }

src/app/util/af-types.h

@@ -24,6 +24,8 @@
  */
 
 #include "att-storage.h"
+#include <platform/CHIPDeviceConfig.h> // For CHIP_DEVICE_CONFIG_DYNAMIC_ENDPOINT_COUNT
+
 #include <stdbool.h> // For bool
 #include <stdint.h>  // For various uint*_t types
 
@@ -48,6 +50,15 @@
  */
 typedef uint8_t EmberAfClusterMask;
 
+/**
+ * @brief Type for specifiying cluster including mask, to differentiate server & client
+ */
+typedef struct
+{
+    chip::ClusterId clusterId;
+    EmberAfClusterMask mask;
+} EmberAfClusterSpec;
+
 /**
  * @brief Generic function type, used for either of the cluster function.
  *
@@ -180,7 +191,7 @@ typedef struct
      */
     uint8_t clusterCount;
     /**
-     * Size of all non-external, non-singlet attribute in this endpoint type.
+     * Size of all non-external, non-singleton attributes in this endpoint type.
      */
     uint16_t endpointSize;
 } EmberAfEndpointType;
@@ -221,6 +232,17 @@ struct EmberAfDefinedEndpoint
      */
     chip::DataVersion * dataVersions = nullptr;
 
+#if CHIP_DEVICE_CONFIG_DYNAMIC_ENDPOINT_COUNT > 0
+    /**
+     * Span describing a memory block to be used for automatic attribute storage if
+     * this is a dynamic endpoint. If not empty, the Span must have
+     * a size at least equal to endpointType->endpointSize (which is
+     * the sum of all endpointType->clusters[*].clustersize).
+     */
+    chip::Span<uint8_t> dynamicAttributeStorage;
+
+#endif
+
     /**
      * Root endpoint id for composed device type.
      */

src/app/util/attribute-storage.cpp

@@ -30,6 +30,7 @@
 #include <app/util/persistence/AttributePersistenceProvider.h>
 #include <lib/core/CHIPConfig.h>
 #include <lib/support/CodeUtils.h>
+#include <lib/support/SafeInt.h>
 #include <lib/support/logging/CHIPLogging.h>
 #include <platform/LockTracker.h>
 #include <protocols/interaction_model/StatusCode.h>
@@ -266,9 +267,82 @@ uint16_t emberAfGetDynamicIndexFromEndpoint(EndpointId id)
     return kEmberInvalidEndpointIndex;
 }
 
+namespace {
+
+const EmberAfCluster * getClusterTypeDefinition(EndpointId endpointId, ClusterId clusterId, EmberAfClusterMask mask)
+{
+    uint16_t index = emberAfIndexFromEndpointIncludingDisabledEndpoints(endpointId);
+    if (index != kEmberInvalidEndpointIndex)
+    {
+        return emberAfFindClusterInType(emAfEndpoints[index].endpointType, clusterId, mask, nullptr);
+    }
+    // not found
+    return nullptr;
+}
+
+} // anonymous namespace
+
+CHIP_ERROR emberAfSetupDynamicEndpointDeclaration(EmberAfEndpointType & endpointType, EndpointId templateEndpointId,
+                                                  const Span<const EmberAfClusterSpec> & templateClusterSpecs)
+{
+    // we want an explicitly empty endpoint to begin with, to make sure no already set-up endpoint is passed in
+    VerifyOrReturnError(endpointType.cluster == nullptr, CHIP_ERROR_INVALID_ARGUMENT);
+    VerifyOrReturnError(endpointType.endpointSize == 0, CHIP_ERROR_INVALID_ARGUMENT);
+    VerifyOrReturnError(endpointType.clusterCount == 0, CHIP_ERROR_INVALID_ARGUMENT);
+    // check cluster count fits target struct's clusterCount field
+    size_t clusterCount = templateClusterSpecs.size();
+    VerifyOrReturnError(CanCastTo<decltype(endpointType.clusterCount)>(clusterCount), CHIP_ERROR_NO_MEMORY);
+    // allocate new cluster array
+    auto newClusters = new EmberAfCluster[clusterCount];
+    VerifyOrReturnError(newClusters != nullptr, CHIP_ERROR_NO_MEMORY);
+    size_t endpointSize = 0;
+    // get the actual cluster pointers and sum up memory size
+    for (size_t i = 0; i < clusterCount; i++)
+    {
+        auto cluster =
+            getClusterTypeDefinition(templateEndpointId, templateClusterSpecs[i].clusterId, templateClusterSpecs[i].mask);
+        if (!cluster)
+        {
+            delete[] newClusters;
+            ChipLogError(DataManagement, "cluster 0x%04x with mask %x could not be found in template endpoint %u",
+                         (unsigned int) templateClusterSpecs[i].clusterId, templateClusterSpecs[i].mask,
+                         (unsigned int) templateEndpointId);
+            return CHIP_ERROR_NOT_FOUND;
+        }
+        // for now, we need to copy the cluster definition, unfortunately.
+        // TODO: make endpointType use a pointer to a list of EmberAfCluster* instead, so we can re-use cluster definitions
+        //   instead of duplicating them here once for every instance.
+        newClusters[i] = *cluster;
+        // sum up the needed storage, result must fit into endpointSize member (which is smaller than size_t)
+        endpointSize += cluster->clusterSize;
+        if (!CanCastTo<decltype(endpointType.endpointSize)>(endpointSize))
+        {
+            delete[] newClusters;
+            return CHIP_ERROR_NO_MEMORY;
+        }
+    }
+    // set up dynamic endpoint
+    endpointType.clusterCount = static_cast<decltype(endpointType.clusterCount)>(clusterCount);
+    endpointType.cluster      = newClusters;
+    endpointType.endpointSize = static_cast<decltype(endpointType.endpointSize)>(endpointSize);
+    return CHIP_NO_ERROR;
+}
+
+void emberAfResetDynamicEndpointDeclaration(EmberAfEndpointType & endpointType)
+{
+    if (endpointType.cluster)
+    {
+        delete[] endpointType.cluster;
+        endpointType.cluster = nullptr;
+    }
+    endpointType.clusterCount = 0;
+    endpointType.endpointSize = 0;
+}
+
 CHIP_ERROR emberAfSetDynamicEndpoint(uint16_t index, EndpointId id, const EmberAfEndpointType * ep,
-                                     const Span<DataVersion> & dataVersionStorage, Span<const EmberAfDeviceType> deviceTypeList,
-                                     EndpointId parentEndpointId)
+                                     const chip::Span<chip::DataVersion> & dataVersionStorage,
+                                     chip::Span<const EmberAfDeviceType> deviceTypeList, EndpointId parentEndpointId,
+                                     Span<uint8_t> dynamicAttributeStorage)
 {
     auto realIndex = index + FIXED_ENDPOINT_COUNT;
 
@@ -296,13 +370,23 @@ CHIP_ERROR emberAfSetDynamicEndpoint(uint16_t index, EndpointId id, const EmberA
         }
     }
 
+#if CHIP_DEVICE_CONFIG_DYNAMIC_ENDPOINT_COUNT > 0
+    if (!dynamicAttributeStorage.empty() && dynamicAttributeStorage.size() < ep->endpointSize)
+    {
+        return CHIP_ERROR_NO_MEMORY; // not enough memory provided for dynamic attribute storage
+    }
+#endif
+
     emAfEndpoints[index].endpoint       = id;
     emAfEndpoints[index].deviceTypeList = deviceTypeList;
     emAfEndpoints[index].endpointType   = ep;
     emAfEndpoints[index].dataVersions   = dataVersionStorage.data();
     // Start the endpoint off as disabled.
     emAfEndpoints[index].bitmask.Clear(EmberAfEndpointOptions::isEnabled);
     emAfEndpoints[index].parentEndpointId = parentEndpointId;
+#if CHIP_DEVICE_CONFIG_DYNAMIC_ENDPOINT_COUNT > 0
+    emAfEndpoints[index].dynamicAttributeStorage = dynamicAttributeStorage;
+#endif
 
     emberAfSetDynamicEndpointCount(MAX_ENDPOINT_COUNT - FIXED_ENDPOINT_COUNT);
 
@@ -317,6 +401,18 @@ CHIP_ERROR emberAfSetDynamicEndpoint(uint16_t index, EndpointId id, const EmberA
         }
     }
 
+#if CHIP_DEVICE_CONFIG_DYNAMIC_ENDPOINT_COUNT > 0
+    if (!dynamicAttributeStorage.empty() && ep->endpointSize > 0)
+    {
+        // Flag the endpoint as enabled here, because otherwise loading attributes cannot work
+        emAfEndpoints[index].bitmask.Set(EmberAfEndpointOptions::isEnabled);
+        // Load attributes from NVstorage or set defaults
+        emberAfInitializeAttributes(id);
+        // set disabled again, so enabling below will detect a transition
+        emAfEndpoints[index].bitmask.Clear(EmberAfEndpointOptions::isEnabled);
+    }
+#endif
+
     // Now enable the endpoint.
     emberAfEndpointEnableDisable(id, true);
 
@@ -580,21 +676,44 @@ Status emAfReadOrWriteAttribute(const EmberAfAttributeSearchRecord * attRecord,
 {
     assertChipStackLockedByCurrentThread();
 
-    uint16_t attributeOffsetIndex = 0;
+    // offset relative to the storage block, which is:
+    // - for static endpoints: attributeData[] global
+    // - for dynamic endpoints: dynamicAttributeStorage (unless nullPtr, then all attributes must be external)
+    uint16_t attributeStorageOffset = 0;
 
     for (uint16_t ep = 0; ep < emberAfEndpointCount(); ep++)
     {
         // Is this a dynamic endpoint?
         bool isDynamicEndpoint = (ep >= emberAfFixedEndpointCount());
 
         if (emAfEndpoints[ep].endpoint == attRecord->endpoint)
-        {
+        { // Got the endpoint
             const EmberAfEndpointType * endpointType = emAfEndpoints[ep].endpointType;
             uint8_t clusterIndex;
             if (!emberAfEndpointIndexIsEnabled(ep))
             {
+                // TODO: I think this is wrong, and should be a break
+                //   It does not harm because usually no other endpointindex will contain
+                //   an endpoint with the same ID, but it would cause catastrophic mess in
+                //   attribute data because attributeOffsetIndex does not get incremented -
+                //   endpoint enabling/disabling is dynamic, so enabled/disabled state
+                //   MUST NOT change the data layout!
                 continue;
             }
+
+#if CHIP_DEVICE_CONFIG_DYNAMIC_ENDPOINT_COUNT > 0
+            bool hasDynamicAttributeStorage = !emAfEndpoints[ep].dynamicAttributeStorage.empty();
+            if (hasDynamicAttributeStorage)
+            {
+                // endpoint storage is not in the static global `attributeData`, but offset
+                // from the per-endpoint dynamicAttributeStorage pointer.
+                // Endpoint processing starts here, so reset the offset.
+                attributeStorageOffset = 0;
+            }
+#else
+            const bool hasDynamicAttributeStorage = false;
+#endif
+
             for (clusterIndex = 0; clusterIndex < endpointType->clusterCount; clusterIndex++)
             {
                 const EmberAfCluster * cluster = &(endpointType->cluster[clusterIndex]);
@@ -613,9 +732,29 @@ Status emAfReadOrWriteAttribute(const EmberAfAttributeSearchRecord * attRecord,
                             }
 
                             {
-                                uint8_t * attributeLocation =
-                                    (am->mask & ATTRIBUTE_MASK_SINGLETON ? singletonAttributeLocation(am)
-                                                                         : attributeData + attributeOffsetIndex);
+                                // Determine the attribute storage base address
+                                // Attribute storage can be
+                                // - singleton: statically allocated in singletonAttributeData global
+                                // - static endpoint: statically allocated in attributeData global
+                                // - dynamic endpoint with dynamic storage: in memory block provided at endpoint instantiation
+                                uint8_t * attributeLocation;
+                                if (am->mask & ATTRIBUTE_MASK_SINGLETON)
+                                {
+                                    attributeLocation = singletonAttributeLocation(am);
+                                }
+#if CHIP_DEVICE_CONFIG_DYNAMIC_ENDPOINT_COUNT > 0
+                                else if (hasDynamicAttributeStorage)
+                                {
+                                    attributeLocation = emAfEndpoints[ep].dynamicAttributeStorage.data();
+                                }
+#endif
+                                else
+                                {
+                                    attributeLocation = attributeData;
+                                }
+                                // Apply the offset to the attribute
+                                attributeLocation += attributeStorageOffset;
+
                                 uint8_t *src, *dst;
                                 if (write)
                                 {
@@ -631,6 +770,7 @@ Status emAfReadOrWriteAttribute(const EmberAfAttributeSearchRecord * attRecord,
                                 {
                                     if (buffer == nullptr)
                                     {
+                                        // only getting metadata
                                         return Status::Success;
                                     }
 
@@ -653,7 +793,8 @@ Status emAfReadOrWriteAttribute(const EmberAfAttributeSearchRecord * attRecord,
                                 }
 
                                 // Internal storage is only supported for fixed endpoints
-                                if (!isDynamicEndpoint)
+                                // and dynamic ones with dynamicAttributeStorage assigned.
+                                if (!isDynamicEndpoint || hasDynamicAttributeStorage)
                                 {
                                     return typeSensitiveMemCopy(attRecord->clusterId, dst, src, am, write, readLength);
                                 }
@@ -666,7 +807,7 @@ Status emAfReadOrWriteAttribute(const EmberAfAttributeSearchRecord * attRecord,
                             // Increase the index if attribute is not externally stored
                             if (!(am->mask & ATTRIBUTE_MASK_EXTERNAL_STORAGE) && !(am->mask & ATTRIBUTE_MASK_SINGLETON))
                             {
-                                attributeOffsetIndex = static_cast<uint16_t>(attributeOffsetIndex + emberAfAttributeSize(am));
+                                attributeStorageOffset = static_cast<uint16_t>(attributeStorageOffset + emberAfAttributeSize(am));
                             }
                         }
                     }
@@ -676,18 +817,18 @@ Status emAfReadOrWriteAttribute(const EmberAfAttributeSearchRecord * attRecord,
                 }
 
                 // Not the cluster we are looking for
-                attributeOffsetIndex = static_cast<uint16_t>(attributeOffsetIndex + cluster->clusterSize);
+                attributeStorageOffset = static_cast<uint16_t>(attributeStorageOffset + cluster->clusterSize);
             }
 
             // Cluster is not in the endpoint.
             return Status::UnsupportedCluster;
         }
 
         // Not the endpoint we are looking for
-        // Dynamic endpoints are external and don't factor into storage size
+        // Dynamic endpoints are external and don't factor into statically allocated global storage size
         if (!isDynamicEndpoint)
         {
-            attributeOffsetIndex = static_cast<uint16_t>(attributeOffsetIndex + emAfEndpoints[ep].endpointType->endpointSize);
+            attributeStorageOffset = static_cast<uint16_t>(attributeStorageOffset + emAfEndpoints[ep].endpointType->endpointSize);
         }
     }
     return Status::UnsupportedEndpoint; // Sorry, endpoint was not found.
@@ -1153,7 +1294,7 @@ void emAfLoadAttributeDefaults(EndpointId endpoint, Optional<ClusterId> clusterI
     uint8_t attrData[ATTRIBUTE_LARGEST];
     auto * attrStorage = GetAttributePersistenceProvider();
     // Don't check whether we actually have an attrStorage here, because it's OK
-    // to have one if none of our attributes have NVM storage.
+    // to have none if none of our attributes have NVM storage.
 
     for (ep = 0; ep < epCount; ep++)
     {