Implement storage statistic printing on console

src/hal/interface/storage.h

@@ -97,3 +97,10 @@ typedef bool (*storageFunc_t)(const char *key, void *buffer, size_t length);
  * @return true in case of success.
  */
 bool storageForeach(const char* prefix, storageFunc_t func);
+
+/**
+ * Print storage information on the debug console
+ * 
+ * This function locks the storage while getting the stats.
+ */
+void storagePrintStats();

src/hal/src/storage.c

@@ -27,6 +27,7 @@
  */
 
 #include "storage.h"
+#include "param.h"
 
 #include "kve/kve.h"
 
@@ -210,3 +211,42 @@ bool storageDelete(const char* key)
 
   return result;
 }
+
+void storagePrintStats()
+{
+  kveStats_t stats;
+
+  xSemaphoreTake(storageMutex, portMAX_DELAY);
+
+  kveGetStats(&kve, &stats);
+
+  xSemaphoreGive(storageMutex);
+
+
+  DEBUG_PRINT("Used storage: %d item stored, %d Bytes/%d Bytes (%d%%)\n", stats.totalItems, stats.itemSize, stats.totalSize, (stats.itemSize*100)/stats.totalSize);
+  DEBUG_PRINT("Fragmentation: %d%%\n", stats.fragmentation);
+  DEBUG_PRINT("Efficiency: Data: %d Bytes (%d%%), Keys: %d Bytes (%d%%), Metadata: %d Bytes (%d%%)\n", 
+    stats.dataSize, (stats.dataSize*100)/stats.totalSize,
+    stats.keySize, (stats.keySize*100)/stats.totalSize,
+    stats.metadataSize, (stats.metadataSize*100)/stats.totalSize);
+}
+
+static bool storageStats;
+
+static void printStats(void)
+{
+  if (storageStats) {
+    storagePrintStats();
+
+    storageStats = false;
+  }
+}
+
+PARAM_GROUP_START(system)
+
+/**
+ * @brief Set to nonzero to dump CPU and stack usage to console
+ */
+PARAM_ADD_WITH_CALLBACK(PARAM_UINT8, storageStats, &storageStats, printStats)
+
+PARAM_GROUP_STOP(system)

src/utils/interface/kve/kve.h

@@ -48,3 +48,18 @@ void kveFormat(kveMemory_t *kve);
 bool kveCheck(kveMemory_t *kve);
 
 bool kveForeach(kveMemory_t *kve, const char *prefix, kveFunc_t func);
+
+typedef struct kveStats {
+    size_t totalSize;
+    size_t totalItems;
+    size_t itemSize;
+    size_t keySize;
+    size_t dataSize;
+    size_t metadataSize;
+    size_t holeSize;
+    size_t freeSpace;
+    size_t fragmentation;
+    size_t spaceLeftUntilForcedDefrag;
+} kveStats_t;
+
+void kveGetStats(kveMemory_t *kve, kveStats_t *stats);

src/utils/interface/kve/kve_storage.h

@@ -44,7 +44,9 @@
 
 #define KVE_STORAGE_INVALID_ADDRESS (SIZE_MAX)
 
-#define END_TAG_LENDTH 2
+#define KVE_END_TAG_LENDTH 2
+
+#define KVE_END_TAG (0xffffu)
 
 typedef struct itemHeader_s {
   uint16_t full_length;

src/utils/src/kve/kve.c

@@ -36,7 +36,7 @@ static bool appendItemToEnd(kveMemory_t *kve, size_t address, const char* key, c
     }
 
     // Test that there is enough space to write the item
-    if ((itemAddress + sizeof(kveItemHeader_t) + strlen(key) + length + END_TAG_LENDTH) < kve->memorySize) {
+    if ((itemAddress + sizeof(kveItemHeader_t) + strlen(key) + length + KVE_END_TAG_LENDTH) < kve->memorySize) {
         itemAddress += kveStorageWriteItem(kve, itemAddress, key, buffer, length);
         kveStorageWriteEnd(kve, itemAddress);
     } else {
@@ -45,7 +45,7 @@ static bool appendItemToEnd(kveMemory_t *kve, size_t address, const char* key, c
 
         itemAddress = kveStorageFindEnd(kve, FIRST_ITEM_ADDRESS);
 
-        if ((itemAddress + sizeof(kveItemHeader_t) + strlen(key) + length + END_TAG_LENDTH) < kve->memorySize) {
+        if ((itemAddress + sizeof(kveItemHeader_t) + strlen(key) + length + KVE_END_TAG_LENDTH) < kve->memorySize) {
             itemAddress += kveStorageWriteItem(kve, itemAddress, key, buffer, length);
             kveStorageWriteEnd(kve, itemAddress);
         } else {
@@ -205,3 +205,51 @@ bool kveCheck(kveMemory_t *kve) {
 
     return true;
 }
+
+void kveGetStats(kveMemory_t *kve, kveStats_t *stats) {
+    size_t item_address = FIRST_ITEM_ADDRESS;
+
+    size_t end_address = kveStorageFindEnd(kve, FIRST_ITEM_ADDRESS);
+
+    size_t total_size = 0;
+    size_t total_items = 0;
+    size_t hole_size = 0;
+    size_t item_size = 0;
+    size_t data_size = 0;
+    size_t key_size = 0;
+    size_t metadata_size = 0;
+
+    while (item_address < end_address) {
+        kveItemHeader_t itemInfo = kveStorageGetItemInfo(kve, item_address);
+
+        if (itemInfo.full_length == KVE_END_TAG) {
+            break;
+        }
+
+        total_size += itemInfo.full_length;
+
+        if (itemInfo.key_length == 0) {
+            hole_size += itemInfo.full_length;
+        } else {
+            item_size += itemInfo.full_length;
+            total_items++;
+
+            key_size += itemInfo.key_length;
+            metadata_size += sizeof(itemInfo);
+            data_size += itemInfo.full_length - itemInfo.key_length - sizeof(itemInfo);
+        }
+
+        item_address = item_address + itemInfo.full_length;
+    }
+
+    stats->totalSize = kve->memorySize;
+    stats->totalItems = total_items;
+    stats->itemSize = item_size;
+    stats->keySize = key_size;
+    stats->dataSize = data_size;
+    stats->metadataSize = metadata_size;
+    stats->holeSize = hole_size;
+    stats->freeSpace = kve->memorySize - item_size;
+    stats->fragmentation = (hole_size * 100) / (kve->memorySize - total_size);
+    stats->spaceLeftUntilForcedDefrag = kve->memorySize - total_size;
+}

src/utils/src/kve/kve_storage.c

@@ -40,8 +40,6 @@ static size_t min(size_t a, size_t b)
     }
 }
 
-#define END_TAG (0xffffu)
-
 int kveStorageWriteItem(kveMemory_t *kve, size_t address, const char* key, const void* buffer, size_t length)
 {
   kveItemHeader_t header;
@@ -70,7 +68,7 @@ uint16_t kveStorageWriteHole(kveMemory_t *kve, size_t address, size_t full_lengt
 }
 
 uint16_t kveStorageWriteEnd(kveMemory_t *kve, size_t address) {
-    uint16_t endTag = END_TAG;
+    uint16_t endTag = KVE_END_TAG;
 
     kve->write(address, &endTag, 2);
 
@@ -110,7 +108,7 @@ size_t kveStorageFindItemByKey(kveMemory_t *kve, size_t address, const char * ke
         length = searchBuffer[0] + (searchBuffer[1]<<8);
         keyLength = searchBuffer[2];
 
-        if (length == END_TAG) {
+        if (length == KVE_END_TAG) {
             return SIZE_MAX;
         }
 
@@ -146,7 +144,7 @@ size_t kveStorageFindItemByPrefix(kveMemory_t *kve, size_t address,
         length = searchBuffer[0] + (searchBuffer[1]<<8);
         keyLength = searchBuffer[2];
 
-        if (length == END_TAG) {
+        if (length == KVE_END_TAG) {
             *itemAddress = SIZE_MAX;
             return SIZE_MAX;
         }
@@ -174,7 +172,7 @@ size_t kveStorageFindEnd(kveMemory_t *kve, size_t address) {
 
     while (currentAddress < (kve->memorySize - 2)) {
         kve->read(currentAddress, &header, sizeof(header));
-        if (header.full_length == END_TAG) {
+        if (header.full_length == KVE_END_TAG) {
             return currentAddress;
         }
 
@@ -213,7 +211,7 @@ size_t kveStorageFindNextItem(kveMemory_t *kve, size_t address)
 
     // Jump over the current item
     kve->read(currentAddress, &header, sizeof(header));
-    if (header.full_length == END_TAG) {
+    if (header.full_length == KVE_END_TAG) {
         return KVE_STORAGE_INVALID_ADDRESS;
     }
     currentAddress += header.full_length;
@@ -222,7 +220,7 @@ size_t kveStorageFindNextItem(kveMemory_t *kve, size_t address)
         kve->read(currentAddress, &header, sizeof(header));
 
 
-        if (header.full_length == END_TAG) {
+        if (header.full_length == KVE_END_TAG) {
             return KVE_STORAGE_INVALID_ADDRESS;
         }
 

test/utils/src/kve/test_kve.c

@@ -213,3 +213,38 @@ void testRemoveAndStoreBiggerWhenMemoryIsFull(void) {
   TEST_ASSERT_EQUAL(true, actualDelete);
   TEST_ASSERT_EQUAL(false, actualStore);
 }
+
+void testStorageStatistics(void) {
+  // Fixture
+  // Fill memory
+  char *name = "hello";
+  uint32_t i = 42;
+  kveStore(&kve, name, &i, sizeof(i));
+  // Test
+  kveStats_t stats;
+  kveGetStats(&kve, &stats);
+  // Assert
+  TEST_ASSERT_EQUAL(strlen(name), stats.keySize);
+  TEST_ASSERT_EQUAL(sizeof(i), stats.dataSize);
+  TEST_ASSERT_EQUAL(3, stats.metadataSize);
+  TEST_ASSERT_EQUAL(strlen(name) + sizeof(i) + 3, stats.itemSize);
+  TEST_ASSERT_EQUAL(1, stats.totalItems);
+  TEST_ASSERT_EQUAL(strlen(name) + sizeof(i) + 3, stats.totalSize - stats.freeSpace);
+  TEST_ASSERT_EQUAL(0, stats.fragmentation);
+  TEST_ASSERT_EQUAL(0, stats.holeSize);
+  TEST_ASSERT_EQUAL(strlen(name) + sizeof(i) + 3, stats.totalSize - stats.spaceLeftUntilForcedDefrag);
+}
+
+void testFragmentationStatisticsIncreaseWhenFragmented(void) {
+  // Fixture
+  // Fill memory
+  char data[128];
+  kveStore(&kve, "hello", data, sizeof(data));
+  kveDelete(&kve, "hello");
+  kveStore(&kve, "hello", data, sizeof(data));
+  // Test
+  kveStats_t stats;
+  kveGetStats(&kve, &stats);
+  // Assert
+  TEST_ASSERT_NOT_EQUAL(0, stats.fragmentation);
+}