Added string deduplication (closes #1303)

CHANGELOG.md

@@ -5,6 +5,7 @@ HEAD
 ----
 
 * Added comparisons (`>`, `>=`, `==`, `!=`, `<`, and `<=`) between `JsonVariant`s
+* Added string deduplication (issue #1303)
 
 v6.15.2 (2020-05-15)
 -------

README.md

@@ -31,6 +31,7 @@ ArduinoJson is a C++ JSON library for Arduino and IoT (Internet Of Things).
     * [Consumes roughly 10% less RAM than the "official" Arduino_JSON library](https://arduinojson.org/2019/11/19/arduinojson-vs-arduino_json/?utm_source=github&utm_medium=readme)
     * [Fixed memory allocation, no heap fragmentation](https://arduinojson.org/v6/api/jsondocument/?utm_source=github&utm_medium=readme)
     * [Optionally works without heap memory (zero malloc)](https://arduinojson.org/v6/api/staticjsondocument/?utm_source=github&utm_medium=readme)
+    * Deduplicates strings
 * Versatile
     * [Supports custom allocators (to use external RAM chip, for example)](https://arduinojson.org/v6/how-to/use-external-ram-on-esp32/?utm_source=github&utm_medium=readme)
     * Supports [Arduino's `String`](https://arduinojson.org/v6/api/config/enable_arduino_string/) and [STL's `std::string`](https://arduinojson.org/v6/api/config/enable_std_string/?utm_source=github&utm_medium=readme)

extras/tests/Helpers/WString.h

@@ -9,6 +9,9 @@
 // Reproduces Arduino's String class
 class String {
  public:
+  String() {}
+  explicit String(const char* s) : _str(s) {}
+
   String& operator+=(const char* rhs) {
     _str += rhs;
     return *this;

extras/tests/JsonDeserializer/filter.cpp

@@ -239,7 +239,7 @@ TEST_CASE("Filtering") {
       10,
       DeserializationError::Ok,
       "[{\"example\":1},{\"example\":3}]",
-      JSON_ARRAY_SIZE(2) + 2 * JSON_OBJECT_SIZE(1) + 16
+      JSON_ARRAY_SIZE(2) + 2 * JSON_OBJECT_SIZE(1) + 8
     },
     {
       "[',2,3]",

extras/tests/JsonDeserializer/string.cpp

@@ -74,16 +74,16 @@ TEST_CASE("Invalid JSON string") {
   }
 }
 
-TEST_CASE("Not enough room to duplicate the string") {
-  DynamicJsonDocument doc(JSON_OBJECT_SIZE(0));
+TEST_CASE("Not enough room to save the key") {
+  DynamicJsonDocument doc(JSON_OBJECT_SIZE(1) + 8);
 
   SECTION("Quoted string") {
-    REQUIRE(deserializeJson(doc, "{\"example\":1}") ==
+    REQUIRE(deserializeJson(doc, "{\"accuracy\":1}") ==
             DeserializationError::NoMemory);
   }
 
   SECTION("Non-quoted string") {
-    REQUIRE(deserializeJson(doc, "{example:1}") ==
+    REQUIRE(deserializeJson(doc, "{accuracy:1}") ==
             DeserializationError::NoMemory);
   }
 }

extras/tests/JsonDocument/StaticJsonDocument.cpp

@@ -212,7 +212,7 @@ TEST_CASE("StaticJsonDocument") {
 
   SECTION("garbageCollect()") {
     StaticJsonDocument<256> doc;
-    doc[std::string("example")] = std::string("example");
+    doc[std::string("example")] = std::string("jukebox");
     doc.remove("example");
     REQUIRE(doc.memoryUsage() == JSON_OBJECT_SIZE(1) + 16);
 

extras/tests/MemoryPool/CMakeLists.txt

@@ -4,8 +4,8 @@
 
 add_executable(MemoryPoolTests 
 	allocVariant.cpp
-	allocString.cpp
 	clear.cpp
+	saveString.cpp
 	size.cpp
 	StringCopier.cpp
 )

extras/tests/MemoryPool/StringCopier.cpp

@@ -38,8 +38,47 @@ TEST_CASE("StringCopier") {
 
     str.startString(&pool);
     str.append('h');
-    str.commit(&pool);
+    str.save(&pool);
 
     REQUIRE(1 == pool.size());
   }
 }
+
+static const char* addStringToPool(MemoryPool* pool, const char* s) {
+  StringCopier str;
+  str.startString(pool);
+  str.append(s);
+  str.append('\0');
+  return str.save(pool);
+}
+
+TEST_CASE("StringCopier::save() deduplicates strings") {
+  char buffer[4096];
+  MemoryPool pool(buffer, 4096);
+
+  SECTION("Basic") {
+    const char* s1 = addStringToPool(&pool, "hello");
+    const char* s2 = addStringToPool(&pool, "world");
+    const char* s3 = addStringToPool(&pool, "hello");
+
+    REQUIRE(s1 == s3);
+    REQUIRE(s2 != s3);
+    REQUIRE(pool.size() == 12);
+  }
+
+  SECTION("Requires terminator") {
+    const char* s1 = addStringToPool(&pool, "hello world");
+    const char* s2 = addStringToPool(&pool, "hello");
+
+    REQUIRE(s2 != s1);
+    REQUIRE(pool.size() == 12 + 6);
+  }
+
+  SECTION("Don't overrun") {
+    const char* s1 = addStringToPool(&pool, "hello world");
+    const char* s2 = addStringToPool(&pool, "wor");
+
+    REQUIRE(s2 != s1);
+    REQUIRE(pool.size() == 12 + 4);
+  }
+}

extras/tests/MemoryPool/clear.cpp

@@ -3,6 +3,7 @@
 // MIT License
 
 #include <ArduinoJson/Memory/MemoryPool.hpp>
+#include <ArduinoJson/Strings/StringAdapters.hpp>
 #include <catch.hpp>
 
 using namespace ARDUINOJSON_NAMESPACE;
@@ -21,8 +22,8 @@ TEST_CASE("MemoryPool::clear()") {
   }
 
   SECTION("Discards allocated strings") {
-    pool.allocFrozenString(10);
-    REQUIRE(pool.size() > 0);
+    pool.saveString(adaptString(const_cast<char *>("123456789")));
+    REQUIRE(pool.size() == 10);
 
     pool.clear();
 

extras/tests/MemoryPool/saveString.cpp

@@ -0,0 +1,81 @@
+// ArduinoJson - arduinojson.org
+// Copyright Benoit Blanchon 2014-2020
+// MIT License
+
+#include <ArduinoJson/Memory/MemoryPool.hpp>
+#include <ArduinoJson/Strings/StringAdapters.hpp>
+#include <catch.hpp>
+
+using namespace ARDUINOJSON_NAMESPACE;
+
+static const char *saveString(MemoryPool &pool, const char *s) {
+  return pool.saveString(adaptString(const_cast<char *>(s)));
+}
+
+TEST_CASE("MemoryPool::saveString()") {
+  char buffer[32];
+  MemoryPool pool(buffer, 32);
+
+  SECTION("Duplicates different strings") {
+    const char *a = saveString(pool, "hello");
+    const char *b = saveString(pool, "world");
+    REQUIRE(a != b);
+    REQUIRE(pool.size() == 6 + 6);
+  }
+
+  SECTION("Deduplicates identical strings") {
+    const char *a = saveString(pool, "hello");
+    const char *b = saveString(pool, "hello");
+    REQUIRE(a == b);
+    REQUIRE(pool.size() == 6);
+  }
+
+  SECTION("Returns NULL when full") {
+    REQUIRE(pool.capacity() == 32);
+
+    const void *p1 = saveString(pool, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+    REQUIRE(p1 != 0);
+    REQUIRE(pool.size() == 32);
+
+    const void *p2 = saveString(pool, "b");
+    REQUIRE(p2 == 0);
+  }
+
+  SECTION("Returns NULL when pool is too small") {
+    const void *p = saveString(pool, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+    REQUIRE(0 == p);
+  }
+
+  SECTION("Returns NULL when buffer is NULL") {
+    MemoryPool pool2(0, 32);
+    REQUIRE(0 == saveString(pool2, "a"));
+  }
+
+  SECTION("Returns NULL when capacity is 0") {
+    MemoryPool pool2(buffer, 0);
+    REQUIRE(0 == saveString(pool2, "a"));
+  }
+
+  SECTION("Returns same address after clear()") {
+    const void *a = saveString(pool, "hello");
+    pool.clear();
+    const void *b = saveString(pool, "world");
+
+    REQUIRE(a == b);
+  }
+
+  SECTION("Can use full capacity when fresh") {
+    const void *a = saveString(pool, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+
+    REQUIRE(a != 0);
+  }
+
+  SECTION("Can use full capacity after clear") {
+    saveString(pool, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+    pool.clear();
+
+    const void *a = saveString(pool, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb");
+
+    REQUIRE(a != 0);
+  }
+}

extras/tests/MemoryPool/size.cpp

@@ -22,24 +22,6 @@ TEST_CASE("MemoryPool::size()") {
     REQUIRE(0 == pool.size());
   }
 
-  SECTION("Decreases after freezeString()") {
-    StringSlot a = pool.allocExpandableString();
-    pool.freezeString(a, 1);
-    REQUIRE(pool.size() == 1);
-
-    StringSlot b = pool.allocExpandableString();
-    pool.freezeString(b, 1);
-    REQUIRE(pool.size() == 2);
-  }
-
-  SECTION("Increases after allocFrozenString()") {
-    pool.allocFrozenString(1);
-    REQUIRE(pool.size() == 1);
-
-    pool.allocFrozenString(2);
-    REQUIRE(pool.size() == 3);
-  }
-
   SECTION("Doesn't grow when memory pool is full") {
     const size_t variantCount = sizeof(buffer) / sizeof(VariantSlot);
 

extras/tests/Misc/StringAdapters.cpp

@@ -6,6 +6,7 @@
 #include "progmem_emulation.hpp"
 #include "weird_strcmp.hpp"
 
+#include <ArduinoJson/Strings/ArduinoStringAdapter.hpp>
 #include <ArduinoJson/Strings/ConstRamStringAdapter.hpp>
 #include <ArduinoJson/Strings/FlashStringAdapter.hpp>
 #include <ArduinoJson/Strings/SizedRamStringAdapter.hpp>
@@ -114,6 +115,21 @@ TEST_CASE("std::string") {
   CHECK(adapter.size() == 5);
 }
 
+TEST_CASE("Arduino String") {
+  ::String str("bravo");
+  ArduinoStringAdapter adapter = adaptString(str);
+
+  CHECK(adapter.compare(NULL) > 0);
+  CHECK(adapter.compare("alpha") > 0);
+  CHECK(adapter.compare("bravo") == 0);
+  CHECK(adapter.compare("charlie") < 0);
+
+  CHECK(adapter.equals("bravo"));
+  CHECK_FALSE(adapter.equals("charlie"));
+
+  CHECK(adapter.size() == 5);
+}
+
 TEST_CASE("custom_string") {
   custom_string str("bravo");
   StlStringAdapter<custom_string> adapter = adaptString(str);

extras/tests/MixedConfiguration/CMakeLists.txt

@@ -18,6 +18,8 @@ add_executable(MixedConfigurationTests
 	enable_nan_0.cpp
 	enable_nan_1.cpp
 	enable_progmem_1.cpp
+	enable_string_deduplication_0.cpp
+	enable_string_deduplication_1.cpp
 	use_double_0.cpp
 	use_double_1.cpp
 	use_long_long_0.cpp