Optimize memory usage with DBString and DBArray

CMakeLists.txt

@@ -35,6 +35,7 @@ add_library(lcf)
 
 set(LCF_SOURCES
 	src/data.cpp
+	src/dbarray.cpp
 	src/encoder.cpp
 	src/ini.cpp
 	src/inireader.cpp
@@ -194,6 +195,9 @@ set(LCF_SOURCES
 
 set(LCF_HEADERS
 	src/lcf/data.h
+	src/lcf/dbarray.h
+	src/lcf/dbarrayalloc.h
+	src/lcf/dbstring.h
 	src/lcf/encoder.h
 	src/lcf/enum_tags.h
 	src/lcf/flag_set.h

Makefile.am

@@ -43,6 +43,7 @@ liblcf_la_LDFLAGS = \
 	-no-undefined
 liblcf_la_SOURCES = \
 	src/data.cpp \
+	src/dbarray.cpp \
 	src/encoder.cpp \
 	src/ini.cpp \
 	src/inireader.cpp \
@@ -201,6 +202,9 @@ liblcf_la_SOURCES = \
 
 lcfinclude_HEADERS = \
 	src/lcf/data.h \
+	src/lcf/dbarray.h \
+	src/lcf/dbarrayalloc.h \
+	src/lcf/dbstring.h \
 	src/lcf/encoder.h \
 	src/lcf/enum_tags.h \
 	src/lcf/flag_set.h \

generator/generate.py

@@ -34,6 +34,7 @@
     'Int16': 'int16_t',
     'Int32': 'int32_t',
     'String': 'std::string',
+    'DBString': 'DBString',
 }
 
 # Additional Jinja 2 functions
@@ -70,6 +71,9 @@ def cpp_type(ty, prefix=True):
     m = re.match(r'(Vector|Array)<(.*)>', ty)
     if m:
         return 'std::vector<%s>' % cpp_type(m.group(2), prefix)
+    m = re.match(r'DBArray<(.*)>', ty)
+    if m:
+        return 'DBArray<%s>' % cpp_type(m.group(1), prefix)
 
     m = re.match(r'Ref<(.*):(.*)>', ty)
     if m:
@@ -100,7 +104,7 @@ def pod_default(field):
     ftype = field.type
 
     # Not a POD, no default
-    if dfl == '' or dfl == '\'\'' or ftype.startswith('Vector') or ftype.startswith('Array'):
+    if dfl == '' or dfl == '\'\'' or ftype.startswith('Vector') or ftype.startswith('Array') or ftype.startswith('DBArray'):
         return ""
 
     if ftype == 'Boolean':
@@ -164,6 +168,9 @@ def struct_headers(ty, header_map):
     if ty == 'String':
         return ['<string>']
 
+    if ty == 'DBString':
+        return ['"lcf/dbstring.h"']
+
     if ty in int_types or ty == "DatabaseVersion":
         return ['<stdint.h>']
 
@@ -191,6 +198,10 @@ def struct_headers(ty, header_map):
     if m:
         return ['<vector>'] + struct_headers(m.group(2), header_map)
 
+    m = re.match(r'DBArray<(.*)>', ty)
+    if m:
+        return ['<lcf/dbarray.h>'] + struct_headers(m.group(1), header_map)
+
     header = header_map.get(ty)
     if header is not None:
         return ['"lcf/rpg/%s.h"' % header]
@@ -336,7 +347,7 @@ def needs_ctor(struct_name):
                                     for method, hdrs in setup[struct_name])
 
 def type_is_array(ty):
-    return re.match(r'(Vector|Array)<(.*)>', ty)
+    return re.match(r'(Vector|Array|DBArray)<(.*)>', ty)
 
 def is_monotonic_from_0(enum):
     expected = 0

src/dbarray.cpp

@@ -0,0 +1,91 @@
+#include "lcf/dbarrayalloc.h"
+#include "lcf/dbarray.h"
+#include "lcf/dbstring.h"
+#include <cassert>
+#include <cstddef>
+
+//#define LCF_DEBUG_DBARRAY
+
+#ifdef LCF_DEBUG_DBARRAY
+#include <iostream>
+#endif
+
+namespace lcf {
+
+constexpr DBArrayAlloc::size_type DBArrayAlloc::_empty_buf;
+constexpr DBString::size_type DBString::npos;
+
+static ptrdiff_t HeaderSize(size_t align) {
+	return std::max(sizeof(DBArrayAlloc::size_type), align);
+}
+
+static size_t AllocSize(size_t size, size_t align) {
+	return HeaderSize(align) + size;
+}
+
+static void* Adjust(void* p, ptrdiff_t off) {
+	return reinterpret_cast<void*>(reinterpret_cast<intptr_t>(p) + off);
+}
+
+void* DBArrayAlloc::alloc(size_type size, size_type field_size, size_type align) {
+	if (field_size == 0) {
+		return empty_buf();
+	}
+	assert(align <= alignof(std::max_align_t));
+	auto* raw = ::operator new(AllocSize(size, align));
+	auto* p = Adjust(raw, HeaderSize(align));
+	*get_size_ptr(p) = field_size;
+#ifdef LCF_DEBUG_DBARRAY
+	std::cout << "DBArray: Allocated"
+		<< " size=" << size
+		<< " field_size=" << *get_size_ptr(p)
+		<< " align=" << align
+		<< " ptr=" << raw
+		<< " adjusted=" << p
+		<< std::endl;
+#endif
+	return p;
+}
+
+void DBArrayAlloc::free(void* p, size_type align) noexcept {
+	assert(p != nullptr);
+	if (p != empty_buf()) {
+		auto* raw = Adjust(p, -HeaderSize(align));
+#ifdef LCF_DEBUG_DBARRAY
+		std::cout << "DBArray: Free"
+			<< " align=" << align
+			<< " ptr=" << raw
+			<< " adjusted=" << p
+			<< " field_size=" << *get_size_ptr(p)
+			<< std::endl;
+#endif
+		::operator delete(raw);
+	}
+}
+
+char* DBString::construct_z(const char* s, size_t len) {
+	auto* p = alloc(len);
+	if (len) {
+		std::memcpy(p, s, len + 1);
+	}
+	return p;
+}
+
+char* DBString::construct_sv(const char* s, size_t len) {
+	auto* p = alloc(len);
+	if (len) {
+		std::memcpy(p, s, len);
+		p[len] = '\0';
+	}
+	return p;
+}
+
+DBString& DBString::operator=(const DBString& o) {
+	if (this != &o) {
+		destroy();
+		_storage = construct_z(o.data(), o.size());
+	}
+	return *this;
+}
+
+} // namespace lcf

src/generated/lcf/rpg/actor.h

@@ -14,8 +14,8 @@
 
 // Headers
 #include <stdint.h>
-#include <string>
 #include <vector>
+#include "lcf/dbstring.h"
 #include "lcf/rpg/equipment.h"
 #include "lcf/rpg/learning.h"
 #include "lcf/rpg/parameters.h"
@@ -31,16 +31,16 @@ namespace rpg {
 	public:
 		void Setup();
 		int ID = 0;
-		std::string name;
-		std::string title;
-		std::string character_name;
+		DBString name;
+		DBString title;
+		DBString character_name;
 		int32_t character_index = 0;
 		bool transparent = false;
 		int32_t initial_level = 1;
 		int32_t final_level = -1;
 		bool critical_hit = true;
 		int32_t critical_hit_chance = 30;
-		std::string face_name;
+		DBString face_name;
 		int32_t face_index = 0;
 		bool two_weapon = false;
 		bool lock_equipment = false;
@@ -58,7 +58,7 @@ namespace rpg {
 		int32_t battler_animation = 1;
 		std::vector<Learning> skills;
 		bool rename_skill = false;
-		std::string skill_name;
+		DBString skill_name;
 		std::vector<uint8_t> state_ranks;
 		std::vector<uint8_t> attribute_ranks;
 		std::vector<int32_t> battle_commands;

src/generated/lcf/rpg/animation.h

@@ -13,8 +13,8 @@
 #define LCF_RPG_ANIMATION_H
 
 // Headers
-#include <string>
 #include <vector>
+#include "lcf/dbstring.h"
 #include "lcf/enum_tags.h"
 #include "lcf/rpg/animationframe.h"
 #include "lcf/rpg/animationtiming.h"
@@ -48,8 +48,8 @@ namespace rpg {
 		);
 
 		int ID = 0;
-		std::string name;
-		std::string animation_name;
+		DBString name;
+		DBString animation_name;
 		bool large = false;
 		std::vector<AnimationTiming> timings;
 		int32_t scope = 0;

src/generated/lcf/rpg/attribute.h

@@ -14,7 +14,7 @@
 
 // Headers
 #include <stdint.h>
-#include <string>
+#include "lcf/dbstring.h"
 #include "lcf/enum_tags.h"
 #include <ostream>
 #include <type_traits>
@@ -36,7 +36,7 @@ namespace rpg {
 		);
 
 		int ID = 0;
-		std::string name;
+		DBString name;
 		int32_t type = 0;
 		int32_t a_rate = 300;
 		int32_t b_rate = 200;

src/generated/lcf/rpg/battlecommand.h

@@ -13,7 +13,7 @@
 #define LCF_RPG_BATTLECOMMAND_H
 
 // Headers
-#include <string>
+#include "lcf/dbstring.h"
 #include "lcf/enum_tags.h"
 #include <ostream>
 #include <type_traits>
@@ -45,7 +45,7 @@ namespace rpg {
 		);
 
 		int ID = 0;
-		std::string name;
+		DBString name;
 		int32_t type = 0;
 	};
 	inline std::ostream& operator<<(std::ostream& os, BattleCommand::Type code) {

src/generated/lcf/rpg/battleranimation.h

@@ -13,8 +13,8 @@
 #define LCF_RPG_BATTLERANIMATION_H
 
 // Headers
-#include <string>
 #include <vector>
+#include "lcf/dbstring.h"
 #include "lcf/enum_tags.h"
 #include "lcf/rpg/battleranimationextension.h"
 #include <ostream>
@@ -34,7 +34,7 @@ namespace rpg {
 		};
 
 		int ID = 0;
-		std::string name;
+		DBString name;
 		int32_t speed = 0;
 		std::vector<BattlerAnimationExtension> base_data;
 		std::vector<BattlerAnimationExtension> weapon_data;

src/generated/lcf/rpg/battleranimationextension.h

@@ -14,7 +14,7 @@
 
 // Headers
 #include <stdint.h>
-#include <string>
+#include "lcf/dbstring.h"
 #include "lcf/enum_tags.h"
 #include <ostream>
 #include <type_traits>
@@ -36,8 +36,8 @@ namespace rpg {
 		);
 
 		int ID = 0;
-		std::string name;
-		std::string battler_name;
+		DBString name;
+		DBString battler_name;
 		int32_t battler_index = 0;
 		int32_t animation_type = 0;
 		int32_t animation_id = 1;

src/generated/lcf/rpg/chipset.h

@@ -14,8 +14,8 @@
 
 // Headers
 #include <stdint.h>
-#include <string>
 #include <vector>
+#include "lcf/dbstring.h"
 #include "lcf/enum_tags.h"
 #include <ostream>
 #include <type_traits>
@@ -39,8 +39,8 @@ namespace rpg {
 		Chipset();
 		void Init();
 		int ID = 0;
-		std::string name;
-		std::string chipset_name;
+		DBString name;
+		DBString chipset_name;
 		std::vector<int16_t> terrain_data;
 		std::vector<uint8_t> passable_data_lower;
 		std::vector<uint8_t> passable_data_upper;

src/generated/lcf/rpg/class.h

@@ -14,8 +14,8 @@
 
 // Headers
 #include <stdint.h>
-#include <string>
 #include <vector>
+#include "lcf/dbstring.h"
 #include "lcf/rpg/learning.h"
 #include "lcf/rpg/parameters.h"
 #include <ostream>
@@ -29,7 +29,7 @@ namespace rpg {
 	class Class {
 	public:
 		int ID = 0;
-		std::string name;
+		DBString name;
 		bool two_weapon = false;
 		bool lock_equipment = false;
 		bool auto_battle = false;

src/generated/lcf/rpg/commonevent.h

@@ -13,8 +13,8 @@
 #define LCF_RPG_COMMONEVENT_H
 
 // Headers
-#include <string>
 #include <vector>
+#include "lcf/dbstring.h"
 #include "lcf/enum_tags.h"
 #include "lcf/rpg/eventcommand.h"
 #include <ostream>
@@ -34,7 +34,7 @@ namespace rpg {
 		};
 
 		int ID = 0;
-		std::string name;
+		DBString name;
 		int32_t trigger = 0;
 		bool switch_flag = false;
 		int32_t switch_id = 1;