From 52b121e068b4f45c67fdfb9408b110304cc8da30 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Fri, 24 Apr 2020 10:43:40 -0400 Subject: [PATCH 01/23] abieos, rocksdb, chain-kv --- .gitmodules | 6 + libraries/CMakeLists.txt | 5 + libraries/abieos | 1 + libraries/chain-kv/.clang-format | 76 ++ libraries/chain-kv/CMakeLists.txt | 13 + .../chain-kv/include/b1/chain_kv/chain_kv.hpp | 886 ++++++++++++++++++ libraries/chain-kv/unit_tests/CMakeLists.txt | 23 + .../chain-kv/unit_tests/chain_kv_tests.hpp | 114 +++ libraries/chain-kv/unit_tests/main.cpp | 36 + .../chain-kv/unit_tests/undo_stack_tests.cpp | 457 +++++++++ libraries/chain-kv/unit_tests/view_tests.cpp | 216 +++++ .../unit_tests/write_session_tests.cpp | 71 ++ libraries/rocksdb | 1 + 13 files changed, 1905 insertions(+) create mode 160000 libraries/abieos create mode 100644 libraries/chain-kv/.clang-format create mode 100644 libraries/chain-kv/CMakeLists.txt create mode 100644 libraries/chain-kv/include/b1/chain_kv/chain_kv.hpp create mode 100644 libraries/chain-kv/unit_tests/CMakeLists.txt create mode 100644 libraries/chain-kv/unit_tests/chain_kv_tests.hpp create mode 100644 libraries/chain-kv/unit_tests/main.cpp create mode 100644 libraries/chain-kv/unit_tests/undo_stack_tests.cpp create mode 100644 libraries/chain-kv/unit_tests/view_tests.cpp create mode 100644 libraries/chain-kv/unit_tests/write_session_tests.cpp create mode 160000 libraries/rocksdb diff --git a/.gitmodules b/.gitmodules index a2e867f9361..8553d4eb424 100644 --- a/.gitmodules +++ b/.gitmodules @@ -21,3 +21,9 @@ [submodule "eosio-wasm-spec-tests"] path = eosio-wasm-spec-tests url = https://github.com/EOSIO/eosio-wasm-spec-tests +[submodule "libraries/abieos"] + path = libraries/abieos + url = https://github.com/EOSIO/abieos.git +[submodule "libraries/rocksdb"] + path = libraries/rocksdb + url = https://github.com/facebook/rocksdb.git diff --git a/libraries/CMakeLists.txt b/libraries/CMakeLists.txt index 77185fbdb56..21cae663cbb 100644 --- a/libraries/CMakeLists.txt +++ b/libraries/CMakeLists.txt @@ -1,3 +1,5 @@ +set(PORTABLE ON) # rocksdb: don't use sse4.2 + add_subdirectory( fc ) add_subdirectory( builtins ) add_subdirectory( softfloat ) @@ -8,6 +10,9 @@ add_subdirectory( chain ) add_subdirectory( testing ) add_subdirectory( version ) add_subdirectory( state_history ) +add_subdirectory( abieos ) +add_subdirectory( rocksdb EXCLUDE_FROM_ALL ) +add_subdirectory( chain-kv ) set(USE_EXISTING_SOFTFLOAT ON CACHE BOOL "use pre-exisiting softfloat lib") set(ENABLE_TOOLS OFF CACHE BOOL "Build tools") diff --git a/libraries/abieos b/libraries/abieos new file mode 160000 index 00000000000..cf15468978c --- /dev/null +++ b/libraries/abieos @@ -0,0 +1 @@ +Subproject commit cf15468978c075ce7aeb0e626071e592fa7ff040 diff --git a/libraries/chain-kv/.clang-format b/libraries/chain-kv/.clang-format new file mode 100644 index 00000000000..b55bfd7924e --- /dev/null +++ b/libraries/chain-kv/.clang-format @@ -0,0 +1,76 @@ +BasedOnStyle: LLVM +IndentWidth: 3 +UseTab: Never +ColumnLimit: 120 + +--- +Language: Cpp +# always align * and & to the type +DerivePointerAlignment: false +PointerAlignment: Left + +# regroup includes to these classes +IncludeCategories: + - Regex: '(<|"(eosio)/)' + Priority: 4 + - Regex: '(<|"(boost)/)' + Priority: 3 + - Regex: '(<|"(llvm|llvm-c|clang|clang-c)/' + Priority: 3 + - Regex: '<[[:alnum:]]+>' + Priority: 2 + - Regex: '.*' + Priority: 1 + +#IncludeBlocks: Regroup + +# set indent for public, private and protected +#AccessModifierOffset: 3 + +# make line continuations twice the normal indent +ContinuationIndentWidth: 6 + +# add missing namespace comments +FixNamespaceComments: true + +# add spaces to braced list i.e. int* foo = { 0, 1, 2 }; instead of int* foo = {0,1,2}; +Cpp11BracedListStyle: false +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: true +AlignConsecutiveDeclarations: true +AlignOperands: true +AlignTrailingComments: true +AllowShortCaseLabelsOnASingleLine: true +AllowShortFunctionsOnASingleLine: All +AllowShortBlocksOnASingleLine: true +#AllowShortIfStatementsOnASingleLine: WithoutElse +#AllowShortIfStatementsOnASingleLine: true +#AllowShortLambdasOnASingleLine: All +AllowShortLoopsOnASingleLine: true +AlwaysBreakTemplateDeclarations: true + +BinPackParameters: true +### use this with clang9 +BreakBeforeBraces: Custom +BraceWrapping: + #AfterCaseLabel: true + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + +BreakConstructorInitializers: BeforeColon +CompactNamespaces: true +IndentCaseLabels: true +IndentPPDirectives: AfterHash +NamespaceIndentation: Inner +ReflowComments: true +SortIncludes: true +SortUsingDeclarations: true +--- diff --git a/libraries/chain-kv/CMakeLists.txt b/libraries/chain-kv/CMakeLists.txt new file mode 100644 index 00000000000..f32c1802d8c --- /dev/null +++ b/libraries/chain-kv/CMakeLists.txt @@ -0,0 +1,13 @@ +file(GLOB_RECURSE HEADERS "include/*.hpp") + +add_library( chain-kv INTERFACE ) + +target_link_libraries( chain-kv + INTERFACE fc rocksdb gflags + ) + +target_include_directories( chain-kv + INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/../rocksdb/include" + ) + +add_subdirectory(unit_tests) diff --git a/libraries/chain-kv/include/b1/chain_kv/chain_kv.hpp b/libraries/chain-kv/include/b1/chain_kv/chain_kv.hpp new file mode 100644 index 00000000000..7f7e31683d5 --- /dev/null +++ b/libraries/chain-kv/include/b1/chain_kv/chain_kv.hpp @@ -0,0 +1,886 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace b1::chain_kv { + +class exception : public std::exception { + std::string msg; + + public: + exception(std::string&& msg) : msg(std::move(msg)) {} + exception(const exception&) = default; + exception(exception&&) = default; + + exception& operator=(const exception&) = default; + exception& operator=(exception&&) = default; + + const char* what() const noexcept override { return msg.c_str(); } +}; + +inline void check(rocksdb::Status s, const char* prefix) { + if (!s.ok()) + throw exception(prefix + s.ToString()); +} + +using bytes = std::vector; + +inline rocksdb::Slice to_slice(const bytes& v) { return { v.data(), v.size() }; } + +inline bytes to_bytes(const rocksdb::Slice& v) { return { v.data(), v.data() + v.size() }; } + +inline std::shared_ptr to_shared_bytes(const rocksdb::Slice& v) { + return std::make_shared(v.data(), v.data() + v.size()); +} + +// Bypasses fc's vector size limit +template +void pack_bytes(Stream& s, const bytes& b) { + fc::unsigned_int size(b.size()); + if (size.value != b.size()) + throw exception("bytes is too big"); + fc::raw::pack(s, size); + s.write(b.data(), b.size()); +} + +template +void pack_optional_bytes(Stream& s, const bytes* b) { + fc::raw::pack(s, bool(b)); + if (b) + pack_bytes(s, *b); +} + +template +std::pair get_bytes(Stream& s) { + fc::unsigned_int size; + fc::raw::unpack(s, size); + if (size > s.remaining()) + throw exception("bad size for bytes"); + auto data = s.pos(); + s.skip(size.value); + return { data, size.value }; +} + +template +std::pair get_optional_bytes(Stream& s) { + bool present; + fc::raw::unpack(s, present); + if (present) + return get_bytes(s); + else + return { nullptr, 0 }; +} + +template +inline int compare_blob(const A& a, const B& b) { + static_assert(std::is_same_v, char> || + std::is_same_v, unsigned char>); + static_assert(std::is_same_v, char> || + std::is_same_v, unsigned char>); + auto r = memcmp(a.data(), b.data(), std::min(a.size(), b.size())); + if (r) + return r; + if (a.size() < b.size()) + return -1; + if (a.size() > b.size()) + return 1; + return 0; +} + +struct less_blob { + using is_transparent = void; + + template + bool operator()(const A& a, const B& b) const { + return compare_blob(a, b) < 0; + } +}; + +inline bytes get_next_prefix(const bytes& prefix) { + bytes next_prefix = prefix; + while (!next_prefix.empty()) { + if (++next_prefix.back()) + break; + next_prefix.pop_back(); + } + return next_prefix; +} + +template +auto append_key(bytes& dest, T value) -> std::enable_if_t, void> { + char buf[sizeof(value)]; + memcpy(buf, &value, sizeof(value)); + std::reverse(std::begin(buf), std::end(buf)); + dest.insert(dest.end(), std::begin(buf), std::end(buf)); +} + +template +bytes create_full_key(const bytes& prefix, uint64_t contract, const T& key) { + bytes result; + result.reserve(prefix.size() + sizeof(contract) + key.size()); + result.insert(result.end(), prefix.begin(), prefix.end()); + append_key(result, contract); + result.insert(result.end(), key.data(), key.data() + key.size()); + return result; +} + +struct database { + std::unique_ptr rdb; + + database(const char* db_path, bool create_if_missing, std::optional threads = {}, + std::optional max_open_files = {}) { + + rocksdb::Options options; + options.create_if_missing = create_if_missing; + options.level_compaction_dynamic_level_bytes = true; + options.bytes_per_sync = 1048576; + + if (threads) + options.IncreaseParallelism(*threads); + + options.OptimizeLevelStyleCompaction(256ull << 20); + + if (max_open_files) + options.max_open_files = *max_open_files; + + rocksdb::BlockBasedTableOptions table_options; + table_options.format_version = 4; + table_options.index_block_restart_interval = 16; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + rocksdb::DB* p; + check(rocksdb::DB::Open(options, db_path, &p), "database::database: rocksdb::DB::Open: "); + rdb.reset(p); + + // Sentinels with keys 0x00 and 0xff simplify iteration logic. + // Views have prefixes which must start with a byte within the range 0x01 - 0xfe. + rocksdb::WriteBatch batch; + bool modified = false; + auto write_sentinal = [&](const bytes& k) { + rocksdb::PinnableSlice v; + auto stat = rdb->Get(rocksdb::ReadOptions(), rdb->DefaultColumnFamily(), to_slice(k), &v); + if (stat.IsNotFound()) { + check(batch.Put(to_slice(k), {}), "database::database: rocksdb::WriteBatch::Put: "); + modified = true; + } else { + check(stat, "database::database: rocksdb::DB::Get: "); + } + }; + write_sentinal({ 0x00 }); + write_sentinal({ (char)0xff }); + if (modified) + write(batch); + } + + database(database&&) = default; + database& operator=(database&&) = default; + + void flush(bool allow_write_stall, bool wait) { + rocksdb::FlushOptions op; + op.allow_write_stall = allow_write_stall; + op.wait = wait; + rdb->Flush(op); + } + + void write(rocksdb::WriteBatch& batch) { + rocksdb::WriteOptions opt; + opt.disableWAL = true; + check(rdb->Write(opt, &batch), "database::write: rocksdb::DB::Write (batch)"); + batch.Clear(); + } +}; // database + +struct key_value { + rocksdb::Slice key = {}; + rocksdb::Slice value = {}; +}; + +inline int compare_key(const std::optional& a, const std::optional& b) { + // nullopt represents end; everything else is before end + if (!a && !b) + return 0; + else if (!a && b) + return 1; + else if (a && !b) + return -1; + else + return compare_blob(a->key, b->key); +} + +inline int compare_value(const std::shared_ptr& a, const std::shared_ptr& b) { + // nullptr represents erased; everything else, including empty, is after erased + if (!a && !b) + return 0; + else if (!a && b) + return -1; + else if (a && !b) + return 1; + else + return compare_blob(*a, *b); +} + +using cache_map = std::map; + +// The cache serves these needs: +// * Keep track of changes that need to be written to rocksdb +// * Support reading writes +// * Support iteration logic +struct cached_value { + uint64_t num_erases = 0; // For iterator invalidation + std::shared_ptr orig_value = {}; + std::shared_ptr current_value = {}; + bool in_change_list = false; + cache_map::iterator change_list_next = {}; +}; + +struct undo_state { + uint8_t format_version = 0; + int64_t revision = 0; + std::vector undo_stack = {}; // Number of undo segments needed to go back each revision + uint64_t next_undo_segment = 0; +}; + +template +void pack_undo_segment(Stream& s, const bytes& key, const bytes* old_value, const bytes* new_value) { + pack_bytes(s, key); + pack_optional_bytes(s, old_value); + pack_optional_bytes(s, new_value); +} + +class undo_stack { + private: + database& db; + bytes undo_prefix; + uint64_t target_segment_size; + bytes state_prefix; + bytes segment_prefix; + bytes segment_next_prefix; + undo_state state; + + public: + undo_stack(database& db, const bytes& undo_prefix, uint64_t target_segment_size = 64 * 1024 * 1024) + : db{ db }, undo_prefix{ undo_prefix }, target_segment_size{ target_segment_size } { + if (this->undo_prefix.empty()) + throw exception("undo_prefix is empty"); + + // Sentinels reserve 0x00 and 0xff. This keeps rocksdb iterators from going + // invalid during iteration. + if (this->undo_prefix[0] == 0x00 || this->undo_prefix[0] == (char)0xff) + throw exception("undo_stack may not have a prefix which begins with 0x00 or 0xff"); + + state_prefix = this->undo_prefix; + state_prefix.push_back(0x00); + segment_prefix = this->undo_prefix; + segment_prefix.push_back(0x80); + segment_next_prefix = get_next_prefix(segment_prefix); + + rocksdb::PinnableSlice v; + auto stat = db.rdb->Get(rocksdb::ReadOptions(), db.rdb->DefaultColumnFamily(), to_slice(this->state_prefix), &v); + if (!stat.IsNotFound()) + check(stat, "undo_stack::undo_stack: rocksdb::DB::Get: "); + if (stat.ok()) { + auto format_version = fc::raw::unpack(v.data(), v.size()); + if (format_version) + throw exception("invalid undo format"); + state = fc::raw::unpack(v.data(), v.size()); + } + } + + int64_t revision() const { return state.revision; } + int64_t first_revision() const { return state.revision - state.undo_stack.size(); } + + void set_revision(uint64_t revision, bool write_now = true) { + if (state.undo_stack.size() != 0) + throw exception("cannot set revision while there is an existing undo stack"); + if (revision > std::numeric_limits::max()) + throw exception("revision to set is too high"); + if (revision < state.revision) + throw exception("revision cannot decrease"); + state.revision = revision; + if (write_now) + write_state(); + } + + // Create a new entry on the undo stack + void push(bool write_now = true) { + state.undo_stack.push_back(0); + ++state.revision; + if (write_now) + write_state(); + } + + // Combine the top two states on the undo stack + void squash(bool write_now = true) { + if (state.undo_stack.empty()) { + return; + } else if (state.undo_stack.size() == 1) { + rocksdb::WriteBatch batch; + check(batch.DeleteRange(to_slice(create_segment_key(0)), + to_slice(create_segment_key(state.next_undo_segment))), + "undo_stack::squash: rocksdb::WriteBatch::DeleteRange: "); + state.undo_stack.clear(); + --state.revision; + write_state(batch); + db.write(batch); + return; + } + auto n = state.undo_stack.back(); + state.undo_stack.pop_back(); + state.undo_stack.back() += n; + --state.revision; + if (write_now) + write_state(); + } + + // Reset the contents to the state at the top of the undo stack + void undo(bool write_now = true) { + if (state.undo_stack.empty()) + throw exception("nothing to undo"); + rocksdb::WriteBatch batch; + + std::unique_ptr rocks_it{ db.rdb->NewIterator(rocksdb::ReadOptions()) }; + auto first = create_segment_key(state.next_undo_segment - state.undo_stack.back()); + rocks_it->Seek(to_slice(segment_next_prefix)); + if (rocks_it->Valid()) + rocks_it->Prev(); + + while (rocks_it->Valid()) { + auto segment_key = rocks_it->key(); + if (compare_blob(segment_key, first) < 0) + break; + write_now = true; + auto segment = rocks_it->value(); + fc::datastream ds(segment.data(), segment.size()); + while (ds.remaining()) { + auto [key, key_size] = get_bytes(ds); + auto [old_value, old_value_size] = get_optional_bytes(ds); + get_optional_bytes(ds); + if (old_value) + check(batch.Put({ key, key_size }, { old_value, old_value_size }), + "undo_stack::undo: rocksdb::WriteBatch::Put: "); + else + check(batch.Delete({ key, key_size }), "undo_stack::undo: rocksdb::WriteBatch::Delete: "); + } + check(batch.Delete(segment_key), "undo_stack::undo: rocksdb::WriteBatch::Delete: "); + rocks_it->Prev(); + } + check(rocks_it->status(), "undo_stack::undo: iterate rocksdb: "); + + state.next_undo_segment -= state.undo_stack.back(); + state.undo_stack.pop_back(); + --state.revision; + if (write_now) { + write_state(batch); + db.write(batch); + } + } + + // Discard all undo history prior to revision + void commit(int64_t revision) { + revision = std::min(revision, state.revision); + auto first_revision = state.revision - state.undo_stack.size(); + if (first_revision < revision) { + rocksdb::WriteBatch batch; + state.undo_stack.erase(state.undo_stack.begin(), state.undo_stack.begin() + (revision - first_revision)); + uint64_t keep_undo_segment = state.next_undo_segment; + for (auto n : state.undo_stack) // + keep_undo_segment -= n; + check(batch.DeleteRange(to_slice(create_segment_key(0)), to_slice(create_segment_key(keep_undo_segment))), + "undo_stack::commit: rocksdb::WriteBatch::DeleteRange: "); + write_state(batch); + db.write(batch); + } + } + + // Write changes in `change_list`. Everything in `change_list` must belong to `cache`. + // + // It is undefined behavior if any `orig_value` in `change_list` doesn't match the + // database's current state. + void write_changes(cache_map& cache, cache_map::iterator change_list) { + rocksdb::WriteBatch batch; + bytes segment; + segment.reserve(target_segment_size); + + auto write_segment = [&] { + if (segment.empty()) + return; + auto key = create_segment_key(state.next_undo_segment++); + check(batch.Put(to_slice(key), to_slice(segment)), "undo_stack::write_changes: rocksdb::WriteBatch::Put: "); + ++state.undo_stack.back(); + segment.clear(); + }; + + auto append_segment = [&](auto f) { + fc::datastream size_stream; + f(size_stream); + if (segment.size() + size_stream.tellp() > target_segment_size) + write_segment(); + auto orig_size = segment.size(); + segment.resize(segment.size() + size_stream.tellp()); + fc::datastream ds(segment.data() + orig_size, size_stream.tellp()); + f(ds); + }; + + auto it = change_list; + while (it != cache.end()) { + if (compare_value(it->second.orig_value, it->second.current_value)) { + if (it->second.current_value) + check(batch.Put(to_slice(it->first), to_slice(*it->second.current_value)), + "undo_stack::write_changes: rocksdb::WriteBatch::Put: "); + else + check(batch.Delete(to_slice(it->first)), "undo_stack::write_changes: rocksdb::WriteBatch::Erase: "); + if (!state.undo_stack.empty()) { + append_segment([&](auto& stream) { + pack_undo_segment(stream, it->first, it->second.orig_value.get(), it->second.current_value.get()); + }); + } + } + it = it->second.change_list_next; + } + + write_segment(); + write_state(batch); + db.write(batch); + } // write_changes() + + void write_state() { + rocksdb::WriteBatch batch; + write_state(batch); + db.write(batch); + } + + private: + void write_state(rocksdb::WriteBatch& batch) { + check(batch.Put(to_slice(state_prefix), to_slice(fc::raw::pack(state))), + "undo_stack::write_state: rocksdb::WriteBatch::Put: "); + } + + bytes create_segment_key(uint64_t segment) { + bytes key; + key.reserve(segment_prefix.size() + sizeof(segment)); + key.insert(key.end(), segment_prefix.begin(), segment_prefix.end()); + append_key(key, segment); + return key; + } +}; // undo_stack + +// Supports reading and writing through a cache_map +// +// Caution: write_session will misbehave if it's used to read or write a key that +// that is changed elsewhere during write_session's lifetime. e.g. through another +// simultaneous write_session, unless that write_session is never written to the +// database. +// +// Extra keys stored in the cache used only as sentinels are exempt from this +// restriction. +struct write_session { + database& db; + const rocksdb::Snapshot* snapshot; + cache_map cache; + cache_map::iterator change_list = cache.end(); + + write_session(database& db, const rocksdb::Snapshot* snapshot = nullptr) : db{ db }, snapshot{ snapshot } {} + + rocksdb::ReadOptions read_options() { + rocksdb::ReadOptions r; + r.snapshot = snapshot; + return r; + } + + // Add item to change_list + void changed(cache_map::iterator it) { + if (it->second.in_change_list) + return; + it->second.in_change_list = true; + it->second.change_list_next = change_list; + change_list = it; + } + + // Get a value. Includes any changes written to cache. Returns nullptr + // if key-value doesn't exist. + std::shared_ptr get(bytes&& k) { + auto it = cache.find(k); + if (it != cache.end()) + return it->second.current_value; + + rocksdb::PinnableSlice v; + auto stat = db.rdb->Get(read_options(), db.rdb->DefaultColumnFamily(), to_slice(k), &v); + if (stat.IsNotFound()) + return nullptr; + check(stat, "write_session::get: rocksdb::DB::Get: "); + + auto value = to_shared_bytes(v); + cache[std::move(k)] = cached_value{ 0, value, value }; + return value; + } + + // Write a key-value to cache and add to change_list if changed. + void set(bytes&& k, const rocksdb::Slice& v) { + auto it = cache.find(k); + if (it != cache.end()) { + if (!it->second.current_value || compare_blob(*it->second.current_value, v)) { + it->second.current_value = to_shared_bytes(v); + changed(it); + } + return; + } + + rocksdb::PinnableSlice orig_v; + auto stat = db.rdb->Get(read_options(), db.rdb->DefaultColumnFamily(), to_slice(k), &orig_v); + if (stat.IsNotFound()) { + auto [it, b] = + cache.insert(cache_map::value_type{ std::move(k), cached_value{ 0, nullptr, to_shared_bytes(v) } }); + changed(it); + return; + } + + check(stat, "write_session::set: rocksdb::DB::Get: "); + if (compare_blob(v, orig_v)) { + auto [it, b] = cache.insert( + cache_map::value_type{ std::move(k), cached_value{ 0, to_shared_bytes(orig_v), to_shared_bytes(v) } }); + changed(it); + } else { + auto value = to_shared_bytes(orig_v); + cache[std::move(k)] = cached_value{ 0, value, value }; + } + } + + // Mark key as erased in the cache and add to change_list if changed. Bumps `num_erases` to invalidate iterators. + void erase(bytes&& k) { + { + auto it = cache.find(k); + if (it != cache.end()) { + if (it->second.current_value) { + ++it->second.num_erases; + it->second.current_value = nullptr; + changed(it); + } + return; + } + } + + rocksdb::PinnableSlice orig_v; + auto stat = db.rdb->Get(read_options(), db.rdb->DefaultColumnFamily(), to_slice(k), &orig_v); + if (stat.IsNotFound()) { + cache[std::move(k)] = cached_value{ 0, nullptr, nullptr }; + return; + } + + check(stat, "write_session::erase: rocksdb::DB::Get: "); + auto [it, b] = + cache.insert(cache_map::value_type{ std::move(k), cached_value{ 1, to_shared_bytes(orig_v), nullptr } }); + changed(it); + } + + // Fill cache with a key-value pair read from the database. Does not undo any changes (e.g. set() or erase()) + // already made to the cache. Returns an iterator to the freshly-created or already-existing cache entry. + cache_map::iterator fill_cache(const rocksdb::Slice& k, const rocksdb::Slice& v) { + cache_map::iterator it; + bool b; + it = cache.find(k); + if (it != cache.end()) + return it; + auto value = to_shared_bytes(v); + std::tie(it, b) = cache.insert(cache_map::value_type{ to_bytes(k), cached_value{ 0, value, value } }); + return it; + } + + // Write changes in `change_list` to database. See undo_stack::write_changes. + // + // Caution: write_changes wipes the cache, which invalidates iterators + void write_changes(undo_stack& u) { + u.write_changes(cache, change_list); + wipe_cache(); + } + + // Wipe the cache. Invalidates iterators. + void wipe_cache() { + cache.clear(); + change_list = cache.end(); + } +}; // write_session + +// A view of the database with a restricted range (prefix). Implements part of +// https://github.com/EOSIO/spec-repo/blob/master/esr_key_value_database_intrinsics.md, +// including iterator wrap-around behavior. +// +// Keys have this format: prefix, contract, user-provided key. +class view { + public: + class iterator; + + chain_kv::write_session& write_session; + const bytes prefix; + + private: + struct iterator_impl { + friend chain_kv::view; + friend iterator; + + chain_kv::view& view; + bytes prefix; + size_t hidden_prefix_size; + bytes next_prefix; + cache_map::iterator cache_it; + uint64_t cache_it_num_erases = 0; + std::unique_ptr rocks_it; + + iterator_impl(chain_kv::view& view, uint64_t contract, const rocksdb::Slice& prefix) + : view{ view }, // + prefix{ create_full_key(view.prefix, contract, prefix) }, // + hidden_prefix_size{ view.prefix.size() + sizeof(contract) }, // + rocks_it{ view.write_session.db.rdb->NewIterator(view.write_session.read_options()) } // + { + next_prefix = get_next_prefix(this->prefix); + + // Fill the cache with sentinel keys to simplify iteration logic. These may be either + // the reserved 0x00 or 0xff sentinels, or keys from regions neighboring prefix. + rocks_it->Seek(to_slice(this->prefix)); + check(rocks_it->status(), "view::iterator_impl::iterator_impl: rocksdb::Iterator::Seek: "); + view.write_session.fill_cache(rocks_it->key(), rocks_it->value()); + rocks_it->Prev(); + check(rocks_it->status(), "view::iterator_impl::iterator_impl: rocksdb::Iterator::Prev: "); + view.write_session.fill_cache(rocks_it->key(), rocks_it->value()); + rocks_it->Seek(to_slice(next_prefix)); + check(rocks_it->status(), "view::iterator_impl::iterator_impl: rocksdb::Iterator::Seek: "); + view.write_session.fill_cache(rocks_it->key(), rocks_it->value()); + + move_to_end(); + } + + iterator_impl(const iterator_impl&) = delete; + iterator_impl& operator=(const iterator_impl&) = delete; + + void move_to_begin() { lower_bound_full_key(prefix); } + + void move_to_end() { cache_it = view.write_session.cache.end(); } + + void lower_bound(const char* key, size_t size) { + auto x = compare_blob(rocksdb::Slice{ key, size }, rocksdb::Slice{ prefix.data() + hidden_prefix_size, + prefix.size() - hidden_prefix_size }); + if (x < 0) { + key = prefix.data() + hidden_prefix_size; + size = prefix.size() - hidden_prefix_size; + } + + bytes full_key; + full_key.reserve(hidden_prefix_size + size); + full_key.insert(full_key.end(), prefix.data(), prefix.data() + hidden_prefix_size); + full_key.insert(full_key.end(), key, key + size); + lower_bound_full_key(full_key); + } + + void lower_bound_full_key(const bytes& full_key) { + rocks_it->Seek(to_slice(full_key)); + check(rocks_it->status(), "view::iterator_impl::lower_bound_full_key: rocksdb::Iterator::Seek: "); + cache_it = view.write_session.fill_cache(rocks_it->key(), rocks_it->value()); + if (compare_blob(cache_it->first, to_slice(full_key))) + cache_it = view.write_session.cache.lower_bound(full_key); + while (!cache_it->second.current_value) { + while (compare_blob(rocks_it->key(), cache_it->first) <= 0) { + rocks_it->Next(); + check(rocks_it->status(), "view::iterator_impl::lower_bound_full_key: rocksdb::Iterator::Next: "); + view.write_session.fill_cache(rocks_it->key(), rocks_it->value()); + } + ++cache_it; + } + if (compare_blob(cache_it->first, next_prefix) >= 0) + cache_it = view.write_session.cache.end(); + else + cache_it_num_erases = cache_it->second.num_erases; + } + + std::optional get_kv() { + if (cache_it == view.write_session.cache.end()) + return {}; + if (cache_it_num_erases != cache_it->second.num_erases) + throw exception("kv iterator is at an erased value"); + return key_value{ rocksdb::Slice{ cache_it->first.data() + hidden_prefix_size, + cache_it->first.size() - hidden_prefix_size }, + to_slice(*cache_it->second.current_value) }; + } + + bool is_end() { return cache_it == view.write_session.cache.end(); } + + bool is_valid() { + return cache_it != view.write_session.cache.end() && cache_it_num_erases == cache_it->second.num_erases; + } + + bool is_erased() { + return cache_it != view.write_session.cache.end() && cache_it_num_erases != cache_it->second.num_erases; + } + + iterator_impl& operator++() { + if (cache_it == view.write_session.cache.end()) { + move_to_begin(); + return *this; + } else if (cache_it_num_erases != cache_it->second.num_erases) + throw exception("kv iterator is at an erased value"); + do { + while (compare_blob(rocks_it->key(), cache_it->first) <= 0) { + rocks_it->Next(); + check(rocks_it->status(), "view::iterator_impl::operator++: rocksdb::Iterator::Next: "); + view.write_session.fill_cache(rocks_it->key(), rocks_it->value()); + } + ++cache_it; + } while (!cache_it->second.current_value); + if (compare_blob(cache_it->first, next_prefix) >= 0) + cache_it = view.write_session.cache.end(); + else + cache_it_num_erases = cache_it->second.num_erases; + return *this; + } + + iterator_impl& operator--() { + if (cache_it == view.write_session.cache.end()) { + rocks_it->Seek(to_slice(next_prefix)); + check(rocks_it->status(), "view::iterator_impl::operator--: rocksdb::Iterator::Seek: "); + cache_it = view.write_session.fill_cache(rocks_it->key(), rocks_it->value()); + if (compare_blob(cache_it->first, to_slice(next_prefix))) + cache_it = view.write_session.cache.lower_bound(next_prefix); + } else if (cache_it_num_erases != cache_it->second.num_erases) + throw exception("kv iterator is at an erased value"); + do { + while (compare_blob(rocks_it->key(), cache_it->first) >= 0) { + rocks_it->Prev(); + check(rocks_it->status(), "view::iterator_impl::operator--: rocksdb::Iterator::Prev: "); + view.write_session.fill_cache(rocks_it->key(), rocks_it->value()); + } + --cache_it; + } while (!cache_it->second.current_value); + if (compare_blob(cache_it->first, prefix) < 0) + cache_it = view.write_session.cache.end(); + else + cache_it_num_erases = cache_it->second.num_erases; + return *this; + } + }; // iterator_impl + + public: + // Iterates through a user-provided prefix within view's keyspace. + class iterator { + friend view; + + private: + std::unique_ptr impl; + + void check_initialized() const { + if (!impl) + throw exception("kv iterator is not initialized"); + } + + public: + iterator(view& view, uint64_t contract, const rocksdb::Slice& prefix) + : impl{ std::make_unique(view, contract, std::move(prefix)) } {} + + iterator(const iterator&) = delete; + iterator(iterator&&) = default; + + iterator& operator=(const iterator&) = delete; + iterator& operator=(iterator&&) = default; + + // Compare 2 iterators. Throws if the iterators are from different views. + // Also throws if either iterator is at an erased kv pair. non-end iterators + // compare less than end iterators. + friend int compare(const iterator& a, const iterator& b) { + a.check_initialized(); + b.check_initialized(); + if (&a.impl->view != &b.impl->view) + throw exception("iterators are from different views"); + return compare_key(a.impl->get_kv(), b.impl->get_kv()); + } + + friend bool operator==(const iterator& a, const iterator& b) { return compare(a, b) == 0; } + friend bool operator!=(const iterator& a, const iterator& b) { return compare(a, b) != 0; } + friend bool operator<(const iterator& a, const iterator& b) { return compare(a, b) < 0; } + friend bool operator<=(const iterator& a, const iterator& b) { return compare(a, b) <= 0; } + friend bool operator>(const iterator& a, const iterator& b) { return compare(a, b) > 0; } + friend bool operator>=(const iterator& a, const iterator& b) { return compare(a, b) >= 0; } + + iterator& operator++() { + check_initialized(); + ++*impl; + return *this; + } + + iterator& operator--() { + check_initialized(); + --*impl; + return *this; + } + + void move_to_begin() { + check_initialized(); + impl->move_to_begin(); + } + + void move_to_end() { + check_initialized(); + impl->move_to_end(); + } + + void lower_bound(const char* key, size_t size) { + check_initialized(); + impl->lower_bound(key, size); + } + + void lower_bound(const bytes& key) { lower_bound(key.data(), key.size()); } + + bool is_end() const { + check_initialized(); + return impl->is_end(); + } + + // true if !is_end() and not at an erased kv pair + bool is_valid() const { + check_initialized(); + return impl->is_valid(); + } + + // true if !is_end() and is at an erased kv pair + bool is_erased() const { + check_initialized(); + return impl->is_erased(); + } + + // Get key_value at current position. Returns nullopt if at end. Throws if at an erased kv pair. + // The returned key does not include the view's prefix or the contract. + std::optional get_kv() const { + check_initialized(); + return impl->get_kv(); + } + }; + + view(struct write_session& write_session, bytes prefix) + : write_session{ write_session }, prefix{ std::move(prefix) } { + if (this->prefix.empty()) + throw exception("kv view may not have empty prefix"); + + // Sentinels reserve 0x00 and 0xff. This keeps rocksdb iterators from going + // invalid during iteration. This also allows get_next_prefix() to function correctly. + if (this->prefix[0] == 0x00 || this->prefix[0] == (char)0xff) + throw exception("view may not have a prefix which begins with 0x00 or 0xff"); + } + + // Get a value. Includes any changes written to cache. Returns nullptr + // if key doesn't exist. + std::shared_ptr get(uint64_t contract, const rocksdb::Slice& k) { + return write_session.get(create_full_key(prefix, contract, k)); + } + + // Set a key-value pair + void set(uint64_t contract, const rocksdb::Slice& k, const rocksdb::Slice& v) { + write_session.set(create_full_key(prefix, contract, k), v); + } + + // Erase a key-value pair + void erase(uint64_t contract, const rocksdb::Slice& k) { write_session.erase(create_full_key(prefix, contract, k)); } +}; // view + +} // namespace b1::chain_kv + +FC_REFLECT(b1::chain_kv::undo_state, (format_version)(revision)(undo_stack)(next_undo_segment)) diff --git a/libraries/chain-kv/unit_tests/CMakeLists.txt b/libraries/chain-kv/unit_tests/CMakeLists.txt new file mode 100644 index 00000000000..ecef2caf378 --- /dev/null +++ b/libraries/chain-kv/unit_tests/CMakeLists.txt @@ -0,0 +1,23 @@ +include(ExternalProject) + +file(GLOB UNIT_TESTS "*.cpp") # find all unit test suites + +add_executable(chain-kv-tests ${UNIT_TESTS}) +target_link_libraries(chain-kv-tests chain-kv) + +enable_testing() + +### MARK TEST SUITES FOR EXECUTION ### +foreach(TEST_SUITE ${UNIT_TESTS}) # create an independent target for each test suite + execute_process(COMMAND bash -c "grep -E 'BOOST_AUTO_TEST_SUITE\\s*[(]' ${TEST_SUITE} | grep -vE '//.*BOOST_AUTO_TEST_SUITE\\s*[(]' | cut -d ')' -f 1 | cut -d '(' -f 2" OUTPUT_VARIABLE SUITE_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) # get the test suite name from the *.cpp file + if (NOT "" STREQUAL "${SUITE_NAME}") # ignore empty lines + execute_process(COMMAND bash -c "echo ${SUITE_NAME} | sed -e 's/s$//' | sed -e 's/_test$//'" OUTPUT_VARIABLE TRIMMED_SUITE_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) # trim "_test" or "_tests" from the end of ${SUITE_NAME} + add_test(NAME ${TRIMMED_SUITE_NAME}_unit_test COMMAND chain-kv-tests --run_test=${SUITE_NAME} --report_level=detailed --color_output --catch_system_errors=no) + # build list of tests to run during coverage testing + if(ctest_tests) + string(APPEND ctest_tests "|") + endif() + string(APPEND ctest_tests ${TRIMMED_SUITE_NAME}_unit_test) + endif() +endforeach(TEST_SUITE) +set(ctest_tests "'${ctest_tests}' -j8") diff --git a/libraries/chain-kv/unit_tests/chain_kv_tests.hpp b/libraries/chain-kv/unit_tests/chain_kv_tests.hpp new file mode 100644 index 00000000000..ed0cb8f2c37 --- /dev/null +++ b/libraries/chain-kv/unit_tests/chain_kv_tests.hpp @@ -0,0 +1,114 @@ +#include +#include + +namespace chain_kv = b1::chain_kv; + +struct kv_values { + std::vector> values; + + friend bool operator==(const kv_values& a, const kv_values& b) { return a.values == b.values; } +}; + +template +S& operator<<(S& s, const kv_values& v) { + s << "{\n"; + for (auto& kv : v.values) { + s << " {{"; + bool first = true; + for (auto b : kv.first) { + if (!first) + s << ", "; + first = false; + char x[10]; + sprintf(x, "0x%02x", (uint8_t)b); + s << x; + } + s << "}, {"; + first = true; + for (auto b : kv.second) { + if (!first) + s << ", "; + first = false; + char x[10]; + sprintf(x, "0x%02x", (uint8_t)b); + s << x; + } + s << "}},\n"; + } + s << "}"; + return s; +} + +#define KV_REQUIRE_EXCEPTION(expr, msg) \ + BOOST_REQUIRE_EXCEPTION(expr, chain_kv::exception, [](auto& e) { \ + if (!strcmp(e.what(), msg)) \ + return true; \ + std::cerr << "expected: '" \ + << "\033[33m" << msg << "\033[0m'\ngot: '\033[33m" << e.what() << "'\033[0m\n"; \ + return false; \ + }); + +inline kv_values get_all(chain_kv::database& db, const chain_kv::bytes& prefix) { + kv_values result; + std::unique_ptr rocks_it{ db.rdb->NewIterator(rocksdb::ReadOptions()) }; + rocks_it->Seek(chain_kv::to_slice(prefix)); + while (rocks_it->Valid()) { + auto k = rocks_it->key(); + if (k.size() < prefix.size() || memcmp(k.data(), prefix.data(), prefix.size())) + break; + auto v = rocks_it->value(); + result.values.push_back({ chain_kv::to_bytes(k), chain_kv::to_bytes(v) }); + rocks_it->Next(); + } + if (!rocks_it->status().IsNotFound()) + chain_kv::check(rocks_it->status(), "iterate: "); + return result; +} + +inline kv_values get_values(chain_kv::write_session& session, const std::vector& keys) { + kv_values result; + for (auto& key : keys) { + chain_kv::bytes value; + if (auto value = session.get(chain_kv::bytes{ key })) + result.values.push_back({ key, *value }); + } + return result; +} + +inline kv_values get_matching(chain_kv::view& view, uint64_t contract, const chain_kv::bytes& prefix = {}) { + kv_values result; + chain_kv::view::iterator it{ view, contract, chain_kv::to_slice(prefix) }; + ++it; + while (!it.is_end()) { + auto kv = it.get_kv(); + if (!kv) + throw chain_kv::exception("iterator read failure"); + result.values.push_back({ chain_kv::to_bytes(kv->key), chain_kv::to_bytes(kv->value) }); + ++it; + } + return result; +} + +inline kv_values get_matching2(chain_kv::view& view, uint64_t contract, const chain_kv::bytes& prefix = {}) { + kv_values result; + chain_kv::view::iterator it{ view, contract, chain_kv::to_slice(prefix) }; + --it; + while (!it.is_end()) { + auto kv = it.get_kv(); + if (!kv) + throw chain_kv::exception("iterator read failure"); + result.values.push_back({ chain_kv::to_bytes(kv->key), chain_kv::to_bytes(kv->value) }); + --it; + } + std::reverse(result.values.begin(), result.values.end()); + return result; +} + +inline kv_values get_it(const chain_kv::view::iterator& it) { + kv_values result; + auto kv = it.get_kv(); + if (!kv) + throw chain_kv::exception("iterator read failure"); + result.values.push_back({ chain_kv::to_bytes(kv->key), chain_kv::to_bytes(kv->value) }); + return result; +} diff --git a/libraries/chain-kv/unit_tests/main.cpp b/libraries/chain-kv/unit_tests/main.cpp new file mode 100644 index 00000000000..99a2beae010 --- /dev/null +++ b/libraries/chain-kv/unit_tests/main.cpp @@ -0,0 +1,36 @@ +#include +#include +#include +#include +#include + +void translate_fc_exception(const fc::exception& e) { + std::cerr << "\033[33m" << e.to_detail_string() << "\033[0m" << std::endl; + BOOST_TEST_FAIL("Caught Unexpected Exception"); +} + +boost::unit_test::test_suite* init_unit_test_suite(int argc, char* argv[]) { + // Turn off logging if no --verbose parameter is not added + // To have verbose enabled, call "unit_tests/unit_test -- --verbose" + bool is_verbose = false; + std::string verbose_arg = "--verbose"; + for (int i = 0; i < argc; i++) { + if (verbose_arg == argv[i]) { + is_verbose = true; + break; + } + } + if (is_verbose) { + fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug); + } else { + fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::off); + } + + // Register fc::exception translator + boost::unit_test::unit_test_monitor.register_exception_translator(&translate_fc_exception); + + std::srand(time(NULL)); + std::cout << "Random number generator seeded to " << time(NULL) << std::endl; + + return nullptr; +} diff --git a/libraries/chain-kv/unit_tests/undo_stack_tests.cpp b/libraries/chain-kv/unit_tests/undo_stack_tests.cpp new file mode 100644 index 00000000000..8cc1e81639a --- /dev/null +++ b/libraries/chain-kv/unit_tests/undo_stack_tests.cpp @@ -0,0 +1,457 @@ +#include "chain_kv_tests.hpp" +#include + +using chain_kv::bytes; +using chain_kv::to_slice; + +BOOST_AUTO_TEST_SUITE(undo_stack_tests) + +void undo_tests(bool reload_undo, uint64_t target_segment_size) { + boost::filesystem::remove_all("test-undo-db"); + chain_kv::database db{ "test-undo-db", true }; + std::unique_ptr undo_stack; + + auto reload = [&] { + if (!undo_stack || reload_undo) + undo_stack = std::make_unique(db, bytes{ 0x10 }, target_segment_size); + }; + reload(); + + KV_REQUIRE_EXCEPTION(undo_stack->undo(), "nothing to undo"); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 0); + { + chain_kv::write_session session{ db }; + session.set({ 0x20, 0x00 }, to_slice({})); + session.set({ 0x20, 0x02 }, to_slice({ 0x50 })); + session.set({ 0x20, 0x01 }, to_slice({ 0x40 })); + session.erase({ 0x20, 0x02 }); + session.set({ 0x20, 0x03 }, to_slice({ 0x60 })); + session.set({ 0x20, 0x01 }, to_slice({ 0x50 })); + session.write_changes(*undo_stack); + } + KV_REQUIRE_EXCEPTION(undo_stack->undo(), "nothing to undo"); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 0); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x10, (char)0x80 }), (kv_values{})); // no undo segments + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x00 }, {} }, + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x03 }, { 0x60 } }, + } })); + + reload(); + undo_stack->push(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 1); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 1); + { + chain_kv::write_session session{ db }; + session.erase({ 0x20, 0x01 }); + session.set({ 0x20, 0x00 }, to_slice({ 0x70 })); + session.write_changes(*undo_stack); + } + BOOST_REQUIRE_NE(get_all(db, { 0x10, (char)0x80 }), (kv_values{})); // has undo segments + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x00 }, { 0x70 } }, + { { 0x20, 0x03 }, { 0x60 } }, + } })); + + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 1); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 1); + KV_REQUIRE_EXCEPTION(undo_stack->set_revision(2), "cannot set revision while there is an existing undo stack"); + undo_stack->undo(); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x10, (char)0x80 }), (kv_values{})); // no undo segments + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 0); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 0); + undo_stack->set_revision(10); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 10); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 10); + + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x00 }, {} }, + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x03 }, { 0x60 } }, + } })); + + { + chain_kv::write_session session{ db }; + session.erase({ 0x20, 0x01 }); + session.set({ 0x20, 0x00 }, to_slice({ 0x70 })); + session.write_changes(*undo_stack); + } + BOOST_REQUIRE_EQUAL(get_all(db, { 0x10, (char)0x80 }), (kv_values{})); // no undo segments + reload(); + undo_stack->push(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 11); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 11); + KV_REQUIRE_EXCEPTION(undo_stack->set_revision(12), "cannot set revision while there is an existing undo stack"); + KV_REQUIRE_EXCEPTION(undo_stack->squash(), "nothing to squash"); + undo_stack->commit(0); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 11); + KV_REQUIRE_EXCEPTION(undo_stack->set_revision(12), "cannot set revision while there is an existing undo stack"); + KV_REQUIRE_EXCEPTION(undo_stack->squash(), "nothing to squash"); + undo_stack->commit(11); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 11); + reload(); + KV_REQUIRE_EXCEPTION(undo_stack->set_revision(9), "revision cannot decrease"); + undo_stack->set_revision(12); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 12); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 12); + + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x00 }, { 0x70 } }, + { { 0x20, 0x03 }, { 0x60 } }, + } })); + +} // undo_tests() + +void squash_tests(bool reload_undo, uint64_t target_segment_size) { + boost::filesystem::remove_all("test-squash-db"); + chain_kv::database db{ "test-squash-db", true }; + std::unique_ptr undo_stack; + + auto reload = [&] { + if (!undo_stack || reload_undo) + undo_stack = std::make_unique(db, bytes{ 0x10 }, target_segment_size); + }; + reload(); + + // set 1 + undo_stack->push(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 1); + { + chain_kv::write_session session{ db }; + session.set({ 0x20, 0x01 }, to_slice({ 0x50 })); + session.set({ 0x20, 0x02 }, to_slice({ 0x60 })); + session.write_changes(*undo_stack); + } + reload(); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x60 } }, + } })); + + // set 2 + undo_stack->push(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 2); + { + chain_kv::write_session session{ db }; + session.erase({ 0x20, 0x01 }); + session.set({ 0x20, 0x02 }, to_slice({ 0x61 })); + session.set({ 0x20, 0x03 }, to_slice({ 0x70 })); + session.set({ 0x20, 0x04 }, to_slice({ 0x10 })); + session.write_changes(*undo_stack); + } + reload(); + undo_stack->push(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 3); + { + chain_kv::write_session session{ db }; + session.set({ 0x20, 0x01 }, to_slice({ 0x50 })); + session.set({ 0x20, 0x02 }, to_slice({ 0x62 })); + session.erase({ 0x20, 0x03 }); + session.set({ 0x20, 0x05 }, to_slice({ 0x05 })); + session.set({ 0x20, 0x06 }, to_slice({ 0x06 })); + session.write_changes(*undo_stack); + } + reload(); + undo_stack->squash(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 2); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x62 } }, + { { 0x20, 0x04 }, { 0x10 } }, + { { 0x20, 0x05 }, { 0x05 } }, + { { 0x20, 0x06 }, { 0x06 } }, + } })); + + // set 3 + undo_stack->push(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 3); + { + chain_kv::write_session session{ db }; + session.set({ 0x20, 0x07 }, to_slice({ 0x07 })); + session.set({ 0x20, 0x08 }, to_slice({ 0x08 })); + session.write_changes(*undo_stack); + } + reload(); + undo_stack->push(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 4); + { + chain_kv::write_session session{ db }; + session.set({ 0x20, 0x09 }, to_slice({ 0x09 })); + session.set({ 0x20, 0x0a }, to_slice({ 0x0a })); + session.write_changes(*undo_stack); + } + reload(); + undo_stack->push(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 5); + { + chain_kv::write_session session{ db }; + session.set({ 0x20, 0x0b }, to_slice({ 0x0b })); + session.set({ 0x20, 0x0c }, to_slice({ 0x0c })); + session.write_changes(*undo_stack); + } + reload(); + undo_stack->squash(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 4); + undo_stack->squash(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 3); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x62 } }, + { { 0x20, 0x04 }, { 0x10 } }, + { { 0x20, 0x05 }, { 0x05 } }, + { { 0x20, 0x06 }, { 0x06 } }, + { { 0x20, 0x07 }, { 0x07 } }, + { { 0x20, 0x08 }, { 0x08 } }, + { { 0x20, 0x09 }, { 0x09 } }, + { { 0x20, 0x0a }, { 0x0a } }, + { { 0x20, 0x0b }, { 0x0b } }, + { { 0x20, 0x0c }, { 0x0c } }, + } })); + + // undo set 3 + undo_stack->undo(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 2); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x62 } }, + { { 0x20, 0x04 }, { 0x10 } }, + { { 0x20, 0x05 }, { 0x05 } }, + { { 0x20, 0x06 }, { 0x06 } }, + } })); + + // undo set 2 + undo_stack->undo(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 1); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x60 } }, + } })); + + // undo set 1 + undo_stack->undo(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 0); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ {} })); +} // squash_tests() + +void commit_tests(bool reload_undo, uint64_t target_segment_size) { + boost::filesystem::remove_all("test-commit-db"); + chain_kv::database db{ "test-commit-db", true }; + std::unique_ptr undo_stack; + + auto reload = [&] { + if (!undo_stack || reload_undo) + undo_stack = std::make_unique(db, bytes{ 0x10 }, target_segment_size); + }; + reload(); + + // revision 1 + undo_stack->push(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 1); + { + chain_kv::write_session session{ db }; + session.set({ 0x20, 0x01 }, to_slice({ 0x50 })); + session.set({ 0x20, 0x02 }, to_slice({ 0x60 })); + session.write_changes(*undo_stack); + } + reload(); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x60 } }, + } })); + + // revision 2 + undo_stack->push(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 2); + { + chain_kv::write_session session{ db }; + session.erase({ 0x20, 0x01 }); + session.set({ 0x20, 0x02 }, to_slice({ 0x61 })); + session.set({ 0x20, 0x03 }, to_slice({ 0x70 })); + session.set({ 0x20, 0x04 }, to_slice({ 0x10 })); + session.set({ 0x20, 0x01 }, to_slice({ 0x50 })); + session.set({ 0x20, 0x02 }, to_slice({ 0x62 })); + session.erase({ 0x20, 0x03 }); + session.set({ 0x20, 0x05 }, to_slice({ 0x05 })); + session.set({ 0x20, 0x06 }, to_slice({ 0x06 })); + session.write_changes(*undo_stack); + } + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 2); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x62 } }, + { { 0x20, 0x04 }, { 0x10 } }, + { { 0x20, 0x05 }, { 0x05 } }, + { { 0x20, 0x06 }, { 0x06 } }, + } })); + + // revision 3 + undo_stack->push(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 3); + { + chain_kv::write_session session{ db }; + session.set({ 0x20, 0x07 }, to_slice({ 0x07 })); + session.set({ 0x20, 0x08 }, to_slice({ 0x08 })); + session.set({ 0x20, 0x09 }, to_slice({ 0x09 })); + session.set({ 0x20, 0x0a }, to_slice({ 0x0a })); + session.set({ 0x20, 0x0b }, to_slice({ 0x0b })); + session.set({ 0x20, 0x0c }, to_slice({ 0x0c })); + session.write_changes(*undo_stack); + } + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 3); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x62 } }, + { { 0x20, 0x04 }, { 0x10 } }, + { { 0x20, 0x05 }, { 0x05 } }, + { { 0x20, 0x06 }, { 0x06 } }, + { { 0x20, 0x07 }, { 0x07 } }, + { { 0x20, 0x08 }, { 0x08 } }, + { { 0x20, 0x09 }, { 0x09 } }, + { { 0x20, 0x0a }, { 0x0a } }, + { { 0x20, 0x0b }, { 0x0b } }, + { { 0x20, 0x0c }, { 0x0c } }, + } })); + + // commit revision 1 + undo_stack->commit(1); + + // undo revision 3 + undo_stack->undo(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 2); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x62 } }, + { { 0x20, 0x04 }, { 0x10 } }, + { { 0x20, 0x05 }, { 0x05 } }, + { { 0x20, 0x06 }, { 0x06 } }, + } })); + + // undo revision 2 + undo_stack->undo(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 1); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x60 } }, + } })); + + // Can't undo revision 1 + KV_REQUIRE_EXCEPTION(undo_stack->undo(), "nothing to undo"); + + BOOST_REQUIRE_EQUAL(get_all(db, { 0x10, (char)0x80 }), (kv_values{})); // no undo segments + + // revision 2 + undo_stack->push(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 2); + { + chain_kv::write_session session{ db }; + session.erase({ 0x20, 0x01 }); + session.set({ 0x20, 0x02 }, to_slice({ 0x61 })); + session.set({ 0x20, 0x03 }, to_slice({ 0x70 })); + session.set({ 0x20, 0x04 }, to_slice({ 0x10 })); + session.set({ 0x20, 0x01 }, to_slice({ 0x50 })); + session.set({ 0x20, 0x02 }, to_slice({ 0x62 })); + session.erase({ 0x20, 0x03 }); + session.set({ 0x20, 0x05 }, to_slice({ 0x05 })); + session.set({ 0x20, 0x06 }, to_slice({ 0x06 })); + session.write_changes(*undo_stack); + } + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 2); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x62 } }, + { { 0x20, 0x04 }, { 0x10 } }, + { { 0x20, 0x05 }, { 0x05 } }, + { { 0x20, 0x06 }, { 0x06 } }, + } })); + + // revision 3 + undo_stack->push(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 3); + { + chain_kv::write_session session{ db }; + session.set({ 0x20, 0x07 }, to_slice({ 0x07 })); + session.set({ 0x20, 0x08 }, to_slice({ 0x08 })); + session.set({ 0x20, 0x09 }, to_slice({ 0x09 })); + session.set({ 0x20, 0x0a }, to_slice({ 0x0a })); + session.set({ 0x20, 0x0b }, to_slice({ 0x0b })); + session.set({ 0x20, 0x0c }, to_slice({ 0x0c })); + session.write_changes(*undo_stack); + } + reload(); + + // commit revision 3 + undo_stack->commit(3); + + // Can't undo + KV_REQUIRE_EXCEPTION(undo_stack->undo(), "nothing to undo"); + + BOOST_REQUIRE_EQUAL(get_all(db, { 0x10, (char)0x80 }), (kv_values{})); // no undo segments + + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 3); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x62 } }, + { { 0x20, 0x04 }, { 0x10 } }, + { { 0x20, 0x05 }, { 0x05 } }, + { { 0x20, 0x06 }, { 0x06 } }, + { { 0x20, 0x07 }, { 0x07 } }, + { { 0x20, 0x08 }, { 0x08 } }, + { { 0x20, 0x09 }, { 0x09 } }, + { { 0x20, 0x0a }, { 0x0a } }, + { { 0x20, 0x0b }, { 0x0b } }, + { { 0x20, 0x0c }, { 0x0c } }, + } })); +} // commit_tests() + +BOOST_AUTO_TEST_CASE(test_undo) { + undo_tests(false, 0); + undo_tests(true, 0); + undo_tests(false, 64 * 1024 * 1024); + undo_tests(true, 64 * 1024 * 1024); +} + +BOOST_AUTO_TEST_CASE(test_squash) { + squash_tests(false, 0); + squash_tests(true, 0); + squash_tests(false, 64 * 1024 * 1024); + squash_tests(true, 64 * 1024 * 1024); +} + +BOOST_AUTO_TEST_CASE(test_commit) { + commit_tests(false, 0); + commit_tests(true, 0); + commit_tests(false, 64 * 1024 * 1024); + commit_tests(true, 64 * 1024 * 1024); +} + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/libraries/chain-kv/unit_tests/view_tests.cpp b/libraries/chain-kv/unit_tests/view_tests.cpp new file mode 100644 index 00000000000..4b29852081a --- /dev/null +++ b/libraries/chain-kv/unit_tests/view_tests.cpp @@ -0,0 +1,216 @@ +#include "chain_kv_tests.hpp" +#include + +using chain_kv::bytes; +using chain_kv::to_slice; + +BOOST_AUTO_TEST_SUITE(view_tests) + +void view_test(bool reload_session) { + boost::filesystem::remove_all("test-write-session-db"); + chain_kv::database db{ "test-write-session-db", true }; + chain_kv::undo_stack undo_stack{ db, { 0x10 } }; + std::unique_ptr session; + std::unique_ptr view; + + auto reload = [&] { + if (session && reload_session) { + session->write_changes(undo_stack); + view = nullptr; + session = nullptr; + } + if (!session) + session = std::make_unique(db); + if (!view) + view = std::make_unique(*session, bytes{ 0x70 }); + }; + reload(); + + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x1234), (kv_values{})); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x1234), get_matching2(*view, 0x1234)); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x5678), (kv_values{})); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x5678), get_matching2(*view, 0x5678)); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x9abc), (kv_values{})); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x9abc), get_matching2(*view, 0x9abc)); + + view->set(0x1234, to_slice({ 0x30, 0x40 }), to_slice({ 0x50, 0x60 })); + view->set(0x5678, to_slice({ 0x30, 0x71 }), to_slice({ 0x59, 0x69 })); + view->set(0x5678, to_slice({ 0x30, 0x00 }), to_slice({ 0x59, 0x69 })); + view->set(0x5678, to_slice({ 0x30, 0x42 }), to_slice({ 0x55, 0x66 })); + view->set(0x5678, to_slice({ 0x30, 0x41 }), to_slice({ 0x51, 0x61 })); + view->set(0x9abc, to_slice({ 0x30, 0x42 }), to_slice({ 0x52, 0x62 })); + reload(); + + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x1234), (kv_values{ { + { { 0x30, 0x40 }, { 0x50, 0x60 } }, + } })); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x1234), get_matching2(*view, 0x1234)); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x5678), (kv_values{ { + { { 0x30, 0x00 }, { 0x59, 0x69 } }, + { { 0x30, 0x41 }, { 0x51, 0x61 } }, + { { 0x30, 0x42 }, { 0x55, 0x66 } }, + { { 0x30, 0x71 }, { 0x59, 0x69 } }, + } })); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x5678), get_matching2(*view, 0x5678)); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x9abc), (kv_values{ { + { { 0x30, 0x42 }, { 0x52, 0x62 } }, + } })); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x9abc), get_matching2(*view, 0x9abc)); + + view->erase(0x5678, to_slice({ 0x30, 0x00 })); + view->erase(0x5678, to_slice({ 0x30, 0x71 })); + view->erase(0x5678, to_slice({ 0x30, 0x42 })); + reload(); + + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x1234), (kv_values{ { + { { 0x30, 0x40 }, { 0x50, 0x60 } }, + } })); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x1234), get_matching2(*view, 0x1234)); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x5678), (kv_values{ { + { { 0x30, 0x41 }, { 0x51, 0x61 } }, + } })); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x5678), get_matching2(*view, 0x5678)); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x9abc), (kv_values{ { + { { 0x30, 0x42 }, { 0x52, 0x62 } }, + } })); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x9abc), get_matching2(*view, 0x9abc)); + + { + chain_kv::view::iterator it{ *view, 0x5678, {} }; + it.lower_bound({ 0x30, 0x22 }); + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { 0x30, 0x41 }, { 0x51, 0x61 } }, + } })); + view->set(0x5678, to_slice({ 0x30, 0x22 }), to_slice({ 0x55, 0x66 })); + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { 0x30, 0x41 }, { 0x51, 0x61 } }, + } })); + it.lower_bound({ 0x30, 0x22 }); + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { 0x30, 0x22 }, { 0x55, 0x66 } }, + } })); + view->set(0x5678, to_slice({ 0x30, 0x22 }), to_slice({ 0x00, 0x11 })); + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { 0x30, 0x22 }, { 0x00, 0x11 } }, + } })); + view->erase(0x5678, to_slice({ 0x30, 0x22 })); + KV_REQUIRE_EXCEPTION(it.get_kv(), "kv iterator is at an erased value"); + KV_REQUIRE_EXCEPTION(--it, "kv iterator is at an erased value"); + KV_REQUIRE_EXCEPTION(++it, "kv iterator is at an erased value"); + view->set(0x5678, to_slice({ 0x30, 0x22 }), to_slice({})); + KV_REQUIRE_EXCEPTION(it.get_kv(), "kv iterator is at an erased value"); + it.lower_bound({ 0x30, 0x22 }); + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { 0x30, 0x22 }, {} }, + } })); + view->set(0x5678, to_slice({ 0x30, 0x22 }), to_slice({ 0x00 })); + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { 0x30, 0x22 }, { 0x00 } }, + } })); + view->erase(0x5678, to_slice({ 0x30, 0x22 })); + KV_REQUIRE_EXCEPTION(it.get_kv(), "kv iterator is at an erased value"); + } + reload(); + + { + chain_kv::view::iterator it{ *view, 0xbeefbeef, {} }; + view->set(0xbeefbeef, to_slice({ (char)0x80 }), to_slice({ (char)0xff })); + view->set(0xbeefbeef, to_slice({ (char)0x90 }), to_slice({ (char)0xfe })); + view->set(0xbeefbeef, to_slice({ (char)0xa0 }), to_slice({ (char)0xfd })); + view->set(0xbeefbeef, to_slice({ (char)0xb0 }), to_slice({ (char)0xfc })); + it.lower_bound({}); + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { (char)0x80 }, { (char)0xff } }, + } })); + it.lower_bound({ (char)0x80 }); + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { (char)0x80 }, { (char)0xff } }, + } })); + it.lower_bound({ (char)0x81 }); + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { (char)0x90 }, { (char)0xfe } }, + } })); + it.lower_bound({ (char)0x90 }); + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { (char)0x90 }, { (char)0xfe } }, + } })); + --it; + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { (char)0x80 }, { (char)0xff } }, + } })); + ++it; + ++it; + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { (char)0xa0 }, { (char)0xfd } }, + } })); + view->erase(0xbeefbeef, to_slice({ (char)0x90 })); + --it; + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { (char)0x80 }, { (char)0xff } }, + } })); + view->erase(0xbeefbeef, to_slice({ (char)0xa0 })); + ++it; + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { (char)0xb0 }, { (char)0xfc } }, + } })); + } + reload(); + + BOOST_REQUIRE_EQUAL(get_matching(*view, 0xbeefbeef), (kv_values{ { + { { (char)0x80 }, { (char)0xff } }, + { { (char)0xb0 }, { (char)0xfc } }, + } })); + + view->set(0xf00df00d, to_slice({ 0x10, 0x20, 0x00 }), to_slice({ (char)0x70 })); + view->set(0xf00df00d, to_slice({ 0x10, 0x20, 0x01 }), to_slice({ (char)0x71 })); + view->set(0xf00df00d, to_slice({ 0x10, 0x20, 0x02 }), to_slice({ (char)0x72 })); + + view->set(0xf00df00d, to_slice({ 0x10, 0x30, 0x00 }), to_slice({ (char)0x70 })); + view->set(0xf00df00d, to_slice({ 0x10, 0x30, 0x01 }), to_slice({ (char)0x71 })); + view->set(0xf00df00d, to_slice({ 0x10, 0x30, 0x02 }), to_slice({ (char)0x72 })); + + view->set(0xf00df00d, to_slice({ 0x20, 0x00 }), to_slice({ (char)0x70 })); + view->set(0xf00df00d, to_slice({ 0x20, 0x01 }), to_slice({ (char)0x71 })); + view->set(0xf00df00d, to_slice({ 0x20, 0x02 }), to_slice({ (char)0x72 })); + reload(); + + BOOST_REQUIRE_EQUAL(get_matching(*view, 0xf00df00d, { 0x10, 0x20 }), (kv_values{ { + { { 0x10, 0x20, 0x00 }, { 0x70 } }, + { { 0x10, 0x20, 0x01 }, { 0x71 } }, + { { 0x10, 0x20, 0x02 }, { 0x72 } }, + } })); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0xf00df00d, { 0x10, 0x20 }), + get_matching2(*view, 0xf00df00d, { 0x10, 0x20 })); + + BOOST_REQUIRE_EQUAL(get_matching(*view, 0xf00df00d, { 0x10, 0x30 }), (kv_values{ { + { { 0x10, 0x30, 0x00 }, { 0x70 } }, + { { 0x10, 0x30, 0x01 }, { 0x71 } }, + { { 0x10, 0x30, 0x02 }, { 0x72 } }, + } })); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0xf00df00d, { 0x10, 0x30 }), + get_matching2(*view, 0xf00df00d, { 0x10, 0x30 })); + + BOOST_REQUIRE_EQUAL(get_matching(*view, 0xf00df00d, { 0x10 }), (kv_values{ { + { { 0x10, 0x20, 0x00 }, { 0x70 } }, + { { 0x10, 0x20, 0x01 }, { 0x71 } }, + { { 0x10, 0x20, 0x02 }, { 0x72 } }, + { { 0x10, 0x30, 0x00 }, { 0x70 } }, + { { 0x10, 0x30, 0x01 }, { 0x71 } }, + { { 0x10, 0x30, 0x02 }, { 0x72 } }, + } })); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0xf00df00d, { 0x10 }), get_matching2(*view, 0xf00df00d, { 0x10 })); + + BOOST_REQUIRE_EQUAL(get_matching(*view, 0xf00df00d, { 0x20 }), (kv_values{ { + { { 0x20, 0x00 }, { 0x70 } }, + { { 0x20, 0x01 }, { 0x71 } }, + { { 0x20, 0x02 }, { 0x72 } }, + } })); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0xf00df00d, { 0x20 }), get_matching2(*view, 0xf00df00d, { 0x20 })); +} // view_test() + +BOOST_AUTO_TEST_CASE(test_view) { + view_test(false); + view_test(true); +} + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/libraries/chain-kv/unit_tests/write_session_tests.cpp b/libraries/chain-kv/unit_tests/write_session_tests.cpp new file mode 100644 index 00000000000..5201f2cca80 --- /dev/null +++ b/libraries/chain-kv/unit_tests/write_session_tests.cpp @@ -0,0 +1,71 @@ +#include "chain_kv_tests.hpp" +#include + +using chain_kv::bytes; +using chain_kv::to_slice; + +BOOST_AUTO_TEST_SUITE(write_session_tests) + +void write_session_test(bool reload_session) { + boost::filesystem::remove_all("test-write-session-db"); + chain_kv::database db{ "test-write-session-db", true }; + chain_kv::undo_stack undo_stack{ db, { 0x10 } }; + std::unique_ptr session; + + auto reload = [&] { + if (session && reload_session) { + session->write_changes(undo_stack); + session = nullptr; + } + if (!session) + session = std::make_unique(db); + }; + reload(); + + std::vector keys = { + { 0x20 }, + { 0x20, 0x00 }, + { 0x30 }, + }; + + BOOST_REQUIRE_EQUAL(get_values(*session, keys), (kv_values{})); + session->erase({ 0x20 }); + reload(); + BOOST_REQUIRE_EQUAL(get_values(*session, keys), (kv_values{})); + session->set({ 0x20, 0x00 }, to_slice({ 0x01 })); + session->set({ 0x30 }, to_slice({ 0x02 })); + reload(); + session->set({ 0x20 }, to_slice({ 0x03 })); + reload(); + BOOST_REQUIRE_EQUAL(get_values(*session, keys), (kv_values{ { + { { 0x20 }, { 0x03 } }, + { { 0x20, 0x00 }, { 0x01 } }, + { { 0x30 }, { 0x02 } }, + } })); + session->erase({ 0x20 }); + reload(); + BOOST_REQUIRE_EQUAL(get_values(*session, keys), (kv_values{ { + { { 0x20, 0x00 }, { 0x01 } }, + { { 0x30 }, { 0x02 } }, + } })); + session->set({ 0x20, 0x00 }, to_slice({ 0x11 })); + reload(); + BOOST_REQUIRE_EQUAL(get_values(*session, keys), (kv_values{ { + { { 0x20, 0x00 }, { 0x11 } }, + { { 0x30 }, { 0x02 } }, + } })); + session->set({ 0x20 }, to_slice({ 0x23 })); + reload(); + BOOST_REQUIRE_EQUAL(get_values(*session, keys), (kv_values{ { + { { 0x20 }, { 0x23 } }, + { { 0x20, 0x00 }, { 0x11 } }, + { { 0x30 }, { 0x02 } }, + } })); +} // write_session_test() + +BOOST_AUTO_TEST_CASE(test_write_session) { + write_session_test(false); + write_session_test(true); +} + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/libraries/rocksdb b/libraries/rocksdb new file mode 160000 index 00000000000..551a1109184 --- /dev/null +++ b/libraries/rocksdb @@ -0,0 +1 @@ +Subproject commit 551a110918493a19d11243f53408b97485de1411 From feac5232bd14b99c531602a5c9303a0c95f1ef10 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Fri, 24 Apr 2020 11:53:07 -0400 Subject: [PATCH 02/23] fix tests --- libraries/abieos | 2 +- libraries/chain-kv/unit_tests/CMakeLists.txt | 4 ++-- libraries/chain-kv/unit_tests/undo_stack_tests.cpp | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/libraries/abieos b/libraries/abieos index cf15468978c..ca85b0d0858 160000 --- a/libraries/abieos +++ b/libraries/abieos @@ -1 +1 @@ -Subproject commit cf15468978c075ce7aeb0e626071e592fa7ff040 +Subproject commit ca85b0d08589e4275bfb0cdd6cfb2d039c35612a diff --git a/libraries/chain-kv/unit_tests/CMakeLists.txt b/libraries/chain-kv/unit_tests/CMakeLists.txt index ecef2caf378..e74dc37a59e 100644 --- a/libraries/chain-kv/unit_tests/CMakeLists.txt +++ b/libraries/chain-kv/unit_tests/CMakeLists.txt @@ -12,12 +12,12 @@ foreach(TEST_SUITE ${UNIT_TESTS}) # create an independent target for each test s execute_process(COMMAND bash -c "grep -E 'BOOST_AUTO_TEST_SUITE\\s*[(]' ${TEST_SUITE} | grep -vE '//.*BOOST_AUTO_TEST_SUITE\\s*[(]' | cut -d ')' -f 1 | cut -d '(' -f 2" OUTPUT_VARIABLE SUITE_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) # get the test suite name from the *.cpp file if (NOT "" STREQUAL "${SUITE_NAME}") # ignore empty lines execute_process(COMMAND bash -c "echo ${SUITE_NAME} | sed -e 's/s$//' | sed -e 's/_test$//'" OUTPUT_VARIABLE TRIMMED_SUITE_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) # trim "_test" or "_tests" from the end of ${SUITE_NAME} - add_test(NAME ${TRIMMED_SUITE_NAME}_unit_test COMMAND chain-kv-tests --run_test=${SUITE_NAME} --report_level=detailed --color_output --catch_system_errors=no) + add_test(NAME chain_kv_${TRIMMED_SUITE_NAME}_unit_test COMMAND chain-kv-tests --run_test=${SUITE_NAME} --report_level=detailed --color_output --catch_system_errors=no) # build list of tests to run during coverage testing if(ctest_tests) string(APPEND ctest_tests "|") endif() - string(APPEND ctest_tests ${TRIMMED_SUITE_NAME}_unit_test) + string(APPEND ctest_tests chain_kv_${TRIMMED_SUITE_NAME}_unit_test) endif() endforeach(TEST_SUITE) set(ctest_tests "'${ctest_tests}' -j8") diff --git a/libraries/chain-kv/unit_tests/undo_stack_tests.cpp b/libraries/chain-kv/unit_tests/undo_stack_tests.cpp index 8cc1e81639a..eda20d9e73e 100644 --- a/libraries/chain-kv/unit_tests/undo_stack_tests.cpp +++ b/libraries/chain-kv/unit_tests/undo_stack_tests.cpp @@ -88,11 +88,9 @@ void undo_tests(bool reload_undo, uint64_t target_segment_size) { reload(); BOOST_REQUIRE_EQUAL(undo_stack->revision(), 11); KV_REQUIRE_EXCEPTION(undo_stack->set_revision(12), "cannot set revision while there is an existing undo stack"); - KV_REQUIRE_EXCEPTION(undo_stack->squash(), "nothing to squash"); undo_stack->commit(0); BOOST_REQUIRE_EQUAL(undo_stack->revision(), 11); KV_REQUIRE_EXCEPTION(undo_stack->set_revision(12), "cannot set revision while there is an existing undo stack"); - KV_REQUIRE_EXCEPTION(undo_stack->squash(), "nothing to squash"); undo_stack->commit(11); BOOST_REQUIRE_EQUAL(undo_stack->revision(), 11); reload(); @@ -106,7 +104,6 @@ void undo_tests(bool reload_undo, uint64_t target_segment_size) { { { 0x20, 0x00 }, { 0x70 } }, { { 0x20, 0x03 }, { 0x60 } }, } })); - } // undo_tests() void squash_tests(bool reload_undo, uint64_t target_segment_size) { @@ -250,6 +247,8 @@ void squash_tests(bool reload_undo, uint64_t target_segment_size) { reload(); BOOST_REQUIRE_EQUAL(undo_stack->revision(), 0); BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ {} })); + + // TODO: test squash with only 1 undo level } // squash_tests() void commit_tests(bool reload_undo, uint64_t target_segment_size) { From 1548d4f5b5dcc2dd96ab282d85f724e0ea27ee6c Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Fri, 24 Apr 2020 14:12:16 -0400 Subject: [PATCH 03/23] rename chain_kv --- libraries/CMakeLists.txt | 2 +- libraries/{chain-kv => chain_kv}/.clang-format | 0 libraries/{chain-kv => chain_kv}/CMakeLists.txt | 6 +++--- .../{chain-kv => chain_kv}/include/b1/chain_kv/chain_kv.hpp | 0 libraries/{chain-kv => chain_kv}/unit_tests/CMakeLists.txt | 2 +- .../{chain-kv => chain_kv}/unit_tests/chain_kv_tests.hpp | 0 libraries/{chain-kv => chain_kv}/unit_tests/main.cpp | 0 .../{chain-kv => chain_kv}/unit_tests/undo_stack_tests.cpp | 0 libraries/{chain-kv => chain_kv}/unit_tests/view_tests.cpp | 0 .../unit_tests/write_session_tests.cpp | 0 10 files changed, 5 insertions(+), 5 deletions(-) rename libraries/{chain-kv => chain_kv}/.clang-format (100%) rename libraries/{chain-kv => chain_kv}/CMakeLists.txt (74%) rename libraries/{chain-kv => chain_kv}/include/b1/chain_kv/chain_kv.hpp (100%) rename libraries/{chain-kv => chain_kv}/unit_tests/CMakeLists.txt (96%) rename libraries/{chain-kv => chain_kv}/unit_tests/chain_kv_tests.hpp (100%) rename libraries/{chain-kv => chain_kv}/unit_tests/main.cpp (100%) rename libraries/{chain-kv => chain_kv}/unit_tests/undo_stack_tests.cpp (100%) rename libraries/{chain-kv => chain_kv}/unit_tests/view_tests.cpp (100%) rename libraries/{chain-kv => chain_kv}/unit_tests/write_session_tests.cpp (100%) diff --git a/libraries/CMakeLists.txt b/libraries/CMakeLists.txt index 21cae663cbb..baf946b4004 100644 --- a/libraries/CMakeLists.txt +++ b/libraries/CMakeLists.txt @@ -12,7 +12,7 @@ add_subdirectory( version ) add_subdirectory( state_history ) add_subdirectory( abieos ) add_subdirectory( rocksdb EXCLUDE_FROM_ALL ) -add_subdirectory( chain-kv ) +add_subdirectory( chain_kv ) set(USE_EXISTING_SOFTFLOAT ON CACHE BOOL "use pre-exisiting softfloat lib") set(ENABLE_TOOLS OFF CACHE BOOL "Build tools") diff --git a/libraries/chain-kv/.clang-format b/libraries/chain_kv/.clang-format similarity index 100% rename from libraries/chain-kv/.clang-format rename to libraries/chain_kv/.clang-format diff --git a/libraries/chain-kv/CMakeLists.txt b/libraries/chain_kv/CMakeLists.txt similarity index 74% rename from libraries/chain-kv/CMakeLists.txt rename to libraries/chain_kv/CMakeLists.txt index f32c1802d8c..90fc9f1fea7 100644 --- a/libraries/chain-kv/CMakeLists.txt +++ b/libraries/chain_kv/CMakeLists.txt @@ -1,12 +1,12 @@ file(GLOB_RECURSE HEADERS "include/*.hpp") -add_library( chain-kv INTERFACE ) +add_library( chain_kv INTERFACE ) -target_link_libraries( chain-kv +target_link_libraries( chain_kv INTERFACE fc rocksdb gflags ) -target_include_directories( chain-kv +target_include_directories( chain_kv INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/../rocksdb/include" ) diff --git a/libraries/chain-kv/include/b1/chain_kv/chain_kv.hpp b/libraries/chain_kv/include/b1/chain_kv/chain_kv.hpp similarity index 100% rename from libraries/chain-kv/include/b1/chain_kv/chain_kv.hpp rename to libraries/chain_kv/include/b1/chain_kv/chain_kv.hpp diff --git a/libraries/chain-kv/unit_tests/CMakeLists.txt b/libraries/chain_kv/unit_tests/CMakeLists.txt similarity index 96% rename from libraries/chain-kv/unit_tests/CMakeLists.txt rename to libraries/chain_kv/unit_tests/CMakeLists.txt index e74dc37a59e..302d3010da1 100644 --- a/libraries/chain-kv/unit_tests/CMakeLists.txt +++ b/libraries/chain_kv/unit_tests/CMakeLists.txt @@ -3,7 +3,7 @@ include(ExternalProject) file(GLOB UNIT_TESTS "*.cpp") # find all unit test suites add_executable(chain-kv-tests ${UNIT_TESTS}) -target_link_libraries(chain-kv-tests chain-kv) +target_link_libraries(chain-kv-tests chain_kv) enable_testing() diff --git a/libraries/chain-kv/unit_tests/chain_kv_tests.hpp b/libraries/chain_kv/unit_tests/chain_kv_tests.hpp similarity index 100% rename from libraries/chain-kv/unit_tests/chain_kv_tests.hpp rename to libraries/chain_kv/unit_tests/chain_kv_tests.hpp diff --git a/libraries/chain-kv/unit_tests/main.cpp b/libraries/chain_kv/unit_tests/main.cpp similarity index 100% rename from libraries/chain-kv/unit_tests/main.cpp rename to libraries/chain_kv/unit_tests/main.cpp diff --git a/libraries/chain-kv/unit_tests/undo_stack_tests.cpp b/libraries/chain_kv/unit_tests/undo_stack_tests.cpp similarity index 100% rename from libraries/chain-kv/unit_tests/undo_stack_tests.cpp rename to libraries/chain_kv/unit_tests/undo_stack_tests.cpp diff --git a/libraries/chain-kv/unit_tests/view_tests.cpp b/libraries/chain_kv/unit_tests/view_tests.cpp similarity index 100% rename from libraries/chain-kv/unit_tests/view_tests.cpp rename to libraries/chain_kv/unit_tests/view_tests.cpp diff --git a/libraries/chain-kv/unit_tests/write_session_tests.cpp b/libraries/chain_kv/unit_tests/write_session_tests.cpp similarity index 100% rename from libraries/chain-kv/unit_tests/write_session_tests.cpp rename to libraries/chain_kv/unit_tests/write_session_tests.cpp From 293cacd56ff101c09199f8e04c944e3e6dab5013 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Fri, 24 Apr 2020 14:45:37 -0400 Subject: [PATCH 04/23] rodeos --- libraries/rodeos/.clang-format | 76 ++++ libraries/rodeos/CMakeLists.txt | 14 + libraries/rodeos/embedded_rodeos.cpp | 273 +++++++++++++ .../include/b1/rodeos/callbacks/action.hpp | 50 +++ .../include/b1/rodeos/callbacks/basic.hpp | 103 +++++ .../include/b1/rodeos/callbacks/chaindb.hpp | 263 ++++++++++++ .../b1/rodeos/callbacks/compiler_builtins.hpp | 320 +++++++++++++++ .../include/b1/rodeos/callbacks/console.hpp | 108 +++++ .../include/b1/rodeos/callbacks/filter.hpp | 26 ++ .../rodeos/include/b1/rodeos/callbacks/kv.hpp | 378 ++++++++++++++++++ .../include/b1/rodeos/callbacks/memory.hpp | 50 +++ .../include/b1/rodeos/callbacks/query.hpp | 44 ++ .../b1/rodeos/callbacks/unimplemented.hpp | 251 ++++++++++++ .../rodeos/callbacks/unimplemented_filter.hpp | 24 ++ .../include/b1/rodeos/embedded_rodeos.h | 151 +++++++ .../include/b1/rodeos/embedded_rodeos.hpp | 198 +++++++++ libraries/rodeos/include/b1/rodeos/filter.hpp | 62 +++ .../include/b1/rodeos/get_state_row.hpp | 43 ++ libraries/rodeos/include/b1/rodeos/rodeos.hpp | 87 ++++ .../include/b1/rodeos/rodeos_tables.hpp | 274 +++++++++++++ .../rodeos/include/b1/rodeos/wasm_ql.hpp | 77 ++++ 21 files changed, 2872 insertions(+) create mode 100644 libraries/rodeos/.clang-format create mode 100644 libraries/rodeos/CMakeLists.txt create mode 100644 libraries/rodeos/embedded_rodeos.cpp create mode 100644 libraries/rodeos/include/b1/rodeos/callbacks/action.hpp create mode 100644 libraries/rodeos/include/b1/rodeos/callbacks/basic.hpp create mode 100644 libraries/rodeos/include/b1/rodeos/callbacks/chaindb.hpp create mode 100644 libraries/rodeos/include/b1/rodeos/callbacks/compiler_builtins.hpp create mode 100644 libraries/rodeos/include/b1/rodeos/callbacks/console.hpp create mode 100644 libraries/rodeos/include/b1/rodeos/callbacks/filter.hpp create mode 100644 libraries/rodeos/include/b1/rodeos/callbacks/kv.hpp create mode 100644 libraries/rodeos/include/b1/rodeos/callbacks/memory.hpp create mode 100644 libraries/rodeos/include/b1/rodeos/callbacks/query.hpp create mode 100644 libraries/rodeos/include/b1/rodeos/callbacks/unimplemented.hpp create mode 100644 libraries/rodeos/include/b1/rodeos/callbacks/unimplemented_filter.hpp create mode 100644 libraries/rodeos/include/b1/rodeos/embedded_rodeos.h create mode 100644 libraries/rodeos/include/b1/rodeos/embedded_rodeos.hpp create mode 100644 libraries/rodeos/include/b1/rodeos/filter.hpp create mode 100644 libraries/rodeos/include/b1/rodeos/get_state_row.hpp create mode 100644 libraries/rodeos/include/b1/rodeos/rodeos.hpp create mode 100644 libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp create mode 100644 libraries/rodeos/include/b1/rodeos/wasm_ql.hpp diff --git a/libraries/rodeos/.clang-format b/libraries/rodeos/.clang-format new file mode 100644 index 00000000000..fa257fc706b --- /dev/null +++ b/libraries/rodeos/.clang-format @@ -0,0 +1,76 @@ +BasedOnStyle: LLVM +IndentWidth: 3 +UseTab: Never +ColumnLimit: 120 + +--- +Language: Cpp +# always align * and & to the type +DerivePointerAlignment: false +PointerAlignment: Left + +# regroup includes to these classes +IncludeCategories: + - Regex: '(<|"(eosio)/)' + Priority: 4 + - Regex: '(<|"(boost)/)' + Priority: 3 + - Regex: '(<|"(llvm|llvm-c|clang|clang-c)/' + Priority: 3 + - Regex: '<[[:alnum:]]+>' + Priority: 2 + - Regex: '.*' + Priority: 1 + +#IncludeBlocks: Regroup + +# set indent for public, private and protected +#AccessModifierOffset: 3 + +# make line continuations twice the normal indent +ContinuationIndentWidth: 6 + +# add missing namespace comments +FixNamespaceComments: true + +# add spaces to braced list i.e. int* foo = { 0, 1, 2 }; instead of int* foo = {0,1,2}; +Cpp11BracedListStyle: false +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: true +AlignConsecutiveDeclarations: true +AlignOperands: true +AlignTrailingComments: true +AllowShortCaseLabelsOnASingleLine: true +AllowShortFunctionsOnASingleLine: All +AllowShortBlocksOnASingleLine: true +#AllowShortIfStatementsOnASingleLine: WithoutElse +#AllowShortIfStatementsOnASingleLine: true +#AllowShortLambdasOnASingleLine: All +AllowShortLoopsOnASingleLine: true +AlwaysBreakTemplateDeclarations: true + +BinPackParameters: true +### use this with clang9 +BreakBeforeBraces: Custom +BraceWrapping: + #AfterCaseLabel: true + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + +BreakConstructorInitializers: BeforeColon +CompactNamespaces: true +IndentCaseLabels: true +IndentPPDirectives: AfterHash +NamespaceIndentation: None +ReflowComments: false +SortIncludes: true +SortUsingDeclarations: true +--- diff --git a/libraries/rodeos/CMakeLists.txt b/libraries/rodeos/CMakeLists.txt new file mode 100644 index 00000000000..7c202b3dbe9 --- /dev/null +++ b/libraries/rodeos/CMakeLists.txt @@ -0,0 +1,14 @@ +file(GLOB_RECURSE HEADERS "include/*.hpp" "include/*.h") + +add_library( rodeos_lib + embedded_rodeos.cpp + ${HEADERS} + ) + +target_link_libraries( rodeos_lib + PUBLIC abieos chain_kv eosio_chain fc softfloat + ) + +target_include_directories( rodeos_lib + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" + ) diff --git a/libraries/rodeos/embedded_rodeos.cpp b/libraries/rodeos/embedded_rodeos.cpp new file mode 100644 index 00000000000..5b1c09c43c6 --- /dev/null +++ b/libraries/rodeos/embedded_rodeos.cpp @@ -0,0 +1,273 @@ +#include +#include + +struct rodeos_error_s { + const char* msg = "no error"; + std::string buffer; + + bool set(const char* m) { + try { + buffer = m; + msg = buffer.c_str(); + } catch (...) { msg = "error storing error message"; } + return false; + } +}; + +struct rodeos_context_s : b1::rodeos::rodeos_context {}; + +struct rodeos_db_partition_s { + std::shared_ptr obj; +}; + +struct rodeos_db_snapshot_s : b1::rodeos::rodeos_db_snapshot { + using rodeos_db_snapshot::rodeos_db_snapshot; +}; + +struct rodeos_filter_s : b1::rodeos::rodeos_filter { + using rodeos_filter::rodeos_filter; +}; + +struct rodeos_query_handler_s : b1::rodeos::rodeos_query_handler { + using rodeos_query_handler::rodeos_query_handler; +}; + +extern "C" rodeos_error* rodeos_create_error() { + try { + return std::make_unique().release(); + } catch (...) { return nullptr; } +} + +extern "C" void rodeos_destroy_error(rodeos_error* error) { std::unique_ptr{ error }; } + +extern "C" const char* rodeos_get_error(rodeos_error* error) { + if (!error) + return "error is null"; + return error->msg; +} + +template +auto handle_exceptions(rodeos_error* error, T errval, F f) noexcept -> decltype(f()) { + if (!error) + return errval; + try { + return f(); + } catch (std::exception& e) { + error->set(e.what()); + return errval; + } catch (...) { + error->set("unknown exception"); + return errval; + } +} + +extern "C" rodeos_context* rodeos_create() { + try { + return std::make_unique().release(); + } catch (...) { return nullptr; } +} + +extern "C" void rodeos_destroy(rodeos_context* context) { std::unique_ptr{ context }; } + +extern "C" rodeos_bool rodeos_open_db(rodeos_error* error, rodeos_context* context, const char* path, + rodeos_bool create_if_missing, int num_threads, int max_open_files) { + return handle_exceptions(error, false, [&] { + if (!context) + return error->set("context is null"); + if (!path) + return error->set("path is null"); + if (context->db) + return error->set("a database is already open on this context"); + context->db = std::make_shared( + path, create_if_missing, num_threads ? std::make_optional(num_threads) : std::nullopt, + max_open_files ? std::make_optional(max_open_files) : std::nullopt); + return true; + }); +} + +extern "C" rodeos_db_partition* rodeos_create_partition(rodeos_error* error, rodeos_context* context, + const char* prefix, uint32_t prefix_size) { + return handle_exceptions(error, nullptr, [&]() -> rodeos_db_partition* { + if (!context) + return error->set("context is null"), nullptr; + if (!prefix) + return error->set("prefix is null"), nullptr; + if (!context->db) + return error->set("database wasn't opened"), nullptr; + auto p = std::make_unique(); + p->obj = std::make_shared(context->db, + std::vector{ prefix, prefix + prefix_size }); + return p.release(); + }); +} + +extern "C" void rodeos_destroy_partition(rodeos_db_partition* partition) { + std::unique_ptr{ partition }; +} + +extern "C" rodeos_db_snapshot* rodeos_create_snapshot(rodeos_error* error, rodeos_db_partition* partition, + rodeos_bool persistent) { + return handle_exceptions(error, nullptr, [&]() -> rodeos_db_snapshot* { + if (!partition) + return error->set("partition is null"), nullptr; + return std::make_unique(partition->obj, persistent).release(); + }); +} + +extern "C" void rodeos_destroy_snapshot(rodeos_db_snapshot* snapshot) { + std::unique_ptr{ snapshot }; +} + +extern "C" rodeos_bool rodeos_refresh_snapshot(rodeos_error* error, rodeos_db_snapshot* snapshot) { + return handle_exceptions(error, false, [&]() { + if (!snapshot) + return error->set("snapshot is null"); + snapshot->refresh(); + return true; + }); +} + +template +void with_result(const char* data, uint64_t size, F f) { + eosio::input_stream bin{ data, data + size }; + eosio::ship_protocol::result result; + eosio::check_discard(from_bin(result, bin)); + auto* result_v0 = std::get_if(&result); + if (!result_v0) + throw std::runtime_error("expected a get_blocks_result_v0"); + f(*result_v0); +} + +extern "C" rodeos_bool rodeos_start_block(rodeos_error* error, rodeos_db_snapshot* snapshot, const char* data, + uint64_t size) { + return handle_exceptions(error, false, [&]() { + if (!snapshot) + return error->set("snapshot is null"); + with_result(data, size, [&](auto& result) { snapshot->start_block(result); }); + return true; + }); +} + +extern "C" rodeos_bool rodeos_end_block(rodeos_error* error, rodeos_db_snapshot* snapshot, const char* data, + uint64_t size, bool force_write) { + return handle_exceptions(error, false, [&]() { + if (!snapshot) + return error->set("snapshot is null"); + with_result(data, size, [&](auto& result) { snapshot->end_block(result, force_write); }); + return true; + }); +} + +extern "C" rodeos_bool rodeos_write_block_info(rodeos_error* error, rodeos_db_snapshot* snapshot, const char* data, + uint64_t size) { + return handle_exceptions(error, false, [&]() { + if (!snapshot) + return error->set("snapshot is null"); + with_result(data, size, [&](auto& result) { snapshot->write_block_info(result); }); + return true; + }); +} + +extern "C" rodeos_bool rodeos_write_deltas(rodeos_error* error, rodeos_db_snapshot* snapshot, const char* data, + uint64_t size, rodeos_bool (*shutdown)(void*), void* shutdown_arg) { + return handle_exceptions(error, false, [&]() { + if (!snapshot) + return error->set("snapshot is null"); + with_result(data, size, [&](auto& result) { + snapshot->write_deltas(result, [=]() -> bool { + if (shutdown) + return shutdown(shutdown_arg); + else + return false; + }); + }); + return true; + }); +} + +extern "C" rodeos_filter* rodeos_create_filter(rodeos_error* error, uint64_t name, const char* wasm_filename) { + return handle_exceptions(error, nullptr, [&]() -> rodeos_filter* { // + return std::make_unique(eosio::name{ name }, wasm_filename).release(); + }); +} + +extern "C" void rodeos_destroy_filter(rodeos_filter* filter) { std::unique_ptr{ filter }; } + +extern "C" rodeos_bool rodeos_run_filter(rodeos_error* error, rodeos_db_snapshot* snapshot, rodeos_filter* filter, + const char* data, uint64_t size, + rodeos_bool (*push_data)(void* arg, const char* data, uint64_t size), + void* push_data_arg) { + return handle_exceptions(error, false, [&]() { + if (!snapshot) + return error->set("snapshot is null"); + if (!filter) + return error->set("filter is null"); + with_result(data, size, [&](auto& result) { + filter->process(*snapshot, result, { data, data + size }, [&](const char* data, uint64_t size) { + if (push_data && !push_data(push_data_arg, data, size)) + throw std::runtime_error("push_data returned false"); + }); + }); + return true; + }); +} + +extern "C" rodeos_query_handler* rodeos_create_query_handler(rodeos_error* error, rodeos_db_partition* partition, + uint32_t max_console_size, uint32_t wasm_cache_size, + uint64_t max_exec_time_ms, const char* contract_dir) { + return handle_exceptions(error, nullptr, [&]() -> rodeos_query_handler* { + if (!partition) + return error->set("partition is null"), nullptr; + auto shared_state = std::make_shared(partition->obj->db); + shared_state->max_console_size = max_console_size; + shared_state->wasm_cache_size = wasm_cache_size; + shared_state->max_exec_time_ms = max_exec_time_ms; + shared_state->contract_dir = contract_dir ? contract_dir : ""; + return std::make_unique(partition->obj, shared_state).release(); + }); +} + +void rodeos_destroy_query_handler(rodeos_query_handler* handler) { std::unique_ptr{ handler }; } + +rodeos_bool rodeos_query_transaction(rodeos_error* error, rodeos_query_handler* handler, rodeos_db_snapshot* snapshot, + const char* data, uint64_t size, char** result, uint64_t* result_size) { + return handle_exceptions(error, false, [&]() { + if (!handler) + return error->set("handler is null"); + if (!result) + return error->set("result is null"); + if (!result_size) + return error->set("result_size is null"); + *result = nullptr; + *result_size = 0; + + std::vector> memory; + eosio::input_stream s{ data, size }; + auto trx = eosio::check(eosio::from_bin(s)).value(); + + auto thread_state = handler->state_cache.get_state(); + eosio::ship_protocol::transaction_trace tt; + if (snapshot->snap.has_value()) { + tt = query_send_transaction(*thread_state, snapshot->partition->contract_kv_prefix, trx, + snapshot->snap->snapshot(), memory, true); + } else { + tt = query_send_transaction(*thread_state, snapshot->partition->contract_kv_prefix, trx, nullptr, memory, + true); + } + + handler->state_cache.store_state(std::move(thread_state)); + + auto packed = eosio::check(eosio::convert_to_bin(tt)).value(); + *result = (char*)malloc(packed.size()); + if (!result) + throw std::bad_alloc(); + *result_size = packed.size(); + memcpy(*result, packed.data(), packed.size()); + return true; + }); +} + +void rodeos_free_result(char* result) { + if (result) + free(result); +} diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/action.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/action.hpp new file mode 100644 index 00000000000..7e513df6011 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/action.hpp @@ -0,0 +1,50 @@ +#pragma once + +#include + +namespace b1::rodeos { + +struct action_state { + eosio::name receiver{}; + eosio::input_stream action_data{}; + std::vector action_return_value{}; +}; + +template +struct action_callbacks { + Derived& derived() { return static_cast(*this); } + + int read_action_data(char* memory, uint32_t buffer_size) { + derived().check_bounds(memory, buffer_size); + auto& state = derived().get_state(); + auto s = state.action_data.end - state.action_data.pos; + if (buffer_size == 0) + return s; + auto copy_size = std::min(static_cast(buffer_size), static_cast(s)); + memcpy(memory, state.action_data.pos, copy_size); + return copy_size; + } + + int action_data_size() { + auto& state = derived().get_state(); + return state.action_data.end - state.action_data.pos; + } + + uint64_t current_receiver() { return derived().get_state().receiver.value; } + + void set_action_return_value(char* packed_blob, uint32_t datalen) { + // todo: limit size + derived().check_bounds(packed_blob, datalen); + derived().get_state().action_return_value.assign(packed_blob, packed_blob + datalen); + } + + template + static void register_callbacks() { + Rft::template add("env", "read_action_data"); + Rft::template add("env", "action_data_size"); + Rft::template add("env", "current_receiver"); + Rft::template add("env", "set_action_return_value"); + } +}; // action_callbacks + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/basic.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/basic.hpp new file mode 100644 index 00000000000..fdb9e6b4d20 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/basic.hpp @@ -0,0 +1,103 @@ +#pragma once + +#include + +#include +#include + +namespace b1::rodeos { + +struct assert_exception : std::exception { + std::string msg; + + assert_exception(std::string&& msg) : msg(std::move(msg)) {} + + const char* what() const noexcept override { return msg.c_str(); } +}; + +template +struct wasm_state { + eosio::vm::wasm_allocator& wa; + Backend& backend; +}; + +inline size_t copy_to_wasm(char* dest, size_t dest_size, const char* src, size_t src_size) { + if (dest_size == 0) + return src_size; + auto copy_size = std::min(dest_size, src_size); + memcpy(dest, src, copy_size); + return copy_size; +} + +template +struct basic_callbacks { + Derived& derived() { return static_cast(*this); } + + void check_bounds(const char* begin, const char* end) { + if (begin > end) + throw std::runtime_error("bad memory"); + // todo: check bounds + } + + void check_bounds(const char* begin, uint32_t size) { + // todo: check bounds + } + + uint32_t check_bounds_get_len(const char* str) { + // todo: check bounds + return strlen(str); + } + + void abort() { throw std::runtime_error("called abort"); } + + void eosio_assert_message(bool test, const char* msg, uint32_t msg_len) { + check_bounds(msg, msg + msg_len); + if (!test) + throw assert_exception(std::string(msg, msg_len)); + } + + // todo: replace with prints_l + void print_range(const char* begin, const char* end) { + check_bounds(begin, end); + std::cerr.write(begin, end - begin); + } + + template + static void register_callbacks() { + Rft::template add("env", "abort"); + Rft::template add("env", "eosio_assert_message"); + Rft::template add("env", "print_range"); + } +}; + +template +struct data_state { + eosio::input_stream input_data; + std::vector output_data; +}; + +template +struct data_callbacks { + Derived& derived() { return static_cast(*this); } + + uint32_t get_input_data(char* dest, uint32_t size) { + derived().check_bounds(dest, size); + auto& input_data = derived().get_state().input_data; + return copy_to_wasm(dest, size, input_data.pos, size_t(input_data.end - input_data.pos)); + } + + void set_output_data(const char* data, uint32_t size) { + derived().check_bounds(data, size); + auto& output_data = derived().get_state().output_data; + output_data.clear(); + output_data.insert(output_data.end(), data, data + size); + } + + template + static void register_callbacks() { + Rft::template add("env", "get_input_data"); + Rft::template add("env", "set_output_data"); + } +}; + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/chaindb.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/chaindb.hpp new file mode 100644 index 00000000000..6e7ae7967ae --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/chaindb.hpp @@ -0,0 +1,263 @@ +#pragma once + +#include + +#include +#include +#include +#include + +namespace b1::rodeos { + +class iterator_cache { + private: + chain_kv::view& view; + + struct table_key { + uint64_t code = {}; + uint64_t table = {}; + uint64_t scope = {}; + + std::tuple order() const { return { code, table, scope }; } + friend bool operator<(const table_key& a, const table_key& b) { return a.order() < b.order(); } + }; + std::vector tables; + std::map table_to_index; + + struct row_key { + int32_t table_index = {}; + uint64_t key = {}; + + std::tuple order() const { return { table_index, key }; } + friend bool operator<(const row_key& a, const row_key& b) { return a.order() < b.order(); } + }; + + struct iterator { + int32_t table_index = {}; + uint64_t primary = {}; + std::vector value; + int32_t next = -1; + std::optional view_it; + }; + std::vector iterators; + std::vector end_iterators; + std::map key_to_iterator_index; + + int32_t get_table_index(table_key key) { + auto map_it = table_to_index.find(key); + if (map_it != table_to_index.end()) + return map_it->second; + if (!view.get(eosio::name{ "state" }.value, + chain_kv::to_slice(eosio::check(eosio::convert_to_key(std::make_tuple( + eosio::name{ "contract.tab" }, eosio::name{ "primary" }, + key.code, key.table, key.scope))) + .value()))) + return -1; + if (tables.size() != table_to_index.size() || tables.size() != end_iterators.size()) + throw std::runtime_error("internal error: tables.size() mismatch"); + auto result = tables.size(); + if (result > std::numeric_limits::max()) + throw std::runtime_error("too many open tables"); + tables.push_back(key); + table_to_index[key] = result; + end_iterators.push_back({}); + auto& end_it = end_iterators.back(); + end_it.table_index = result; + return result; + } + + int32_t get_iterator(row_key rk, chain_kv::view::iterator&& view_it) { + iterator* it; + int32_t result; + if (view_it.is_end()) { + // std::cout << "...end\n"; + it = &end_iterators[rk.table_index]; + result = index_to_end_iterator(rk.table_index); + } else { + auto map_it = key_to_iterator_index.find(rk); + if (map_it != key_to_iterator_index.end()) { + // std::cout << "...existing it (b)\n"; + it = &iterators[map_it->second]; + result = map_it->second; + } else { + // std::cout << "...new it\n"; + if (iterators.size() > std::numeric_limits::max()) + throw std::runtime_error("too many iterators"); + result = iterators.size(); + iterators.emplace_back(); + it = &iterators.back(); + eosio::input_stream stream{ view_it.get_kv()->value.data(), view_it.get_kv()->value.size() }; + auto row = std::get<0>(eosio::check(eosio::from_bin(stream)).value()); + it->table_index = rk.table_index; + it->primary = row.primary_key; + it->value.insert(it->value.end(), row.value.pos, row.value.end); + } + } + if (!it->view_it) + it->view_it = std::move(view_it); + return result; + } + + // Precondition: std::numeric_limits::min() < ei < -1 + // Iterator of -1 is reserved for invalid iterators (i.e. when the appropriate table has not yet been created). + size_t end_iterator_to_index(int32_t ei) const { return (-ei - 2); } + // Precondition: indx < tables.size() <= std::numeric_limits::max() + int32_t index_to_end_iterator(size_t indx) const { return -(indx + 2); } + + public: + iterator_cache(chain_kv::view& view) : view{ view } {} + + size_t db_get_i64(int itr, char* buffer, uint32_t buffer_size) { + if (itr == -1) + throw std::runtime_error("dereference invalid iterator"); + if (itr < 0) + throw std::runtime_error("dereference end iterator"); + if (itr >= iterators.size()) + throw std::runtime_error("dereference non-existing iterator"); + auto& it = iterators[itr]; + return copy_to_wasm(buffer, buffer_size, it.value.data(), it.value.size()); + } + + int db_next_i64(int itr, uint64_t& primary) { + if (itr == -1) + throw std::runtime_error("increment invalid iterator"); + if (itr < 0) + return -1; + if (itr >= iterators.size()) + throw std::runtime_error("increment non-existing iterator"); + auto& it = iterators[itr]; + if (it.next >= 0) { + primary = iterators[it.next].primary; + return it.next; + } else if (it.next < -1) { + return it.next; + } + std::optional view_it = std::move(it.view_it); + it.view_it.reset(); + if (!view_it) { + // std::cout << "db_next_i64: db_view::iterator\n"; + const auto& table_key = tables[it.table_index]; + view_it = chain_kv::view::iterator{ + view, eosio::name{ "state" }.value, + chain_kv::to_slice(eosio::check(eosio::convert_to_key(std::make_tuple( + eosio::name{ "contract.row" }, eosio::name{ "primary" }, + table_key.code, table_key.table, table_key.scope))) + .value()) + }; + view_it->lower_bound(eosio::check(eosio::convert_to_key(std::make_tuple( + eosio::name{ "contract.row" }, eosio::name{ "primary" }, + table_key.code, table_key.table, table_key.scope, it.primary))) + .value()); + } + ++*view_it; + if (view_it->is_end()) { + it.next = index_to_end_iterator(itr); + return it.next; + } else { + eosio::input_stream stream{ view_it->get_kv()->value.data(), view_it->get_kv()->value.size() }; + auto row = std::get<0>(eosio::check(eosio::from_bin(stream)).value()); + primary = row.primary_key; + it.next = get_iterator({ it.table_index, primary }, std::move(*view_it)); + return it.next; + } + } + + int32_t lower_bound(uint64_t code, uint64_t scope, uint64_t table, uint64_t key) { + int32_t table_index = get_table_index({ code, table, scope }); + if (table_index < 0) { + // std::cout << "...no table\n"; + return -1; + } + row_key rk{ table_index, key }; + auto map_it = key_to_iterator_index.find(rk); + if (map_it != key_to_iterator_index.end()) { + // std::cout << "...existing it (a)\n"; + return map_it->second; + } + // std::cout << "lower_bound: db_view::iterator\n"; + chain_kv::view::iterator it{ view, eosio::name{ "state" }.value, + chain_kv::to_slice(eosio::check(eosio::convert_to_key(std::make_tuple( + eosio::name{ "contract.row" }, + eosio::name{ "primary" }, code, table, scope))) + .value()) }; + it.lower_bound( + eosio::check(eosio::convert_to_key(std::make_tuple(eosio::name{ "contract.row" }, eosio::name{ "primary" }, + code, table, scope, key))) + .value()); + return get_iterator(rk, std::move(it)); + } +}; // iterator_cache + +struct chaindb_state { + std::unique_ptr iterator_cache; +}; + +template +struct chaindb_callbacks { + Derived& derived() { return static_cast(*this); } + + iterator_cache& get_iterator_cache() { + auto& chaindb_state = derived().get_chaindb_state(); + if (!chaindb_state.iterator_cache) + chaindb_state.iterator_cache = std::make_unique(derived().get_db_view_state().kv_state.view); + return *chaindb_state.iterator_cache; + } + + int db_store_i64(uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, const char* buffer, + uint32_t buffer_size) { + throw std::runtime_error("unimplemented: db_store_i64"); + } + + void db_update_i64(int itr, uint64_t payer, const char* buffer, uint32_t buffer_size) { + throw std::runtime_error("unimplemented: db_update_i64"); + } + + void db_remove_i64(int itr) { throw std::runtime_error("unimplemented: db_remove_i64"); } + + int db_get_i64(int itr, char* buffer, uint32_t buffer_size) { + derived().check_bounds(buffer, buffer_size); + return get_iterator_cache().db_get_i64(itr, buffer, buffer_size); + } + + int db_next_i64(int itr, uint64_t& primary) { return get_iterator_cache().db_next_i64(itr, primary); } + + int db_previous_i64(int itr, uint64_t& primary) { + // + throw std::runtime_error("unimplemented: db_previous_i64"); + } + + int db_find_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) { + // + throw std::runtime_error("unimplemented: db_find_i64"); + } + + int db_lowerbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) { + return get_iterator_cache().lower_bound(code, scope, table, id); + } + + int db_upperbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) { + // + throw std::runtime_error("unimplemented: db_upperbound_i64"); + } + + int db_end_i64(uint64_t code, uint64_t scope, uint64_t table) { + // + throw std::runtime_error("unimplemented: db_end_i64"); + } + + template + static void register_callbacks() { + Rft::template add("env", "db_store_i64"); + Rft::template add("env", "db_update_i64"); + Rft::template add("env", "db_remove_i64"); + Rft::template add("env", "db_get_i64"); + Rft::template add("env", "db_next_i64"); + Rft::template add("env", "db_previous_i64"); + Rft::template add("env", "db_find_i64"); + Rft::template add("env", "db_lowerbound_i64"); + Rft::template add("env", "db_upperbound_i64"); + Rft::template add("env", "db_end_i64"); + } +}; + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/compiler_builtins.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/compiler_builtins.hpp new file mode 100644 index 00000000000..b9bc654b0d5 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/compiler_builtins.hpp @@ -0,0 +1,320 @@ +#pragma once + +#include +#include +#include + +namespace b1::rodeos { + +template +struct compiler_builtins_callbacks { + Derived& derived() { return static_cast(*this); } + + void __ashlti3(__int128& ret, uint64_t low, uint64_t high, uint32_t shift) { + if (shift >= 128) { + ret = 0; + } else { + unsigned __int128 i = high; + i <<= 64; + i |= low; + i <<= shift; + ret = (__int128)i; + } + } + + void __ashrti3(__int128& ret, uint64_t low, uint64_t high, uint32_t shift) { + // retain the signedness + ret = high; + ret <<= 64; + ret |= low; + ret >>= shift; + } + + void __lshlti3(__int128& ret, uint64_t low, uint64_t high, uint32_t shift) { + if (shift >= 128) { + ret = 0; + } else { + unsigned __int128 i = high; + i <<= 64; + i |= low; + i <<= shift; + ret = (__int128)i; + } + } + + void __lshrti3(__int128& ret, uint64_t low, uint64_t high, uint32_t shift) { + unsigned __int128 i = high; + i <<= 64; + i |= low; + i >>= shift; + ret = (unsigned __int128)i; + } + + void __divti3(__int128& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + __int128 lhs = ha; + __int128 rhs = hb; + + lhs <<= 64; + lhs |= la; + + rhs <<= 64; + rhs |= lb; + + if (rhs == 0) + throw std::runtime_error("divide by zero"); + + lhs /= rhs; + + ret = lhs; + } + + void __udivti3(unsigned __int128& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + unsigned __int128 lhs = ha; + unsigned __int128 rhs = hb; + + lhs <<= 64; + lhs |= la; + + rhs <<= 64; + rhs |= lb; + + if (rhs == 0) + throw std::runtime_error("divide by zero"); + + lhs /= rhs; + ret = lhs; + } + + void __modti3(__int128& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + __int128 lhs = ha; + __int128 rhs = hb; + + lhs <<= 64; + lhs |= la; + + rhs <<= 64; + rhs |= lb; + + if (rhs == 0) + throw std::runtime_error("divide by zero"); + + lhs %= rhs; + ret = lhs; + } + + void __umodti3(unsigned __int128& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + unsigned __int128 lhs = ha; + unsigned __int128 rhs = hb; + + lhs <<= 64; + lhs |= la; + + rhs <<= 64; + rhs |= lb; + + if (rhs == 0) + throw std::runtime_error("divide by zero"); + + lhs %= rhs; + ret = lhs; + } + + void __multi3(__int128& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + __int128 lhs = ha; + __int128 rhs = hb; + + lhs <<= 64; + lhs |= la; + + rhs <<= 64; + rhs |= lb; + + lhs *= rhs; + ret = lhs; + } + + void __addtf3(float128_t& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + float128_t a = { { la, ha } }; + float128_t b = { { lb, hb } }; + ret = f128_add(a, b); + } + + void __subtf3(float128_t& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + float128_t a = { { la, ha } }; + float128_t b = { { lb, hb } }; + ret = f128_sub(a, b); + } + + void __multf3(float128_t& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + float128_t a = { { la, ha } }; + float128_t b = { { lb, hb } }; + ret = f128_mul(a, b); + } + + void __divtf3(float128_t& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + float128_t a = { { la, ha } }; + float128_t b = { { lb, hb } }; + ret = f128_div(a, b); + } + + int __unordtf2(uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + float128_t a = { { la, ha } }; + float128_t b = { { lb, hb } }; + if (f128_is_nan(a) || f128_is_nan(b)) + return 1; + return 0; + } + + int ___cmptf2(uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb, int return_value_if_nan) { + float128_t a = { { la, ha } }; + float128_t b = { { lb, hb } }; + if (__unordtf2(la, ha, lb, hb)) + return return_value_if_nan; + if (f128_lt(a, b)) + return -1; + if (f128_eq(a, b)) + return 0; + return 1; + } + + int __eqtf2(uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { return ___cmptf2(la, ha, lb, hb, 1); } + + int __netf2(uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { return ___cmptf2(la, ha, lb, hb, 1); } + + int __getf2(uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { return ___cmptf2(la, ha, lb, hb, -1); } + + int __gttf2(uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { return ___cmptf2(la, ha, lb, hb, 0); } + + int __letf2(uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { return ___cmptf2(la, ha, lb, hb, 1); } + + int __lttf2(uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { return ___cmptf2(la, ha, lb, hb, 0); } + + int __cmptf2(uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { return ___cmptf2(la, ha, lb, hb, 1); } + + void __negtf2(float128_t& ret, uint64_t la, uint64_t ha) { ret = { { la, (ha ^ (uint64_t)1 << 63) } }; } + + void __floatsitf(float128_t& ret, int32_t i) { ret = i32_to_f128(i); } + + void __floatditf(float128_t& ret, uint64_t a) { ret = i64_to_f128(a); } + + void __floatunsitf(float128_t& ret, uint32_t i) { ret = ui32_to_f128(i); } + + void __floatunditf(float128_t& ret, uint64_t a) { ret = ui64_to_f128(a); } + + double __floattidf(uint64_t l, uint64_t h) { + unsigned __int128 val = h; + val <<= 64; + val |= l; + return ___floattidf(*(__int128*)&val); + } + + double __floatuntidf(uint64_t l, uint64_t h) { + unsigned __int128 val = h; + val <<= 64; + val |= l; + return ___floatuntidf((unsigned __int128)val); + } + + double __floatsidf(int32_t i) { return from_softfloat64(i32_to_f64(i)); } + + void __extendsftf2(float128_t& ret, float f) { ret = f32_to_f128(to_softfloat32(f)); } + + void __extenddftf2(float128_t& ret, double d) { ret = f64_to_f128(to_softfloat64(d)); } + + void __fixtfti(__int128& ret, uint64_t l, uint64_t h) { + float128_t f = { { l, h } }; + ret = ___fixtfti(f); + } + + int32_t __fixtfsi(uint64_t l, uint64_t h) { + float128_t f = { { l, h } }; + return f128_to_i32(f, 0, false); + } + + int64_t __fixtfdi(uint64_t l, uint64_t h) { + float128_t f = { { l, h } }; + return f128_to_i64(f, 0, false); + } + + void __fixunstfti(unsigned __int128& ret, uint64_t l, uint64_t h) { + float128_t f = { { l, h } }; + ret = ___fixunstfti(f); + } + + uint32_t __fixunstfsi(uint64_t l, uint64_t h) { + float128_t f = { { l, h } }; + return f128_to_ui32(f, 0, false); + } + + uint64_t __fixunstfdi(uint64_t l, uint64_t h) { + float128_t f = { { l, h } }; + return f128_to_ui64(f, 0, false); + } + + void __fixsfti(__int128& ret, float a) { ret = ___fixsfti(to_softfloat32(a).v); } + + void __fixdfti(__int128& ret, double a) { ret = ___fixdfti(to_softfloat64(a).v); } + + void __fixunssfti(unsigned __int128& ret, float a) { ret = ___fixunssfti(to_softfloat32(a).v); } + + void __fixunsdfti(unsigned __int128& ret, double a) { ret = ___fixunsdfti(to_softfloat64(a).v); } + + double __trunctfdf2(uint64_t l, uint64_t h) { + float128_t f = { { l, h } }; + return from_softfloat64(f128_to_f64(f)); + } + + float __trunctfsf2(uint64_t l, uint64_t h) { + float128_t f = { { l, h } }; + return from_softfloat32(f128_to_f32(f)); + } + + template + static void register_callbacks() { + Rft::template add("env", "__ashlti3"); + Rft::template add("env", "__ashrti3"); + Rft::template add("env", "__lshlti3"); + Rft::template add("env", "__lshrti3"); + Rft::template add("env", "__divti3"); + Rft::template add("env", "__udivti3"); + Rft::template add("env", "__modti3"); + Rft::template add("env", "__umodti3"); + Rft::template add("env", "__multi3"); + Rft::template add("env", "__addtf3"); + Rft::template add("env", "__subtf3"); + Rft::template add("env", "__multf3"); + Rft::template add("env", "__divtf3"); + Rft::template add("env", "__eqtf2"); + Rft::template add("env", "__netf2"); + Rft::template add("env", "__getf2"); + Rft::template add("env", "__gttf2"); + Rft::template add("env", "__lttf2"); + Rft::template add("env", "__letf2"); + Rft::template add("env", "__cmptf2"); + Rft::template add("env", "__unordtf2"); + Rft::template add("env", "__negtf2"); + Rft::template add("env", "__floatsitf"); + Rft::template add("env", "__floatunsitf"); + Rft::template add("env", "__floatditf"); + Rft::template add("env", "__floatunditf"); + Rft::template add("env", "__floattidf"); + Rft::template add("env", "__floatuntidf"); + Rft::template add("env", "__floatsidf"); + Rft::template add("env", "__extendsftf2"); + Rft::template add("env", "__extenddftf2"); + Rft::template add("env", "__fixtfti"); + Rft::template add("env", "__fixtfdi"); + Rft::template add("env", "__fixtfsi"); + Rft::template add("env", "__fixunstfti"); + Rft::template add("env", "__fixunstfdi"); + Rft::template add("env", "__fixunstfsi"); + Rft::template add("env", "__fixsfti"); + Rft::template add("env", "__fixdfti"); + Rft::template add("env", "__fixunssfti"); + Rft::template add("env", "__fixunsdfti"); + Rft::template add("env", "__trunctfdf2"); + Rft::template add("env", "__trunctfsf2"); + } +}; + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/console.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/console.hpp new file mode 100644 index 00000000000..e95b5889d11 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/console.hpp @@ -0,0 +1,108 @@ +#pragma once + +#include +#include + +namespace b1::rodeos { + +struct console_state { + uint32_t max_console_size = 0; + std::string console = {}; +}; + +template +struct console_callbacks { + Derived& derived() { return static_cast(*this); } + + void append_console(const char* str, uint32_t len) { + auto& state = derived().get_state(); + state.console.append(str, std::min(size_t(len), state.max_console_size - state.console.size())); + } + + void prints(const char* str) { + auto len = derived().check_bounds_get_len(str); + append_console(str, len); + } + + void prints_l(const char* str, uint32_t len) { + derived().check_bounds(str, len); + append_console(str, len); + } + + void printi(int64_t value) { + auto& state = derived().get_state(); + if (!state.max_console_size) + return; + auto s = std::to_string(value); + append_console(s.c_str(), s.size()); + } + + void printui(uint64_t value) { + auto& state = derived().get_state(); + if (!state.max_console_size) + return; + auto s = std::to_string(value); + append_console(s.c_str(), s.size()); + } + + void printi128(void* value) { + // todo + append_console("{unimplemented}", 15); + } + + void printui128(void* value) { + // todo + append_console("{unimplemented}", 15); + } + + void printsf(float value) { printdf(value); } + + void printdf(double value) { + auto& state = derived().get_state(); + if (!state.max_console_size) + return; + auto s = std::to_string(value); + append_console(s.c_str(), s.size()); + } + + void printqf(void*) { + // don't bother + append_console("{unimplemented}", 15); + } + + void printn(uint64_t value) { + auto& state = derived().get_state(); + if (!state.max_console_size) + return; + auto s = std::to_string(value); + append_console(s.c_str(), s.size()); + } + + void printhex(const char* data, uint32_t len) { + derived().check_bounds(data, len); + auto& state = derived().get_state(); + for (uint32_t i = 0; i < len && state.console.size() + 2 < state.max_console_size; ++i) { + static const char hex_digits[] = "0123456789ABCDEF"; + uint8_t byte = data[i]; + state.console.push_back(hex_digits[byte >> 4]); + state.console.push_back(hex_digits[byte & 15]); + } + } + + template + static void register_callbacks() { + Rft::template add("env", "prints"); + Rft::template add("env", "prints_l"); + Rft::template add("env", "printi"); + Rft::template add("env", "printui"); + Rft::template add("env", "printi128"); + Rft::template add("env", "printui128"); + Rft::template add("env", "printsf"); + Rft::template add("env", "printdf"); + Rft::template add("env", "printqf"); + Rft::template add("env", "printn"); + Rft::template add("env", "printhex"); + } +}; // console_callbacks + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/filter.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/filter.hpp new file mode 100644 index 00000000000..39455974519 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/filter.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include + +namespace b1::rodeos { + +struct filter_callback_state { + std::function push_data; +}; + +template +struct filter_callbacks { + Derived& derived() { return static_cast(*this); } + + void push_data(const char* data, uint32_t size) { + derived().check_bounds(data, size); + derived().get_filter_callback_state().push_data(data, size); + } + + template + static void register_callbacks() { + Rft::template add("env", "push_data"); + } +}; // query_callbacks + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/kv.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/kv.hpp new file mode 100644 index 00000000000..bd21ebdbee1 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/kv.hpp @@ -0,0 +1,378 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace b1::rodeos { + +inline constexpr eosio::name kvram_db_id{ "eosio.kvram" }; +inline constexpr eosio::name kvdisk_db_id{ "eosio.kvdisk" }; +inline constexpr eosio::name state_db_id{ "eosio.state" }; + +enum class kv_it_stat { + iterator_ok = 0, // Iterator is positioned at a key-value pair + iterator_erased = -1, // The key-value pair that the iterator used to be positioned at was erased + iterator_end = -2, // Iterator is out-of-bounds +}; + +struct kv_iterator_rocksdb { + uint32_t& num_iterators; + chain_kv::view& view; + uint64_t contract; + chain_kv::view::iterator kv_it; + + kv_iterator_rocksdb(uint32_t& num_iterators, chain_kv::view& view, uint64_t contract, const char* prefix, + uint32_t size) + : num_iterators(num_iterators), view{ view }, contract{ contract }, kv_it{ view, contract, { prefix, size } } { + ++num_iterators; + } + + ~kv_iterator_rocksdb() { --num_iterators; } + + bool is_kv_chainbase_context_iterator() const { return false; } + bool is_kv_rocksdb_context_iterator() const { return true; } + + kv_it_stat kv_it_status() { + if (kv_it.is_end()) + return kv_it_stat::iterator_end; + else if (kv_it.is_erased()) + return kv_it_stat::iterator_erased; + else + return kv_it_stat::iterator_ok; + } + + int32_t kv_it_compare(const kv_iterator_rocksdb& rhs) { + eosio::check(rhs.is_kv_rocksdb_context_iterator(), "Incompatible key-value iterators"); + auto& r = static_cast(rhs); + eosio::check(&view == &r.view && contract == r.contract, "Incompatible key-value iterators"); + eosio::check(!kv_it.is_erased(), "Iterator to erased element"); + eosio::check(!r.kv_it.is_erased(), "Iterator to erased element"); + return compare(kv_it, r.kv_it); + } + + int32_t kv_it_key_compare(const char* key, uint32_t size) { + eosio::check(!kv_it.is_erased(), "Iterator to erased element"); + return chain_kv::compare_key(kv_it.get_kv(), chain_kv::key_value{ { key, size }, {} }); + } + + kv_it_stat kv_it_move_to_end() { + kv_it.move_to_end(); + return kv_it_stat::iterator_end; + } + + kv_it_stat kv_it_next() { + eosio::check(!kv_it.is_erased(), "Iterator to erased element"); + ++kv_it; + return kv_it_status(); + } + + kv_it_stat kv_it_prev() { + eosio::check(!kv_it.is_erased(), "Iterator to erased element"); + --kv_it; + return kv_it_status(); + } + + kv_it_stat kv_it_lower_bound(const char* key, uint32_t size) { + kv_it.lower_bound(key, size); + return kv_it_status(); + } + + kv_it_stat kv_it_key(uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) { + eosio::check(!kv_it.is_erased(), "Iterator to erased element"); + + std::optional kv; + kv = kv_it.get_kv(); + + if (kv) { + if (offset < kv->key.size()) + memcpy(dest, kv->key.data() + offset, std::min((size_t)size, kv->key.size() - offset)); + actual_size = kv->key.size(); + return kv_it_stat::iterator_ok; + } else { + actual_size = 0; + return kv_it_stat::iterator_end; + } + } + + kv_it_stat kv_it_value(uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) { + eosio::check(!kv_it.is_erased(), "Iterator to erased element"); + + std::optional kv; + kv = kv_it.get_kv(); + + if (kv) { + if (offset < kv->value.size()) + memcpy(dest, kv->value.data() + offset, std::min((size_t)size, kv->value.size() - offset)); + actual_size = kv->value.size(); + return kv_it_stat::iterator_ok; + } else { + actual_size = 0; + return kv_it_stat::iterator_end; + } + } +}; // kv_iterator_rocksdb + +struct kv_database_config { + std::uint32_t max_key_size = 1024; + std::uint32_t max_value_size = 256 * 1024; // Large enough to hold most contracts + std::uint32_t max_iterators = 1024; +}; + +struct kv_context_rocksdb { + chain_kv::database& database; + chain_kv::write_session& write_session; + std::vector contract_kv_prefix; + eosio::name database_id; + chain_kv::view view; + bool enable_write = false; + bool bypass_receiver_check = false; + eosio::name receiver; + const kv_database_config& limits; + uint32_t num_iterators = 0; + std::shared_ptr> temp_data_buffer; + + kv_context_rocksdb(chain_kv::database& database, chain_kv::write_session& write_session, + std::vector contract_kv_prefix, eosio::name database_id, eosio::name receiver, + const kv_database_config& limits) + : database{ database }, write_session{ write_session }, contract_kv_prefix{ std::move(contract_kv_prefix) }, + database_id{ database_id }, view{ write_session, make_prefix() }, receiver{ receiver }, limits{ limits } {} + + std::vector make_prefix() { + std::vector prefix = contract_kv_prefix; + chain_kv::append_key(prefix, database_id.value); + return prefix; + } + + int64_t kv_erase(uint64_t contract, const char* key, uint32_t key_size) { + eosio::check(enable_write && (bypass_receiver_check || eosio::name{ contract } == receiver), + "Can not write to this key"); + temp_data_buffer = nullptr; + view.erase(contract, { key, key_size }); + return 0; + } + + int64_t kv_set(uint64_t contract, const char* key, uint32_t key_size, const char* value, uint32_t value_size) { + eosio::check(enable_write && (bypass_receiver_check || eosio::name{ contract } == receiver), + "Can not write to this key"); + eosio::check(key_size <= limits.max_key_size, "Key too large"); + eosio::check(value_size <= limits.max_value_size, "Value too large"); + temp_data_buffer = nullptr; + view.set(contract, { key, key_size }, { value, value_size }); + return 0; + } + + bool kv_get(uint64_t contract, const char* key, uint32_t key_size, uint32_t& value_size) { + temp_data_buffer = view.get(contract, { key, key_size }); + if (temp_data_buffer) { + value_size = temp_data_buffer->size(); + return true; + } else { + value_size = 0; + return false; + } + } + + uint32_t kv_get_data(uint32_t offset, char* data, uint32_t data_size) { + const char* temp = nullptr; + uint32_t temp_size = 0; + if (temp_data_buffer) { + temp = temp_data_buffer->data(); + temp_size = temp_data_buffer->size(); + } + if (offset < temp_size) + memcpy(data, temp + offset, std::min(data_size, temp_size - offset)); + return temp_size; + } + + std::unique_ptr kv_it_create(uint64_t contract, const char* prefix, uint32_t size) { + eosio::check(num_iterators < limits.max_iterators, "Too many iterators"); + return std::make_unique(num_iterators, view, contract, prefix, size); + } +}; // kv_context_rocksdb + +struct db_view_state { + eosio::name receiver; + chain_kv::database& database; + const kv_database_config limits; + const kv_database_config kv_state_limits{ 1024, std::numeric_limits::max() }; + kv_context_rocksdb kv_ram; + kv_context_rocksdb kv_disk; + kv_context_rocksdb kv_state; + std::vector> kv_iterators; + std::vector kv_destroyed_iterators; + + db_view_state(eosio::name receiver, chain_kv::database& database, chain_kv::write_session& write_session, + const std::vector& contract_kv_prefix) + : receiver{ receiver }, database{ database }, // + kv_ram{ database, write_session, contract_kv_prefix, kvram_db_id, receiver, limits }, + kv_disk{ database, write_session, contract_kv_prefix, kvdisk_db_id, receiver, limits }, + kv_state{ database, write_session, contract_kv_prefix, state_db_id, receiver, kv_state_limits }, + kv_iterators(1) {} + + void reset() { + eosio::check(kv_iterators.size() == kv_destroyed_iterators.size() + 1, "iterators are still alive"); + kv_iterators.resize(1); + kv_destroyed_iterators.clear(); + } +}; + +template +struct db_callbacks { + Derived& derived() { return static_cast(*this); } + + int64_t kv_erase(uint64_t db, uint64_t contract, const char* key, uint32_t key_size) { + derived().check_bounds(key, key_size); + return kv_get_db(db).kv_erase(contract, key, key_size); + } + + int64_t kv_set(uint64_t db, uint64_t contract, const char* key, uint32_t key_size, const char* value, + uint32_t value_size) { + derived().check_bounds(key, key_size); + derived().check_bounds(value, value_size); + return kv_get_db(db).kv_set(contract, key, key_size, value, value_size); + } + + bool kv_get(uint64_t db, uint64_t contract, const char* key, uint32_t key_size, uint32_t& value_size) { + derived().check_bounds(key, key_size); + return kv_get_db(db).kv_get(contract, key, key_size, value_size); + } + + uint32_t kv_get_data(uint64_t db, uint32_t offset, char* data, uint32_t data_size) { + derived().check_bounds(data, data_size); + return kv_get_db(db).kv_get_data(offset, data, data_size); + } + + uint32_t kv_it_create(uint64_t db, uint64_t contract, const char* prefix, uint32_t size) { + derived().check_bounds(prefix, size); + auto& kdb = kv_get_db(db); + uint32_t itr; + if (!derived().get_db_view_state().kv_destroyed_iterators.empty()) { + itr = derived().get_db_view_state().kv_destroyed_iterators.back(); + derived().get_db_view_state().kv_destroyed_iterators.pop_back(); + } else { + // Sanity check in case the per-database limits are set poorly + eosio::check(derived().get_db_view_state().kv_iterators.size() <= 0xFFFFFFFFu, "Too many iterators"); + itr = derived().get_db_view_state().kv_iterators.size(); + derived().get_db_view_state().kv_iterators.emplace_back(); + } + derived().get_db_view_state().kv_iterators[itr] = kdb.kv_it_create(contract, prefix, size); + return itr; + } + + void kv_it_destroy(uint32_t itr) { + kv_check_iterator(itr); + derived().get_db_view_state().kv_destroyed_iterators.push_back(itr); + derived().get_db_view_state().kv_iterators[itr].reset(); + } + + int32_t kv_it_status(uint32_t itr) { + kv_check_iterator(itr); + return static_cast(derived().get_db_view_state().kv_iterators[itr]->kv_it_status()); + } + + int32_t kv_it_compare(uint32_t itr_a, uint32_t itr_b) { + kv_check_iterator(itr_a); + kv_check_iterator(itr_b); + return derived().get_db_view_state().kv_iterators[itr_a]->kv_it_compare( + *derived().get_db_view_state().kv_iterators[itr_b]); + } + + int32_t kv_it_key_compare(uint32_t itr, const char* key, uint32_t size) { + derived().check_bounds(key, size); + kv_check_iterator(itr); + return derived().get_db_view_state().kv_iterators[itr]->kv_it_key_compare(key, size); + } + + int32_t kv_it_move_to_end(uint32_t itr) { + kv_check_iterator(itr); + return static_cast(derived().get_db_view_state().kv_iterators[itr]->kv_it_move_to_end()); + } + + int32_t kv_it_next(uint32_t itr) { + kv_check_iterator(itr); + return static_cast(derived().get_db_view_state().kv_iterators[itr]->kv_it_next()); + } + + int32_t kv_it_prev(uint32_t itr) { + kv_check_iterator(itr); + return static_cast(derived().get_db_view_state().kv_iterators[itr]->kv_it_prev()); + } + + int32_t kv_it_lower_bound(uint32_t itr, const char* key, uint32_t size) { + derived().check_bounds(key, size); + kv_check_iterator(itr); + return static_cast(derived().get_db_view_state().kv_iterators[itr]->kv_it_lower_bound(key, size)); + } + + int32_t kv_it_key(uint32_t itr, uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) { + derived().check_bounds(dest, size); + kv_check_iterator(itr); + return static_cast( + derived().get_db_view_state().kv_iterators[itr]->kv_it_key(offset, dest, size, actual_size)); + } + + int32_t kv_it_value(uint32_t itr, uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) { + derived().check_bounds(dest, size); + kv_check_iterator(itr); + return static_cast( + derived().get_db_view_state().kv_iterators[itr]->kv_it_value(offset, dest, size, actual_size)); + } + + kv_context_rocksdb& kv_get_db(uint64_t db) { + if (db == kvram_db_id.value) + return derived().get_db_view_state().kv_ram; + else if (db == kvdisk_db_id.value) + return derived().get_db_view_state().kv_disk; + else if (db == state_db_id.value) + return derived().get_db_view_state().kv_state; + throw std::runtime_error("Bad key-value database ID"); + } + + void kv_check_iterator(uint32_t itr) { + eosio::check(itr < derived().get_db_view_state().kv_iterators.size() && + derived().get_db_view_state().kv_iterators[itr], + "Bad key-value iterator"); + } + + template + static void register_callbacks() { + Rft::template add("env", "kv_erase"); + Rft::template add("env", "kv_set"); + Rft::template add("env", "kv_get"); + Rft::template add("env", "kv_get_data"); + Rft::template add("env", "kv_it_create"); + Rft::template add("env", "kv_it_destroy"); + Rft::template add("env", "kv_it_status"); + Rft::template add("env", "kv_it_compare"); + Rft::template add("env", "kv_it_key_compare"); + Rft::template add("env", "kv_it_move_to_end"); + Rft::template add("env", "kv_it_next"); + Rft::template add("env", "kv_it_prev"); + Rft::template add("env", "kv_it_lower_bound"); + Rft::template add("env", "kv_it_key"); + Rft::template add("env", "kv_it_value"); + } +}; // db_callbacks + +class kv_environment : public db_callbacks { + public: + using base = db_callbacks; + db_view_state& state; + + kv_environment(db_view_state& state) : state{ state } {} + kv_environment(const kv_environment&) = default; + + auto& get_db_view_state() { return state; } + void check_bounds(const char*, uint32_t) {} + + using base::kv_set; + void kv_set(uint64_t db, uint64_t contract, const std::vector& k, const std::vector& v) { + base::kv_set(db, contract, k.data(), k.size(), v.data(), v.size()); + } +}; + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/memory.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/memory.hpp new file mode 100644 index 00000000000..94c58e996f0 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/memory.hpp @@ -0,0 +1,50 @@ +#pragma once + +#include + +namespace b1::rodeos { + +template +struct memory_callbacks { + Derived& derived() { return static_cast(*this); } + + char* memcpy(char* dest, const char* src, uint32_t length) { + derived().check_bounds(dest, length); + derived().check_bounds(src, length); + if ((size_t)(std::abs((ptrdiff_t)dest - (ptrdiff_t)src)) < length) + throw std::runtime_error("memcpy can only accept non-aliasing pointers"); + return (char*)::memcpy(dest, src, length); + } + + char* memmove(char* dest, const char* src, uint32_t length) { + derived().check_bounds(dest, length); + derived().check_bounds(src, length); + return (char*)::memmove(dest, src, length); + } + + int memcmp(const char* dest, const char* src, uint32_t length) { + derived().check_bounds(dest, length); + derived().check_bounds(src, length); + int ret = ::memcmp(dest, src, length); + if (ret < 0) + return -1; + if (ret > 0) + return 1; + return 0; + } + + char* memset(char* dest, int value, uint32_t length) { + derived().check_bounds(dest, length); + return (char*)::memset(dest, value, length); + } + + template + static void register_callbacks() { + Rft::template add("env", "memcpy"); + Rft::template add("env", "memmove"); + Rft::template add("env", "memcmp"); + Rft::template add("env", "memset"); + } +}; // memory_callbacks + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/query.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/query.hpp new file mode 100644 index 00000000000..9a0162686a4 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/query.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include +#include + +namespace b1::rodeos { + +struct query_state { + uint32_t block_num; + std::optional block_info; +}; + +template +struct query_callbacks { + Derived& derived() { return static_cast(*this); } + + void load_block_info() { + auto& state = derived().get_state(); + if (state.block_info) + return; + auto info = get_state_row( + derived().get_db_view_state().kv_state.view, + std::make_tuple(eosio::name{ "block.info" }, eosio::name{ "primary" }, state.block_num)); + if (!info) + throw std::runtime_error("database is missing block.info for block " + std::to_string(state.block_num)); + state.block_info = info->second; + } + + int64_t current_time() { + load_block_info(); + return std::visit( + [](auto& b) { // + return b.timestamp.to_time_point().time_since_epoch().count(); + }, + *derived().get_state().block_info); + } + + template + static void register_callbacks() { + Rft::template add("env", "current_time"); + } +}; // query_callbacks + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented.hpp new file mode 100644 index 00000000000..228b9beff9b --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented.hpp @@ -0,0 +1,251 @@ +#pragma once + +#include + +namespace b1::rodeos { + +template +struct unimplemented_callbacks { + template + T unimplemented(const char* name) { + throw std::runtime_error("wasm called " + std::string(name) + ", which is unimplemented"); + } + + // privileged_api + int is_feature_active(int64_t) { return unimplemented("is_feature_active"); } + void activate_feature(int64_t) { return unimplemented("activate_feature"); } + void get_resource_limits(int64_t, int, int, int) { return unimplemented("get_resource_limits"); } + void set_resource_limits(int64_t, int64_t, int64_t, int64_t) { return unimplemented("set_resource_limits"); } + int64_t set_proposed_producers(int, int) { return unimplemented("set_proposed_producers"); } + int get_blockchain_parameters_packed(int, int) { return unimplemented("get_blockchain_parameters_packed"); } + void set_blockchain_parameters_packed(int, int) { return unimplemented("set_blockchain_parameters_packed"); } + int is_privileged(int64_t) { return unimplemented("is_privileged"); } + void set_privileged(int64_t, int) { return unimplemented("set_privileged"); } + void preactivate_feature(int) { return unimplemented("preactivate_feature"); } + + // producer_api + int get_active_producers(int, int) { return unimplemented("get_active_producers"); } + +#define DB_SECONDARY_INDEX_METHODS_SIMPLE(IDX) \ + int db_##IDX##_store(int64_t, int64_t, int64_t, int64_t, int) { return unimplemented("db_" #IDX "_store"); } \ + void db_##IDX##_remove(int) { return unimplemented("db_" #IDX "_remove"); } \ + void db_##IDX##_update(int, int64_t, int) { return unimplemented("db_" #IDX "_update"); } \ + int db_##IDX##_find_primary(int64_t, int64_t, int64_t, int, int64_t) { \ + return unimplemented("db_" #IDX "_find_primary"); \ + } \ + int db_##IDX##_find_secondary(int64_t, int64_t, int64_t, int, int) { \ + return unimplemented("db_" #IDX "_find_secondary"); \ + } \ + int db_##IDX##_lowerbound(int64_t, int64_t, int64_t, int, int) { \ + return unimplemented("db_" #IDX "_lowerbound"); \ + } \ + int db_##IDX##_upperbound(int64_t, int64_t, int64_t, int, int) { \ + return unimplemented("db_" #IDX "_upperbound"); \ + } \ + int db_##IDX##_end(int64_t, int64_t, int64_t) { return unimplemented("db_" #IDX "_end"); } \ + int db_##IDX##_next(int, int) { return unimplemented("db_" #IDX "_next"); } \ + int db_##IDX##_previous(int, int) { return unimplemented("db_" #IDX "_previous"); } + +#define DB_SECONDARY_INDEX_METHODS_ARRAY(IDX) \ + int db_##IDX##_store(int64_t, int64_t, int64_t, int64_t, int, int) { \ + return unimplemented("db_" #IDX "_store"); \ + } \ + void db_##IDX##_remove(int) { return unimplemented("db_" #IDX "_remove"); } \ + void db_##IDX##_update(int, int64_t, int, int) { return unimplemented("db_" #IDX "_update"); } \ + int db_##IDX##_find_primary(int64_t, int64_t, int64_t, int, int, int64_t) { \ + return unimplemented("db_" #IDX "_find_primary"); \ + } \ + int db_##IDX##_find_secondary(int64_t, int64_t, int64_t, int, int, int) { \ + return unimplemented("db_" #IDX "_find_secondary"); \ + } \ + int db_##IDX##_lowerbound(int64_t, int64_t, int64_t, int, int, int) { \ + return unimplemented("db_" #IDX "_lowerbound"); \ + } \ + int db_##IDX##_upperbound(int64_t, int64_t, int64_t, int, int, int) { \ + return unimplemented("db_" #IDX "_upperbound"); \ + } \ + int db_##IDX##_end(int64_t, int64_t, int64_t) { return unimplemented("db_" #IDX "_end"); } \ + int db_##IDX##_next(int, int) { return unimplemented("db_" #IDX "_next"); } \ + int db_##IDX##_previous(int, int) { return unimplemented("db_" #IDX "_previous"); } + + // database_api + DB_SECONDARY_INDEX_METHODS_SIMPLE(idx64) + DB_SECONDARY_INDEX_METHODS_SIMPLE(idx128) + DB_SECONDARY_INDEX_METHODS_ARRAY(idx256) + DB_SECONDARY_INDEX_METHODS_SIMPLE(idx_double) + DB_SECONDARY_INDEX_METHODS_SIMPLE(idx_long_double) + +#undef DB_SECONDARY_INDEX_METHODS_SIMPLE +#undef DB_SECONDARY_INDEX_METHODS_ARRAY + + // crypto_api + void assert_recover_key(int, int, int, int, int) { return unimplemented("assert_recover_key"); } + int recover_key(int, int, int, int, int) { return unimplemented("recover_key"); } + void assert_sha256(int, int, int) { return unimplemented("assert_sha256"); } + void assert_sha1(int, int, int) { return unimplemented("assert_sha1"); } + void assert_sha512(int, int, int) { return unimplemented("assert_sha512"); } + void assert_ripemd160(int, int, int) { return unimplemented("assert_ripemd160"); } + void sha1(int, int, int) { return unimplemented("sha1"); } + void sha256(int, int, int) { return unimplemented("sha256"); } + void sha512(int, int, int) { return unimplemented("sha512"); } + void ripemd160(int, int, int) { return unimplemented("ripemd160"); } + + // permission_api + int check_transaction_authorization(int, int, int, int, int, int) { + return unimplemented("check_transaction_authorization"); + } + int check_permission_authorization(int64_t, int64_t, int, int, int, int, int64_t) { + return unimplemented("check_permission_authorization"); + } + int64_t get_permission_last_used(int64_t, int64_t) { return unimplemented("get_permission_last_used"); } + int64_t get_account_creation_time(int64_t) { return unimplemented("get_account_creation_time"); } + + // system_api + int64_t publication_time() { return unimplemented("publication_time"); } + int is_feature_activated(int) { return unimplemented("is_feature_activated"); } + int64_t get_sender() { return unimplemented("get_sender"); } + + // context_free_system_api + void eosio_assert(uint32_t test, const char* msg) { + // todo: bounds check + // todo: move out of unimplemented_callbacks.hpp + if (test == 0) + throw std::runtime_error(std::string("assertion failed: ") + msg); + } + void eosio_assert_code(int, int64_t) { return unimplemented("eosio_assert_code"); } + void eosio_exit(int) { return unimplemented("eosio_exit"); } + + // authorization_api + void require_recipient(int64_t) { return unimplemented("require_recipient"); } + void require_auth(int64_t) { return unimplemented("require_auth"); } + void require_auth2(int64_t, int64_t) { return unimplemented("require_auth2"); } + int has_auth(int64_t) { return unimplemented("has_auth"); } + int is_account(int64_t) { return unimplemented("is_account"); } + + // context_free_transaction_api + int read_transaction(int, int) { return unimplemented("read_transaction"); } + int transaction_size() { return unimplemented("transaction_size"); } + int expiration() { return unimplemented("expiration"); } + int tapos_block_prefix() { return unimplemented("tapos_block_prefix"); } + int tapos_block_num() { return unimplemented("tapos_block_num"); } + int get_action(int, int, int, int) { return unimplemented("get_action"); } + + // transaction_api + void send_inline(int, int) { return unimplemented("send_inline"); } + void send_context_free_inline(int, int) { return unimplemented("send_context_free_inline"); } + void send_deferred(int, int64_t, int, int, int32_t) { return unimplemented("send_deferred"); } + int cancel_deferred(int) { return unimplemented("cancel_deferred"); } + + // context_free_api + int get_context_free_data(int, int, int) { return unimplemented("get_context_free_data"); } + + template + static void register_callbacks() { + // privileged_api + Rft::template add("env", "is_feature_active"); + Rft::template add("env", "activate_feature"); + Rft::template add("env", "get_resource_limits"); + Rft::template add("env", "set_resource_limits"); + Rft::template add("env", "set_proposed_producers"); + Rft::template add( + "env", "get_blockchain_parameters_packed"); + Rft::template add( + "env", "set_blockchain_parameters_packed"); + Rft::template add("env", "is_privileged"); + Rft::template add("env", "set_privileged"); + Rft::template add("env", "preactivate_feature"); + + // producer_api + Rft::template add("env", "get_active_producers"); + +#define DB_SECONDARY_INDEX_METHODS_SIMPLE(IDX) \ + Rft::template add("env", "db_" #IDX "_store"); \ + Rft::template add("env", "db_" #IDX "_remove"); \ + Rft::template add("env", "db_" #IDX "_update"); \ + Rft::template add("env", "db_" #IDX "_find_primary"); \ + Rft::template add("env", "db_" #IDX "_find_secondary"); \ + Rft::template add("env", "db_" #IDX "_lowerbound"); \ + Rft::template add("env", "db_" #IDX "_upperbound"); \ + Rft::template add("env", "db_" #IDX "_end"); \ + Rft::template add("env", "db_" #IDX "_next"); \ + Rft::template add("env", "db_" #IDX "_previous"); + +#define DB_SECONDARY_INDEX_METHODS_ARRAY(IDX) \ + Rft::template add("env", "db_" #IDX "_store"); \ + Rft::template add("env", "db_" #IDX "_remove"); \ + Rft::template add("env", "db_" #IDX "_update"); \ + Rft::template add("env", "db_" #IDX "_find_primary"); \ + Rft::template add("env", "db_" #IDX "_find_secondary"); \ + Rft::template add("env", "db_" #IDX "_lowerbound"); \ + Rft::template add("env", "db_" #IDX "_upperbound"); \ + Rft::template add("env", "db_" #IDX "_end"); \ + Rft::template add("env", "db_" #IDX "_next"); \ + Rft::template add("env", "db_" #IDX "_previous"); + + // database_api + DB_SECONDARY_INDEX_METHODS_SIMPLE(idx64) + DB_SECONDARY_INDEX_METHODS_SIMPLE(idx128) + DB_SECONDARY_INDEX_METHODS_ARRAY(idx256) + DB_SECONDARY_INDEX_METHODS_SIMPLE(idx_double) + DB_SECONDARY_INDEX_METHODS_SIMPLE(idx_long_double) + +#undef DB_SECONDARY_INDEX_METHODS_SIMPLE +#undef DB_SECONDARY_INDEX_METHODS_ARRAY + + // crypto_api + Rft::template add("env", "assert_recover_key"); + Rft::template add("env", "recover_key"); + Rft::template add("env", "assert_sha256"); + Rft::template add("env", "assert_sha1"); + Rft::template add("env", "assert_sha512"); + Rft::template add("env", "assert_ripemd160"); + Rft::template add("env", "sha1"); + Rft::template add("env", "sha256"); + Rft::template add("env", "sha512"); + Rft::template add("env", "ripemd160"); + + // permission_api + Rft::template add( + "env", "check_transaction_authorization"); + Rft::template add("env", + "check_permission_authorization"); + Rft::template add("env", "get_permission_last_used"); + Rft::template add("env", "get_account_creation_time"); + + // system_api + Rft::template add("env", "publication_time"); + Rft::template add("env", "is_feature_activated"); + Rft::template add("env", "get_sender"); + + // context_free_system_api + Rft::template add("env", "eosio_assert"); + Rft::template add("env", "eosio_assert_code"); + Rft::template add("env", "eosio_exit"); + + // authorization_api + Rft::template add("env", "require_recipient"); + Rft::template add("env", "require_auth"); + Rft::template add("env", "require_auth2"); + Rft::template add("env", "has_auth"); + Rft::template add("env", "is_account"); + + // context_free_transaction_api + Rft::template add("env", "read_transaction"); + Rft::template add("env", "transaction_size"); + Rft::template add("env", "expiration"); + Rft::template add("env", "tapos_block_prefix"); + Rft::template add("env", "tapos_block_num"); + Rft::template add("env", "get_action"); + + // transaction_api + Rft::template add("env", "send_inline"); + Rft::template add("env", "send_context_free_inline"); + Rft::template add("env", "send_deferred"); + Rft::template add("env", "cancel_deferred"); + + // context_free_api + Rft::template add("env", "get_context_free_data"); + } // register_callbacks() +}; // unimplemented_callbacks + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented_filter.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented_filter.hpp new file mode 100644 index 00000000000..3c754c16e6a --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented_filter.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include + +namespace b1::rodeos { + +template +struct unimplemented_filter_callbacks { + template + T unimplemented(const char* name) { + throw std::runtime_error("wasm called " + std::string(name) + ", which is unimplemented"); + } + + // system_api + int64_t current_time() { return unimplemented("current_time"); } + + template + static void register_callbacks() { + // system_api + Rft::template add("env", "current_time"); + } +}; // unimplemented_filter_callbacks + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/embedded_rodeos.h b/libraries/rodeos/include/b1/rodeos/embedded_rodeos.h new file mode 100644 index 00000000000..2fc58e399fe --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/embedded_rodeos.h @@ -0,0 +1,151 @@ +#pragma once + +#include "stdint.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct rodeos_error_s rodeos_error; +typedef struct rodeos_context_s rodeos_context; +typedef struct rodeos_db_partition_s rodeos_db_partition; +typedef struct rodeos_db_snapshot_s rodeos_db_snapshot; +typedef struct rodeos_filter_s rodeos_filter; +typedef struct rodeos_query_handler_s rodeos_query_handler; +typedef int rodeos_bool; + +// Create an error object. If multiple threads use an error object, then they must synchronize it. Returns NULL on +// failure. +rodeos_error* rodeos_create_error(); + +// Destroy an error object. This is a no-op if error == NULL. +void rodeos_destroy_error(rodeos_error* error); + +// Get last error in this object. Never returns NULL. The error object owns the returned string. +const char* rodeos_get_error(rodeos_error* error); + +// Create a context. Returns NULL on failure. +rodeos_context* rodeos_create(); + +// Destroy a context. It is undefined behavior if the context is used between threads without synchronization, or if any +// other objects currently exist for this context. This is a no-op if context == NULL. +void rodeos_destroy(rodeos_context* context); + +// Open database. num_threads is the target number of rocksdb background threads; use 0 for default. max_open_files is +// the max number of open rocksdb files; 0 to make this unlimited. +// +// It is undefined behavior if the context is used between threads without synchronization. Returns false on error. +rodeos_bool rodeos_open_db(rodeos_error* error, rodeos_context* context, const char* path, + rodeos_bool create_if_missing, int num_threads, int max_open_files); + +// Create or open a database partition. It is undefined behavior if more than 1 partition is opened for a given prefix, +// if any partitions have overlapping prefixes, or if the context is used between threads without synchronization. +// Returns NULL on failure. +rodeos_db_partition* rodeos_create_partition(rodeos_error* error, rodeos_context* context, const char* prefix, + uint32_t prefix_size); + +// Destroy a partition. It is undefined behavior if the partition is used between threads without synchronization. This +// is a no-op if partition == NULL. +void rodeos_destroy_partition(rodeos_db_partition* partition); + +// Create a database snapshot. Snapshots isolate changes from each other. All database reads and writes happen through +// snapshots. Snapshot objects may safely outlive partition objects. +// +// A single partition supports any number of simultaneous non-persistent snapshots, but only a single persistent +// snapshot at any time. persistent and non-persistent may coexist. Only persistent snapshots make permanent changes to +// the database. +// +// Each snapshot may be used by a different thread, even if they're created from a single partition. +// +// It is undefined behavior if more than 1 persistent snapshot exists on a partition, or if the partition is used +// between threads without synchronization. Returns NULL on failure. +rodeos_db_snapshot* rodeos_create_snapshot(rodeos_error* error, rodeos_db_partition* partition, rodeos_bool persistent); + +// Destroy a snapshot. It is undefined behavior if the snapshot is used between threads without synchronization. This is +// a no-op if snapshot == NULL. +void rodeos_destroy_snapshot(rodeos_db_snapshot* snapshot); + +// Refresh a snapshot so it may read recently-written database changes. This operation is invalid if the snapshot is +// persistent. It is undefined behavior if the snapshot is used between threads without synchronization. +rodeos_bool rodeos_refresh_snapshot(rodeos_error* error, rodeos_db_snapshot* snapshot); + +// Start writing a block. Aborts any block in progress and rolls back reversible blocks if needed. `data` must be the +// serialized `result` type defined by the state-history plugin's ABI. Currently only supports `get_blocks_result_v0`. +// It is undefined behavior if the snapshot is used between threads without synchronization. +rodeos_bool rodeos_start_block(rodeos_error* error, rodeos_db_snapshot* snapshot, const char* data, uint64_t size); + +// Finish writing a block. `data` must be the serialized `result` type defined by the state-history plugin's ABI. +// Currently only supports `get_blocks_result_v0`. If `force_write` is true, then the data will become immediately +// available to newly-created or newly-refreshed snapshots to read. If `force_write` is false, then the write may be +// delayed until a future rodeos_end_block call. It is undefined behavior if the snapshot is used between threads +// without synchronization. +rodeos_bool rodeos_end_block(rodeos_error* error, rodeos_db_snapshot* snapshot, const char* data, uint64_t size, + bool force_write); + +// Write block info. `data` must be the serialized `result` type defined by the state-history plugin's ABI. Currently +// only supports `get_blocks_result_v0`. If `rodeos_write_block_info` returns false, the snapshot will be in an +// inconsistent state; call `start_block` to abandon the current write and start another. It is undefined behavior if +// the snapshot is used between threads without synchronization. +rodeos_bool rodeos_write_block_info(rodeos_error* error, rodeos_db_snapshot* snapshot, const char* data, uint64_t size); + +// Write state-history deltas to a block. `data` must be the serialized `result` type defined by the state-history +// plugin's ABI. Currently only supports `get_blocks_result_v0`. If `shutdown` isn't null, then `rodeos_write_deltas` +// may call it during long operations. If `shutdown` returns true, then `rodeos_write_deltas` abandons the writes. If +// `rodeos_write_deltas` returns false, the snapshot will be in an inconsistent state; call `start_block` to abandon the +// current write and start another. It is undefined behavior if the snapshot is used between threads without +// synchronization. +rodeos_bool rodeos_write_deltas(rodeos_error* error, rodeos_db_snapshot* snapshot, const char* data, uint64_t size, + rodeos_bool (*shutdown)(void*), void* shutdown_arg); + +// Create a filter. Returns NULL on failure. +rodeos_filter* rodeos_create_filter(rodeos_error* error, uint64_t name, const char* wasm_filename); + +// Destroy a filter. It is undefined behavior if the filter is used between threads without synchronization. This is a +// no-op if filter == NULL. +void rodeos_destroy_filter(rodeos_filter* filter); + +// Run filter. data must be the serialized `result` type defined by the state-history plugin's ABI. The `push_data` +// callback receives data from the filter's `push_data` intrinsic. If `push_data` is null, then the intrinsic is a +// no-op. +// +// If `rodeos_run_filter` returns false, the snapshot will be in an inconsistent state; call `start_block` to abandon +// the current write and start another. It is undefined behavior if `snapshot` or `filter` is used between threads +// without synchronization. +rodeos_bool rodeos_run_filter(rodeos_error* error, rodeos_db_snapshot* snapshot, rodeos_filter* filter, + const char* data, uint64_t size, + rodeos_bool (*push_data)(void* arg, const char* data, uint64_t size), + void* push_data_arg); + +// Create a query handler. This object manages pools of resources for running queries simultaneously. +// +// Query handlers may safely outlive partition objects. It is undefined behavior if the partition is used between +// threads without synchronization. Returns NULL on failure. +// +// TODO: remove partition arg; redundant with snapshot in rodeos_query_transaction +rodeos_query_handler* rodeos_create_query_handler(rodeos_error* error, rodeos_db_partition* partition, + uint32_t max_console_size, uint32_t wasm_cache_size, + uint64_t max_exec_time_ms, const char* contract_dir); + +// Destroy a query handler. It is undefined behavior if the handler is used between threads without synchronization. +// This is a no-op if handler == NULL. +void rodeos_destroy_query_handler(rodeos_query_handler* handler); + +// Run a query. data is a serialized ship_protocol::packed_transaction. Returns false on serious error and sets *result +// to NULL and *result_size to 0. Otherwise, sets *result and *result_size to memory containing a serialized +// ship_protocol::transaction_trace. If the query failed, the error result will be in the transaction trace. Caller must +// use rodeos_free_result to free the memory. +// +// It is safe to use the same handler from multiple threads if: +// * The return from rodeos_create_query_handler happens-before any calls to rodeos_query_transaction +// * The return from all rodeos_query_transaction calls happens-before the call to rodeos_destroy_query_handler +// +// It is undefined behavior if `snapshot` is used between threads without synchronization. +rodeos_bool rodeos_query_transaction(rodeos_error* error, rodeos_query_handler* handler, rodeos_db_snapshot* snapshot, + const char* data, uint64_t size, char** result, uint64_t* result_size); + +// Frees memory from rodeos_query_transaction. Does nothing if result == NULL. +void rodeos_free_result(char* result); + +#ifdef __cplusplus +} +#endif diff --git a/libraries/rodeos/include/b1/rodeos/embedded_rodeos.hpp b/libraries/rodeos/include/b1/rodeos/embedded_rodeos.hpp new file mode 100644 index 00000000000..407b2513d3f --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/embedded_rodeos.hpp @@ -0,0 +1,198 @@ +#pragma once + +#include +#include +#include + +namespace b1::embedded_rodeos { + +struct error { + rodeos_error* obj; + + error() : obj{ rodeos_create_error() } { + if (!obj) + throw std::bad_alloc{}; + } + + error(const error&) = delete; + + ~error() { rodeos_destroy_error(obj); } + + error& operator=(const error&) = delete; + + operator rodeos_error*() { return obj; } + + const char* msg() { return rodeos_get_error(obj); } + + template + auto check(F f) -> decltype(f()) { + auto result = f(); + if (!result) + throw std::runtime_error(msg()); + return result; + } +}; + +struct context { + struct error error; + rodeos_context* obj; + + context() : obj{ rodeos_create() } { + if (!obj) + throw std::bad_alloc{}; + } + + context(const context&) = delete; + + ~context() { rodeos_destroy(obj); } + + context& operator=(const context&) = delete; + + operator rodeos_context*() { return obj; } + + void open_db(const char* path, bool create_if_missing, int num_threads = 0, int max_open_files = 0) { + error.check([&] { return rodeos_open_db(error, obj, path, create_if_missing, num_threads, max_open_files); }); + } +}; + +struct partition { + struct error error; + rodeos_db_partition* obj; + + partition(rodeos_context* context, const char* prefix, uint32_t prefix_size) { + obj = error.check([&] { return rodeos_create_partition(error, context, prefix, prefix_size); }); + } + + partition(const partition&) = delete; + + ~partition() { rodeos_destroy_partition(obj); } + + partition& operator=(const partition&) = delete; + + operator rodeos_db_partition*() { return obj; } +}; + +struct snapshot { + struct error error; + rodeos_db_snapshot* obj; + + snapshot(rodeos_db_partition* partition, bool persistent) { + obj = error.check([&] { return rodeos_create_snapshot(error, partition, persistent); }); + } + + snapshot(const snapshot&) = delete; + + ~snapshot() { rodeos_destroy_snapshot(obj); } + + snapshot& operator=(const snapshot&) = delete; + + operator rodeos_db_snapshot*() { return obj; } + + void refresh() { + error.check([&] { return rodeos_refresh_snapshot(error, obj); }); + } + + void start_block(const char* data, uint64_t size) { + error.check([&] { return rodeos_start_block(error, obj, data, size); }); + } + + void end_block(const char* data, uint64_t size, bool force_write) { + error.check([&] { return rodeos_end_block(error, obj, data, size, force_write); }); + } + + void write_block_info(const char* data, uint64_t size) { + error.check([&] { return rodeos_write_block_info(error, obj, data, size); }); + } + + template + void write_deltas(const char* data, uint64_t size, F shutdown) { + error.check([&] { + return rodeos_write_deltas( + error, obj, data, size, [](void* f) -> rodeos_bool { return (*static_cast(f))(); }, &shutdown); + }); + } +}; + +struct filter { + struct error error; + rodeos_filter* obj; + + filter(uint64_t name, const char* wasm_filename) { + obj = error.check([&] { return rodeos_create_filter(error, name, wasm_filename); }); + } + + filter(const filter&) = delete; + + ~filter() { rodeos_destroy_filter(obj); } + + filter& operator=(const filter&) = delete; + + operator rodeos_filter*() { return obj; } + + void run(rodeos_db_snapshot* snapshot, const char* data, uint64_t size) { + error.check([&] { return rodeos_run_filter(error, snapshot, obj, data, size, nullptr, nullptr); }); + } + + template + void run(rodeos_db_snapshot* snapshot, const char* data, uint64_t size, F push_data) { + error.check([&] { + return rodeos_run_filter( + error, snapshot, obj, data, size, + [](void* arg, const char* data, uint64_t size) -> rodeos_bool { + try { + return (*reinterpret_cast(arg))(data, size); + } catch (...) { return false; } + }, + &push_data); + }); + } +}; + +struct result { + char* data = {}; + uint64_t size = {}; + + result() = default; + result(const result&) = delete; + result(result&& src) { *this = std::move(src); } + ~result() { rodeos_free_result(data); } + + result& operator=(const result& src) = delete; + + result& operator=(result&& src) { + data = src.data; + size = src.size; + src.data = nullptr; + src.size = 0; + return *this; + } +}; + +struct query_handler { + struct error error; + rodeos_query_handler* obj; + + query_handler(rodeos_db_partition* partition, uint32_t max_console_size, uint32_t wasm_cache_size, + uint64_t max_exec_time_ms, const char* contract_dir) { + obj = error.check([&] { + return rodeos_create_query_handler(error, partition, max_console_size, wasm_cache_size, max_exec_time_ms, + contract_dir); + }); + } + + query_handler(const query_handler&) = delete; + + ~query_handler() { rodeos_destroy_query_handler(obj); } + + query_handler& operator=(const query_handler&) = delete; + + operator rodeos_query_handler*() { return obj; } + + result query_transaction(rodeos_db_snapshot* snapshot, const char* data, uint64_t size) { + result r; + error.check([&] { return rodeos_query_transaction(error, obj, snapshot, data, size, &r.data, &r.size); }); + return r; + } +}; + +} // namespace b1::embedded_rodeos diff --git a/libraries/rodeos/include/b1/rodeos/filter.hpp b/libraries/rodeos/include/b1/rodeos/filter.hpp new file mode 100644 index 00000000000..5dfed0649a2 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/filter.hpp @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +// todo: configure limits +// todo: timeout +namespace b1::rodeos::filter { + +struct callbacks; +using backend_t = eosio::vm::backend; +using rhf_t = eosio::vm::registered_host_functions; + +struct filter_state : b1::rodeos::data_state, b1::rodeos::console_state, b1::rodeos::filter_callback_state { + eosio::vm::wasm_allocator wa = {}; +}; + +// todo: remove basic_callbacks +struct callbacks : b1::rodeos::basic_callbacks, + b1::rodeos::chaindb_callbacks, + b1::rodeos::compiler_builtins_callbacks, + b1::rodeos::console_callbacks, + b1::rodeos::data_callbacks, + b1::rodeos::db_callbacks, + b1::rodeos::filter_callbacks, + b1::rodeos::memory_callbacks, + b1::rodeos::unimplemented_callbacks, + b1::rodeos::unimplemented_filter_callbacks { + filter::filter_state& filter_state; + b1::rodeos::chaindb_state& chaindb_state; + b1::rodeos::db_view_state& db_view_state; + + callbacks(filter::filter_state& filter_state, b1::rodeos::chaindb_state& chaindb_state, + b1::rodeos::db_view_state& db_view_state) + : filter_state{ filter_state }, chaindb_state{ chaindb_state }, db_view_state{ db_view_state } {} + + auto& get_state() { return filter_state; } + auto& get_filter_callback_state() { return filter_state; } + auto& get_chaindb_state() { return chaindb_state; } + auto& get_db_view_state() { return db_view_state; } +}; + +inline void register_callbacks() { + b1::rodeos::basic_callbacks::register_callbacks(); + b1::rodeos::chaindb_callbacks::register_callbacks(); + b1::rodeos::compiler_builtins_callbacks::register_callbacks(); + b1::rodeos::console_callbacks::register_callbacks(); + b1::rodeos::data_callbacks::register_callbacks(); + b1::rodeos::db_callbacks::register_callbacks(); + b1::rodeos::filter_callbacks::register_callbacks(); + b1::rodeos::memory_callbacks::register_callbacks(); + b1::rodeos::unimplemented_callbacks::register_callbacks(); + b1::rodeos::unimplemented_filter_callbacks::register_callbacks(); +} + +} // namespace b1::rodeos::filter diff --git a/libraries/rodeos/include/b1/rodeos/get_state_row.hpp b/libraries/rodeos/include/b1/rodeos/get_state_row.hpp new file mode 100644 index 00000000000..9821533ff83 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/get_state_row.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include +#include + +template +std::optional, T>> get_state_row(chain_kv::view& view, const K& key) { + std::optional, T>> result; + result.emplace(); + result->first = + view.get(eosio::name{ "state" }.value, chain_kv::to_slice(eosio::check(eosio::convert_to_key(key)).value())); + if (!result->first) { + result.reset(); + return result; + } + + eosio::input_stream stream{ *result->first }; + if (auto r = from_bin(result->second, stream); !r) + throw std::runtime_error("An error occurred deserializing state: " + r.error().message()); + return result; +} + +template +std::optional, T>> get_state_row_secondary(chain_kv::view& view, + const K& key) { + std::optional, T>> result; + auto pk = + view.get(eosio::name{ "state" }.value, chain_kv::to_slice(eosio::check(eosio::convert_to_key(key)).value())); + if (!pk) + return result; + + result.emplace(); + result->first = view.get(eosio::name{ "state" }.value, chain_kv::to_slice(*pk)); + if (!result->first) { + result.reset(); + return result; + } + + eosio::input_stream stream{ *result->first }; + if (auto r = from_bin(result->second, stream); !r) + throw std::runtime_error("An error occurred deserializing state: " + r.error().message()); + return result; +} diff --git a/libraries/rodeos/include/b1/rodeos/rodeos.hpp b/libraries/rodeos/include/b1/rodeos/rodeos.hpp new file mode 100644 index 00000000000..e0116a4b2fb --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/rodeos.hpp @@ -0,0 +1,87 @@ +#include +#include +#include +#include +#include + +namespace b1::rodeos { + +static constexpr char undo_prefix_byte = 0x01; +static constexpr char contract_kv_prefix_byte = 0x02; + +struct rodeos_context { + std::shared_ptr db; +}; + +struct rodeos_db_partition { + const std::shared_ptr db; + const std::vector undo_prefix; + const std::vector contract_kv_prefix; + + // todo: move rocksdb::ManagedSnapshot to here to prevent optimization in cloner from + // defeating non-persistent snapshots. + + rodeos_db_partition(std::shared_ptr db, const std::vector& prefix) + : db{ std::move(db) }, // + undo_prefix{ [&] { + auto x = prefix; + x.push_back(undo_prefix_byte); + return x; + }() }, + contract_kv_prefix{ [&] { + auto x = prefix; + x.push_back(contract_kv_prefix_byte); + return x; + }() } {} +}; + +struct rodeos_db_snapshot { + std::shared_ptr partition = {}; + std::shared_ptr db = {}; + std::optional undo_stack = {}; // only if persistent + std::optional snap = {}; // only if !persistent + std::optional write_session = {}; + eosio::checksum256 chain_id = {}; + uint32_t head = 0; + eosio::checksum256 head_id = {}; + uint32_t irreversible = 0; + eosio::checksum256 irreversible_id = {}; + uint32_t first = 0; + std::optional writing_block = {}; + + rodeos_db_snapshot(std::shared_ptr partition, bool persistent); + + void refresh(); + void end_write(bool write_fill); + void start_block(const ship_protocol::get_blocks_result_v0& result); + void end_block(const ship_protocol::get_blocks_result_v0& result, bool force_write); + void check_write(const ship_protocol::get_blocks_result_v0& result); + void write_block_info(const ship_protocol::get_blocks_result_v0& result); + void write_deltas(const ship_protocol::get_blocks_result_v0& result, std::function shutdown); + + private: + void write_fill_status(); +}; + +struct rodeos_filter { + eosio::name name = {}; + std::unique_ptr backend = {}; + std::unique_ptr filter_state = {}; + + rodeos_filter(eosio::name name, const std::string& wasm_filename); + + void process(rodeos_db_snapshot& snapshot, const ship_protocol::get_blocks_result_v0& result, + eosio::input_stream bin, const std::function& push_data); +}; + +struct rodeos_query_handler { + std::shared_ptr partition; + const std::shared_ptr shared_state; + wasm_ql::thread_state_cache state_cache; + + rodeos_query_handler(std::shared_ptr partition, + std::shared_ptr shared_state); + rodeos_query_handler(const rodeos_query_handler&) = delete; +}; + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp b/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp new file mode 100644 index 00000000000..424b634736d --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp @@ -0,0 +1,274 @@ +#pragma once + +#include +#include +#include + +namespace eosio { +using b1::rodeos::kv_environment; +} + +#include "../wasms/table.hpp" + +namespace b1::rodeos { + +struct fill_status_v0 { + eosio::checksum256 chain_id = {}; + uint32_t head = {}; + eosio::checksum256 head_id = {}; + uint32_t irreversible = {}; + eosio::checksum256 irreversible_id = {}; + uint32_t first = {}; +}; + +EOSIO_REFLECT(fill_status_v0, chain_id, head, head_id, irreversible, irreversible_id, first) + +using fill_status = std::variant; + +inline bool operator==(const fill_status_v0& a, fill_status_v0& b) { + return std::tie(a.head, a.head_id, a.irreversible, a.irreversible_id, a.first) == + std::tie(b.head, b.head_id, b.irreversible, b.irreversible_id, b.first); +} + +inline bool operator!=(const fill_status_v0& a, fill_status_v0& b) { return !(a == b); } + +struct fill_status_kv : eosio::table { + index primary_index{ eosio::name{ "primary" }, [](const auto& var) { return std::vector{}; } }; + + fill_status_kv(eosio::kv_environment environment) : eosio::table{ std::move(environment) } { + init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "fill.status" }, primary_index); + } +}; + +struct block_info_v0 { + uint32_t num = {}; + eosio::checksum256 id = {}; + eosio::block_timestamp timestamp = {}; + eosio::name producer = {}; + uint16_t confirmed = {}; + eosio::checksum256 previous = {}; + eosio::checksum256 transaction_mroot = {}; + eosio::checksum256 action_mroot = {}; + uint32_t schedule_version = {}; + std::optional new_producers = {}; + eosio::signature producer_signature = {}; +}; + +EOSIO_REFLECT(block_info_v0, num, id, timestamp, producer, confirmed, previous, transaction_mroot, action_mroot, + schedule_version, new_producers, producer_signature) + +using block_info = std::variant; + +// todo: move out of "state"? +struct block_info_kv : eosio::table { + index primary_index{ eosio::name{ "primary" }, [](const auto& var) { + return std::visit( + [](const auto& obj) { return eosio::check(eosio::convert_to_key(obj.num)).value(); }, + var); + } }; + + index id_index{ eosio::name{ "id" }, [](const auto& var) { + return std::visit( + [](const auto& obj) { return eosio::check(eosio::convert_to_key(obj.id)).value(); }, var); + } }; + + block_info_kv(eosio::kv_environment environment) : eosio::table{ std::move(environment) } { + init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "block.info" }, primary_index, id_index); + } +}; + +struct global_property_kv : eosio::table { + index primary_index{ eosio::name{ "primary" }, [](const auto& var) { return std::vector{}; } }; + + global_property_kv(eosio::kv_environment environment) : eosio::table{ std::move(environment) } { + init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "global.prop" }, primary_index); + } +}; + +struct account_kv : eosio::table { + index primary_index{ eosio::name{ "primary" }, [](const auto& var) { + return std::visit( + [](const auto& obj) { + return eosio::check(eosio::convert_to_key(std::tie(obj.name))).value(); + }, + var); + } }; + + account_kv(eosio::kv_environment environment) : eosio::table{ std::move(environment) } { + init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "account" }, primary_index); + } +}; + +struct account_metadata_kv : eosio::table { + index primary_index{ eosio::name{ "primary" }, [](const auto& var) { + return std::visit( + [](const auto& obj) { + return eosio::check(eosio::convert_to_key(std::tie(obj.name))).value(); + }, + var); + } }; + + account_metadata_kv(eosio::kv_environment environment) : eosio::table{ std::move(environment) } { + init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "account.meta" }, primary_index); + } +}; + +struct code_kv : eosio::table { + index primary_index{ eosio::name{ "primary" }, [](const auto& var) { + return std::visit( + [](const auto& obj) { + return eosio::check(eosio::convert_to_key( + std::tie(obj.vm_type, obj.vm_version, obj.code_hash))) + .value(); + }, + var); + } }; + + code_kv(eosio::kv_environment environment) : eosio::table{ std::move(environment) } { + init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "code" }, primary_index); + } +}; + +struct contract_table_kv : eosio::table { + index primary_index{ + eosio::name{ "primary" }, + [](const auto& var) { + return std::visit( + [](const auto& obj) { + return eosio::check(eosio::convert_to_key(std::tie(obj.code, obj.table, obj.scope))).value(); + }, + var); + } + }; + + contract_table_kv(eosio::kv_environment environment) : eosio::table{ std::move(environment) } { + init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "contract.tab" }, primary_index); + } +}; + +struct contract_row_kv : eosio::table { + index primary_index{ eosio::name{ "primary" }, [](const auto& var) { + return std::visit( + [](const auto& obj) { + return eosio::check(eosio::convert_to_key( + std::tie(obj.code, obj.table, obj.scope, obj.primary_key))) + .value(); + }, + var); + } }; + + contract_row_kv(eosio::kv_environment environment) : eosio::table{ std::move(environment) } { + init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "contract.row" }, primary_index); + } +}; + +struct contract_index64_kv : eosio::table { + index primary_index{ eosio::name{ "primary" }, [](const auto& var) { + return std::visit( + [](const auto& obj) { + return eosio::check(eosio::convert_to_key( + std::tie(obj.code, obj.table, obj.scope, obj.primary_key))) + .value(); + }, + var); + } }; + index secondary_index{ eosio::name{ "secondary" }, [](const auto& var) { + return std::visit( + [](const auto& obj) { + return eosio::check( + eosio::convert_to_key(std::tie(obj.code, obj.table, obj.scope, + obj.secondary_key, obj.primary_key))) + .value(); + }, + var); + } }; + + contract_index64_kv(eosio::kv_environment environment) : eosio::table{ std::move(environment) } { + init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "contract.i1" }, primary_index, + secondary_index); + } +}; + +struct contract_index128_kv : eosio::table { + index primary_index{ eosio::name{ "primary" }, [](const auto& var) { + return std::visit( + [](const auto& obj) { + return eosio::check(eosio::convert_to_key( + std::tie(obj.code, obj.table, obj.scope, obj.primary_key))) + .value(); + }, + var); + } }; + index secondary_index{ eosio::name{ "secondary" }, [](const auto& var) { + return std::visit( + [](const auto& obj) { + return eosio::check( + eosio::convert_to_key(std::tie(obj.code, obj.table, obj.scope, + obj.secondary_key, obj.primary_key))) + .value(); + }, + var); + } }; + + contract_index128_kv(eosio::kv_environment environment) : eosio::table{ std::move(environment) } { + init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "contract.i2" }, primary_index, + secondary_index); + } +}; + +template +void store_delta_typed(eosio::kv_environment environment, table_delta_v0& delta, bool bypass_preexist_check, F f) { + Table table{ environment }; + for (auto& row : delta.rows) { + f(); + auto obj = eosio::check(eosio::from_bin(row.data)).value(); + if (row.present) + table.insert(obj, bypass_preexist_check); + else + table.erase(obj); + } +} + +template +void store_delta_kv(eosio::kv_environment environment, table_delta_v0& delta, F f) { + for (auto& row : delta.rows) { + f(); + auto obj = eosio::check(eosio::from_bin(row.data)).value(); + auto& obj0 = std::get(obj); + if (row.present) + environment.kv_set(obj0.database.value, obj0.contract.value, obj0.key.pos, obj0.key.remaining(), + obj0.value.pos, obj0.value.remaining()); + else + environment.kv_erase(obj0.database.value, obj0.contract.value, obj0.key.pos, obj0.key.remaining()); + } +} + +template +inline void store_delta(eosio::kv_environment environment, table_delta_v0& delta, bool bypass_preexist_check, F f) { + if (delta.name == "global_property") + store_delta_typed(environment, delta, bypass_preexist_check, f); + if (delta.name == "account") + store_delta_typed(environment, delta, bypass_preexist_check, f); + if (delta.name == "account_metadata") + store_delta_typed(environment, delta, bypass_preexist_check, f); + if (delta.name == "code") + store_delta_typed(environment, delta, bypass_preexist_check, f); + if (delta.name == "contract_table") + store_delta_typed(environment, delta, bypass_preexist_check, f); + if (delta.name == "contract_row") + store_delta_typed(environment, delta, bypass_preexist_check, f); + if (delta.name == "contract_index64") + store_delta_typed(environment, delta, bypass_preexist_check, f); + if (delta.name == "contract_index128") + store_delta_typed(environment, delta, bypass_preexist_check, f); + if (delta.name == "key_value") + store_delta_kv(environment, delta, f); +} + +inline void store_deltas(eosio::kv_environment environment, std::vector& deltas, + bool bypass_preexist_check) { + for (auto& delta : deltas) // + store_delta(environment, std::get(delta), bypass_preexist_check, [] {}); +} + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/wasm_ql.hpp b/libraries/rodeos/include/b1/rodeos/wasm_ql.hpp new file mode 100644 index 00000000000..21347f32205 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/wasm_ql.hpp @@ -0,0 +1,77 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +namespace b1::rodeos::wasm_ql { + +class backend_cache; + +struct shared_state { + uint32_t max_console_size = {}; + uint32_t wasm_cache_size = {}; + uint64_t max_exec_time_ms = {}; + std::string contract_dir = {}; + std::unique_ptr backend_cache = {}; + std::shared_ptr db; + + shared_state(std::shared_ptr db); + ~shared_state(); +}; + +struct thread_state : eosio::history_tools::action_state, + eosio::history_tools::console_state, + eosio::history_tools::query_state { + std::shared_ptr shared = {}; + eosio::vm::wasm_allocator wa = {}; +}; + +class thread_state_cache { + private: + std::mutex mutex; + std::shared_ptr shared_state; + std::vector> states; + + public: + thread_state_cache(const std::shared_ptr& shared_state) : shared_state(shared_state) {} + + std::unique_ptr get_state() { + std::lock_guard lock{ mutex }; + if (states.empty()) { + auto result = std::make_unique(); + result->shared = shared_state; + return result; + } + auto result = std::move(states.back()); + states.pop_back(); + return result; + } + + void store_state(std::unique_ptr state) { + std::lock_guard lock{ mutex }; + states.push_back(std::move(state)); + } +}; + +const std::vector& query_get_info(wasm_ql::thread_state& thread_state, + const std::vector& contract_kv_prefix); +const std::vector& query_get_block(wasm_ql::thread_state& thread_state, + const std::vector& contract_kv_prefix, std::string_view body); +const std::vector& query_get_abi(wasm_ql::thread_state& thread_state, const std::vector& contract_kv_prefix, + std::string_view body); +const std::vector& query_get_required_keys(wasm_ql::thread_state& thread_state, std::string_view body); +const std::vector& query_send_transaction(wasm_ql::thread_state& thread_state, + const std::vector& contract_kv_prefix, std::string_view body, + bool return_trace_on_except); +ship_protocol::transaction_trace_v0 +query_send_transaction(wasm_ql::thread_state& thread_state, const std::vector& contract_kv_prefix, + const ship_protocol::packed_transaction& trx, const rocksdb::Snapshot* snapshot, + std::vector>& memory, bool return_trace_on_except); + +} // namespace b1::rodeos::wasm_ql From af6b67a9cb8330347b9e652867435cf34776a0ab Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Sat, 25 Apr 2020 15:29:03 -0400 Subject: [PATCH 05/23] rodeos --- libraries/rodeos/embedded_rodeos.cpp | 4 +- .../include/b1/rodeos/callbacks/action.hpp | 32 +- .../include/b1/rodeos/callbacks/basic.hpp | 76 +- .../include/b1/rodeos/callbacks/chaindb.hpp | 69 +- .../b1/rodeos/callbacks/compiler_builtins.hpp | 183 ++-- .../include/b1/rodeos/callbacks/console.hpp | 88 +- .../b1/rodeos/callbacks/definitions.hpp | 76 ++ .../include/b1/rodeos/callbacks/filter.hpp | 12 +- .../rodeos/include/b1/rodeos/callbacks/kv.hpp | 163 +-- .../include/b1/rodeos/callbacks/memory.hpp | 58 +- .../include/b1/rodeos/callbacks/query.hpp | 11 +- .../b1/rodeos/callbacks/unimplemented.hpp | 149 ++- .../rodeos/callbacks/unimplemented_filter.hpp | 8 +- libraries/rodeos/include/b1/rodeos/filter.hpp | 29 +- .../include/b1/rodeos/get_state_row.hpp | 10 +- libraries/rodeos/include/b1/rodeos/rodeos.hpp | 14 +- .../include/b1/rodeos/rodeos_tables.hpp | 229 ++--- .../rodeos/include/b1/rodeos/wasm_ql.hpp | 11 +- libraries/rodeos/include/eosio/datastream.hpp | 453 +++++++++ libraries/rodeos/include/eosio/key_value.hpp | 957 ++++++++++++++++++ 20 files changed, 2044 insertions(+), 588 deletions(-) create mode 100644 libraries/rodeos/include/b1/rodeos/callbacks/definitions.hpp create mode 100644 libraries/rodeos/include/eosio/datastream.hpp create mode 100644 libraries/rodeos/include/eosio/key_value.hpp diff --git a/libraries/rodeos/embedded_rodeos.cpp b/libraries/rodeos/embedded_rodeos.cpp index 5b1c09c43c6..162502aab94 100644 --- a/libraries/rodeos/embedded_rodeos.cpp +++ b/libraries/rodeos/embedded_rodeos.cpp @@ -78,7 +78,7 @@ extern "C" rodeos_bool rodeos_open_db(rodeos_error* error, rodeos_context* conte return error->set("path is null"); if (context->db) return error->set("a database is already open on this context"); - context->db = std::make_shared( + context->db = std::make_shared( path, create_if_missing, num_threads ? std::make_optional(num_threads) : std::nullopt, max_open_files ? std::make_optional(max_open_files) : std::nullopt); return true; @@ -218,7 +218,7 @@ extern "C" rodeos_query_handler* rodeos_create_query_handler(rodeos_error* error return handle_exceptions(error, nullptr, [&]() -> rodeos_query_handler* { if (!partition) return error->set("partition is null"), nullptr; - auto shared_state = std::make_shared(partition->obj->db); + auto shared_state = std::make_shared(partition->obj->db); shared_state->max_console_size = max_console_size; shared_state->wasm_cache_size = wasm_cache_size; shared_state->max_exec_time_ms = max_exec_time_ms; diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/action.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/action.hpp index 7e513df6011..eed2d097c86 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/action.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/action.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace b1::rodeos { @@ -14,15 +14,11 @@ template struct action_callbacks { Derived& derived() { return static_cast(*this); } - int read_action_data(char* memory, uint32_t buffer_size) { - derived().check_bounds(memory, buffer_size); - auto& state = derived().get_state(); - auto s = state.action_data.end - state.action_data.pos; - if (buffer_size == 0) - return s; - auto copy_size = std::min(static_cast(buffer_size), static_cast(s)); - memcpy(memory, state.action_data.pos, copy_size); - return copy_size; + int read_action_data(eosio::vm::span data) { + auto& state = derived().get_state(); + size_t s = state.action_data.end - state.action_data.pos; + memcpy(data.data(), state.action_data.pos, std::min(data.size(), s)); + return s; } int action_data_size() { @@ -32,18 +28,18 @@ struct action_callbacks { uint64_t current_receiver() { return derived().get_state().receiver.value; } - void set_action_return_value(char* packed_blob, uint32_t datalen) { + void set_action_return_value(eosio::vm::span packed_blob) { // todo: limit size - derived().check_bounds(packed_blob, datalen); - derived().get_state().action_return_value.assign(packed_blob, packed_blob + datalen); + derived().get_state().action_return_value.assign(packed_blob.begin(), packed_blob.end()); } - template + template static void register_callbacks() { - Rft::template add("env", "read_action_data"); - Rft::template add("env", "action_data_size"); - Rft::template add("env", "current_receiver"); - Rft::template add("env", "set_action_return_value"); + // todo: preconditions + Rft::template add<&Derived::read_action_data>("env", "read_action_data"); + Rft::template add<&Derived::action_data_size>("env", "action_data_size"); + Rft::template add<&Derived::current_receiver>("env", "current_receiver"); + Rft::template add<&Derived::set_action_return_value>("env", "set_action_return_value"); } }; // action_callbacks diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/basic.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/basic.hpp index fdb9e6b4d20..be23d5371de 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/basic.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/basic.hpp @@ -1,9 +1,7 @@ #pragma once -#include - +#include #include -#include namespace b1::rodeos { @@ -15,58 +13,28 @@ struct assert_exception : std::exception { const char* what() const noexcept override { return msg.c_str(); } }; -template -struct wasm_state { - eosio::vm::wasm_allocator& wa; - Backend& backend; -}; - -inline size_t copy_to_wasm(char* dest, size_t dest_size, const char* src, size_t src_size) { - if (dest_size == 0) - return src_size; - auto copy_size = std::min(dest_size, src_size); - memcpy(dest, src, copy_size); - return copy_size; -} - template -struct basic_callbacks { +struct context_free_system_callbacks { Derived& derived() { return static_cast(*this); } - void check_bounds(const char* begin, const char* end) { - if (begin > end) - throw std::runtime_error("bad memory"); - // todo: check bounds - } - - void check_bounds(const char* begin, uint32_t size) { - // todo: check bounds - } - - uint32_t check_bounds_get_len(const char* str) { - // todo: check bounds - return strlen(str); - } - void abort() { throw std::runtime_error("called abort"); } - void eosio_assert_message(bool test, const char* msg, uint32_t msg_len) { - check_bounds(msg, msg + msg_len); - if (!test) - throw assert_exception(std::string(msg, msg_len)); + void eosio_assert(uint32_t condition, null_terminated_ptr msg) { + if (!condition) + throw assert_exception(std::string(msg.data(), msg.size())); } - // todo: replace with prints_l - void print_range(const char* begin, const char* end) { - check_bounds(begin, end); - std::cerr.write(begin, end - begin); + void eosio_assert_message(bool condition, legacy_span msg) { + if (!condition) + throw assert_exception(std::string(msg.data(), msg.size())); } - template + template static void register_callbacks() { - Rft::template add("env", "abort"); - Rft::template add("env", "eosio_assert_message"); - Rft::template add("env", "print_range"); + // todo: preconditions + Rft::template add<&Derived::abort>("env", "abort"); + Rft::template add<&Derived::eosio_assert>("env", "eosio_assert"); + Rft::template add<&Derived::eosio_assert_message>("env", "eosio_assert_message"); } }; @@ -80,23 +48,23 @@ template struct data_callbacks { Derived& derived() { return static_cast(*this); } - uint32_t get_input_data(char* dest, uint32_t size) { - derived().check_bounds(dest, size); + uint32_t get_input_data(eosio::vm::span dest) { auto& input_data = derived().get_state().input_data; - return copy_to_wasm(dest, size, input_data.pos, size_t(input_data.end - input_data.pos)); + memcpy(dest.data(), input_data.pos, std::min(dest.size(), size_t(input_data.end - input_data.pos))); + return input_data.end - input_data.pos; } - void set_output_data(const char* data, uint32_t size) { - derived().check_bounds(data, size); + void set_output_data(eosio::vm::span data) { auto& output_data = derived().get_state().output_data; output_data.clear(); - output_data.insert(output_data.end(), data, data + size); + output_data.insert(output_data.end(), data.begin(), data.end()); } - template + template static void register_callbacks() { - Rft::template add("env", "get_input_data"); - Rft::template add("env", "set_output_data"); + // todo: preconditions + Rft::template add<&Derived::get_input_data>("env", "get_input_data"); + Rft::template add<&Derived::set_output_data>("env", "set_output_data"); } }; diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/chaindb.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/chaindb.hpp index 6e7ae7967ae..62d1fe1e575 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/chaindb.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/chaindb.hpp @@ -1,11 +1,8 @@ #pragma once -#include - #include #include #include -#include namespace b1::rodeos { @@ -49,8 +46,8 @@ class iterator_cache { return map_it->second; if (!view.get(eosio::name{ "state" }.value, chain_kv::to_slice(eosio::check(eosio::convert_to_key(std::make_tuple( - eosio::name{ "contract.tab" }, eosio::name{ "primary" }, - key.code, key.table, key.scope))) + (uint8_t)0x01, eosio::name{ "contract.tab" }, + eosio::name{ "primary" }, key.code, key.table, key.scope))) .value()))) return -1; if (tables.size() != table_to_index.size() || tables.size() != end_iterators.size()) @@ -115,7 +112,7 @@ class iterator_cache { if (itr >= iterators.size()) throw std::runtime_error("dereference non-existing iterator"); auto& it = iterators[itr]; - return copy_to_wasm(buffer, buffer_size, it.value.data(), it.value.size()); + return legacy_copy_to_wasm(buffer, buffer_size, it.value.data(), it.value.size()); } int db_next_i64(int itr, uint64_t& primary) { @@ -139,13 +136,14 @@ class iterator_cache { const auto& table_key = tables[it.table_index]; view_it = chain_kv::view::iterator{ view, eosio::name{ "state" }.value, - chain_kv::to_slice(eosio::check(eosio::convert_to_key(std::make_tuple( - eosio::name{ "contract.row" }, eosio::name{ "primary" }, - table_key.code, table_key.table, table_key.scope))) - .value()) + chain_kv::to_slice( + eosio::check(eosio::convert_to_key(std::make_tuple( // + (uint8_t)0x01, eosio::name{ "contract.row" }, eosio::name{ "primary" }, + table_key.code, table_key.table, table_key.scope))) + .value()) }; view_it->lower_bound(eosio::check(eosio::convert_to_key(std::make_tuple( - eosio::name{ "contract.row" }, eosio::name{ "primary" }, + (uint8_t)0x01, eosio::name{ "contract.row" }, eosio::name{ "primary" }, table_key.code, table_key.table, table_key.scope, it.primary))) .value()); } @@ -177,12 +175,12 @@ class iterator_cache { // std::cout << "lower_bound: db_view::iterator\n"; chain_kv::view::iterator it{ view, eosio::name{ "state" }.value, chain_kv::to_slice(eosio::check(eosio::convert_to_key(std::make_tuple( - eosio::name{ "contract.row" }, + (uint8_t)0x01, eosio::name{ "contract.row" }, eosio::name{ "primary" }, code, table, scope))) .value()) }; it.lower_bound( - eosio::check(eosio::convert_to_key(std::make_tuple(eosio::name{ "contract.row" }, eosio::name{ "primary" }, - code, table, scope, key))) + eosio::check(eosio::convert_to_key(std::make_tuple((uint8_t)0x01, eosio::name{ "contract.row" }, + eosio::name{ "primary" }, code, table, scope, key))) .value()); return get_iterator(rk, std::move(it)); } @@ -203,31 +201,29 @@ struct chaindb_callbacks { return *chaindb_state.iterator_cache; } - int db_store_i64(uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, const char* buffer, - uint32_t buffer_size) { + int32_t db_store_i64(uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, legacy_span buffer) { throw std::runtime_error("unimplemented: db_store_i64"); } - void db_update_i64(int itr, uint64_t payer, const char* buffer, uint32_t buffer_size) { + void db_update_i64(int32_t itr, uint64_t payer, legacy_span buffer) { throw std::runtime_error("unimplemented: db_update_i64"); } void db_remove_i64(int itr) { throw std::runtime_error("unimplemented: db_remove_i64"); } - int db_get_i64(int itr, char* buffer, uint32_t buffer_size) { - derived().check_bounds(buffer, buffer_size); - return get_iterator_cache().db_get_i64(itr, buffer, buffer_size); + int32_t db_get_i64(int32_t itr, legacy_span buffer) { + return get_iterator_cache().db_get_i64(itr, buffer.data(), buffer.size()); } - int db_next_i64(int itr, uint64_t& primary) { return get_iterator_cache().db_next_i64(itr, primary); } + int32_t db_next_i64(int32_t itr, legacy_ptr primary) { + return get_iterator_cache().db_next_i64(itr, *primary); + } - int db_previous_i64(int itr, uint64_t& primary) { - // + int32_t db_previous_i64(int32_t itr, legacy_ptr primary) { throw std::runtime_error("unimplemented: db_previous_i64"); } int db_find_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) { - // throw std::runtime_error("unimplemented: db_find_i64"); } @@ -236,27 +232,26 @@ struct chaindb_callbacks { } int db_upperbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) { - // throw std::runtime_error("unimplemented: db_upperbound_i64"); } int db_end_i64(uint64_t code, uint64_t scope, uint64_t table) { - // throw std::runtime_error("unimplemented: db_end_i64"); } - template + template static void register_callbacks() { - Rft::template add("env", "db_store_i64"); - Rft::template add("env", "db_update_i64"); - Rft::template add("env", "db_remove_i64"); - Rft::template add("env", "db_get_i64"); - Rft::template add("env", "db_next_i64"); - Rft::template add("env", "db_previous_i64"); - Rft::template add("env", "db_find_i64"); - Rft::template add("env", "db_lowerbound_i64"); - Rft::template add("env", "db_upperbound_i64"); - Rft::template add("env", "db_end_i64"); + // todo: preconditions + Rft::template add<&Derived::db_store_i64>("env", "db_store_i64"); + Rft::template add<&Derived::db_update_i64>("env", "db_update_i64"); + Rft::template add<&Derived::db_remove_i64>("env", "db_remove_i64"); + Rft::template add<&Derived::db_get_i64>("env", "db_get_i64"); + Rft::template add<&Derived::db_next_i64>("env", "db_next_i64"); + Rft::template add<&Derived::db_previous_i64>("env", "db_previous_i64"); + Rft::template add<&Derived::db_find_i64>("env", "db_find_i64"); + Rft::template add<&Derived::db_lowerbound_i64>("env", "db_lowerbound_i64"); + Rft::template add<&Derived::db_upperbound_i64>("env", "db_upperbound_i64"); + Rft::template add<&Derived::db_end_i64>("env", "db_end_i64"); } }; diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/compiler_builtins.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/compiler_builtins.hpp index b9bc654b0d5..8f07591d885 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/compiler_builtins.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/compiler_builtins.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include @@ -10,47 +10,47 @@ template struct compiler_builtins_callbacks { Derived& derived() { return static_cast(*this); } - void __ashlti3(__int128& ret, uint64_t low, uint64_t high, uint32_t shift) { + void __ashlti3(legacy_ptr<__int128> ret, uint64_t low, uint64_t high, uint32_t shift) { if (shift >= 128) { - ret = 0; + *ret = 0; } else { unsigned __int128 i = high; i <<= 64; i |= low; i <<= shift; - ret = (__int128)i; + *ret = (__int128)i; } } - void __ashrti3(__int128& ret, uint64_t low, uint64_t high, uint32_t shift) { + void __ashrti3(legacy_ptr<__int128> ret, uint64_t low, uint64_t high, uint32_t shift) { // retain the signedness - ret = high; - ret <<= 64; - ret |= low; - ret >>= shift; + *ret = high; + *ret <<= 64; + *ret |= low; + *ret >>= shift; } - void __lshlti3(__int128& ret, uint64_t low, uint64_t high, uint32_t shift) { + void __lshlti3(legacy_ptr<__int128> ret, uint64_t low, uint64_t high, uint32_t shift) { if (shift >= 128) { - ret = 0; + *ret = 0; } else { unsigned __int128 i = high; i <<= 64; i |= low; i <<= shift; - ret = (__int128)i; + *ret = (__int128)i; } } - void __lshrti3(__int128& ret, uint64_t low, uint64_t high, uint32_t shift) { + void __lshrti3(legacy_ptr<__int128> ret, uint64_t low, uint64_t high, uint32_t shift) { unsigned __int128 i = high; i <<= 64; i |= low; i >>= shift; - ret = (unsigned __int128)i; + *ret = (unsigned __int128)i; } - void __divti3(__int128& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + void __divti3(legacy_ptr<__int128> ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { __int128 lhs = ha; __int128 rhs = hb; @@ -65,10 +65,10 @@ struct compiler_builtins_callbacks { lhs /= rhs; - ret = lhs; + *ret = lhs; } - void __udivti3(unsigned __int128& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + void __udivti3(legacy_ptr ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { unsigned __int128 lhs = ha; unsigned __int128 rhs = hb; @@ -82,10 +82,10 @@ struct compiler_builtins_callbacks { throw std::runtime_error("divide by zero"); lhs /= rhs; - ret = lhs; + *ret = lhs; } - void __modti3(__int128& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + void __modti3(legacy_ptr<__int128> ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { __int128 lhs = ha; __int128 rhs = hb; @@ -99,10 +99,10 @@ struct compiler_builtins_callbacks { throw std::runtime_error("divide by zero"); lhs %= rhs; - ret = lhs; + *ret = lhs; } - void __umodti3(unsigned __int128& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + void __umodti3(legacy_ptr ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { unsigned __int128 lhs = ha; unsigned __int128 rhs = hb; @@ -116,10 +116,10 @@ struct compiler_builtins_callbacks { throw std::runtime_error("divide by zero"); lhs %= rhs; - ret = lhs; + *ret = lhs; } - void __multi3(__int128& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + void __multi3(legacy_ptr<__int128> ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { __int128 lhs = ha; __int128 rhs = hb; @@ -130,31 +130,31 @@ struct compiler_builtins_callbacks { rhs |= lb; lhs *= rhs; - ret = lhs; + *ret = lhs; } - void __addtf3(float128_t& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + void __addtf3(legacy_ptr ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { float128_t a = { { la, ha } }; float128_t b = { { lb, hb } }; - ret = f128_add(a, b); + *ret = f128_add(a, b); } - void __subtf3(float128_t& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + void __subtf3(legacy_ptr ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { float128_t a = { { la, ha } }; float128_t b = { { lb, hb } }; - ret = f128_sub(a, b); + *ret = f128_sub(a, b); } - void __multf3(float128_t& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + void __multf3(legacy_ptr ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { float128_t a = { { la, ha } }; float128_t b = { { lb, hb } }; - ret = f128_mul(a, b); + *ret = f128_mul(a, b); } - void __divtf3(float128_t& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + void __divtf3(legacy_ptr ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { float128_t a = { { la, ha } }; float128_t b = { { lb, hb } }; - ret = f128_div(a, b); + *ret = f128_div(a, b); } int __unordtf2(uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { @@ -191,15 +191,15 @@ struct compiler_builtins_callbacks { int __cmptf2(uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { return ___cmptf2(la, ha, lb, hb, 1); } - void __negtf2(float128_t& ret, uint64_t la, uint64_t ha) { ret = { { la, (ha ^ (uint64_t)1 << 63) } }; } + void __negtf2(legacy_ptr ret, uint64_t la, uint64_t ha) { *ret = { { la, (ha ^ (uint64_t)1 << 63) } }; } - void __floatsitf(float128_t& ret, int32_t i) { ret = i32_to_f128(i); } + void __floatsitf(legacy_ptr ret, int32_t i) { *ret = i32_to_f128(i); } - void __floatditf(float128_t& ret, uint64_t a) { ret = i64_to_f128(a); } + void __floatditf(legacy_ptr ret, uint64_t a) { *ret = i64_to_f128(a); } - void __floatunsitf(float128_t& ret, uint32_t i) { ret = ui32_to_f128(i); } + void __floatunsitf(legacy_ptr ret, uint32_t i) { *ret = ui32_to_f128(i); } - void __floatunditf(float128_t& ret, uint64_t a) { ret = ui64_to_f128(a); } + void __floatunditf(legacy_ptr ret, uint64_t a) { *ret = ui64_to_f128(a); } double __floattidf(uint64_t l, uint64_t h) { unsigned __int128 val = h; @@ -217,13 +217,13 @@ struct compiler_builtins_callbacks { double __floatsidf(int32_t i) { return from_softfloat64(i32_to_f64(i)); } - void __extendsftf2(float128_t& ret, float f) { ret = f32_to_f128(to_softfloat32(f)); } + void __extendsftf2(legacy_ptr ret, float f) { *ret = f32_to_f128(to_softfloat32(f)); } - void __extenddftf2(float128_t& ret, double d) { ret = f64_to_f128(to_softfloat64(d)); } + void __extenddftf2(legacy_ptr ret, double d) { *ret = f64_to_f128(to_softfloat64(d)); } - void __fixtfti(__int128& ret, uint64_t l, uint64_t h) { + void __fixtfti(legacy_ptr<__int128> ret, uint64_t l, uint64_t h) { float128_t f = { { l, h } }; - ret = ___fixtfti(f); + *ret = ___fixtfti(f); } int32_t __fixtfsi(uint64_t l, uint64_t h) { @@ -236,9 +236,9 @@ struct compiler_builtins_callbacks { return f128_to_i64(f, 0, false); } - void __fixunstfti(unsigned __int128& ret, uint64_t l, uint64_t h) { + void __fixunstfti(legacy_ptr ret, uint64_t l, uint64_t h) { float128_t f = { { l, h } }; - ret = ___fixunstfti(f); + *ret = ___fixunstfti(f); } uint32_t __fixunstfsi(uint64_t l, uint64_t h) { @@ -251,13 +251,13 @@ struct compiler_builtins_callbacks { return f128_to_ui64(f, 0, false); } - void __fixsfti(__int128& ret, float a) { ret = ___fixsfti(to_softfloat32(a).v); } + void __fixsfti(legacy_ptr<__int128> ret, float a) { *ret = ___fixsfti(to_softfloat32(a).v); } - void __fixdfti(__int128& ret, double a) { ret = ___fixdfti(to_softfloat64(a).v); } + void __fixdfti(legacy_ptr<__int128> ret, double a) { *ret = ___fixdfti(to_softfloat64(a).v); } - void __fixunssfti(unsigned __int128& ret, float a) { ret = ___fixunssfti(to_softfloat32(a).v); } + void __fixunssfti(legacy_ptr ret, float a) { *ret = ___fixunssfti(to_softfloat32(a).v); } - void __fixunsdfti(unsigned __int128& ret, double a) { ret = ___fixunsdfti(to_softfloat64(a).v); } + void __fixunsdfti(legacy_ptr ret, double a) { *ret = ___fixunsdfti(to_softfloat64(a).v); } double __trunctfdf2(uint64_t l, uint64_t h) { float128_t f = { { l, h } }; @@ -269,51 +269,52 @@ struct compiler_builtins_callbacks { return from_softfloat32(f128_to_f32(f)); } - template + template static void register_callbacks() { - Rft::template add("env", "__ashlti3"); - Rft::template add("env", "__ashrti3"); - Rft::template add("env", "__lshlti3"); - Rft::template add("env", "__lshrti3"); - Rft::template add("env", "__divti3"); - Rft::template add("env", "__udivti3"); - Rft::template add("env", "__modti3"); - Rft::template add("env", "__umodti3"); - Rft::template add("env", "__multi3"); - Rft::template add("env", "__addtf3"); - Rft::template add("env", "__subtf3"); - Rft::template add("env", "__multf3"); - Rft::template add("env", "__divtf3"); - Rft::template add("env", "__eqtf2"); - Rft::template add("env", "__netf2"); - Rft::template add("env", "__getf2"); - Rft::template add("env", "__gttf2"); - Rft::template add("env", "__lttf2"); - Rft::template add("env", "__letf2"); - Rft::template add("env", "__cmptf2"); - Rft::template add("env", "__unordtf2"); - Rft::template add("env", "__negtf2"); - Rft::template add("env", "__floatsitf"); - Rft::template add("env", "__floatunsitf"); - Rft::template add("env", "__floatditf"); - Rft::template add("env", "__floatunditf"); - Rft::template add("env", "__floattidf"); - Rft::template add("env", "__floatuntidf"); - Rft::template add("env", "__floatsidf"); - Rft::template add("env", "__extendsftf2"); - Rft::template add("env", "__extenddftf2"); - Rft::template add("env", "__fixtfti"); - Rft::template add("env", "__fixtfdi"); - Rft::template add("env", "__fixtfsi"); - Rft::template add("env", "__fixunstfti"); - Rft::template add("env", "__fixunstfdi"); - Rft::template add("env", "__fixunstfsi"); - Rft::template add("env", "__fixsfti"); - Rft::template add("env", "__fixdfti"); - Rft::template add("env", "__fixunssfti"); - Rft::template add("env", "__fixunsdfti"); - Rft::template add("env", "__trunctfdf2"); - Rft::template add("env", "__trunctfsf2"); + // todo: preconditions + Rft::template add<&Derived::__ashlti3>("env", "__ashlti3"); + Rft::template add<&Derived::__ashrti3>("env", "__ashrti3"); + Rft::template add<&Derived::__lshlti3>("env", "__lshlti3"); + Rft::template add<&Derived::__lshrti3>("env", "__lshrti3"); + Rft::template add<&Derived::__divti3>("env", "__divti3"); + Rft::template add<&Derived::__udivti3>("env", "__udivti3"); + Rft::template add<&Derived::__modti3>("env", "__modti3"); + Rft::template add<&Derived::__umodti3>("env", "__umodti3"); + Rft::template add<&Derived::__multi3>("env", "__multi3"); + Rft::template add<&Derived::__addtf3>("env", "__addtf3"); + Rft::template add<&Derived::__subtf3>("env", "__subtf3"); + Rft::template add<&Derived::__multf3>("env", "__multf3"); + Rft::template add<&Derived::__divtf3>("env", "__divtf3"); + Rft::template add<&Derived::__eqtf2>("env", "__eqtf2"); + Rft::template add<&Derived::__netf2>("env", "__netf2"); + Rft::template add<&Derived::__getf2>("env", "__getf2"); + Rft::template add<&Derived::__gttf2>("env", "__gttf2"); + Rft::template add<&Derived::__lttf2>("env", "__lttf2"); + Rft::template add<&Derived::__letf2>("env", "__letf2"); + Rft::template add<&Derived::__cmptf2>("env", "__cmptf2"); + Rft::template add<&Derived::__unordtf2>("env", "__unordtf2"); + Rft::template add<&Derived::__negtf2>("env", "__negtf2"); + Rft::template add<&Derived::__floatsitf>("env", "__floatsitf"); + Rft::template add<&Derived::__floatunsitf>("env", "__floatunsitf"); + Rft::template add<&Derived::__floatditf>("env", "__floatditf"); + Rft::template add<&Derived::__floatunditf>("env", "__floatunditf"); + Rft::template add<&Derived::__floattidf>("env", "__floattidf"); + Rft::template add<&Derived::__floatuntidf>("env", "__floatuntidf"); + Rft::template add<&Derived::__floatsidf>("env", "__floatsidf"); + Rft::template add<&Derived::__extendsftf2>("env", "__extendsftf2"); + Rft::template add<&Derived::__extenddftf2>("env", "__extenddftf2"); + Rft::template add<&Derived::__fixtfti>("env", "__fixtfti"); + Rft::template add<&Derived::__fixtfdi>("env", "__fixtfdi"); + Rft::template add<&Derived::__fixtfsi>("env", "__fixtfsi"); + Rft::template add<&Derived::__fixunstfti>("env", "__fixunstfti"); + Rft::template add<&Derived::__fixunstfdi>("env", "__fixunstfdi"); + Rft::template add<&Derived::__fixunstfsi>("env", "__fixunstfsi"); + Rft::template add<&Derived::__fixsfti>("env", "__fixsfti"); + Rft::template add<&Derived::__fixdfti>("env", "__fixdfti"); + Rft::template add<&Derived::__fixunssfti>("env", "__fixunssfti"); + Rft::template add<&Derived::__fixunsdfti>("env", "__fixunsdfti"); + Rft::template add<&Derived::__trunctfdf2>("env", "__trunctfdf2"); + Rft::template add<&Derived::__trunctfsf2>("env", "__trunctfsf2"); } }; diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/console.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/console.hpp index e95b5889d11..8b7890e5cf3 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/console.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/console.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include #include namespace b1::rodeos { @@ -19,45 +19,16 @@ struct console_callbacks { state.console.append(str, std::min(size_t(len), state.max_console_size - state.console.size())); } - void prints(const char* str) { - auto len = derived().check_bounds_get_len(str); - append_console(str, len); + template + void append_console(const char (&str)[size]) { + return append_console(str, size - 1); // omit 0-termination } - void prints_l(const char* str, uint32_t len) { - derived().check_bounds(str, len); - append_console(str, len); - } - - void printi(int64_t value) { - auto& state = derived().get_state(); - if (!state.max_console_size) - return; - auto s = std::to_string(value); - append_console(s.c_str(), s.size()); - } - - void printui(uint64_t value) { - auto& state = derived().get_state(); - if (!state.max_console_size) - return; - auto s = std::to_string(value); - append_console(s.c_str(), s.size()); - } - - void printi128(void* value) { - // todo - append_console("{unimplemented}", 15); - } + void prints(null_terminated_ptr str) { append_console(str.data(), str.size()); } + void prints_l(legacy_span str) { append_console(str.data(), str.size()); } - void printui128(void* value) { - // todo - append_console("{unimplemented}", 15); - } - - void printsf(float value) { printdf(value); } - - void printdf(double value) { + template + void print_numeric(T value) { auto& state = derived().get_state(); if (!state.max_console_size) return; @@ -65,23 +36,25 @@ struct console_callbacks { append_console(s.c_str(), s.size()); } - void printqf(void*) { - // don't bother - append_console("{unimplemented}", 15); - } + void printi(int64_t value) { print_numeric(value); } + void printui(uint64_t value) { print_numeric(value); } + void printi128(legacy_ptr p) { append_console("{printi128 unimplemented}"); } + void printui128(legacy_ptr p) { append_console("{printui128 unimplemented}"); } + void printsf(float value) { print_numeric(value); } + void printdf(double value) { print_numeric(value); } + void printqf(legacy_ptr) { append_console("{printqf unimplemented}"); } void printn(uint64_t value) { auto& state = derived().get_state(); if (!state.max_console_size) return; - auto s = std::to_string(value); + auto s = eosio::name{ value }.to_string(); append_console(s.c_str(), s.size()); } - void printhex(const char* data, uint32_t len) { - derived().check_bounds(data, len); + void printhex(legacy_span data) { auto& state = derived().get_state(); - for (uint32_t i = 0; i < len && state.console.size() + 2 < state.max_console_size; ++i) { + for (uint32_t i = 0; i < data.size() && state.console.size() + 2 < state.max_console_size; ++i) { static const char hex_digits[] = "0123456789ABCDEF"; uint8_t byte = data[i]; state.console.push_back(hex_digits[byte >> 4]); @@ -89,19 +62,20 @@ struct console_callbacks { } } - template + template static void register_callbacks() { - Rft::template add("env", "prints"); - Rft::template add("env", "prints_l"); - Rft::template add("env", "printi"); - Rft::template add("env", "printui"); - Rft::template add("env", "printi128"); - Rft::template add("env", "printui128"); - Rft::template add("env", "printsf"); - Rft::template add("env", "printdf"); - Rft::template add("env", "printqf"); - Rft::template add("env", "printn"); - Rft::template add("env", "printhex"); + // todo: preconditions + Rft::template add<&Derived::prints>("env", "prints"); + Rft::template add<&Derived::prints_l>("env", "prints_l"); + Rft::template add<&Derived::printi>("env", "printi"); + Rft::template add<&Derived::printui>("env", "printui"); + Rft::template add<&Derived::printi128>("env", "printi128"); + Rft::template add<&Derived::printui128>("env", "printui128"); + Rft::template add<&Derived::printsf>("env", "printsf"); + Rft::template add<&Derived::printdf>("env", "printdf"); + Rft::template add<&Derived::printqf>("env", "printqf"); + Rft::template add<&Derived::printn>("env", "printn"); + Rft::template add<&Derived::printhex>("env", "printhex"); } }; // console_callbacks diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/definitions.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/definitions.hpp new file mode 100644 index 00000000000..0278938d372 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/definitions.hpp @@ -0,0 +1,76 @@ +#pragma once + +#include + +#include + +namespace b1::rodeos { + +using wasm_size_t = eosio::vm::wasm_size_t; + +template +using legacy_ptr = eosio::vm::argument_proxy; + +template +using legacy_span = eosio::vm::argument_proxy, Align>; + +struct null_terminated_ptr : eosio::vm::span { + using base_type = eosio::vm::span; + null_terminated_ptr(const char* ptr) : base_type(ptr, strlen(ptr)) {} +}; + +// wrapper pointer type to keep the validations from being done +template +struct unvalidated_ptr { + static_assert(sizeof(T) == 1); // currently only going to support these for char and const char + operator T*() { return ptr; } + T* operator->() { return ptr; } + T& operator*() { return *ptr; } + T* ptr; +}; + +template +void ignore_unused_variable_warning(T&...) {} + +inline size_t legacy_copy_to_wasm(char* dest, size_t dest_size, const char* src, size_t src_size) { + if (dest_size == 0) + return src_size; + auto copy_size = std::min(dest_size, src_size); + memcpy(dest, src, copy_size); + return copy_size; +} + +template +struct type_converter : eosio::vm::type_converter { + using base_type = eosio::vm::type_converter; + using base_type::base_type; + using base_type::from_wasm; + + template + auto from_wasm(const void* ptr) const -> std::enable_if_t>, T> { + return { static_cast(ptr) }; + } + + template + auto from_wasm(void* ptr) const -> std::enable_if_t>, T> { + return { static_cast(ptr) }; + } + + template + auto from_wasm(void* ptr) const -> std::enable_if_t, eosio::vm::argument_proxy> { + this->template validate_pointer>(ptr, 1); + return { ptr }; + } + + EOS_VM_FROM_WASM(null_terminated_ptr, (const void* ptr)) { + this->validate_null_terminated_pointer(ptr); + return { static_cast(ptr) }; + } +}; // type_converter + +template +using registered_host_functions = + eosio::vm::registered_host_functions>; + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/filter.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/filter.hpp index 39455974519..a2fef0e5550 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/filter.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/filter.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace b1::rodeos { @@ -12,14 +12,14 @@ template struct filter_callbacks { Derived& derived() { return static_cast(*this); } - void push_data(const char* data, uint32_t size) { - derived().check_bounds(data, size); - derived().get_filter_callback_state().push_data(data, size); + void push_data(eosio::vm::span data) { + derived().get_filter_callback_state().push_data(data.data(), data.size()); } - template + template static void register_callbacks() { - Rft::template add("env", "push_data"); + // todo: preconditions + Rft::template add<&Derived::push_data>("env", "push_data"); } }; // query_callbacks diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/kv.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/kv.hpp index bd21ebdbee1..c0d4730a31f 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/kv.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/kv.hpp @@ -1,9 +1,9 @@ #pragma once #include +#include #include #include -#include #include #include @@ -45,6 +45,21 @@ struct kv_iterator_rocksdb { return kv_it_stat::iterator_ok; } + void fill_found(uint32_t* found_key_size, uint32_t* found_value_size) { + auto kv = kv_it.get_kv(); + if (kv) { + if (found_key_size) + *found_key_size = kv->key.size(); + if (found_value_size) + *found_value_size = kv->value.size(); + } else { + if (found_key_size) + *found_key_size = 0; + if (found_value_size) + *found_value_size = 0; + } + } + int32_t kv_it_compare(const kv_iterator_rocksdb& rhs) { eosio::check(rhs.is_kv_rocksdb_context_iterator(), "Incompatible key-value iterators"); auto& r = static_cast(rhs); @@ -64,20 +79,24 @@ struct kv_iterator_rocksdb { return kv_it_stat::iterator_end; } - kv_it_stat kv_it_next() { + kv_it_stat kv_it_next(uint32_t* found_key_size = nullptr, uint32_t* found_value_size = nullptr) { eosio::check(!kv_it.is_erased(), "Iterator to erased element"); ++kv_it; + fill_found(found_key_size, found_value_size); return kv_it_status(); } - kv_it_stat kv_it_prev() { + kv_it_stat kv_it_prev(uint32_t* found_key_size = nullptr, uint32_t* found_value_size = nullptr) { eosio::check(!kv_it.is_erased(), "Iterator to erased element"); --kv_it; + fill_found(found_key_size, found_value_size); return kv_it_status(); } - kv_it_stat kv_it_lower_bound(const char* key, uint32_t size) { + kv_it_stat kv_it_lower_bound(const char* key, uint32_t size, uint32_t* found_key_size = nullptr, + uint32_t* found_value_size = nullptr) { kv_it.lower_bound(key, size); + fill_found(found_key_size, found_value_size); return kv_it_status(); } @@ -224,30 +243,23 @@ template struct db_callbacks { Derived& derived() { return static_cast(*this); } - int64_t kv_erase(uint64_t db, uint64_t contract, const char* key, uint32_t key_size) { - derived().check_bounds(key, key_size); - return kv_get_db(db).kv_erase(contract, key, key_size); + int64_t kv_erase(uint64_t db, uint64_t contract, eosio::vm::span key) { + return kv_get_db(db).kv_erase(contract, key.data(), key.size()); } - int64_t kv_set(uint64_t db, uint64_t contract, const char* key, uint32_t key_size, const char* value, - uint32_t value_size) { - derived().check_bounds(key, key_size); - derived().check_bounds(value, value_size); - return kv_get_db(db).kv_set(contract, key, key_size, value, value_size); + int64_t kv_set(uint64_t db, uint64_t contract, eosio::vm::span key, eosio::vm::span value) { + return kv_get_db(db).kv_set(contract, key.data(), key.size(), value.data(), value.size()); } - bool kv_get(uint64_t db, uint64_t contract, const char* key, uint32_t key_size, uint32_t& value_size) { - derived().check_bounds(key, key_size); - return kv_get_db(db).kv_get(contract, key, key_size, value_size); + bool kv_get(uint64_t db, uint64_t contract, eosio::vm::span key, uint32_t* value_size) { + return kv_get_db(db).kv_get(contract, key.data(), key.size(), *value_size); } - uint32_t kv_get_data(uint64_t db, uint32_t offset, char* data, uint32_t data_size) { - derived().check_bounds(data, data_size); - return kv_get_db(db).kv_get_data(offset, data, data_size); + uint32_t kv_get_data(uint64_t db, uint32_t offset, eosio::vm::span data) { + return kv_get_db(db).kv_get_data(offset, data.data(), data.size()); } - uint32_t kv_it_create(uint64_t db, uint64_t contract, const char* prefix, uint32_t size) { - derived().check_bounds(prefix, size); + uint32_t kv_it_create(uint64_t db, uint64_t contract, eosio::vm::span prefix) { auto& kdb = kv_get_db(db); uint32_t itr; if (!derived().get_db_view_state().kv_destroyed_iterators.empty()) { @@ -259,7 +271,7 @@ struct db_callbacks { itr = derived().get_db_view_state().kv_iterators.size(); derived().get_db_view_state().kv_iterators.emplace_back(); } - derived().get_db_view_state().kv_iterators[itr] = kdb.kv_it_create(contract, prefix, size); + derived().get_db_view_state().kv_iterators[itr] = kdb.kv_it_create(contract, prefix.data(), prefix.size()); return itr; } @@ -281,10 +293,9 @@ struct db_callbacks { *derived().get_db_view_state().kv_iterators[itr_b]); } - int32_t kv_it_key_compare(uint32_t itr, const char* key, uint32_t size) { - derived().check_bounds(key, size); + int32_t kv_it_key_compare(uint32_t itr, eosio::vm::span key) { kv_check_iterator(itr); - return derived().get_db_view_state().kv_iterators[itr]->kv_it_key_compare(key, size); + return derived().get_db_view_state().kv_iterators[itr]->kv_it_key_compare(key.data(), key.size()); } int32_t kv_it_move_to_end(uint32_t itr) { @@ -292,34 +303,35 @@ struct db_callbacks { return static_cast(derived().get_db_view_state().kv_iterators[itr]->kv_it_move_to_end()); } - int32_t kv_it_next(uint32_t itr) { + int32_t kv_it_next(uint32_t itr, uint32_t* found_key_size, uint32_t* found_value_size) { kv_check_iterator(itr); - return static_cast(derived().get_db_view_state().kv_iterators[itr]->kv_it_next()); + return static_cast( + derived().get_db_view_state().kv_iterators[itr]->kv_it_next(found_key_size, found_value_size)); } - int32_t kv_it_prev(uint32_t itr) { + int32_t kv_it_prev(uint32_t itr, uint32_t* found_key_size, uint32_t* found_value_size) { kv_check_iterator(itr); - return static_cast(derived().get_db_view_state().kv_iterators[itr]->kv_it_prev()); + return static_cast( + derived().get_db_view_state().kv_iterators[itr]->kv_it_prev(found_key_size, found_value_size)); } - int32_t kv_it_lower_bound(uint32_t itr, const char* key, uint32_t size) { - derived().check_bounds(key, size); + int32_t kv_it_lower_bound(uint32_t itr, eosio::vm::span key, uint32_t* found_key_size, + uint32_t* found_value_size) { kv_check_iterator(itr); - return static_cast(derived().get_db_view_state().kv_iterators[itr]->kv_it_lower_bound(key, size)); + return static_cast(derived().get_db_view_state().kv_iterators[itr]->kv_it_lower_bound( + key.data(), key.size(), found_key_size, found_value_size)); } - int32_t kv_it_key(uint32_t itr, uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) { - derived().check_bounds(dest, size); + int32_t kv_it_key(uint32_t itr, uint32_t offset, eosio::vm::span dest, uint32_t* actual_size) { kv_check_iterator(itr); return static_cast( - derived().get_db_view_state().kv_iterators[itr]->kv_it_key(offset, dest, size, actual_size)); + derived().get_db_view_state().kv_iterators[itr]->kv_it_key(offset, dest.data(), dest.size(), *actual_size)); } - int32_t kv_it_value(uint32_t itr, uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) { - derived().check_bounds(dest, size); + int32_t kv_it_value(uint32_t itr, uint32_t offset, eosio::vm::span dest, uint32_t* actual_size) { kv_check_iterator(itr); - return static_cast( - derived().get_db_view_state().kv_iterators[itr]->kv_it_value(offset, dest, size, actual_size)); + return static_cast(derived().get_db_view_state().kv_iterators[itr]->kv_it_value( + offset, dest.data(), dest.size(), *actual_size)); } kv_context_rocksdb& kv_get_db(uint64_t db) { @@ -338,23 +350,24 @@ struct db_callbacks { "Bad key-value iterator"); } - template + template static void register_callbacks() { - Rft::template add("env", "kv_erase"); - Rft::template add("env", "kv_set"); - Rft::template add("env", "kv_get"); - Rft::template add("env", "kv_get_data"); - Rft::template add("env", "kv_it_create"); - Rft::template add("env", "kv_it_destroy"); - Rft::template add("env", "kv_it_status"); - Rft::template add("env", "kv_it_compare"); - Rft::template add("env", "kv_it_key_compare"); - Rft::template add("env", "kv_it_move_to_end"); - Rft::template add("env", "kv_it_next"); - Rft::template add("env", "kv_it_prev"); - Rft::template add("env", "kv_it_lower_bound"); - Rft::template add("env", "kv_it_key"); - Rft::template add("env", "kv_it_value"); + // todo: preconditions + Rft::template add<&Derived::kv_erase>("env", "kv_erase"); + Rft::template add<&Derived::kv_set>("env", "kv_set"); + Rft::template add<&Derived::kv_get>("env", "kv_get"); + Rft::template add<&Derived::kv_get_data>("env", "kv_get_data"); + Rft::template add<&Derived::kv_it_create>("env", "kv_it_create"); + Rft::template add<&Derived::kv_it_destroy>("env", "kv_it_destroy"); + Rft::template add<&Derived::kv_it_status>("env", "kv_it_status"); + Rft::template add<&Derived::kv_it_compare>("env", "kv_it_compare"); + Rft::template add<&Derived::kv_it_key_compare>("env", "kv_it_key_compare"); + Rft::template add<&Derived::kv_it_move_to_end>("env", "kv_it_move_to_end"); + Rft::template add<&Derived::kv_it_next>("env", "kv_it_next"); + Rft::template add<&Derived::kv_it_prev>("env", "kv_it_prev"); + Rft::template add<&Derived::kv_it_lower_bound>("env", "kv_it_lower_bound"); + Rft::template add<&Derived::kv_it_key>("env", "kv_it_key"); + Rft::template add<&Derived::kv_it_value>("env", "kv_it_value"); } }; // db_callbacks @@ -367,11 +380,47 @@ class kv_environment : public db_callbacks { kv_environment(const kv_environment&) = default; auto& get_db_view_state() { return state; } - void check_bounds(const char*, uint32_t) {} - using base::kv_set; + int64_t kv_erase(uint64_t db, uint64_t contract, const char* key, uint32_t key_size) { + return base::kv_erase(db, contract, { key, key_size }); + } + + int64_t kv_set(uint64_t db, uint64_t contract, const char* key, uint32_t key_size, const char* value, + uint32_t value_size) { + return base::kv_set(db, contract, { key, key_size }, { value, value_size }); + } + void kv_set(uint64_t db, uint64_t contract, const std::vector& k, const std::vector& v) { - base::kv_set(db, contract, k.data(), k.size(), v.data(), v.size()); + base::kv_set(db, contract, { k.data(), k.size() }, { v.data(), v.size() }); + } + + bool kv_get(uint64_t db, uint64_t contract, const char* key, uint32_t key_size, uint32_t& value_size) { + return base::kv_get(db, contract, { key, key_size }, &value_size); + } + + uint32_t kv_get_data(uint64_t db, uint32_t offset, char* data, uint32_t data_size) { + return base::kv_get_data(db, offset, { data, data_size }); + } + + uint32_t kv_it_create(uint64_t db, uint64_t contract, const char* prefix, uint32_t prefix_size) { + return base::kv_it_create(db, contract, { prefix, prefix_size }); + } + + int32_t kv_it_key_compare(uint32_t itr, const char* key, uint32_t key_size) { + return base::kv_it_key_compare(itr, { key, key_size }); + } + + int32_t kv_it_lower_bound(uint32_t itr, const char* key, uint32_t key_size, uint32_t* found_key_size = nullptr, + uint32_t* found_value_size = nullptr) { + return base::kv_it_lower_bound(itr, { key, key_size }, found_key_size, found_value_size); + } + + int32_t kv_it_key(uint32_t itr, uint32_t offset, char* dest, uint32_t dest_size, uint32_t* actual_size) { + return base::kv_it_key(itr, offset, { dest, dest_size }, actual_size); + } + + int32_t kv_it_value(uint32_t itr, uint32_t offset, char* dest, uint32_t dest_size, uint32_t* actual_size) { + return base::kv_it_value(itr, offset, { dest, dest_size }, actual_size); } }; diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/memory.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/memory.hpp index 94c58e996f0..7486ff253d0 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/memory.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/memory.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace b1::rodeos { @@ -8,42 +8,46 @@ template struct memory_callbacks { Derived& derived() { return static_cast(*this); } - char* memcpy(char* dest, const char* src, uint32_t length) { - derived().check_bounds(dest, length); - derived().check_bounds(src, length); - if ((size_t)(std::abs((ptrdiff_t)dest - (ptrdiff_t)src)) < length) + void* memcpy_impl(unvalidated_ptr dest, unvalidated_ptr src, wasm_size_t length) const { + volatile auto check_src = *((const char*)src + length - 1); + volatile auto check_dest = *((const char*)dest + length - 1); + volatile auto check_result = *(const char*)dest; + ignore_unused_variable_warning(check_src, check_dest, check_result); + if ((size_t)(std::abs((ptrdiff_t)(char*)dest - (ptrdiff_t)(const char*)src)) < length) throw std::runtime_error("memcpy can only accept non-aliasing pointers"); - return (char*)::memcpy(dest, src, length); + return (char*)std::memcpy((char*)dest, (const char*)src, length); } - char* memmove(char* dest, const char* src, uint32_t length) { - derived().check_bounds(dest, length); - derived().check_bounds(src, length); - return (char*)::memmove(dest, src, length); + void* memmove_impl(unvalidated_ptr dest, unvalidated_ptr src, wasm_size_t length) const { + volatile auto check_src = *((const char*)src + length - 1); + volatile auto check_dest = *((const char*)dest + length - 1); + volatile auto check_result = *(const char*)dest; + ignore_unused_variable_warning(check_src, check_dest, check_result); + return (char*)std::memmove((char*)dest, (const char*)src, length); } - int memcmp(const char* dest, const char* src, uint32_t length) { - derived().check_bounds(dest, length); - derived().check_bounds(src, length); - int ret = ::memcmp(dest, src, length); - if (ret < 0) - return -1; - if (ret > 0) - return 1; - return 0; + int32_t memcmp_impl(unvalidated_ptr dest, unvalidated_ptr src, wasm_size_t length) const { + volatile auto check_src = *((const char*)src + length - 1); + volatile auto check_dest = *((const char*)dest + length - 1); + ignore_unused_variable_warning(check_src, check_dest); + int32_t ret = std::memcmp((const char*)dest, (const char*)src, length); + return ret < 0 ? -1 : ret > 0 ? 1 : 0; } - char* memset(char* dest, int value, uint32_t length) { - derived().check_bounds(dest, length); - return (char*)::memset(dest, value, length); + void* memset_impl(unvalidated_ptr dest, int32_t value, wasm_size_t length) const { + volatile auto check_dest = *((const char*)dest + length - 1); + volatile auto check_result = *(const char*)dest; + ignore_unused_variable_warning(check_dest, check_result); + return (char*)std::memset((char*)dest, value, length); } - template + template static void register_callbacks() { - Rft::template add("env", "memcpy"); - Rft::template add("env", "memmove"); - Rft::template add("env", "memcmp"); - Rft::template add("env", "memset"); + // todo: preconditions + Rft::template add<&Derived::memcpy_impl>("env", "memcpy"); + Rft::template add<&Derived::memmove_impl>("env", "memmove"); + Rft::template add<&Derived::memcmp_impl>("env", "memcmp"); + Rft::template add<&Derived::memset_impl>("env", "memset"); } }; // memory_callbacks diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/query.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/query.hpp index 9a0162686a4..5b8070bda5a 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/query.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/query.hpp @@ -6,8 +6,8 @@ namespace b1::rodeos { struct query_state { - uint32_t block_num; - std::optional block_info; + uint32_t block_num; + std::optional block_info; }; template @@ -18,7 +18,7 @@ struct query_callbacks { auto& state = derived().get_state(); if (state.block_info) return; - auto info = get_state_row( + auto info = get_state_row( derived().get_db_view_state().kv_state.view, std::make_tuple(eosio::name{ "block.info" }, eosio::name{ "primary" }, state.block_num)); if (!info) @@ -35,9 +35,10 @@ struct query_callbacks { *derived().get_state().block_info); } - template + template static void register_callbacks() { - Rft::template add("env", "current_time"); + // todo: preconditions + Rft::template add<&Derived::current_time>("env", "current_time"); } }; // query_callbacks diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented.hpp index 228b9beff9b..0d9290aecce 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace b1::rodeos { @@ -106,12 +106,6 @@ struct unimplemented_callbacks { int64_t get_sender() { return unimplemented("get_sender"); } // context_free_system_api - void eosio_assert(uint32_t test, const char* msg) { - // todo: bounds check - // todo: move out of unimplemented_callbacks.hpp - if (test == 0) - throw std::runtime_error(std::string("assertion failed: ") + msg); - } void eosio_assert_code(int, int64_t) { return unimplemented("eosio_assert_code"); } void eosio_exit(int) { return unimplemented("eosio_exit"); } @@ -139,48 +133,48 @@ struct unimplemented_callbacks { // context_free_api int get_context_free_data(int, int, int) { return unimplemented("get_context_free_data"); } - template + template static void register_callbacks() { + // todo: preconditions + // privileged_api - Rft::template add("env", "is_feature_active"); - Rft::template add("env", "activate_feature"); - Rft::template add("env", "get_resource_limits"); - Rft::template add("env", "set_resource_limits"); - Rft::template add("env", "set_proposed_producers"); - Rft::template add( - "env", "get_blockchain_parameters_packed"); - Rft::template add( - "env", "set_blockchain_parameters_packed"); - Rft::template add("env", "is_privileged"); - Rft::template add("env", "set_privileged"); - Rft::template add("env", "preactivate_feature"); + Rft::template add<&Derived::is_feature_active>("env", "is_feature_active"); + Rft::template add<&Derived::activate_feature>("env", "activate_feature"); + Rft::template add<&Derived::get_resource_limits>("env", "get_resource_limits"); + Rft::template add<&Derived::set_resource_limits>("env", "set_resource_limits"); + Rft::template add<&Derived::set_proposed_producers>("env", "set_proposed_producers"); + Rft::template add<&Derived::get_blockchain_parameters_packed>("env", "get_blockchain_parameters_packed"); + Rft::template add<&Derived::set_blockchain_parameters_packed>("env", "set_blockchain_parameters_packed"); + Rft::template add<&Derived::is_privileged>("env", "is_privileged"); + Rft::template add<&Derived::set_privileged>("env", "set_privileged"); + Rft::template add<&Derived::preactivate_feature>("env", "preactivate_feature"); // producer_api - Rft::template add("env", "get_active_producers"); + Rft::template add<&Derived::get_active_producers>("env", "get_active_producers"); #define DB_SECONDARY_INDEX_METHODS_SIMPLE(IDX) \ - Rft::template add("env", "db_" #IDX "_store"); \ - Rft::template add("env", "db_" #IDX "_remove"); \ - Rft::template add("env", "db_" #IDX "_update"); \ - Rft::template add("env", "db_" #IDX "_find_primary"); \ - Rft::template add("env", "db_" #IDX "_find_secondary"); \ - Rft::template add("env", "db_" #IDX "_lowerbound"); \ - Rft::template add("env", "db_" #IDX "_upperbound"); \ - Rft::template add("env", "db_" #IDX "_end"); \ - Rft::template add("env", "db_" #IDX "_next"); \ - Rft::template add("env", "db_" #IDX "_previous"); + Rft::template add<&Derived::db_##IDX##_store>("env", "db_" #IDX "_store"); \ + Rft::template add<&Derived::db_##IDX##_remove>("env", "db_" #IDX "_remove"); \ + Rft::template add<&Derived::db_##IDX##_update>("env", "db_" #IDX "_update"); \ + Rft::template add<&Derived::db_##IDX##_find_primary>("env", "db_" #IDX "_find_primary"); \ + Rft::template add<&Derived::db_##IDX##_find_secondary>("env", "db_" #IDX "_find_secondary"); \ + Rft::template add<&Derived::db_##IDX##_lowerbound>("env", "db_" #IDX "_lowerbound"); \ + Rft::template add<&Derived::db_##IDX##_upperbound>("env", "db_" #IDX "_upperbound"); \ + Rft::template add<&Derived::db_##IDX##_end>("env", "db_" #IDX "_end"); \ + Rft::template add<&Derived::db_##IDX##_next>("env", "db_" #IDX "_next"); \ + Rft::template add<&Derived::db_##IDX##_previous>("env", "db_" #IDX "_previous"); #define DB_SECONDARY_INDEX_METHODS_ARRAY(IDX) \ - Rft::template add("env", "db_" #IDX "_store"); \ - Rft::template add("env", "db_" #IDX "_remove"); \ - Rft::template add("env", "db_" #IDX "_update"); \ - Rft::template add("env", "db_" #IDX "_find_primary"); \ - Rft::template add("env", "db_" #IDX "_find_secondary"); \ - Rft::template add("env", "db_" #IDX "_lowerbound"); \ - Rft::template add("env", "db_" #IDX "_upperbound"); \ - Rft::template add("env", "db_" #IDX "_end"); \ - Rft::template add("env", "db_" #IDX "_next"); \ - Rft::template add("env", "db_" #IDX "_previous"); + Rft::template add<&Derived::db_##IDX##_store>("env", "db_" #IDX "_store"); \ + Rft::template add<&Derived::db_##IDX##_remove>("env", "db_" #IDX "_remove"); \ + Rft::template add<&Derived::db_##IDX##_update>("env", "db_" #IDX "_update"); \ + Rft::template add<&Derived::db_##IDX##_find_primary>("env", "db_" #IDX "_find_primary"); \ + Rft::template add<&Derived::db_##IDX##_find_secondary>("env", "db_" #IDX "_find_secondary"); \ + Rft::template add<&Derived::db_##IDX##_lowerbound>("env", "db_" #IDX "_lowerbound"); \ + Rft::template add<&Derived::db_##IDX##_upperbound>("env", "db_" #IDX "_upperbound"); \ + Rft::template add<&Derived::db_##IDX##_end>("env", "db_" #IDX "_end"); \ + Rft::template add<&Derived::db_##IDX##_next>("env", "db_" #IDX "_next"); \ + Rft::template add<&Derived::db_##IDX##_previous>("env", "db_" #IDX "_previous"); // database_api DB_SECONDARY_INDEX_METHODS_SIMPLE(idx64) @@ -193,58 +187,55 @@ struct unimplemented_callbacks { #undef DB_SECONDARY_INDEX_METHODS_ARRAY // crypto_api - Rft::template add("env", "assert_recover_key"); - Rft::template add("env", "recover_key"); - Rft::template add("env", "assert_sha256"); - Rft::template add("env", "assert_sha1"); - Rft::template add("env", "assert_sha512"); - Rft::template add("env", "assert_ripemd160"); - Rft::template add("env", "sha1"); - Rft::template add("env", "sha256"); - Rft::template add("env", "sha512"); - Rft::template add("env", "ripemd160"); + Rft::template add<&Derived::assert_recover_key>("env", "assert_recover_key"); + Rft::template add<&Derived::recover_key>("env", "recover_key"); + Rft::template add<&Derived::assert_sha256>("env", "assert_sha256"); + Rft::template add<&Derived::assert_sha1>("env", "assert_sha1"); + Rft::template add<&Derived::assert_sha512>("env", "assert_sha512"); + Rft::template add<&Derived::assert_ripemd160>("env", "assert_ripemd160"); + Rft::template add<&Derived::sha1>("env", "sha1"); + Rft::template add<&Derived::sha256>("env", "sha256"); + Rft::template add<&Derived::sha512>("env", "sha512"); + Rft::template add<&Derived::ripemd160>("env", "ripemd160"); // permission_api - Rft::template add( - "env", "check_transaction_authorization"); - Rft::template add("env", - "check_permission_authorization"); - Rft::template add("env", "get_permission_last_used"); - Rft::template add("env", "get_account_creation_time"); + Rft::template add<&Derived::check_transaction_authorization>("env", "check_transaction_authorization"); + Rft::template add<&Derived::check_permission_authorization>("env", "check_permission_authorization"); + Rft::template add<&Derived::get_permission_last_used>("env", "get_permission_last_used"); + Rft::template add<&Derived::get_account_creation_time>("env", "get_account_creation_time"); // system_api - Rft::template add("env", "publication_time"); - Rft::template add("env", "is_feature_activated"); - Rft::template add("env", "get_sender"); + Rft::template add<&Derived::publication_time>("env", "publication_time"); + Rft::template add<&Derived::is_feature_activated>("env", "is_feature_activated"); + Rft::template add<&Derived::get_sender>("env", "get_sender"); // context_free_system_api - Rft::template add("env", "eosio_assert"); - Rft::template add("env", "eosio_assert_code"); - Rft::template add("env", "eosio_exit"); + Rft::template add<&Derived::eosio_assert_code>("env", "eosio_assert_code"); + Rft::template add<&Derived::eosio_exit>("env", "eosio_exit"); // authorization_api - Rft::template add("env", "require_recipient"); - Rft::template add("env", "require_auth"); - Rft::template add("env", "require_auth2"); - Rft::template add("env", "has_auth"); - Rft::template add("env", "is_account"); + Rft::template add<&Derived::require_recipient>("env", "require_recipient"); + Rft::template add<&Derived::require_auth>("env", "require_auth"); + Rft::template add<&Derived::require_auth2>("env", "require_auth2"); + Rft::template add<&Derived::has_auth>("env", "has_auth"); + Rft::template add<&Derived::is_account>("env", "is_account"); // context_free_transaction_api - Rft::template add("env", "read_transaction"); - Rft::template add("env", "transaction_size"); - Rft::template add("env", "expiration"); - Rft::template add("env", "tapos_block_prefix"); - Rft::template add("env", "tapos_block_num"); - Rft::template add("env", "get_action"); + Rft::template add<&Derived::read_transaction>("env", "read_transaction"); + Rft::template add<&Derived::transaction_size>("env", "transaction_size"); + Rft::template add<&Derived::expiration>("env", "expiration"); + Rft::template add<&Derived::tapos_block_prefix>("env", "tapos_block_prefix"); + Rft::template add<&Derived::tapos_block_num>("env", "tapos_block_num"); + Rft::template add<&Derived::get_action>("env", "get_action"); // transaction_api - Rft::template add("env", "send_inline"); - Rft::template add("env", "send_context_free_inline"); - Rft::template add("env", "send_deferred"); - Rft::template add("env", "cancel_deferred"); + Rft::template add<&Derived::send_inline>("env", "send_inline"); + Rft::template add<&Derived::send_context_free_inline>("env", "send_context_free_inline"); + Rft::template add<&Derived::send_deferred>("env", "send_deferred"); + Rft::template add<&Derived::cancel_deferred>("env", "cancel_deferred"); // context_free_api - Rft::template add("env", "get_context_free_data"); + Rft::template add<&Derived::get_context_free_data>("env", "get_context_free_data"); } // register_callbacks() }; // unimplemented_callbacks diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented_filter.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented_filter.hpp index 3c754c16e6a..850125e267c 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented_filter.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented_filter.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace b1::rodeos { @@ -14,10 +14,10 @@ struct unimplemented_filter_callbacks { // system_api int64_t current_time() { return unimplemented("current_time"); } - template + template static void register_callbacks() { - // system_api - Rft::template add("env", "current_time"); + // todo: preconditions + Rft::template add<&Derived::current_time>("env", "current_time"); } }; // unimplemented_filter_callbacks diff --git a/libraries/rodeos/include/b1/rodeos/filter.hpp b/libraries/rodeos/include/b1/rodeos/filter.hpp index 5dfed0649a2..8b4b405d288 100644 --- a/libraries/rodeos/include/b1/rodeos/filter.hpp +++ b/libraries/rodeos/include/b1/rodeos/filter.hpp @@ -14,18 +14,17 @@ namespace b1::rodeos::filter { struct callbacks; -using backend_t = eosio::vm::backend; -using rhf_t = eosio::vm::registered_host_functions; +using rhf_t = registered_host_functions; +using backend_t = eosio::vm::backend; struct filter_state : b1::rodeos::data_state, b1::rodeos::console_state, b1::rodeos::filter_callback_state { eosio::vm::wasm_allocator wa = {}; }; -// todo: remove basic_callbacks -struct callbacks : b1::rodeos::basic_callbacks, - b1::rodeos::chaindb_callbacks, +struct callbacks : b1::rodeos::chaindb_callbacks, b1::rodeos::compiler_builtins_callbacks, b1::rodeos::console_callbacks, + b1::rodeos::context_free_system_callbacks, b1::rodeos::data_callbacks, b1::rodeos::db_callbacks, b1::rodeos::filter_callbacks, @@ -47,16 +46,16 @@ struct callbacks : b1::rodeos::basic_callbacks, }; inline void register_callbacks() { - b1::rodeos::basic_callbacks::register_callbacks(); - b1::rodeos::chaindb_callbacks::register_callbacks(); - b1::rodeos::compiler_builtins_callbacks::register_callbacks(); - b1::rodeos::console_callbacks::register_callbacks(); - b1::rodeos::data_callbacks::register_callbacks(); - b1::rodeos::db_callbacks::register_callbacks(); - b1::rodeos::filter_callbacks::register_callbacks(); - b1::rodeos::memory_callbacks::register_callbacks(); - b1::rodeos::unimplemented_callbacks::register_callbacks(); - b1::rodeos::unimplemented_filter_callbacks::register_callbacks(); + b1::rodeos::chaindb_callbacks::register_callbacks(); + b1::rodeos::compiler_builtins_callbacks::register_callbacks(); + b1::rodeos::console_callbacks::register_callbacks(); + b1::rodeos::context_free_system_callbacks::register_callbacks(); + b1::rodeos::data_callbacks::register_callbacks(); + b1::rodeos::db_callbacks::register_callbacks(); + b1::rodeos::filter_callbacks::register_callbacks(); + b1::rodeos::memory_callbacks::register_callbacks(); + b1::rodeos::unimplemented_callbacks::register_callbacks(); + b1::rodeos::unimplemented_filter_callbacks::register_callbacks(); } } // namespace b1::rodeos::filter diff --git a/libraries/rodeos/include/b1/rodeos/get_state_row.hpp b/libraries/rodeos/include/b1/rodeos/get_state_row.hpp index 9821533ff83..ab440a9ad06 100644 --- a/libraries/rodeos/include/b1/rodeos/get_state_row.hpp +++ b/libraries/rodeos/include/b1/rodeos/get_state_row.hpp @@ -3,12 +3,15 @@ #include #include +namespace b1::rodeos { + template std::optional, T>> get_state_row(chain_kv::view& view, const K& key) { std::optional, T>> result; result.emplace(); result->first = - view.get(eosio::name{ "state" }.value, chain_kv::to_slice(eosio::check(eosio::convert_to_key(key)).value())); + view.get(eosio::name{ "state" }.value, + chain_kv::to_slice(eosio::check(eosio::convert_to_key(std::make_tuple((uint8_t)0x01, key))).value())); if (!result->first) { result.reset(); return result; @@ -25,7 +28,8 @@ std::optional, T>> get_state_ro const K& key) { std::optional, T>> result; auto pk = - view.get(eosio::name{ "state" }.value, chain_kv::to_slice(eosio::check(eosio::convert_to_key(key)).value())); + view.get(eosio::name{ "state" }.value, + chain_kv::to_slice(eosio::check(eosio::convert_to_key(std::make_tuple((uint8_t)0x01, key))).value())); if (!pk) return result; @@ -41,3 +45,5 @@ std::optional, T>> get_state_ro throw std::runtime_error("An error occurred deserializing state: " + r.error().message()); return result; } + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/rodeos.hpp b/libraries/rodeos/include/b1/rodeos/rodeos.hpp index e0116a4b2fb..67a9b9af930 100644 --- a/libraries/rodeos/include/b1/rodeos/rodeos.hpp +++ b/libraries/rodeos/include/b1/rodeos/rodeos.hpp @@ -1,6 +1,6 @@ +#include #include #include -#include #include #include @@ -53,11 +53,11 @@ struct rodeos_db_snapshot { void refresh(); void end_write(bool write_fill); - void start_block(const ship_protocol::get_blocks_result_v0& result); - void end_block(const ship_protocol::get_blocks_result_v0& result, bool force_write); - void check_write(const ship_protocol::get_blocks_result_v0& result); - void write_block_info(const ship_protocol::get_blocks_result_v0& result); - void write_deltas(const ship_protocol::get_blocks_result_v0& result, std::function shutdown); + void start_block(const eosio::ship_protocol::get_blocks_result_v0& result); + void end_block(const eosio::ship_protocol::get_blocks_result_v0& result, bool force_write); + void check_write(const eosio::ship_protocol::get_blocks_result_v0& result); + void write_block_info(const eosio::ship_protocol::get_blocks_result_v0& result); + void write_deltas(const eosio::ship_protocol::get_blocks_result_v0& result, std::function shutdown); private: void write_fill_status(); @@ -70,7 +70,7 @@ struct rodeos_filter { rodeos_filter(eosio::name name, const std::string& wasm_filename); - void process(rodeos_db_snapshot& snapshot, const ship_protocol::get_blocks_result_v0& result, + void process(rodeos_db_snapshot& snapshot, const eosio::ship_protocol::get_blocks_result_v0& result, eosio::input_stream bin, const std::function& push_data); }; diff --git a/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp b/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp index 424b634736d..576d0f75c5e 100644 --- a/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp +++ b/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp @@ -8,10 +8,24 @@ namespace eosio { using b1::rodeos::kv_environment; } -#include "../wasms/table.hpp" +#include namespace b1::rodeos { +using account = eosio::ship_protocol::account; +using account_metadata = eosio::ship_protocol::account_metadata; +using code = eosio::ship_protocol::code; +using contract_index128 = eosio::ship_protocol::contract_index128; +using contract_index64 = eosio::ship_protocol::contract_index64; +using contract_row = eosio::ship_protocol::contract_row; +using contract_table = eosio::ship_protocol::contract_table; +using global_property = eosio::ship_protocol::global_property; +using key_value = eosio::ship_protocol::key_value; +using key_value_v0 = eosio::ship_protocol::key_value_v0; +using producer_schedule = eosio::ship_protocol::producer_schedule; +using table_delta = eosio::ship_protocol::table_delta; +using table_delta_v0 = eosio::ship_protocol::table_delta_v0; + struct fill_status_v0 { eosio::checksum256 chain_id = {}; uint32_t head = {}; @@ -32,13 +46,7 @@ inline bool operator==(const fill_status_v0& a, fill_status_v0& b) { inline bool operator!=(const fill_status_v0& a, fill_status_v0& b) { return !(a == b); } -struct fill_status_kv : eosio::table { - index primary_index{ eosio::name{ "primary" }, [](const auto& var) { return std::vector{}; } }; - - fill_status_kv(eosio::kv_environment environment) : eosio::table{ std::move(environment) } { - init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "fill.status" }, primary_index); - } -}; +using fill_status_kv = eosio::kv_singleton; struct block_info_v0 { uint32_t num = {}; @@ -60,157 +68,140 @@ EOSIO_REFLECT(block_info_v0, num, id, timestamp, producer, confirmed, previous, using block_info = std::variant; // todo: move out of "state"? -struct block_info_kv : eosio::table { - index primary_index{ eosio::name{ "primary" }, [](const auto& var) { - return std::visit( - [](const auto& obj) { return eosio::check(eosio::convert_to_key(obj.num)).value(); }, - var); - } }; - - index id_index{ eosio::name{ "id" }, [](const auto& var) { - return std::visit( - [](const auto& obj) { return eosio::check(eosio::convert_to_key(obj.id)).value(); }, var); - } }; - - block_info_kv(eosio::kv_environment environment) : eosio::table{ std::move(environment) } { +struct block_info_kv : eosio::kv_table { + index primary_index{ eosio::name{ "primary" }, [](const auto& var) { + return std::visit([](const auto& obj) { return obj.num; }, *var); + } }; + + index id_index{ eosio::name{ "id" }, [](const auto& var) { + return std::visit([](const auto& obj) { return obj.id; }, *var); + } }; + + block_info_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "block.info" }, primary_index, id_index); } }; -struct global_property_kv : eosio::table { - index primary_index{ eosio::name{ "primary" }, [](const auto& var) { return std::vector{}; } }; +struct global_property_kv : eosio::kv_table { + index> primary_index{ eosio::name{ "primary" }, + [](const auto& var) { return std::vector{}; } }; - global_property_kv(eosio::kv_environment environment) : eosio::table{ std::move(environment) } { + global_property_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "global.prop" }, primary_index); } }; -struct account_kv : eosio::table { - index primary_index{ eosio::name{ "primary" }, [](const auto& var) { - return std::visit( - [](const auto& obj) { - return eosio::check(eosio::convert_to_key(std::tie(obj.name))).value(); - }, - var); - } }; +struct account_kv : eosio::kv_table { + index primary_index{ eosio::name{ "primary" }, [](const auto& var) { + return std::visit([](const auto& obj) { return obj.name; }, *var); + } }; - account_kv(eosio::kv_environment environment) : eosio::table{ std::move(environment) } { + account_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "account" }, primary_index); } }; -struct account_metadata_kv : eosio::table { - index primary_index{ eosio::name{ "primary" }, [](const auto& var) { - return std::visit( - [](const auto& obj) { - return eosio::check(eosio::convert_to_key(std::tie(obj.name))).value(); - }, - var); - } }; +struct account_metadata_kv : eosio::kv_table { + index primary_index{ eosio::name{ "primary" }, [](const auto& var) { + return std::visit([](const auto& obj) { return obj.name; }, *var); + } }; - account_metadata_kv(eosio::kv_environment environment) : eosio::table{ std::move(environment) } { + account_metadata_kv(eosio::kv_environment environment) + : eosio::kv_table{ std::move(environment) } { init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "account.meta" }, primary_index); } }; -struct code_kv : eosio::table { - index primary_index{ eosio::name{ "primary" }, [](const auto& var) { - return std::visit( - [](const auto& obj) { - return eosio::check(eosio::convert_to_key( - std::tie(obj.vm_type, obj.vm_version, obj.code_hash))) - .value(); - }, - var); - } }; - - code_kv(eosio::kv_environment environment) : eosio::table{ std::move(environment) } { +struct code_kv : eosio::kv_table { + index> primary_index{ + eosio::name{ "primary" }, + [](const auto& var) { + return std::visit([](const auto& obj) { return std::tie(obj.vm_type, obj.vm_version, obj.code_hash); }, *var); + } + }; + + code_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "code" }, primary_index); } }; -struct contract_table_kv : eosio::table { - index primary_index{ +struct contract_table_kv : eosio::kv_table { + index> primary_index{ eosio::name{ "primary" }, [](const auto& var) { - return std::visit( - [](const auto& obj) { - return eosio::check(eosio::convert_to_key(std::tie(obj.code, obj.table, obj.scope))).value(); - }, - var); + return std::visit([](const auto& obj) { return std::tie(obj.code, obj.table, obj.scope); }, *var); } }; - contract_table_kv(eosio::kv_environment environment) : eosio::table{ std::move(environment) } { + contract_table_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "contract.tab" }, primary_index); } }; -struct contract_row_kv : eosio::table { - index primary_index{ eosio::name{ "primary" }, [](const auto& var) { - return std::visit( - [](const auto& obj) { - return eosio::check(eosio::convert_to_key( - std::tie(obj.code, obj.table, obj.scope, obj.primary_key))) - .value(); - }, - var); - } }; - - contract_row_kv(eosio::kv_environment environment) : eosio::table{ std::move(environment) } { +struct contract_row_kv : eosio::kv_table { + using PT = typename std::tuple; + index primary_index{ eosio::name{ "primary" }, [](const auto& var) { + return std::visit( + [](const auto& obj) { + return std::tie(obj.code, obj.table, obj.scope, obj.primary_key); + }, + *var); + } }; + + contract_row_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "contract.row" }, primary_index); } }; -struct contract_index64_kv : eosio::table { - index primary_index{ eosio::name{ "primary" }, [](const auto& var) { - return std::visit( - [](const auto& obj) { - return eosio::check(eosio::convert_to_key( - std::tie(obj.code, obj.table, obj.scope, obj.primary_key))) - .value(); - }, - var); - } }; - index secondary_index{ eosio::name{ "secondary" }, [](const auto& var) { - return std::visit( - [](const auto& obj) { - return eosio::check( - eosio::convert_to_key(std::tie(obj.code, obj.table, obj.scope, - obj.secondary_key, obj.primary_key))) - .value(); - }, - var); - } }; - - contract_index64_kv(eosio::kv_environment environment) : eosio::table{ std::move(environment) } { +struct contract_index64_kv : eosio::kv_table { + using PT = typename std::tuple; + index primary_index{ eosio::name{ "primary" }, [](const auto& var) { + return std::visit( + [](const auto& obj) { + return std::tie(obj.code, obj.table, obj.scope, obj.primary_key); + }, + *var); + } }; + using ST = typename std::tuple; + index secondary_index{ eosio::name{ "secondary" }, [](const auto& var) { + return std::visit( + [](const auto& obj) { + return std::tie(obj.code, obj.table, obj.scope, obj.secondary_key, + obj.primary_key); + }, + *var); + } }; + + contract_index64_kv(eosio::kv_environment environment) + : eosio::kv_table{ std::move(environment) } { init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "contract.i1" }, primary_index, secondary_index); } }; -struct contract_index128_kv : eosio::table { - index primary_index{ eosio::name{ "primary" }, [](const auto& var) { - return std::visit( - [](const auto& obj) { - return eosio::check(eosio::convert_to_key( - std::tie(obj.code, obj.table, obj.scope, obj.primary_key))) - .value(); - }, - var); - } }; - index secondary_index{ eosio::name{ "secondary" }, [](const auto& var) { - return std::visit( - [](const auto& obj) { - return eosio::check( - eosio::convert_to_key(std::tie(obj.code, obj.table, obj.scope, - obj.secondary_key, obj.primary_key))) - .value(); - }, - var); - } }; - - contract_index128_kv(eosio::kv_environment environment) : eosio::table{ std::move(environment) } { +struct contract_index128_kv : eosio::kv_table { + using PT = typename std::tuple; + index primary_index{ eosio::name{ "primary" }, [](const auto& var) { + return std::visit( + [](const auto& obj) { + return std::tie(obj.code, obj.table, obj.scope, obj.primary_key); + }, + *var); + } }; + using ST = typename std::tuple; + index secondary_index{ eosio::name{ "secondary" }, [](const auto& var) { + return std::visit( + [](const auto& obj) { + return std::tie(obj.code, obj.table, obj.scope, obj.secondary_key, + obj.primary_key); + }, + *var); + } }; + + contract_index128_kv(eosio::kv_environment environment) + : eosio::kv_table{ std::move(environment) } { init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "contract.i2" }, primary_index, secondary_index); } @@ -223,7 +214,7 @@ void store_delta_typed(eosio::kv_environment environment, table_delta_v0& delta, f(); auto obj = eosio::check(eosio::from_bin(row.data)).value(); if (row.present) - table.insert(obj, bypass_preexist_check); + table.put(obj); else table.erase(obj); } diff --git a/libraries/rodeos/include/b1/rodeos/wasm_ql.hpp b/libraries/rodeos/include/b1/rodeos/wasm_ql.hpp index 21347f32205..05d16bac5ed 100644 --- a/libraries/rodeos/include/b1/rodeos/wasm_ql.hpp +++ b/libraries/rodeos/include/b1/rodeos/wasm_ql.hpp @@ -1,13 +1,10 @@ #pragma once -#include - #include #include #include #include #include -#include namespace b1::rodeos::wasm_ql { @@ -25,9 +22,7 @@ struct shared_state { ~shared_state(); }; -struct thread_state : eosio::history_tools::action_state, - eosio::history_tools::console_state, - eosio::history_tools::query_state { +struct thread_state : action_state, console_state, query_state { std::shared_ptr shared = {}; eosio::vm::wasm_allocator wa = {}; }; @@ -69,9 +64,9 @@ const std::vector& query_get_required_keys(wasm_ql::thread_state& thread_s const std::vector& query_send_transaction(wasm_ql::thread_state& thread_state, const std::vector& contract_kv_prefix, std::string_view body, bool return_trace_on_except); -ship_protocol::transaction_trace_v0 +eosio::ship_protocol::transaction_trace_v0 query_send_transaction(wasm_ql::thread_state& thread_state, const std::vector& contract_kv_prefix, - const ship_protocol::packed_transaction& trx, const rocksdb::Snapshot* snapshot, + const eosio::ship_protocol::packed_transaction& trx, const rocksdb::Snapshot* snapshot, std::vector>& memory, bool return_trace_on_except); } // namespace b1::rodeos::wasm_ql diff --git a/libraries/rodeos/include/eosio/datastream.hpp b/libraries/rodeos/include/eosio/datastream.hpp new file mode 100644 index 00000000000..3db72ba5cbc --- /dev/null +++ b/libraries/rodeos/include/eosio/datastream.hpp @@ -0,0 +1,453 @@ +// clang-format off +/** + * @file datastream.hpp + * @copyright defined in eos/LICENSE + */ +#pragma once +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +//#include +//#include +//#include +//#include + +//#include +//#include + +namespace eosio { + +/** + * @defgroup datastream Data Stream + * @ingroup core + * @brief Defines data stream for reading and writing data in the form of bytes + */ + +/** + * A data stream for reading and writing data in the form of bytes + * + * @tparam T - Type of the datastream buffer + */ +template +class datastream { + public: + /** + * Construct a new datastream object + * + * @details Construct a new datastream object given the size of the buffer and start position of the buffer + * @param start - The start position of the buffer + * @param s - The size of the buffer + */ + datastream( T start, size_t s ) + :_start(start),_pos(start),_end(start+s){} + + result check_available(size_t size) { + if (size > size_t(_end - _pos)) + return stream_error::overrun; + return outcome::success(); + } + + result read_reuse_storage(const char*& result, size_t size) { + if (size > size_t(_end - _pos)) + return stream_error::overrun; + result = _pos; + _pos += size; + return outcome::success(); + } + + /** + * Skips a specified number of bytes from this stream + * + * @param s - The number of bytes to skip + */ + inline result skip( size_t s ){ + _pos += s; + if (_pos > _end ) + return stream_error::overrun; + return outcome::success(); + } + + /** + * Reads a specified number of bytes from the stream into a buffer + * + * @param d - The pointer to the destination buffer + * @param s - the number of bytes to read + * @return true + */ + inline result read( char* d, size_t s ) { + eosio::check( size_t(_end - _pos) >= (size_t)s, "read" ); + memcpy( d, _pos, s ); + _pos += s; + return outcome::success(); + } + + /** + * Writes a specified number of bytes into the stream from a buffer + * + * @param d - The pointer to the source buffer + * @param s - The number of bytes to write + * @return true + */ + inline result write( const char* d, size_t s ) { + eosio::check( _end - _pos >= (int32_t)s, "write" ); + memcpy( (void*)_pos, d, s ); + _pos += s; + return outcome::success(); + } + + /** + * Writes a byte into the stream + * + * @brief Writes a byte into the stream + * @param c byte to write + * @return true + */ + inline result put(char c) { + eosio::check( _pos < _end, "put" ); + *_pos = c; + ++_pos; + return outcome::success(); + } + + inline result write(char c) { return put(c); } + + /** + * Reads a byte from the stream + * + * @brief Reads a byte from the stream + * @param c - The reference to destination byte + * @return true + */ + inline result get( unsigned char& c ) { return get( *(char*)&c ); } + + /** + * Reads a byte from the stream + * + * @brief Reads a byte from the stream + * @param c - The reference to destination byte + * @return true + */ + inline result get( char& c ) + { + eosio::check( _pos < _end, "get" ); + c = *_pos; + ++_pos; + return outcome::success(); + } + + /** + * Retrieves the current position of the stream + * + * @brief Retrieves the current position of the stream + * @return T - The current position of the stream + */ + T pos()const { return _pos; } + T get_pos()const { return _pos; } + inline bool valid()const { return _pos <= _end && _pos >= _start; } + + /** + * Sets the position within the current stream + * + * @brief Sets the position within the current stream + * @param p - The offset relative to the origin + * @return true if p is within the range + * @return false if p is not within the rawnge + */ + inline bool seekp(size_t p) { _pos = _start + p; return _pos <= _end; } + + /** + * Gets the position within the current stream + * + * @brief Gets the position within the current stream + * @return p - The position within the current stream + */ + inline size_t tellp()const { return size_t(_pos - _start); } + + /** + * Returns the number of remaining bytes that can be read/skipped + * + * @brief Returns the number of remaining bytes that can be read/skipped + * @return size_t - The number of remaining bytes + */ + inline size_t remaining()const { return _end - _pos; } + private: + /** + * The start position of the buffer + * + * @brief The start position of the buffer + */ + T _start; + /** + * The current position of the buffer + * + * @brief The current position of the buffer + */ + T _pos; + /** + * The end position of the buffer + * + * @brief The end position of the buffer + */ + T _end; +}; + +/** + * Specialization of datastream used to help determine the final size of a serialized value + */ +template<> +class datastream { + public: + /** + * Construct a new specialized datastream object given the initial size + * + * @param init_size - The initial size + */ + datastream( size_t init_size = 0):_size(init_size){} + + /** + * Increment the size by s. This behaves the same as write( const char* ,size_t s ). + * + * @param s - The amount of size to increase + * @return true + */ + inline result skip( size_t s ) { _size += s; return outcome::success(); } + + /** + * Increment the size by s. This behaves the same as skip( size_t s ) + * + * @param s - The amount of size to increase + * @return true + */ + inline result write( const char* ,size_t s ) { _size += s; return outcome::success(); } + + /** + * Increment the size by one + * + * @return true + */ + inline result put(char ) { ++_size; return outcome::success(); } + + inline result write(char c) { return put(c); } + + /** + * Check validity. It's always valid + * + * @return true + */ + inline bool valid()const { return true; } + + /** + * Set new size + * + * @brief Set new size + * @param p - The new size + * @return true + */ + inline bool seekp(size_t p) { _size = p; return true; } + + /** + * Get the size + * + * @return size_t - The size + */ + inline size_t tellp()const { return _size; } + + /** + * Always returns 0 + * + * @return size_t - 0 + */ + inline size_t remaining()const { return 0; } + private: + /** + * The size used to determine the final size of a serialized value. + */ + size_t _size; +}; + +namespace _datastream_detail { + /** + * Check if type T is a pointer + * + * @brief Check if type T is a pointer + * @tparam T - The type to be checked + * @return true if T is a pointer + * @return false otherwise + */ + template + constexpr bool is_pointer() { + return std::is_pointer::value || + std::is_null_pointer::value || + std::is_member_pointer::value; + } + + /** + * Check if type T is a primitive type + * + * @brief Check if type T is a primitive type + * @tparam T - The type to be checked + * @return true if T is a primitive type + * @return false otherwise + */ + template + constexpr bool is_primitive() { + return std::is_arithmetic::value || + std::is_enum::value; + } + + /* + * Check if type T is a specialization of datastream + * + * @brief Check if type T is a datastream + * @tparam T - The type to be checked + */ + template + struct is_datastream { static constexpr bool value = false; }; + template + struct is_datastream> { static constexpr bool value = true; }; + + namespace operator_detection { + // These overloads will be ambiguous with the operator in eosio, unless + // the user has provided an operator that is a better match. + template::value>* = nullptr> + DataStream& operator>>( DataStream& ds, T& v ); + template::value>* = nullptr> + DataStream& operator<<( DataStream& ds, const T& v ); + template + using require_specialized_right_shift = std::void_t() >> std::declval())>; + template + using require_specialized_left_shift = std::void_t() << std::declval())>; + } + // Going through enable_if/is_detected is necessary for reasons that I don't entirely understand. + template + using require_specialized_right_shift = std::enable_if_t>; + template + using require_specialized_left_shift = std::enable_if_t>; +} + +/** + * Serialize a class + * + * @brief Serialize a class + * @param ds - The stream to write + * @param v - The value to serialize + * @tparam DataStream - Type of datastream + * @tparam T - Type of class + * @return DataStream& - Reference to the datastream + */ +template::value>* = nullptr> +DataStream& operator<<( DataStream& ds, const T& v ) { + check_discard(to_bin(v, ds)); + return ds; +} + +// Backwards compatibility: allow user defined datastream operators to work with from_bin +template,T>* = nullptr> +result to_bin( const T& v, datastream& ds ) { + ds << v; + return outcome::success(); +} + +/** + * Deserialize a class + * + * @brief Deserialize a class + * @param ds - The stream to read + * @param v - The destination for deserialized value + * @tparam DataStream - Type of datastream + * @tparam T - Type of class + * @return DataStream& - Reference to the datastream + */ +template::value>* = nullptr> +DataStream& operator>>( DataStream& ds, T& v ) { + check_discard(from_bin(v, ds)); + return ds; +} + +template,T>* = nullptr> +result from_bin( T& v, datastream& ds ) { + ds >> v; + return outcome::success(); +} + +/** + * Unpack data inside a fixed size buffer as T + * + * @ingroup datastream + * @brief Unpack data inside a fixed size buffer as T + * @tparam T - Type of the unpacked data + * @param buffer - Pointer to the buffer + * @param len - Length of the buffer + * @return T - The unpacked data + */ +template +T unpack( const char* buffer, size_t len ) { + T result; + datastream ds(buffer,len); + ds >> result; + return result; +} + +/** + * Unpack data inside a variable size buffer as T + * + * @ingroup datastream + * @brief Unpack data inside a variable size buffer as T + * @tparam T - Type of the unpacked data + * @param bytes - Buffer + * @return T - The unpacked data + */ +template +T unpack( const std::vector& bytes ) { + return unpack( bytes.data(), bytes.size() ); +} + +/** + * Get the size of the packed data + * + * @ingroup datastream + * @brief Get the size of the packed data + * @tparam T - Type of the data to be packed + * @param value - Data to be packed + * @return size_t - Size of the packed data + */ +template +size_t pack_size( const T& value ) { + datastream ps; + ps << value; + return ps.tellp(); +} + +/** + * Get packed data + * + * @ingroup datastream + * @brief Get packed data + * @tparam T - Type of the data to be packed + * @param value - Data to be packed + * @return bytes - The packed data + */ +template +std::vector pack( const T& value ) { + std::vector result; + result.resize(pack_size(value)); + + datastream ds( result.data(), result.size() ); + ds << value; + return result; +} +} diff --git a/libraries/rodeos/include/eosio/key_value.hpp b/libraries/rodeos/include/eosio/key_value.hpp new file mode 100644 index 00000000000..36fd66880ba --- /dev/null +++ b/libraries/rodeos/include/eosio/key_value.hpp @@ -0,0 +1,957 @@ +// clang-format off +#pragma once +#include +#include +#include + +#include + +#include +#include +#include + +#define EOSIO_CDT_GET_RETURN_T(value_class, index_name) std::decay_t()))> + +/** + * @brief Macro to define an index. + * @details In the case where the autogenerated index names created by DEFINE_TABLE are not enough, a user can instead + * manually define the table and indices. This macro allows users to conveniently define an index without having to specify + * the index template type, as those can be large/unwieldy to type out. + * + * @param index_name - The index name. + * @param member_name - The name of the member pointer used for the index. This also defines the index's C++ variable name. + */ +#define KV_NAMED_INDEX(index_name, member_name) \ + index member_name{eosio::name{index_name}, &value_type::member_name}; + +namespace eosio { + namespace internal_use_do_not_use { + +#ifdef __eosio_cdt__ + +#define IMPORT extern "C" __attribute__((eosio_wasm_import)) + + // clang-format off + IMPORT void kv_erase(uint64_t db, uint64_t contract, const char* key, uint32_t key_size); + IMPORT void kv_set(uint64_t db, uint64_t contract, const char* key, uint32_t key_size, const char* value, uint32_t value_size); + IMPORT bool kv_get(uint64_t db, uint64_t contract, const char* key, uint32_t key_size, uint32_t& value_size); + IMPORT uint32_t kv_get_data(uint64_t db, uint32_t offset, char* data, uint32_t data_size); + IMPORT uint32_t kv_it_create(uint64_t db, uint64_t contract, const char* prefix, uint32_t size); + IMPORT void kv_it_destroy(uint32_t itr); + IMPORT int32_t kv_it_status(uint32_t itr); + IMPORT int32_t kv_it_compare(uint32_t itr_a, uint32_t itr_b); + IMPORT int32_t kv_it_key_compare(uint32_t itr, const char* key, uint32_t size); + IMPORT int32_t kv_it_move_to_end(uint32_t itr); + IMPORT int32_t kv_it_next(uint32_t itr); + IMPORT int32_t kv_it_prev(uint32_t itr); + IMPORT int32_t kv_it_lower_bound(uint32_t itr, const char* key, uint32_t size); + IMPORT int32_t kv_it_key(uint32_t itr, uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size); + IMPORT int32_t kv_it_value(uint32_t itr, uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size); + // xclang-format on + +#undef IMPORT + +#endif + } + +#ifdef __eosio_cdt__ +class kv_environment { + public: + kv_environment() {} + + void kv_set(const std::vector& k, const std::vector& v) { + internal_use_do_not_use::kv_set(k.data(), k.size(), v.data(), v.size()); + } + + // clang-format off + void kv_erase(uint64_t db, uint64_t contract, const char* key, uint32_t key_size) {return internal_use_do_not_use::kv_erase(db, contract, key, key_size);} + void kv_set(uint64_t db, uint64_t contract, const char* key, uint32_t key_size, const char* value, uint32_t value_size) {return internal_use_do_not_use::kv_set(db, contract, key, key_size, value, value_size);} + bool kv_get(uint64_t db, uint64_t contract, const char* key, uint32_t key_size, uint32_t& value_size) {return internal_use_do_not_use::kv_get(db, contract, key, key_size, value_size);} + uint32_t kv_get_data(uint64_t db, uint32_t offset, char* data, uint32_t data_size) {return internal_use_do_not_use::kv_get_data(db, offset, data, data_size);} + uint32_t kv_it_create(uint64_t db, uint64_t contract, const char* prefix, uint32_t size) {return internal_use_do_not_use::kv_it_create(db, contract, prefix, size);} + void kv_it_destroy(uint32_t itr) {return internal_use_do_not_use::kv_it_destroy(itr);} + int32_t kv_it_status(uint32_t itr) {return internal_use_do_not_use::kv_it_status(itr);} + int32_t kv_it_compare(uint32_t itr_a, uint32_t itr_b) {return internal_use_do_not_use::kv_it_compare(itr_a, itr_b);} + int32_t kv_it_key_compare(uint32_t itr, const char* key, uint32_t size) {return internal_use_do_not_use::kv_it_key_compare(itr, key, size);} + int32_t kv_it_move_to_end(uint32_t itr) {return internal_use_do_not_use::kv_it_move_to_end(itr);} + int32_t kv_it_next(uint32_t itr) {return internal_use_do_not_use::kv_it_next(itr);} + int32_t kv_it_prev(uint32_t itr) {return internal_use_do_not_use::kv_it_prev(itr);} + int32_t kv_it_lower_bound(uint32_t itr, const char* key, uint32_t size) {return internal_use_do_not_use::kv_it_lower_bound(itr, key, size);} + int32_t kv_it_key(uint32_t itr, uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) {return internal_use_do_not_use::kv_it_key(itr, offset, dest, size, actual_size);} + int32_t kv_it_value(uint32_t itr, uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) {return internal_use_do_not_use::kv_it_value(itr, offset, dest, size, actual_size);} + // xclang-format on +}; +#endif + +namespace detail { + constexpr inline size_t max_stack_buffer_size = 512; + + template + static void serialize(const V& value, void* buffer, size_t size) { + datastream ds((char*)buffer, size); + unsigned_int i{0}; + ds << i; + ds << value; + } + + template + static void serialize(const std::variant& value, void* buffer, size_t size) { + datastream ds((char*)buffer, size); + ds << value; + } + + template + static void deserialize(V& value, void* buffer, size_t size) { + unsigned_int idx; + datastream ds((char*)buffer, size); + + ds >> idx; + eosio::check(idx==unsigned_int(0), "there was an error deserializing this value."); + ds >> value; + } + + template + static void deserialize(std::variant& value, void* buffer, size_t size) { + datastream ds((char*)buffer, size); + ds >> value; + } + + template + static size_t get_size(const V& value) { + auto size = pack_size(value); + return size + 1; + } + + template + static size_t get_size(const std::variant& value) { + auto size = pack_size(value); + return size; + } +} + +/** + * The key_type struct is used to store the binary representation of a key. + */ +struct key_type : private std::vector { + key_type() = default; + + explicit key_type(std::vector&& v) : std::vector(v) {} + + key_type(char* str, size_t size) : std::vector(str, str+size) {} + + key_type operator+(const key_type& b) const { + key_type ret = *this; + ret += b; + return ret; + } + + key_type& operator+=(const key_type& b) { + this->insert(this->end(), b.begin(), b.end()); + return *this; + } + + static key_type from_hex( const std::string_view& str ) { + key_type out; + + check( str.size() % 2 == 0, "invalid hex string length" ); + out.reserve( str.size() / 2 ); + + auto start = str.data(); + auto end = start + str.size(); + for(const char* p = start; p != end; p+=2 ) { + auto hic = p[0]; + auto lowc = p[1]; + + uint8_t hi = hic <= '9' ? hic-'0' : 10+(hic-'a'); + uint8_t low = lowc <= '9' ? lowc-'0' : 10+(lowc-'a'); + + out.push_back( char((hi << 4) | low) ); + } + + return out; + } + + std::string to_hex() const { + const char* hex_characters = "0123456789abcdef"; + + uint32_t buffer_size = 2 * size(); + check(buffer_size >= size(), "length passed into printhex is too large"); + + void* buffer = buffer_size > detail::max_stack_buffer_size ? malloc(buffer_size) : alloca(buffer_size); + + char* b = reinterpret_cast(buffer); + const uint8_t* d = reinterpret_cast(data()); + for(uint32_t i = 0; i < size(); ++i) { + *b = hex_characters[d[i] >> 4]; + ++b; + *b = hex_characters[d[i] & 0x0f]; + ++b; + } + + std::string ret{reinterpret_cast(buffer), buffer_size}; + + if (buffer_size > detail::max_stack_buffer_size) { + free(buffer); + } + + return ret; + } + + using std::vector::data; + using std::vector::size; + using std::vector::resize; +}; + +/* @cond PRIVATE */ +template +inline key_type make_key(T&& t) { + auto bytes = convert_to_key(std::forward(t)); + eosio::check((bool)bytes, "There was a failure in make_key."); + return key_type(std::move(bytes.value())); +} + +inline key_type make_prefix(eosio::name table_name, eosio::name index_name, uint8_t status = 1) { + return make_key(std::make_tuple(status, table_name, index_name)); +} + +inline key_type table_key(const key_type& prefix, const key_type& key) { + return prefix + key; +} +/* @endcond */ + +// This is the "best" way to document a function that does not technically exist using Doxygen. +#if EOSIO_CDT_DOXYGEN +/** + * @brief A function for converting types to the appropriate binary representation for the EOSIO Key Value database. + * @details The CDT provides implementations of this function for many of the common primitives and for structs/tuples. + * If sticking with standard types, contract developers should not need to interact with this function. + * If doing something more advanced, contract developers may need to provide their own implementation for a special type. + */ +template +inline key_type make_key(T val) { + return {}; +} +#endif + +static constexpr eosio::name kv_ram = "eosio.kvram"_n; +static constexpr eosio::name kv_disk = "eosio.kvdisk"_n; + +struct default_constructor_tag; + +/** + * @defgroup keyvalue Key Value Table + * @ingroup contracts + * + * @brief Defines an EOSIO Key Value Table + * @details EOSIO Key Value API provides a C++ interface to the EOSIO Key Value database. + * Key Value Tables require 1 primary index, of any type that can be serialized to a binary representation. + * Key Value Tables support 0 or more secondary index, of any type that can be serialized to a binary representation. + * Indexes must be a member variable or a member function. + * + * @tparam T - the type of the data stored as the value of the table + */ +template +class kv_table { + + class kv_index; + + class iterator { + public: + enum class status { + iterator_ok = 0, // Iterator is positioned at a key-value pair + iterator_erased = -1, // The key-value pair that the iterator used to be positioned at was erased + iterator_end = -2, // Iterator is out-of-bounds + }; + + iterator() = default; + + iterator(uint32_t itr, status itr_stat, const kv_index* index) : itr{itr}, itr_stat{itr_stat}, index{index} {} + + iterator(iterator&& other) : + itr(std::exchange(other.itr, 0)), + itr_stat(std::move(other.itr_stat)) + {} + + ~iterator() { + if (itr) { + index->tbl->environment.kv_it_destroy(itr); + } + } + + iterator& operator=(iterator&& other) { + if (itr) { + index->tbl->environment.kv_it_destroy(itr); + } + itr = std::exchange(other.itr, 0); + itr_stat = std::move(other.itr_stat); + return *this; + } + + bool good()const { return itr_stat != status::iterator_end; } + + /** + * Returns the value that the iterator points to. + * @ingroup keyvalue + * + * @return The value that the iterator points to. + */ + T value() const { + using namespace detail; + + eosio::check(itr_stat != status::iterator_end, "Cannot read end iterator"); + + uint32_t value_size; + uint32_t actual_value_size; + uint32_t actual_data_size; + uint32_t offset = 0; + + // call once to get the value_size + index->tbl->environment.kv_it_value(itr, 0, (char*)nullptr, 0, value_size); + + void* buffer = value_size > detail::max_stack_buffer_size ? malloc(value_size) : alloca(value_size); + auto stat = index->tbl->environment.kv_it_value(itr, offset, (char*)buffer, value_size, actual_value_size); + + eosio::check(static_cast(stat) == status::iterator_ok, "Error reading value"); + + void* deserialize_buffer = buffer; + size_t deserialize_size = actual_value_size; + + bool is_primary = index->index_name == index->tbl->primary_index_name; + if (!is_primary) { + auto success = index->tbl->environment.kv_get(index->tbl->db_name, index->contract_name.value, (char*)buffer, actual_value_size, actual_data_size); + eosio::check(success, "failure getting primary key in `value()`"); + + void* pk_buffer = actual_data_size > detail::max_stack_buffer_size ? malloc(actual_data_size) : alloca(actual_data_size); + index->tbl->environment.kv_get_data(index->tbl->db_name, 0, (char*)pk_buffer, actual_data_size); + + deserialize_buffer = pk_buffer; + deserialize_size = actual_data_size; + } + + T val; + detail::deserialize(val, deserialize_buffer, deserialize_size); + + if (value_size > detail::max_stack_buffer_size) { + free(buffer); + } + + if (is_primary && actual_data_size > detail::max_stack_buffer_size) { + free(deserialize_buffer); + } + return val; + } + + key_type key() const { + uint32_t actual_value_size; + uint32_t value_size; + + // call once to get the value size + index->tbl->environment.kv_it_key(itr, 0, (char*)nullptr, 0, value_size); + + void* buffer = value_size > detail::max_stack_buffer_size ? malloc(value_size) : alloca(value_size); + auto stat = index->tbl->environment.kv_it_key(itr, 0, (char*)buffer, value_size, actual_value_size); + + eosio::check(static_cast(stat) == status::iterator_ok, "Error getting key"); + + return {(char*)buffer, actual_value_size}; + } + + iterator& operator++() { + eosio::check(itr_stat != status::iterator_end, "cannot increment end iterator"); + itr_stat = static_cast(index->tbl->environment.kv_it_next(itr)); + return *this; + } + + iterator& operator--() { + if (!itr) { + itr = index->tbl->environment.kv_it_create(index->tbl->db_name, index->contract_name.value, index->prefix.data(), index->prefix.size()); + } + itr_stat = static_cast(index->tbl->environment.kv_it_prev(itr)); + eosio::check(itr_stat != status::iterator_end, "decremented past the beginning"); + return *this; + } + + int32_t key_compare(key_type kt) const { + if (itr == 0 || itr_stat == status::iterator_end) { + return 1; + } else { + return index->tbl->environment.kv_it_key_compare(itr, kt.data(), kt.size()); + } + } + + bool operator==(const iterator& b) const { + return compare(b) == 0; + } + + bool operator!=(const iterator& b) const { + return compare(b) != 0; + } + + bool operator<(const iterator& b) const { + return compare(b) < 0; + } + + bool operator<=(const iterator& b) const { + return compare(b) <= 0; + } + + bool operator>(const iterator& b) const { + return compare(b) > 0; + } + + bool operator>=(const iterator& b) const { + return compare(b) >= 0; + } + + private: + uint32_t itr; + status itr_stat; + + const kv_index* index; + + int compare(const iterator& b) const { + bool a_is_end = !itr || itr_stat == status::iterator_end; + bool b_is_end = !b.itr || b.itr_stat == status::iterator_end; + if (a_is_end && b_is_end) { + return 0; + } else if (a_is_end && b.itr) { + return 1; + } else if (itr && b_is_end) { + return -1; + } else { + return index->tbl->environment.kv_it_compare(itr, b.itr); + } + } + }; + + class kv_index { + + public: + eosio::name index_name; + eosio::name table_name; + eosio::name contract_name; + + key_type to_table_key( const key_type& k )const{ return table_key( prefix, k ); } + + protected: + kv_index() = default; + + template + kv_index(eosio::name index_name, KF&& kf) : index_name{index_name} { + key_function = [=](const T& t) { + return make_key(std::invoke(kf, &t)); + }; + } + + key_type get_key(const T& inst) const { return key_function(inst); } + kv_table* tbl; + key_type prefix; + + private: + friend kv_table; + + std::function key_function; + + virtual void setup() = 0; + }; + +public: + + using iterator = kv_table::iterator; + using value_type = T; + + /** + * @ingroup keyvalue + * + * @brief Defines an index on an EOSIO Key Value Table + * @details A Key Value Index allows a user of the table to search based on a given field. + * The only restrictions on that field are that it is serializable to a binary representation sortable by the KV intrinsics. + * Convenience functions exist to handle most of the primitive types as well as some more complex types, and are + * used automatically where possible. + * + * @tparam K - The type of the key used in the index. + */ + template + class index : public kv_index { + public: + using kv_table::kv_index::tbl; + using kv_table::kv_index::table_name; + using kv_table::kv_index::contract_name; + using kv_table::kv_index::index_name; + using kv_table::kv_index::prefix; + + template + index(eosio::name name, KF&& kf) : kv_index{name, kf} { + static_assert(std::is_same_v()))>>>, + "Make sure the variable/function passed to the constructor returns the same type as the template parameter."); + } + + /** + * Search for an existing object in a table by the index, using the given key. + * @ingroup keyvalue + * + * @param key - The key to search for. + * @return An iterator to the found object OR the `end` iterator if the given key was not found. + */ + iterator find(const K& key) const { + auto t_key = table_key(prefix, make_key(key)); + + return find(t_key); + } + + iterator find(const key_type& key) const { + uint32_t itr = tbl->environment.kv_it_create(tbl->db_name, contract_name.value, prefix.data(), prefix.size()); + int32_t itr_stat = tbl->environment.kv_it_lower_bound(itr, key.data(), key.size()); + + auto cmp = tbl->environment.kv_it_key_compare(itr, key.data(), key.size()); + + if (cmp != 0) { + tbl->environment.kv_it_destroy(itr); + return end(); + } + + return {itr, static_cast(itr_stat), this}; + } + + /** + * Check if a given key exists in the index. + * @ingroup keyvalue + * + * @param key - The key to check for. + * @return If the key exists or not. + */ + bool exists(const K& key) const { + auto t_key = table_key(prefix, make_key(key)); + return exists(t_key); + } + + bool exists(const key_type& key) const { + uint32_t value_size; + return tbl->environment.kv_get(tbl->db_name, contract_name.value, key.data(), key.size(), value_size); + } + + /** + * Get the value for an existing object in a table by the index, using the given key. + * @ingroup keyvalue + * + * @param key - The key to search for. + * @return The value corresponding to the key. + */ + T operator[](const K& key) const { + return operator[](make_key(key)); + } + + T operator[](const key_type& key) const { + auto opt = get(key); + eosio::check(opt.has_value(), __FILE__ ":" + std::to_string(__LINE__) + " Key not found in `[]`"); + return *opt; + } + + /** + * Get the value for an existing object in a table by the index, using the given key. + * @ingroup keyvalue + * + * @param key - The key to search for. + * @return A std::optional of the value corresponding to the key. + */ + std::optional get(const K& key) const { + return get(make_key(key)); + } + + std::optional get(const key_type& k ) const { + auto key = table_key( prefix, k ); + uint32_t value_size; + uint32_t actual_data_size; + std::optional ret_val; + + auto success = tbl->environment.kv_get(tbl->db_name, contract_name.value, key.data(), key.size(), value_size); + if (!success) { + return ret_val; + } + + void* buffer = value_size > detail::max_stack_buffer_size ? malloc(value_size) : alloca(value_size); + auto copy_size = tbl->environment.kv_get_data(tbl->db_name, 0, (char*)buffer, value_size); + + void* deserialize_buffer = buffer; + size_t deserialize_size = copy_size; + + bool is_primary = index_name == tbl->primary_index_name; + if (!is_primary) { + auto success = tbl->environment.kv_get(tbl->db_name, contract_name.value, (char*)buffer, copy_size, actual_data_size); + eosio::check(success, "failure getting primary key"); + + void* pk_buffer = actual_data_size > detail::max_stack_buffer_size ? malloc(actual_data_size) : alloca(actual_data_size); + auto pk_copy_size = tbl->environment.kv_get_data(tbl->db_name, 0, (char*)pk_buffer, actual_data_size); + + deserialize_buffer = pk_buffer; + deserialize_size = pk_copy_size; + } + + ret_val.emplace(); + detail::deserialize(*ret_val, deserialize_buffer, deserialize_size); + + if (value_size > detail::max_stack_buffer_size) { + free(buffer); + } + + if (is_primary && actual_data_size > detail::max_stack_buffer_size) { + free(deserialize_buffer); + } + + return ret_val; + } + + /** + * Returns an iterator to the object with the lowest key (by this index) in the table. + * @ingroup keyvalue + * + * @return An iterator to the object with the lowest key (by this index) in the table. + */ + iterator begin() const { + uint32_t itr = tbl->environment.kv_it_create(tbl->db_name, contract_name.value, prefix.data(), prefix.size()); + int32_t itr_stat = tbl->environment.kv_it_lower_bound(itr, "", 0); + + return {itr, static_cast(itr_stat), this}; + } + + /** + * Returns an iterator pointing past the end. It does not point to any element, therefore `value` should not be called on it. + * @ingroup keyvalue + * + * @return An iterator pointing past the end. + */ + iterator end() const { + return {0, iterator::status::iterator_end, this}; + } + + /** + * Returns an iterator pointing to the element with the lowest key greater than or equal to the given key. + * @ingroup keyvalue + * + * @return An iterator pointing to the element with the lowest key greater than or equal to the given key. + */ + iterator lower_bound(const K& key) const { + return lower_bound(make_key(key)); + } + + iterator lower_bound(const key_type& k ) const { + auto key = table_key( prefix, k ); + uint32_t itr = tbl->environment.kv_it_create(tbl->db_name, contract_name.value, prefix.data(), prefix.size()); + int32_t itr_stat = tbl->environment.kv_it_lower_bound(itr, key.data(), key.size()); + + return {itr, static_cast(itr_stat), this}; + } + + /** + * Returns an iterator pointing to the first element greater than the given key. + * @ingroup keyvalue + * + * @return An iterator pointing to the first element greater than the given key. + */ + iterator upper_bound(const K& key) const { + return upper_bound(make_key(key)); + } + + iterator upper_bound(const key_type& key) const { + auto it = lower_bound(key); + + auto cmp = it.key_compare(key); + if (cmp == 0) { + ++it; + } + + return it; + } + + /** + * Returns a vector of objects that fall between the specifed range. The range is inclusive, exclusive. + * @ingroup keyvalue + * + * @param begin - The beginning of the range (inclusive). + * @param end - The end of the range (exclusive). + * @return A vector containing all the objects that fall between the range. + */ + std::vector range(const K& b, const K& e) const { + return range(make_key(b), make_key(e)); + } + + std::vector range(const key_type& b_key, const key_type& e_key) const { + auto b = table_key(prefix, make_key(b_key)); + auto e = table_key(prefix, make_key(e_key)); + std::vector return_values; + + for(auto itr = lower_bound(b), end_itr = lower_bound(e); itr < end_itr; ++itr) { + return_values.push_back(itr.value()); + } + + return return_values; + } + + void setup() override { + prefix = make_prefix(table_name, index_name); + } + }; + + /** + * @ingroup keyvalue + * Puts a value into the table. If the value already exists, it updates the existing entry. + * The key is determined from the defined primary index. + * If the put attempts to store over an existing secondary index, the transaction will be aborted. + * + * @param value - The entry to be stored in the table. + */ + void put(const T& value) { + uint32_t value_size; + T old_value; + + auto primary_key = primary_index->get_key(value); + auto tbl_key = table_key(make_prefix(table_name, primary_index->index_name), primary_key); + + auto primary_key_found = environment.kv_get(db_name, contract_name.value, tbl_key.data(), tbl_key.size(), value_size); + + if (primary_key_found) { + void* buffer = value_size > detail::max_stack_buffer_size ? malloc(value_size) : alloca(value_size); + auto copy_size = environment.kv_get_data(db_name, 0, (char*)buffer, value_size); + + detail::deserialize(old_value, buffer, copy_size); + + if (value_size > detail::max_stack_buffer_size) { + free(buffer); + } + } + + for (const auto& idx : secondary_indices) { + auto sec_tbl_key = table_key(make_prefix(table_name, idx->index_name), idx->get_key(value)); + auto sec_found = environment.kv_get(db_name, contract_name.value, sec_tbl_key.data(), sec_tbl_key.size(), value_size); + + if (!primary_key_found) { + eosio::check(!sec_found, "Attempted to store an existing secondary index."); + environment.kv_set(db_name, contract_name.value, sec_tbl_key.data(), sec_tbl_key.size(), tbl_key.data(), tbl_key.size()); + } else { + if (sec_found) { + void* buffer = value_size > detail::max_stack_buffer_size ? malloc(value_size) : alloca(value_size); + auto copy_size = environment.kv_get_data(db_name, 0, (char*)buffer, value_size); + + auto res = memcmp(buffer, tbl_key.data(), copy_size); + eosio::check(copy_size == tbl_key.size() && res == 0, "Attempted to update an existing secondary index."); + + if (copy_size > detail::max_stack_buffer_size) { + free(buffer); + } + } else { + auto old_sec_key = table_key(make_prefix(table_name, idx->index_name), idx->get_key(old_value)); + environment.kv_erase(db_name, contract_name.value, old_sec_key.data(), old_sec_key.size()); + environment.kv_set(db_name, contract_name.value, sec_tbl_key.data(), sec_tbl_key.size(), tbl_key.data(), tbl_key.size()); + } + } + + } + + size_t data_size = detail::get_size(value); + void* data_buffer = data_size > detail::max_stack_buffer_size ? malloc(data_size) : alloca(data_size); + + detail::serialize(value, data_buffer, data_size); + + environment.kv_set(db_name, contract_name.value, tbl_key.data(), tbl_key.size(), (const char*)data_buffer, data_size); + + if (data_size > detail::max_stack_buffer_size) { + free(data_buffer); + } + } + + /** + * Removes a value from the table. + * @ingroup keyvalue + * + * @param key - The key of the value to be removed. + */ + void erase(const T& value) { + uint32_t value_size; + + auto primary_key = primary_index->get_key(value); + auto tbl_key = table_key(make_prefix(table_name, primary_index->index_name), primary_key); + auto primary_key_found = environment.kv_get(db_name, contract_name.value, tbl_key.data(), tbl_key.size(), value_size); + + if (!primary_key_found) { + return; + } + + for (const auto& idx : secondary_indices) { + auto sec_tbl_key = table_key(make_prefix(table_name, idx->index_name), idx->get_key(value)); + environment.kv_erase(db_name, contract_name.value, sec_tbl_key.data(), sec_tbl_key.size()); + } + + environment.kv_erase(db_name, contract_name.value, tbl_key.data(), tbl_key.size()); + } + +protected: + kv_table() = default; + + kv_environment environment; + + kv_table(kv_environment environment) : environment{std::move(environment)} {} + + kv_table(const kv_table&) = delete; + kv_table(kv_table&&) = delete; + + template + void setup_indices(I& index) { + kv_index* idx = &index; + idx->contract_name = contract_name; + idx->table_name = table_name; + idx->tbl = this; + + idx->setup(); + secondary_indices.push_back(idx); + } + + template + void init(eosio::name contract, eosio::name table, eosio::name db, PrimaryIndex& prim_index, SecondaryIndices&... indices) { + validate_types(prim_index); + (validate_types(indices), ...); + + contract_name = contract; + table_name = table; + db_name = db.value; + + primary_index = &prim_index; + primary_index->contract_name = contract_name; + primary_index->table_name = table_name; + primary_index->tbl = this; + + primary_index->setup(); + + primary_index_name = primary_index->index_name; + + (setup_indices(indices), ...); + } + +private: + eosio::name contract_name; + eosio::name table_name; + uint64_t db_name; + + eosio::name primary_index_name; + + kv_index* primary_index; + std::vector secondary_indices; + + constexpr void validate_types() {} + + template + constexpr void validate_types(Type& t) { + constexpr bool is_kv_index = std::is_base_of_v>; + static_assert(is_kv_index, "Incorrect type passed to init. Must be a reference to an index."); + } + +}; + +template +class kv_singleton { + kv_environment environment; + + struct state { + T value; + char* raw_original; + size_t raw_original_size; + + bool is_dirty = false; + bool is_cached = false; + + ~state() { + if (raw_original_size) { + free(raw_original); + } + } + }; + +public: + using value_type = T; + + explicit kv_singleton(eosio::name contract_name) : contract_name{contract_name} { + key = make_prefix(); + } + + kv_singleton(eosio::name contract_name, kv_environment environment) : environment{environment}, contract_name{contract_name} {} + + ~kv_singleton() { + if (get_state().is_dirty) { + store(); + } + } + + const T& get() { + auto& ste = get_state(); + if (!ste.is_cached) { + uint32_t copy_size; + uint32_t value_size; + + auto success = environment.kv_get(db_name, contract_name.value, key.data(), key.size(), value_size); + + eosio::check(success, "tried to get a singleton that does not exist"); + + ste.raw_original = (char*)malloc(value_size); + ste.raw_original_size = value_size; + copy_size = environment.kv_get_data(db_name, 0, ste.raw_original, value_size); + + detail::deserialize(ste.value, ste.raw_original, copy_size); + ste.is_cached = true; + } + + return get_state().value; + } + + void set(const T& val) { + auto& ste = get_state(); + ste.value = val; + ste.is_dirty = true; + ste.is_cached = true; + } + + bool exists() { + uint32_t value_size; + + return environment.kv_get(db_name, contract_name.value, key.data(), key.size(), value_size); + } + + void erase() { + environment.kv_erase(db_name, contract_name.value, key.data(), key.size()); + auto& ste = get_state(); + ste.is_cached = false; + ste.is_dirty = false; + ste.raw_original_size = 0; + free(ste.raw_original); + } + + void store() { + auto& ste = get_state(); + if (ste.is_dirty) { + size_t data_size = detail::get_size(ste.value); + void* data_buffer = data_size > detail::max_stack_buffer_size ? malloc(data_size) : alloca(data_size); + + detail::serialize(ste.value, data_buffer, data_size); + + if (ste.raw_original_size != data_size || memcmp(ste.raw_original, data_buffer, data_size) != 0) { + environment.kv_set(db_name, contract_name.value, key.data(), key.size(), (const char*)data_buffer, data_size); + } + } + } + +private: + constexpr static uint64_t db_name = static_cast(DbName); + constexpr static uint64_t singleton_name = static_cast(SingletonName); + + eosio::name contract_name; + key_type key; + + key_type make_prefix() { + return make_key(std::make_tuple(0x02, singleton_name)); + } + + state& get_state() { + static state value; + return value; + } +}; + +} // eosio From 5e5f0cae99c135f596c1795d73e2210eecfdefde Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Sat, 25 Apr 2020 17:43:26 -0400 Subject: [PATCH 06/23] rodeos --- libraries/CMakeLists.txt | 2 + libraries/rodeos/CMakeLists.txt | 2 + .../include/b1/rodeos/callbacks/action.hpp | 4 +- .../include/b1/rodeos/callbacks/basic.hpp | 2 +- .../b1/rodeos/callbacks/compiler_builtins.hpp | 2 +- .../include/b1/rodeos/callbacks/console.hpp | 2 +- .../include/b1/rodeos/callbacks/filter.hpp | 2 +- .../rodeos/include/b1/rodeos/callbacks/kv.hpp | 10 +- .../include/b1/rodeos/callbacks/memory.hpp | 2 +- .../b1/rodeos/callbacks/unimplemented.hpp | 2 +- .../rodeos/callbacks/unimplemented_filter.hpp | 2 +- .../{definitions.hpp => vm_types.hpp} | 0 libraries/rodeos/rodeos.cpp | 261 ++++++++ libraries/rodeos/wasm_ql.cpp | 625 ++++++++++++++++++ 14 files changed, 905 insertions(+), 13 deletions(-) rename libraries/rodeos/include/b1/rodeos/callbacks/{definitions.hpp => vm_types.hpp} (100%) create mode 100644 libraries/rodeos/rodeos.cpp create mode 100644 libraries/rodeos/wasm_ql.cpp diff --git a/libraries/CMakeLists.txt b/libraries/CMakeLists.txt index baf946b4004..898590dd020 100644 --- a/libraries/CMakeLists.txt +++ b/libraries/CMakeLists.txt @@ -37,3 +37,5 @@ set_property(GLOBAL PROPERTY CTEST_CUSTOM_TESTS_IGNORE "change_authkey import_ed decrypt_ec decrypt_rsa ssh logs generate_rsa import_ec echo\ yubico_otp wrap_data wrap info import_rsa import_authkey generate_hmac generate_ec\ attest pbkdf2 parsing ${_CTEST_CUSTOM_TESTS_IGNORE}") + +add_subdirectory( rodeos ) diff --git a/libraries/rodeos/CMakeLists.txt b/libraries/rodeos/CMakeLists.txt index 7c202b3dbe9..3d657b18457 100644 --- a/libraries/rodeos/CMakeLists.txt +++ b/libraries/rodeos/CMakeLists.txt @@ -2,6 +2,8 @@ file(GLOB_RECURSE HEADERS "include/*.hpp" "include/*.h") add_library( rodeos_lib embedded_rodeos.cpp + rodeos.cpp + wasm_ql.cpp ${HEADERS} ) diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/action.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/action.hpp index eed2d097c86..40da3e49154 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/action.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/action.hpp @@ -1,6 +1,8 @@ #pragma once -#include +#include +#include +#include namespace b1::rodeos { diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/basic.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/basic.hpp index be23d5371de..6249a9f1a15 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/basic.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/basic.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include #include namespace b1::rodeos { diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/compiler_builtins.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/compiler_builtins.hpp index 8f07591d885..aa161a437eb 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/compiler_builtins.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/compiler_builtins.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/console.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/console.hpp index 8b7890e5cf3..9dda9f26a68 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/console.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/console.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include #include namespace b1::rodeos { diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/filter.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/filter.hpp index a2fef0e5550..fff307cafed 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/filter.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/filter.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace b1::rodeos { diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/kv.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/kv.hpp index c0d4730a31f..495e4ad959a 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/kv.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/kv.hpp @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include #include @@ -415,12 +415,12 @@ class kv_environment : public db_callbacks { return base::kv_it_lower_bound(itr, { key, key_size }, found_key_size, found_value_size); } - int32_t kv_it_key(uint32_t itr, uint32_t offset, char* dest, uint32_t dest_size, uint32_t* actual_size) { - return base::kv_it_key(itr, offset, { dest, dest_size }, actual_size); + int32_t kv_it_key(uint32_t itr, uint32_t offset, char* dest, uint32_t dest_size, uint32_t& actual_size) { + return base::kv_it_key(itr, offset, { dest, dest_size }, &actual_size); } - int32_t kv_it_value(uint32_t itr, uint32_t offset, char* dest, uint32_t dest_size, uint32_t* actual_size) { - return base::kv_it_value(itr, offset, { dest, dest_size }, actual_size); + int32_t kv_it_value(uint32_t itr, uint32_t offset, char* dest, uint32_t dest_size, uint32_t& actual_size) { + return base::kv_it_value(itr, offset, { dest, dest_size }, &actual_size); } }; diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/memory.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/memory.hpp index 7486ff253d0..d8f336593b7 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/memory.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/memory.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace b1::rodeos { diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented.hpp index 0d9290aecce..a995bd46cb7 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace b1::rodeos { diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented_filter.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented_filter.hpp index 850125e267c..92143646f37 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented_filter.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented_filter.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace b1::rodeos { diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/definitions.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/vm_types.hpp similarity index 100% rename from libraries/rodeos/include/b1/rodeos/callbacks/definitions.hpp rename to libraries/rodeos/include/b1/rodeos/callbacks/vm_types.hpp diff --git a/libraries/rodeos/rodeos.cpp b/libraries/rodeos/rodeos.cpp new file mode 100644 index 00000000000..336ac301d9d --- /dev/null +++ b/libraries/rodeos/rodeos.cpp @@ -0,0 +1,261 @@ +#include + +#include +#include + +namespace b1::rodeos { + +namespace ship_protocol = eosio::ship_protocol; + +using ship_protocol::get_blocks_result_v0; +using ship_protocol::signed_block; + +rodeos_db_snapshot::rodeos_db_snapshot(std::shared_ptr partition, bool persistent) + : partition{ std::move(partition) }, db{ this->partition->db } { + if (persistent) { + undo_stack.emplace(*db, this->partition->undo_prefix); + write_session.emplace(*db); + } else { + snap.emplace(db->rdb.get()); + write_session.emplace(*db, snap->snapshot()); + } + + db_view_state view_state{ eosio::name{ "state" }, *db, *write_session, this->partition->contract_kv_prefix }; + fill_status_kv table{ eosio::name{ "state" }, view_state }; + if (table.exists()) { + auto status = std::get<0>(table.get()); + chain_id = status.chain_id; + head = status.head; + head_id = status.head_id; + irreversible = status.irreversible; + irreversible_id = status.irreversible_id; + first = status.first; + } +} + +void rodeos_db_snapshot::refresh() { + if (undo_stack) + throw std::runtime_error("can not refresh a persistent snapshot"); + snap.emplace(db->rdb.get()); + write_session->snapshot = snap->snapshot(); + write_session->wipe_cache(); +} + +void rodeos_db_snapshot::write_fill_status() { + if (!undo_stack) + throw std::runtime_error("Can only write to persistent snapshots"); + fill_status status; + if (irreversible < head) + status = fill_status_v0{ .chain_id = chain_id, + .head = head, + .head_id = head_id, + .irreversible = irreversible, + .irreversible_id = irreversible_id, + .first = first }; + else + status = fill_status_v0{ .chain_id = chain_id, + .head = head, + .head_id = head_id, + .irreversible = head, + .irreversible_id = head_id, + .first = first }; + + db_view_state view_state{ eosio::name{ "state" }, *db, *write_session, partition->contract_kv_prefix }; + view_state.kv_state.enable_write = true; + fill_status_kv table{ eosio::name{ "state" }, view_state }; + table.set(status); +} + +void rodeos_db_snapshot::end_write(bool write_fill) { + if (!undo_stack) + throw std::runtime_error("Can only write to persistent snapshots"); + if (write_fill) + write_fill_status(); + write_session->write_changes(*undo_stack); +} + +void rodeos_db_snapshot::start_block(const get_blocks_result_v0& result) { + if (!undo_stack) + throw std::runtime_error("Can only write to persistent snapshots"); + if (!result.this_block) + throw std::runtime_error("get_blocks_result this_block is empty"); + + if (result.this_block->block_num <= head) { + ilog("switch forks at block ${b}; database contains revisions ${f} - ${h}", + ("b", result.this_block->block_num)("f", undo_stack->first_revision())("h", undo_stack->revision())); + if (undo_stack->first_revision() >= result.this_block->block_num) + throw std::runtime_error("can't switch forks since database doesn't contain revision " + + std::to_string(result.this_block->block_num - 1)); + write_session->wipe_cache(); + while (undo_stack->revision() >= result.this_block->block_num) // + undo_stack->undo(true); + } + + if (head_id != eosio::checksum256{} && (!result.prev_block || result.prev_block->block_id != head_id)) + throw std::runtime_error("prev_block does not match"); + + if (result.this_block->block_num <= result.last_irreversible.block_num) { + undo_stack->commit(std::min(result.last_irreversible.block_num, head)); + undo_stack->set_revision(result.this_block->block_num, false); + } else { + end_write(false); + undo_stack->commit(std::min(result.last_irreversible.block_num, head)); + undo_stack->push(false); + } + writing_block = result.this_block->block_num; +} + +void rodeos_db_snapshot::end_block(const get_blocks_result_v0& result, bool force_write) { + if (!undo_stack) + throw std::runtime_error("Can only write to persistent snapshots"); + if (!result.this_block) + throw std::runtime_error("get_blocks_result this_block is empty"); + if (!writing_block || result.this_block->block_num != *writing_block) + throw std::runtime_error("call start_block first"); + + bool near = result.this_block->block_num + 4 >= result.last_irreversible.block_num; + bool write_now = !(result.this_block->block_num % 200) || near || force_write; + head = result.this_block->block_num; + head_id = result.this_block->block_id; + irreversible = result.last_irreversible.block_num; + irreversible_id = result.last_irreversible.block_id; + if (!first || head < first) + first = head; + if (write_now) + end_write(write_now); + if (near) + db->flush(false, false); +} + +void rodeos_db_snapshot::check_write(const ship_protocol::get_blocks_result_v0& result) { + if (!undo_stack) + throw std::runtime_error("Can only write to persistent snapshots"); + if (!result.this_block) + throw std::runtime_error("get_blocks_result this_block is empty"); + if (!writing_block || result.this_block->block_num != *writing_block) + throw std::runtime_error("call start_block first"); +} + +void rodeos_db_snapshot::write_block_info(const ship_protocol::get_blocks_result_v0& result) { + check_write(result); + if (!result.block) + return; + + uint32_t block_num = result.this_block->block_num; + eosio::input_stream bin = *result.block; + signed_block block; + eosio::check_discard(from_bin(block, bin)); + + db_view_state view_state{ eosio::name{ "state" }, *db, *write_session, partition->contract_kv_prefix }; + view_state.kv_state.enable_write = true; + + block_info_v0 info; + info.num = block_num; + info.id = result.this_block->block_id; + info.timestamp = block.timestamp; + info.producer = block.producer; + info.confirmed = block.confirmed; + info.previous = block.previous; + info.transaction_mroot = block.transaction_mroot; + info.action_mroot = block.action_mroot; + info.schedule_version = block.schedule_version; + info.new_producers = block.new_producers; + info.producer_signature = block.producer_signature; + + block_info_kv table{ kv_environment{ view_state } }; + table.put(info); +} + +void rodeos_db_snapshot::write_deltas(const ship_protocol::get_blocks_result_v0& result, + std::function shutdown) { + check_write(result); + if (!result.deltas) + return; + + uint32_t block_num = result.this_block->block_num; + eosio::input_stream bin = *result.deltas; + + db_view_state view_state{ eosio::name{ "state" }, *db, *write_session, partition->contract_kv_prefix }; + view_state.kv_ram.enable_write = true; + view_state.kv_ram.bypass_receiver_check = true; + view_state.kv_disk.enable_write = true; + view_state.kv_disk.bypass_receiver_check = true; + view_state.kv_state.enable_write = true; + uint32_t num; + eosio::check_discard(eosio::varuint32_from_bin(num, bin)); + for (uint32_t i = 0; i < num; ++i) { + ship_protocol::table_delta delta; + eosio::check_discard(from_bin(delta, bin)); + auto& delta_v0 = std::get<0>(delta); + size_t num_processed = 0; + store_delta({ view_state }, delta_v0, head == 0, [&]() { + if (delta_v0.rows.size() > 10000 && !(num_processed % 10000)) { + if (shutdown()) + throw std::runtime_error("shutting down"); + ilog("block ${b} ${t} ${n} of ${r}", + ("b", block_num)("t", delta_v0.name)("n", num_processed)("r", delta_v0.rows.size())); + if (head == 0) { + end_write(false); + view_state.reset(); + } + } + ++num_processed; + }); + } +} + +std::once_flag registered_filter_callbacks; + +rodeos_filter::rodeos_filter(eosio::name name, const std::string& wasm_filename) : name{ name } { + std::call_once(registered_filter_callbacks, filter::register_callbacks); + + std::ifstream wasm_file(wasm_filename, std::ios::binary); + if (!wasm_file.is_open()) + throw std::runtime_error("can not open " + wasm_filename); + ilog("compiling ${f}", ("f", wasm_filename)); + wasm_file.seekg(0, std::ios::end); + int len = wasm_file.tellg(); + if (len < 0) + throw std::runtime_error("wasm file length is -1"); + std::vector code(len); + wasm_file.seekg(0, std::ios::beg); + wasm_file.read((char*)code.data(), code.size()); + wasm_file.close(); + backend = std::make_unique(code, nullptr); + filter_state = std::make_unique(); + filter::rhf_t::resolve(backend->get_module()); +} + +void rodeos_filter::process(rodeos_db_snapshot& snapshot, const ship_protocol::get_blocks_result_v0& result, + eosio::input_stream bin, + const std::function& push_data) { + // todo: timeout + snapshot.check_write(result); + chaindb_state chaindb_state; + db_view_state view_state{ name, *snapshot.db, *snapshot.write_session, snapshot.partition->contract_kv_prefix }; + view_state.kv_disk.enable_write = true; + view_state.kv_ram.enable_write = true; + filter::callbacks cb{ *filter_state, chaindb_state, view_state }; + filter_state->max_console_size = 10000; + filter_state->console.clear(); + filter_state->input_data = bin; + filter_state->push_data = push_data; + backend->set_wasm_allocator(&filter_state->wa); + backend->initialize(&cb); + try { + (*backend)(cb, "env", "apply", uint64_t(0), uint64_t(0), uint64_t(0)); + } catch (...) { + if (!filter_state->console.empty()) + ilog("filter ${n} console output before exception: <<<\n${c}>>>", + ("n", name.to_string())("c", filter_state->console)); + throw; + } + if (!filter_state->console.empty()) + ilog("filter ${n} console output: <<<\n${c}>>>", ("n", name.to_string())("c", filter_state->console)); +} + +rodeos_query_handler::rodeos_query_handler(std::shared_ptr partition, + std::shared_ptr shared_state) + : partition{ partition }, shared_state{ std::move(shared_state) }, state_cache{ this->shared_state } {} + +} // namespace b1::rodeos diff --git a/libraries/rodeos/wasm_ql.cpp b/libraries/rodeos/wasm_ql.cpp new file mode 100644 index 00000000000..83cae66fb1e --- /dev/null +++ b/libraries/rodeos/wasm_ql.cpp @@ -0,0 +1,625 @@ +// copyright defined in LICENSE.txt + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std::literals; +namespace ship_protocol = eosio::ship_protocol; + +using boost::multi_index::indexed_by; +using boost::multi_index::member; +using boost::multi_index::multi_index_container; +using boost::multi_index::ordered_non_unique; +using boost::multi_index::sequenced; +using boost::multi_index::tag; + +using eosio::ship_protocol::action_receipt_v0; +using eosio::ship_protocol::action_trace_v1; +using eosio::ship_protocol::transaction_trace_v0; + +namespace eosio { + +// todo: move to abieos +template +result to_json(const might_not_exist& val, S& stream) { + return to_json(val.value, stream); +} + +// todo: abieos support for pair. Used by extensions_type. +template +result to_json(const std::pair>&, S& stream) { + return stream_error::bad_variant_index; +} + +} // namespace eosio + +namespace b1::rodeos::wasm_ql { + +// todo: relax some of these limits +// todo: restore max_function_section_elements to 1023 and use nodeos's hard fork +struct wasm_ql_backend_options { + static constexpr std::uint32_t max_mutable_global_bytes = 1024; + static constexpr std::uint32_t max_table_elements = 1024; + static constexpr std::uint32_t max_section_elements = 8191; + static constexpr std::uint32_t max_function_section_elements = 8000; + static constexpr std::uint32_t max_import_section_elements = 1023; + static constexpr std::uint32_t max_element_segment_elements = 8191; + static constexpr std::uint32_t max_data_segment_bytes = 8191; + static constexpr std::uint32_t max_linear_memory_init = 64 * 1024; + static constexpr std::uint32_t max_func_local_bytes = 8192; + static constexpr std::uint32_t max_local_sets = 1023; + static constexpr std::uint32_t eosio_max_nested_structures = 1023; + static constexpr std::uint32_t max_br_table_elements = 8191; + static constexpr std::uint32_t max_symbol_bytes = 8191; + static constexpr std::uint32_t max_memory_offset = (33 * 1024 * 1024 - 1); + static constexpr std::uint32_t max_pages = 528; // 33 MiB + static constexpr std::uint32_t max_call_depth = 251; +}; + +struct callbacks; +using rhf_t = registered_host_functions; +using backend_t = eosio::vm::backend; + +// todo: remove context_free_system_callbacks +struct callbacks : action_callbacks, + chaindb_callbacks, + compiler_builtins_callbacks, + console_callbacks, + context_free_system_callbacks, + db_callbacks, + memory_callbacks, + query_callbacks, + unimplemented_callbacks { + wasm_ql::thread_state& thread_state; + rodeos::chaindb_state& chaindb_state; + rodeos::db_view_state& db_view_state; + + callbacks(wasm_ql::thread_state& thread_state, rodeos::chaindb_state& chaindb_state, + rodeos::db_view_state& db_view_state) + : thread_state{ thread_state }, chaindb_state{ chaindb_state }, db_view_state{ db_view_state } {} + + auto& get_state() { return thread_state; } + auto& get_chaindb_state() { return chaindb_state; } + auto& get_db_view_state() { return db_view_state; } +}; + +std::once_flag registered_callbacks; + +void register_callbacks() { + action_callbacks::register_callbacks(); + chaindb_callbacks::register_callbacks(); + compiler_builtins_callbacks::register_callbacks(); + console_callbacks::register_callbacks(); + context_free_system_callbacks::register_callbacks(); + db_callbacks::register_callbacks(); + memory_callbacks::register_callbacks(); + query_callbacks::register_callbacks(); + unimplemented_callbacks::register_callbacks(); +} + +struct backend_entry { + eosio::name name; // only for wasms loaded from disk + eosio::checksum256 hash; // only for wasms loaded from chain + std::unique_ptr backend; +}; + +struct by_age; +struct by_name; +struct by_hash; + +using backend_container = multi_index_container< + backend_entry, + indexed_by>, // + ordered_non_unique, member>, + ordered_non_unique, member>>>; + +class backend_cache { + private: + std::mutex mutex; + const wasm_ql::shared_state& shared_state; + backend_container backends; + + public: + backend_cache(const wasm_ql::shared_state& shared_state) : shared_state{ shared_state } {} + + void add(backend_entry&& entry) { + std::lock_guard lock{ mutex }; + auto& ind = backends.get(); + ind.push_back(std::move(entry)); + while (ind.size() > shared_state.wasm_cache_size) ind.pop_front(); + } + + std::optional get(eosio::name name) { + std::optional result; + std::lock_guard lock{ mutex }; + auto& ind = backends.get(); + auto it = ind.find(name); + if (it == ind.end()) + return result; + ind.modify(it, [&](auto& x) { result = std::move(x); }); + ind.erase(it); + return result; + } + + std::optional get(const eosio::checksum256& hash) { + std::optional result; + std::lock_guard lock{ mutex }; + auto& ind = backends.get(); + auto it = ind.find(hash); + if (it == ind.end()) + return result; + ind.modify(it, [&](auto& x) { result = std::move(x); }); + ind.erase(it); + return result; + } +}; + +shared_state::shared_state(std::shared_ptr db) + : backend_cache(std::make_unique(*this)), db(std::move(db)) {} + +shared_state::~shared_state() {} + +std::optional> read_code(wasm_ql::thread_state& thread_state, eosio::name account) { + std::optional> code; + if (!thread_state.shared->contract_dir.empty()) { + auto filename = thread_state.shared->contract_dir + "/" + (std::string)account + ".wasm"; + std::ifstream wasm_file(filename, std::ios::binary); + if (wasm_file.is_open()) { + ilog("compiling ${f}", ("f", filename)); + wasm_file.seekg(0, std::ios::end); + int len = wasm_file.tellg(); + if (len < 0) + throw std::runtime_error("wasm file length is -1"); + code.emplace(len); + wasm_file.seekg(0, std::ios::beg); + wasm_file.read((char*)code->data(), code->size()); + wasm_file.close(); + } + } + return code; +} + +std::optional get_contract_hash(db_view_state& db_view_state, eosio::name account) { + std::optional result; + auto meta = get_state_row( + db_view_state.kv_state.view, + std::make_tuple(eosio::name{ "account.meta" }, eosio::name{ "primary" }, account)); + if (!meta) + return result; + auto& meta0 = std::get(meta->second); + if (!meta0.code->vm_type && !meta0.code->vm_version) + result = meta0.code->code_hash; + return result; +} + +std::optional> read_contract(db_view_state& db_view_state, const eosio::checksum256& hash, + eosio::name account) { + std::optional> result; + auto code_row = get_state_row( + db_view_state.kv_state.view, + std::make_tuple(eosio::name{ "code" }, eosio::name{ "primary" }, uint8_t(0), uint8_t(0), hash)); + if (!code_row) + return result; + auto& code0 = std::get(code_row->second); + + // todo: avoid copy + result.emplace(code0.code.pos, code0.code.end); + ilog("compiling ${h}: ${a}", ("h", eosio::check(eosio::convert_to_json(hash)).value())("a", (std::string)account)); + return result; +} + +void run_action(wasm_ql::thread_state& thread_state, const std::vector& contract_kv_prefix, + ship_protocol::action& action, action_trace_v1& atrace, const rocksdb::Snapshot* snapshot, + const std::chrono::steady_clock::time_point& stop_time, std::vector>& memory) { + if (std::chrono::steady_clock::now() >= stop_time) + throw eosio::vm::timeout_exception("execution timed out"); + + chain_kv::write_session write_session{ *thread_state.shared->db, snapshot }; + db_view_state db_view_state{ eosio::name{ "state" }, *thread_state.shared->db, write_session, contract_kv_prefix }; + + std::optional entry = thread_state.shared->backend_cache->get(action.account); + std::optional> code; + if (!entry) + code = read_code(thread_state, action.account); + std::optional hash; + if (!entry && !code) { + hash = get_contract_hash(db_view_state, action.account); + if (hash) { + entry = thread_state.shared->backend_cache->get(*hash); + if (!entry) + code = read_contract(db_view_state, *hash, action.account); + } + } + + // todo: fail? silent success like normal transactions? + if (!entry && !code) + throw std::runtime_error("account " + (std::string)action.account + " has no code"); + + if (!entry) { + entry.emplace(); + if (hash) + entry->hash = *hash; + else + entry->name = action.account; + entry->backend = std::make_unique(*code, nullptr); + + std::call_once(registered_callbacks, register_callbacks); + rhf_t::resolve(entry->backend->get_module()); + } + auto se = fc::make_scoped_exit([&] { thread_state.shared->backend_cache->add(std::move(*entry)); }); + + fill_status_kv fill_status_table{ eosio::name{ "state" }, db_view_state }; + if (!fill_status_table.exists()) + throw std::runtime_error("No fill_status records found; is filler running?"); + auto& fill_status = fill_status_table.get(); + + // todo: move these out of thread_state since future enhancements could cause state to accidentally leak between + // queries + thread_state.max_console_size = thread_state.shared->max_console_size; + thread_state.receiver = action.account; + thread_state.action_data = action.data; + thread_state.action_return_value.clear(); + std::visit([&](auto& stat) { thread_state.block_num = stat.head; }, fill_status); + thread_state.block_info.reset(); + + chaindb_state chaindb_state; + callbacks cb{ thread_state, chaindb_state, db_view_state }; + entry->backend->set_wasm_allocator(&thread_state.wa); + + try { + eosio::vm::watchdog wd{ stop_time - std::chrono::steady_clock::now() }; + entry->backend->timed_run(wd, [&] { + entry->backend->initialize(&cb); + (*entry->backend)(cb, "env", "apply", action.account.value, action.account.value, action.name.value); + }); + } catch (...) { + atrace.console = std::move(thread_state.console); + throw; + } + + atrace.console = std::move(thread_state.console); + memory.push_back(std::move(thread_state.action_return_value)); + atrace.return_value = memory.back(); +} // run_action + +const std::vector& query_get_info(wasm_ql::thread_state& thread_state, + const std::vector& contract_kv_prefix) { + rocksdb::ManagedSnapshot snapshot{ thread_state.shared->db->rdb.get() }; + chain_kv::write_session write_session{ *thread_state.shared->db, snapshot.snapshot() }; + db_view_state db_view_state{ eosio::name{ "state" }, *thread_state.shared->db, write_session, contract_kv_prefix }; + + std::string result = "{\"server_type\":\"wasm-ql\""; + + { + global_property_kv table{ { db_view_state } }; + bool found = false; + if (table.primary_index.begin() != table.primary_index.end()) { + auto record = table.primary_index.begin().value(); + if (auto* obj = std::get_if(&record)) { + found = true; + result += ",\"chain_id\":" + eosio::check(eosio::convert_to_json(obj->chain_id)).value(); + } + } + if (!found) + throw std::runtime_error("No global_property_v1 records found; is filler running?"); + } + + { + fill_status_kv table{ eosio::name{ "state" }, db_view_state }; + if (table.exists()) { + std::visit( + [&](auto& obj) { + result += ",\"head_block_num\":\"" + std::to_string(obj.head) + "\""; + result += ",\"head_block_id\":" + eosio::check(eosio::convert_to_json(obj.head_id)).value(); + result += ",\"last_irreversible_block_num\":\"" + std::to_string(obj.irreversible) + "\""; + result += ",\"last_irreversible_block_id\":" + + eosio::check(eosio::convert_to_json(obj.irreversible_id)).value(); + }, + table.get()); + } else + throw std::runtime_error("No fill_status records found; is filler running?"); + } + + result += "}"; + + thread_state.action_return_value.assign(result.data(), result.data() + result.size()); + return thread_state.action_return_value; +} + +struct get_block_params { + std::string block_num_or_id = {}; +}; + +EOSIO_REFLECT(get_block_params, block_num_or_id) + +const std::vector& query_get_block(wasm_ql::thread_state& thread_state, + const std::vector& contract_kv_prefix, std::string_view body) { + get_block_params params; + std::string s{ body.begin(), body.end() }; + eosio::json_token_stream stream{ s.data() }; + if (auto r = from_json(params, stream); !r) + throw std::runtime_error("An error occurred deserializing get_block_params: " + r.error().message()); + + rocksdb::ManagedSnapshot snapshot{ thread_state.shared->db->rdb.get() }; + chain_kv::write_session write_session{ *thread_state.shared->db, snapshot.snapshot() }; + db_view_state db_view_state{ eosio::name{ "state" }, *thread_state.shared->db, write_session, contract_kv_prefix }; + + std::string bn_json = "\"" + params.block_num_or_id + "\""; + eosio::json_token_stream bn_stream{ bn_json.data() }; + + std::optional, block_info>> info; + if (params.block_num_or_id.size() == 64) { + eosio::checksum256 id; + if (auto r = from_json(id, bn_stream); !r) + throw std::runtime_error("An error occurred deserializing block_num_or_id: " + r.error().message()); + info = get_state_row_secondary(db_view_state.kv_state.view, + std::make_tuple(eosio::name{ "block.info" }, eosio::name{ "id" }, id)); + } else { + uint32_t num; + if (auto r = from_json(num, bn_stream); !r) + throw std::runtime_error("An error occurred deserializing block_num_or_id: " + r.error().message()); + info = get_state_row(db_view_state.kv_state.view, + std::make_tuple(eosio::name{ "block.info" }, eosio::name{ "primary" }, num)); + } + + if (info) { + auto& obj = std::get(info->second); + uint32_t ref_block_prefix; + memcpy(&ref_block_prefix, obj.id.value.begin() + 8, sizeof(ref_block_prefix)); + + std::string result = "{"; + result += "\"block_num\":" + eosio::check(eosio::convert_to_json(obj.num)).value(); + result += ",\"id\":" + eosio::check(eosio::convert_to_json(obj.id)).value(); + result += ",\"timestamp\":" + eosio::check(eosio::convert_to_json(obj.timestamp)).value(); + result += ",\"producer\":" + eosio::check(eosio::convert_to_json(obj.producer)).value(); + result += ",\"confirmed\":" + eosio::check(eosio::convert_to_json(obj.confirmed)).value(); + result += ",\"previous\":" + eosio::check(eosio::convert_to_json(obj.previous)).value(); + result += ",\"transaction_mroot\":" + eosio::check(eosio::convert_to_json(obj.transaction_mroot)).value(); + result += ",\"action_mroot\":" + eosio::check(eosio::convert_to_json(obj.action_mroot)).value(); + result += ",\"schedule_version\":" + eosio::check(eosio::convert_to_json(obj.schedule_version)).value(); + result += ",\"producer_signature\":" + eosio::check(eosio::convert_to_json(obj.producer_signature)).value(); + result += ",\"ref_block_prefix\":" + eosio::check(eosio::convert_to_json(ref_block_prefix)).value(); + result += "}"; + + thread_state.action_return_value.assign(result.data(), result.data() + result.size()); + return thread_state.action_return_value; + } + + throw std::runtime_error("block " + params.block_num_or_id + " not found"); +} // query_get_block + +struct get_abi_params { + eosio::name account_name = {}; +}; + +EOSIO_REFLECT(get_abi_params, account_name) + +struct get_abi_result { + eosio::name account_name; + std::optional abi; +}; + +EOSIO_REFLECT(get_abi_result, account_name, abi) + +const std::vector& query_get_abi(wasm_ql::thread_state& thread_state, const std::vector& contract_kv_prefix, + std::string_view body) { + get_abi_params params; + std::string s{ body.begin(), body.end() }; + eosio::json_token_stream stream{ s.data() }; + if (auto r = from_json(params, stream); !r) + throw std::runtime_error("An error occurred deserializing get_abi_params: " + r.error().message()); + + rocksdb::ManagedSnapshot snapshot{ thread_state.shared->db->rdb.get() }; + chain_kv::write_session write_session{ *thread_state.shared->db, snapshot.snapshot() }; + db_view_state db_view_state{ eosio::name{ "state" }, *thread_state.shared->db, write_session, contract_kv_prefix }; + + auto acc = get_state_row( + db_view_state.kv_state.view, + std::make_tuple(eosio::name{ "account" }, eosio::name{ "primary" }, params.account_name)); + if (!acc) + throw std::runtime_error("account " + (std::string)params.account_name + " not found"); + auto& acc0 = std::get(acc->second); + + get_abi_result result; + result.account_name = acc0.name; + if (acc0.abi.pos != acc0.abi.end) { + result.abi.emplace(); + eosio::check_discard(eosio::from_bin(*result.abi, acc0.abi)); + } + + // todo: avoid the extra copy + auto json = eosio::check(eosio::convert_to_json(result)); + thread_state.action_return_value.assign(json.value().begin(), json.value().end()); + return thread_state.action_return_value; +} // query_get_abi + +// Ignores data field +struct action_no_data { + eosio::name account = {}; + eosio::name name = {}; + std::vector authorization = {}; +}; + +struct extension_hex_data { + uint16_t type = {}; + eosio::bytes data = {}; +}; + +EOSIO_REFLECT(extension_hex_data, type, data) + +EOSIO_REFLECT(action_no_data, account, name, authorization) + +struct transaction_for_get_keys : ship_protocol::transaction_header { + std::vector context_free_actions = {}; + std::vector actions = {}; + std::vector transaction_extensions = {}; +}; + +EOSIO_REFLECT(transaction_for_get_keys, base ship_protocol::transaction_header, context_free_actions, actions, + transaction_extensions) + +struct get_required_keys_params { + transaction_for_get_keys transaction = {}; + std::vector available_keys = {}; +}; + +EOSIO_REFLECT(get_required_keys_params, transaction, available_keys) + +struct get_required_keys_result { + std::vector required_keys = {}; +}; + +EOSIO_REFLECT(get_required_keys_result, required_keys) + +const std::vector& query_get_required_keys(wasm_ql::thread_state& thread_state, std::string_view body) { + get_required_keys_params params; + std::string s{ body.begin(), body.end() }; + eosio::json_token_stream stream{ s.data() }; + if (auto r = from_json(params, stream); !r) + throw std::runtime_error("An error occurred deserializing get_required_keys_params: " + r.error().message()); + + get_required_keys_result result; + for (auto& action : params.transaction.context_free_actions) + if (!action.authorization.empty()) + throw std::runtime_error("Context-free actions may not have authorizations"); + for (auto& action : params.transaction.actions) + if (!action.authorization.empty()) + throw std::runtime_error("Actions may not have authorizations"); // todo + + // todo: avoid the extra copy + auto json = eosio::check(eosio::convert_to_json(result)); + thread_state.action_return_value.assign(json.value().begin(), json.value().end()); + return thread_state.action_return_value; +} // query_get_required_keys + +struct send_transaction_params { + std::vector signatures = {}; + std::string compression = {}; + eosio::bytes packed_context_free_data = {}; + eosio::bytes packed_trx = {}; +}; + +EOSIO_REFLECT(send_transaction_params, signatures, compression, packed_context_free_data, packed_trx) + +struct send_transaction_results { + eosio::checksum256 transaction_id; // todo: redundant with processed.id + transaction_trace_v0 processed; +}; + +EOSIO_REFLECT(send_transaction_results, transaction_id, processed) + +const std::vector& query_send_transaction(wasm_ql::thread_state& thread_state, + const std::vector& contract_kv_prefix, std::string_view body, + bool return_trace_on_except) { + send_transaction_params params; + { + std::string s{ body.begin(), body.end() }; + eosio::json_token_stream stream{ s.data() }; + if (auto r = from_json(params, stream); !r) + throw std::runtime_error("An error occurred deserializing send_transaction_params: "s + r.error().message()); + } + if (params.compression != "0" && params.compression != "none") + throw std::runtime_error("Compression must be 0 or none"); // todo + ship_protocol::packed_transaction trx{ std::move(params.signatures), 0, params.packed_context_free_data.data, + params.packed_trx.data }; + rocksdb::ManagedSnapshot snapshot{ thread_state.shared->db->rdb.get() }; + + std::vector> memory; + send_transaction_results results; + results.processed = query_send_transaction(thread_state, contract_kv_prefix, trx, snapshot.snapshot(), memory, + return_trace_on_except); + + // todo: hide variants during json conversion + // todo: avoid the extra copy + auto json = eosio::check(eosio::convert_to_json(results)); + thread_state.action_return_value.assign(json.value().begin(), json.value().end()); + return thread_state.action_return_value; +} // query_send_transaction + +transaction_trace_v0 query_send_transaction(wasm_ql::thread_state& thread_state, // + const std::vector& contract_kv_prefix, // + const ship_protocol::packed_transaction& trx, // + const rocksdb::Snapshot* snapshot, // + std::vector>& memory, // + bool return_trace_on_except) { + eosio::input_stream s{ trx.packed_trx }; + auto r = eosio::from_bin(s); + if (!r) + throw std::runtime_error("An error occurred deserializing packed_trx: "s + r.error().message()); + if (s.end != s.pos) + throw std::runtime_error("Extra data in packed_trx"); + ship_protocol::transaction& unpacked = r.value(); + + if (!trx.signatures.empty()) + throw std::runtime_error("Signatures must be empty"); // todo + if (trx.compression) + throw std::runtime_error("Compression must be 0 or none"); // todo + if (trx.packed_context_free_data.pos != trx.packed_context_free_data.end) + throw std::runtime_error("packed_context_free_data must be empty"); + // todo: verify query transaction extension is present, but no others + // todo: redirect if transaction extension not present? + if (!unpacked.transaction_extensions.empty()) + throw std::runtime_error("transaction_extensions must be empty"); + // todo: check expiration, ref_block_num, ref_block_prefix + if (unpacked.delay_sec.value) + throw std::runtime_error("delay_sec must be 0"); // queries can't be deferred + if (!unpacked.context_free_actions.empty()) + throw std::runtime_error("context_free_actions must be empty"); // todo: is there a case where CFA makes sense? + for (auto& action : unpacked.actions) + if (!action.authorization.empty()) + throw std::runtime_error("authorization must be empty"); // todo + + // todo: fill transaction_id + transaction_trace_v0 tt; + tt.action_traces.reserve(unpacked.actions.size()); + + auto start_time = std::chrono::steady_clock::now(); + auto stop_time = start_time + std::chrono::milliseconds{ thread_state.shared->max_exec_time_ms }; + + for (auto& action : unpacked.actions) { + tt.action_traces.emplace_back(); + auto& at = tt.action_traces.back().emplace(); + at.action_ordinal.value = tt.action_traces.size(); // starts at 1 + at.receiver = action.account; + at.act = action; + + try { + run_action(thread_state, contract_kv_prefix, action, at, snapshot, stop_time, memory); + } catch (eosio::vm::timeout_exception&) { // + throw std::runtime_error( + "timeout after " + + std::to_string(std::chrono::duration_cast(stop_time - start_time).count()) + + " ms"); + } catch (std::exception& e) { + if (!return_trace_on_except) + throw; + // todo: errorcode + at.except = tt.except = e.what(); + tt.status = ship_protocol::transaction_status::soft_fail; + break; + } + + at.receipt.emplace(); + auto& r = at.receipt->emplace(); + r.receiver = action.account; + } + + return tt; +} // query_send_transaction + +} // namespace b1::rodeos::wasm_ql From 3c4dcdd2b8bdd59ec433660d29a6bf09fc679609 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Sat, 25 Apr 2020 20:01:23 -0400 Subject: [PATCH 07/23] rodeos --- CMakeLists.txt | 1 + libraries/rodeos/wasm_ql.cpp | 2 - programs/CMakeLists.txt | 1 + programs/rodeos/.clang-format | 76 +++++++++ programs/rodeos/CMakeLists.txt | 39 +++++ programs/rodeos/cloner_plugin.cpp | 238 +++++++++++++++++++++++++++++ programs/rodeos/cloner_plugin.hpp | 22 +++ programs/rodeos/config.hpp.in | 16 ++ programs/rodeos/main.cpp | 116 ++++++++++++++ programs/rodeos/rocksdb_plugin.cpp | 57 +++++++ programs/rodeos/rocksdb_plugin.hpp | 25 +++ programs/rodeos/ship_client.hpp | 209 +++++++++++++++++++++++++ 12 files changed, 800 insertions(+), 2 deletions(-) create mode 100644 programs/rodeos/.clang-format create mode 100644 programs/rodeos/CMakeLists.txt create mode 100644 programs/rodeos/cloner_plugin.cpp create mode 100644 programs/rodeos/cloner_plugin.hpp create mode 100644 programs/rodeos/config.hpp.in create mode 100644 programs/rodeos/main.cpp create mode 100644 programs/rodeos/rocksdb_plugin.cpp create mode 100644 programs/rodeos/rocksdb_plugin.hpp create mode 100644 programs/rodeos/ship_client.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index fe52de6fd74..a96e205ceba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,6 +33,7 @@ endif() set( CLI_CLIENT_EXECUTABLE_NAME cleos ) set( NODE_EXECUTABLE_NAME nodeos ) set( KEY_STORE_EXECUTABLE_NAME keosd ) +set( RODEOS_EXECUTABLE_NAME rodeos ) # http://stackoverflow.com/a/18369825 if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") diff --git a/libraries/rodeos/wasm_ql.cpp b/libraries/rodeos/wasm_ql.cpp index 83cae66fb1e..545d9c4c176 100644 --- a/libraries/rodeos/wasm_ql.cpp +++ b/libraries/rodeos/wasm_ql.cpp @@ -1,5 +1,3 @@ -// copyright defined in LICENSE.txt - #include #include diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index a40ef3b423c..ca64a63795b 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -3,3 +3,4 @@ add_subdirectory( cleos ) add_subdirectory( keosd ) add_subdirectory( eosio-launcher ) add_subdirectory( eosio-blocklog ) +add_subdirectory( rodeos ) diff --git a/programs/rodeos/.clang-format b/programs/rodeos/.clang-format new file mode 100644 index 00000000000..fa257fc706b --- /dev/null +++ b/programs/rodeos/.clang-format @@ -0,0 +1,76 @@ +BasedOnStyle: LLVM +IndentWidth: 3 +UseTab: Never +ColumnLimit: 120 + +--- +Language: Cpp +# always align * and & to the type +DerivePointerAlignment: false +PointerAlignment: Left + +# regroup includes to these classes +IncludeCategories: + - Regex: '(<|"(eosio)/)' + Priority: 4 + - Regex: '(<|"(boost)/)' + Priority: 3 + - Regex: '(<|"(llvm|llvm-c|clang|clang-c)/' + Priority: 3 + - Regex: '<[[:alnum:]]+>' + Priority: 2 + - Regex: '.*' + Priority: 1 + +#IncludeBlocks: Regroup + +# set indent for public, private and protected +#AccessModifierOffset: 3 + +# make line continuations twice the normal indent +ContinuationIndentWidth: 6 + +# add missing namespace comments +FixNamespaceComments: true + +# add spaces to braced list i.e. int* foo = { 0, 1, 2 }; instead of int* foo = {0,1,2}; +Cpp11BracedListStyle: false +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: true +AlignConsecutiveDeclarations: true +AlignOperands: true +AlignTrailingComments: true +AllowShortCaseLabelsOnASingleLine: true +AllowShortFunctionsOnASingleLine: All +AllowShortBlocksOnASingleLine: true +#AllowShortIfStatementsOnASingleLine: WithoutElse +#AllowShortIfStatementsOnASingleLine: true +#AllowShortLambdasOnASingleLine: All +AllowShortLoopsOnASingleLine: true +AlwaysBreakTemplateDeclarations: true + +BinPackParameters: true +### use this with clang9 +BreakBeforeBraces: Custom +BraceWrapping: + #AfterCaseLabel: true + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + +BreakConstructorInitializers: BeforeColon +CompactNamespaces: true +IndentCaseLabels: true +IndentPPDirectives: AfterHash +NamespaceIndentation: None +ReflowComments: false +SortIncludes: true +SortUsingDeclarations: true +--- diff --git a/programs/rodeos/CMakeLists.txt b/programs/rodeos/CMakeLists.txt new file mode 100644 index 00000000000..fb452ce266a --- /dev/null +++ b/programs/rodeos/CMakeLists.txt @@ -0,0 +1,39 @@ +add_executable( ${RODEOS_EXECUTABLE_NAME} + cloner_plugin.cpp + main.cpp + rocksdb_plugin.cpp +) + +if( UNIX AND NOT APPLE ) + set(rt_library rt ) +endif() + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/../../.git) + find_package(Git) + if(GIT_FOUND) + execute_process( + COMMAND ${GIT_EXECUTABLE} rev-parse --short=8 HEAD + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../.." + OUTPUT_VARIABLE "rodeos_BUILD_VERSION" + ERROR_QUIET + OUTPUT_STRIP_TRAILING_WHITESPACE) + message(STATUS "Git commit revision: ${rodeos_BUILD_VERSION}") + else() + set(rodeos_BUILD_VERSION 0) + endif() +else() + set(rodeos_BUILD_VERSION 0) +endif() + +configure_file(config.hpp.in config.hpp ESCAPE_QUOTES) + +target_include_directories(${RODEOS_EXECUTABLE_NAME} PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../../libraries/abieos/src ) + +target_link_libraries( ${RODEOS_EXECUTABLE_NAME} + PRIVATE appbase version + PRIVATE rodeos_lib fc ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS} ) + +copy_bin( ${RODEOS_EXECUTABLE_NAME} ) +install( TARGETS + ${RODEOS_EXECUTABLE_NAME} RUNTIME DESTINATION ${CMAKE_INSTALL_FULL_BINDIR} COMPONENT base +) diff --git a/programs/rodeos/cloner_plugin.cpp b/programs/rodeos/cloner_plugin.cpp new file mode 100644 index 00000000000..b5c2fda5a11 --- /dev/null +++ b/programs/rodeos/cloner_plugin.cpp @@ -0,0 +1,238 @@ +#include "cloner_plugin.hpp" +#include "ship_client.hpp" + +#include +#include +#include +#include +#include + +namespace b1 { + +using namespace appbase; +using namespace std::literals; +using namespace eosio::ship_protocol; + +namespace asio = boost::asio; +namespace bpo = boost::program_options; +namespace ship_protocol = eosio::ship_protocol; +namespace websocket = boost::beast::websocket; + +using asio::ip::tcp; +using boost::beast::flat_buffer; +using boost::system::error_code; + +using rodeos::rodeos_db_partition; +using rodeos::rodeos_db_snapshot; +using rodeos::rodeos_filter; + +struct cloner_session; + +struct cloner_config : ship_client::connection_config { + uint32_t skip_to = 0; + uint32_t stop_before = 0; + eosio::name filter_name = {}; // todo: remove + std::string filter_wasm = {}; // todo: remove +}; + +struct cloner_plugin_impl : std::enable_shared_from_this { + std::shared_ptr config = std::make_shared(); + std::shared_ptr session; + boost::asio::deadline_timer timer; + + cloner_plugin_impl() : timer(app().get_io_service()) {} + + ~cloner_plugin_impl(); + + void schedule_retry() { + timer.expires_from_now(boost::posix_time::seconds(1)); + timer.async_wait([this](auto&) { + ilog("retry..."); + start(); + }); + } + + void start(); +}; + +struct cloner_session : ship_client::connection_callbacks, std::enable_shared_from_this { + cloner_plugin_impl* my = nullptr; + std::shared_ptr config; + std::shared_ptr db = app().find_plugin()->get_db(); + std::shared_ptr partition = + std::make_shared(db, std::vector{}); // todo: prefix + + std::optional rodeos_snapshot; + std::shared_ptr connection; + bool reported_block = false; + std::unique_ptr filter = {}; // todo: remove + + cloner_session(cloner_plugin_impl* my) : my(my), config(my->config) { + // todo: remove + if (!config->filter_wasm.empty()) + filter = std::make_unique(config->filter_name, config->filter_wasm); + } + + void connect(asio::io_context& ioc) { + rodeos_snapshot.emplace(partition, true); + + ilog("cloner database status:"); + ilog(" revisions: ${f} - ${r}", + ("f", rodeos_snapshot->undo_stack->first_revision())("r", rodeos_snapshot->undo_stack->revision())); + ilog(" chain: ${a}", ("a", eosio::check(eosio::convert_to_json(rodeos_snapshot->chain_id)).value())); + ilog(" head: ${a} ${b}", + ("a", rodeos_snapshot->head)("b", eosio::check(eosio::convert_to_json(rodeos_snapshot->head_id)).value())); + ilog(" irreversible: ${a} ${b}", + ("a", rodeos_snapshot->irreversible)( + "b", eosio::check(eosio::convert_to_json(rodeos_snapshot->irreversible_id)).value())); + + rodeos_snapshot->end_write(true); + db->flush(true, true); + + connection = std::make_shared(ioc, *config, shared_from_this()); + connection->connect(); + } + + void received_abi() override { + ilog("request status"); + connection->send(get_status_request_v0{}); + } + + bool received(get_status_result_v0& status, eosio::input_stream bin) override { + ilog("nodeos has chain ${c}", ("c", eosio::check(eosio::convert_to_json(status.chain_id)).value())); + if (rodeos_snapshot->chain_id == eosio::checksum256{}) + rodeos_snapshot->chain_id = status.chain_id; + if (rodeos_snapshot->chain_id != status.chain_id) + throw std::runtime_error( + "database is for chain " + eosio::check(eosio::convert_to_json(rodeos_snapshot->chain_id)).value() + + " but nodeos has chain " + eosio::check(eosio::convert_to_json(status.chain_id)).value()); + ilog("request blocks"); + connection->request_blocks(status, std::max(config->skip_to, rodeos_snapshot->head + 1), get_positions(), + ship_client::request_block | ship_client::request_traces | + ship_client::request_deltas); + return true; + } + + std::vector get_positions() { + std::vector result; + if (rodeos_snapshot->head) { + rodeos::db_view_state view_state{ eosio::name{ "state" }, *db, *rodeos_snapshot->write_session, + partition->contract_kv_prefix }; + for (uint32_t i = rodeos_snapshot->irreversible; i <= rodeos_snapshot->head; ++i) { + auto info = rodeos::get_state_row( + view_state.kv_state.view, std::make_tuple(eosio::name{ "block.info" }, eosio::name{ "primary" }, i)); + if (!info) + throw std::runtime_error("database is missing block.info for block " + std::to_string(i)); + auto& info0 = std::get(info->second); + result.push_back({ info0.num, info0.id }); + } + } + return result; + } + + bool received(get_blocks_result_v0& result, eosio::input_stream bin) override { + if (!result.this_block) + return true; + if (config->stop_before && result.this_block->block_num >= config->stop_before) { + ilog("block ${b}: stop requested", ("b", result.this_block->block_num)); + rodeos_snapshot->end_write(true); + db->flush(false, false); + return false; + } + if (rodeos_snapshot->head && result.this_block->block_num > rodeos_snapshot->head + 1) + throw std::runtime_error("state-history plugin is missing block " + std::to_string(rodeos_snapshot->head + 1)); + + rodeos_snapshot->start_block(result); + if (result.this_block->block_num <= rodeos_snapshot->head) + reported_block = false; + + bool near = result.this_block->block_num + 4 >= result.last_irreversible.block_num; + bool write_now = !(result.this_block->block_num % 200) || near; + if (write_now || !reported_block) + ilog("block ${b} ${i}", + ("b", result.this_block->block_num)( + "i", result.this_block->block_num <= result.last_irreversible.block_num ? "irreversible" : "")); + reported_block = true; + + rodeos_snapshot->write_block_info(result); + rodeos_snapshot->write_deltas(result, [] { return app().is_quiting(); }); + + // todo: remove + if (filter) + filter->process(*rodeos_snapshot, result, bin, [](const char*, uint64_t) {}); + + rodeos_snapshot->end_block(result, false); + return true; + } // receive_result() + + void closed(bool retry) override { + if (my) { + my->session.reset(); + if (retry) + my->schedule_retry(); + } + } + + ~cloner_session() {} +}; // cloner_session + +static abstract_plugin& _cloner_plugin = app().register_plugin(); + +cloner_plugin_impl::~cloner_plugin_impl() { + if (session) + session->my = nullptr; +} + +void cloner_plugin_impl::start() { + session = std::make_shared(this); + session->connect(app().get_io_service()); +} + +cloner_plugin::cloner_plugin() : my(std::make_shared()) {} + +cloner_plugin::~cloner_plugin() {} + +void cloner_plugin::set_program_options(options_description& cli, options_description& cfg) { + auto op = cfg.add_options(); + auto clop = cli.add_options(); + op("clone-connect-to,f", bpo::value()->default_value("127.0.0.1:8080"), + "State-history endpoint to connect to (nodeos)"); + clop("clone-skip-to,k", bpo::value(), "Skip blocks before [arg]"); + clop("clone-stop,x", bpo::value(), "Stop before block [arg]"); + // todo: remove + op("filter-name", bpo::value(), "Filter name"); + op("filter-wasm", bpo::value(), "Filter wasm"); +} + +void cloner_plugin::plugin_initialize(const variables_map& options) { + try { + auto endpoint = options.at("clone-connect-to").as(); + if (endpoint.find(':') == std::string::npos) + throw std::runtime_error("invalid endpoint: " + endpoint); + + auto port = endpoint.substr(endpoint.find(':') + 1, endpoint.size()); + auto host = endpoint.substr(0, endpoint.find(':')); + my->config->host = host; + my->config->port = port; + my->config->skip_to = options.count("clone-skip-to") ? options["clone-skip-to"].as() : 0; + my->config->stop_before = options.count("clone-stop") ? options["clone-stop"].as() : 0; + if (options.count("filter-name") && options.count("filter-wasm")) { + my->config->filter_name = eosio::name{ options["filter-name"].as() }; + my->config->filter_wasm = options["filter-wasm"].as(); + } else if (options.count("filter-name") || options.count("filter-wasm")) { + throw std::runtime_error("filter-name and filter-wasm must be used together"); + } + } + FC_LOG_AND_RETHROW() +} + +void cloner_plugin::plugin_startup() { my->start(); } + +void cloner_plugin::plugin_shutdown() { + if (my->session) + my->session->connection->close(false); + my->timer.cancel(); + ilog("cloner_plugin stopped"); +} + +} // namespace b1 diff --git a/programs/rodeos/cloner_plugin.hpp b/programs/rodeos/cloner_plugin.hpp new file mode 100644 index 00000000000..d9c382a6aeb --- /dev/null +++ b/programs/rodeos/cloner_plugin.hpp @@ -0,0 +1,22 @@ +#pragma once +#include "rocksdb_plugin.hpp" + +namespace b1 { + +class cloner_plugin : public appbase::plugin { + public: + APPBASE_PLUGIN_REQUIRES((rocksdb_plugin)) + + cloner_plugin(); + virtual ~cloner_plugin(); + + virtual void set_program_options(appbase::options_description& cli, appbase::options_description& cfg) override; + void plugin_initialize(const appbase::variables_map& options); + void plugin_startup(); + void plugin_shutdown(); + + private: + std::shared_ptr my; +}; + +} // namespace b1 diff --git a/programs/rodeos/config.hpp.in b/programs/rodeos/config.hpp.in new file mode 100644 index 00000000000..c83b5e280aa --- /dev/null +++ b/programs/rodeos/config.hpp.in @@ -0,0 +1,16 @@ +// clang-format off + +/** + * \warning This file is machine generated. DO NOT EDIT. See config.hpp.in for changes. + */ +#pragma once + +#ifndef CONFIG_HPP_IN +# define CONFIG_HPP_IN + +namespace b1::rodeos::config { + constexpr uint64_t version = 0x${rodeos_BUILD_VERSION}; + const std::string rodeos_executable_name = "${RODEOS_EXECUTABLE_NAME}"; +} + +#endif // CONFIG_HPP_IN diff --git a/programs/rodeos/main.cpp b/programs/rodeos/main.cpp new file mode 100644 index 00000000000..6c0f115cd96 --- /dev/null +++ b/programs/rodeos/main.cpp @@ -0,0 +1,116 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cloner_plugin.hpp" +#include "config.hpp" + +using namespace appbase; + +namespace detail { + +void configure_logging(const bfs::path& config_path) { + try { + try { + fc::configure_logging(config_path); + } catch (...) { + elog("Error reloading logging.json"); + throw; + } + } catch (const fc::exception& e) { // + elog("${e}", ("e", e.to_detail_string())); + } catch (const boost::exception& e) { + elog("${e}", ("e", boost::diagnostic_information(e))); + } catch (const std::exception& e) { // + elog("${e}", ("e", e.what())); + } catch (...) { + // empty + } +} + +} // namespace detail + +void logging_conf_handler() { + auto config_path = app().get_logging_conf(); + if (fc::exists(config_path)) { + ilog("Received HUP. Reloading logging configuration from ${p}.", ("p", config_path.string())); + } else { + ilog("Received HUP. No log config found at ${p}, setting to default.", ("p", config_path.string())); + } + ::detail::configure_logging(config_path); + fc::log_config::initialize_appenders(app().get_io_service()); +} + +void initialize_logging() { + auto config_path = app().get_logging_conf(); + if (fc::exists(config_path)) + fc::configure_logging(config_path); // intentionally allowing exceptions to escape + fc::log_config::initialize_appenders(app().get_io_service()); + + app().set_sighup_callback(logging_conf_handler); +} + +enum return_codes { + other_fail = -2, + initialize_fail = -1, + success = 0, + bad_alloc = 1, +}; + +int main(int argc, char** argv) { + try { + app().set_version(b1::rodeos::config::version); + app().set_version_string(eosio::version::version_client()); + app().set_full_version_string(eosio::version::version_full()); + + auto root = fc::app_path(); + app().set_default_data_dir(root / "eosio" / b1::rodeos::config::rodeos_executable_name / "data"); + app().set_default_config_dir(root / "eosio" / b1::rodeos::config::rodeos_executable_name / "config"); + if (!app().initialize(argc, argv)) { + const auto& opts = app().get_options(); + if (opts.count("help") || opts.count("version") || opts.count("full-version") || + opts.count("print-default-config")) { + return success; + } + return initialize_fail; + } + initialize_logging(); + ilog("${name} version ${ver} ${fv}", + ("name", b1::rodeos::config::rodeos_executable_name)("ver", app().version_string())( + "fv", app().version_string() == app().full_version_string() ? "" : app().full_version_string())); + ilog("${name} using configuration file ${c}", + ("name", b1::rodeos::config::rodeos_executable_name)("c", app().full_config_file_path().string())); + ilog("${name} data directory is ${d}", + ("name", b1::rodeos::config::rodeos_executable_name)("d", app().data_dir().string())); + app().startup(); + app().set_thread_priority_max(); + app().exec(); + } catch (const fc::std_exception_wrapper& e) { + elog("${e}", ("e", e.to_detail_string())); + return other_fail; + } catch (const fc::exception& e) { + elog("${e}", ("e", e.to_detail_string())); + return other_fail; + } catch (const boost::interprocess::bad_alloc& e) { + elog("bad alloc"); + return bad_alloc; + } catch (const boost::exception& e) { + elog("${e}", ("e", boost::diagnostic_information(e))); + return other_fail; + } catch (const std::exception& e) { + elog("${e}", ("e", e.what())); + return other_fail; + } catch (...) { + elog("unknown exception"); + return other_fail; + } + + ilog("${name} successfully exiting", ("name", b1::rodeos::config::rodeos_executable_name)); + return success; +} diff --git a/programs/rodeos/rocksdb_plugin.cpp b/programs/rodeos/rocksdb_plugin.cpp new file mode 100644 index 00000000000..523a21f3c8c --- /dev/null +++ b/programs/rodeos/rocksdb_plugin.cpp @@ -0,0 +1,57 @@ +#include "rocksdb_plugin.hpp" + +#include + +namespace b1 { + +using namespace appbase; +using namespace std::literals; + +struct rocksdb_plugin_impl { + boost::filesystem::path db_path = {}; + std::optional threads = {}; + std::optional max_open_files = {}; + std::shared_ptr database = {}; + std::mutex mutex = {}; +}; + +static abstract_plugin& _rocksdb_plugin = app().register_plugin(); + +rocksdb_plugin::rocksdb_plugin() : my(std::make_shared()) {} + +rocksdb_plugin::~rocksdb_plugin() {} + +void rocksdb_plugin::set_program_options(options_description& cli, options_description& cfg) { + auto op = cfg.add_options(); + op("rdb-database", bpo::value()->default_value("./chain.rocksdb"), "Primary database path"); + op("rdb-threads", bpo::value(), + "Increase number of background RocksDB threads. Only used with cloner_plugin. Recommend 8 for full history " + "on large chains."); + op("rdb-max-files", bpo::value(), + "RocksDB limit max number of open files (default unlimited). This should be smaller than 'ulimit -n #'. " + "# should be a very large number for full-history nodes."); +} + +void rocksdb_plugin::plugin_initialize(const variables_map& options) { + try { + my->db_path = options["rdb-database"].as(); + if (!options["rdb-threads"].empty()) + my->threads = options["rdb-threads"].as(); + if (!options["rdb-max-files"].empty()) + my->max_open_files = options["rdb-max-files"].as(); + } + FC_LOG_AND_RETHROW() +} + +void rocksdb_plugin::plugin_startup() {} + +void rocksdb_plugin::plugin_shutdown() {} + +std::shared_ptr rocksdb_plugin::get_db() { + std::lock_guard lock(my->mutex); + if (!my->database) + my->database = std::make_shared(my->db_path.c_str(), true, my->threads, my->max_open_files); + return my->database; +} + +} // namespace b1 diff --git a/programs/rodeos/rocksdb_plugin.hpp b/programs/rodeos/rocksdb_plugin.hpp new file mode 100644 index 00000000000..45e366de5a9 --- /dev/null +++ b/programs/rodeos/rocksdb_plugin.hpp @@ -0,0 +1,25 @@ +#pragma once +#include +#include + +namespace b1 { + +class rocksdb_plugin : public appbase::plugin { + public: + APPBASE_PLUGIN_REQUIRES() + + rocksdb_plugin(); + virtual ~rocksdb_plugin(); + + virtual void set_program_options(appbase::options_description& cli, appbase::options_description& cfg) override; + void plugin_initialize(const appbase::variables_map& options); + void plugin_startup(); + void plugin_shutdown(); + + std::shared_ptr get_db(); + + private: + std::shared_ptr my; +}; + +} // namespace b1 diff --git a/programs/rodeos/ship_client.hpp b/programs/rodeos/ship_client.hpp new file mode 100644 index 00000000000..9bd67f10e37 --- /dev/null +++ b/programs/rodeos/ship_client.hpp @@ -0,0 +1,209 @@ +// TODO: move to a library + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +namespace b1::ship_client { + +namespace ship = eosio::ship_protocol; + +enum request_flags { + request_irreversible_only = 1, + request_block = 2, + request_traces = 4, + request_deltas = 8, +}; + +struct connection_callbacks { + virtual ~connection_callbacks() = default; + virtual void received_abi() {} + virtual bool received(ship::get_status_result_v0& status, eosio::input_stream bin) { return true; } + virtual bool received(ship::get_blocks_result_v0& result, eosio::input_stream bin) { return true; } + virtual void closed(bool retry) = 0; +}; + +struct connection_config { + std::string host; + std::string port; +}; + +struct abi_def_skip_table : eosio::abi_def {}; + +EOSIO_REFLECT(abi_def_skip_table, version, types, structs, actions, ricardian_clauses, error_messages, abi_extensions, + variants); + +struct connection : std::enable_shared_from_this { + using error_code = boost::system::error_code; + using flat_buffer = boost::beast::flat_buffer; + using tcp = boost::asio::ip::tcp; + using abi_type = eosio::abi_type; + + connection_config config; + std::shared_ptr callbacks; + tcp::resolver resolver; + boost::beast::websocket::stream stream; + bool have_abi = false; + abi_def_skip_table abi = {}; + std::map abi_types = {}; + + connection(boost::asio::io_context& ioc, const connection_config& config, + std::shared_ptr callbacks) + : config(config), callbacks(callbacks), resolver(ioc), stream(ioc) { + + stream.binary(true); + stream.read_message_max(10ull * 1024 * 1024 * 1024); + } + + void connect() { + ilog("connect to ${h}:${p}", ("h", config.host)("p", config.port)); + resolver.async_resolve( // + config.host, config.port, + [self = shared_from_this(), this](error_code ec, tcp::resolver::results_type results) { + enter_callback(ec, "resolve", [&] { + boost::asio::async_connect( // + stream.next_layer(), results.begin(), results.end(), + [self = shared_from_this(), this](error_code ec, auto&) { + enter_callback(ec, "connect", [&] { + stream.async_handshake( // + config.host, "/", [self = shared_from_this(), this](error_code ec) { + enter_callback(ec, "handshake", [&] { // + start_read(); + }); + }); + }); + }); + }); + }); + } + + void start_read() { + auto in_buffer = std::make_shared(); + stream.async_read(*in_buffer, [self = shared_from_this(), this, in_buffer](error_code ec, size_t) { + enter_callback(ec, "async_read", [&] { + if (!have_abi) + receive_abi(in_buffer); + else { + if (!receive_result(in_buffer)) { + close(false); + return; + } + } + start_read(); + }); + }); + } + + void receive_abi(const std::shared_ptr& p) { + auto data = p->data(); + std::string json{ (const char*)data.data(), data.size() }; + eosio::json_token_stream stream{ json.data() }; + eosio::check_discard(from_json(abi, stream)); + std::string error; + if (!abieos::check_abi_version(abi.version, error)) + throw std::runtime_error(error); + eosio::abi a; + eosio::check_discard(convert(abi, a)); + abi_types = std::move(a.abi_types); + have_abi = true; + if (callbacks) + callbacks->received_abi(); + } + + bool receive_result(const std::shared_ptr& p) { + auto data = p->data(); + eosio::input_stream bin{ (const char*)data.data(), (const char*)data.data() + data.size() }; + auto orig = bin; + ship::result result; + auto r = from_bin(result, bin); + if (!r) + throw std::runtime_error{ r.error().message() }; + return callbacks && std::visit([&](auto& r) { return callbacks->received(r, orig); }, result); + } + + void request_blocks(uint32_t start_block_num, const std::vector& positions, int flags) { + ship::get_blocks_request_v0 req; + req.start_block_num = start_block_num; + req.end_block_num = 0xffff'ffff; + req.max_messages_in_flight = 0xffff'ffff; + req.have_positions = positions; + req.irreversible_only = flags & request_irreversible_only; + req.fetch_block = flags & request_block; + req.fetch_traces = flags & request_traces; + req.fetch_deltas = flags & request_deltas; + send(req); + } + + void request_blocks(const ship::get_status_result_v0& status, uint32_t start_block_num, + const std::vector& positions, int flags) { + uint32_t nodeos_start = 0xffff'ffff; + if (status.trace_begin_block < status.trace_end_block) + nodeos_start = std::min(nodeos_start, status.trace_begin_block); + if (status.chain_state_begin_block < status.chain_state_end_block) + nodeos_start = std::min(nodeos_start, status.chain_state_begin_block); + if (nodeos_start == 0xffff'ffff) + nodeos_start = 0; + request_blocks(std::max(start_block_num, nodeos_start), positions, flags); + } + + const abi_type& get_type(const std::string& name) { + auto it = abi_types.find(name); + if (it == abi_types.end()) + throw std::runtime_error(std::string("unknown type ") + name); + return it->second; + } + + void send(const ship::request& req) { + auto bin = std::make_shared>(); + auto r = eosio::convert_to_bin(req, *bin); + if (!r) + throw std::runtime_error{ r.error().message() }; + stream.async_write(boost::asio::buffer(*bin), [self = shared_from_this(), bin, this](error_code ec, size_t) { + enter_callback(ec, "async_write", [&] {}); + }); + } + + template + void catch_and_close(F f) { + try { + f(); + } catch (const std::exception& e) { + elog("${e}", ("e", e.what())); + close(false); + } catch (...) { + elog("unknown exception"); + close(false); + } + } + + template + void enter_callback(error_code ec, const char* what, F f) { + if (ec) + return on_fail(ec, what); + catch_and_close(f); + } + + void on_fail(error_code ec, const char* what) { + try { + elog("${w}: ${m}", ("w", what)("m", ec.message())); + close(true); + } catch (...) { elog("exception while closing"); } + } + + void close(bool retry) { + ilog("closing state-history socket"); + stream.next_layer().close(); + if (callbacks) + callbacks->closed(retry); + callbacks.reset(); + } +}; // connection + +} // namespace b1::ship_client From 76809cd1ce24c285f3ad7d445a48c4e7b3c2e4ca Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Sun, 26 Apr 2020 10:51:56 -0400 Subject: [PATCH 08/23] rodeos --- .../include/b1/rodeos/rodeos_tables.hpp | 20 +++++++++---------- libraries/rodeos/include/eosio/key_value.hpp | 7 ++++--- libraries/rodeos/rodeos.cpp | 13 ++++++------ libraries/rodeos/wasm_ql.cpp | 12 +++++------ 4 files changed, 27 insertions(+), 25 deletions(-) diff --git a/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp b/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp index 576d0f75c5e..c6b5246a034 100644 --- a/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp +++ b/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp @@ -46,7 +46,7 @@ inline bool operator==(const fill_status_v0& a, fill_status_v0& b) { inline bool operator!=(const fill_status_v0& a, fill_status_v0& b) { return !(a == b); } -using fill_status_kv = eosio::kv_singleton; +using fill_status_sing = eosio::kv_singleton; struct block_info_v0 { uint32_t num = {}; @@ -78,7 +78,7 @@ struct block_info_kv : eosio::kv_table { } }; block_info_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "block.info" }, primary_index, id_index); + init(eosio::name{ "state" }, eosio::name{ "block.info" }, eosio::name{ "eosio.state" }, primary_index, id_index); } }; @@ -87,7 +87,7 @@ struct global_property_kv : eosio::kv_table { [](const auto& var) { return std::vector{}; } }; global_property_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "global.prop" }, primary_index); + init(eosio::name{ "state" }, eosio::name{ "global.prop" }, eosio::name{ "eosio.state" }, primary_index); } }; @@ -97,7 +97,7 @@ struct account_kv : eosio::kv_table { } }; account_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "account" }, primary_index); + init(eosio::name{ "state" }, eosio::name{ "account" }, eosio::name{ "eosio.state" }, primary_index); } }; @@ -108,7 +108,7 @@ struct account_metadata_kv : eosio::kv_table { account_metadata_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "account.meta" }, primary_index); + init(eosio::name{ "state" }, eosio::name{ "account.meta" }, eosio::name{ "eosio.state" }, primary_index); } }; @@ -121,7 +121,7 @@ struct code_kv : eosio::kv_table { }; code_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "code" }, primary_index); + init(eosio::name{ "state" }, eosio::name{ "code" }, eosio::name{ "eosio.state" }, primary_index); } }; @@ -134,7 +134,7 @@ struct contract_table_kv : eosio::kv_table { }; contract_table_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "contract.tab" }, primary_index); + init(eosio::name{ "state" }, eosio::name{ "contract.tab" }, eosio::name{ "eosio.state" }, primary_index); } }; @@ -149,7 +149,7 @@ struct contract_row_kv : eosio::kv_table { } }; contract_row_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "contract.row" }, primary_index); + init(eosio::name{ "state" }, eosio::name{ "contract.row" }, eosio::name{ "eosio.state" }, primary_index); } }; @@ -175,7 +175,7 @@ struct contract_index64_kv : eosio::kv_table { contract_index64_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "contract.i1" }, primary_index, + init(eosio::name{ "state" }, eosio::name{ "contract.i1" }, eosio::name{ "eosio.state" }, primary_index, secondary_index); } }; @@ -202,7 +202,7 @@ struct contract_index128_kv : eosio::kv_table { contract_index128_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(eosio::name{ "eosio.state" }, eosio::name{ "state" }, eosio::name{ "contract.i2" }, primary_index, + init(eosio::name{ "state" }, eosio::name{ "contract.i2" }, eosio::name{ "eosio.state" }, primary_index, secondary_index); } }; diff --git a/libraries/rodeos/include/eosio/key_value.hpp b/libraries/rodeos/include/eosio/key_value.hpp index 36fd66880ba..ac669cb9bda 100644 --- a/libraries/rodeos/include/eosio/key_value.hpp +++ b/libraries/rodeos/include/eosio/key_value.hpp @@ -868,14 +868,14 @@ class kv_singleton { public: using value_type = T; - explicit kv_singleton(eosio::name contract_name) : contract_name{contract_name} { + explicit kv_singleton(eosio::name contract_name, bool write_on_destruct) : contract_name{contract_name}, write_on_destruct{write_on_destruct} { key = make_prefix(); } - kv_singleton(eosio::name contract_name, kv_environment environment) : environment{environment}, contract_name{contract_name} {} + kv_singleton(eosio::name contract_name, kv_environment environment, bool write_on_destruct) : environment{environment}, contract_name{contract_name}, write_on_destruct{write_on_destruct} {} ~kv_singleton() { - if (get_state().is_dirty) { + if (get_state().is_dirty && write_on_destruct) { store(); } } @@ -942,6 +942,7 @@ class kv_singleton { constexpr static uint64_t singleton_name = static_cast(SingletonName); eosio::name contract_name; + bool write_on_destruct; key_type key; key_type make_prefix() { diff --git a/libraries/rodeos/rodeos.cpp b/libraries/rodeos/rodeos.cpp index 336ac301d9d..501579d43e5 100644 --- a/libraries/rodeos/rodeos.cpp +++ b/libraries/rodeos/rodeos.cpp @@ -20,10 +20,10 @@ rodeos_db_snapshot::rodeos_db_snapshot(std::shared_ptr part write_session.emplace(*db, snap->snapshot()); } - db_view_state view_state{ eosio::name{ "state" }, *db, *write_session, this->partition->contract_kv_prefix }; - fill_status_kv table{ eosio::name{ "state" }, view_state }; - if (table.exists()) { - auto status = std::get<0>(table.get()); + db_view_state view_state{ eosio::name{ "state" }, *db, *write_session, this->partition->contract_kv_prefix }; + fill_status_sing sing{ eosio::name{ "state" }, view_state, false }; + if (sing.exists()) { + auto status = std::get<0>(sing.get()); chain_id = status.chain_id; head = status.head; head_id = status.head_id; @@ -62,8 +62,9 @@ void rodeos_db_snapshot::write_fill_status() { db_view_state view_state{ eosio::name{ "state" }, *db, *write_session, partition->contract_kv_prefix }; view_state.kv_state.enable_write = true; - fill_status_kv table{ eosio::name{ "state" }, view_state }; - table.set(status); + fill_status_sing sing{ eosio::name{ "state" }, view_state, false }; + sing.set(status); + sing.store(); } void rodeos_db_snapshot::end_write(bool write_fill) { diff --git a/libraries/rodeos/wasm_ql.cpp b/libraries/rodeos/wasm_ql.cpp index 545d9c4c176..57acc49c0c6 100644 --- a/libraries/rodeos/wasm_ql.cpp +++ b/libraries/rodeos/wasm_ql.cpp @@ -261,10 +261,10 @@ void run_action(wasm_ql::thread_state& thread_state, const std::vector& co } auto se = fc::make_scoped_exit([&] { thread_state.shared->backend_cache->add(std::move(*entry)); }); - fill_status_kv fill_status_table{ eosio::name{ "state" }, db_view_state }; - if (!fill_status_table.exists()) + fill_status_sing sing{ eosio::name{ "state" }, db_view_state, false }; + if (!sing.exists()) throw std::runtime_error("No fill_status records found; is filler running?"); - auto& fill_status = fill_status_table.get(); + auto& fill_status = sing.get(); // todo: move these out of thread_state since future enhancements could cause state to accidentally leak between // queries @@ -318,8 +318,8 @@ const std::vector& query_get_info(wasm_ql::thread_state& thread_state, } { - fill_status_kv table{ eosio::name{ "state" }, db_view_state }; - if (table.exists()) { + fill_status_sing sing{ eosio::name{ "state" }, db_view_state, false }; + if (sing.exists()) { std::visit( [&](auto& obj) { result += ",\"head_block_num\":\"" + std::to_string(obj.head) + "\""; @@ -328,7 +328,7 @@ const std::vector& query_get_info(wasm_ql::thread_state& thread_state, result += ",\"last_irreversible_block_id\":" + eosio::check(eosio::convert_to_json(obj.irreversible_id)).value(); }, - table.get()); + sing.get()); } else throw std::runtime_error("No fill_status records found; is filler running?"); } From 1ab263555ca2928f844fb1b50a4771839c3f31f9 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Sun, 26 Apr 2020 11:31:09 -0400 Subject: [PATCH 09/23] rodeos --- .../include/b1/rodeos/callbacks/chaindb.hpp | 7 +++--- .../rodeos/include/b1/rodeos/constants.hpp | 13 +++++++++++ .../include/b1/rodeos/get_state_row.hpp | 6 ++--- .../include/b1/rodeos/rodeos_tables.hpp | 23 +++++++++---------- libraries/rodeos/rodeos.cpp | 12 +++++----- libraries/rodeos/wasm_ql.cpp | 12 +++++----- programs/rodeos/cloner_plugin.cpp | 2 +- programs/rodeos/rocksdb_plugin.cpp | 2 +- 8 files changed, 45 insertions(+), 32 deletions(-) create mode 100644 libraries/rodeos/include/b1/rodeos/constants.hpp diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/chaindb.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/chaindb.hpp index 62d1fe1e575..a0ec57a12e3 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/chaindb.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/chaindb.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -44,7 +45,7 @@ class iterator_cache { auto map_it = table_to_index.find(key); if (map_it != table_to_index.end()) return map_it->second; - if (!view.get(eosio::name{ "state" }.value, + if (!view.get(state_account.value, chain_kv::to_slice(eosio::check(eosio::convert_to_key(std::make_tuple( (uint8_t)0x01, eosio::name{ "contract.tab" }, eosio::name{ "primary" }, key.code, key.table, key.scope))) @@ -135,7 +136,7 @@ class iterator_cache { // std::cout << "db_next_i64: db_view::iterator\n"; const auto& table_key = tables[it.table_index]; view_it = chain_kv::view::iterator{ - view, eosio::name{ "state" }.value, + view, state_account.value, chain_kv::to_slice( eosio::check(eosio::convert_to_key(std::make_tuple( // (uint8_t)0x01, eosio::name{ "contract.row" }, eosio::name{ "primary" }, @@ -173,7 +174,7 @@ class iterator_cache { return map_it->second; } // std::cout << "lower_bound: db_view::iterator\n"; - chain_kv::view::iterator it{ view, eosio::name{ "state" }.value, + chain_kv::view::iterator it{ view, state_account.value, chain_kv::to_slice(eosio::check(eosio::convert_to_key(std::make_tuple( (uint8_t)0x01, eosio::name{ "contract.row" }, eosio::name{ "primary" }, code, table, scope))) diff --git a/libraries/rodeos/include/b1/rodeos/constants.hpp b/libraries/rodeos/include/b1/rodeos/constants.hpp new file mode 100644 index 00000000000..094bed9587b --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/constants.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include + +namespace b1::rodeos { + +// kv database which stores rodeos state, including a mirror of nodeos state +inline constexpr eosio::name state_database{ "eosio.state" }; + +// account within state_database which stores state +inline constexpr eosio::name state_account{ "state" }; + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/get_state_row.hpp b/libraries/rodeos/include/b1/rodeos/get_state_row.hpp index ab440a9ad06..29bf3353234 100644 --- a/libraries/rodeos/include/b1/rodeos/get_state_row.hpp +++ b/libraries/rodeos/include/b1/rodeos/get_state_row.hpp @@ -10,7 +10,7 @@ std::optional, T>> get_state_ro std::optional, T>> result; result.emplace(); result->first = - view.get(eosio::name{ "state" }.value, + view.get(state_account.value, chain_kv::to_slice(eosio::check(eosio::convert_to_key(std::make_tuple((uint8_t)0x01, key))).value())); if (!result->first) { result.reset(); @@ -28,13 +28,13 @@ std::optional, T>> get_state_ro const K& key) { std::optional, T>> result; auto pk = - view.get(eosio::name{ "state" }.value, + view.get(state_account.value, chain_kv::to_slice(eosio::check(eosio::convert_to_key(std::make_tuple((uint8_t)0x01, key))).value())); if (!pk) return result; result.emplace(); - result->first = view.get(eosio::name{ "state" }.value, chain_kv::to_slice(*pk)); + result->first = view.get(state_account.value, chain_kv::to_slice(*pk)); if (!result->first) { result.reset(); return result; diff --git a/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp b/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp index c6b5246a034..c64de1a20eb 100644 --- a/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp +++ b/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -46,7 +47,7 @@ inline bool operator==(const fill_status_v0& a, fill_status_v0& b) { inline bool operator!=(const fill_status_v0& a, fill_status_v0& b) { return !(a == b); } -using fill_status_sing = eosio::kv_singleton; +using fill_status_sing = eosio::kv_singleton; struct block_info_v0 { uint32_t num = {}; @@ -78,7 +79,7 @@ struct block_info_kv : eosio::kv_table { } }; block_info_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(eosio::name{ "state" }, eosio::name{ "block.info" }, eosio::name{ "eosio.state" }, primary_index, id_index); + init(state_account, eosio::name{ "block.info" }, state_database, primary_index, id_index); } }; @@ -87,7 +88,7 @@ struct global_property_kv : eosio::kv_table { [](const auto& var) { return std::vector{}; } }; global_property_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(eosio::name{ "state" }, eosio::name{ "global.prop" }, eosio::name{ "eosio.state" }, primary_index); + init(state_account, eosio::name{ "global.prop" }, state_database, primary_index); } }; @@ -97,7 +98,7 @@ struct account_kv : eosio::kv_table { } }; account_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(eosio::name{ "state" }, eosio::name{ "account" }, eosio::name{ "eosio.state" }, primary_index); + init(state_account, eosio::name{ "account" }, state_database, primary_index); } }; @@ -108,7 +109,7 @@ struct account_metadata_kv : eosio::kv_table { account_metadata_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(eosio::name{ "state" }, eosio::name{ "account.meta" }, eosio::name{ "eosio.state" }, primary_index); + init(state_account, eosio::name{ "account.meta" }, state_database, primary_index); } }; @@ -121,7 +122,7 @@ struct code_kv : eosio::kv_table { }; code_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(eosio::name{ "state" }, eosio::name{ "code" }, eosio::name{ "eosio.state" }, primary_index); + init(state_account, eosio::name{ "code" }, state_database, primary_index); } }; @@ -134,7 +135,7 @@ struct contract_table_kv : eosio::kv_table { }; contract_table_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(eosio::name{ "state" }, eosio::name{ "contract.tab" }, eosio::name{ "eosio.state" }, primary_index); + init(state_account, eosio::name{ "contract.tab" }, state_database, primary_index); } }; @@ -149,7 +150,7 @@ struct contract_row_kv : eosio::kv_table { } }; contract_row_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(eosio::name{ "state" }, eosio::name{ "contract.row" }, eosio::name{ "eosio.state" }, primary_index); + init(state_account, eosio::name{ "contract.row" }, state_database, primary_index); } }; @@ -175,8 +176,7 @@ struct contract_index64_kv : eosio::kv_table { contract_index64_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(eosio::name{ "state" }, eosio::name{ "contract.i1" }, eosio::name{ "eosio.state" }, primary_index, - secondary_index); + init(state_account, eosio::name{ "contract.i1" }, state_database, primary_index, secondary_index); } }; @@ -202,8 +202,7 @@ struct contract_index128_kv : eosio::kv_table { contract_index128_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(eosio::name{ "state" }, eosio::name{ "contract.i2" }, eosio::name{ "eosio.state" }, primary_index, - secondary_index); + init(state_account, eosio::name{ "contract.i2" }, state_database, primary_index, secondary_index); } }; diff --git a/libraries/rodeos/rodeos.cpp b/libraries/rodeos/rodeos.cpp index 501579d43e5..38851bd07c9 100644 --- a/libraries/rodeos/rodeos.cpp +++ b/libraries/rodeos/rodeos.cpp @@ -20,8 +20,8 @@ rodeos_db_snapshot::rodeos_db_snapshot(std::shared_ptr part write_session.emplace(*db, snap->snapshot()); } - db_view_state view_state{ eosio::name{ "state" }, *db, *write_session, this->partition->contract_kv_prefix }; - fill_status_sing sing{ eosio::name{ "state" }, view_state, false }; + db_view_state view_state{ state_account, *db, *write_session, this->partition->contract_kv_prefix }; + fill_status_sing sing{ state_account, view_state, false }; if (sing.exists()) { auto status = std::get<0>(sing.get()); chain_id = status.chain_id; @@ -60,9 +60,9 @@ void rodeos_db_snapshot::write_fill_status() { .irreversible_id = head_id, .first = first }; - db_view_state view_state{ eosio::name{ "state" }, *db, *write_session, partition->contract_kv_prefix }; + db_view_state view_state{ state_account, *db, *write_session, partition->contract_kv_prefix }; view_state.kv_state.enable_write = true; - fill_status_sing sing{ eosio::name{ "state" }, view_state, false }; + fill_status_sing sing{ state_account, view_state, false }; sing.set(status); sing.store(); } @@ -147,7 +147,7 @@ void rodeos_db_snapshot::write_block_info(const ship_protocol::get_blocks_result signed_block block; eosio::check_discard(from_bin(block, bin)); - db_view_state view_state{ eosio::name{ "state" }, *db, *write_session, partition->contract_kv_prefix }; + db_view_state view_state{ state_account, *db, *write_session, partition->contract_kv_prefix }; view_state.kv_state.enable_write = true; block_info_v0 info; @@ -176,7 +176,7 @@ void rodeos_db_snapshot::write_deltas(const ship_protocol::get_blocks_result_v0& uint32_t block_num = result.this_block->block_num; eosio::input_stream bin = *result.deltas; - db_view_state view_state{ eosio::name{ "state" }, *db, *write_session, partition->contract_kv_prefix }; + db_view_state view_state{ state_account, *db, *write_session, partition->contract_kv_prefix }; view_state.kv_ram.enable_write = true; view_state.kv_ram.bypass_receiver_check = true; view_state.kv_disk.enable_write = true; diff --git a/libraries/rodeos/wasm_ql.cpp b/libraries/rodeos/wasm_ql.cpp index 57acc49c0c6..d349a15260e 100644 --- a/libraries/rodeos/wasm_ql.cpp +++ b/libraries/rodeos/wasm_ql.cpp @@ -228,7 +228,7 @@ void run_action(wasm_ql::thread_state& thread_state, const std::vector& co throw eosio::vm::timeout_exception("execution timed out"); chain_kv::write_session write_session{ *thread_state.shared->db, snapshot }; - db_view_state db_view_state{ eosio::name{ "state" }, *thread_state.shared->db, write_session, contract_kv_prefix }; + db_view_state db_view_state{ state_account, *thread_state.shared->db, write_session, contract_kv_prefix }; std::optional entry = thread_state.shared->backend_cache->get(action.account); std::optional> code; @@ -261,7 +261,7 @@ void run_action(wasm_ql::thread_state& thread_state, const std::vector& co } auto se = fc::make_scoped_exit([&] { thread_state.shared->backend_cache->add(std::move(*entry)); }); - fill_status_sing sing{ eosio::name{ "state" }, db_view_state, false }; + fill_status_sing sing{ state_account, db_view_state, false }; if (!sing.exists()) throw std::runtime_error("No fill_status records found; is filler running?"); auto& fill_status = sing.get(); @@ -299,7 +299,7 @@ const std::vector& query_get_info(wasm_ql::thread_state& thread_state, const std::vector& contract_kv_prefix) { rocksdb::ManagedSnapshot snapshot{ thread_state.shared->db->rdb.get() }; chain_kv::write_session write_session{ *thread_state.shared->db, snapshot.snapshot() }; - db_view_state db_view_state{ eosio::name{ "state" }, *thread_state.shared->db, write_session, contract_kv_prefix }; + db_view_state db_view_state{ state_account, *thread_state.shared->db, write_session, contract_kv_prefix }; std::string result = "{\"server_type\":\"wasm-ql\""; @@ -318,7 +318,7 @@ const std::vector& query_get_info(wasm_ql::thread_state& thread_state, } { - fill_status_sing sing{ eosio::name{ "state" }, db_view_state, false }; + fill_status_sing sing{ state_account, db_view_state, false }; if (sing.exists()) { std::visit( [&](auto& obj) { @@ -355,7 +355,7 @@ const std::vector& query_get_block(wasm_ql::thread_state& thread_state, rocksdb::ManagedSnapshot snapshot{ thread_state.shared->db->rdb.get() }; chain_kv::write_session write_session{ *thread_state.shared->db, snapshot.snapshot() }; - db_view_state db_view_state{ eosio::name{ "state" }, *thread_state.shared->db, write_session, contract_kv_prefix }; + db_view_state db_view_state{ state_account, *thread_state.shared->db, write_session, contract_kv_prefix }; std::string bn_json = "\"" + params.block_num_or_id + "\""; eosio::json_token_stream bn_stream{ bn_json.data() }; @@ -424,7 +424,7 @@ const std::vector& query_get_abi(wasm_ql::thread_state& thread_state, cons rocksdb::ManagedSnapshot snapshot{ thread_state.shared->db->rdb.get() }; chain_kv::write_session write_session{ *thread_state.shared->db, snapshot.snapshot() }; - db_view_state db_view_state{ eosio::name{ "state" }, *thread_state.shared->db, write_session, contract_kv_prefix }; + db_view_state db_view_state{ state_account, *thread_state.shared->db, write_session, contract_kv_prefix }; auto acc = get_state_row( db_view_state.kv_state.view, diff --git a/programs/rodeos/cloner_plugin.cpp b/programs/rodeos/cloner_plugin.cpp index b5c2fda5a11..15901801468 100644 --- a/programs/rodeos/cloner_plugin.cpp +++ b/programs/rodeos/cloner_plugin.cpp @@ -116,7 +116,7 @@ struct cloner_session : ship_client::connection_callbacks, std::enable_shared_fr std::vector get_positions() { std::vector result; if (rodeos_snapshot->head) { - rodeos::db_view_state view_state{ eosio::name{ "state" }, *db, *rodeos_snapshot->write_session, + rodeos::db_view_state view_state{ rodeos::state_account, *db, *rodeos_snapshot->write_session, partition->contract_kv_prefix }; for (uint32_t i = rodeos_snapshot->irreversible; i <= rodeos_snapshot->head; ++i) { auto info = rodeos::get_state_row( diff --git a/programs/rodeos/rocksdb_plugin.cpp b/programs/rodeos/rocksdb_plugin.cpp index 523a21f3c8c..d80899f1e16 100644 --- a/programs/rodeos/rocksdb_plugin.cpp +++ b/programs/rodeos/rocksdb_plugin.cpp @@ -23,7 +23,7 @@ rocksdb_plugin::~rocksdb_plugin() {} void rocksdb_plugin::set_program_options(options_description& cli, options_description& cfg) { auto op = cfg.add_options(); - op("rdb-database", bpo::value()->default_value("./chain.rocksdb"), "Primary database path"); + op("rdb-database", bpo::value()->default_value("./rodeos.rocksdb"), "Primary database path"); op("rdb-threads", bpo::value(), "Increase number of background RocksDB threads. Only used with cloner_plugin. Recommend 8 for full history " "on large chains."); From 0d97166576d0e68cbf8fa31b3badd4ff200e404c Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Sun, 26 Apr 2020 11:58:04 -0400 Subject: [PATCH 10/23] rodeos --- programs/rodeos/rocksdb_plugin.cpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/programs/rodeos/rocksdb_plugin.cpp b/programs/rodeos/rocksdb_plugin.cpp index d80899f1e16..5f1a3e15ac1 100644 --- a/programs/rodeos/rocksdb_plugin.cpp +++ b/programs/rodeos/rocksdb_plugin.cpp @@ -1,5 +1,6 @@ #include "rocksdb_plugin.hpp" +#include #include namespace b1 { @@ -23,7 +24,8 @@ rocksdb_plugin::~rocksdb_plugin() {} void rocksdb_plugin::set_program_options(options_description& cli, options_description& cfg) { auto op = cfg.add_options(); - op("rdb-database", bpo::value()->default_value("./rodeos.rocksdb"), "Primary database path"); + op("rdb-database", bpo::value()->default_value("rodeos.rocksdb"), + "Database path (absolute path or relative to application data dir)"); op("rdb-threads", bpo::value(), "Increase number of background RocksDB threads. Only used with cloner_plugin. Recommend 8 for full history " "on large chains."); @@ -34,7 +36,11 @@ void rocksdb_plugin::set_program_options(options_description& cli, options_descr void rocksdb_plugin::plugin_initialize(const variables_map& options) { try { - my->db_path = options["rdb-database"].as(); + auto db_path = options.at("rdb-database").as(); + if (db_path.is_relative()) + my->db_path = app().data_dir() / db_path; + else + my->db_path = db_path; if (!options["rdb-threads"].empty()) my->threads = options["rdb-threads"].as(); if (!options["rdb-max-files"].empty()) @@ -49,8 +55,12 @@ void rocksdb_plugin::plugin_shutdown() {} std::shared_ptr rocksdb_plugin::get_db() { std::lock_guard lock(my->mutex); - if (!my->database) + if (!my->database) { + ilog("rodeos database is ${d}", ("d", my->db_path.string())); + if (!bfs::exists(my->db_path.parent_path())) + bfs::create_directories(my->db_path.parent_path()); my->database = std::make_shared(my->db_path.c_str(), true, my->threads, my->max_open_files); + } return my->database; } From 9c7c98523de13448fe6949a7b5c94acbffb53af5 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Mon, 27 Apr 2020 14:42:00 -0400 Subject: [PATCH 11/23] eosio-tester --- CMakeLists.txt | 1 + .../include/b1/rodeos/callbacks/memory.hpp | 8 +- .../include/b1/rodeos/callbacks/vm_types.hpp | 3 - libraries/rodeos/include/eosio/datastream.hpp | 2 + libraries/rodeos/include/eosio/key_value.hpp | 2 + programs/CMakeLists.txt | 1 + programs/eosio-tester/.clang-format | 76 + programs/eosio-tester/CMakeLists.txt | 13 + programs/eosio-tester/chain_types.hpp | 261 ++++ programs/eosio-tester/main.cpp | 1229 +++++++++++++++++ 10 files changed, 1589 insertions(+), 7 deletions(-) create mode 100644 programs/eosio-tester/.clang-format create mode 100644 programs/eosio-tester/CMakeLists.txt create mode 100644 programs/eosio-tester/chain_types.hpp create mode 100644 programs/eosio-tester/main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a96e205ceba..bbe3dcc0ab4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,6 +34,7 @@ set( CLI_CLIENT_EXECUTABLE_NAME cleos ) set( NODE_EXECUTABLE_NAME nodeos ) set( KEY_STORE_EXECUTABLE_NAME keosd ) set( RODEOS_EXECUTABLE_NAME rodeos ) +set( TESTER_EXECUTABLE_NAME eosio-tester ) # http://stackoverflow.com/a/18369825 if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/memory.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/memory.hpp index d8f336593b7..d44c4df86bc 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/memory.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/memory.hpp @@ -12,7 +12,7 @@ struct memory_callbacks { volatile auto check_src = *((const char*)src + length - 1); volatile auto check_dest = *((const char*)dest + length - 1); volatile auto check_result = *(const char*)dest; - ignore_unused_variable_warning(check_src, check_dest, check_result); + eosio::vm::ignore_unused_variable_warning(check_src, check_dest, check_result); if ((size_t)(std::abs((ptrdiff_t)(char*)dest - (ptrdiff_t)(const char*)src)) < length) throw std::runtime_error("memcpy can only accept non-aliasing pointers"); return (char*)std::memcpy((char*)dest, (const char*)src, length); @@ -22,14 +22,14 @@ struct memory_callbacks { volatile auto check_src = *((const char*)src + length - 1); volatile auto check_dest = *((const char*)dest + length - 1); volatile auto check_result = *(const char*)dest; - ignore_unused_variable_warning(check_src, check_dest, check_result); + eosio::vm::ignore_unused_variable_warning(check_src, check_dest, check_result); return (char*)std::memmove((char*)dest, (const char*)src, length); } int32_t memcmp_impl(unvalidated_ptr dest, unvalidated_ptr src, wasm_size_t length) const { volatile auto check_src = *((const char*)src + length - 1); volatile auto check_dest = *((const char*)dest + length - 1); - ignore_unused_variable_warning(check_src, check_dest); + eosio::vm::ignore_unused_variable_warning(check_src, check_dest); int32_t ret = std::memcmp((const char*)dest, (const char*)src, length); return ret < 0 ? -1 : ret > 0 ? 1 : 0; } @@ -37,7 +37,7 @@ struct memory_callbacks { void* memset_impl(unvalidated_ptr dest, int32_t value, wasm_size_t length) const { volatile auto check_dest = *((const char*)dest + length - 1); volatile auto check_result = *(const char*)dest; - ignore_unused_variable_warning(check_dest, check_result); + eosio::vm::ignore_unused_variable_warning(check_dest, check_result); return (char*)std::memset((char*)dest, value, length); } diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/vm_types.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/vm_types.hpp index 0278938d372..1afae82a6f0 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/vm_types.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/vm_types.hpp @@ -29,9 +29,6 @@ struct unvalidated_ptr { T* ptr; }; -template -void ignore_unused_variable_warning(T&...) {} - inline size_t legacy_copy_to_wasm(char* dest, size_t dest_size, const char* src, size_t src_size) { if (dest_size == 0) return src_size; diff --git a/libraries/rodeos/include/eosio/datastream.hpp b/libraries/rodeos/include/eosio/datastream.hpp index 3db72ba5cbc..88e814b9632 100644 --- a/libraries/rodeos/include/eosio/datastream.hpp +++ b/libraries/rodeos/include/eosio/datastream.hpp @@ -1,3 +1,5 @@ +// TODO: this file duplicates a CDT file + // clang-format off /** * @file datastream.hpp diff --git a/libraries/rodeos/include/eosio/key_value.hpp b/libraries/rodeos/include/eosio/key_value.hpp index ac669cb9bda..01f93665280 100644 --- a/libraries/rodeos/include/eosio/key_value.hpp +++ b/libraries/rodeos/include/eosio/key_value.hpp @@ -1,3 +1,5 @@ +// TODO: this file duplicates a CDT file + // clang-format off #pragma once #include diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index ca64a63795b..92acbee1f56 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -4,3 +4,4 @@ add_subdirectory( keosd ) add_subdirectory( eosio-launcher ) add_subdirectory( eosio-blocklog ) add_subdirectory( rodeos ) +add_subdirectory( eosio-tester ) diff --git a/programs/eosio-tester/.clang-format b/programs/eosio-tester/.clang-format new file mode 100644 index 00000000000..f067826dc2f --- /dev/null +++ b/programs/eosio-tester/.clang-format @@ -0,0 +1,76 @@ +BasedOnStyle: LLVM +IndentWidth: 3 +UseTab: Never +ColumnLimit: 120 + +--- +Language: Cpp +# always align * and & to the type +DerivePointerAlignment: false +PointerAlignment: Left + +# regroup includes to these classes +IncludeCategories: + - Regex: '(<|"(eosio)/)' + Priority: 4 + - Regex: '(<|"(boost)/)' + Priority: 3 + - Regex: '(<|"(llvm|llvm-c|clang|clang-c)/' + Priority: 3 + - Regex: '<[[:alnum:]]+>' + Priority: 2 + - Regex: '.*' + Priority: 1 + +#IncludeBlocks: Regroup + +# set indent for public, private and protected +#AccessModifierOffset: 3 + +# make line continuations twice the normal indent +ContinuationIndentWidth: 6 + +# add missing namespace comments +FixNamespaceComments: true + +# add spaces to braced list i.e. int* foo = { 0, 1, 2 }; instead of int* foo = {0,1,2}; +Cpp11BracedListStyle: false +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: true +AlignConsecutiveDeclarations: true +AlignOperands: true +AlignTrailingComments: true +AllowShortCaseLabelsOnASingleLine: true +AllowShortFunctionsOnASingleLine: All +AllowShortBlocksOnASingleLine: true +#AllowShortIfStatementsOnASingleLine: WithoutElse +#AllowShortIfStatementsOnASingleLine: true +#AllowShortLambdasOnASingleLine: All +AllowShortLoopsOnASingleLine: true +AlwaysBreakTemplateDeclarations: true + +BinPackParameters: true +### use this with clang9 +BreakBeforeBraces: Custom +BraceWrapping: + #AfterCaseLabel: true + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + +BreakConstructorInitializers: BeforeColon +CompactNamespaces: true +IndentCaseLabels: true +IndentPPDirectives: None +NamespaceIndentation: None +ReflowComments: false +SortIncludes: true +SortUsingDeclarations: true +--- diff --git a/programs/eosio-tester/CMakeLists.txt b/programs/eosio-tester/CMakeLists.txt new file mode 100644 index 00000000000..6775719f19f --- /dev/null +++ b/programs/eosio-tester/CMakeLists.txt @@ -0,0 +1,13 @@ +add_executable( ${TESTER_EXECUTABLE_NAME} main.cpp ) +if( UNIX AND NOT APPLE ) + set(rt_library rt ) +endif() + +target_link_libraries( ${TESTER_EXECUTABLE_NAME} + PRIVATE rodeos_lib state_history eosio_chain ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS} ) +target_include_directories(${TESTER_EXECUTABLE_NAME} PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) + +copy_bin( ${TESTER_EXECUTABLE_NAME} ) +install( TARGETS + ${TESTER_EXECUTABLE_NAME} RUNTIME DESTINATION ${CMAKE_INSTALL_FULL_BINDIR} COMPONENT base +) diff --git a/programs/eosio-tester/chain_types.hpp b/programs/eosio-tester/chain_types.hpp new file mode 100644 index 00000000000..f05785d3613 --- /dev/null +++ b/programs/eosio-tester/chain_types.hpp @@ -0,0 +1,261 @@ +// TODO: this is a duplicate of a file in CDT + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace chain_types { + +struct extension { + uint16_t type = {}; + eosio::input_stream data = {}; +}; + +EOSIO_REFLECT(extension, type, data); + +enum class transaction_status : uint8_t { + executed = 0, // succeed, no error handler executed + soft_fail = 1, // objectively failed (not executed), error handler executed + hard_fail = 2, // objectively failed and error handler objectively failed thus no state change + delayed = 3, // transaction delayed/deferred/scheduled for future execution + expired = 4, // transaction expired and storage space refunded to user +}; + +inline std::string to_string(transaction_status status) { + switch (status) { + case transaction_status::executed: return "executed"; + case transaction_status::soft_fail: return "soft_fail"; + case transaction_status::hard_fail: return "hard_fail"; + case transaction_status::delayed: return "delayed"; + case transaction_status::expired: return "expired"; + } + + eosio::check(false, "unknown status: " + std::to_string((uint8_t)status)); + return {}; // suppress warning +} + +inline transaction_status get_transaction_status(const std::string& s) { + if (s == "executed") + return transaction_status::executed; + if (s == "soft_fail") + return transaction_status::soft_fail; + if (s == "hard_fail") + return transaction_status::hard_fail; + if (s == "delayed") + return transaction_status::delayed; + if (s == "expired") + return transaction_status::expired; + + eosio::check(false, "unknown status: " + s); + return {}; // suppress warning +} + +struct permission_level { + eosio::name actor = {}; + eosio::name permission = {}; +}; + +EOSIO_REFLECT(permission_level, actor, permission); + +struct account_auth_sequence { + eosio::name account = {}; + uint64_t sequence = {}; +}; + +EOSIO_REFLECT(account_auth_sequence, account, sequence); +EOSIO_COMPARE(account_auth_sequence); + +struct account_delta { + eosio::name account = {}; + int64_t delta = {}; +}; + +EOSIO_REFLECT(account_delta, account, delta); +EOSIO_COMPARE(account_delta); + +struct action_receipt_v0 { + eosio::name receiver = {}; + eosio::checksum256 act_digest = {}; + uint64_t global_sequence = {}; + uint64_t recv_sequence = {}; + std::vector auth_sequence = {}; + eosio::varuint32 code_sequence = {}; + eosio::varuint32 abi_sequence = {}; +}; + +EOSIO_REFLECT(action_receipt_v0, receiver, act_digest, global_sequence, recv_sequence, auth_sequence, code_sequence, + abi_sequence); + +using action_receipt = std::variant; + +struct action { + eosio::name account = {}; + eosio::name name = {}; + std::vector authorization = {}; + eosio::input_stream data = {}; +}; + +EOSIO_REFLECT(action, account, name, authorization, data); + +struct action_trace_v0 { + eosio::varuint32 action_ordinal = {}; + eosio::varuint32 creator_action_ordinal = {}; + std::optional receipt = {}; + eosio::name receiver = {}; + action act = {}; + bool context_free = {}; + int64_t elapsed = {}; + std::string console = {}; + std::vector account_ram_deltas = {}; + std::optional except = {}; + std::optional error_code = {}; +}; + +EOSIO_REFLECT(action_trace_v0, action_ordinal, creator_action_ordinal, receipt, receiver, act, context_free, elapsed, + console, account_ram_deltas, except, error_code); + +struct action_trace_v1 { + eosio::varuint32 action_ordinal = {}; + eosio::varuint32 creator_action_ordinal = {}; + std::optional receipt = {}; + eosio::name receiver = {}; + action act = {}; + bool context_free = {}; + int64_t elapsed = {}; + std::string console = {}; + std::vector account_ram_deltas = {}; + std::vector account_disk_deltas = {}; + std::optional except = {}; + std::optional error_code = {}; + eosio::input_stream return_value = {}; +}; + +EOSIO_REFLECT(action_trace_v1, action_ordinal, creator_action_ordinal, receipt, receiver, act, context_free, elapsed, + console, account_ram_deltas, account_disk_deltas, except, error_code, return_value) + +using action_trace = std::variant; + +struct partial_transaction_v0 { + eosio::time_point_sec expiration = {}; + uint16_t ref_block_num = {}; + uint32_t ref_block_prefix = {}; + eosio::varuint32 max_net_usage_words = {}; + uint8_t max_cpu_usage_ms = {}; + eosio::varuint32 delay_sec = {}; + std::vector transaction_extensions = {}; + std::vector signatures = {}; + std::vector context_free_data = {}; +}; + +EOSIO_REFLECT(partial_transaction_v0, expiration, ref_block_num, ref_block_prefix, max_net_usage_words, + max_cpu_usage_ms, delay_sec, transaction_extensions, signatures, context_free_data); + +using partial_transaction = std::variant; + +struct transaction_trace_v0; +using transaction_trace = std::variant; + +struct transaction_trace_v0 { + eosio::checksum256 id = {}; + transaction_status status = {}; + uint32_t cpu_usage_us = {}; + eosio::varuint32 net_usage_words = {}; + int64_t elapsed = {}; + uint64_t net_usage = {}; + bool scheduled = {}; + std::vector action_traces = {}; + std::optional account_ram_delta = {}; + std::optional except = {}; + std::optional error_code = {}; + std::vector failed_dtrx_trace = {}; + std::optional reserved_do_not_use = {}; +}; + +EOSIO_REFLECT(transaction_trace_v0, id, status, cpu_usage_us, net_usage_words, elapsed, net_usage, scheduled, + action_traces, account_ram_delta, except, error_code, failed_dtrx_trace, reserved_do_not_use); + +struct producer_key { + eosio::name producer_name = {}; + eosio::public_key block_signing_key = {}; +}; + +EOSIO_REFLECT(producer_key, producer_name, block_signing_key); + +struct producer_schedule { + uint32_t version = {}; + std::vector producers = {}; +}; + +EOSIO_REFLECT(producer_schedule, version, producers); + +struct transaction_receipt_header { + transaction_status status = {}; + uint32_t cpu_usage_us = {}; + eosio::varuint32 net_usage_words = {}; +}; + +EOSIO_REFLECT(transaction_receipt_header, status, cpu_usage_us, net_usage_words); + +struct packed_transaction { + std::vector signatures = {}; + uint8_t compression = {}; + eosio::input_stream packed_context_free_data = {}; + eosio::input_stream packed_trx = {}; +}; + +EOSIO_REFLECT(packed_transaction, signatures, compression, packed_context_free_data, packed_trx); + +using transaction_variant = std::variant; + +struct transaction_receipt : transaction_receipt_header { + transaction_variant trx = {}; +}; + +EOSIO_REFLECT(transaction_receipt, base transaction_receipt_header, trx); + +struct block_header { + eosio::block_timestamp timestamp; + eosio::name producer = {}; + uint16_t confirmed = {}; + eosio::checksum256 previous = {}; + eosio::checksum256 transaction_mroot = {}; + eosio::checksum256 action_mroot = {}; + uint32_t schedule_version = {}; + std::optional new_producers = {}; + std::vector header_extensions = {}; +}; + +EOSIO_REFLECT(block_header, timestamp, producer, confirmed, previous, transaction_mroot, action_mroot, schedule_version, + new_producers, header_extensions) + +struct signed_block_header : block_header { + eosio::signature producer_signature = {}; +}; + +EOSIO_REFLECT(signed_block_header, base block_header, producer_signature); + +struct signed_block : signed_block_header { + std::vector transactions = {}; + std::vector block_extensions = {}; +}; + +EOSIO_REFLECT(signed_block, base signed_block_header, transactions, block_extensions); + +struct block_info { + uint32_t block_num = {}; + eosio::checksum256 block_id = {}; + eosio::block_timestamp timestamp; +}; + +EOSIO_REFLECT(block_info, block_num, block_id, timestamp); + +} // namespace chain_types diff --git a/programs/eosio-tester/main.cpp b/programs/eosio-tester/main.cpp new file mode 100644 index 00000000000..198da39f6ca --- /dev/null +++ b/programs/eosio-tester/main.cpp @@ -0,0 +1,1229 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#undef N + +#include "chain_types.hpp" +#include +#include +#include +#include +#include + +#include +#include + +using namespace eosio::literals; +using namespace std::literals; + +using std::optional; + +using boost::signals2::scoped_connection; +using eosio::check; +using eosio::convert_to_bin; +using eosio::chain::block_state_ptr; +using eosio::chain::builtin_protocol_feature_t; +using eosio::chain::digest_type; +using eosio::chain::kv_bad_db_id; +using eosio::chain::kv_bad_iter; +using eosio::chain::kv_context; +using eosio::chain::kvdisk_id; +using eosio::chain::kvram_id; +using eosio::chain::protocol_feature_exception; +using eosio::chain::protocol_feature_set; +using eosio::chain::signed_transaction; +using eosio::chain::transaction_trace_ptr; +using eosio::state_history::block_position; +using eosio::state_history::create_deltas; +using eosio::state_history::get_blocks_result_v0; +using eosio::state_history::state_result; +using eosio::state_history::trace_converter; +using eosio::vm::span; + +struct callbacks; +using rhf_t = eosio::vm::registered_host_functions; +using backend_t = eosio::vm::backend; + +inline constexpr int block_interval_ms = 500; +inline constexpr int block_interval_us = block_interval_ms * 1000; +inline constexpr uint32_t billed_cpu_time_use = 2000; + +// Handle eosio version differences +namespace { +template +auto to_uint64_t(T n) -> std::enable_if_t, decltype(n.value)> { + return n.value; +} +template +auto to_uint64_t(T n) -> std::enable_if_t, decltype(n.to_uint64_t())> { + return n.to_uint64_t(); +} + +template +auto do_startup(C&& control, F0&&, F1&& f1, G &&) + -> std::enable_if_t, eosio::chain::controller::config, + protocol_feature_set>> { + return control->startup([]() { return false; }, nullptr); +} +template +auto do_startup(C&& control, F0&&, F1&& f1, G&& genesis) -> decltype(control->startup(f1, genesis)) { + return control->startup([]() { return false; }, genesis); +} +template +auto do_startup(C&& control, F0&& f0, F1&& f1, G&& genesis) -> decltype(control->startup(f0, f1, genesis)) { + return control->startup(f0, f1, genesis); +} +} // namespace + +struct assert_exception : std::exception { + std::string msg; + + assert_exception(std::string&& msg) : msg(std::move(msg)) {} + + const char* what() const noexcept override { return msg.c_str(); } +}; + +// HACK: UB. Unfortunately, I can't think of a way to allow a transaction_context +// to be constructed outside of controller in 2.0 that doesn't have undefined behavior. +// A better solution would be to factor database access out of apply_context, but +// that can't really be backported to 2.0 at this point. +namespace { +struct __attribute__((__may_alias__)) xxx_transaction_checktime_timer { + std::atomic_bool& expired; + eosio::chain::platform_timer& _timer; +}; +struct transaction_checktime_factory { + eosio::chain::platform_timer timer; + std::atomic_bool expired; + eosio::chain::transaction_checktime_timer get() { + xxx_transaction_checktime_timer result{ expired, timer }; + return std::move(*reinterpret_cast(&result)); + } +}; +}; // namespace + +struct intrinsic_context { + eosio::chain::controller& control; + eosio::chain::signed_transaction trx; + std::unique_ptr trx_ctx; + std::unique_ptr apply_context; + + intrinsic_context(eosio::chain::controller& control) : control{ control } { + static transaction_checktime_factory xxx_timer; + + trx.actions.emplace_back(); + trx.actions.back().account = eosio::chain::name{ "eosio.null" }; + trx.actions.back().authorization.push_back({ eosio::chain::name{ "eosio" }, eosio::chain::name{ "active" } }); + trx_ctx = std::make_unique(control, trx, trx.id(), xxx_timer.get(), + fc::time_point::now()); + trx_ctx->init_for_implicit_trx(0); + trx_ctx->exec(); + apply_context = std::make_unique(control, *trx_ctx, 1, 0); + apply_context->exec_one(); + } +}; + +protocol_feature_set make_protocol_feature_set() { + protocol_feature_set pfs; + std::map> visited_builtins; + + std::function add_builtins = + [&pfs, &visited_builtins, &add_builtins](builtin_protocol_feature_t codename) -> digest_type { + auto res = visited_builtins.emplace(codename, optional()); + if (!res.second) { + EOS_ASSERT(res.first->second, protocol_feature_exception, + "invariant failure: cycle found in builtin protocol feature dependencies"); + return *res.first->second; + } + + auto f = protocol_feature_set::make_default_builtin_protocol_feature( + codename, [&add_builtins](builtin_protocol_feature_t d) { return add_builtins(d); }); + + const auto& pf = pfs.add_feature(f); + res.first->second = pf.feature_digest; + + return pf.feature_digest; + }; + + for (const auto& p : eosio::chain::builtin_protocol_feature_codenames) { add_builtins(p.first); } + + return pfs; +} + +template +using wasm_ptr = eosio::vm::argument_proxy; + +struct test_chain; + +struct test_chain_ref { + test_chain* chain = {}; + + test_chain_ref() = default; + test_chain_ref(test_chain&); + test_chain_ref(const test_chain_ref&); + ~test_chain_ref(); + + test_chain_ref& operator=(const test_chain_ref&); +}; + +struct test_chain { + eosio::chain::private_key_type producer_key{ "5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3"s }; + + fc::temp_directory dir; + std::unique_ptr cfg; + std::unique_ptr control; + fc::optional applied_transaction_connection; + fc::optional accepted_block_connection; + trace_converter trace_converter; + fc::optional prev_block; + std::map> history; + std::unique_ptr intr_ctx; + std::set refs; + + test_chain(const char* snapshot) { + eosio::chain::genesis_state genesis; + genesis.initial_timestamp = fc::time_point::from_iso_string("2020-01-01T00:00:00.000"); + cfg = std::make_unique(); + cfg->blocks_dir = dir.path() / "blocks"; + cfg->state_dir = dir.path() / "state"; + cfg->contracts_console = true; + cfg->wasm_runtime = eosio::chain::wasm_interface::vm_type::eos_vm_jit; + + std::optional snapshot_file; + std::shared_ptr snapshot_reader; + if (snapshot && *snapshot) { + std::optional chain_id; + { + std::ifstream temp_file(snapshot, (std::ios::in | std::ios::binary)); + if (!temp_file.is_open()) + throw std::runtime_error("can not open " + std::string{ snapshot }); + eosio::chain::istream_snapshot_reader tmp_reader(temp_file); + tmp_reader.validate(); + chain_id = eosio::chain::controller::extract_chain_id(tmp_reader); + } + snapshot_file.emplace(snapshot, std::ios::in | std::ios::binary); + snapshot_reader = std::make_shared(*snapshot_file); + control = std::make_unique(*cfg, make_protocol_feature_set(), *chain_id); + } else { + control = std::make_unique(*cfg, make_protocol_feature_set(), + genesis.compute_chain_id()); + } + + control->add_indices(); + + applied_transaction_connection.emplace(control->applied_transaction.connect( + [&](std::tuple t) { + on_applied_transaction(std::get<0>(t), std::get<1>(t)); + })); + accepted_block_connection.emplace( + control->accepted_block.connect([&](const block_state_ptr& p) { on_accepted_block(p); })); + + if (snapshot_reader) { + control->startup([] {}, [] { return false; }, snapshot_reader); + } else { + do_startup( + control, [] {}, [] { return false; }, genesis); + control->start_block(control->head_block_time() + fc::microseconds(block_interval_us), 0, + { *control->get_protocol_feature_manager().get_builtin_digest( + eosio::chain::builtin_protocol_feature_t::preactivate_feature) }); + } + } + + test_chain(const test_chain&) = delete; + test_chain& operator=(const test_chain&) = delete; + + ~test_chain() { + for (auto* ref : refs) ref->chain = nullptr; + } + + void on_applied_transaction(const transaction_trace_ptr& p, const signed_transaction& t) { + trace_converter.add_transaction(p, t); + } + + void on_accepted_block(const block_state_ptr& block_state) { + auto block_bin = fc::raw::pack(*block_state->block); + auto traces_bin = trace_converter.pack(control->db(), false, block_state); + auto deltas_bin = fc::raw::pack(create_deltas(control->db(), !prev_block)); + + get_blocks_result_v0 message; + message.head = block_position{ control->head_block_num(), control->head_block_id() }; + message.last_irreversible = + block_position{ control->last_irreversible_block_num(), control->last_irreversible_block_id() }; + message.this_block = block_position{ block_state->block->block_num(), block_state->block->id() }; + message.prev_block = prev_block; + message.block = std::move(block_bin); + message.traces = std::move(traces_bin); + message.deltas = std::move(deltas_bin); + + prev_block = message.this_block; + history[control->head_block_num()] = fc::raw::pack(state_result{ message }); + } + + void mutating() { intr_ctx.reset(); } + + auto& get_apply_context() { + if (!intr_ctx) { + start_if_needed(); + intr_ctx = std::make_unique(*control); + } + return *intr_ctx->apply_context; + } + + void start_block(int64_t skip_miliseconds = 0) { + mutating(); + if (control->is_building_block()) + finish_block(); + control->start_block(control->head_block_time() + fc::microseconds(skip_miliseconds * 1000ll + block_interval_us), + 0); + } + + void start_if_needed() { + mutating(); + if (!control->is_building_block()) + control->start_block(control->head_block_time() + fc::microseconds(block_interval_us), 0, {}); + } + + void finish_block() { + start_if_needed(); + ilog("finish block ${n}", ("n", control->head_block_num())); + control->finalize_block([&](eosio::chain::digest_type d) { return std::vector{ producer_key.sign(d) }; }); + control->commit_block(); + } +}; + +test_chain_ref::test_chain_ref(test_chain& chain) { + chain.refs.insert(this); + this->chain = &chain; +} + +test_chain_ref::test_chain_ref(const test_chain_ref& src) { + chain = src.chain; + if (chain) + chain->refs.insert(this); +} + +test_chain_ref::~test_chain_ref() { + if (chain) + chain->refs.erase(this); +} + +test_chain_ref& test_chain_ref::operator=(const test_chain_ref& src) { + if (chain) + chain->refs.erase(this); + chain = nullptr; + if (src.chain) + src.chain->refs.insert(this); + chain = src.chain; + return *this; +} + +struct test_rodeos { + fc::temp_directory dir; + b1::embedded_rodeos::context context; + std::optional partition; + std::optional write_snapshot; + std::list filters; + std::optional query_handler; + test_chain_ref chain; + uint32_t next_block = 0; + std::vector> pushed_data; + + test_rodeos() { + context.open_db(dir.path().string().c_str(), true); + partition.emplace(context, "", 0); + write_snapshot.emplace(partition->obj, true); + } +}; + +eosio::checksum256 convert(const eosio::chain::checksum_type& obj) { + std::array bytes; + static_assert(bytes.size() == sizeof(obj)); + memcpy(bytes.data(), &obj, bytes.size()); + return eosio::checksum256(bytes); +} + +chain_types::account_delta convert(const eosio::chain::account_delta& obj) { + chain_types::account_delta result; + result.account.value = to_uint64_t(obj.account); + result.delta = obj.delta; + return result; +} + +chain_types::action_receipt_v0 convert(const eosio::chain::action_receipt& obj) { + chain_types::action_receipt_v0 result; + result.receiver.value = to_uint64_t(obj.receiver); + result.act_digest = convert(obj.act_digest); + result.global_sequence = obj.global_sequence; + result.recv_sequence = obj.recv_sequence; + for (auto& auth : obj.auth_sequence) + result.auth_sequence.push_back({ eosio::name{ to_uint64_t(auth.first) }, auth.second }); + result.code_sequence.value = obj.code_sequence.value; + result.abi_sequence.value = obj.abi_sequence.value; + return result; +} + +chain_types::action convert(const eosio::chain::action& obj) { + chain_types::action result; + result.account.value = to_uint64_t(obj.account); + result.name.value = to_uint64_t(obj.name); + for (auto& auth : obj.authorization) + result.authorization.push_back( + { eosio::name{ to_uint64_t(auth.actor) }, eosio::name{ to_uint64_t(auth.permission) } }); + result.data = { obj.data.data(), obj.data.data() + obj.data.size() }; + return result; +} + +chain_types::action_trace_v1 convert(const eosio::chain::action_trace& obj) { + chain_types::action_trace_v1 result; + result.action_ordinal.value = obj.action_ordinal.value; + result.creator_action_ordinal.value = obj.creator_action_ordinal.value; + if (obj.receipt) + result.receipt = convert(*obj.receipt); + result.receiver.value = to_uint64_t(obj.receiver); + result.act = convert(obj.act); + result.context_free = obj.context_free; + result.elapsed = obj.elapsed.count(); + result.console = obj.console; + for (auto& delta : obj.account_ram_deltas) result.account_ram_deltas.push_back(convert(delta)); + for (auto& delta : obj.account_disk_deltas) result.account_disk_deltas.push_back(convert(delta)); + if (obj.except) + result.except = obj.except->to_string(); + if (obj.error_code) + result.error_code = *obj.error_code; + result.return_value = { obj.return_value.data(), obj.return_value.size() }; + return result; +} + +chain_types::transaction_trace_v0 convert(const eosio::chain::transaction_trace& obj) { + chain_types::transaction_trace_v0 result{}; + result.id = convert(obj.id); + if (obj.receipt) { + result.status = (chain_types::transaction_status)obj.receipt->status.value; + result.cpu_usage_us = obj.receipt->cpu_usage_us; + result.net_usage_words = obj.receipt->net_usage_words.value; + } else { + result.status = chain_types::transaction_status::hard_fail; + } + result.elapsed = obj.elapsed.count(); + result.net_usage = obj.net_usage; + result.scheduled = obj.scheduled; + for (auto& at : obj.action_traces) result.action_traces.push_back(convert(at)); + if (obj.account_ram_delta) + result.account_ram_delta = convert(*obj.account_ram_delta); + if (obj.except) + result.except = obj.except->to_string(); + if (obj.error_code) + result.error_code = *obj.error_code; + if (obj.failed_dtrx_trace) + result.failed_dtrx_trace.push_back({ convert(*obj.failed_dtrx_trace) }); + return result; +} + +struct contract_row { + uint32_t block_num = {}; + bool present = {}; + eosio::name code = {}; + eosio::name scope = {}; + eosio::name table = {}; + uint64_t primary_key = {}; + eosio::name payer = {}; + eosio::input_stream value = {}; +}; + +EOSIO_REFLECT(contract_row, block_num, present, code, scope, table, primary_key, payer, value); + +struct file { + FILE* f = nullptr; + bool owns = false; + + file(FILE* f = nullptr, bool owns = true) : f(f), owns(owns) {} + + file(const file&) = delete; + file(file&& src) { *this = std::move(src); } + + ~file() { close(); } + + file& operator=(const file&) = delete; + + file& operator=(file&& src) { + close(); + this->f = src.f; + this->owns = src.owns; + src.f = nullptr; + src.owns = false; + return *this; + } + + void close() { + if (owns && f) + fclose(f); + f = nullptr; + owns = false; + } +}; + +struct state { + const char* wasm; + eosio::vm::wasm_allocator& wa; + backend_t& backend; + std::vector args; + std::vector files; + std::vector> chains; + std::vector> rodeoses; + std::optional selected_chain_index; +}; + +struct push_trx_args { + eosio::chain::bytes transaction; + std::vector context_free_data; + std::vector signatures; + std::vector keys; +}; +FC_REFLECT(push_trx_args, (transaction)(context_free_data)(signatures)(keys)) + +#define DB_WRAPPERS_SIMPLE_SECONDARY(IDX, TYPE) \ + int32_t db_##IDX##_find_secondary(uint64_t code, uint64_t scope, uint64_t table, wasm_ptr secondary, \ + wasm_ptr primary) { \ + return selected().IDX.find_secondary(code, scope, table, *secondary, *primary); \ + } \ + int32_t db_##IDX##_find_primary(uint64_t code, uint64_t scope, uint64_t table, wasm_ptr secondary, \ + uint64_t primary) { \ + return selected().IDX.find_primary(code, scope, table, *secondary, primary); \ + } \ + int32_t db_##IDX##_lowerbound(uint64_t code, uint64_t scope, uint64_t table, wasm_ptr secondary, \ + wasm_ptr primary) { \ + return selected().IDX.lowerbound_secondary(code, scope, table, *secondary, *primary); \ + } \ + int32_t db_##IDX##_upperbound(uint64_t code, uint64_t scope, uint64_t table, wasm_ptr secondary, \ + wasm_ptr primary) { \ + return selected().IDX.upperbound_secondary(code, scope, table, *secondary, *primary); \ + } \ + int32_t db_##IDX##_end(uint64_t code, uint64_t scope, uint64_t table) { \ + return selected().IDX.end_secondary(code, scope, table); \ + } \ + int32_t db_##IDX##_next(int32_t iterator, wasm_ptr primary) { \ + return selected().IDX.next_secondary(iterator, *primary); \ + } \ + int32_t db_##IDX##_previous(int32_t iterator, wasm_ptr primary) { \ + return selected().IDX.previous_secondary(iterator, *primary); \ + } + +#define DB_WRAPPERS_ARRAY_SECONDARY(IDX, ARR_SIZE, ARR_ELEMENT_TYPE) \ + int db_##IDX##_find_secondary(uint64_t code, uint64_t scope, uint64_t table, \ + eosio::chain::array_ptr data, uint32_t data_len, \ + uint64_t& primary) { \ + EOS_ASSERT(data_len == ARR_SIZE, eosio::chain::db_api_exception, \ + "invalid size of secondary key array for " #IDX \ + ": given ${given} bytes but expected ${expected} bytes", \ + ("given", data_len)("expected", ARR_SIZE)); \ + return selected().IDX.find_secondary(code, scope, table, data, primary); \ + } \ + int db_##IDX##_find_primary(uint64_t code, uint64_t scope, uint64_t table, \ + eosio::chain::array_ptr data, uint32_t data_len, uint64_t primary) { \ + EOS_ASSERT(data_len == ARR_SIZE, eosio::chain::db_api_exception, \ + "invalid size of secondary key array for " #IDX \ + ": given ${given} bytes but expected ${expected} bytes", \ + ("given", data_len)("expected", ARR_SIZE)); \ + return selected().IDX.find_primary(code, scope, table, data.value, primary); \ + } \ + int db_##IDX##_lowerbound(uint64_t code, uint64_t scope, uint64_t table, \ + eosio::chain::array_ptr data, uint32_t data_len, uint64_t& primary) { \ + EOS_ASSERT(data_len == ARR_SIZE, eosio::chain::db_api_exception, \ + "invalid size of secondary key array for " #IDX \ + ": given ${given} bytes but expected ${expected} bytes", \ + ("given", data_len)("expected", ARR_SIZE)); \ + return selected().IDX.lowerbound_secondary(code, scope, table, data.value, primary); \ + } \ + int db_##IDX##_upperbound(uint64_t code, uint64_t scope, uint64_t table, \ + eosio::chain::array_ptr data, uint32_t data_len, uint64_t& primary) { \ + EOS_ASSERT(data_len == ARR_SIZE, eosio::chain::db_api_exception, \ + "invalid size of secondary key array for " #IDX \ + ": given ${given} bytes but expected ${expected} bytes", \ + ("given", data_len)("expected", ARR_SIZE)); \ + return selected().IDX.upperbound_secondary(code, scope, table, data.value, primary); \ + } \ + int db_##IDX##_end(uint64_t code, uint64_t scope, uint64_t table) { \ + return selected().IDX.end_secondary(code, scope, table); \ + } \ + int db_##IDX##_next(int iterator, uint64_t& primary) { return selected().IDX.next_secondary(iterator, primary); } \ + int db_##IDX##_previous(int iterator, uint64_t& primary) { \ + return selected().IDX.previous_secondary(iterator, primary); \ + } + +#define DB_WRAPPERS_FLOAT_SECONDARY(IDX, TYPE) \ + int db_##IDX##_find_secondary(uint64_t code, uint64_t scope, uint64_t table, const TYPE& secondary, \ + uint64_t& primary) { \ + /* EOS_ASSERT(!softfloat_api::is_nan(secondary), transaction_exception, "NaN is not an allowed value for a \ + * secondary key"); */ \ + return selected().IDX.find_secondary(code, scope, table, secondary, primary); \ + } \ + int db_##IDX##_find_primary(uint64_t code, uint64_t scope, uint64_t table, TYPE& secondary, uint64_t primary) { \ + return selected().IDX.find_primary(code, scope, table, secondary, primary); \ + } \ + int db_##IDX##_lowerbound(uint64_t code, uint64_t scope, uint64_t table, TYPE& secondary, uint64_t& primary) { \ + /* EOS_ASSERT(!softfloat_api::is_nan(secondary), transaction_exception, "NaN is not an allowed value for a \ + * secondary key"); */ \ + return selected().IDX.lowerbound_secondary(code, scope, table, secondary, primary); \ + } \ + int db_##IDX##_upperbound(uint64_t code, uint64_t scope, uint64_t table, TYPE& secondary, uint64_t& primary) { \ + /* EOS_ASSERT(!softfloat_api::is_nan(secondary), transaction_exception, "NaN is not an allowed value for a \ + * secondary key"); */ \ + return selected().IDX.upperbound_secondary(code, scope, table, secondary, primary); \ + } \ + int db_##IDX##_end(uint64_t code, uint64_t scope, uint64_t table) { \ + return selected().IDX.end_secondary(code, scope, table); \ + } \ + int db_##IDX##_next(int iterator, uint64_t& primary) { return selected().IDX.next_secondary(iterator, primary); } \ + int db_##IDX##_previous(int iterator, uint64_t& primary) { \ + return selected().IDX.previous_secondary(iterator, primary); \ + } + +struct callbacks { + ::state& state; + + void check_bounds(void* data, size_t size) { + volatile auto check = *((const char*)data + size - 1); + eosio::vm::ignore_unused_variable_warning(check); + } + + template + T unpack(const char* begin, size_t size) { + fc::datastream ds(begin, size); + T args; + fc::raw::unpack(ds, args); + return args; + } + + template + T unpack(span data) { + return unpack(data.begin(), data.size()); + } + + template + T unpack(const eosio::input_stream& data) { + return unpack(data.pos, data.end - data.pos); + } + + std::string span_str(span str) { return { str.data(), str.size() }; } + + char* alloc(uint32_t cb_alloc_data, uint32_t cb_alloc, uint32_t size) { + // todo: verify cb_alloc isn't in imports + if (state.backend.get_module().tables.size() < 0 || state.backend.get_module().tables[0].table.size() < cb_alloc) + throw std::runtime_error("cb_alloc is out of range"); + auto result = state.backend.get_context().execute( // + this, eosio::vm::jit_visitor(42), state.backend.get_module().tables[0].table[cb_alloc], cb_alloc_data, + size); + if (!result || !result->is_a()) + throw std::runtime_error("cb_alloc returned incorrect type"); + char* begin = state.wa.get_base_ptr() + result->to_ui32(); + check_bounds(begin, size); + return begin; + } + + template + void set_data(uint32_t cb_alloc_data, uint32_t cb_alloc, const T& data) { + memcpy(alloc(cb_alloc_data, cb_alloc, data.size()), data.data(), data.size()); + } + + void set_data(uint32_t cb_alloc_data, uint32_t cb_alloc, const b1::embedded_rodeos::result& data) { + memcpy(alloc(cb_alloc_data, cb_alloc, data.size), data.data, data.size); + } + + void abort() { throw std::runtime_error("called abort"); } + + void eosio_assert_message(bool condition, span msg) { + if (!condition) + throw ::assert_exception(span_str(msg)); + } + + void prints_l(span str) { std::cerr.write(str.data(), str.size()); } + + uint32_t get_args(span dest) { + memcpy(dest.data(), state.args.data(), std::min(dest.size(), state.args.size())); + return state.args.size(); + } + + int32_t clock_gettime(int32_t id, void* data) { + check_bounds((char*)data, 8); + std::chrono::nanoseconds result; + if (id == 0) { // CLOCK_REALTIME + result = std::chrono::system_clock::now().time_since_epoch(); + } else if (id == 1) { // CLOCK_MONOTONIC + result = std::chrono::steady_clock::now().time_since_epoch(); + } else { + return -1; + } + int32_t sec = result.count() / 1000000000; + int32_t nsec = result.count() % 1000000000; + fc::datastream ds((char*)data, 8); + fc::raw::pack(ds, sec); + fc::raw::pack(ds, nsec); + return 0; + } + + int32_t open_file(span filename, span mode) { + file f = fopen(span_str(filename).c_str(), span_str(mode).c_str()); + if (!f.f) + return -1; + state.files.push_back(std::move(f)); + return state.files.size() - 1; + } + + file& assert_file(int32_t file_index) { + if (file_index < 0 || file_index >= state.files.size() || !state.files[file_index].f) + throw std::runtime_error("file is not opened"); + return state.files[file_index]; + } + + bool isatty(int32_t file_index) { return !assert_file(file_index).owns; } + + void close_file(int32_t file_index) { assert_file(file_index).close(); } + + bool write_file(int32_t file_index, span content) { + auto& f = assert_file(file_index); + return fwrite(content.data(), content.size(), 1, f.f) == 1; + } + + int32_t read_file(int32_t file_index, span content) { + auto& f = assert_file(file_index); + return fread(content.data(), 1, content.size(), f.f); + } + + bool read_whole_file(span filename, uint32_t cb_alloc_data, uint32_t cb_alloc) { + file f = fopen(span_str(filename).c_str(), "r"); + if (!f.f) + return false; + if (fseek(f.f, 0, SEEK_END)) + return false; + auto size = ftell(f.f); + if (size < 0 || (long)(uint32_t)size != size) + return false; + if (fseek(f.f, 0, SEEK_SET)) + return false; + std::vector buf(size); + if (fread(buf.data(), size, 1, f.f) != 1) + return false; + set_data(cb_alloc_data, cb_alloc, buf); + return true; + } + + int32_t execute(span command) { return system(span_str(command).c_str()); } + + test_chain& assert_chain(uint32_t chain, bool require_control = true) { + if (chain >= state.chains.size() || !state.chains[chain]) + throw std::runtime_error("chain does not exist or was destroyed"); + auto& result = *state.chains[chain]; + if (require_control && !result.control) + throw std::runtime_error("chain was shut down"); + return result; + } + + uint32_t create_chain(span snapshot) { + state.chains.push_back(std::make_unique(span_str(snapshot).c_str())); + if (state.chains.size() == 1) + state.selected_chain_index = 0; + return state.chains.size() - 1; + } + + void destroy_chain(uint32_t chain) { + assert_chain(chain, false); + if (state.selected_chain_index && *state.selected_chain_index == chain) + state.selected_chain_index.reset(); + state.chains[chain].reset(); + while (!state.chains.empty() && !state.chains.back()) { state.chains.pop_back(); } + } + + void shutdown_chain(uint32_t chain) { + auto& c = assert_chain(chain); + c.control.reset(); + } + + uint32_t get_chain_path(uint32_t chain, span dest) { + auto& c = assert_chain(chain, false); + auto s = c.dir.path().string(); + memcpy(dest.data(), s.c_str(), std::min(dest.size(), s.size())); + return s.size(); + } + + void replace_producer_keys(uint32_t chain_index, span key) { + auto& chain = assert_chain(chain_index); + auto k = unpack(key); + chain.control->replace_producer_keys(k); + } + + void replace_account_keys(uint32_t chain_index, uint64_t account, uint64_t permission, span key) { + auto& chain = assert_chain(chain_index); + auto k = unpack(key); + chain.control->replace_account_keys(eosio::chain::name{ account }, eosio::chain::name{ permission }, k); + } + + void start_block(uint32_t chain_index, int64_t skip_miliseconds) { + assert_chain(chain_index).start_block(skip_miliseconds); + } + + void finish_block(uint32_t chain_index) { assert_chain(chain_index).finish_block(); } + + void get_head_block_info(uint32_t chain_index, uint32_t cb_alloc_data, uint32_t cb_alloc) { + auto& chain = assert_chain(chain_index); + chain_types::block_info info; + info.block_num = chain.control->head_block_num(); + info.block_id = convert(chain.control->head_block_id()); + info.timestamp.slot = chain.control->head_block_state()->header.timestamp.slot; + set_data(cb_alloc_data, cb_alloc, check(convert_to_bin(info)).value()); + } + + void push_transaction(uint32_t chain_index, span args_packed, uint32_t cb_alloc_data, + uint32_t cb_alloc) { + auto args = unpack(args_packed); + auto transaction = unpack(args.transaction); + eosio::chain::signed_transaction signed_trx{ std::move(transaction), std::move(args.signatures), + std::move(args.context_free_data) }; + auto& chain = assert_chain(chain_index); + chain.start_if_needed(); + for (auto& key : args.keys) signed_trx.sign(key, chain.control->get_chain_id()); + auto ptrx = std::make_shared( + signed_trx, eosio::chain::packed_transaction::compression_type::none); + auto fut = eosio::chain::transaction_metadata::start_recover_keys( + ptrx, chain.control->get_thread_pool(), chain.control->get_chain_id(), fc::microseconds::maximum()); + auto start_time = std::chrono::steady_clock::now(); + auto result = chain.control->push_transaction(fut.get(), fc::time_point::maximum(), 2000, true); + auto us = std::chrono::duration_cast(std::chrono::steady_clock::now() - start_time); + ilog("chainlib transaction took ${u} us", ("u", us.count())); + // ilog("${r}", ("r", fc::json::to_pretty_string(result))); + set_data(cb_alloc_data, cb_alloc, + check(convert_to_bin(chain_types::transaction_trace{ convert(*result) })).value()); + } + + bool exec_deferred(uint32_t chain_index, uint32_t cb_alloc_data, uint32_t cb_alloc) { + auto& chain = assert_chain(chain_index); + chain.start_if_needed(); + const auto& idx = + chain.control->db().get_index(); + auto itr = idx.begin(); + if (itr != idx.end() && itr->delay_until <= chain.control->pending_block_time()) { + auto trace = chain.control->push_scheduled_transaction(itr->trx_id, fc::time_point::maximum(), + billed_cpu_time_use, true); + set_data(cb_alloc_data, cb_alloc, + check(convert_to_bin(chain_types::transaction_trace{ convert(*trace) })).value()); + return true; + } + return false; + } + + uint32_t get_history(uint32_t chain_index, uint32_t block_num, span dest) { + auto& chain = assert_chain(chain_index); + std::map>::iterator it; + if (block_num == 0xffff'ffff && !chain.history.empty()) + it = --chain.history.end(); + else { + it = chain.history.find(block_num); + if (it == chain.history.end()) + return 0; + } + memcpy(dest.data(), it->second.data(), std::min(dest.size(), it->second.size())); + return it->second.size(); + } + + void select_chain_for_db(uint32_t chain_index) { + assert_chain(chain_index); + state.selected_chain_index = chain_index; + } + + auto& selected() { + if (!state.selected_chain_index || *state.selected_chain_index >= state.chains.size() || + !state.chains[*state.selected_chain_index] || !state.chains[*state.selected_chain_index]->control) + throw std::runtime_error("select_chain_for_db() must be called before using multi_index"); + return state.chains[*state.selected_chain_index]->get_apply_context(); + } + + test_rodeos& assert_rodeos(uint32_t rodeos) { + if (rodeos >= state.rodeoses.size() || !state.rodeoses[rodeos]) + throw std::runtime_error("rodeos does not exist or was destroyed"); + return *state.rodeoses[rodeos]; + } + + uint32_t create_rodeos() { + state.rodeoses.push_back(std::make_unique()); + return state.rodeoses.size() - 1; + } + + void destroy_rodeos(uint32_t rodeos) { + assert_rodeos(rodeos); + state.rodeoses[rodeos].reset(); + while (!state.rodeoses.empty() && !state.rodeoses.back()) { state.rodeoses.pop_back(); } + } + + void rodeos_add_filter(uint32_t rodeos, uint64_t name, span wasm_filename) { + auto& r = assert_rodeos(rodeos); + r.filters.emplace_back(name, span_str(wasm_filename).c_str()); + } + + void rodeos_enable_queries(uint32_t rodeos, uint32_t max_console_size, uint32_t wasm_cache_size, + uint64_t max_exec_time_ms, span contract_dir) { + auto& r = assert_rodeos(rodeos); + r.query_handler.emplace(*r.partition, max_console_size, wasm_cache_size, max_exec_time_ms, + span_str(contract_dir).c_str()); + } + + void connect_rodeos(uint32_t rodeos, uint32_t chain) { + auto& r = assert_rodeos(rodeos); + auto& c = assert_chain(chain); + if (r.chain.chain) + throw std::runtime_error("rodeos is already connected"); + r.chain = test_chain_ref{ c }; + } + + bool rodeos_sync_block(uint32_t rodeos) { + auto& r = assert_rodeos(rodeos); + if (!r.chain.chain) + throw std::runtime_error("rodeos is not connected to a chain"); + auto it = r.chain.chain->history.lower_bound(r.next_block); + if (it == r.chain.chain->history.end()) + return false; + r.write_snapshot->start_block(it->second.data(), it->second.size()); + r.write_snapshot->write_block_info(it->second.data(), it->second.size()); + r.write_snapshot->write_deltas(it->second.data(), it->second.size(), [] { return false; }); + for (auto& filter : r.filters) { + try { + filter.run(*r.write_snapshot, it->second.data(), it->second.size(), [&](const char* data, uint64_t size) { + r.pushed_data.emplace_back(data, data + size); + return true; + }); + } catch (std::exception& e) { throw std::runtime_error("filter failed: " + std::string{ e.what() }); } + } + r.write_snapshot->end_block(it->second.data(), it->second.size(), true); + r.next_block = it->first + 1; + return true; + } + + void rodeos_push_transaction(uint32_t rodeos, span packed_args, uint32_t cb_alloc_data, + uint32_t cb_alloc) { + auto& r = assert_rodeos(rodeos); + if (!r.chain.chain) + throw std::runtime_error("rodeos is not connected to a chain"); + if (!r.query_handler) + throw std::runtime_error("call rodeos_enable_queries before rodeos_push_transaction"); + auto& chain = *r.chain.chain; + chain.start_if_needed(); + + auto args = unpack(packed_args); + auto transaction = unpack(args.transaction); + eosio::chain::signed_transaction signed_trx{ std::move(transaction), std::move(args.signatures), + std::move(args.context_free_data) }; + for (auto& key : args.keys) signed_trx.sign(key, chain.control->get_chain_id()); + eosio::chain::packed_transaction ptrx{ signed_trx, eosio::chain::packed_transaction::compression_type::none }; + auto data = fc::raw::pack(ptrx); + auto start_time = std::chrono::steady_clock::now(); + auto result = r.query_handler->query_transaction(*r.write_snapshot, data.data(), data.size()); + auto us = std::chrono::duration_cast(std::chrono::steady_clock::now() - start_time); + ilog("rodeos transaction took ${u} us", ("u", us.count())); + auto tt = eosio::check(eosio::convert_from_bin( + { result.data, result.data + result.size })) + .value(); + auto& tt0 = std::get(tt); + for (auto& at : tt0.action_traces) { + auto& at1 = std::get(at); + if (!at1.console.empty()) + ilog("rodeos query console: <<<\n${c}>>>", ("c", at1.console)); + } + set_data(cb_alloc_data, cb_alloc, result); + } + + uint32_t rodeos_get_num_pushed_data(uint32_t rodeos) { + auto& r = assert_rodeos(rodeos); + return r.pushed_data.size(); + } + + uint32_t rodeos_get_pushed_data(uint32_t rodeos, uint32_t index, span dest) { + auto& r = assert_rodeos(rodeos); + if (index >= r.pushed_data.size()) + throw std::runtime_error("rodeos_get_pushed_data: index is out of range"); + memcpy(dest.data(), r.pushed_data[index].data(), std::min(dest.size(), r.pushed_data[index].size())); + return r.pushed_data[index].size(); + } + + // clang-format off + int32_t db_get_i64(int32_t iterator, span buffer) {return selected().db_get_i64(iterator, buffer.data(), buffer.size());} + int32_t db_next_i64(int32_t iterator, wasm_ptr primary) {return selected().db_next_i64(iterator, *primary);} + int32_t db_previous_i64(int32_t iterator, wasm_ptr primary) {return selected().db_previous_i64(iterator, *primary);} + int32_t db_find_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) {return selected().db_find_i64(eosio::chain::name{code}, eosio::chain::name{scope}, eosio::chain::name{table}, id);} + int32_t db_lowerbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) {return selected().db_lowerbound_i64(eosio::chain::name{code}, eosio::chain::name{scope}, eosio::chain::name{table}, id);} + int32_t db_upperbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) {return selected().db_upperbound_i64(eosio::chain::name{code}, eosio::chain::name{scope}, eosio::chain::name{table}, id);} + int32_t db_end_i64(uint64_t code, uint64_t scope, uint64_t table) {return selected().db_end_i64(eosio::chain::name{code}, eosio::chain::name{scope}, eosio::chain::name{table});} + + DB_WRAPPERS_SIMPLE_SECONDARY(idx64, uint64_t) + DB_WRAPPERS_SIMPLE_SECONDARY(idx128, unsigned __int128) + // DB_WRAPPERS_ARRAY_SECONDARY(idx256, 2, unsigned __int128) + // DB_WRAPPERS_FLOAT_SECONDARY(idx_double, float64_t) + // DB_WRAPPERS_FLOAT_SECONDARY(idx_long_double, float128_t) + // clang-format on + + int64_t kv_erase(uint64_t db, uint64_t contract, span key) { + throw std::runtime_error("kv_erase not implemented in tester"); + } + + int64_t kv_set(uint64_t db, uint64_t contract, span key, span value) { + throw std::runtime_error("kv_set not implemented in tester"); + } + + bool kv_get(uint64_t db, uint64_t contract, span key, wasm_ptr value_size) { + return kv_get_db(db).kv_get(contract, key.data(), key.size(), *value_size); + } + + uint32_t kv_get_data(uint64_t db, uint32_t offset, span data) { + return kv_get_db(db).kv_get_data(offset, data.data(), data.size()); + } + + uint32_t kv_it_create(uint64_t db, uint64_t contract, span prefix) { + auto& kdb = kv_get_db(db); + uint32_t itr; + if (!selected().kv_destroyed_iterators.empty()) { + itr = selected().kv_destroyed_iterators.back(); + selected().kv_destroyed_iterators.pop_back(); + } else { + // Sanity check in case the per-database limits are set poorly + EOS_ASSERT(selected().kv_iterators.size() <= 0xFFFFFFFFu, kv_bad_iter, "Too many iterators"); + itr = selected().kv_iterators.size(); + selected().kv_iterators.emplace_back(); + } + selected().kv_iterators[itr] = kdb.kv_it_create(contract, prefix.data(), prefix.size()); + return itr; + } + + void kv_it_destroy(uint32_t itr) { + kv_check_iterator(itr); + selected().kv_destroyed_iterators.push_back(itr); + selected().kv_iterators[itr].reset(); + } + + int32_t kv_it_status(uint32_t itr) { + kv_check_iterator(itr); + return static_cast(selected().kv_iterators[itr]->kv_it_status()); + } + + int32_t kv_it_compare(uint32_t itr_a, uint32_t itr_b) { + kv_check_iterator(itr_a); + kv_check_iterator(itr_b); + return selected().kv_iterators[itr_a]->kv_it_compare(*selected().kv_iterators[itr_b]); + } + + int32_t kv_it_key_compare(uint32_t itr, span key) { + kv_check_iterator(itr); + return selected().kv_iterators[itr]->kv_it_key_compare(key.data(), key.size()); + } + + int32_t kv_it_move_to_end(uint32_t itr) { + kv_check_iterator(itr); + return static_cast(selected().kv_iterators[itr]->kv_it_move_to_end()); + } + + int32_t kv_it_next(uint32_t itr, wasm_ptr found_key_size, wasm_ptr found_value_size) { + kv_check_iterator(itr); + return static_cast(selected().kv_iterators[itr]->kv_it_next(found_key_size, found_value_size)); + } + + int32_t kv_it_prev(uint32_t itr, wasm_ptr found_key_size, wasm_ptr found_value_size) { + kv_check_iterator(itr); + return static_cast(selected().kv_iterators[itr]->kv_it_prev(found_key_size, found_value_size)); + } + + int32_t kv_it_lower_bound(uint32_t itr, span key, wasm_ptr found_key_size, + wasm_ptr found_value_size) { + kv_check_iterator(itr); + return static_cast( + selected().kv_iterators[itr]->kv_it_lower_bound(key.data(), key.size(), found_key_size, found_value_size)); + } + + int32_t kv_it_key(uint32_t itr, uint32_t offset, span dest, wasm_ptr actual_size) { + kv_check_iterator(itr); + return static_cast( + selected().kv_iterators[itr]->kv_it_key(offset, dest.data(), dest.size(), *actual_size)); + } + + int32_t kv_it_value(uint32_t itr, uint32_t offset, span dest, wasm_ptr actual_size) { + kv_check_iterator(itr); + return static_cast( + selected().kv_iterators[itr]->kv_it_value(offset, dest.data(), dest.size(), *actual_size)); + } + + kv_context& kv_get_db(uint64_t db) { + if (db == kvram_id.to_uint64_t()) + return *selected().kv_ram; + else if (db == kvdisk_id.to_uint64_t()) + return *selected().kv_disk; + EOS_ASSERT(false, kv_bad_db_id, "Bad key-value database ID"); + } + + void kv_check_iterator(uint32_t itr) { + EOS_ASSERT(itr < selected().kv_iterators.size() && selected().kv_iterators[itr], kv_bad_iter, + "Bad key-value iterator"); + } + + void sha1(span data, void* hash_val) { + auto hash = fc::sha1::hash(data.data(), data.size()); + check_bounds(hash_val, hash.data_size()); + std::memcpy(hash_val, hash.data(), hash.data_size()); + } + + void sha256(span data, void* hash_val) { + auto hash = fc::sha256::hash(data.data(), data.size()); + check_bounds(hash_val, hash.data_size()); + std::memcpy(hash_val, hash.data(), hash.data_size()); + } + + void sha512(span data, void* hash_val) { + auto hash = fc::sha512::hash(data.data(), data.size()); + check_bounds(hash_val, hash.data_size()); + std::memcpy(hash_val, hash.data(), hash.data_size()); + } + + void ripemd160(span data, void* hash_val) { + auto hash = fc::ripemd160::hash(data.data(), data.size()); + check_bounds(hash_val, hash.data_size()); + std::memcpy(hash_val, hash.data(), hash.data_size()); + } +}; // callbacks + +#define DB_REGISTER_SECONDARY(IDX) \ + rhf_t::add<&callbacks::db_##IDX##_find_secondary>("env", "db_" #IDX "_find_secondary"); \ + rhf_t::add<&callbacks::db_##IDX##_find_primary>("env", "db_" #IDX "_find_primary"); \ + rhf_t::add<&callbacks::db_##IDX##_lowerbound>("env", "db_" #IDX "_lowerbound"); \ + rhf_t::add<&callbacks::db_##IDX##_upperbound>("env", "db_" #IDX "_upperbound"); \ + rhf_t::add<&callbacks::db_##IDX##_end>("env", "db_" #IDX "_end"); \ + rhf_t::add<&callbacks::db_##IDX##_next>("env", "db_" #IDX "_next"); \ + rhf_t::add<&callbacks::db_##IDX##_previous>("env", "db_" #IDX "_previous"); + +void register_callbacks() { + rhf_t::add<&callbacks::abort>("env", "abort"); + rhf_t::add<&callbacks::eosio_assert_message>("env", "eosio_assert_message"); + rhf_t::add<&callbacks::prints_l>("env", "prints_l"); + rhf_t::add<&callbacks::get_args>("env", "get_args"); + rhf_t::add<&callbacks::clock_gettime>("env", "clock_gettime"); + rhf_t::add<&callbacks::open_file>("env", "open_file"); + rhf_t::add<&callbacks::isatty>("env", "isatty"); + rhf_t::add<&callbacks::close_file>("env", "close_file"); + rhf_t::add<&callbacks::write_file>("env", "write_file"); + rhf_t::add<&callbacks::read_file>("env", "read_file"); + rhf_t::add<&callbacks::read_whole_file>("env", "read_whole_file"); + rhf_t::add<&callbacks::execute>("env", "execute"); + rhf_t::add<&callbacks::create_chain>("env", "create_chain"); + rhf_t::add<&callbacks::destroy_chain>("env", "destroy_chain"); + rhf_t::add<&callbacks::shutdown_chain>("env", "shutdown_chain"); + rhf_t::add<&callbacks::get_chain_path>("env", "get_chain_path"); + rhf_t::add<&callbacks::replace_producer_keys>("env", "replace_producer_keys"); + rhf_t::add<&callbacks::replace_account_keys>("env", "replace_account_keys"); + rhf_t::add<&callbacks::start_block>("env", "start_block"); + rhf_t::add<&callbacks::finish_block>("env", "finish_block"); + rhf_t::add<&callbacks::get_head_block_info>("env", "get_head_block_info"); + rhf_t::add<&callbacks::push_transaction>("env", "push_transaction"); + rhf_t::add<&callbacks::exec_deferred>("env", "exec_deferred"); + rhf_t::add<&callbacks::get_history>("env", "get_history"); + rhf_t::add<&callbacks::select_chain_for_db>("env", "select_chain_for_db"); + + rhf_t::add<&callbacks::create_rodeos>("env", "create_rodeos"); + rhf_t::add<&callbacks::destroy_rodeos>("env", "destroy_rodeos"); + rhf_t::add<&callbacks::rodeos_add_filter>("env", "rodeos_add_filter"); + rhf_t::add<&callbacks::rodeos_enable_queries>("env", "rodeos_enable_queries"); + rhf_t::add<&callbacks::connect_rodeos>("env", "connect_rodeos"); + rhf_t::add<&callbacks::rodeos_sync_block>("env", "rodeos_sync_block"); + rhf_t::add<&callbacks::rodeos_push_transaction>("env", "rodeos_push_transaction"); + rhf_t::add<&callbacks::rodeos_get_num_pushed_data>("env", "rodeos_get_num_pushed_data"); + rhf_t::add<&callbacks::rodeos_get_pushed_data>("env", "rodeos_get_pushed_data"); + + rhf_t::add<&callbacks::db_get_i64>("env", "db_get_i64"); + rhf_t::add<&callbacks::db_next_i64>("env", "db_next_i64"); + rhf_t::add<&callbacks::db_previous_i64>("env", "db_previous_i64"); + rhf_t::add<&callbacks::db_find_i64>("env", "db_find_i64"); + rhf_t::add<&callbacks::db_lowerbound_i64>("env", "db_lowerbound_i64"); + rhf_t::add<&callbacks::db_upperbound_i64>("env", "db_upperbound_i64"); + rhf_t::add<&callbacks::db_end_i64>("env", "db_end_i64"); + DB_REGISTER_SECONDARY(idx64) + DB_REGISTER_SECONDARY(idx128) + // DB_REGISTER_SECONDARY(idx256) + // DB_REGISTER_SECONDARY(idx_double) + // DB_REGISTER_SECONDARY(idx_long_double) + rhf_t::add<&callbacks::kv_erase>("env", "kv_erase"); + rhf_t::add<&callbacks::kv_set>("env", "kv_set"); + rhf_t::add<&callbacks::kv_get>("env", "kv_get"); + rhf_t::add<&callbacks::kv_get_data>("env", "kv_get_data"); + rhf_t::add<&callbacks::kv_it_create>("env", "kv_it_create"); + rhf_t::add<&callbacks::kv_it_destroy>("env", "kv_it_destroy"); + rhf_t::add<&callbacks::kv_it_status>("env", "kv_it_status"); + rhf_t::add<&callbacks::kv_it_compare>("env", "kv_it_compare"); + rhf_t::add<&callbacks::kv_it_key_compare>("env", "kv_it_key_compare"); + rhf_t::add<&callbacks::kv_it_move_to_end>("env", "kv_it_move_to_end"); + rhf_t::add<&callbacks::kv_it_next>("env", "kv_it_next"); + rhf_t::add<&callbacks::kv_it_prev>("env", "kv_it_prev"); + rhf_t::add<&callbacks::kv_it_lower_bound>("env", "kv_it_lower_bound"); + rhf_t::add<&callbacks::kv_it_key>("env", "kv_it_key"); + rhf_t::add<&callbacks::kv_it_value>("env", "kv_it_value"); + rhf_t::add<&callbacks::sha1>("env", "sha1"); + rhf_t::add<&callbacks::sha256>("env", "sha256"); + rhf_t::add<&callbacks::sha512>("env", "sha512"); + rhf_t::add<&callbacks::ripemd160>("env", "ripemd160"); +} + +static void run(const char* wasm, const std::vector& args) { + eosio::vm::wasm_allocator wa; + auto code = eosio::vm::read_wasm(wasm); + backend_t backend(code, nullptr); + ::state state{ wasm, wa, backend, eosio::check(eosio::convert_to_bin(args)).value() }; + callbacks cb{ state }; + state.files.emplace_back(stdin, false); + state.files.emplace_back(stdout, false); + state.files.emplace_back(stderr, false); + backend.set_wasm_allocator(&wa); + + rhf_t::resolve(backend.get_module()); + backend.initialize(&cb); + backend(cb, "env", "start", 0); +} + +const char usage[] = "usage: eosio-tester [-h or --help] [-v or --verbose] file.wasm [args for wasm]\n"; + +int main(int argc, char* argv[]) { + fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::off); + + bool show_usage = false; + bool error = false; + int next_arg = 1; + while (next_arg < argc && argv[next_arg][0] == '-') { + if (!strcmp(argv[next_arg], "-h") || !strcmp(argv[next_arg], "--help")) + show_usage = true; + else if (!strcmp(argv[next_arg], "-v") || !strcmp(argv[next_arg], "--verbose")) + fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug); + else { + std::cerr << "unknown option: " << argv[next_arg] << "\n"; + error = true; + } + ++next_arg; + } + if (next_arg >= argc) + error = true; + if (show_usage || error) { + std::cerr << usage; + return error; + } + try { + std::vector args{ argv + next_arg + 1, argv + argc }; + register_callbacks(); + run(argv[next_arg], args); + return 0; + } catch (::assert_exception& e) { + std::cerr << "tester wasm asserted: " << e.what() << "\n"; + } catch (eosio::vm::exception& e) { + std::cerr << "vm::exception: " << e.detail() << "\n"; + } catch (std::exception& e) { std::cerr << "std::exception: " << e.what() << "\n"; } catch (fc::exception& e) { + std::cerr << "fc::exception: " << e.to_string() << "\n"; + } + return 1; +} From 218ea53def882c2167ee077c5399bb4a7ea2c6b2 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Mon, 27 Apr 2020 20:32:15 -0400 Subject: [PATCH 12/23] wasm fixes --- libraries/rodeos/include/b1/rodeos/callbacks/basic.hpp | 2 +- libraries/rodeos/wasm_ql.cpp | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/basic.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/basic.hpp index 6249a9f1a15..12c0b499175 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/basic.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/basic.hpp @@ -19,7 +19,7 @@ struct context_free_system_callbacks { void abort() { throw std::runtime_error("called abort"); } - void eosio_assert(uint32_t condition, null_terminated_ptr msg) { + void eosio_assert(bool condition, null_terminated_ptr msg) { if (!condition) throw assert_exception(std::string(msg.data(), msg.size())); } diff --git a/libraries/rodeos/wasm_ql.cpp b/libraries/rodeos/wasm_ql.cpp index d349a15260e..9abb786c646 100644 --- a/libraries/rodeos/wasm_ql.cpp +++ b/libraries/rodeos/wasm_ql.cpp @@ -73,7 +73,6 @@ struct callbacks; using rhf_t = registered_host_functions; using backend_t = eosio::vm::backend; -// todo: remove context_free_system_callbacks struct callbacks : action_callbacks, chaindb_callbacks, compiler_builtins_callbacks, @@ -254,9 +253,9 @@ void run_action(wasm_ql::thread_state& thread_state, const std::vector& co entry->hash = *hash; else entry->name = action.account; - entry->backend = std::make_unique(*code, nullptr); std::call_once(registered_callbacks, register_callbacks); + entry->backend = std::make_unique(*code, nullptr); rhf_t::resolve(entry->backend->get_module()); } auto se = fc::make_scoped_exit([&] { thread_state.shared->backend_cache->add(std::move(*entry)); }); From 89cb9bfca6d8404a8fca598c35b00aa7a7b1303c Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Mon, 27 Apr 2020 21:29:41 -0400 Subject: [PATCH 13/23] rodeos --- libraries/rodeos/include/eosio/key_value.hpp | 2 +- programs/rodeos/CMakeLists.txt | 2 + programs/rodeos/main.cpp | 3 +- programs/rodeos/wasm_ql_http.cpp | 604 +++++++++++++++++++ programs/rodeos/wasm_ql_http.hpp | 25 + programs/rodeos/wasm_ql_plugin.cpp | 87 +++ programs/rodeos/wasm_ql_plugin.hpp | 22 + 7 files changed, 743 insertions(+), 2 deletions(-) create mode 100644 programs/rodeos/wasm_ql_http.cpp create mode 100644 programs/rodeos/wasm_ql_http.hpp create mode 100644 programs/rodeos/wasm_ql_plugin.cpp create mode 100644 programs/rodeos/wasm_ql_plugin.hpp diff --git a/libraries/rodeos/include/eosio/key_value.hpp b/libraries/rodeos/include/eosio/key_value.hpp index 01f93665280..46ed3fd9197 100644 --- a/libraries/rodeos/include/eosio/key_value.hpp +++ b/libraries/rodeos/include/eosio/key_value.hpp @@ -337,7 +337,7 @@ class kv_table { free(buffer); } - if (is_primary && actual_data_size > detail::max_stack_buffer_size) { + if (!is_primary && actual_data_size > detail::max_stack_buffer_size) { free(deserialize_buffer); } return val; diff --git a/programs/rodeos/CMakeLists.txt b/programs/rodeos/CMakeLists.txt index fb452ce266a..1690aea5c92 100644 --- a/programs/rodeos/CMakeLists.txt +++ b/programs/rodeos/CMakeLists.txt @@ -2,6 +2,8 @@ add_executable( ${RODEOS_EXECUTABLE_NAME} cloner_plugin.cpp main.cpp rocksdb_plugin.cpp + wasm_ql_http.cpp + wasm_ql_plugin.cpp ) if( UNIX AND NOT APPLE ) diff --git a/programs/rodeos/main.cpp b/programs/rodeos/main.cpp index 6c0f115cd96..985df5b34fe 100644 --- a/programs/rodeos/main.cpp +++ b/programs/rodeos/main.cpp @@ -10,6 +10,7 @@ #include "cloner_plugin.hpp" #include "config.hpp" +#include "wasm_ql_plugin.hpp" using namespace appbase; @@ -72,7 +73,7 @@ int main(int argc, char** argv) { auto root = fc::app_path(); app().set_default_data_dir(root / "eosio" / b1::rodeos::config::rodeos_executable_name / "data"); app().set_default_config_dir(root / "eosio" / b1::rodeos::config::rodeos_executable_name / "config"); - if (!app().initialize(argc, argv)) { + if (!app().initialize(argc, argv)) { const auto& opts = app().get_options(); if (opts.count("help") || opts.count("version") || opts.count("full-version") || opts.count("print-default-config")) { diff --git a/programs/rodeos/wasm_ql_http.cpp b/programs/rodeos/wasm_ql_http.cpp new file mode 100644 index 00000000000..d1094314d4f --- /dev/null +++ b/programs/rodeos/wasm_ql_http.cpp @@ -0,0 +1,604 @@ +// Adapted from Boost Beast Advanced Server example +// +// Copyright (c) 2016-2019 Vinnie Falco (vinnie dot falco at gmail dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include "wasm_ql_http.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +static const std::vector temp_contract_kv_prefix{ 0x02 }; // todo: replace + +namespace beast = boost::beast; // from +namespace http = beast::http; // from +namespace net = boost::asio; // from +using tcp = boost::asio::ip::tcp; // from + +using namespace std::literals; + +struct error_info { + int64_t code = {}; + std::string name = {}; + std::string what = {}; + std::vector details = {}; +}; + +EOSIO_REFLECT(error_info, code, name, what, details) + +struct error_results { + uint16_t code = {}; + std::string message = {}; + error_info error = {}; +}; + +EOSIO_REFLECT(error_results, code, message, error) + +namespace b1::rodeos::wasm_ql { + +// Report a failure +static void fail(beast::error_code ec, const char* what) { elog("${w}: ${s}", ("w", what)("s", ec.message())); } + +// Return a reasonable mime type based on the extension of a file. +beast::string_view mime_type(beast::string_view path) { + using beast::iequals; + const auto ext = [&path] { + const auto pos = path.rfind("."); + if (pos == beast::string_view::npos) + return beast::string_view{}; + return path.substr(pos); + }(); + if (iequals(ext, ".htm")) + return "text/html"; + if (iequals(ext, ".html")) + return "text/html"; + if (iequals(ext, ".php")) + return "text/html"; + if (iequals(ext, ".css")) + return "text/css"; + if (iequals(ext, ".txt")) + return "text/plain"; + if (iequals(ext, ".js")) + return "application/javascript"; + if (iequals(ext, ".json")) + return "application/json"; + if (iequals(ext, ".wasm")) + return "application/wasm"; + if (iequals(ext, ".xml")) + return "application/xml"; + if (iequals(ext, ".swf")) + return "application/x-shockwave-flash"; + if (iequals(ext, ".flv")) + return "video/x-flv"; + if (iequals(ext, ".png")) + return "image/png"; + if (iequals(ext, ".jpe")) + return "image/jpeg"; + if (iequals(ext, ".jpeg")) + return "image/jpeg"; + if (iequals(ext, ".jpg")) + return "image/jpeg"; + if (iequals(ext, ".gif")) + return "image/gif"; + if (iequals(ext, ".bmp")) + return "image/bmp"; + if (iequals(ext, ".ico")) + return "image/vnd.microsoft.icon"; + if (iequals(ext, ".tiff")) + return "image/tiff"; + if (iequals(ext, ".tif")) + return "image/tiff"; + if (iequals(ext, ".svg")) + return "image/svg+xml"; + if (iequals(ext, ".svgz")) + return "image/svg+xml"; + return "application/text"; +} // mime_type + +// Append an HTTP rel-path to a local filesystem path. +// The returned path is normalized for the platform. +std::string path_cat(beast::string_view base, beast::string_view path) { + if (base.empty()) + return std::string(path); + std::string result(base); +#ifdef BOOST_MSVC + char constexpr path_separator = '\\'; + if (result.back() == path_separator) + result.resize(result.size() - 1); + result.append(path.data(), path.size()); + for (auto& c : result) + if (c == '/') + c = path_separator; +#else + char constexpr path_separator = '/'; + if (result.back() == path_separator) + result.resize(result.size() - 1); + result.append(path.data(), path.size()); +#endif + return result; +} + +// This function produces an HTTP response for the given +// request. The type of the response object depends on the +// contents of the request, so the interface requires the +// caller to pass a generic lambda for receiving the response. +template +void handle_request(const wasm_ql::http_config& http_config, const wasm_ql::shared_state& shared_state, + thread_state_cache& state_cache, http::request>&& req, + Send&& send) { + // Returns a bad request response + const auto bad_request = [&http_config, &req](beast::string_view why) { + http::response res{ http::status::bad_request, req.version() }; + res.set(http::field::server, BOOST_BEAST_VERSION_STRING); + res.set(http::field::content_type, "text/html"); + if (!http_config.allow_origin.empty()) + res.set(http::field::access_control_allow_origin, http_config.allow_origin); + res.keep_alive(req.keep_alive()); + res.body() = why.to_string(); + res.prepare_payload(); + return res; + }; + + // Returns a not found response + const auto not_found = [&http_config, &req](beast::string_view target) { + http::response res{ http::status::not_found, req.version() }; + res.set(http::field::server, BOOST_BEAST_VERSION_STRING); + res.set(http::field::content_type, "text/html"); + if (!http_config.allow_origin.empty()) + res.set(http::field::access_control_allow_origin, http_config.allow_origin); + res.keep_alive(req.keep_alive()); + res.body() = "The resource '" + target.to_string() + "' was not found."; + res.prepare_payload(); + return res; + }; + + // Returns an error response + const auto error = [&http_config, &req](http::status status, beast::string_view why, + const char* content_type = "text/html") { + http::response res{ status, req.version() }; + res.set(http::field::server, BOOST_BEAST_VERSION_STRING); + res.set(http::field::content_type, content_type); + if (!http_config.allow_origin.empty()) + res.set(http::field::access_control_allow_origin, http_config.allow_origin); + res.keep_alive(req.keep_alive()); + res.body() = why.to_string(); + res.prepare_payload(); + return res; + }; + + const auto ok = [&http_config, &req](std::vector reply, const char* content_type) { + http::response> res{ http::status::ok, req.version() }; + res.set(http::field::server, BOOST_BEAST_VERSION_STRING); + res.set(http::field::content_type, content_type); + if (!http_config.allow_origin.empty()) + res.set(http::field::access_control_allow_origin, http_config.allow_origin); + res.keep_alive(req.keep_alive()); + res.body() = std::move(reply); + res.prepare_payload(); + return res; + }; + + // todo: pack error messages in json + // todo: replace "query failed" + try { + if (req.target() == "/v1/chain/get_info") { + auto thread_state = state_cache.get_state(); + send(ok(query_get_info(*thread_state, temp_contract_kv_prefix), "application/json")); + state_cache.store_state(std::move(thread_state)); + return; + } else if (req.target() == + "/v1/chain/get_block") { // todo: replace with /v1/chain/get_block_header. upgrade cleos. + if (req.method() != http::verb::post) + return send( + error(http::status::bad_request, "Unsupported HTTP-method for " + req.target().to_string() + "\n")); + auto thread_state = state_cache.get_state(); + send(ok(query_get_block(*thread_state, temp_contract_kv_prefix, + std::string_view{ req.body().data(), req.body().size() }), + "application/json")); + state_cache.store_state(std::move(thread_state)); + return; + } else if (req.target() == "/v1/chain/get_abi") { // todo: get_raw_abi. upgrade cleos to use get_raw_abi. + if (req.method() != http::verb::post) + return send( + error(http::status::bad_request, "Unsupported HTTP-method for " + req.target().to_string() + "\n")); + auto thread_state = state_cache.get_state(); + send(ok(query_get_abi(*thread_state, temp_contract_kv_prefix, + std::string_view{ req.body().data(), req.body().size() }), + "application/json")); + state_cache.store_state(std::move(thread_state)); + return; + } else if (req.target() == "/v1/chain/get_required_keys") { // todo: replace with a binary endpoint? + if (req.method() != http::verb::post) + return send( + error(http::status::bad_request, "Unsupported HTTP-method for " + req.target().to_string() + "\n")); + auto thread_state = state_cache.get_state(); + send(ok(query_get_required_keys(*thread_state, std::string_view{ req.body().data(), req.body().size() }), + "application/json")); + state_cache.store_state(std::move(thread_state)); + return; + } else if (req.target() == "/v1/chain/send_transaction") { + // todo: replace with /v1/chain/send_transaction2? + // or: change nodeos to not do abi deserialization if transaction extension present? + if (req.method() != http::verb::post) + return send( + error(http::status::bad_request, "Unsupported HTTP-method for " + req.target().to_string() + "\n")); + auto thread_state = state_cache.get_state(); + send(ok(query_send_transaction(*thread_state, temp_contract_kv_prefix, + std::string_view{ req.body().data(), req.body().size() }, + false // todo: switch to true when /v1/chain/send_transaction2 + ), + "application/json")); + state_cache.store_state(std::move(thread_state)); + return; + } else if (req.target().starts_with("/v1/") || http_config.static_dir.empty()) { + // todo: redirect if /v1/? + return send( + error(http::status::not_found, "The resource '" + req.target().to_string() + "' was not found.\n")); + } else { + // Make sure we can handle the method + if (req.method() != http::verb::get && req.method() != http::verb::head) + return send(bad_request("Unknown HTTP-method")); + + // Request path must be absolute and not contain "..". + if (req.target().empty() || req.target()[0] != '/' || req.target().find("..") != beast::string_view::npos) + return send(bad_request("Illegal request-target")); + + // Build the path to the requested file + std::string path = path_cat(http_config.static_dir, req.target()); + if (req.target().back() == '/') + path.append("index.html"); + + // Attempt to open the file + beast::error_code ec; + http::file_body::value_type body; + body.open(path.c_str(), beast::file_mode::scan, ec); + + // Handle the case where the file doesn't exist + if (ec == beast::errc::no_such_file_or_directory) + return send(not_found(req.target())); + + // Handle an unknown error + if (ec) + return send(error(http::status::internal_server_error, "An error occurred: "s + ec.message())); + + // Cache the size since we need it after the move + const auto size = body.size(); + + // Respond to HEAD request + if (req.method() == http::verb::head) { + http::response res{ http::status::ok, req.version() }; + res.set(http::field::server, BOOST_BEAST_VERSION_STRING); + res.set(http::field::content_type, mime_type(path)); + if (!http_config.allow_origin.empty()) + res.set(http::field::access_control_allow_origin, http_config.allow_origin); + res.content_length(size); + res.keep_alive(req.keep_alive()); + return send(std::move(res)); + } + + // Respond to GET request + http::response res{ std::piecewise_construct, std::make_tuple(std::move(body)), + std::make_tuple(http::status::ok, req.version()) }; + res.set(http::field::server, BOOST_BEAST_VERSION_STRING); + res.set(http::field::content_type, mime_type(path)); + if (!http_config.allow_origin.empty()) + res.set(http::field::access_control_allow_origin, http_config.allow_origin); + res.content_length(size); + res.keep_alive(req.keep_alive()); + return send(std::move(res)); + } + } catch (const std::exception& e) { + try { + // elog("query failed: ${s}", ("s", e.what())); + error_results err; + err.code = (uint16_t)http::status::internal_server_error; + err.message = "Internal Service Error"; + err.error.name = "exception"; + err.error.what = e.what(); + return send(error(http::status::internal_server_error, eosio::check(eosio::convert_to_json(err)).value(), + "application/json")); + } catch (...) { // + return send(error(http::status::internal_server_error, "failure reporting failure\n")); + } + } catch (...) { + elog("query failed: unknown exception"); + return send(error(http::status::internal_server_error, "query failed: unknown exception\n")); + } +} // handle_request + +// Handles an HTTP server connection +class http_session : public std::enable_shared_from_this { + // This queue is used for HTTP pipelining. + class queue { + enum { + // Maximum number of responses we will queue + limit = 8 + }; + + // The type-erased, saved work item + struct work { + virtual ~work() = default; + virtual void operator()() = 0; + }; + + http_session& self; + std::vector> items; + + public: + explicit queue(http_session& self) : self(self) { + static_assert(limit > 0, "queue limit must be positive"); + items.reserve(limit); + } + + // Returns `true` if we have reached the queue limit + bool is_full() const { return items.size() >= limit; } + + // Called when a message finishes sending + // Returns `true` if the caller should initiate a read + bool on_write() { + BOOST_ASSERT(!items.empty()); + const auto was_full = is_full(); + items.erase(items.begin()); + if (!items.empty()) + (*items.front())(); + return was_full; + } + + // Called by the HTTP handler to send a response. + template + void operator()(http::message&& msg) { + // This holds a work item + struct work_impl : work { + http_session& self; + http::message msg; + + work_impl(http_session& self, http::message&& msg) + : self(self), msg(std::move(msg)) {} + + void operator()() { + http::async_write( + self.stream, msg, + beast::bind_front_handler(&http_session::on_write, self.shared_from_this(), msg.need_eof())); + } + }; + + // Allocate and store the work + items.push_back(boost::make_unique(self, std::move(msg))); + + // If there was no previous work, start this one + if (items.size() == 1) + (*items.front())(); + } + }; + + beast::tcp_stream stream; + beast::flat_buffer buffer; + std::shared_ptr http_config; + std::shared_ptr shared_state; + std::shared_ptr state_cache; + queue queue_; + + // The parser is stored in an optional container so we can + // construct it from scratch it at the beginning of each new message. + boost::optional>> parser; + + public: + // Take ownership of the socket + http_session(const std::shared_ptr& http_config, + const std::shared_ptr& shared_state, + const std::shared_ptr& state_cache, tcp::socket&& socket) + : stream(std::move(socket)), http_config(http_config), shared_state(shared_state), state_cache(state_cache), + queue_(*this) {} + + // Start the session + void run() { do_read(); } + + private: + void do_read() { + // Construct a new parser for each message + parser.emplace(); + + // Apply a reasonable limit to the allowed size + // of the body in bytes to prevent abuse. + // todo: make configurable + parser->body_limit(http_config->max_request_size); + + // Set the timeout. + stream.expires_after(std::chrono::milliseconds(http_config->idle_timeout_ms)); + + // Read a request using the parser-oriented interface + http::async_read(stream, buffer, *parser, beast::bind_front_handler(&http_session::on_read, shared_from_this())); + } + + void on_read(beast::error_code ec, std::size_t bytes_transferred) { + boost::ignore_unused(bytes_transferred); + + // This means they closed the connection + if (ec == http::error::end_of_stream) + return do_close(); + + if (ec) + return fail(ec, "read"); + + // Send the response + handle_request(*http_config, *shared_state, *state_cache, parser->release(), queue_); + + // If we aren't at the queue limit, try to pipeline another request + if (!queue_.is_full()) + do_read(); + } + + void on_write(bool close, beast::error_code ec, std::size_t bytes_transferred) { + boost::ignore_unused(bytes_transferred); + + if (ec) + return fail(ec, "write"); + + if (close) { + // This means we should close the connection, usually because + // the response indicated the "Connection: close" semantic. + return do_close(); + } + + // Inform the queue that a write completed + if (queue_.on_write()) { + // Read another request + do_read(); + } + } + + void do_close() { + // Send a TCP shutdown + beast::error_code ec; + stream.socket().shutdown(tcp::socket::shutdown_send, ec); + + // At this point the connection is closed gracefully + } +}; // http_session + +// Accepts incoming connections and launches the sessions +class listener : public std::enable_shared_from_this { + std::shared_ptr http_config; + std::shared_ptr shared_state; + net::io_context& ioc; + tcp::acceptor acceptor; + bool acceptor_ready = false; + std::shared_ptr state_cache; + + public: + listener(const std::shared_ptr& http_config, + const std::shared_ptr& shared_state, net::io_context& ioc, + tcp::endpoint endpoint) + : http_config{ http_config }, shared_state{ shared_state }, ioc(ioc), acceptor(net::make_strand(ioc)), + state_cache(std::make_shared(shared_state)) { + + beast::error_code ec; + + // Open the acceptor + acceptor.open(endpoint.protocol(), ec); + if (ec) { + fail(ec, "open"); + return; + } + + // Bind to the server address + acceptor.bind(endpoint, ec); + if (ec) { + fail(ec, "bind"); + return; + } + + // Start listening for connections + acceptor.listen(net::socket_base::max_listen_connections, ec); + if (ec) { + fail(ec, "listen"); + return; + } + + acceptor_ready = true; + } + + // Start accepting incoming connections + void run() { + if (acceptor_ready) + do_accept(); + } + + private: + void do_accept() { + // The new connection gets its own strand + acceptor.async_accept(net::make_strand(ioc), beast::bind_front_handler(&listener::on_accept, shared_from_this())); + } + + void on_accept(beast::error_code ec, tcp::socket socket) { + if (ec) { + fail(ec, "accept"); + } else { + // Create the http session and run it + std::make_shared(http_config, shared_state, state_cache, std::move(socket))->run(); + } + + // Accept another connection + do_accept(); + } +}; // listener + +struct server_impl : http_server, std::enable_shared_from_this { + net::io_service ioc; + std::shared_ptr http_config = {}; + std::shared_ptr shared_state = {}; + std::vector threads = {}; + std::unique_ptr acceptor = {}; + + server_impl(const std::shared_ptr& http_config, + const std::shared_ptr& shared_state) + : http_config{ http_config }, shared_state{ shared_state } {} + + virtual ~server_impl() {} + + virtual void stop() override { + ioc.stop(); + for (auto& t : threads) t.join(); + threads.clear(); + } + + void start() { + boost::system::error_code ec; + auto check_ec = [&](const char* what) { + if (!ec) + return; + elog("${w}: ${m}", ("w", what)("m", ec.message())); + FC_ASSERT(false, "unable to open listen socket"); + }; + + ilog("listen on ${a}:${p}", ("a", http_config->address)("p", http_config->port)); + boost::asio::ip::address a; + try { + a = net::ip::make_address(http_config->address); + } catch (std::exception& e) { + throw std::runtime_error("make_address(): "s + http_config->address + ": " + e.what()); + } + std::make_shared(http_config, shared_state, ioc, + tcp::endpoint{ a, (unsigned short)std::atoi(http_config->port.c_str()) }) + ->run(); + + threads.reserve(http_config->num_threads); + for (int i = 0; i < http_config->num_threads; ++i) + threads.emplace_back([self = shared_from_this()] { self->ioc.run(); }); + } +}; // server_impl + +std::shared_ptr http_server::create(const std::shared_ptr& http_config, + const std::shared_ptr& shared_state) { + FC_ASSERT(http_config->num_threads > 0, "too few threads"); + auto server = std::make_shared(http_config, shared_state); + server->start(); + return server; +} + +} // namespace b1::rodeos::wasm_ql diff --git a/programs/rodeos/wasm_ql_http.hpp b/programs/rodeos/wasm_ql_http.hpp new file mode 100644 index 00000000000..054b490d3fe --- /dev/null +++ b/programs/rodeos/wasm_ql_http.hpp @@ -0,0 +1,25 @@ +#pragma once +#include + +namespace b1::rodeos::wasm_ql { + +struct http_config { + uint32_t num_threads = {}; + uint32_t max_request_size = {}; + uint64_t idle_timeout_ms = {}; + std::string allow_origin = {}; + std::string static_dir = {}; + std::string address = {}; + std::string port = {}; +}; + +struct http_server { + virtual ~http_server() {} + + static std::shared_ptr create(const std::shared_ptr& http_config, + const std::shared_ptr& shared_state); + + virtual void stop() = 0; +}; + +} // namespace b1::rodeos::wasm_ql diff --git a/programs/rodeos/wasm_ql_plugin.cpp b/programs/rodeos/wasm_ql_plugin.cpp new file mode 100644 index 00000000000..db1e169c287 --- /dev/null +++ b/programs/rodeos/wasm_ql_plugin.cpp @@ -0,0 +1,87 @@ +#include "wasm_ql_plugin.hpp" +#include "wasm_ql_http.hpp" + +#include +#include +#include + +using namespace appbase; +using namespace b1::rodeos; +using namespace std::literals; + +using wasm_ql_plugin = b1::wasm_ql_plugin; + +static abstract_plugin& _wasm_ql_plugin = app().register_plugin(); + +namespace b1 { + +struct wasm_ql_plugin_impl : std::enable_shared_from_this { + bool stopping = false; + std::shared_ptr http_config = {}; + std::shared_ptr shared_state = {}; + std::shared_ptr http_server = {}; + + void start_http() { http_server = wasm_ql::http_server::create(http_config, shared_state); } + + void shutdown() { + stopping = true; + if (http_server) + http_server->stop(); + } +}; // wasm_ql_plugin_impl + +} // namespace b1 + +wasm_ql_plugin::wasm_ql_plugin() : my(std::make_shared()) {} + +wasm_ql_plugin::~wasm_ql_plugin() { + if (my->stopping) + ilog("wasm_ql_plugin stopped"); +} + +void wasm_ql_plugin::set_program_options(options_description& cli, options_description& cfg) { + auto op = cfg.add_options(); + op("wql-threads", bpo::value()->default_value(8), "Number of threads to process requests"); + op("wql-listen", bpo::value()->default_value("127.0.0.1:8880"), "Endpoint to listen on"); + op("wql-allow-origin", bpo::value(), "Access-Control-Allow-Origin header. Use \"*\" to allow any."); + op("wql-contract-dir", bpo::value(), + "Directory to fetch contracts from. These override contracts on the chain. (default: disabled)"); + op("wql-static-dir", bpo::value(), "Directory to serve static files from (default: disabled)"); + op("wql-console-size", bpo::value()->default_value(0), "Maximum size of console data"); + op("wql-wasm-cache-size", bpo::value()->default_value(100), "Maximum number of compiled wasms to cache"); + op("wql-max-request-size", bpo::value()->default_value(10000), "HTTP maximum request body size (bytes)"); + op("wql-idle-timeout", bpo::value()->default_value(30000), "HTTP idle connection timeout (ms)"); + op("wql-exec-time", bpo::value()->default_value(200), "Max query execution time (ms)"); +} + +void wasm_ql_plugin::plugin_initialize(const variables_map& options) { + try { + auto ip_port = options.at("wql-listen").as(); + if (ip_port.find(':') == std::string::npos) + throw std::runtime_error("invalid --wql-listen value: " + ip_port); + + auto http_config = std::make_shared(); + auto shared_state = std::make_shared(app().find_plugin()->get_db()); + my->http_config = http_config; + my->shared_state = shared_state; + + http_config->num_threads = options.at("wql-threads").as(); + http_config->port = ip_port.substr(ip_port.find(':') + 1, ip_port.size()); + http_config->address = ip_port.substr(0, ip_port.find(':')); + shared_state->max_console_size = options.at("wql-console-size").as(); + shared_state->wasm_cache_size = options.at("wql-wasm-cache-size").as(); + http_config->max_request_size = options.at("wql-max-request-size").as(); + http_config->idle_timeout_ms = options.at("wql-idle-timeout").as(); + shared_state->max_exec_time_ms = options.at("wql-exec-time").as(); + if (options.count("wql-contract-dir")) + shared_state->contract_dir = options.at("wql-contract-dir").as(); + if (options.count("wql-allow-origin")) + http_config->allow_origin = options.at("wql-allow-origin").as(); + if (options.count("wql-static-dir")) + http_config->static_dir = options.at("wql-static-dir").as(); + } + FC_LOG_AND_RETHROW() +} + +void wasm_ql_plugin::plugin_startup() { my->start_http(); } +void wasm_ql_plugin::plugin_shutdown() { my->shutdown(); } diff --git a/programs/rodeos/wasm_ql_plugin.hpp b/programs/rodeos/wasm_ql_plugin.hpp new file mode 100644 index 00000000000..f37879fa521 --- /dev/null +++ b/programs/rodeos/wasm_ql_plugin.hpp @@ -0,0 +1,22 @@ +#pragma once +#include "rocksdb_plugin.hpp" + +namespace b1 { + +class wasm_ql_plugin : public appbase::plugin { + public: + APPBASE_PLUGIN_REQUIRES((rocksdb_plugin)) + + wasm_ql_plugin(); + virtual ~wasm_ql_plugin(); + + virtual void set_program_options(appbase::options_description& cli, appbase::options_description& cfg) override; + void plugin_initialize(const appbase::variables_map& options); + void plugin_startup(); + void plugin_shutdown(); + + private: + std::shared_ptr my; +}; + +} // namespace b1 From cb333fb7776fe4269b418ecce26b5798803194d1 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Tue, 28 Apr 2020 15:09:36 -0400 Subject: [PATCH 14/23] gcc 8 fixes --- .../include/b1/rodeos/callbacks/chaindb.hpp | 6 ++--- .../rodeos/include/b1/rodeos/wasm_ql.hpp | 15 ++++++++----- libraries/rodeos/include/eosio/datastream.hpp | 4 ++++ libraries/rodeos/wasm_ql.cpp | 2 +- programs/eosio-tester/main.cpp | 3 +-- programs/rodeos/wasm_ql_http.cpp | 22 +++++++++---------- 6 files changed, 29 insertions(+), 23 deletions(-) diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/chaindb.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/chaindb.hpp index a0ec57a12e3..70699d9de58 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/chaindb.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/chaindb.hpp @@ -110,7 +110,7 @@ class iterator_cache { throw std::runtime_error("dereference invalid iterator"); if (itr < 0) throw std::runtime_error("dereference end iterator"); - if (itr >= iterators.size()) + if (size_t(itr) >= iterators.size()) throw std::runtime_error("dereference non-existing iterator"); auto& it = iterators[itr]; return legacy_copy_to_wasm(buffer, buffer_size, it.value.data(), it.value.size()); @@ -121,7 +121,7 @@ class iterator_cache { throw std::runtime_error("increment invalid iterator"); if (itr < 0) return -1; - if (itr >= iterators.size()) + if (size_t(itr) >= iterators.size()) throw std::runtime_error("increment non-existing iterator"); auto& it = iterators[itr]; if (it.next >= 0) { @@ -188,7 +188,7 @@ class iterator_cache { }; // iterator_cache struct chaindb_state { - std::unique_ptr iterator_cache; + std::unique_ptr iterator_cache; }; template diff --git a/libraries/rodeos/include/b1/rodeos/wasm_ql.hpp b/libraries/rodeos/include/b1/rodeos/wasm_ql.hpp index 05d16bac5ed..6d251ce7b6c 100644 --- a/libraries/rodeos/include/b1/rodeos/wasm_ql.hpp +++ b/libraries/rodeos/include/b1/rodeos/wasm_ql.hpp @@ -11,15 +11,18 @@ namespace b1::rodeos::wasm_ql { class backend_cache; struct shared_state { - uint32_t max_console_size = {}; - uint32_t wasm_cache_size = {}; - uint64_t max_exec_time_ms = {}; - std::string contract_dir = {}; - std::unique_ptr backend_cache = {}; - std::shared_ptr db; + uint32_t max_console_size = {}; + uint32_t wasm_cache_size = {}; + uint64_t max_exec_time_ms = {}; + std::string contract_dir = {}; + std::shared_ptr backend_cache = {}; + std::shared_ptr db; shared_state(std::shared_ptr db); + shared_state(const shared_state&) = delete; ~shared_state(); + + shared_state& operator=(const shared_state&) = delete; }; struct thread_state : action_state, console_state, query_state { diff --git a/libraries/rodeos/include/eosio/datastream.hpp b/libraries/rodeos/include/eosio/datastream.hpp index 88e814b9632..4968fb0f380 100644 --- a/libraries/rodeos/include/eosio/datastream.hpp +++ b/libraries/rodeos/include/eosio/datastream.hpp @@ -357,12 +357,14 @@ DataStream& operator<<( DataStream& ds, const T& v ) { return ds; } +/* throws off gcc // Backwards compatibility: allow user defined datastream operators to work with from_bin template,T>* = nullptr> result to_bin( const T& v, datastream& ds ) { ds << v; return outcome::success(); } +*/ /** * Deserialize a class @@ -380,11 +382,13 @@ DataStream& operator>>( DataStream& ds, T& v ) { return ds; } +/* throws off gcc template,T>* = nullptr> result from_bin( T& v, datastream& ds ) { ds >> v; return outcome::success(); } +*/ /** * Unpack data inside a fixed size buffer as T diff --git a/libraries/rodeos/wasm_ql.cpp b/libraries/rodeos/wasm_ql.cpp index 9abb786c646..1b46de41d5f 100644 --- a/libraries/rodeos/wasm_ql.cpp +++ b/libraries/rodeos/wasm_ql.cpp @@ -167,7 +167,7 @@ class backend_cache { }; shared_state::shared_state(std::shared_ptr db) - : backend_cache(std::make_unique(*this)), db(std::move(db)) {} + : backend_cache(std::make_shared(*this)), db(std::move(db)) {} shared_state::~shared_state() {} diff --git a/programs/eosio-tester/main.cpp b/programs/eosio-tester/main.cpp index 198da39f6ca..1ca5f7c462e 100644 --- a/programs/eosio-tester/main.cpp +++ b/programs/eosio-tester/main.cpp @@ -46,7 +46,6 @@ using eosio::state_history::block_position; using eosio::state_history::create_deltas; using eosio::state_history::get_blocks_result_v0; using eosio::state_history::state_result; -using eosio::state_history::trace_converter; using eosio::vm::span; struct callbacks; @@ -183,7 +182,7 @@ struct test_chain { std::unique_ptr control; fc::optional applied_transaction_connection; fc::optional accepted_block_connection; - trace_converter trace_converter; + eosio::state_history::trace_converter trace_converter; fc::optional prev_block; std::map> history; std::unique_ptr intr_ctx; diff --git a/programs/rodeos/wasm_ql_http.cpp b/programs/rodeos/wasm_ql_http.cpp index d1094314d4f..96d0f5378be 100644 --- a/programs/rodeos/wasm_ql_http.cpp +++ b/programs/rodeos/wasm_ql_http.cpp @@ -396,12 +396,12 @@ class http_session : public std::enable_shared_from_this { } }; - beast::tcp_stream stream; - beast::flat_buffer buffer; - std::shared_ptr http_config; - std::shared_ptr shared_state; - std::shared_ptr state_cache; - queue queue_; + beast::tcp_stream stream; + beast::flat_buffer buffer; + std::shared_ptr http_config; + std::shared_ptr shared_state; + std::shared_ptr state_cache; + queue queue_; // The parser is stored in an optional container so we can // construct it from scratch it at the beginning of each new message. @@ -549,11 +549,11 @@ class listener : public std::enable_shared_from_this { }; // listener struct server_impl : http_server, std::enable_shared_from_this { - net::io_service ioc; - std::shared_ptr http_config = {}; - std::shared_ptr shared_state = {}; - std::vector threads = {}; - std::unique_ptr acceptor = {}; + net::io_service ioc; + std::shared_ptr http_config = {}; + std::shared_ptr shared_state = {}; + std::vector threads = {}; + std::unique_ptr acceptor = {}; server_impl(const std::shared_ptr& http_config, const std::shared_ptr& shared_state) From 04f6d84f4e4e1b8b2683e70b609a2f490ffc8239 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Tue, 28 Apr 2020 16:39:41 -0400 Subject: [PATCH 15/23] build issues --- libraries/abieos | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/abieos b/libraries/abieos index ca85b0d0858..e9a4dca327d 160000 --- a/libraries/abieos +++ b/libraries/abieos @@ -1 +1 @@ -Subproject commit ca85b0d08589e4275bfb0cdd6cfb2d039c35612a +Subproject commit e9a4dca327dcf81596f1ace10f272a77510ec8d4 From 15e2e38adb26da9764898a3cc4b05d997f24ac1e Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Wed, 29 Apr 2020 08:48:35 -0400 Subject: [PATCH 16/23] Build issues --- libraries/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/CMakeLists.txt b/libraries/CMakeLists.txt index 898590dd020..5b313b5314f 100644 --- a/libraries/CMakeLists.txt +++ b/libraries/CMakeLists.txt @@ -1,4 +1,5 @@ -set(PORTABLE ON) # rocksdb: don't use sse4.2 +set(PORTABLE ON) # rocksdb: don't use sse4.2 +set(FAIL_ON_WARNINGS OFF) # rocksdb: stop the madness: warnings change over time add_subdirectory( fc ) add_subdirectory( builtins ) From dba1daa84c84eb205e25b7a1f64711f3f95c7b4c Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Wed, 29 Apr 2020 09:09:22 -0400 Subject: [PATCH 17/23] gflags dependency --- .cicd/platforms/pinned/amazon_linux-2-pinned.dockerfile | 2 +- .cicd/platforms/pinned/centos-7.7-pinned.dockerfile | 2 +- .cicd/platforms/pinned/macos-10.14-pinned.sh | 2 +- .cicd/platforms/pinned/macos-10.15-pinned.sh | 2 +- .cicd/platforms/pinned/ubuntu-16.04-pinned.dockerfile | 3 ++- .cicd/platforms/pinned/ubuntu-18.04-pinned.dockerfile | 2 +- .cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile | 2 +- .cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile | 3 ++- .cicd/platforms/unpinned/macos-10.14-unpinned.sh | 2 +- .cicd/platforms/unpinned/macos-10.15-unpinned.sh | 2 +- .cicd/platforms/unpinned/ubuntu-18.04-unpinned.dockerfile | 3 ++- 11 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.cicd/platforms/pinned/amazon_linux-2-pinned.dockerfile b/.cicd/platforms/pinned/amazon_linux-2-pinned.dockerfile index 69e0cdec45b..623620d72f5 100644 --- a/.cicd/platforms/pinned/amazon_linux-2-pinned.dockerfile +++ b/.cicd/platforms/pinned/amazon_linux-2-pinned.dockerfile @@ -5,7 +5,7 @@ RUN yum update -y && \ yum install -y which git sudo procps-ng util-linux autoconf automake \ libtool make bzip2 bzip2-devel openssl-devel gmp-devel libstdc++ libcurl-devel \ libusbx-devel python3 python3-devel python-devel libedit-devel doxygen \ - graphviz patch gcc gcc-c++ vim-common jq + graphviz patch gcc gcc-c++ vim-common jq gflags-devel # build cmake RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ tar -xzf cmake-3.16.2.tar.gz && \ diff --git a/.cicd/platforms/pinned/centos-7.7-pinned.dockerfile b/.cicd/platforms/pinned/centos-7.7-pinned.dockerfile index 34c5d4caff2..60eaf5a8935 100644 --- a/.cicd/platforms/pinned/centos-7.7-pinned.dockerfile +++ b/.cicd/platforms/pinned/centos-7.7-pinned.dockerfile @@ -8,7 +8,7 @@ RUN yum update -y && \ yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 doxygen \ graphviz bzip2-devel openssl-devel gmp-devel ocaml libicu-devel \ python python-devel rh-python36 file libusbx-devel \ - libcurl-devel patch vim-common jq + libcurl-devel patch vim-common jq gflags-devel # build cmake RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ tar -xzf cmake-3.16.2.tar.gz && \ diff --git a/.cicd/platforms/pinned/macos-10.14-pinned.sh b/.cicd/platforms/pinned/macos-10.14-pinned.sh index 769d0299116..072d703fbf8 100755 --- a/.cicd/platforms/pinned/macos-10.14-pinned.sh +++ b/.cicd/platforms/pinned/macos-10.14-pinned.sh @@ -2,7 +2,7 @@ set -eo pipefail VERSION=1 brew update -brew install git cmake python libtool libusb graphviz automake wget gmp pkgconfig doxygen openssl@1.1 jq || : +brew install git cmake python libtool libusb graphviz automake wget gmp pkgconfig doxygen openssl@1.1 jq gflags || : # install clang from source git clone --single-branch --branch llvmorg-10.0.0 https://github.com/llvm/llvm-project clang10 mkdir clang10/build diff --git a/.cicd/platforms/pinned/macos-10.15-pinned.sh b/.cicd/platforms/pinned/macos-10.15-pinned.sh index 45a35a2b638..af9727f0798 100755 --- a/.cicd/platforms/pinned/macos-10.15-pinned.sh +++ b/.cicd/platforms/pinned/macos-10.15-pinned.sh @@ -3,7 +3,7 @@ set -eo pipefail VERSION=1 export SDKROOT="$(xcrun --sdk macosx --show-sdk-path)" brew update -brew install git cmake python libtool libusb graphviz automake wget gmp pkgconfig doxygen openssl jq || : +brew install git cmake python libtool libusb graphviz automake wget gmp pkgconfig doxygen openssl jq gflags || : # install clang from source git clone --single-branch --branch llvmorg-10.0.0 https://github.com/llvm/llvm-project clang10 mkdir clang10/build diff --git a/.cicd/platforms/pinned/ubuntu-16.04-pinned.dockerfile b/.cicd/platforms/pinned/ubuntu-16.04-pinned.dockerfile index 4e91307bca1..2ec27938dbe 100644 --- a/.cicd/platforms/pinned/ubuntu-16.04-pinned.dockerfile +++ b/.cicd/platforms/pinned/ubuntu-16.04-pinned.dockerfile @@ -6,7 +6,8 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential git automake \ libbz2-dev libssl-dev doxygen graphviz libgmp3-dev autotools-dev libicu-dev \ python2.7 python2.7-dev python3 python3-dev autoconf libtool curl zlib1g-dev \ - sudo ruby libusb-1.0-0-dev libcurl4-gnutls-dev pkg-config apt-transport-https vim-common jq + sudo ruby libusb-1.0-0-dev libcurl4-gnutls-dev pkg-config apt-transport-https vim-common jq \ + gflags # build cmake RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ tar -xzf cmake-3.16.2.tar.gz && \ diff --git a/.cicd/platforms/pinned/ubuntu-18.04-pinned.dockerfile b/.cicd/platforms/pinned/ubuntu-18.04-pinned.dockerfile index 1e0a1a86fc8..8478a5758b8 100644 --- a/.cicd/platforms/pinned/ubuntu-18.04-pinned.dockerfile +++ b/.cicd/platforms/pinned/ubuntu-18.04-pinned.dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && \ autotools-dev libicu-dev python2.7 python2.7-dev python3 \ python3-dev python-configparser python-requests python-pip \ autoconf libtool g++ gcc curl zlib1g-dev sudo ruby libusb-1.0-0-dev \ - libcurl4-gnutls-dev pkg-config patch vim-common jq + libcurl4-gnutls-dev pkg-config patch vim-common jq libgflags-dev # build cmake RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ tar -xzf cmake-3.16.2.tar.gz && \ diff --git a/.cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile b/.cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile index 4b0d5359d66..d16baa9b461 100644 --- a/.cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile +++ b/.cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile @@ -5,7 +5,7 @@ RUN yum update -y && \ yum install -y which git sudo procps-ng util-linux autoconf automake \ libtool make bzip2 bzip2-devel openssl-devel gmp-devel libstdc++ libcurl-devel \ libusbx-devel python3 python3-devel python-devel libedit-devel doxygen \ - graphviz clang patch llvm-devel llvm-static vim-common jq + graphviz clang patch llvm-devel llvm-static vim-common jq gflags-devel # build cmake RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ tar -xzf cmake-3.16.2.tar.gz && \ diff --git a/.cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile b/.cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile index c33cc6fef00..85eb50290d8 100644 --- a/.cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile +++ b/.cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile @@ -8,7 +8,8 @@ RUN yum update -y && \ yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 doxygen \ graphviz bzip2-devel openssl-devel gmp-devel ocaml libicu-devel \ python python-devel rh-python36 file libusbx-devel \ - libcurl-devel patch vim-common jq llvm-toolset-7.0-llvm-devel llvm-toolset-7.0-llvm-static + libcurl-devel patch vim-common jq llvm-toolset-7.0-llvm-devel llvm-toolset-7.0-llvm-static \ + gflags-devel # build cmake RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ tar -xzf cmake-3.16.2.tar.gz && \ diff --git a/.cicd/platforms/unpinned/macos-10.14-unpinned.sh b/.cicd/platforms/unpinned/macos-10.14-unpinned.sh index 585e4366486..b8867e0caa5 100755 --- a/.cicd/platforms/unpinned/macos-10.14-unpinned.sh +++ b/.cicd/platforms/unpinned/macos-10.14-unpinned.sh @@ -2,7 +2,7 @@ set -eo pipefail VERSION=1 brew update -brew install git cmake python libtool libusb graphviz automake wget gmp pkgconfig doxygen openssl@1.1 jq boost || : +brew install git cmake python libtool libusb graphviz automake wget gmp pkgconfig doxygen openssl@1.1 jq boost gflags || : # install mongodb cd ~ && curl -OL https://fastdl.mongodb.org/osx/mongodb-osx-ssl-x86_64-3.6.3.tgz tar -xzf mongodb-osx-ssl-x86_64-3.6.3.tgz && rm -f mongodb-osx-ssl-x86_64-3.6.3.tgz && \ diff --git a/.cicd/platforms/unpinned/macos-10.15-unpinned.sh b/.cicd/platforms/unpinned/macos-10.15-unpinned.sh index 23d71a57473..36a06b37678 100755 --- a/.cicd/platforms/unpinned/macos-10.15-unpinned.sh +++ b/.cicd/platforms/unpinned/macos-10.15-unpinned.sh @@ -3,7 +3,7 @@ set -eo pipefail VERSION=1 export SDKROOT="$(xcrun --sdk macosx --show-sdk-path)" brew update -brew install git cmake python libtool libusb graphviz automake wget gmp pkgconfig doxygen openssl jq boost || : +brew install git cmake python libtool libusb graphviz automake wget gmp pkgconfig doxygen openssl jq boost gflags || : # install mongoDB cd ~ curl -OL https://fastdl.mongodb.org/osx/mongodb-osx-ssl-x86_64-3.6.3.tgz diff --git a/.cicd/platforms/unpinned/ubuntu-18.04-unpinned.dockerfile b/.cicd/platforms/unpinned/ubuntu-18.04-unpinned.dockerfile index 0eb67529c16..24f909d7c97 100644 --- a/.cicd/platforms/unpinned/ubuntu-18.04-unpinned.dockerfile +++ b/.cicd/platforms/unpinned/ubuntu-18.04-unpinned.dockerfile @@ -7,7 +7,8 @@ RUN apt-get update && \ bzip2 automake libbz2-dev libssl-dev doxygen graphviz libgmp3-dev \ autotools-dev libicu-dev python2.7 python2.7-dev python3 python3-dev \ autoconf libtool curl zlib1g-dev sudo ruby libusb-1.0-0-dev \ - libcurl4-gnutls-dev pkg-config patch llvm-7-dev clang-7 vim-common jq + libcurl4-gnutls-dev pkg-config patch llvm-7-dev clang-7 vim-common jq \ + libgflags-dev # build cmake RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ tar -xzf cmake-3.16.2.tar.gz && \ From ad1c1a6d52e3b44394dbcf7ea127e83052c064e7 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Wed, 29 Apr 2020 10:18:23 -0400 Subject: [PATCH 18/23] cicd --- .cicd/platforms/pinned/ubuntu-16.04-pinned.dockerfile | 2 +- libraries/CMakeLists.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.cicd/platforms/pinned/ubuntu-16.04-pinned.dockerfile b/.cicd/platforms/pinned/ubuntu-16.04-pinned.dockerfile index 2ec27938dbe..1d4a278721f 100644 --- a/.cicd/platforms/pinned/ubuntu-16.04-pinned.dockerfile +++ b/.cicd/platforms/pinned/ubuntu-16.04-pinned.dockerfile @@ -7,7 +7,7 @@ RUN apt-get update && \ libbz2-dev libssl-dev doxygen graphviz libgmp3-dev autotools-dev libicu-dev \ python2.7 python2.7-dev python3 python3-dev autoconf libtool curl zlib1g-dev \ sudo ruby libusb-1.0-0-dev libcurl4-gnutls-dev pkg-config apt-transport-https vim-common jq \ - gflags + libgflags-dev # build cmake RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ tar -xzf cmake-3.16.2.tar.gz && \ diff --git a/libraries/CMakeLists.txt b/libraries/CMakeLists.txt index 5b313b5314f..89dd018e0df 100644 --- a/libraries/CMakeLists.txt +++ b/libraries/CMakeLists.txt @@ -1,5 +1,5 @@ -set(PORTABLE ON) # rocksdb: don't use sse4.2 -set(FAIL_ON_WARNINGS OFF) # rocksdb: stop the madness: warnings change over time +option(PORTABLE CACHE ON) # rocksdb: don't use sse4.2 +option(FAIL_ON_WARNINGS CACHE OFF) # rocksdb: stop the madness: warnings change over time add_subdirectory( fc ) add_subdirectory( builtins ) From 69d067f1c6825ef1b7c4373ec9cd547cc27d989a Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Wed, 29 Apr 2020 11:55:58 -0400 Subject: [PATCH 19/23] cicd --- libraries/abieos | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/abieos b/libraries/abieos index e9a4dca327d..8e74e353d89 160000 --- a/libraries/abieos +++ b/libraries/abieos @@ -1 +1 @@ -Subproject commit e9a4dca327dcf81596f1ace10f272a77510ec8d4 +Subproject commit 8e74e353d89d743dc2e0e409a60f051d690896e5 From 47d1b9e8a15aaa793269646a1d2f329ad25326d0 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Wed, 29 Apr 2020 12:32:46 -0400 Subject: [PATCH 20/23] Don't use gflags --- .cicd/platforms/pinned/amazon_linux-2-pinned.dockerfile | 2 +- .cicd/platforms/pinned/centos-7.7-pinned.dockerfile | 2 +- .cicd/platforms/pinned/macos-10.14-pinned.sh | 2 +- .cicd/platforms/pinned/macos-10.15-pinned.sh | 2 +- .cicd/platforms/pinned/ubuntu-16.04-pinned.dockerfile | 3 +-- .cicd/platforms/pinned/ubuntu-18.04-pinned.dockerfile | 2 +- .cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile | 2 +- .cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile | 3 +-- .cicd/platforms/unpinned/macos-10.14-unpinned.sh | 2 +- .cicd/platforms/unpinned/macos-10.15-unpinned.sh | 2 +- .cicd/platforms/unpinned/ubuntu-18.04-unpinned.dockerfile | 3 +-- libraries/CMakeLists.txt | 1 + libraries/abieos | 2 +- libraries/chain_kv/CMakeLists.txt | 2 +- 14 files changed, 14 insertions(+), 16 deletions(-) diff --git a/.cicd/platforms/pinned/amazon_linux-2-pinned.dockerfile b/.cicd/platforms/pinned/amazon_linux-2-pinned.dockerfile index 623620d72f5..69e0cdec45b 100644 --- a/.cicd/platforms/pinned/amazon_linux-2-pinned.dockerfile +++ b/.cicd/platforms/pinned/amazon_linux-2-pinned.dockerfile @@ -5,7 +5,7 @@ RUN yum update -y && \ yum install -y which git sudo procps-ng util-linux autoconf automake \ libtool make bzip2 bzip2-devel openssl-devel gmp-devel libstdc++ libcurl-devel \ libusbx-devel python3 python3-devel python-devel libedit-devel doxygen \ - graphviz patch gcc gcc-c++ vim-common jq gflags-devel + graphviz patch gcc gcc-c++ vim-common jq # build cmake RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ tar -xzf cmake-3.16.2.tar.gz && \ diff --git a/.cicd/platforms/pinned/centos-7.7-pinned.dockerfile b/.cicd/platforms/pinned/centos-7.7-pinned.dockerfile index 60eaf5a8935..34c5d4caff2 100644 --- a/.cicd/platforms/pinned/centos-7.7-pinned.dockerfile +++ b/.cicd/platforms/pinned/centos-7.7-pinned.dockerfile @@ -8,7 +8,7 @@ RUN yum update -y && \ yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 doxygen \ graphviz bzip2-devel openssl-devel gmp-devel ocaml libicu-devel \ python python-devel rh-python36 file libusbx-devel \ - libcurl-devel patch vim-common jq gflags-devel + libcurl-devel patch vim-common jq # build cmake RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ tar -xzf cmake-3.16.2.tar.gz && \ diff --git a/.cicd/platforms/pinned/macos-10.14-pinned.sh b/.cicd/platforms/pinned/macos-10.14-pinned.sh index 072d703fbf8..769d0299116 100755 --- a/.cicd/platforms/pinned/macos-10.14-pinned.sh +++ b/.cicd/platforms/pinned/macos-10.14-pinned.sh @@ -2,7 +2,7 @@ set -eo pipefail VERSION=1 brew update -brew install git cmake python libtool libusb graphviz automake wget gmp pkgconfig doxygen openssl@1.1 jq gflags || : +brew install git cmake python libtool libusb graphviz automake wget gmp pkgconfig doxygen openssl@1.1 jq || : # install clang from source git clone --single-branch --branch llvmorg-10.0.0 https://github.com/llvm/llvm-project clang10 mkdir clang10/build diff --git a/.cicd/platforms/pinned/macos-10.15-pinned.sh b/.cicd/platforms/pinned/macos-10.15-pinned.sh index af9727f0798..45a35a2b638 100755 --- a/.cicd/platforms/pinned/macos-10.15-pinned.sh +++ b/.cicd/platforms/pinned/macos-10.15-pinned.sh @@ -3,7 +3,7 @@ set -eo pipefail VERSION=1 export SDKROOT="$(xcrun --sdk macosx --show-sdk-path)" brew update -brew install git cmake python libtool libusb graphviz automake wget gmp pkgconfig doxygen openssl jq gflags || : +brew install git cmake python libtool libusb graphviz automake wget gmp pkgconfig doxygen openssl jq || : # install clang from source git clone --single-branch --branch llvmorg-10.0.0 https://github.com/llvm/llvm-project clang10 mkdir clang10/build diff --git a/.cicd/platforms/pinned/ubuntu-16.04-pinned.dockerfile b/.cicd/platforms/pinned/ubuntu-16.04-pinned.dockerfile index 1d4a278721f..4e91307bca1 100644 --- a/.cicd/platforms/pinned/ubuntu-16.04-pinned.dockerfile +++ b/.cicd/platforms/pinned/ubuntu-16.04-pinned.dockerfile @@ -6,8 +6,7 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential git automake \ libbz2-dev libssl-dev doxygen graphviz libgmp3-dev autotools-dev libicu-dev \ python2.7 python2.7-dev python3 python3-dev autoconf libtool curl zlib1g-dev \ - sudo ruby libusb-1.0-0-dev libcurl4-gnutls-dev pkg-config apt-transport-https vim-common jq \ - libgflags-dev + sudo ruby libusb-1.0-0-dev libcurl4-gnutls-dev pkg-config apt-transport-https vim-common jq # build cmake RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ tar -xzf cmake-3.16.2.tar.gz && \ diff --git a/.cicd/platforms/pinned/ubuntu-18.04-pinned.dockerfile b/.cicd/platforms/pinned/ubuntu-18.04-pinned.dockerfile index 8478a5758b8..1e0a1a86fc8 100644 --- a/.cicd/platforms/pinned/ubuntu-18.04-pinned.dockerfile +++ b/.cicd/platforms/pinned/ubuntu-18.04-pinned.dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && \ autotools-dev libicu-dev python2.7 python2.7-dev python3 \ python3-dev python-configparser python-requests python-pip \ autoconf libtool g++ gcc curl zlib1g-dev sudo ruby libusb-1.0-0-dev \ - libcurl4-gnutls-dev pkg-config patch vim-common jq libgflags-dev + libcurl4-gnutls-dev pkg-config patch vim-common jq # build cmake RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ tar -xzf cmake-3.16.2.tar.gz && \ diff --git a/.cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile b/.cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile index d16baa9b461..4b0d5359d66 100644 --- a/.cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile +++ b/.cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile @@ -5,7 +5,7 @@ RUN yum update -y && \ yum install -y which git sudo procps-ng util-linux autoconf automake \ libtool make bzip2 bzip2-devel openssl-devel gmp-devel libstdc++ libcurl-devel \ libusbx-devel python3 python3-devel python-devel libedit-devel doxygen \ - graphviz clang patch llvm-devel llvm-static vim-common jq gflags-devel + graphviz clang patch llvm-devel llvm-static vim-common jq # build cmake RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ tar -xzf cmake-3.16.2.tar.gz && \ diff --git a/.cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile b/.cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile index 85eb50290d8..c33cc6fef00 100644 --- a/.cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile +++ b/.cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile @@ -8,8 +8,7 @@ RUN yum update -y && \ yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 doxygen \ graphviz bzip2-devel openssl-devel gmp-devel ocaml libicu-devel \ python python-devel rh-python36 file libusbx-devel \ - libcurl-devel patch vim-common jq llvm-toolset-7.0-llvm-devel llvm-toolset-7.0-llvm-static \ - gflags-devel + libcurl-devel patch vim-common jq llvm-toolset-7.0-llvm-devel llvm-toolset-7.0-llvm-static # build cmake RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ tar -xzf cmake-3.16.2.tar.gz && \ diff --git a/.cicd/platforms/unpinned/macos-10.14-unpinned.sh b/.cicd/platforms/unpinned/macos-10.14-unpinned.sh index b8867e0caa5..585e4366486 100755 --- a/.cicd/platforms/unpinned/macos-10.14-unpinned.sh +++ b/.cicd/platforms/unpinned/macos-10.14-unpinned.sh @@ -2,7 +2,7 @@ set -eo pipefail VERSION=1 brew update -brew install git cmake python libtool libusb graphviz automake wget gmp pkgconfig doxygen openssl@1.1 jq boost gflags || : +brew install git cmake python libtool libusb graphviz automake wget gmp pkgconfig doxygen openssl@1.1 jq boost || : # install mongodb cd ~ && curl -OL https://fastdl.mongodb.org/osx/mongodb-osx-ssl-x86_64-3.6.3.tgz tar -xzf mongodb-osx-ssl-x86_64-3.6.3.tgz && rm -f mongodb-osx-ssl-x86_64-3.6.3.tgz && \ diff --git a/.cicd/platforms/unpinned/macos-10.15-unpinned.sh b/.cicd/platforms/unpinned/macos-10.15-unpinned.sh index 36a06b37678..23d71a57473 100755 --- a/.cicd/platforms/unpinned/macos-10.15-unpinned.sh +++ b/.cicd/platforms/unpinned/macos-10.15-unpinned.sh @@ -3,7 +3,7 @@ set -eo pipefail VERSION=1 export SDKROOT="$(xcrun --sdk macosx --show-sdk-path)" brew update -brew install git cmake python libtool libusb graphviz automake wget gmp pkgconfig doxygen openssl jq boost gflags || : +brew install git cmake python libtool libusb graphviz automake wget gmp pkgconfig doxygen openssl jq boost || : # install mongoDB cd ~ curl -OL https://fastdl.mongodb.org/osx/mongodb-osx-ssl-x86_64-3.6.3.tgz diff --git a/.cicd/platforms/unpinned/ubuntu-18.04-unpinned.dockerfile b/.cicd/platforms/unpinned/ubuntu-18.04-unpinned.dockerfile index 24f909d7c97..0eb67529c16 100644 --- a/.cicd/platforms/unpinned/ubuntu-18.04-unpinned.dockerfile +++ b/.cicd/platforms/unpinned/ubuntu-18.04-unpinned.dockerfile @@ -7,8 +7,7 @@ RUN apt-get update && \ bzip2 automake libbz2-dev libssl-dev doxygen graphviz libgmp3-dev \ autotools-dev libicu-dev python2.7 python2.7-dev python3 python3-dev \ autoconf libtool curl zlib1g-dev sudo ruby libusb-1.0-0-dev \ - libcurl4-gnutls-dev pkg-config patch llvm-7-dev clang-7 vim-common jq \ - libgflags-dev + libcurl4-gnutls-dev pkg-config patch llvm-7-dev clang-7 vim-common jq # build cmake RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ tar -xzf cmake-3.16.2.tar.gz && \ diff --git a/libraries/CMakeLists.txt b/libraries/CMakeLists.txt index 89dd018e0df..ecbaf0f0071 100644 --- a/libraries/CMakeLists.txt +++ b/libraries/CMakeLists.txt @@ -1,4 +1,5 @@ option(PORTABLE CACHE ON) # rocksdb: don't use sse4.2 +option(WITH_GFLAGS CACHE OFF) # rocksdb: don't use gflags option(FAIL_ON_WARNINGS CACHE OFF) # rocksdb: stop the madness: warnings change over time add_subdirectory( fc ) diff --git a/libraries/abieos b/libraries/abieos index 8e74e353d89..537935dbb4b 160000 --- a/libraries/abieos +++ b/libraries/abieos @@ -1 +1 @@ -Subproject commit 8e74e353d89d743dc2e0e409a60f051d690896e5 +Subproject commit 537935dbb4b6ead30021548042c1d1275a31e38e diff --git a/libraries/chain_kv/CMakeLists.txt b/libraries/chain_kv/CMakeLists.txt index 90fc9f1fea7..2ae0c510314 100644 --- a/libraries/chain_kv/CMakeLists.txt +++ b/libraries/chain_kv/CMakeLists.txt @@ -3,7 +3,7 @@ file(GLOB_RECURSE HEADERS "include/*.hpp") add_library( chain_kv INTERFACE ) target_link_libraries( chain_kv - INTERFACE fc rocksdb gflags + INTERFACE fc rocksdb ) target_include_directories( chain_kv From 479ceba5002eb7e141757438b8d4d48ee1264e06 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Wed, 29 Apr 2020 15:09:48 -0400 Subject: [PATCH 21/23] rocksdb license files --- CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index f06ce41ac97..36e6d9deb92 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -217,6 +217,10 @@ configure_file(${CMAKE_SOURCE_DIR}/libraries/yubihsm/LICENSE ${CMAKE_BINARY_DIR}/licenses/eosio/LICENSE.yubihsm COPYONLY) configure_file(${CMAKE_SOURCE_DIR}/libraries/eos-vm/LICENSE ${CMAKE_BINARY_DIR}/licenses/eosio/LICENSE.eos-vm COPYONLY) +configure_file(${CMAKE_SOURCE_DIR}/libraries/rocksdb/LICENSE.Apache + ${CMAKE_BINARY_DIR}/licenses/eosio/LICENSE.rocksdb COPYONLY) +configure_file(${CMAKE_SOURCE_DIR}/libraries/rocksdb/LICENSE.leveldb + ${CMAKE_BINARY_DIR}/licenses/eosio/LICENSE.leveldb COPYONLY) install(FILES LICENSE DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ COMPONENT base) install(FILES libraries/softfloat/COPYING.txt DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.softfloat COMPONENT base) @@ -225,6 +229,8 @@ install(FILES libraries/fc/secp256k1/upstream/COPYING DESTINATION ${CMAKE_INSTAL install(FILES libraries/fc/src/network/LICENSE.go DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ COMPONENT base) install(FILES libraries/yubihsm/LICENSE DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.yubihsm COMPONENT base) install(FILES libraries/eos-vm/LICENSE DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.eos-vm COMPONENT base) +install(FILES libraries/rocksdb/LICENSE.Apache DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.rocksdb COMPONENT base) +install(FILES libraries/rocksdb/LICENSE.leveldb DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.leveldb COMPONENT base) add_custom_target(base-install COMMAND "${CMAKE_COMMAND}" --build "${CMAKE_BINARY_DIR}" From 8e2f46bf780097d177bca62524927ce510a0b986 Mon Sep 17 00:00:00 2001 From: Leo Ribeiro Date: Mon, 4 May 2020 11:02:07 -0400 Subject: [PATCH 22/23] comments out some wasmql options --- libraries/rodeos/wasm_ql.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/libraries/rodeos/wasm_ql.cpp b/libraries/rodeos/wasm_ql.cpp index 1b46de41d5f..a4dbafa519a 100644 --- a/libraries/rodeos/wasm_ql.cpp +++ b/libraries/rodeos/wasm_ql.cpp @@ -51,20 +51,20 @@ namespace b1::rodeos::wasm_ql { // todo: relax some of these limits // todo: restore max_function_section_elements to 1023 and use nodeos's hard fork struct wasm_ql_backend_options { - static constexpr std::uint32_t max_mutable_global_bytes = 1024; - static constexpr std::uint32_t max_table_elements = 1024; - static constexpr std::uint32_t max_section_elements = 8191; - static constexpr std::uint32_t max_function_section_elements = 8000; - static constexpr std::uint32_t max_import_section_elements = 1023; - static constexpr std::uint32_t max_element_segment_elements = 8191; - static constexpr std::uint32_t max_data_segment_bytes = 8191; - static constexpr std::uint32_t max_linear_memory_init = 64 * 1024; - static constexpr std::uint32_t max_func_local_bytes = 8192; - static constexpr std::uint32_t max_local_sets = 1023; - static constexpr std::uint32_t eosio_max_nested_structures = 1023; - static constexpr std::uint32_t max_br_table_elements = 8191; - static constexpr std::uint32_t max_symbol_bytes = 8191; - static constexpr std::uint32_t max_memory_offset = (33 * 1024 * 1024 - 1); + // static constexpr std::uint32_t max_mutable_global_bytes = 1024; + // static constexpr std::uint32_t max_table_elements = 1024; + // static constexpr std::uint32_t max_section_elements = 8191; + // static constexpr std::uint32_t max_function_section_elements = 8000; + // static constexpr std::uint32_t max_import_section_elements = 1023; + // static constexpr std::uint32_t max_element_segment_elements = 8191; + // static constexpr std::uint32_t max_data_segment_bytes = 8191; + // static constexpr std::uint32_t max_linear_memory_init = 64 * 1024; + // static constexpr std::uint32_t max_func_local_bytes = 8192; + // static constexpr std::uint32_t max_local_sets = 1023; + // static constexpr std::uint32_t eosio_max_nested_structures = 1023; + // static constexpr std::uint32_t max_br_table_elements = 8191; + // static constexpr std::uint32_t max_symbol_bytes = 8191; + // static constexpr std::uint32_t max_memory_offset = (33 * 1024 * 1024 - 1); static constexpr std::uint32_t max_pages = 528; // 33 MiB static constexpr std::uint32_t max_call_depth = 251; }; From affb99ae95fb88cbb8362ffd7cb8b5a78097fdb4 Mon Sep 17 00:00:00 2001 From: Todd Fleming Date: Mon, 4 May 2020 16:37:54 -0400 Subject: [PATCH 23/23] Address feedback --- libraries/CMakeLists.txt | 9 ++++++--- libraries/abieos | 2 +- plugins/state_history_plugin/state_history_plugin.cpp | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/libraries/CMakeLists.txt b/libraries/CMakeLists.txt index ecbaf0f0071..e7b08972c61 100644 --- a/libraries/CMakeLists.txt +++ b/libraries/CMakeLists.txt @@ -1,6 +1,9 @@ -option(PORTABLE CACHE ON) # rocksdb: don't use sse4.2 -option(WITH_GFLAGS CACHE OFF) # rocksdb: don't use gflags -option(FAIL_ON_WARNINGS CACHE OFF) # rocksdb: stop the madness: warnings change over time +option(PORTABLE CACHE ON) # rocksdb: don't use sse4.2 +option(WITH_GFLAGS CACHE OFF) # rocksdb: don't use gflags +option(WITH_TESTS CACHE OFF) # rocksdb: don't build this +option(WITH_TOOLS CACHE OFF) # rocksdb: don't build this +option(WITH_BENCHMARK_TOOLS CACHE OFF) # rocksdb: don't build this +option(FAIL_ON_WARNINGS CACHE OFF) # rocksdb: stop the madness: warnings change over time add_subdirectory( fc ) add_subdirectory( builtins ) diff --git a/libraries/abieos b/libraries/abieos index 537935dbb4b..79f00408511 160000 --- a/libraries/abieos +++ b/libraries/abieos @@ -1 +1 @@ -Subproject commit 537935dbb4b6ead30021548042c1d1275a31e38e +Subproject commit 79f00408511b652e7f6c9b3b406357d8fb852b9e diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 3afc0eb265e..e63fa0dc948 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -50,7 +50,7 @@ struct state_history_plugin_impl : std::enable_shared_from_this acceptor; - trace_converter trace_converter; + state_history::trace_converter trace_converter; void get_log_entry(state_history_log& log, uint32_t block_num, fc::optional& result) { if (block_num < log.begin_block() || block_num >= log.end_block())