From 228b8af4aaa3799d35f9dfc614e2acf892c11908 Mon Sep 17 00:00:00 2001 From: Stefano Date: Wed, 30 Mar 2022 13:38:03 +0200 Subject: [PATCH 01/11] First attempt to allow storing multiple types in the same BufferManager --- CMakeLists.txt | 4 +- src/examples/CB_to_matfile_example.cpp | 31 +- .../circular_buffer_record_example.cpp | 63 ++-- src/examples/telemetry_buffer_example.cpp | 48 +-- src/libYARP_telemetry/src/CMakeLists.txt | 1 + .../yarp/telemetry/experimental/Buffer.cpp | 78 +++++ .../src/yarp/telemetry/experimental/Buffer.h | 88 ++---- .../telemetry/experimental/BufferConfig.cpp | 10 +- .../telemetry/experimental/BufferConfig.h | 69 ++++- .../telemetry/experimental/BufferManager.h | 288 +++++++++++++----- .../src/yarp/telemetry/experimental/Record.h | 60 +--- 11 files changed, 456 insertions(+), 284 deletions(-) create mode 100644 src/libYARP_telemetry/src/yarp/telemetry/experimental/Buffer.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index d32b3cf..ff634b7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,8 +65,8 @@ set(CMAKE_C_FLAGS ${YARP_C_FLAGS}) set(CMAKE_CXX_FLAGS ${YARP_CXX_FLAGS}) -#### INSERT find_package(MATIO) and other libraries here -find_package(matioCpp 0.1.1 REQUIRED) +#### Dependencies +find_package(matioCpp 0.2.0 REQUIRED) find_package(Boost REQUIRED) find_package(Threads REQUIRED) diff --git a/src/examples/CB_to_matfile_example.cpp b/src/examples/CB_to_matfile_example.cpp index fec0c40..512bf4f 100644 --- a/src/examples/CB_to_matfile_example.cpp +++ b/src/examples/CB_to_matfile_example.cpp @@ -32,12 +32,12 @@ class storeData { bool closing; double period; double wait_interval; - vector > local_collection; // stores on the read-thread the values from the buffer - std::shared_ptr > > cb; // shared pointer to circular buffer + vector local_collection; // stores on the read-thread the values from the buffer + std::shared_ptr> cb; // shared pointer to circular buffer public: // constructor of the read/save class. Initialized with the shared pointer and the read period - storeData(std::shared_ptr > > _cb, const double& _period) : cb(_cb), period(_period) + storeData(std::shared_ptr> _cb, const double& _period) : cb(_cb), period(_period) { closing = false; } @@ -66,10 +66,11 @@ class storeData { // here we read and remove all elements. Each element we retrieve from the circular buffer should be removed (pop_back) to prevent reread lock_mut.lock(); - for (long unsigned int i=0; i < cb->size(); i++) + for (long unsigned int i=0; i < cb->size(); i++) { // print the elements stored in the vector (for confirmation) - for (auto f = cb->back().m_datum.begin(); f != cb->back().m_datum.end(); ++f) + auto castedDatum = any_cast>(cb->back().m_datum); + for (auto f = castedDatum.begin(); f != castedDatum.end(); ++f) cout << *f << ' '; cout << endl; // store the elements into a local collection (independent of the circular buffer to allow reading more elements) @@ -95,7 +96,7 @@ class storeData { // create copy of the local collection (this acts as a second buffer) lock_mut.lock(); - vector > _collection_copy = local_collection; + vector _collection_copy = local_collection; lock_mut.unlock(); cout << "local copy created " << endl; @@ -111,14 +112,14 @@ class storeData { return false; } // we assume the size of the data is the same for every timestep (is there any situation this would not be the case?) - int size_datum = _collection_copy[0].m_datum.size(); + int size_datum = any_cast>(_collection_copy[0].m_datum).size(); cout << "size of datum: " << size_datum << endl; // we first collapse the matrix of data into a single vector, in preparation for matioCpp convertion cout << "linearizing matrix..." << endl; for (auto& _cell : _collection_copy) { - for (auto& _el : _cell.m_datum) + for (auto& _el : any_cast>(_cell.m_datum)) { linear_matrix.push_back(_el); } @@ -131,7 +132,7 @@ class storeData { // first create timestamps vector matioCpp::Vector timestamps("timestamps"); timestamps = timestamp_vector; - + // and the structures for the actual data too vector test_data; @@ -157,11 +158,11 @@ class storeData { // and the matioCpp struct for these signals matioCpp::Struct signals("signals", signalsVect); - + // now we initialize the proto-timeseries structure vector timeSeries_data; - // the timestamps vector is stored in parallel to the signals + // the timestamps vector is stored in parallel to the signals timeSeries_data.emplace_back(timestamps); timeSeries_data.emplace_back(signals); @@ -170,8 +171,8 @@ class storeData { // and finally we write the file matioCpp::File file = matioCpp::File::Create("test_cb.mat"); file.write(timeSeries); - - return true; + + return true; } bool close() @@ -199,7 +200,7 @@ int main() /**************************************************/ // Initialization of our Buffer (3 entries, type vector) - Buffer cb(3); + Buffer cb(3); double period = 5; // period for the reading of the buffer // Initialization of our reading and saving to file class - uses the shared-pointer for reading the circular buffer @@ -215,7 +216,7 @@ int main() // we lock before we populate the circular buffer to prevent conflicts with reading lock_mut.lock(); - cb.push_back(Record(yarp::os::Time::now(), vec)); + cb.push_back(Record({yarp::os::Time::now(), vec})); lock_mut.unlock(); // user input -> say "no" to close the loop and generate the mat file diff --git a/src/examples/circular_buffer_record_example.cpp b/src/examples/circular_buffer_record_example.cpp index dcc80ea..3f865ae 100644 --- a/src/examples/circular_buffer_record_example.cpp +++ b/src/examples/circular_buffer_record_example.cpp @@ -27,120 +27,111 @@ using namespace yarp::telemetry::experimental; Network yarp; std::cout<<"XXXXXXXX CIRCULAR BUFFER OF INT XXXXXXXX"< structures. - boost::circular_buffer> cb_i(3); + // Create a circular buffer with a capacity for 3 Record structures. + boost::circular_buffer cb_i(3); size_t total_payload = 0; cout<<"The capacity is: "<(yarp::os::Time::now(), vector{ 1 })); + cb_i.push_back(Record({yarp::os::Time::now(), vector{ 1 }})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); - cb_i.push_back(Record(yarp::os::Time::now(), vector{ 2 })); + cb_i.push_back(Record({yarp::os::Time::now(), vector{ 2 }})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); - cb_i.push_back(Record(yarp::os::Time::now(), vector{ 3 })); + cb_i.push_back(Record({yarp::os::Time::now(), vector{ 3 }})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); cout<<"The capacity is: "<>(c_el.m_datum)[0]<(yarp::os::Time::now(), vector{ 4 })); + cb_i.push_back(Record({yarp::os::Time::now(), vector{ 4 }})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); - cb_i.push_back(Record(yarp::os::Time::now(), vector{ 5 })); + cb_i.push_back(Record({yarp::os::Time::now(), vector{ 5 }})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); cout<<"The capacity is: "<>(c_el.m_datum)[0]< structures. - boost::circular_buffer> cb_d(3); + // Create a circular buffer with a capacity for 3 Record structures. + boost::circular_buffer cb_d(3); - total_payload = 0; cout<<"The capacity is: "<(yarp::os::Time::now(), vector{ 0.1 })); + cb_d.push_back(Record({yarp::os::Time::now(), vector{ 0.1 }})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); - cb_d.push_back(Record(yarp::os::Time::now(), vector{ 0.2 })); + cb_d.push_back(Record({yarp::os::Time::now(), vector{ 0.2 }})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); - cb_d.push_back(Record(yarp::os::Time::now(), vector{ 0.3 })); + cb_d.push_back(Record({yarp::os::Time::now(), vector{ 0.3 }})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); cout<<"The capacity is: "<>(c_el.m_datum)[0]<(yarp::os::Time::now(), vector{ 0.4 })); + cb_d.push_back(Record({yarp::os::Time::now(), vector{ 0.4 }})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); - cb_d.push_back(Record(yarp::os::Time::now(), vector{ 0.5 })); + cb_d.push_back(Record({yarp::os::Time::now(), vector{ 0.5 }})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); cout<<"The capacity is: "<>(c_el.m_datum)[0]<> structures. - boost::circular_buffer> cb_v(3); + // Create a circular buffer with a capacity for 3 Record structures. + boost::circular_buffer cb_v(3); - total_payload = 0; cout<<"The capacity is: "<(yarp::os::Time::now(), vector{0.1, 0.2, 0.3})); + cb_v.push_back(Record({yarp::os::Time::now(), vector{0.1, 0.2, 0.3}})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); - cb_v.push_back(Record(yarp::os::Time::now(), vector{0.3, 0.4, 0.5})); + cb_v.push_back(Record({yarp::os::Time::now(), vector{0.3, 0.4, 0.5}})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); - cb_v.push_back(Record(yarp::os::Time::now(), vector{0.6, 0.7, 0.8})); + cb_v.push_back(Record({yarp::os::Time::now(), vector{0.6, 0.7, 0.8}})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); cout<<"The capacity is: "<>(c_el.m_datum)) { cout<(yarp::os::Time::now(), vector{0.9, 1.0, 1.1})); + cb_v.push_back(Record({yarp::os::Time::now(), vector{0.9, 1.0, 1.1}})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); - cb_v.push_back(Record(yarp::os::Time::now(), vector{1.2, 1.3, 1.4})); + cb_v.push_back(Record({yarp::os::Time::now(), vector{1.2, 1.3, 1.4}})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); cout<<"The capacity is: "<>(c_el.m_datum)) { cout< cb_i(3); + yarp::telemetry::experimental::Buffer cb_i(3); cout<<"The space available is: "<(yarp::os::Time::now(), vector{ 1 })); + cb_i.push_back(Record({yarp::os::Time::now(), vector{ 1 }})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); - cb_i.push_back(Record(yarp::os::Time::now(), vector{ 2 })); + cb_i.push_back(Record({yarp::os::Time::now(), vector{ 2 }})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); - cb_i.push_back(Record(yarp::os::Time::now(), vector{ 3 })); + cb_i.push_back(Record({yarp::os::Time::now(), vector{ 3 }})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); cout<<"The space available is: "<>(c_el.m_datum)[0]<(yarp::os::Time::now(), vector{ 4 })); + cb_i.push_back(Record({yarp::os::Time::now(), vector{ 4 }})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); - cb_i.push_back(Record(yarp::os::Time::now(), vector{ 5 })); + cb_i.push_back(Record({yarp::os::Time::now(), vector{ 5 }})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); cout<<"The space available is: "<>(c_el.m_datum)[0]< cb_d(3); + yarp::telemetry::experimental::Buffer cb_d(3); cout<<"The space available is: "<(yarp::os::Time::now(), vector{ 0.1 })); + cb_d.push_back(Record({yarp::os::Time::now(), vector{ 0.1 }})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); - cb_d.push_back(Record(yarp::os::Time::now(), vector{ 0.2 })); + cb_d.push_back(Record({yarp::os::Time::now(), vector{ 0.2 }})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); - cb_d.push_back(Record(yarp::os::Time::now(), vector{ 0.3 })); + cb_d.push_back(Record({yarp::os::Time::now(), vector{ 0.3 }})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); cout<<"The space available is: "<>(c_el.m_datum)[0]<(yarp::os::Time::now(), vector{ 0.4 })); + cb_d.push_back(Record({yarp::os::Time::now(), vector{ 0.4 }})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); - cb_d.push_back(Record(yarp::os::Time::now(), vector{ 0.5 })); + cb_d.push_back(Record({yarp::os::Time::now(), vector{ 0.5 }})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); cout<<"The space available is: "<>(c_el.m_datum)[0]< cb_v(3); + yarp::telemetry::experimental::Buffer cb_v(3); cout<<"The space available is: "<(yarp::os::Time::now(), vector{0.1, 0.2, 0.3})); + cb_v.push_back(Record({yarp::os::Time::now(), vector{0.1, 0.2, 0.3}})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); - cb_v.push_back(Record(yarp::os::Time::now(), vector{0.3, 0.4, 0.5})); + cb_v.push_back(Record({yarp::os::Time::now(), vector{0.3, 0.4, 0.5}})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); - cb_v.push_back(Record(yarp::os::Time::now(), vector{0.6, 0.7, 0.8})); + cb_v.push_back(Record({yarp::os::Time::now(), vector{0.6, 0.7, 0.8}})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); cout<<"The space available is: "<>(c_el.m_datum)) { cout<(yarp::os::Time::now(), vector{0.9, 1.0, 1.1})); + cb_v.push_back(Record({yarp::os::Time::now(), vector{0.9, 1.0, 1.1}})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); - cb_v.push_back(Record(yarp::os::Time::now(), vector{1.2, 1.3, 1.4})); + cb_v.push_back(Record({yarp::os::Time::now(), vector{1.2, 1.3, 1.4}})); std::this_thread::sleep_for(std::chrono::milliseconds(200)); @@ -122,7 +122,7 @@ using namespace yarp::telemetry::experimental; cout<<"The circular buffer contains:"<>(c_el.m_datum)) { cout< + +yarp::telemetry::experimental::Buffer::Buffer(size_t num_elements): m_buffer_ptr(std::make_shared>(num_elements)) +{ + +} + +void yarp::telemetry::experimental::Buffer::push_back(const Record &elem) +{ + m_buffer_ptr->push_back(elem); +} + +void yarp::telemetry::experimental::Buffer::push_back(Record &&elem) +{ + m_buffer_ptr->push_back(std::move(elem)); +} + +size_t yarp::telemetry::experimental::Buffer::getBufferFreeSpace() const { + return m_buffer_ptr->capacity() - m_buffer_ptr->size(); +} + +size_t yarp::telemetry::experimental::Buffer::size() const { + return m_buffer_ptr->size(); +} + +size_t yarp::telemetry::experimental::Buffer::capacity() const { + return m_buffer_ptr->capacity(); + +} + +bool yarp::telemetry::experimental::Buffer::empty() const { + return m_buffer_ptr->empty(); +} + +void yarp::telemetry::experimental::Buffer::resize(size_t new_size) { + return m_buffer_ptr->resize(new_size); +} + +void yarp::telemetry::experimental::Buffer::set_capacity(size_t new_size) { + return m_buffer_ptr->set_capacity(new_size); +} + +bool yarp::telemetry::experimental::Buffer::full() const { + return m_buffer_ptr->full(); +} + +yarp::telemetry::experimental::Buffer::const_iterator yarp::telemetry::experimental::Buffer::begin() const noexcept { + return m_buffer_ptr->begin(); +} + +yarp::telemetry::experimental::Buffer::iterator yarp::telemetry::experimental::Buffer::end() noexcept { + return m_buffer_ptr->end(); +} + +yarp::telemetry::experimental::Buffer::iterator yarp::telemetry::experimental::Buffer::begin() noexcept { + return m_buffer_ptr->begin(); +} + +yarp::telemetry::experimental::Buffer::const_iterator yarp::telemetry::experimental::Buffer::end() const noexcept { + return m_buffer_ptr->end(); +} + +void yarp::telemetry::experimental::Buffer::clear() noexcept { + return m_buffer_ptr->clear(); + +} + +std::shared_ptr > yarp::telemetry::experimental::Buffer::getBufferSharedPtr() const { + return m_buffer_ptr; +} diff --git a/src/libYARP_telemetry/src/yarp/telemetry/experimental/Buffer.h b/src/libYARP_telemetry/src/yarp/telemetry/experimental/Buffer.h index eb70d2f..8f88726 100644 --- a/src/libYARP_telemetry/src/yarp/telemetry/experimental/Buffer.h +++ b/src/libYARP_telemetry/src/yarp/telemetry/experimental/Buffer.h @@ -18,15 +18,14 @@ namespace yarp::telemetry::experimental { /** - * @brief A class to represent the buffer of yarp::telemetry::experimental::Record. + * @brief A class to represent the buffer of yarp::telemetry::experimental::Record. * */ -template class Buffer { public: - using iterator = typename boost::circular_buffer>::iterator; - using const_iterator = typename boost::circular_buffer>::const_iterator; + using iterator = typename boost::circular_buffer::iterator; + using const_iterator = typename boost::circular_buffer::const_iterator; Buffer() = default; @@ -48,18 +47,18 @@ class Buffer { * @brief Copy assignment operator. * * @param[in] _other Buffer to be copied. - * @return Buffer& Resulting Buffer. + * @return Buffer& Resulting Buffer. */ - Buffer& operator=(const Buffer& _other) = default; + Buffer& operator=(const Buffer& _other) = default; /** * @brief Move assignment operator. * * @param[in] _other Buffer to be moved. - * @return Buffer& Resulting Buffer. + * @return Buffer& Resulting Buffer. */ - Buffer& operator=(Buffer&& _other) noexcept = default; + Buffer& operator=(Buffer&& _other) noexcept = default; /** * @brief Destroy the Buffer object @@ -71,150 +70,113 @@ class Buffer { * * @param[in] num_elements Number of elements of the Buffer to be constructed. */ - explicit Buffer(size_t num_elements): m_buffer_ptr(std::make_shared>>(num_elements)) - { - - } + Buffer(size_t num_elements); /** * @brief Push back copying the new Record. * * @param[in] elem Record to be copied */ - inline void push_back(const Record &elem) - { - m_buffer_ptr->push_back(elem); - } + void push_back(const Record &elem); /** * @brief Push back moving the new Record. * * @param[in] elem Record to be moved. */ - inline void push_back(Record&& elem) - { - m_buffer_ptr->push_back(std::move(elem)); - } + void push_back(Record&& elem); /** * @brief Get the Buffer free space. * * @return size_t The free space expressed in bytes. */ - size_t getBufferFreeSpace() const { - return m_buffer_ptr->capacity() - m_buffer_ptr->size(); - } + size_t getBufferFreeSpace() const; /** * @brief Get the size of the Buffer. * * @return size_t The size of the buffer */ - size_t size() const { - return m_buffer_ptr->size(); - } + size_t size() const; /** * @brief Get the capacity of Buffer * * @return size_t The capacity of the buffer. */ - size_t capacity() const { - return m_buffer_ptr->capacity(); - - } + size_t capacity() const; /** * @brief Return true if the Buffer is empty, false otherwise. * */ - bool empty() const { - return m_buffer_ptr->empty(); - } + bool empty() const; /** * @brief Resize the Buffer. * * @param[in] new_size The new size to be resized to. */ - void resize(size_t new_size) { - return m_buffer_ptr->resize(new_size); - } + void resize(size_t new_size); /** * @brief Change the capacity of the Buffer. * * @param[in] new_size The new size. */ - void set_capacity(size_t new_size) { - return m_buffer_ptr->set_capacity(new_size); - } + void set_capacity(size_t new_size); /** * @brief Return true if the Buffer is full, false otherwise. * */ - bool full() const { - return m_buffer_ptr->full(); - } + bool full() const; /** * @brief Return the iterator referred to the begin of the Buffer. * * @return iterator */ - iterator begin() noexcept { - return m_buffer_ptr->begin(); - } + iterator begin() noexcept; /** * @brief Return the iterator referred to the end of the Buffer. * * @return iterator */ - iterator end() noexcept { - return m_buffer_ptr->end(); - } + iterator end() noexcept; /** * @brief Return the const iterator referred to the begin of the Buffer. * * @return const_iterator */ - const_iterator begin() const noexcept { - return m_buffer_ptr->begin(); - } + const_iterator begin() const noexcept; /** * @brief Return the const iterator referred to the end of the Buffer. * * @return const_iterator */ - const_iterator end() const noexcept { - return m_buffer_ptr->end(); - } + const_iterator end() const noexcept; /** * @brief Clear the content of the buffer. * */ - void clear() noexcept { - return m_buffer_ptr->clear(); - - } + void clear() noexcept; /** * @brief Get the Buffer shared_ptr object. * - * @return std::shared_ptr>> + * @return std::shared_ptr> */ - std::shared_ptr>> getBufferSharedPtr() const { - return m_buffer_ptr; - } + std::shared_ptr> getBufferSharedPtr() const; private: - std::shared_ptr>> m_buffer_ptr; + std::shared_ptr> m_buffer_ptr; }; diff --git a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.cpp b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.cpp index 287bdc7..8fb336a 100644 --- a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.cpp +++ b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.cpp @@ -24,10 +24,11 @@ namespace matioCpp { namespace yarp::telemetry::experimental { ChannelInfo::ChannelInfo(const std::string& name, - const dimensions_t& dimensions, + const dimensions_t& dimensions, const std::string &type, const elements_names_t& elements_names) : name(name), dimensions(dimensions), + type_name(type), elements_names(elements_names) { const unsigned int elements = std::accumulate(dimensions.begin(), @@ -42,9 +43,10 @@ namespace yarp::telemetry::experimental { } } - ChannelInfo::ChannelInfo(const std::string& name, const dimensions_t& dimensions) + ChannelInfo::ChannelInfo(const std::string& name, const dimensions_t& dimensions, const std::string &type) : name(name), - dimensions(dimensions) + dimensions(dimensions), + type_name(type) { const unsigned int elements = std::accumulate(dimensions.begin(), dimensions.end(), @@ -60,6 +62,7 @@ namespace yarp::telemetry::experimental { { j = nlohmann::json{{"name", info.name}, {"dimensions", info.dimensions}, + {"type_name", info.type_name}, {"elements_names", info.elements_names}, }; } @@ -68,6 +71,7 @@ namespace yarp::telemetry::experimental { { j.at("name").get_to(info.name); j.at("dimensions").get_to(info.dimensions); + j.at("type_name").get_to(info.type_name); j.at("elements_names").get_to(info.elements_names); } diff --git a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.h b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.h index e2bff16..06e59cb 100644 --- a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.h +++ b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.h @@ -9,6 +9,9 @@ #ifndef YARP_TELEMETRY_BUFFER_CONFIG_H #define YARP_TELEMETRY_BUFFER_CONFIG_H +#include +#include + #include #include #include @@ -19,13 +22,17 @@ namespace yarp::telemetry::experimental { using dimensions_t = std::vector; using elements_names_t = std::vector; + /** * @brief Struct representing a channel(variable) in terms of * name and dimensions and names of the each element of a variable. */ struct YARP_telemetry_API ChannelInfo { + + static constexpr char type_name_not_set_tag[] = "type_name_not_set"; std::string name; /**< Name of the channel */ dimensions_t dimensions; /**< Dimension of the channel */ + std::string type_name{type_name_not_set_tag}; /**< The name of the type of data used in the channel. */ elements_names_t elements_names; /**< Vector containing the names of each element of the channel */ /** @@ -38,12 +45,52 @@ struct YARP_telemetry_API ChannelInfo { * the elements associated to the channel. * @param name name of the channel. * @param dimensions dimension associated to the channel. + * @param type the human readable type of the buffer (e.g. "std::vector") * @param elements_names Vector containing the names of each element of the channel. */ ChannelInfo(const std::string& name, const dimensions_t& dimensions, + const std::string& type, const elements_names_t& elements_names); + /** + * @brief Construct a ChannelInfo from name, dimensions and a vector containing the name of + * the elements associated to the channel. + * @param name name of the channel. + * @param dimensions dimension associated to the channel. + * @param elements_names Vector containing the names of each element of the channel. + */ + template + explicit ChannelInfo(const std::string& name, + const dimensions_t& dimensions, + const elements_names_t& elements_names) + : ChannelInfo(name, dimensions, getTypeName(), elements_names){ + } + + /** + * @brief Construct a ChannelInfo from name and dimensions. + * @param name name of the channel. + * @param dimensions dimension associated to the channel. + * @param type the human readable type of the buffer (e.g. "std::vector") + * @note If the constructor is called the elements_names are set as + * elements_names = [element_0, element_1, ..., element_n], where n is given by the + * product of the dimensions. + */ + ChannelInfo(const std::string& name, const dimensions_t& dimensions, const std::string& type); + + /** + * @brief Construct a ChannelInfo from name and dimensions. + * @param name name of the channel. + * @param dimensions dimension associated to the channel. + * @note If the constructor is called the elements_names are set as + * elements_names = [element_0, element_1, ..., element_n], where n is given by the + * product of the dimensions. + */ + template + explicit ChannelInfo(const std::string& name, const dimensions_t& dimensions) + : ChannelInfo(name, dimensions, getTypeName()){ + } + /** * @brief Construct a ChannelInfo from name and dimensions. * @param name name of the channel. @@ -52,7 +99,27 @@ struct YARP_telemetry_API ChannelInfo { * elements_names = [element_0, element_1, ..., element_n], where n is given by the * product of the dimensions. */ - ChannelInfo(const std::string& name, const dimensions_t& dimensions); + ChannelInfo(const std::string& name, const dimensions_t& dimensions) + : ChannelInfo(name, dimensions, type_name_not_set_tag){ + } + + /** + * @brief Get the type name as string + */ + template + static std::string getTypeName(const T& someInput) + { + return boost::core::demangle(typeid(someInput).name()); + } + + /** + * @brief Get the type name as string + */ + template + static std::string getTypeName() + { + return boost::core::demangle(typeid(T).name()); + } }; /** diff --git a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h index adc9785..beaaa90 100644 --- a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h +++ b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h @@ -29,6 +29,7 @@ #include #include #include +#include #ifndef __has_include static_assert(false, "__has_include not supported"); @@ -53,17 +54,30 @@ namespace yarp::telemetry::experimental { * info(e.g. dimensions) used by the yarp::telemetry::experimental::BufferManager * */ -template struct BufferInfo { - Buffer m_buffer; + Buffer m_buffer; std::mutex m_buff_mutex; dimensions_t m_dimensions; + std::string m_type_name; elements_names_t m_elements_names; + std::function m_matioCpp_create_storage; + std::function m_append_function; BufferInfo() = default; - BufferInfo(const BufferInfo& other) : m_buffer(other.m_buffer), m_dimensions(other.m_dimensions) { + BufferInfo(const BufferInfo& other) + : m_buffer(other.m_buffer), + m_dimensions(other.m_dimensions), + m_type_name(other.m_type_name), + m_elements_names(other.m_elements_names), + m_append_function(other.m_append_function){ } - BufferInfo(BufferInfo&& other) : m_buffer(std::move(other.m_buffer)), m_dimensions(std::move(other.m_dimensions)) { + + BufferInfo(BufferInfo&& other) + : m_buffer(std::move(other.m_buffer)), + m_dimensions(std::move(other.m_dimensions)), + m_type_name(std::move(other.m_type_name)), + m_elements_names(std::move(other.m_elements_names)), + m_append_function(std::move(other.m_append_function)){ } }; /** @@ -75,7 +89,7 @@ struct BufferInfo { * to/from a json file. * */ -template +template class BufferManager { public: @@ -85,7 +99,7 @@ class BufferManager { * */ BufferManager() { - m_tree = std::make_shared>>(); + m_tree = std::make_shared>(); } /** @@ -95,7 +109,7 @@ class BufferManager { * @param[in] _bufferConfig The struct containing the configuration for the BufferManager. */ BufferManager(const BufferConfig& _bufferConfig) { - m_tree = std::make_shared>>(); + m_tree = std::make_shared>(); bool ok = configure(_bufferConfig); assert(ok); } @@ -176,7 +190,6 @@ class BufferManager { BufferConfig getBufferConfig() const { return m_bufferConfig; } - /** * @brief Set the file name that will be created by the BufferManager. * @@ -248,9 +261,27 @@ class BufferManager { * @return true on success, false otherwise. */ bool addChannel(const ChannelInfo& channel) { - auto buffInfo = std::make_shared>(); - buffInfo->m_buffer = Buffer(m_bufferConfig.n_samples); + auto buffInfo = std::make_shared(); + buffInfo->m_buffer = Buffer(m_bufferConfig.n_samples); buffInfo->m_dimensions = channel.dimensions; + + bool userDidNotSetTypeName = channel.type_name == ChannelInfo::type_name_not_set_tag || channel.type_name.empty(); + + if (std::is_same_v && userDidNotSetTypeName) + { + std::cerr << "Unable to determine the type of the channel " << channel.name << std::endl; + return false; + } + + if (userDidNotSetTypeName) + { + buffInfo->m_type_name = ChannelInfo::getTypeName>();; + } + else + { + buffInfo->m_type_name = channel.type_name; + } + buffInfo->m_elements_names = channel.elements_names; const bool ok = addLeaf(channel.name, buffInfo, m_tree); @@ -283,12 +314,55 @@ class BufferManager { * The var_name channels must exist, otherwise an exception is thrown. * * @param[in] elem The element to be pushed(via copy) in the channel. + * @param[in] ts The timestamp of the element to be pushed. * @param[in] var_name The name of the channel. */ - inline void push_back(matioCpp::Span elem, const std::string& var_name) + template//-----------------------------Here I need to mak sure that T is a numeric quantity + inline void push_back(matioCpp::Span elem, double ts, const std::string& var_name) { - assert(m_nowFunction != nullptr); - this->push_back(elem, m_nowFunction(), var_name); + auto leaf = getLeaf(var_name, m_tree).lock(); + if (leaf == nullptr) + { + throw std::invalid_argument("The channel " + var_name + " does not exist."); + } + + auto bufferInfo = leaf->getValue(); + assert(bufferInfo != nullptr); + + if (bufferInfo->m_type_name != ChannelInfo::getTypeName>()) + { + std::cout << "Cannot push to the channel " << var_name + << ". Expected type: " << bufferInfo->m_type_name + << ". Input type: " << ChannelInfo::getTypeName>(); + return; + } + + std::scoped_lock lock{ bufferInfo->m_buff_mutex }; + //Create the saving functions if they were not present already + if (!bufferInfo->m_matioCpp_create_storage) + { + bufferInfo->m_matioCpp_create_storage = [](const std::string& name, const dimensions_t& bufferInfoDimensions, size_t num_instants) -> matioCpp::Variable { + dimensions_t fullDimensions = bufferInfoDimensions; + fullDimensions.push_back(num_instants); + return matioCpp::MultiDimensionalArray(name, bufferInfoDimensions); + }; + } + if (!bufferInfo->m_append_function) + { + bufferInfo->m_append_function = [](const std::any& input, size_t instantIndex, matioCpp::Variable& concatenated) { + matioCpp::MultiDimensionalArray concatenatedCasted = concatenated.asMultiDimensionalArray(); + + const std::vector& inputCasted = std::any_cast>(input); + + for (size_t i = 0; i < inputCasted.size(); ++i) + { + concatenatedCasted[{i, 0, instantIndex}] = inputCasted[i]; + } + }; + } + + assert(elem.size() == bufferInfo->m_dimensions[0] * bufferInfo->m_dimensions[1]); + bufferInfo->m_buffer.push_back({ts, std::vector(elem.begin(), elem.end())}); } /** @@ -299,17 +373,50 @@ class BufferManager { * @param[in] ts The timestamp of the element to be pushed. * @param[in] var_name The name of the channel. */ - inline void push_back(matioCpp::Span elem, double ts, const std::string& var_name) + template//-----------------------------Here I need to mak sure that T is a numeric quantity + inline void push_back(const std::initializer_list& elem, double ts, const std::string& var_name) { auto leaf = getLeaf(var_name, m_tree).lock(); - assert(leaf != nullptr); - + if (leaf == nullptr) + { + throw std::invalid_argument("The channel " + var_name + " does not exist."); + } auto bufferInfo = leaf->getValue(); assert(bufferInfo != nullptr); - assert(elem.size() == bufferInfo->m_dimensions[0] * bufferInfo->m_dimensions[1]); + if (bufferInfo->m_type_name != ChannelInfo::getTypeName>()) + { + std::cout << "Cannot push to the channel " << var_name + << ". Expected type: " << bufferInfo->m_type_name + << ". Input type: " << ChannelInfo::getTypeName>(); + return; + } + std::scoped_lock lock{ bufferInfo->m_buff_mutex }; - bufferInfo->m_buffer.push_back(Record(ts, elem)); + //Create the saving functions if they were not present already + if (!bufferInfo->m_matioCpp_create_storage) + { + bufferInfo->m_matioCpp_create_storage = [](const std::string& name, const dimensions_t& bufferInfoDimensions, size_t num_instants) -> matioCpp::Variable { + dimensions_t fullDimensions = bufferInfoDimensions; + fullDimensions.push_back(num_instants); + return matioCpp::MultiDimensionalArray(name, bufferInfoDimensions); + }; + } + if (!bufferInfo->m_append_function) + { + bufferInfo->m_append_function = [](const std::any& input, size_t instantIndex, matioCpp::Variable& concatenated) { + matioCpp::MultiDimensionalArray concatenatedCasted = concatenated.asMultiDimensionalArray(); + + const std::vector& inputCasted = std::any_cast>(input); + + for (size_t i = 0; i < inputCasted.size(); ++i) + { + concatenatedCasted[{i, 0, instantIndex}] = inputCasted[i]; + } + }; + } + assert(elem.size() == bufferInfo->m_dimensions[0] * bufferInfo->m_dimensions[1]); + bufferInfo->m_buffer.push_back({ts, std::vector(elem.begin(), elem.end())}); } /** @@ -319,53 +426,72 @@ class BufferManager { * @param[in] elem The element to be pushed(via copy) in the channel. * @param[in] var_name The name of the channel. */ + template//-----------------------------Here I need to mak sure that T is a numeric quantity inline void push_back(const std::initializer_list& elem, const std::string& var_name) { - assert(m_nowFunction != nullptr); - this->push_back(elem, m_nowFunction(), var_name); + push_back(elem, m_nowFunction(), var_name); } /** * @brief Push a new element in the var_name channel. * The var_name channels must exist, otherwise an exception is thrown. * - * @param[in] elem The element to be pushed(via copy) in the channel. + * @param[in] elem The element to be pushed in the channel. * @param[in] ts The timestamp of the element to be pushed. * @param[in] var_name The name of the channel. */ - inline void push_back(const std::initializer_list& elem, double ts, const std::string& var_name) + template//----------------------------Here I probably need to avoid ambiguity with the two methods above. std::vector!!!!! + inline void push_back(const T& elem, double ts, const std::string& var_name) { - auto leaf = getLeaf(var_name, m_tree).lock(); - assert(leaf != nullptr); + static_assert(matioCpp::is_make_variable_callable::value, "The selected type cannot be used with matioCpp."); + auto leaf = getLeaf(var_name, m_tree).lock(); + if (leaf == nullptr) + { + throw std::invalid_argument("The channel " + var_name + " does not exist."); + } auto bufferInfo = leaf->getValue(); assert(bufferInfo != nullptr); - assert(elem.size() == bufferInfo->m_dimensions[0] * bufferInfo->m_dimensions[1]); + if (bufferInfo->m_type_name != ChannelInfo::getTypeName()) + { + std::cout << "Cannot push to the channel " << var_name + << ". Expected type: " << bufferInfo->m_type_name + << ". Input type: " << ChannelInfo::getTypeName(); + return; + } + std::scoped_lock lock{ bufferInfo->m_buff_mutex }; - bufferInfo->m_buffer.push_back(Record(ts, elem)); + //Create the saving functions if they were not present already + if (!bufferInfo->m_matioCpp_create_storage) + { + bufferInfo->m_matioCpp_create_storage = [](const std::string& name, const dimensions_t&, size_t num_instants) -> matioCpp::Variable { + return matioCpp::CellArray(name, {num_instants, 1}); + }; + } + + if (!bufferInfo->m_append_function) + { + bufferInfo->m_append_function = [](const std::any& input, size_t instantIndex, matioCpp::Variable& concatenated) { + matioCpp::CellArray concatenatedCasted = concatenated.asCellArray(); + concatenatedCasted.setElement(instantIndex, matioCpp::make_variable("element", std::any_cast(input))); + }; + } + + bufferInfo->m_buffer.push_back({ts, elem}); } /** * @brief Push a new element in the var_name channel. * The var_name channels must exist, otherwise an exception is thrown. * - * @param[in] elem The element to be pushed(via move) in the channel. + * @param[in] elem The element to be pushed in the channel. * @param[in] var_name The name of the channel. */ - inline void push_back(std::vector&& elem, const std::string& var_name) + template + inline void push_back(const T& elem, const std::string& var_name) { - auto leaf = getLeaf(var_name, m_tree).lock(); - assert(leaf != nullptr); - - auto bufferInfo = leaf->getValue(); - assert(bufferInfo != nullptr); - - assert(m_nowFunction != nullptr); - - assert(elem.size() == bufferInfo->m_dimensions[0] * bufferInfo->m_dimensions[1]); - std::scoped_lock lock{ bufferInfo->m_buff_mutex }; - bufferInfo->m_buffer.push_back(Record(m_nowFunction(), std::move(elem))); + push_back(elem, m_nowFunction(), var_name); } /** @@ -435,6 +561,7 @@ class BufferManager { static double DefaultClock() { return std::chrono::duration(std::chrono::system_clock::now().time_since_epoch()).count(); } + void periodicSave() { std::unique_lock lk_cv(m_mutex_cv); @@ -452,7 +579,7 @@ class BufferManager { } matioCpp::Struct createTreeStruct(const std::string& node_name, - std::shared_ptr>> tree_node, + std::shared_ptr> tree_node, bool flush_all) { const auto& children = tree_node->getChildren(); if (children.size() == 0) { @@ -467,8 +594,25 @@ class BufferManager { return tmp; } + matioCpp::Variable concatenateElements(std::shared_ptr buffInfo) const + { + // the number of timesteps is the size of our collection + auto num_timesteps = buffInfo->m_buffer.size(); + + matioCpp::Variable out = buffInfo->m_matioCpp_create_storage("data", buffInfo->m_dimensions, num_timesteps); + + size_t i = 0; + for (auto& _cell : buffInfo->m_buffer) { + buffInfo->m_append_function(_cell.m_datum, i, out); + ++i; + } + assert(i == buffInfo->m_buffer.size()); + + return out; + } + matioCpp::Struct createElementStruct(const std::string& var_name, - std::shared_ptr> buffInfo, + std::shared_ptr buffInfo, bool flush_all) const { assert(buffInfo); @@ -484,57 +628,37 @@ class BufferManager { return matioCpp::Struct(); } - std::vector linear_matrix; - std::vector timestamp_vector; - // the number of timesteps is the size of our collection auto num_timesteps = buffInfo->m_buffer.size(); + // We concatenate all the data of the buffer into a single variable + matioCpp::Variable data = concatenateElements(buffInfo); - // we first collapse the matrix of data into a single vector, in preparation for matioCpp convertion - // TODO put mutexes here.... + //We construct the timestamp vector + matioCpp::Vector timestamps("timestamps", num_timesteps); + size_t i = 0; for (auto& _cell : buffInfo->m_buffer) { - for (auto& _el : _cell.m_datum) { - linear_matrix.push_back(_el); - } - timestamp_vector.push_back(_cell.m_ts); + timestamps[i] = _cell.m_ts; + ++i; } - buffInfo->m_buffer.clear(); - - // now we start the matioCpp convertion process + assert(i == buffInfo->m_buffer.size()); - // first create timestamps vector - matioCpp::Vector timestamps("timestamps"); - timestamps = timestamp_vector; + //Clear the buffer, we don't need it anymore + buffInfo->m_buffer.clear(); - // and the structures for the actual data too + //Create the set of variables to be used in the output struct std::vector test_data; // now we create the vector for the dimensions // The first two dimensions are the r and c of the sample, the number of sample has to be the last dimension. std::vector dimensions_data_vect {static_cast(buffInfo->m_dimensions[0]), - static_cast(buffInfo->m_dimensions[1]), - static_cast(num_timesteps)}; - matioCpp::Vector dimensions_data("dimensions"); - dimensions_data = dimensions_data_vect; - - // now we populate the matioCpp matrix - matioCpp::MultiDimensionalArray out("data", - {buffInfo->m_dimensions[0] , - buffInfo->m_dimensions[1], - static_cast(num_timesteps) }, - linear_matrix.data()); - - std::vector elements_names_vector; - for (const auto& str : buffInfo->m_elements_names) { - elements_names_vector.emplace_back(matioCpp::String("useless_name",str)); - } - matioCpp::CellArray elements_names_list("elements_names", { buffInfo->m_elements_names.size(), 1 }, elements_names_vector); + static_cast(buffInfo->m_dimensions[1]), + static_cast(num_timesteps)}; - test_data.emplace_back(out); // Data + test_data.emplace_back(data); // Data - test_data.emplace_back(dimensions_data); // dimensions vector - test_data.emplace_back(elements_names_list); // elements names + test_data.emplace_back(matioCpp::make_variable("dimensions", dimensions_data_vect)); // dimensions vector + test_data.emplace_back(matioCpp::make_variable("elements_names", buffInfo->m_elements_names)); // elements names test_data.emplace_back(matioCpp::String("name", var_name)); // name of the signal test_data.emplace_back(timestamps); @@ -573,12 +697,12 @@ class BufferManager { m_description_cell_array = description_list; } - void resize(size_t new_size, std::shared_ptr>> node) { + void resize(size_t new_size, std::shared_ptr> node) { // resize the variable auto variable = node->getValue(); if (variable != nullptr) { - variable->m_buffer.resize(new_size); + variable->m_buffer.resize(new_size); } for (auto& [var_name, child] : node->getChildren()) { @@ -586,12 +710,12 @@ class BufferManager { } } - void set_capacity(size_t new_size, std::shared_ptr>> node) { + void set_capacity(size_t new_size, std::shared_ptr> node) { // resize the variable auto variable = node->getValue(); if (variable != nullptr) { - variable->m_buffer.set_capacity(new_size); + variable->m_buffer.set_capacity(new_size); } for (auto& [var_name, child] : node->getChildren()) { @@ -604,7 +728,7 @@ class BufferManager { bool m_should_stop_thread{ false }; std::mutex m_mutex_cv; std::condition_variable m_cv; - std::shared_ptr>> m_tree; + std::shared_ptr> m_tree; std::function m_nowFunction{DefaultClock}; std::thread m_save_thread; diff --git a/src/libYARP_telemetry/src/yarp/telemetry/experimental/Record.h b/src/libYARP_telemetry/src/yarp/telemetry/experimental/Record.h index 6b4d51b..682adbe 100644 --- a/src/libYARP_telemetry/src/yarp/telemetry/experimental/Record.h +++ b/src/libYARP_telemetry/src/yarp/telemetry/experimental/Record.h @@ -10,6 +10,7 @@ #define YARP_TELEMETRY_RECORD_H #include +#include #include @@ -19,69 +20,12 @@ namespace yarp::telemetry::experimental { * @brief A structure to represent a Record. * */ -template struct Record { double m_ts;/**< timestamp */ - std::vector m_datum;/**< the actual data of the record */ - - /** - * @brief Construct an empty Record object - * - */ - Record() = default; - - /** - * @brief Construct a new Record object copying the _datum - * - * @param[in] _ts Timestamp to assign to the record. - * @param[in] _datum Datum to be copied. - */ - Record(const double& _ts, - matioCpp::Span _datum) : m_ts(_ts), m_datum(_datum.begin(), _datum.end()) { - m_payload = sizeof(m_ts) + sizeof(m_datum) + sizeof(T) * m_datum.capacity(); - } - - /** - * @brief Construct a new Record object copying the _datum - * - * @param[in] _ts Timestamp to assign to the record. - * @param[in] _datum Datum to be copied. - */ - Record(const double& _ts, - const std::initializer_list& _datum) - : m_ts(_ts), m_datum(_datum.begin(), _datum.end()) { - m_payload = sizeof(m_ts) + sizeof(m_datum) + sizeof(T) * m_datum.capacity(); - } - - /** - * @brief Construct a new Record object moving the _datum - * - * @param[in] _ts Timestamp to assign to the record. - * @param[in] _datum Datum to be moved. - */ - Record(const double& _ts, - std::vector&& _datum) : m_ts(_ts), m_datum(std::move(_datum)) { - m_payload = sizeof(m_ts) + sizeof(m_datum) + sizeof(T) * m_datum.capacity(); - } - - /** - * @brief Get the payload of the Record as nr of bytes - * - * @return size_t The payload of the Record. - */ - size_t getPayload() const { - return m_payload; - } - - private: - size_t m_payload{0}; - - // Trying to apply the rule of zero - + std::any m_datum;/**< the actual data of the record */ }; - } // yarp::telemetry #endif From 61a591f31cde005b6f2e9d3516f237282bc364a4 Mon Sep 17 00:00:00 2001 From: Stefano Date: Wed, 30 Mar 2022 13:44:37 +0200 Subject: [PATCH 02/11] Fixed test compilation. --- .../src/yarp/telemetry/experimental/BufferConfig.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.h b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.h index 06e59cb..4310850 100644 --- a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.h +++ b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.h @@ -67,6 +67,20 @@ struct YARP_telemetry_API ChannelInfo { : ChannelInfo(name, dimensions, getTypeName(), elements_names){ } + /** + * @brief Construct a ChannelInfo from name, dimensions and a vector containing the name of + * the elements associated to the channel. + * @param name name of the channel. + * @param dimensions dimension associated to the channel. + * @param elements_names Vector containing the names of each element of the channel. + */ + ChannelInfo(const std::string& name, + const dimensions_t& dimensions, + const elements_names_t& elements_names) + : ChannelInfo(name, dimensions, type_name_not_set_tag, elements_names){ + } + + /** * @brief Construct a ChannelInfo from name and dimensions. * @param name name of the channel. From b5d7a8b14bd25bad7d29e7f6f65e8dc7267f5131 Mon Sep 17 00:00:00 2001 From: Stefano Date: Wed, 30 Mar 2022 15:03:44 +0200 Subject: [PATCH 03/11] Fixed concatenation in a multidimensional array. --- .../telemetry/experimental/BufferManager.h | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h index beaaa90..f31c97b 100644 --- a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h +++ b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h @@ -344,7 +344,7 @@ class BufferManager { bufferInfo->m_matioCpp_create_storage = [](const std::string& name, const dimensions_t& bufferInfoDimensions, size_t num_instants) -> matioCpp::Variable { dimensions_t fullDimensions = bufferInfoDimensions; fullDimensions.push_back(num_instants); - return matioCpp::MultiDimensionalArray(name, bufferInfoDimensions); + return matioCpp::MultiDimensionalArray(name, fullDimensions); }; } if (!bufferInfo->m_append_function) @@ -354,9 +354,16 @@ class BufferManager { const std::vector& inputCasted = std::any_cast>(input); + auto dims = concatenatedCasted.dimensions(); + std::vector elem; + elem.resize(dims.size(), 0); + elem.back() = instantIndex; + + size_t startIndex = concatenatedCasted.rawIndexFromIndices(elem); //The start raw index is {0,0,0,...., instantIndex} + for (size_t i = 0; i < inputCasted.size(); ++i) { - concatenatedCasted[{i, 0, instantIndex}] = inputCasted[i]; + concatenatedCasted[startIndex + i] = inputCasted[i]; } }; } @@ -399,7 +406,7 @@ class BufferManager { bufferInfo->m_matioCpp_create_storage = [](const std::string& name, const dimensions_t& bufferInfoDimensions, size_t num_instants) -> matioCpp::Variable { dimensions_t fullDimensions = bufferInfoDimensions; fullDimensions.push_back(num_instants); - return matioCpp::MultiDimensionalArray(name, bufferInfoDimensions); + return matioCpp::MultiDimensionalArray(name, fullDimensions); }; } if (!bufferInfo->m_append_function) @@ -409,9 +416,16 @@ class BufferManager { const std::vector& inputCasted = std::any_cast>(input); + auto dims = concatenatedCasted.dimensions(); + std::vector elem; + elem.resize(dims.size(), 0); + elem.back() = instantIndex; + + size_t startIndex = concatenatedCasted.rawIndexFromIndices(elem); //The start raw index is {0,0,0,...., instantIndex} + for (size_t i = 0; i < inputCasted.size(); ++i) { - concatenatedCasted[{i, 0, instantIndex}] = inputCasted[i]; + concatenatedCasted[startIndex + i] = inputCasted[i]; } }; } From 45d880e28963bab3370b751b5af8ca81ff351c09 Mon Sep 17 00:00:00 2001 From: Stefano Date: Thu, 31 Mar 2022 16:46:15 +0200 Subject: [PATCH 04/11] Used a function to determine the save lambda --- .../telemetry/experimental/BufferManager.h | 249 ++++++++---------- 1 file changed, 104 insertions(+), 145 deletions(-) diff --git a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h index f31c97b..cce5008 100644 --- a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h +++ b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h @@ -58,27 +58,110 @@ struct BufferInfo { Buffer m_buffer; std::mutex m_buff_mutex; dimensions_t m_dimensions; + size_t m_dimensions_factorial{0}; std::string m_type_name; elements_names_t m_elements_names; - std::function m_matioCpp_create_storage; - std::function m_append_function; + std::function m_convert_to_matioCpp; BufferInfo() = default; BufferInfo(const BufferInfo& other) : m_buffer(other.m_buffer), m_dimensions(other.m_dimensions), + m_dimensions_factorial(other.m_dimensions_factorial), m_type_name(other.m_type_name), m_elements_names(other.m_elements_names), - m_append_function(other.m_append_function){ + m_convert_to_matioCpp(other.m_convert_to_matioCpp){ } BufferInfo(BufferInfo&& other) : m_buffer(std::move(other.m_buffer)), m_dimensions(std::move(other.m_dimensions)), + m_dimensions_factorial(std::move(other.m_dimensions_factorial)), m_type_name(std::move(other.m_type_name)), m_elements_names(std::move(other.m_elements_names)), - m_append_function(std::move(other.m_append_function)){ + m_convert_to_matioCpp(std::move(other.m_convert_to_matioCpp)){ } + + template + void createMatioCppConvertFunction() + { + static_assert(matioCpp::is_make_variable_callable::value, "The selected type cannot be used with matioCpp."); + + if (m_convert_to_matioCpp) + { + return; + } + + using matioCppType = typename matioCpp::make_variable_output::type; + m_convert_to_matioCpp = [this](const std::string& name) + { + size_t num_instants = this->m_buffer.size(); + + //We need to find the value_type of the matioCppType. For example, the value_type of matioCpp::Vector is double. + //This is needed for is_same_v to work. On the other hand, if matioCppType is a Struct for example, the value_type is not defined. + //The conditional allows compiling anyway, using void as elementType + using elementType = std::conditional_t::value, typename matioCppType::value_type, void>; + + //if the input data is numeric, then we concatenate on the last dimension + if constexpr (std::is_same_v> || + std::is_same_v> || + std::is_same_v>) + { + dimensions_t fullDimensions = this->m_dimensions; + fullDimensions.push_back(num_instants); + + matioCpp::MultiDimensionalArray outputVariable(name, fullDimensions); + + size_t i = 0; + for (auto& _cell : this->m_buffer) { + const T& cellCasted = std::any_cast(_cell.m_datum); + + + auto matioCppVariable = matioCpp::make_variable("element", cellCasted); //this is a little of a waste. + //We copy each buffer element into a matioCpp variable. + //But in this way, the code remains generic and usable with all the types that matioCpp supports + + auto matioCppSpan = matioCppVariable.toSpan(); + + size_t startIndex = this->m_dimensions_factorial * i; //We concatenate on the last dimension. Suppose that the channel stores matrices of size 3x2. + //The output variable is a 3x2xn matrix, where n is the number of elements in the buffer. + //If we consider the output buffer as a linear vector, the element at position i starts from location 6*i + //and ends at 6*(i+1) + + for (size_t i = 0; i < std::min(static_cast(matioCppSpan.size()), this->m_dimensions_factorial); ++i) + { + outputVariable[startIndex + i] = matioCppSpan[i]; //we copy the new element in the corresponding position inside the variable + } + ++i; + } + return outputVariable; + } + else if constexpr(std::is_same_v) //if the input is a struct, we use a struct array + { + matioCpp::StructArray outputVariable(name, {num_instants, 1}); + + size_t i = 0; + for (auto& _cell : this->m_buffer) { + outputVariable.setElement(i, matioCpp::make_variable("element", std::any_cast(_cell.m_datum))); + ++i; + } + return outputVariable; + } + else //otherwise we use a cell array + { + matioCpp::CellArray outputVariable(name, {num_instants, 1}); + + size_t i = 0; + for (auto& _cell : this->m_buffer) { + outputVariable.setElement(i, matioCpp::make_variable("element", std::any_cast(_cell.m_datum))); + ++i; + } + return outputVariable; + } + }; + } + + }; /** * @brief Class that manages the buffers associated to the channels of the telemetry. @@ -265,6 +348,11 @@ class BufferManager { buffInfo->m_buffer = Buffer(m_bufferConfig.n_samples); buffInfo->m_dimensions = channel.dimensions; + buffInfo->m_dimensions_factorial = std::accumulate(channel.dimensions.begin(), + channel.dimensions.end(), + 1, + std::multiplies<>()); + bool userDidNotSetTypeName = channel.type_name == ChannelInfo::type_name_not_set_tag || channel.type_name.empty(); if (std::is_same_v && userDidNotSetTypeName) @@ -317,59 +405,10 @@ class BufferManager { * @param[in] ts The timestamp of the element to be pushed. * @param[in] var_name The name of the channel. */ - template//-----------------------------Here I need to mak sure that T is a numeric quantity + template inline void push_back(matioCpp::Span elem, double ts, const std::string& var_name) { - auto leaf = getLeaf(var_name, m_tree).lock(); - if (leaf == nullptr) - { - throw std::invalid_argument("The channel " + var_name + " does not exist."); - } - - auto bufferInfo = leaf->getValue(); - assert(bufferInfo != nullptr); - - if (bufferInfo->m_type_name != ChannelInfo::getTypeName>()) - { - std::cout << "Cannot push to the channel " << var_name - << ". Expected type: " << bufferInfo->m_type_name - << ". Input type: " << ChannelInfo::getTypeName>(); - return; - } - - std::scoped_lock lock{ bufferInfo->m_buff_mutex }; - //Create the saving functions if they were not present already - if (!bufferInfo->m_matioCpp_create_storage) - { - bufferInfo->m_matioCpp_create_storage = [](const std::string& name, const dimensions_t& bufferInfoDimensions, size_t num_instants) -> matioCpp::Variable { - dimensions_t fullDimensions = bufferInfoDimensions; - fullDimensions.push_back(num_instants); - return matioCpp::MultiDimensionalArray(name, fullDimensions); - }; - } - if (!bufferInfo->m_append_function) - { - bufferInfo->m_append_function = [](const std::any& input, size_t instantIndex, matioCpp::Variable& concatenated) { - matioCpp::MultiDimensionalArray concatenatedCasted = concatenated.asMultiDimensionalArray(); - - const std::vector& inputCasted = std::any_cast>(input); - - auto dims = concatenatedCasted.dimensions(); - std::vector elem; - elem.resize(dims.size(), 0); - elem.back() = instantIndex; - - size_t startIndex = concatenatedCasted.rawIndexFromIndices(elem); //The start raw index is {0,0,0,...., instantIndex} - - for (size_t i = 0; i < inputCasted.size(); ++i) - { - concatenatedCasted[startIndex + i] = inputCasted[i]; - } - }; - } - - assert(elem.size() == bufferInfo->m_dimensions[0] * bufferInfo->m_dimensions[1]); - bufferInfo->m_buffer.push_back({ts, std::vector(elem.begin(), elem.end())}); + push_back(std::vector(elem.begin(), elem.end()), ts, var_name); } /** @@ -380,57 +419,10 @@ class BufferManager { * @param[in] ts The timestamp of the element to be pushed. * @param[in] var_name The name of the channel. */ - template//-----------------------------Here I need to mak sure that T is a numeric quantity + template inline void push_back(const std::initializer_list& elem, double ts, const std::string& var_name) { - auto leaf = getLeaf(var_name, m_tree).lock(); - if (leaf == nullptr) - { - throw std::invalid_argument("The channel " + var_name + " does not exist."); - } - auto bufferInfo = leaf->getValue(); - assert(bufferInfo != nullptr); - - if (bufferInfo->m_type_name != ChannelInfo::getTypeName>()) - { - std::cout << "Cannot push to the channel " << var_name - << ". Expected type: " << bufferInfo->m_type_name - << ". Input type: " << ChannelInfo::getTypeName>(); - return; - } - - std::scoped_lock lock{ bufferInfo->m_buff_mutex }; - //Create the saving functions if they were not present already - if (!bufferInfo->m_matioCpp_create_storage) - { - bufferInfo->m_matioCpp_create_storage = [](const std::string& name, const dimensions_t& bufferInfoDimensions, size_t num_instants) -> matioCpp::Variable { - dimensions_t fullDimensions = bufferInfoDimensions; - fullDimensions.push_back(num_instants); - return matioCpp::MultiDimensionalArray(name, fullDimensions); - }; - } - if (!bufferInfo->m_append_function) - { - bufferInfo->m_append_function = [](const std::any& input, size_t instantIndex, matioCpp::Variable& concatenated) { - matioCpp::MultiDimensionalArray concatenatedCasted = concatenated.asMultiDimensionalArray(); - - const std::vector& inputCasted = std::any_cast>(input); - - auto dims = concatenatedCasted.dimensions(); - std::vector elem; - elem.resize(dims.size(), 0); - elem.back() = instantIndex; - - size_t startIndex = concatenatedCasted.rawIndexFromIndices(elem); //The start raw index is {0,0,0,...., instantIndex} - - for (size_t i = 0; i < inputCasted.size(); ++i) - { - concatenatedCasted[startIndex + i] = inputCasted[i]; - } - }; - } - assert(elem.size() == bufferInfo->m_dimensions[0] * bufferInfo->m_dimensions[1]); - bufferInfo->m_buffer.push_back({ts, std::vector(elem.begin(), elem.end())}); + push_back(std::vector(elem.begin(), elem.end()), ts, var_name); } /** @@ -440,7 +432,7 @@ class BufferManager { * @param[in] elem The element to be pushed(via copy) in the channel. * @param[in] var_name The name of the channel. */ - template//-----------------------------Here I need to mak sure that T is a numeric quantity + template inline void push_back(const std::initializer_list& elem, const std::string& var_name) { push_back(elem, m_nowFunction(), var_name); @@ -454,11 +446,9 @@ class BufferManager { * @param[in] ts The timestamp of the element to be pushed. * @param[in] var_name The name of the channel. */ - template//----------------------------Here I probably need to avoid ambiguity with the two methods above. std::vector!!!!! + template inline void push_back(const T& elem, double ts, const std::string& var_name) { - static_assert(matioCpp::is_make_variable_callable::value, "The selected type cannot be used with matioCpp."); - auto leaf = getLeaf(var_name, m_tree).lock(); if (leaf == nullptr) { @@ -477,20 +467,7 @@ class BufferManager { std::scoped_lock lock{ bufferInfo->m_buff_mutex }; //Create the saving functions if they were not present already - if (!bufferInfo->m_matioCpp_create_storage) - { - bufferInfo->m_matioCpp_create_storage = [](const std::string& name, const dimensions_t&, size_t num_instants) -> matioCpp::Variable { - return matioCpp::CellArray(name, {num_instants, 1}); - }; - } - - if (!bufferInfo->m_append_function) - { - bufferInfo->m_append_function = [](const std::any& input, size_t instantIndex, matioCpp::Variable& concatenated) { - matioCpp::CellArray concatenatedCasted = concatenated.asCellArray(); - concatenatedCasted.setElement(instantIndex, matioCpp::make_variable("element", std::any_cast(input))); - }; - } + bufferInfo->createMatioCppConvertFunction(); bufferInfo->m_buffer.push_back({ts, elem}); } @@ -608,23 +585,6 @@ class BufferManager { return tmp; } - matioCpp::Variable concatenateElements(std::shared_ptr buffInfo) const - { - // the number of timesteps is the size of our collection - auto num_timesteps = buffInfo->m_buffer.size(); - - matioCpp::Variable out = buffInfo->m_matioCpp_create_storage("data", buffInfo->m_dimensions, num_timesteps); - - size_t i = 0; - for (auto& _cell : buffInfo->m_buffer) { - buffInfo->m_append_function(_cell.m_datum, i, out); - ++i; - } - assert(i == buffInfo->m_buffer.size()); - - return out; - } - matioCpp::Struct createElementStruct(const std::string& var_name, std::shared_ptr buffInfo, bool flush_all) const { @@ -645,8 +605,9 @@ class BufferManager { // the number of timesteps is the size of our collection auto num_timesteps = buffInfo->m_buffer.size(); + assert(buffInfo->m_convert_to_matioCpp); // We concatenate all the data of the buffer into a single variable - matioCpp::Variable data = concatenateElements(buffInfo); + matioCpp::Variable data = buffInfo->m_convert_to_matioCpp("data"); //We construct the timestamp vector matioCpp::Vector timestamps("timestamps", num_timesteps); @@ -664,14 +625,12 @@ class BufferManager { std::vector test_data; // now we create the vector for the dimensions - // The first two dimensions are the r and c of the sample, the number of sample has to be the last dimension. - std::vector dimensions_data_vect {static_cast(buffInfo->m_dimensions[0]), - static_cast(buffInfo->m_dimensions[1]), - static_cast(num_timesteps)}; + dimensions_t fullDimensions = buffInfo->m_dimensions; + fullDimensions.push_back(num_timesteps); test_data.emplace_back(data); // Data - test_data.emplace_back(matioCpp::make_variable("dimensions", dimensions_data_vect)); // dimensions vector + test_data.emplace_back(matioCpp::make_variable("dimensions", fullDimensions)); // dimensions vector test_data.emplace_back(matioCpp::make_variable("elements_names", buffInfo->m_elements_names)); // elements names test_data.emplace_back(matioCpp::String("name", var_name)); // name of the signal From 27953db65d5920f9c8f79fb52413b60e42b312f5 Mon Sep 17 00:00:00 2001 From: Stefano Date: Fri, 1 Apr 2022 15:51:13 +0200 Subject: [PATCH 05/11] First working version with multiple types. --- .../telemetry/experimental/BufferConfig.cpp | 8 ++++ .../telemetry/experimental/BufferConfig.h | 43 ++++++++--------- .../telemetry/experimental/BufferManager.h | 48 ++++++++++++++----- test/BufferManagerTest.cpp | 48 +++++++++++++++++++ 4 files changed, 112 insertions(+), 35 deletions(-) diff --git a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.cpp b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.cpp index 8fb336a..8ff88fa 100644 --- a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.cpp +++ b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.cpp @@ -43,6 +43,10 @@ namespace yarp::telemetry::experimental { } } + ChannelInfo::ChannelInfo(const std::string &name, const dimensions_t &dimensions, const elements_names_t &elements_names) + : ChannelInfo(name, dimensions, type_name_not_set_tag, elements_names){ + } + ChannelInfo::ChannelInfo(const std::string& name, const dimensions_t& dimensions, const std::string &type) : name(name), dimensions(dimensions), @@ -58,6 +62,10 @@ namespace yarp::telemetry::experimental { } } + ChannelInfo::ChannelInfo(const std::string &name, const dimensions_t &dimensions) + : ChannelInfo(name, dimensions, type_name_not_set_tag){ + } + void to_json(nlohmann::json& j, const ChannelInfo& info) { j = nlohmann::json{{"name", info.name}, diff --git a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.h b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.h index 4310850..00d0740 100644 --- a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.h +++ b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.h @@ -23,6 +23,24 @@ namespace yarp::telemetry::experimental { using dimensions_t = std::vector; using elements_names_t = std::vector; +/** + * @brief Get the type name as string + */ +template +static std::string getTypeName(const T& someInput) +{ + return boost::core::demangle(typeid(someInput).name()); +} + +/** + * @brief Get the type name as string + */ +template +static std::string getTypeName() +{ + return boost::core::demangle(typeid(T).name()); +} + /** * @brief Struct representing a channel(variable) in terms of * name and dimensions and names of the each element of a variable. @@ -76,9 +94,7 @@ struct YARP_telemetry_API ChannelInfo { */ ChannelInfo(const std::string& name, const dimensions_t& dimensions, - const elements_names_t& elements_names) - : ChannelInfo(name, dimensions, type_name_not_set_tag, elements_names){ - } + const elements_names_t& elements_names); /** @@ -113,27 +129,8 @@ struct YARP_telemetry_API ChannelInfo { * elements_names = [element_0, element_1, ..., element_n], where n is given by the * product of the dimensions. */ - ChannelInfo(const std::string& name, const dimensions_t& dimensions) - : ChannelInfo(name, dimensions, type_name_not_set_tag){ - } - - /** - * @brief Get the type name as string - */ - template - static std::string getTypeName(const T& someInput) - { - return boost::core::demangle(typeid(someInput).name()); - } + ChannelInfo(const std::string& name, const dimensions_t& dimensions); - /** - * @brief Get the type name as string - */ - template - static std::string getTypeName() - { - return boost::core::demangle(typeid(T).name()); - } }; /** diff --git a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h index cce5008..2ec2d58 100644 --- a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h +++ b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h @@ -48,6 +48,21 @@ namespace yarp::telemetry::experimental { +template +struct matioCppCanConcatenate : std::false_type {}; + +// matiomatioCppCanConcatenate::value is true when T has the T::value_type memeber. If this is true, then we check +// if T is either an Element, a Vector (but not a String), or a MultidimensionalArray +template +struct matioCppCanConcatenate::value>, + typename std::enable_if_t<(std::is_same_v> || + (std::is_same_v> && + !std::is_same_v) || + std::is_same_v>)>> + : std::true_type {}; + + /** * @brief Class that aggregates the yarp::telemetry::experimental::Buffer and some other @@ -97,16 +112,11 @@ struct BufferInfo { { size_t num_instants = this->m_buffer.size(); - //We need to find the value_type of the matioCppType. For example, the value_type of matioCpp::Vector is double. - //This is needed for is_same_v to work. On the other hand, if matioCppType is a Struct for example, the value_type is not defined. - //The conditional allows compiling anyway, using void as elementType - using elementType = std::conditional_t::value, typename matioCppType::value_type, void>; - //if the input data is numeric, then we concatenate on the last dimension - if constexpr (std::is_same_v> || - std::is_same_v> || - std::is_same_v>) + if constexpr (matioCppCanConcatenate::value) { + using elementType = typename matioCppType::value_type; + dimensions_t fullDimensions = this->m_dimensions; fullDimensions.push_back(num_instants); @@ -142,7 +152,14 @@ struct BufferInfo { size_t i = 0; for (auto& _cell : this->m_buffer) { - outputVariable.setElement(i, matioCpp::make_variable("element", std::any_cast(_cell.m_datum))); + matioCpp::Struct element = matioCpp::make_variable("element", std::any_cast(_cell.m_datum)); + + if (i == 0) + { + outputVariable.addFields(element.fields()); + } + + outputVariable.setElement(i, element); ++i; } return outputVariable; @@ -361,9 +378,16 @@ class BufferManager { return false; } + std::string defaultTypeName = ChannelInfo::type_name_not_set_tag; + + if constexpr (!std::is_same_v) + { + defaultTypeName = getTypeName>(); + } + if (userDidNotSetTypeName) { - buffInfo->m_type_name = ChannelInfo::getTypeName>();; + buffInfo->m_type_name = defaultTypeName; } else { @@ -457,11 +481,11 @@ class BufferManager { auto bufferInfo = leaf->getValue(); assert(bufferInfo != nullptr); - if (bufferInfo->m_type_name != ChannelInfo::getTypeName()) + if (bufferInfo->m_type_name != getTypeName()) { std::cout << "Cannot push to the channel " << var_name << ". Expected type: " << bufferInfo->m_type_name - << ". Input type: " << ChannelInfo::getTypeName(); + << ". Input type: " << getTypeName() <()}; + yarp::telemetry::experimental::ChannelInfo var_double{ "double", {1}, yte::getTypeName() }; + yarp::telemetry::experimental::ChannelInfo var_string{ "string", {1}, yte::getTypeName()}; + yarp::telemetry::experimental::ChannelInfo var_vector{ "vector", {4, 1}, yte::getTypeName>()}; + yarp::telemetry::experimental::ChannelInfo var_struct{ "struct", {1}, yte::getTypeName()}; + +// First add channels that will be handling empty buffers + REQUIRE(bm.addChannel(var_int)); + REQUIRE(bm.addChannel(var_double)); + REQUIRE(bm.addChannel(var_string)); + REQUIRE(bm.addChannel(var_vector)); + REQUIRE(bm.addChannel(var_struct)); + + bufferConfig.n_samples = n_samples; + bufferConfig.filename = "buffer_manager_test_multiple_types"; + bufferConfig.auto_save = true; + + REQUIRE(bm.configure(bufferConfig)); + + testStruct item; + + for (int i = 0; i < 40; i++) { + bm.push_back(i, "int"); + bm.push_back(i * 1.0, "double"); + bm.push_back("iter" + std::to_string(i), "string"); + bm.push_back({i + 0.0, i + 1.0, i + 2.0, i + 3.0}, "vector"); + item.a = i; + item.b = i; + bm.push_back(item, "struct"); + + yarp::os::Time::delay(0.01); + } + } + #if defined CATCH_CONFIG_ENABLE_BENCHMARKING SECTION("Benchmarking section scalar int") { From 385791fdbdf3361b3a353475e39c9b469c2cafb7 Mon Sep 17 00:00:00 2001 From: Stefano Date: Fri, 1 Apr 2022 19:28:04 +0200 Subject: [PATCH 06/11] Using Span when possible when saving to file. --- .../telemetry/experimental/BufferManager.h | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h index 2ec2d58..0fe0439 100644 --- a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h +++ b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h @@ -126,12 +126,22 @@ struct BufferInfo { for (auto& _cell : this->m_buffer) { const T& cellCasted = std::any_cast(_cell.m_datum); + matioCpp::Span matioCppSpan; - auto matioCppVariable = matioCpp::make_variable("element", cellCasted); //this is a little of a waste. - //We copy each buffer element into a matioCpp variable. - //But in this way, the code remains generic and usable with all the types that matioCpp supports + matioCppType matioCppVariable; - auto matioCppSpan = matioCppVariable.toSpan(); + //We convert the cell to a matioCpp variable only if we are not able to create a Span. + //In this way we avoid duplicating memory + if constexpr (matioCpp::SpanUtils::is_make_span_callable::value) + { + matioCppSpan = matioCpp::make_span(cellCasted); + } + else + { + matioCppVariable = matioCpp::make_variable("element", cellCasted); + + matioCppSpan = matioCppVariable.toSpan(); + } size_t startIndex = this->m_dimensions_factorial * i; //We concatenate on the last dimension. Suppose that the channel stores matrices of size 3x2. //The output variable is a 3x2xn matrix, where n is the number of elements in the buffer. From 64961cce082bb9149e60b29c6917a369fdb9290e Mon Sep 17 00:00:00 2001 From: Stefano Date: Mon, 4 Apr 2022 11:21:19 +0200 Subject: [PATCH 07/11] The type of the channel is automatically obtained when pushing the first time. --- .../telemetry/experimental/BufferConfig.cpp | 20 ++---- .../telemetry/experimental/BufferConfig.h | 72 ------------------- .../telemetry/experimental/BufferManager.h | 56 ++++++++------- test/BufferManagerTest.cpp | 11 ++- 4 files changed, 41 insertions(+), 118 deletions(-) diff --git a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.cpp b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.cpp index 8ff88fa..f742765 100644 --- a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.cpp +++ b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.cpp @@ -24,11 +24,10 @@ namespace matioCpp { namespace yarp::telemetry::experimental { ChannelInfo::ChannelInfo(const std::string& name, - const dimensions_t& dimensions, const std::string &type, + const dimensions_t& dimensions, const elements_names_t& elements_names) : name(name), dimensions(dimensions), - type_name(type), elements_names(elements_names) { const unsigned int elements = std::accumulate(dimensions.begin(), @@ -43,14 +42,9 @@ namespace yarp::telemetry::experimental { } } - ChannelInfo::ChannelInfo(const std::string &name, const dimensions_t &dimensions, const elements_names_t &elements_names) - : ChannelInfo(name, dimensions, type_name_not_set_tag, elements_names){ - } - - ChannelInfo::ChannelInfo(const std::string& name, const dimensions_t& dimensions, const std::string &type) + ChannelInfo::ChannelInfo(const std::string& name, const dimensions_t& dimensions) : name(name), - dimensions(dimensions), - type_name(type) + dimensions(dimensions) { const unsigned int elements = std::accumulate(dimensions.begin(), dimensions.end(), @@ -62,24 +56,18 @@ namespace yarp::telemetry::experimental { } } - ChannelInfo::ChannelInfo(const std::string &name, const dimensions_t &dimensions) - : ChannelInfo(name, dimensions, type_name_not_set_tag){ - } - void to_json(nlohmann::json& j, const ChannelInfo& info) { j = nlohmann::json{{"name", info.name}, {"dimensions", info.dimensions}, - {"type_name", info.type_name}, {"elements_names", info.elements_names}, - }; + }; } void from_json(const nlohmann::json& j, ChannelInfo& info) { j.at("name").get_to(info.name); j.at("dimensions").get_to(info.dimensions); - j.at("type_name").get_to(info.type_name); j.at("elements_names").get_to(info.elements_names); } diff --git a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.h b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.h index 00d0740..560dde7 100644 --- a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.h +++ b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.h @@ -23,34 +23,14 @@ namespace yarp::telemetry::experimental { using dimensions_t = std::vector; using elements_names_t = std::vector; -/** - * @brief Get the type name as string - */ -template -static std::string getTypeName(const T& someInput) -{ - return boost::core::demangle(typeid(someInput).name()); -} - -/** - * @brief Get the type name as string - */ -template -static std::string getTypeName() -{ - return boost::core::demangle(typeid(T).name()); -} - /** * @brief Struct representing a channel(variable) in terms of * name and dimensions and names of the each element of a variable. */ struct YARP_telemetry_API ChannelInfo { - static constexpr char type_name_not_set_tag[] = "type_name_not_set"; std::string name; /**< Name of the channel */ dimensions_t dimensions; /**< Dimension of the channel */ - std::string type_name{type_name_not_set_tag}; /**< The name of the type of data used in the channel. */ elements_names_t elements_names; /**< Vector containing the names of each element of the channel */ /** @@ -63,64 +43,12 @@ struct YARP_telemetry_API ChannelInfo { * the elements associated to the channel. * @param name name of the channel. * @param dimensions dimension associated to the channel. - * @param type the human readable type of the buffer (e.g. "std::vector") * @param elements_names Vector containing the names of each element of the channel. */ ChannelInfo(const std::string& name, const dimensions_t& dimensions, - const std::string& type, const elements_names_t& elements_names); - /** - * @brief Construct a ChannelInfo from name, dimensions and a vector containing the name of - * the elements associated to the channel. - * @param name name of the channel. - * @param dimensions dimension associated to the channel. - * @param elements_names Vector containing the names of each element of the channel. - */ - template - explicit ChannelInfo(const std::string& name, - const dimensions_t& dimensions, - const elements_names_t& elements_names) - : ChannelInfo(name, dimensions, getTypeName(), elements_names){ - } - - /** - * @brief Construct a ChannelInfo from name, dimensions and a vector containing the name of - * the elements associated to the channel. - * @param name name of the channel. - * @param dimensions dimension associated to the channel. - * @param elements_names Vector containing the names of each element of the channel. - */ - ChannelInfo(const std::string& name, - const dimensions_t& dimensions, - const elements_names_t& elements_names); - - - /** - * @brief Construct a ChannelInfo from name and dimensions. - * @param name name of the channel. - * @param dimensions dimension associated to the channel. - * @param type the human readable type of the buffer (e.g. "std::vector") - * @note If the constructor is called the elements_names are set as - * elements_names = [element_0, element_1, ..., element_n], where n is given by the - * product of the dimensions. - */ - ChannelInfo(const std::string& name, const dimensions_t& dimensions, const std::string& type); - - /** - * @brief Construct a ChannelInfo from name and dimensions. - * @param name name of the channel. - * @param dimensions dimension associated to the channel. - * @note If the constructor is called the elements_names are set as - * elements_names = [element_0, element_1, ..., element_n], where n is given by the - * product of the dimensions. - */ - template - explicit ChannelInfo(const std::string& name, const dimensions_t& dimensions) - : ChannelInfo(name, dimensions, getTypeName()){ - } - /** * @brief Construct a ChannelInfo from name and dimensions. * @param name name of the channel. diff --git a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h index 0fe0439..68264cb 100644 --- a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h +++ b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h @@ -48,11 +48,29 @@ namespace yarp::telemetry::experimental { -template -struct matioCppCanConcatenate : std::false_type {}; +/** + * @brief Get the type name as string + */ +template +static std::string getTypeName(const T& someInput) +{ + return boost::core::demangle(typeid(someInput).name()); +} + +/** + * @brief Get the type name as string + */ +template +static std::string getTypeName() +{ + return boost::core::demangle(typeid(T).name()); +} // matiomatioCppCanConcatenate::value is true when T has the T::value_type memeber. If this is true, then we check // if T is either an Element, a Vector (but not a String), or a MultidimensionalArray +template +struct matioCppCanConcatenate : std::false_type {}; + template struct matioCppCanConcatenate::value>, @@ -70,11 +88,12 @@ struct matioCppCanConcatenate m_convert_to_matioCpp; @@ -380,28 +399,9 @@ class BufferManager { 1, std::multiplies<>()); - bool userDidNotSetTypeName = channel.type_name == ChannelInfo::type_name_not_set_tag || channel.type_name.empty(); - - if (std::is_same_v && userDidNotSetTypeName) - { - std::cerr << "Unable to determine the type of the channel " << channel.name << std::endl; - return false; - } - - std::string defaultTypeName = ChannelInfo::type_name_not_set_tag; - if constexpr (!std::is_same_v) { - defaultTypeName = getTypeName>(); - } - - if (userDidNotSetTypeName) - { - buffInfo->m_type_name = defaultTypeName; - } - else - { - buffInfo->m_type_name = channel.type_name; + buffInfo->m_type_name = getTypeName>(); } buffInfo->m_elements_names = channel.elements_names; @@ -491,7 +491,9 @@ class BufferManager { auto bufferInfo = leaf->getValue(); assert(bufferInfo != nullptr); - if (bufferInfo->m_type_name != getTypeName()) + bool typename_set = bufferInfo->m_type_name != BufferInfo::type_name_not_set_tag; + + if (typename_set && (bufferInfo->m_type_name != getTypeName())) { std::cout << "Cannot push to the channel " << var_name << ". Expected type: " << bufferInfo->m_type_name @@ -500,6 +502,12 @@ class BufferManager { } std::scoped_lock lock{ bufferInfo->m_buff_mutex }; + + if (!typename_set) + { + bufferInfo->m_type_name = getTypeName(); + } + //Create the saving functions if they were not present already bufferInfo->createMatioCppConvertFunction(); diff --git a/test/BufferManagerTest.cpp b/test/BufferManagerTest.cpp index 8bc3152..8a21618 100644 --- a/test/BufferManagerTest.cpp +++ b/test/BufferManagerTest.cpp @@ -348,16 +348,15 @@ TEST_CASE("Buffer Manager Test") SECTION("Multiple types") { - namespace yte = yarp::telemetry::experimental; yarp::telemetry::experimental::BufferManager bm; yarp::telemetry::experimental::BufferConfig bufferConfig; - yarp::telemetry::experimental::ChannelInfo var_int{ "int", {1}, yte::getTypeName()}; - yarp::telemetry::experimental::ChannelInfo var_double{ "double", {1}, yte::getTypeName() }; - yarp::telemetry::experimental::ChannelInfo var_string{ "string", {1}, yte::getTypeName()}; - yarp::telemetry::experimental::ChannelInfo var_vector{ "vector", {4, 1}, yte::getTypeName>()}; - yarp::telemetry::experimental::ChannelInfo var_struct{ "struct", {1}, yte::getTypeName()}; + yarp::telemetry::experimental::ChannelInfo var_int{ "int", {1}}; + yarp::telemetry::experimental::ChannelInfo var_double{ "double", {1}}; + yarp::telemetry::experimental::ChannelInfo var_string{ "string", {1}}; + yarp::telemetry::experimental::ChannelInfo var_vector{ "vector", {4, 1}}; + yarp::telemetry::experimental::ChannelInfo var_struct{ "struct", {1}}; // First add channels that will be handling empty buffers REQUIRE(bm.addChannel(var_int)); From 10177dc1c90389cc9ba9e771d59b76057fe5979c Mon Sep 17 00:00:00 2001 From: Stefano Date: Mon, 4 Apr 2022 11:31:20 +0200 Subject: [PATCH 08/11] Added push_back with Span without ts. --- .../yarp/telemetry/experimental/BufferConfig.h | 6 ------ .../yarp/telemetry/experimental/BufferManager.h | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.h b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.h index 560dde7..e2bff16 100644 --- a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.h +++ b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferConfig.h @@ -9,9 +9,6 @@ #ifndef YARP_TELEMETRY_BUFFER_CONFIG_H #define YARP_TELEMETRY_BUFFER_CONFIG_H -#include -#include - #include #include #include @@ -22,13 +19,11 @@ namespace yarp::telemetry::experimental { using dimensions_t = std::vector; using elements_names_t = std::vector; - /** * @brief Struct representing a channel(variable) in terms of * name and dimensions and names of the each element of a variable. */ struct YARP_telemetry_API ChannelInfo { - std::string name; /**< Name of the channel */ dimensions_t dimensions; /**< Dimension of the channel */ elements_names_t elements_names; /**< Vector containing the names of each element of the channel */ @@ -58,7 +53,6 @@ struct YARP_telemetry_API ChannelInfo { * product of the dimensions. */ ChannelInfo(const std::string& name, const dimensions_t& dimensions); - }; /** diff --git a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h index 68264cb..d6c0c6a 100644 --- a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h +++ b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -30,6 +31,8 @@ #include #include #include +#include + #ifndef __has_include static_assert(false, "__has_include not supported"); @@ -459,6 +462,19 @@ class BufferManager { push_back(std::vector(elem.begin(), elem.end()), ts, var_name); } + /** + * @brief Push a new element in the var_name channel. + * The var_name channels must exist, otherwise an exception is thrown. + * + * @param[in] elem The element to be pushed(via copy) in the channel. + * @param[in] var_name The name of the channel. + */ + template + inline void push_back(matioCpp::Span elem, const std::string& var_name) + { + push_back(elem, m_nowFunction(), var_name); + } + /** * @brief Push a new element in the var_name channel. * The var_name channels must exist, otherwise an exception is thrown. From dfe8e8f8c3994443d39f04d3bcf09148d24ee1e1 Mon Sep 17 00:00:00 2001 From: Stefano Date: Thu, 28 Apr 2022 17:25:21 +0200 Subject: [PATCH 09/11] Updated README. - Fixed a typo - Align multiple types test to the README. --- README.md | 112 ++++++++++++++++-- .../telemetry/experimental/BufferManager.h | 2 +- test/BufferManagerTest.cpp | 24 ++-- 3 files changed, 115 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 9feb27e..98442f2 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ target_link_libraries(myApp YARP::YARP_telemetry) ### Example scalar variable -Here is the code snippet for dumping in a `.mat` file 3 samples of the scalar varibles `"one"` and `"two"`. +Here is the code snippet for dumping in a `.mat` file 3 samples of the scalar variables `"one"` and `"two"`. The type of the channel is inferred when pushing the first time ```c++ yarp::telemetry::experimental::BufferConfig bufferConfig; @@ -99,10 +99,10 @@ Here is the code snippet for dumping in a `.mat` file 3 samples of the scalar va // We use the default config, setting only the number of samples (no auto/periodic saving) bufferConfig.n_samples = n_samples; - yarp::telemetry::experimental::BufferManager bm(bufferConfig); + yarp::telemetry::experimental::BufferManager bm(bufferConfig); bm.setFileName("buffer_manager_test"); - yarp::telemetry::experimental::ChannelInfo var_one{ "one", {1,1} }; - yarp::telemetry::experimental::ChannelInfo var_two{ "two", {1,1} }; + yarp::telemetry::experimental::ChannelInfo var_one{ "one", {1} }; + yarp::telemetry::experimental::ChannelInfo var_two{ "two", {1} }; bool ok = bm.addChannel(var_one); ok = ok && bm.addChannel(var_two); @@ -112,9 +112,9 @@ Here is the code snippet for dumping in a `.mat` file 3 samples of the scalar va } for (int i = 0; i < 10; i++) { - bm.push_back({ i }, "one"); + bm.push_back(i , "one"); yarp::os::Time::delay(0.2); - bm.push_back({ i + 1 }, "two"); + bm.push_back(i + 1.0, "two"); } if (bm.saveToFile()) @@ -139,8 +139,8 @@ buffer_manager_test.one = struct with fields: - data: [1×1×3 int32] - dimensions: [1 1 3] + data: [1×3 int32] + dimensions: [1 3] elements_names: {'element_0'} name: 'one' timestamps: [1.6481e+09 1.6481e+09 1.6481e+09] @@ -150,6 +150,7 @@ buffer_manager_test.one = It is possible to save and dump also vector variables. Here is the code snippet for dumping in a `.mat` file 3 samples of the 4x1 vector variables `"one"` and `"two"`. +If ``BufferManager`` is used with a template ``type`` (e.g. ``BufferManager``), it expects all the inputs to be of type ``std::vector``. ```c++ yarp::telemetry::experimental::BufferConfig bufferConfig; @@ -158,7 +159,7 @@ Here is the code snippet for dumping in a `.mat` file 3 samples of the 4x1 vecto bufferConfig.filename = "buffer_manager_test_vector"; bufferConfig.n_samples = 3; - yarp::telemetry::experimental::BufferManager bm_v(bufferConfig); + yarp::telemetry::experimental::BufferManager bm_v(bufferConfig); //Only vectors of doubles are accepted for (int i = 0; i < 10; i++) { bm_v.push_back({ i+1.0, i+2.0, i+3.0, i+4.0 }, "one"); yarp::os::Time::delay(0.2); @@ -211,6 +212,7 @@ yarp::telemetry::experimental::ChannelInfo var_one{ "one", {4,1}, {"A", "B", "C" ### Example matrix variable Here is the code snippet for dumping in a `.mat` file 3 samples of the 2x3 matrix variable`"one"` and of the 3x2 matrix variable `"two"`. +If ``BufferManager`` is used with a template ``type`` (e.g. ``BufferManager``), it expects all the inputs to be of type ``std::vector``, but then input is remapped into a matrix of the specified type. ```c++ yarp::telemetry::experimental::BufferManager bm_m; @@ -302,6 +304,98 @@ ans = timestamps: [1.6415e+09 1.6415e+09 1.6415e+09] ``` +### Example multiple types + +``BufferManager`` can be used to store channels of different types, including ``struct``s. In order to store a ``struct``, it is necessary to use the ``VISITABLE_STRUCT`` macro (see https://github.com/garbageslam/visit_struct). The available conversions depend on [``matio-cpp``](https://github.com/ami-iit/matio-cpp). +```c++ +struct testStruct +{ + int a; + double b; +}; +VISITABLE_STRUCT(testStruct, a, b); + +... + + yarp::telemetry::experimental::BufferManager bm; + yarp::telemetry::experimental::BufferConfig bufferConfig; + + yarp::telemetry::experimental::ChannelInfo var_int{ "int_channel", {1}}; + yarp::telemetry::experimental::ChannelInfo var_double{ "double_channel", {1}}; + yarp::telemetry::experimental::ChannelInfo var_string{ "string_channel", {1}}; + yarp::telemetry::experimental::ChannelInfo var_vector{ "vector_channel", {4, 1}}; + yarp::telemetry::experimental::ChannelInfo var_struct{ "struct_channel", {1}}; + + bm.addChannel(var_int); + bm.addChannel(var_double); + bm.addChannel(var_string); + bm.addChannel(var_vector); + bm.addChannel(var_struct); + + bufferConfig.n_samples = 3; + bufferConfig.filename = "buffer_manager_test_multiple_types"; + bufferConfig.auto_save = true; + + bm.configure(bufferConfig); + + testStruct item; + + for (int i = 0; i < 10; i++) { + bm.push_back(i, "int_channel"); + bm.push_back(i * 1.0, "double_channel"); + bm.push_back("iter" + std::to_string(i), "string_channel"); + bm.push_back({i + 0.0, i + 1.0, i + 2.0, i + 3.0}, "vector_channel"); + item.a = i; + item.b = i; + bm.push_back(item, "struct_channel"); + + yarp::os::Time::delay(0.01); + } +} + +``` +The above snippet of code generates channels of different types. It produces the following output. +``` +>> buffer_manager_test_multiple_types + +buffer_manager_test_multiple_types = + + struct with fields: + + description_list: {[1×0 char]} + yarp_robot_name: [1×0 char] + struct_channel: [1×1 struct] + vector_channel: [1×1 struct] + string_channel: [1×1 struct] + double_channel: [1×1 struct] + int_channel: [1×1 struct] + +>> buffer_manager_test_multiple_types.string_channel + +ans = + + struct with fields: + + data: {3×1 cell} + dimensions: [1 3] + elements_names: {'element_0'} + name: 'string_channel' + timestamps: [1.6512e+09 1.6512e+09 1.6512e+09] + +>> buffer_manager_test_multiple_types.vector_channel + +ans = + + struct with fields: + + data: [4×1×3 double] + dimensions: [4 1 3] + elements_names: {4×1 cell} + name: 'vector_channel' + timestamps: [1.6512e+09 1.6512e+09 1.6512e+09] + +``` + ### Example configuration file It is possible to load the configuration of a BufferManager **from a json file** diff --git a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h index d6c0c6a..f18a752 100644 --- a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h +++ b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h @@ -69,7 +69,7 @@ static std::string getTypeName() return boost::core::demangle(typeid(T).name()); } -// matiomatioCppCanConcatenate::value is true when T has the T::value_type memeber. If this is true, then we check +// matiomatioCppCanConcatenate::value is true when T has the T::value_type member. If this is true, then we check // if T is either an Element, a Vector (but not a String), or a MultidimensionalArray template struct matioCppCanConcatenate : std::false_type {}; diff --git a/test/BufferManagerTest.cpp b/test/BufferManagerTest.cpp index 8a21618..1d156a8 100644 --- a/test/BufferManagerTest.cpp +++ b/test/BufferManagerTest.cpp @@ -348,17 +348,15 @@ TEST_CASE("Buffer Manager Test") SECTION("Multiple types") { - yarp::telemetry::experimental::BufferManager bm; yarp::telemetry::experimental::BufferConfig bufferConfig; - yarp::telemetry::experimental::ChannelInfo var_int{ "int", {1}}; - yarp::telemetry::experimental::ChannelInfo var_double{ "double", {1}}; - yarp::telemetry::experimental::ChannelInfo var_string{ "string", {1}}; - yarp::telemetry::experimental::ChannelInfo var_vector{ "vector", {4, 1}}; - yarp::telemetry::experimental::ChannelInfo var_struct{ "struct", {1}}; + yarp::telemetry::experimental::ChannelInfo var_int{ "int_channel", {1}}; + yarp::telemetry::experimental::ChannelInfo var_double{ "double_channel", {1}}; + yarp::telemetry::experimental::ChannelInfo var_string{ "string_channel", {1}}; + yarp::telemetry::experimental::ChannelInfo var_vector{ "vector_channel", {4, 1}}; + yarp::telemetry::experimental::ChannelInfo var_struct{ "struct_channel", {1}}; -// First add channels that will be handling empty buffers REQUIRE(bm.addChannel(var_int)); REQUIRE(bm.addChannel(var_double)); REQUIRE(bm.addChannel(var_string)); @@ -373,14 +371,14 @@ TEST_CASE("Buffer Manager Test") testStruct item; - for (int i = 0; i < 40; i++) { - bm.push_back(i, "int"); - bm.push_back(i * 1.0, "double"); - bm.push_back("iter" + std::to_string(i), "string"); - bm.push_back({i + 0.0, i + 1.0, i + 2.0, i + 3.0}, "vector"); + for (int i = 0; i < 10; i++) { + bm.push_back(i, "int_channel"); + bm.push_back(i * 1.0, "double_channel"); + bm.push_back("iter" + std::to_string(i), "string_channel"); + bm.push_back({i + 0.0, i + 1.0, i + 2.0, i + 3.0}, "vector_channel"); item.a = i; item.b = i; - bm.push_back(item, "struct"); + bm.push_back(item, "struct_channel"); yarp::os::Time::delay(0.01); } From 0063940f1452aa0427884a9e3587f0a2b07f23c1 Mon Sep 17 00:00:00 2001 From: Stefano Date: Thu, 28 Apr 2022 17:57:38 +0200 Subject: [PATCH 10/11] Attempt to fix Windows compilation by exporting Buffer and Record symbols manually. --- .../src/yarp/telemetry/experimental/Buffer.h | 3 ++- .../src/yarp/telemetry/experimental/Record.h | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libYARP_telemetry/src/yarp/telemetry/experimental/Buffer.h b/src/libYARP_telemetry/src/yarp/telemetry/experimental/Buffer.h index 8f88726..33d5912 100644 --- a/src/libYARP_telemetry/src/yarp/telemetry/experimental/Buffer.h +++ b/src/libYARP_telemetry/src/yarp/telemetry/experimental/Buffer.h @@ -9,6 +9,7 @@ #ifndef YARP_TELEMETRY_BUFFER_H #define YARP_TELEMETRY_BUFFER_H +#include #include #include #include @@ -21,7 +22,7 @@ namespace yarp::telemetry::experimental { * @brief A class to represent the buffer of yarp::telemetry::experimental::Record. * */ -class Buffer { +class YARP_telemetry_API Buffer { public: using iterator = typename boost::circular_buffer::iterator; diff --git a/src/libYARP_telemetry/src/yarp/telemetry/experimental/Record.h b/src/libYARP_telemetry/src/yarp/telemetry/experimental/Record.h index 682adbe..ae1954e 100644 --- a/src/libYARP_telemetry/src/yarp/telemetry/experimental/Record.h +++ b/src/libYARP_telemetry/src/yarp/telemetry/experimental/Record.h @@ -9,6 +9,8 @@ #ifndef YARP_TELEMETRY_RECORD_H #define YARP_TELEMETRY_RECORD_H +#include + #include #include @@ -20,7 +22,7 @@ namespace yarp::telemetry::experimental { * @brief A structure to represent a Record. * */ -struct Record +struct YARP_telemetry_API Record { double m_ts;/**< timestamp */ std::any m_datum;/**< the actual data of the record */ From a53690195f8dfcabf0b53280e7cdb603ebcae5ac Mon Sep 17 00:00:00 2001 From: Stefano Date: Mon, 2 May 2022 10:20:51 +0200 Subject: [PATCH 11/11] Added some docs on the function to convert to matioCpp. Edited the CHANGELOG. --- CHANGELOG.md | 1 + .../telemetry/experimental/BufferManager.h | 42 +++++++++++++++---- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cec37d..178fb8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add `yarp_robot_name` variable in the saved mat file - Added the possibility to specify the names of the each element of a channel. - BufferInfo is now a struct that contains the name, the dimension and the elements_names. +- Added the possibility to have channels with different types (including custom structs) in a single ``BufferManager``. ## [0.4.0] - 2022-02-22 diff --git a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h index f18a752..cc9e6d5 100644 --- a/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h +++ b/src/libYARP_telemetry/src/yarp/telemetry/experimental/BufferManager.h @@ -119,40 +119,56 @@ struct BufferInfo { m_convert_to_matioCpp(std::move(other.m_convert_to_matioCpp)){ } + // This method fills the m_convert_to_matioCpp lambda with a function able to convert the Buffer + // into a matioCpp variable. This method is called when pushing the first time to a channel, + // exploiting the fact that the push_back method is a template method + // (and thus it is clear the type of input, necessary when casting std::any). template void createMatioCppConvertFunction() { + // First check if we can use the matioCpp::make_variable function with the input type T. + // If not, the input type is not compatible with matioCpp static_assert(matioCpp::is_make_variable_callable::value, "The selected type cannot be used with matioCpp."); + // The lambda is generated only the first time that we push to a channel. + // We are enforcing that the type pushed with push_back is always the same. if (m_convert_to_matioCpp) { return; } + // The matioCpp::make_variable_output metafunction provides the type that would be output by matioCpp::make_variable using matioCppType = typename matioCpp::make_variable_output::type; + + // Start filling the m_convert_to_matioCpp lambda. The lambda will take as input the desired name and will output a matioCpp::Variable. m_convert_to_matioCpp = [this](const std::string& name) { size_t num_instants = this->m_buffer.size(); - //if the input data is numeric, then we concatenate on the last dimension + //--- + //SCALAR CASE + //if the input data is numeric (scalar, vector, multi-dimensional array), then we concatenate on the last dimension if constexpr (matioCppCanConcatenate::value) { + //the scalar types in matioCpp have the member T::value_type, that is the type of each single element. using elementType = typename matioCppType::value_type; dimensions_t fullDimensions = this->m_dimensions; fullDimensions.push_back(num_instants); + // The output is a multi dimensional array of dimensions n+1, where the last dimension is the number of time instants. matioCpp::MultiDimensionalArray outputVariable(name, fullDimensions); - size_t i = 0; + size_t t = 0; for (auto& _cell : this->m_buffer) { + //We convert std::any type using the input type T. const T& cellCasted = std::any_cast(_cell.m_datum); matioCpp::Span matioCppSpan; matioCppType matioCppVariable; - //We convert the cell to a matioCpp variable only if we are not able to create a Span. + //We convert the cell to a matioCpp variable only if we are not able to create a Span (for example in case of scalars). //In this way we avoid duplicating memory if constexpr (matioCpp::SpanUtils::is_make_span_callable::value) { @@ -165,21 +181,27 @@ struct BufferInfo { matioCppSpan = matioCppVariable.toSpan(); } - size_t startIndex = this->m_dimensions_factorial * i; //We concatenate on the last dimension. Suppose that the channel stores matrices of size 3x2. + size_t startIndex = this->m_dimensions_factorial * t; //We concatenate on the last dimension. Suppose that the channel stores matrices of size 3x2. //The output variable is a 3x2xn matrix, where n is the number of elements in the buffer. - //If we consider the output buffer as a linear vector, the element at position i starts from location 6*i - //and ends at 6*(i+1) + //If we consider the output buffer as a linear vector, the element at time t would start + //from location 6*t and end at 6*(t+1) + //matioCppSpan.size() should be equal to m_dimensions_factorial, but we avoid to perform this check for each input. + //Hence, with std::min we make sure to avoid reading or wrinting in wrong pieces of memory for (size_t i = 0; i < std::min(static_cast(matioCppSpan.size()), this->m_dimensions_factorial); ++i) { outputVariable[startIndex + i] = matioCppSpan[i]; //we copy the new element in the corresponding position inside the variable } - ++i; + ++t; } return outputVariable; } + + //--- + //STRUCT CASE else if constexpr(std::is_same_v) //if the input is a struct, we use a struct array { + //The output variable would be a struct array of dimensions t. matioCpp::StructArray outputVariable(name, {num_instants, 1}); size_t i = 0; @@ -188,7 +210,7 @@ struct BufferInfo { if (i == 0) { - outputVariable.addFields(element.fields()); + outputVariable.addFields(element.fields()); //Just for the first element, we specify the set of fields in the array } outputVariable.setElement(i, element); @@ -196,8 +218,12 @@ struct BufferInfo { } return outputVariable; } + + //--- + //CELL CASE (default) else //otherwise we use a cell array { + //The output variable would be a cell array of dimensions t. matioCpp::CellArray outputVariable(name, {num_instants, 1}); size_t i = 0;