From 8fbff9e262297a1f22b13d938cf3f97c44fa04c9 Mon Sep 17 00:00:00 2001
From: Marcin Junczys-Dowmunt <marcinjd@microsoft.com>
Date: Thu, 4 Mar 2021 04:47:50 +0000
Subject: [PATCH 1/4] unit tests for binary file operations

---
 src/tests/units/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/tests/units/CMakeLists.txt b/src/tests/units/CMakeLists.txt
index 1ea7da2ee..7745fee14 100644
--- a/src/tests/units/CMakeLists.txt
+++ b/src/tests/units/CMakeLists.txt
@@ -6,6 +6,7 @@ set(UNIT_TESTS
     attention_tests
     fastopt_tests
     utils_tests
+    binary_tests
     # cosmos_tests # optional, uncomment to test with specific files.
 )
 

From 6671ebceef4152cc632c8bd44e339a48814b0693 Mon Sep 17 00:00:00 2001
From: Marcin Junczys-Dowmunt <marcinjd@microsoft.com>
Date: Thu, 4 Mar 2021 04:49:59 +0000
Subject: [PATCH 2/4] adjust changelog

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6c0514b89..e9fd85156 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Correct training with fp16 via `--fp16`. 
 - Dynamic cost-scaling with `--cost-scaling`.
 - Dynamic gradient-scaling with `--dynamic-gradient-scaling`.
+- Add unit tests for binary files.
 
 ### Fixed
 - Find MKL installed under Ubuntu 20.04 via apt-get

From f9653dcb45d691cb9fe6426118bb3d6aefafcdd4 Mon Sep 17 00:00:00 2001
From: Marcin Junczys-Dowmunt <marcinjd@microsoft.com>
Date: Thu, 4 Mar 2021 04:53:23 +0000
Subject: [PATCH 3/4] actually add file

---
 src/tests/units/binary_tests.cpp | 61 ++++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)
 create mode 100644 src/tests/units/binary_tests.cpp

diff --git a/src/tests/units/binary_tests.cpp b/src/tests/units/binary_tests.cpp
new file mode 100644
index 000000000..569b4dec1
--- /dev/null
+++ b/src/tests/units/binary_tests.cpp
@@ -0,0 +1,61 @@
+#include "catch.hpp"
+#include "common/binary.h"
+#include "common/file_stream.h"
+
+#include "3rd_party/mio/mio.hpp"
+
+using namespace marian;
+
+TEST_CASE("a few operations on binary files", "[binary]") {
+
+  SECTION("Save two items to temporary binary file and then load and map") {
+
+    // Create a temporary file that we will only use for the file name
+    io::TemporaryFile temp("/tmp/", /*earlyUnlink=*/false);
+    io::Item item1, item2;
+
+    {
+      std::vector<float>    v1 = { 3.14, 2.71, 1.0, 0.0, 1.41 };
+      std::vector<uint16_t> v2 = { 5, 4, 3, 2, 1, 0 };
+
+      item1.name  = "item1";
+      item1.shape = { 5, 1 };
+      item1.type  = Type::float32;
+      item1.bytes.resize(v1.size() * sizeof(float));
+      std::copy((char*)v1.data(), (char*)v1.data() + v1.size() * sizeof(float), item1.bytes.data());
+
+      item2.name  = "item2";
+      item2.shape = { 2, 3 };
+      item2.type = Type::uint16;
+      item2.bytes.resize(v2.size() * sizeof(uint32_t));
+      std::copy((char*)v2.data(), (char*)v2.data() + v2.size() * sizeof(uint16_t), item2.bytes.data());
+      
+      std::vector<io::Item> items = {item1, item2};
+      io::binary::saveItems(temp.getFileName(), items);
+    }
+
+    { // test loading
+      std::vector<io::Item> items;
+      io::binary::loadItems(temp.getFileName(), items);
+
+      CHECK( item1.name == items[0].name );
+      CHECK( item2.name == items[1].name );
+    
+      CHECK( std::equal(item1.data(), item1.data() + item1.size(), items[0].data()) );
+      CHECK( std::equal(item2.data(), item2.data() + item2.size(), items[1].data()) );
+    }
+  
+    { // test mmapping
+      mio::mmap_source mmap(temp.getFileName());
+      
+      std::vector<io::Item> items;
+      io::binary::loadItems(mmap.data(), items, /*mapped=*/true);
+
+      CHECK( item1.name == items[0].name );
+      CHECK( item2.name == items[1].name );
+
+      CHECK( std::equal(item1.data(), item1.data() + item1.size(), items[0].data()) );
+      CHECK( std::equal(item2.data(), item2.data() + item2.size(), items[1].data()) );
+    }
+  }
+}

From 3eca5ed8829f729f650819729994638201eda22e Mon Sep 17 00:00:00 2001
From: Roman Grundkiewicz <rgrundkiewicz@gmail.com>
Date: Fri, 19 Mar 2021 09:19:28 +0000
Subject: [PATCH 4/4] Set file_ in TemporaryFile for MSVC

---
 src/common/file_stream.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/common/file_stream.cpp b/src/common/file_stream.cpp
index 78cbb12fa..14da7c9b4 100644
--- a/src/common/file_stream.cpp
+++ b/src/common/file_stream.cpp
@@ -161,6 +161,7 @@ void TemporaryFile::MakeTemp(const std::string &base) {
   int fd = open(name, oflag, _S_IREAD | _S_IWRITE);
   ABORT_IF(fd == -1, "Error while making a temporary based on '{}'", base);
 
+  file_ = std::string(name);
 #else
   // create temp file
   std::string name(base);