fix: unit test info number updates for new pango-lineages in test data

endToEndTests/test/info.test.js

@@ -7,7 +7,7 @@ describe('The /info endpoint', () => {
       .get('/info')
       .expect(200)
       .expect('Content-Type', 'application/json')
-      .expect({ nBitmapsSize: 3898, sequenceCount: 100, totalSize: 66467326 })
+      .expect({ nBitmapsSize: 3898, sequenceCount: 100, totalSize: 68915226 })
       .end(done);
   });
 
@@ -26,7 +26,7 @@ describe('The /info endpoint', () => {
           'bitmapContainerSizeStatistic'
         );
         expect(returnedInfo.bitmapContainerSizePerGenomeSection.bitmapContainerSizeStatistic).to.deep.equal({
-          numberOfArrayContainers: 716007,
+          numberOfArrayContainers: 745081,
           numberOfBitsetContainers: 0,
           numberOfRunContainers: 0,
           numberOfValuesStoredInArrayContainers: 2929577,
@@ -61,22 +61,22 @@ describe('The /info endpoint', () => {
 
         expect(returnedInfo).to.have.property('bitmapSizePerSymbol');
         expect(returnedInfo.bitmapSizePerSymbol).to.deep.equal({
-          '-': 5779958,
-          'A': 9190510,
-          'B': 5741376,
-          'C': 7859992,
-          'D': 5741376,
-          'G': 8006876,
-          'H': 5741376,
-          'K': 5741498,
-          'M': 5741466,
-          'N': 5741376,
-          'R': 5741426,
-          'S': 5741376,
-          'T': 9456412,
-          'V': 5741376,
-          'W': 5741426,
-          'Y': 5741406,
+          '-': 6019718,
+          'A': 9498982,
+          'B': 5980600,
+          'C': 8141784,
+          'D': 5980600,
+          'G': 8291636,
+          'H': 5980600,
+          'K': 5980730,
+          'M': 5980690,
+          'N': 5980600,
+          'R': 5980650,
+          'S': 5980600,
+          'T': 9770332,
+          'V': 5980600,
+          'W': 5980650,
+          'Y': 5980630,
         });
       })
       .end(done);

endToEndTests/test/queries/GroupByLineage.json

@@ -3,12 +3,8 @@
   "query": {
     "action": {
       "type": "Aggregated",
-      "groupByFields": [
-        "pango_lineage"
-      ],
-      "orderByFields": [
-        "pango_lineage"
-      ]
+      "groupByFields": ["pango_lineage"],
+      "orderByFields": ["pango_lineage"]
     },
     "filterExpression": {
       "type": "True"

endToEndTests/test/queries/GroupByLineageOrderByCountLimit.json

@@ -3,9 +3,7 @@
   "query": {
     "action": {
       "type": "Aggregated",
-      "groupByFields": [
-        "pango_lineage"
-      ],
+      "groupByFields": ["pango_lineage"],
       "orderByFields": [
         {
           "field": "count",

endToEndTests/test/queries/OffsetLimitOverlap.json

@@ -3,9 +3,7 @@
   "query": {
     "action": {
       "type": "Details",
-      "orderByFields": [
-        "gisaid_epi_isl"
-      ],
+      "orderByFields": ["gisaid_epi_isl"],
       "offset": 90,
       "limit": 90
     },

include/silo/storage/column_group.h

@@ -41,7 +41,9 @@ class DatabaseConfig;
 
 namespace silo::storage {
 
-struct ColumnPartitionGroup {
+class ColumnPartitionGroup {
+   friend class boost::serialization::access;
+
    template <class Archive>
    [[maybe_unused]] void serialize(Archive& archive, const uint32_t /* version */) {
       // clang-format off
@@ -66,6 +68,7 @@ struct ColumnPartitionGroup {
       // clang-format on
    }
 
+  public:
    std::vector<config::DatabaseMetadata> metadata;
 
    std::map<std::string, storage::column::StringColumnPartition&> string_columns;
@@ -85,7 +88,9 @@ struct ColumnPartitionGroup {
    ) const;
 };
 
-struct ColumnGroup {
+class ColumnGroup {
+   friend class boost::serialization::access;
+
    template <class Archive>
    [[maybe_unused]] void serialize(Archive& archive, const uint32_t /* version */) {
       // clang-format off
@@ -110,6 +115,7 @@ struct ColumnGroup {
       // clang-format on
    }
 
+  public:
    std::vector<config::DatabaseMetadata> metadata;
 
    std::map<std::string, storage::column::StringColumn> string_columns;

src/silo/config/database_config.test.cpp

@@ -118,28 +118,29 @@ TEST(DatabaseConfigReader, shouldReadConfigWithCorrectParameters) {
 
 TEST(DatabaseConfigReader, shouldThrowExceptionWhenConfigFileDoesNotExist) {
    ASSERT_THROW(
-      DatabaseConfigReader().readConfig("testBaseData/does_not_exist.yaml"), std::runtime_error
+      (void)DatabaseConfigReader().readConfig("testBaseData/does_not_exist.yaml"),
+      std::runtime_error
    );
 }
 
 TEST(DatabaseConfigReader, shouldThrowErrorForInvalidMetadataType) {
    ASSERT_THROW(
-      DatabaseConfigReader().readConfig(
+      (void)DatabaseConfigReader().readConfig(
          "testBaseData/test_database_config_with_invalid_metadata_type.yaml"
       ),
       ConfigException
    );
 }
 
 TEST(DatabaseConfigReader, shouldNotThrowIfThereAreAdditionalEntries) {
-   ASSERT_NO_THROW(DatabaseConfigReader().readConfig(
+   ASSERT_NO_THROW((void)DatabaseConfigReader().readConfig(
       "testBaseData/test_database_config_with_additional_entries.yaml"
    ));
 }
 
 TEST(DatabaseConfigReader, shouldThrowIfTheConfigHasAnInvalidStructure) {
    ASSERT_THROW(
-      DatabaseConfigReader().readConfig(
+      (void)DatabaseConfigReader().readConfig(
          "testBaseData/test_database_config_with_invalid_structure.yaml"
       ),
       YAML::InvalidNode

src/silo/database.test.cpp

@@ -40,10 +40,10 @@ TEST(DatabaseTest, shouldReturnCorrectDatabaseInfo) {
    const auto simple_info = database.getDatabaseInfo();
 
    EXPECT_EQ(
-      detailed_info.bitmap_size_per_symbol.size_in_bytes.at(silo::NUCLEOTIDE_SYMBOL::A), 9190510
+      detailed_info.bitmap_size_per_symbol.size_in_bytes.at(silo::NUCLEOTIDE_SYMBOL::A), 9498982
    );
    EXPECT_EQ(
-      detailed_info.bitmap_size_per_symbol.size_in_bytes.at(silo::NUCLEOTIDE_SYMBOL::GAP), 5779958
+      detailed_info.bitmap_size_per_symbol.size_in_bytes.at(silo::NUCLEOTIDE_SYMBOL::GAP), 6019718
    );
 
    EXPECT_EQ(
@@ -63,18 +63,18 @@ TEST(DatabaseTest, shouldReturnCorrectDatabaseInfo) {
    );
 
    EXPECT_EQ(
-      detailed_info.bitmap_container_size_per_genome_section.total_bitmap_size_computed, 103449226
+      detailed_info.bitmap_container_size_per_genome_section.total_bitmap_size_computed, 107509402
    );
    EXPECT_EQ(
-      detailed_info.bitmap_container_size_per_genome_section.total_bitmap_size_frozen, 55370197
+      detailed_info.bitmap_container_size_per_genome_section.total_bitmap_size_frozen, 57429359
    );
    EXPECT_EQ(
       detailed_info.bitmap_container_size_per_genome_section.bitmap_container_size_statistic
          .total_bitmap_size_array_containers,
       5859154
    );
 
-   EXPECT_EQ(simple_info.total_size, 66467326);
+   EXPECT_EQ(simple_info.total_size, 68915226);
    EXPECT_EQ(simple_info.sequence_count, 100);
    EXPECT_EQ(simple_info.n_bitmaps_size, 3898);
 }

src/silo/preprocessing/pango_lineage_count.test.cpp

@@ -17,13 +17,13 @@ TEST(PangoLineageCounts, buildPangoLineageCounts) {
       database_config
    );
 
-   ASSERT_EQ(result.pango_lineage_counts.size(), 24);
+   ASSERT_EQ(result.pango_lineage_counts.size(), 25);
    ASSERT_EQ(result.pango_lineage_counts[0].pango_lineage.value, "");
    ASSERT_EQ(result.pango_lineage_counts[0].count_of_sequences, 1);
    ASSERT_EQ(result.pango_lineage_counts[1].pango_lineage.value, "B.1");
    ASSERT_EQ(result.pango_lineage_counts[1].count_of_sequences, 3);
    ASSERT_EQ(result.pango_lineage_counts[7].pango_lineage.value, "B.1.1.7");
    ASSERT_EQ(result.pango_lineage_counts[7].count_of_sequences, 48);
-   ASSERT_EQ(result.pango_lineage_counts[23].pango_lineage.value, "B.1.617.2.9.2");
+   ASSERT_EQ(result.pango_lineage_counts[23].pango_lineage.value, "XA.1");
    ASSERT_EQ(result.pango_lineage_counts[23].count_of_sequences, 1);
 }

src/silo/preprocessing/partition.cpp

@@ -259,3 +259,64 @@ std::size_t std::hash<silo::preprocessing::PartitionChunk>::operator()(
           (hash<uint32_t>()(partition_chunk.chunk) << 3) +
           (hash<uint32_t>()(partition_chunk.chunk) >> 2);
 }
+
+namespace silo::common {
+
+// NOLINTNEXTLINE(readability-identifier-naming)
+void from_json(
+   const nlohmann::json& js_object,
+   silo::common::UnaliasedPangoLineage& pango_lineage
+) {
+   pango_lineage.value = js_object;
+}
+
+// NOLINTNEXTLINE(readability-identifier-naming)
+void to_json(nlohmann::json& js_object, const silo::common::UnaliasedPangoLineage& pango_lineage) {
+   js_object = pango_lineage.value;
+}
+}  // namespace silo::common
+
+template <>
+struct nlohmann::adl_serializer<silo::preprocessing::Chunk> {
+   // NOLINTNEXTLINE(readability-identifier-naming)
+   static silo::preprocessing::Chunk from_json(const nlohmann::json& js_object) {
+      return silo::preprocessing::Chunk{
+         js_object["lineages"].template get<std::vector<silo::common::UnaliasedPangoLineage>>(),
+         js_object["count"].template get<uint32_t>()};
+   }
+
+   // NOLINTNEXTLINE(readability-identifier-naming)
+   static void to_json(nlohmann::json& js_object, const silo::preprocessing::Chunk& chunk) {
+      js_object["lineages"] = chunk.getPangoLineages();
+      js_object["count"] = chunk.getCountOfSequences();
+   }
+};
+
+template <>
+struct nlohmann::adl_serializer<silo::preprocessing::Partition> {
+   // NOLINTNEXTLINE(readability-identifier-naming)
+   static silo::preprocessing::Partition from_json(const nlohmann::json& js_object) {
+      return silo::preprocessing::Partition{
+         js_object.template get<std::vector<silo::preprocessing::Chunk>>()};
+   }
+
+   // NOLINTNEXTLINE(readability-identifier-naming)
+   static void to_json(nlohmann::json& js_object, const silo::preprocessing::Partition& partition) {
+      js_object = partition.getChunks();
+   }
+};
+
+namespace silo::preprocessing {
+
+void Partitions::save(std::ostream& output_file) const {
+   const nlohmann::json json(partitions);
+   output_file << json.dump(4);
+}
+
+Partitions Partitions::load(std::istream& input_file) {
+   nlohmann::json json;
+   json = nlohmann::json::parse(input_file);
+   const std::vector<Partition> partitions = json.get<std::vector<Partition>>();
+   return Partitions{partitions};
+}
+}  // namespace silo::preprocessing

src/silo/preprocessing/partition.test.cpp

@@ -1,5 +1,6 @@
 #include "silo/preprocessing/partition.h"
 
+#include "silo/common/pango_lineage.h"
 #include "silo/preprocessing/preprocessing_config.h"
 
 #include <gtest/gtest.h>
@@ -11,12 +12,24 @@ using silo::preprocessing::Partitions;
 
 Partitions createSimplePartitionsObject() {
    std::vector<Chunk> chunks1;
-   chunks1.emplace_back(std::vector<std::string>{{"A.1", "A.3", "B.1", "A.2"}}, 8);
-   chunks1.emplace_back(std::vector<std::string>{{"B.2", "C.3", "C.1", "C.2"}}, 11123);
+   chunks1.emplace_back(
+      std::vector<silo::common::UnaliasedPangoLineage>{{{"A.1"}, {"A.3"}, {"B.1"}, {"A.2"}}}, 8
+   );
+   chunks1.emplace_back(
+      std::vector<silo::common::UnaliasedPangoLineage>{{{"B.2"}, {"C.3"}, {"C.1"}, {"C.2"}}}, 11123
+   );
    Partition partition1(std::move(chunks1));
    std::vector<Chunk> chunks2;
-   chunks2.emplace_back(std::vector<std::string>{{"XY.1", "XY.3", "XY.A.A.A.3", "XY.2312"}}, 123);
-   chunks2.emplace_back(std::vector<std::string>{{"XT.1", "XT.3", "XTA.A.3", "XT.2312"}}, 512);
+   chunks2.emplace_back(
+      std::vector<silo::common::UnaliasedPangoLineage>{
+         {{"XY.1"}, {"XY.3"}, {"XY.A.A.A.3"}, {"XY.2312"}}},
+      123
+   );
+   chunks2.emplace_back(
+      std::vector<silo::common::UnaliasedPangoLineage>{
+         {{"XT.1"}, {"XT.3"}, {"XTA.A.3"}, {"XT.2312"}}},
+      512
+   );
    Partition partition2(std::move(chunks2));
    return Partitions{{std::move(partition1), std::move(partition2)}};
 }
@@ -81,7 +94,7 @@ TEST(Partitions, shouldSaveSimpleConfig) {
    ASSERT_EQ(file_contents, expected_file_contents);
 }
 
-TEST(Partitions, shouldSaveAndLoadSimpleConfig) {
+TEST(Partitions, shouldSaveAndLoadSimplePartitionsObject) {
    const Partitions partitions = createSimplePartitionsObject();
    std::ofstream out_file("output/test.partitions");
    partitions.save(out_file);

src/silo/storage/database_partition.cpp

@@ -16,7 +16,7 @@ class StringColumnPartition;
 }  // namespace storage::column
 
 DatabasePartition::DatabasePartition(std::vector<silo::preprocessing::Chunk> chunks)
-    : chunks(chunks) {}
+    : chunks(std::move(chunks)) {}
 
 const std::vector<preprocessing::Chunk>& DatabasePartition::getChunks() const {
    return chunks;