feat(dup): add dsn_replica_dup_test to CI testing and fix http outputs

scripts/linux/build.sh

@@ -190,7 +190,7 @@ echo "################################# start testing ##########################
 if [ -z "$TEST_MODULE" ]
 then
     # supported test module
-    TEST_MODULE="dsn.core.tests,dsn.tests,dsn_nfs_test,dsn.replication.simple_kv,dsn.rep_tests.simple_kv,dsn.meta.test,dsn.replica.test,dsn_http_test"
+    TEST_MODULE="dsn.core.tests,dsn.tests,dsn_nfs_test,dsn.replication.simple_kv,dsn.rep_tests.simple_kv,dsn.meta.test,dsn.replica.test,dsn_http_test,dsn_replica_dup_test"
 fi
 
 echo "TEST_MODULE=$TEST_MODULE"

src/dist/replication/common/duplication_common.cpp

@@ -123,13 +123,6 @@ static nlohmann::json duplication_entry_to_json(const duplication_entry &ent)
         {"remote", ent.remote},
         {"status", duplication_status_to_string(ent.status)},
     };
-    if (ent.__isset.not_confirmed) {
-        nlohmann::json sub_json;
-        for (const auto &p : ent.not_confirmed) {
-            sub_json[std::to_string(p.first)] = p.second;
-        }
-        json["not_confirmed_mutations_num"] = sub_json;
-    }
     if (ent.__isset.progress) {
         nlohmann::json sub_json;
         for (const auto &p : ent.progress) {

src/dist/replication/lib/duplication/test/replica_http_service_test.cpp

@@ -15,18 +15,13 @@ class replica_http_service_test : public duplication_test_base
 TEST_F(replica_http_service_test, query_duplication_handler)
 {
     auto pri = stub->add_primary_replica(1, 1);
-    auto sec = stub->add_non_primary_replica(1, 2);
-    sec->as_secondary();
 
     // primary confirmed_decree
     duplication_entry ent;
     ent.dupid = 1583306653;
-    ent.progress[pri->get_gpid().get_partition_index()] = 1000;
+    ent.progress[pri->get_gpid().get_partition_index()] = 0;
     ent.status = duplication_status::DS_PAUSE;
-    auto dup = dsn::make_unique<replica_duplicator>(ent, pri);
-    add_dup(pri, std::move(dup));
-
-    sec->get_duplication_manager()->update_confirmed_decree_if_secondary(899);
+    add_dup(pri, make_unique<replica_duplicator>(ent, pri));
 
     replica_http_service http_svc(stub.get());
 
@@ -43,16 +38,18 @@ TEST_F(replica_http_service_test, query_duplication_handler)
     http_svc.query_duplication_handler(req, resp);
     ASSERT_EQ(resp.status_code, http_status_code::bad_request);
 
+    auto dup = find_dup(pri, ent.dupid);
+    dup->update_progress(duplication_progress().set_last_decree(1050).set_confirmed_decree(1000));
+    pri->set_last_committed_decree(1100);
     req.query_args["appid"] = "1";
     http_svc.query_duplication_handler(req, resp);
     ASSERT_EQ(resp.status_code, http_status_code::ok);
-    ASSERT_EQ(resp.body,
-              R"({)"
-              R"("1583306653":)"
-              R"({"1.1":{"confirmed_decree":1000,"duplicating":false,"last_decree":1000}},)"
-              R"("non-primaries":)"
-              R"({"1.2":{"confirmed_decree":899}})"
-              R"(})");
+    ASSERT_EQ(
+        resp.body,
+        R"({)"
+        R"("1583306653":)"
+        R"({"1.1":{"duplicating":false,"not_confirmed_mutations_num":100,"not_duplicated_mutations_num":50}})"
+        R"(})");
 }
 
 } // namespace replication

src/dist/replication/lib/replica_http_service.cpp

@@ -46,8 +46,8 @@ void replica_http_service::query_duplication_handler(const http_request &req, ht
     for (const auto &s : states) {
         json[std::to_string(s.first)][s.second.id.to_string()] = nlohmann::json{
             {"duplicating", s.second.duplicating},
-            {"not_confirmed", s.second.not_confirmed},
-            {"not_duplicated", s.second.not_duplicated},
+            {"not_confirmed_mutations_num", s.second.not_confirmed},
+            {"not_duplicated_mutations_num", s.second.not_duplicated},
         };
     }
     resp.status_code = http_status_code::ok;

src/dist/replication/meta_server/duplication/duplication_info.cpp

@@ -201,16 +201,9 @@ void duplication_info::append_if_valid_for_query(
 
     entry_list.emplace_back(to_duplication_entry());
     duplication_entry &ent = entry_list.back();
-    ent.__isset.not_confirmed = true;
     // the confirmed decree is not useful for displaying
-    // the overall state of duplication, instead we show pending mutations.
+    // the overall state of duplication
     ent.__isset.progress = false;
-    for (const partition_configuration &part : app.partitions) {
-        int pid = part.pid.get_partition_index();
-        auto it = _progress.find(pid);
-        int64_t pending = part.last_committed_decree - it->second.stored_decree;
-        ent.not_confirmed[pid] = std::max(pending, int64_t(0));
-    }
 }
 
 } // namespace replication

src/dist/replication/test/meta_test/unit_test/meta_duplication_service_test.cpp

@@ -675,12 +675,10 @@ TEST_F(meta_duplication_service_test, query_duplication_handler)
     char ts_buf[32];
     utils::time_ms_to_date_time(
         static_cast<uint64_t>(dup->create_timestamp_ms), ts_buf, sizeof(ts_buf));
-    ASSERT_EQ(
-        fake_resp.body,
-        std::string() +
-            R"({"1":{"create_ts":")" + ts_buf + R"(","dupid":)" + std::to_string(dup->id) +
-            R"(,"not_confirmed_mutations_num":{"0":1,"1":1,"2":1,"3":1,"4":1,"5":1,"6":1,"7":1})"
-            R"(,"remote":"slave-cluster","status":"DS_START"},"appid":2})");
+    ASSERT_EQ(fake_resp.body,
+              std::string() + R"({"1":{"create_ts":")" + ts_buf + R"(","dupid":)" +
+                  std::to_string(dup->id) +
+                  R"(,"remote":"slave-cluster","status":"DS_START"},"appid":2})");
 }
 
 } // namespace replication