temp save

JaySon-Huang · Nov 1, 2023 · 388e44d · 388e44d
1 parent 668c1a3
commit 388e44d
Show file tree

Hide file tree

Showing 2 changed files with 33 additions and 7 deletions.
diff --git a/dbms/src/Storages/S3/S3Common.cpp b/dbms/src/Storages/S3/S3Common.cpp
@@ -1132,6 +1132,7 @@ void rawListPrefix(
     {
         req.SetDelimiter(String(delimiter));
     }
+    req.SetEncodingType(Aws::S3::Model::EncodingType::url);
 
     static auto log = Logger::get("S3RawListPrefix");
 

diff --git a/dbms/src/Storages/S3/tests/gtest_s3client.cpp b/dbms/src/Storages/S3/tests/gtest_s3client.cpp
@@ -27,8 +27,10 @@ class S3ClientTest : public ::testing::Test
     void SetUp() override
     {
         client = ClientFactory::instance().sharedTiFlashClient();
+#if 0
         ::DB::tests::TiFlashTestEnv::deleteBucket(*client);
         ::DB::tests::TiFlashTestEnv::createBucketIfNotExist(*client);
+#endif
     }
 
     std::shared_ptr<TiFlashS3Client> client;
@@ -44,6 +46,7 @@ CATCH
 TEST_F(S3ClientTest, UploadRead)
 try
 {
+#if 0
     deleteObject(*client, "s999/manifest/mf_1");
     ASSERT_FALSE(objectExists(*client, "s999/manifest/mf_1"));
     uploadEmptyFile(*client, "s999/manifest/mf_1");
@@ -56,18 +59,28 @@ try
     uploadEmptyFile(*client, "s999/data/dat_790_0");
 
     uploadEmptyFile(*client, "s999/abcd");
+#endif
 
     {
         Strings prefixes;
-        listPrefixWithDelimiter(*client, "s999/", "/", [&](const Aws::S3::Model::CommonPrefix & p) {
-            prefixes.emplace_back(p.GetPrefix());
-            return PageResult{.num_keys = 1, .more = true};
-        });
+        rawListPrefix(
+            *client,
+            client->bucket(),
+            "tiflash_ut/",
+            "/",
+            [&](const Aws::S3::Model::ListObjectsV2Result & result) {
+                const auto & ps = result.GetCommonPrefixes();
+                for (const auto & p : ps)
+                {
+                    prefixes.emplace_back(p.GetPrefix());
+                }
+                return PageResult{.num_keys = ps.size(), .more = true};
+            });
         ASSERT_EQ(prefixes.size(), 2) << fmt::format("{}", prefixes);
-        EXPECT_EQ(prefixes[0], "s999/data/");
-        EXPECT_EQ(prefixes[1], "s999/manifest/");
+        EXPECT_EQ(prefixes[0], client->root() + "s999/data/");
+        EXPECT_EQ(prefixes[1], client->root() + "s999/manifest/");
     }
-
+#if 0
     // check the keys with raw `LIST` request
     {
         Strings prefixes;
@@ -88,6 +101,18 @@ try
         EXPECT_EQ(prefixes[0], client->root() + "s999/data/");
         EXPECT_EQ(prefixes[1], client->root() + "s999/manifest/");
     }
+
+    {
+        Strings prefixes;
+        listPrefixWithDelimiter(*client, "s999/", "/", [&](const Aws::S3::Model::CommonPrefix & p) {
+            prefixes.emplace_back(p.GetPrefix());
+            return PageResult{.num_keys = 1, .more = true};
+        });
+        ASSERT_EQ(prefixes.size(), 2) << fmt::format("{}", prefixes);
+        EXPECT_EQ(prefixes[0], "s999/data/");
+        EXPECT_EQ(prefixes[1], "s999/manifest/");
+    }
+#endif
 }
 CATCH
-Original file line number
+Diff line change
@@ Expand Up / @@ -1132,6 +1132,7 @@ void rawListPrefix( @@
         {
             req.SetDelimiter(String(delimiter));
         }
+        req.SetEncodingType(Aws::S3::Model::EncodingType::url);
         static auto log = Logger::get("S3RawListPrefix");
@@ Expand Down @@