Fix flaky tests for TLS (#824)

Summary:
X-link: facebookresearch/fbpcs#824

Pull Request resolved: #194

I think this is happening on stress runs because the helper function to generate certs just stores it in the same place (/tmp), so two parallel test runs could interfere with each other

Reviewed By: chualynn

Differential Revision: D35824204

fbshipit-source-id: b03bc0314076fdf6866a648e24f18712613dc15a

fbpcf/engine/communication/test/PartyCommunicationAgentTest.cpp

@@ -62,8 +62,7 @@ TEST(InMemoryPartyCommunicationAgentTest, testSendAndReceive) {
 }
 
 TEST(SocketPartyCommunicationAgentTest, testSendAndReceiveWithTls) {
-  auto tempdir = std::filesystem::temp_directory_path();
-  setUpTlsFiles(tempdir);
+  auto createdDir = setUpTlsFiles();
 
   std::random_device rd;
   std::default_random_engine defEngine(rd());
@@ -79,9 +78,9 @@ TEST(SocketPartyCommunicationAgentTest, testSendAndReceiveWithTls) {
       {1, {"127.0.0.1", intDistro(defEngine)}}};
 
   auto factory0 = std::make_unique<SocketPartyCommunicationAgentFactory>(
-      0, partyInfo, true, tempdir);
+      0, partyInfo, true, createdDir);
   auto factory1 = std::make_unique<SocketPartyCommunicationAgentFactory>(
-      1, partyInfo, true, tempdir);
+      1, partyInfo, true, createdDir);
 
   int size = 1048576; // 1024 ^ 2
   auto thread0 = std::thread(testAgentFactory, 0, size, std::move(factory0));
@@ -90,7 +89,7 @@ TEST(SocketPartyCommunicationAgentTest, testSendAndReceiveWithTls) {
   thread1.join();
   thread0.join();
 
-  deleteTlsFiles(tempdir);
+  deleteTlsFiles(createdDir);
 }
 
 TEST(SocketPartyCommunicationAgentTest, testSendAndReceiveWithoutTls) {

fbpcf/engine/communication/test/TlsCommunicationUtils.h

@@ -9,14 +9,30 @@
 #include <openssl/rsa.h>
 #include <openssl/x509.h>
 #include <stdio.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <filesystem>
 #include <fstream>
 #include <ostream>
+#include <random>
 
 namespace fbpcf::engine::communication {
 
 // creates a cert file, key file, and passphrase file
 // in the provided directory
-inline void setUpTlsFiles(std::string dir) {
+// it returns a sub-directory where the certs are stored.
+// this is so that when multiple runs are happening in
+// parallel, we don't overwrite certs that other tests generate
+inline std::string setUpTlsFiles() {
+  std::random_device rd;
+  std::mt19937_64 e(rd());
+  std::uniform_int_distribution<int64_t> dist(0, 1000000);
+  auto randomInt = dist(e);
+  std::string tmpDir = std::filesystem::temp_directory_path();
+  std::string dirToCreate = tmpDir + "/" + std::to_string(randomInt);
+
+  mkdir(dirToCreate.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
+
   EVP_PKEY* pkey;
   pkey = EVP_PKEY_new();
 
@@ -57,7 +73,7 @@ inline void setUpTlsFiles(std::string dir) {
 
   // write private key to file
   FILE* keyfile;
-  keyfile = fopen((dir + "/key.pem").c_str(), "wb");
+  keyfile = fopen((dirToCreate + "/key.pem").c_str(), "wb");
   PEM_write_PrivateKey(
       keyfile,
       pkey,
@@ -71,21 +87,21 @@ inline void setUpTlsFiles(std::string dir) {
 
   // write cert to file
   FILE* certfile;
-  certfile = fopen((dir + "/cert.pem").c_str(), "wb");
+  certfile = fopen((dirToCreate + "/cert.pem").c_str(), "wb");
   PEM_write_X509(certfile, x509);
   fclose(certfile);
 
   // write passphrase to file
-  std::ofstream passfile(dir + "/passphrase.pem");
+  std::ofstream passfile(dirToCreate + "/passphrase.pem");
   passfile << "test_passphrase";
   passfile.close();
+
+  return dirToCreate;
 }
 
 // deletes TLS files generated by the function above
 inline void deleteTlsFiles(std::string dir) {
-  remove((dir + "/passphrase.pem").c_str());
-  remove((dir + "/key.pem").c_str());
-  remove((dir + "/cert.pem").c_str());
+  std::filesystem::remove_all(dir);
 }
 
 } // namespace fbpcf::engine::communication