[ML] Fix deletion forecast overflow (elastic#110)

Use forecast ID to create a subdirectory when overflowing models to disk for forecasting. This fixes a failing 2nd forecast call because the 1st one deleted the tmp directory.
davidkyle · Jun 1, 2018 · 782972d · 782972d
1 parent 22f2c6b
commit 782972d
Show file tree

Hide file tree

Showing 5 changed files with 39 additions and 15 deletions.
diff --git a/lib/api/CForecastRunner.cc b/lib/api/CForecastRunner.cc
@@ -350,6 +350,10 @@ bool CForecastRunner::pushForecastJob(const std::string& controlMessage,
         LOG_INFO(<< "Forecast of large model requested (requires "
                  << std::to_string(1 + (totalMemoryUsage >> 20)) << " MB), using disk.");
 
+        // create a subdirectory using the unique forecast id
+        temporaryFolder /= forecastJob.s_ForecastId;
+        forecastJob.s_TemporaryFolder = temporaryFolder.string();
+
         boost::system::error_code errorCode;
         boost::filesystem::create_directories(temporaryFolder, errorCode);
         if (errorCode) {

diff --git a/lib/seccomp/CSystemCallFilter_Linux.cc b/lib/seccomp/CSystemCallFilter_Linux.cc
@@ -51,26 +51,27 @@ const struct sock_filter FILTER[] = {
     // Load the system call number into accumulator
     BPF_STMT(BPF_LD | BPF_W | BPF_ABS, SECCOMP_DATA_NR_OFFSET),
     // Only applies to X86_64 arch. Jump to disallow for calls using the x32 ABI
-    BPF_JUMP(BPF_JMP | BPF_JGT | BPF_K, UPPER_NR_LIMIT, 34, 0),
+    BPF_JUMP(BPF_JMP | BPF_JGT | BPF_K, UPPER_NR_LIMIT, 35, 0),
     // If any sys call filters are added or removed then the jump
     // destination for each statement including the one above must
     // be updated accordingly
 
     // Allowed sys calls, jump to return allow on match
-    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_read, 34, 0),
-    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_write, 33, 0),
-    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_writev, 32, 0),
-    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_lseek, 31, 0),
-    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_lstat, 30, 0),
-    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_readlink, 29, 0),
-    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_stat, 28, 0),
-    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_fstat, 27, 0),
-    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_open, 26, 0),
-    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_close, 25, 0),
-    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_connect, 24, 0),
-    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_clone, 23, 0),
-    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_statfs, 22, 0),
-    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_dup2, 21, 0),
+    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_read, 35, 0),
+    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_write, 34, 0),
+    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_writev, 33, 0),
+    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_lseek, 32, 0),
+    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_lstat, 31, 0),
+    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_readlink, 30, 0),
+    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_stat, 29, 0),
+    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_fstat, 28, 0),
+    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_open, 27, 0),
+    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_close, 26, 0),
+    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_connect, 25, 0),
+    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_clone, 24, 0),
+    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_statfs, 23, 0),
+    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_dup2, 22, 0),
+    BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_mkdir, 21, 0), // for forecast temp storage
     BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_rmdir, 20, 0), // for forecast temp storage
     BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_getdents, 19, 0), // for forecast temp storage
     BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_openat, 18, 0), // for forecast temp storage

diff --git a/lib/seccomp/unittest/CSystemCallFilterTest.cc b/lib/seccomp/unittest/CSystemCallFilterTest.cc
@@ -18,6 +18,9 @@
 
 #include <test/CTestTmpDir.h>
 
+#include <boost/filesystem.hpp>
+#include <boost/system/error_code.hpp>
+
 #include <cstdlib>
 #include <string>
 
@@ -172,6 +175,8 @@ void CSystemCallFilterTest::testSystemCallFilter() {
     // Operations that must function after seccomp is initialised
     openPipeAndRead(readPipeName);
     openPipeAndWrite(writePipeName);
+
+    makeAndRemoveDirectory(ml::test::CTestTmpDir::tmpDir());
 }
 
 void CSystemCallFilterTest::openPipeAndRead(const std::string& filename) {
@@ -229,3 +234,15 @@ void CSystemCallFilterTest::openPipeAndWrite(const std::string& filename) {
     CPPUNIT_ASSERT_EQUAL(TEST_SIZE, threadReader.data().length());
     CPPUNIT_ASSERT_EQUAL(std::string(TEST_SIZE, TEST_CHAR), threadReader.data());
 }
+
+void CSystemCallFilterTest::makeAndRemoveDirectory(const std::string& dirname) {
+
+    boost::filesystem::path temporaryFolder(dirname);
+    temporaryFolder /= "test-directory";
+
+    boost::system::error_code errorCode;
+    boost::filesystem::create_directories(temporaryFolder, errorCode);
+    CPPUNIT_ASSERT(errorCode == 0);
+    boost::filesystem::remove_all(temporaryFolder, errorCode);
+    CPPUNIT_ASSERT(errorCode == 0);
+}
diff --git a/lib/seccomp/unittest/CSystemCallFilterTest.h b/lib/seccomp/unittest/CSystemCallFilterTest.h
@@ -19,6 +19,7 @@ class CSystemCallFilterTest : public CppUnit::TestFixture {
 private:
     void openPipeAndRead(const std::string& filename);
     void openPipeAndWrite(const std::string& filename);
+    void makeAndRemoveDirectory(const std::string& dirname);
 };
 
 #endif // INCLUDED_CSystemCallFilterTest_h
diff --git a/lib/seccomp/unittest/Makefile b/lib/seccomp/unittest/Makefile
@@ -8,6 +8,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
 TARGET=ml_test$(EXE_EXT)
 
 USE_BOOST=1
+USE_BOOST_FILESYSTEM_LIBS=1
 LIBS:=$(LIB_ML_SECCOMP)
 LDFLAGS:=$(ML_SECCOMP_LDFLAGS)