Follow-up of PR153

.github/workflows/build_and_test_mac.yml

@@ -77,4 +77,4 @@ jobs:
 
     - name: Run PyTest
       working-directory: ${{github.workspace}}
-      run: python -m pytest tests/python/
+      run: python -m pytest tests/python/ -vv

.github/workflows/build_and_test_ubuntu.yml

@@ -76,4 +76,4 @@ jobs:
 
     - name: Run PyTest
       working-directory: ${{github.workspace}}
-      run: python -m pytest tests/python/
+      run: python -m pytest tests/python/  -vv

.github/workflows/build_and_test_windows.yml

@@ -80,4 +80,4 @@ jobs:
 
     - name: Run PyTest
       working-directory: ${{github.workspace}}
-      run: python -m pytest tests/python/
+      run: python -m pytest tests/python/  -vv

CMakeLists.txt

@@ -412,7 +412,7 @@ endif()
 
 if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
 	SET(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} -fPIC")
-	list(APPEND Nyxus_LIBRARIES stdc++fs)
+	list(APPEND Nyxus_LIBRARIES stdc++fs -static-libgcc -static-libstdc++)
 endif()
 
 if(BUILD_LIB)

src/nyx/arrow_output_stream.cpp

@@ -1,34 +1,45 @@
 #include "arrow_output_stream.h"
-
 #ifdef USE_ARROW
 
-std::shared_ptr<ApacheArrowWriter> ArrowOutputStream::create_arrow_file(const Nyxus::SaveOption& arrow_file_type,
+std::tuple<bool, std::optional<std::string>> ArrowOutputStream::create_arrow_file(const Nyxus::SaveOption& arrow_file_type,
                                                          const std::string& arrow_file_path,
                                                          const std::vector<std::string>& header) {
 
-    if(arrow_file_path != "" && !fs::is_directory(arrow_file_path) && !(Nyxus::ends_with_substr(arrow_file_path, ".arrow") || Nyxus::ends_with_substr(arrow_file_path, ".feather") || Nyxus::ends_with_substr(arrow_file_path, ".parquet"))) {
-        throw std::invalid_argument("The arrow file path must end in \".arrow\"");
-    }
 
-    if (arrow_file_type != Nyxus::SaveOption::saveArrowIPC && arrow_file_type != Nyxus::SaveOption::saveParquet) {
-        throw std::invalid_argument("The valid save options are Nyxus::SaveOption::saveArrowIPC or Nyxus::SaveOption::saveParquet.");
-    }
 
-    std::string extension = (arrow_file_type == Nyxus::SaveOption::saveParquet) ? ".parquet" : ".arrow";
+    auto valid_extension = [&arrow_file_path](){
+        if(	arrow_file_path != "" && 
+            !fs::is_directory(arrow_file_path)){
+                if( auto ext = fs::path(arrow_file_path).extension(); 
+                    ext == ".arrow" || ext == ".feather" || ext == ".arrow"){
+                        return true;
+                } else {
+                    return false;
+                } 
+            }
+        return false;
+    }(); 
 
-    if (arrow_file_path == "") {
-        arrow_file_path_ = "NyxusFeatures" + extension;
-    } else {
+    if (valid_extension) {
         arrow_file_path_ = arrow_file_path;
+    } else {
+        std::string extension = (arrow_file_type == Nyxus::SaveOption::saveParquet) ? "parquet" : "arrow";
+        if (arrow_file_path == "") {
+            arrow_file_path_ = "NyxusFeatures." + extension;
+        } else if (fs::is_directory(arrow_file_path)) {
+            arrow_file_path_ = arrow_file_path + "/NyxusFeatures." + extension;
+        } else {
+            arrow_file_path_ = fs::path(arrow_file_path).replace_extension(extension).string();
+        }
     }
 
-    if (fs::is_directory(arrow_file_path)) {
-        arrow_file_path_ += "/NyxusFeatures" + extension;
+    std::optional<std::string> error_msg;
+    std::tie(writer_, error_msg) = WriterFactory::create_writer(arrow_file_path_, header);
+    if (writer_) {
+        return {true, std::nullopt};
+    } else {
+        return {false, error_msg};
     }
-
-    writer_ = WriterFactory::create_writer(arrow_file_path_, header);
-
-    return writer_;
 }
 
 
@@ -45,30 +56,50 @@ std::string ArrowOutputStream::get_arrow_path() {
     return arrow_file_path_;
 }
 
+std::tuple<bool, std::optional<std::string>> ArrowOutputStream::write_arrow_file (const std::vector<std::tuple<std::vector<std::string>, int, std::vector<double>>>& features){
+    if (writer_){
+        auto status = writer_->write(features);
+        if (status.ok()) {
+            return {true, std::nullopt};
+        }
+        else {
+            return {false, status.ToString()};
+        }
+    }
+    return {false, "Arrow Writer is not initialized."};
+}
+std::tuple<bool, std::optional<std::string>> ArrowOutputStream::close_arrow_file (){
+    if (writer_){
+        auto status = writer_->close();
+        if (status.ok()) {
+            return {true, std::nullopt};
+        }
+        else {
+            return {false, status.ToString()};
+        }
+    }
+    return {false, "Arrow Writer is not initialized."};
+}
+
 #else 
 
-std::shared_ptr<ApacheArrowWriter> ArrowOutputStream::create_arrow_file(const Nyxus::SaveOption& arrow_file_type,
+std::tuple<bool, std::optional<std::string>> ArrowOutputStream::create_arrow_file(const Nyxus::SaveOption& arrow_file_type,
                                                          const std::string& arrow_file_path,
                                                          const std::vector<std::string>& header) {
 
     std::cerr << "Apache Arrow functionality is not available. Please install Nyxus with Arrow enabled to use this functionality." << std::endl;
 
-    return nullptr;
+    return {false, "Apache Arrow functionality is not available."};
 }
 
-
-std::shared_ptr<arrow::Table> ArrowOutputStream::get_arrow_table(const std::string& file_path) {
-
+std::tuple<bool, std::optional<std::string>> ArrowOutputStream::write_arrow_file (const std::vector<std::tuple<std::vector<std::string>, int, std::vector<double>>>& features){
     std::cerr << "Apache Arrow functionality is not available. Please install Nyxus with Arrow enabled to use this functionality." << std::endl;
-
-    return nullptr;
+    return {false, "Apache Arrow functionality is not available."};
 }
-
-std::string ArrowOutputStream::get_arrow_path() {
-
+std::tuple<bool, std::optional<std::string>> ArrowOutputStream::close_arrow_file (){
     std::cerr << "Apache Arrow functionality is not available. Please install Nyxus with Arrow enabled to use this functionality." << std::endl;
-
-    return "";
+    return {false, "Apache Arrow functionality is not available."};
 }
 
+
 #endif

src/nyx/arrow_output_stream.h

@@ -1,7 +1,9 @@
 #pragma once
 
+#include <optional>
 #include <string>
 #include <memory>
+#include <tuple>
 
 #include "output_writers.h"
 #include "helpers/helpers.h"
@@ -31,25 +33,22 @@ class ArrowOutputStream {
 private:
 
   std::string arrow_file_path_ = "";
-	std::shared_ptr<ApacheArrowWriter> writer_ = nullptr;
+	std::unique_ptr<ApacheArrowWriter> writer_ = nullptr;
 	std::string arrow_output_type_ = "";
   std::shared_ptr<arrow::Table> arrow_table_ = nullptr;
 
 public:
-    std::shared_ptr<ApacheArrowWriter> create_arrow_file(const Nyxus::SaveOption& arrow_file_type,
+    std::tuple<bool, std::optional<std::string>> create_arrow_file(const Nyxus::SaveOption& arrow_file_type,
                                                          const std::string& arrow_file_path,
                                                          const std::vector<std::string>& header);
     std::shared_ptr<arrow::Table> get_arrow_table(const std::string& file_path);
     std::string get_arrow_path();
+    std::tuple<bool, std::optional<std::string>> write_arrow_file (const std::vector<std::tuple<std::vector<std::string>, int, std::vector<double>>>& features);
+    std::tuple<bool, std::optional<std::string>> close_arrow_file ();
 };
 
 #else 
 
-// Replace arrow::Table with a dummy variable
-namespace arrow {
-  using Table = bool;
-};
-
 /**
  * @brief Class to write to Apache Arrow formats
  * 
@@ -58,19 +57,12 @@ namespace arrow {
  */
 class ArrowOutputStream {
 
-private:
-
-  std::string arrow_file_path_ = "";
-	std::shared_ptr<ApacheArrowWriter> writer_ = nullptr;
-	std::string arrow_output_type_ = "";
-  std::shared_ptr<arrow::Table> arrow_table_ = nullptr;
-
 public:
-    std::shared_ptr<ApacheArrowWriter> create_arrow_file(const Nyxus::SaveOption& arrow_file_type,
+    std::tuple<bool, std::optional<std::string>> create_arrow_file(const Nyxus::SaveOption& arrow_file_type,
                                                          const std::string& arrow_file_path,
                                                          const std::vector<std::string>& header);
-    std::shared_ptr<arrow::Table> get_arrow_table(const std::string& file_path);
-    std::string get_arrow_path();
+    std::tuple<bool, std::optional<std::string>> write_arrow_file (const std::vector<std::tuple<std::vector<std::string>, int, std::vector<double>>>& features);
+    std::tuple<bool, std::optional<std::string>> close_arrow_file ();
 };
 
 

src/nyx/environment.cpp

@@ -838,16 +838,21 @@ bool Environment::parse_cmdline(int argc, char **argv)
 	}
 
 	//==== Output type
-#ifdef USE_ARROW 
 	VERBOSLVL1(std::cout << "\n*-*-*-*- Using Apache output -*-*-*-*\n");
 
 	auto rawOutpTypeUC = Nyxus::toupper(rawOutpType);
-	if (rawOutpTypeUC != Nyxus::toupper(OT_SINGLECSV) && 
-	    rawOutpTypeUC != Nyxus::toupper(OT_SEPCSV) && 
-		rawOutpTypeUC != Nyxus::toupper(OT_ARROWIPC) &&
-		rawOutpTypeUC != Nyxus::toupper(OT_PARQUET))
+	if (!((rawOutpTypeUC == Nyxus::toupper(OT_SINGLECSV))  || 
+	      (rawOutpTypeUC == Nyxus::toupper(OT_SEPCSV)) || 
+		  (rawOutpTypeUC == Nyxus::toupper(OT_ARROWIPC)) ||
+		  (rawOutpTypeUC == Nyxus::toupper(OT_PARQUET))
+		))	
 	{
-		std::cout << "Error: valid values of " << OUTPUTTYPE << " are " << OT_SEPCSV << ", " << OT_SINGLECSV << ", or" << OT_PARQUET << "."  "\n";
+		std::cout << "Error: valid values of " << OUTPUTTYPE << " are " << OT_SEPCSV << ", " 
+					<< OT_SINGLECSV <<", "
+					#ifdef USE_ARROW
+					<< OT_ARROWIPC <<", or" << OT_PARQUET <<
+					#endif 
+					"."  "\n";
 		return false;
 	}
 
@@ -861,31 +866,16 @@ bool Environment::parse_cmdline(int argc, char **argv)
 		} 
 	}();
 
-	if (saveOption == SaveOption::saveCSV) {
-		separateCsv = rawOutpTypeUC == Nyxus::toupper(OT_SEPCSV);
-	}
-
-#else // no Apache support
-	auto rawOutpTypeUC = Nyxus::toupper(rawOutpType);
-	if (rawOutpTypeUC != Nyxus::toupper(OT_SINGLECSV) &&
-		rawOutpTypeUC != Nyxus::toupper(OT_SEPCSV))
-	{
-		std::cout << "Error: valid values of " << OUTPUTTYPE << " are " << OT_SEPCSV << ", " << OT_SINGLECSV << "\n";
-
-		// Intercept an attempt of running Nyxus with Apache options
-		if (rawOutpTypeUC != Nyxus::toupper(OT_ARROWIPC) ||
-			rawOutpTypeUC != Nyxus::toupper(OT_PARQUET)) 
-			std::cout << "Error: Nyxus must be built with Apache Arrow enabled to use Arrow output types. Please rebuild with the flag USEARROW=ON." << std::endl;
-
+#ifndef USE_ARROW // no Apache support
+	if (saveOption == SaveOption::saveArrowIPC || saveOption == SaveOption::saveParquet) {
+		std::cout << "Error: Nyxus must be built with Apache Arrow enabled to use Arrow output types. Please rebuild with the flag USEARROW=ON." << std::endl;
 		return false;
 	}
+#endif
 
-	separateCsv = rawOutpTypeUC == Nyxus::toupper(OT_SEPCSV);
-
-	if (rawOutpTypeUC == Nyxus::toupper(OT_SINGLECSV) || rawOutpTypeUC == Nyxus::toupper(OT_SEPCSV)) {
-		saveOption = Nyxus::SaveOption::saveCSV;
+	if (saveOption == SaveOption::saveCSV) {
+		separateCsv = rawOutpTypeUC == Nyxus::toupper(OT_SEPCSV);
 	}
-#endif
 
 	//==== Check numeric parameters
 	if (!loader_threads.empty())

src/nyx/environment.h

@@ -8,8 +8,6 @@
 #include "cli_gabor_options.h"
 #include "cli_nested_roi_options.h"
 #include "save_option.h"
-
-#include "output_writers.h"
 #include "arrow_output_stream.h"
 
 
@@ -119,7 +117,6 @@ class Environment: public BasicEnvironment
 
 
 	ArrowOutputStream  arrow_stream;
-	std::shared_ptr<ApacheArrowWriter> arrow_writer = nullptr;
 
 	std::string embedded_pixel_size = "";
 

src/nyx/globals.h

@@ -40,7 +40,7 @@ namespace Nyxus
 
 	bool scanFilePairParallel(const std::string& intens_fpath, const std::string& label_fpath, int num_fastloader_threads, int num_sensemaker_threads, int filepair_index, int tot_num_filepairs);
 	std::string getPureFname(const std::string& fpath);
-	int processDataset(const std::vector<std::string>& intensFiles, const std::vector<std::string>& labelFiles, int numFastloaderThreads, int numSensemakerThreads, int numReduceThreads, int min_online_roi_size, SaveOption saveOption, const std::string& outputDir);
+	int processDataset(const std::vector<std::string>& intensFiles, const std::vector<std::string>& labelFiles, int numFastloaderThreads, int numSensemakerThreads, int numReduceThreads, int min_online_roi_size, const SaveOption saveOption, const std::string& outputPath);
 	bool gatherRoisMetrics(const std::string& intens_fpath, const std::string& label_fpath, int num_FL_threads);
 	bool processTrivialRois (const std::vector<int>& trivRoiLabels, const std::string& intens_fpath, const std::string& label_fpath, int num_FL_threads, size_t memory_limit);
 	bool processNontrivialRois (const std::vector<int>& nontrivRoiLabels, const std::string& intens_fpath, const std::string& label_fpath, int num_FL_threads);
@@ -54,7 +54,7 @@ namespace Nyxus
 	bool gatherRoisMetricsInMemory (const py::array_t<unsigned int, py::array::c_style | py::array::forcecast>& intens_image, const py::array_t<unsigned int, py::array::c_style | py::array::forcecast>& label_image, int start_idx);
 	bool processIntSegImagePairInMemory (const std::string& intens_fpath, const std::string& label_fpath, int filepair_index, const std::string& intens_name, const std::string& seg_name);
 	int processMontage(const py::array_t<unsigned int, py::array::c_style | py::array::forcecast>& intensFiles, const py::array_t<unsigned int, py::array::c_style | py::array::forcecast>& labelFiles, int numReduceThreads, const std::vector<std::string>& intensity_names,
-		const std::vector<std::string>& seg_names, std::string& error_message, SaveOption saveOption,  const std::string& outputDir="");
+		const std::vector<std::string>& seg_names, std::string& error_message, const SaveOption saveOption,  const std::string& outputPath="");
 	bool scanTrivialRois (const std::vector<int>& batch_labels, const py::array_t<unsigned int, py::array::c_style | py::array::forcecast>& intens_images, const py::array_t<unsigned int, py::array::c_style | py::array::forcecast>& label_images, int start_idx);
 	bool processTrivialRoisInMemory (const std::vector<int>& trivRoiLabels, const py::array_t<unsigned int, py::array::c_style | py::array::forcecast>& intens_fpath, const py::array_t<unsigned int, py::array::c_style | py::array::forcecast>& label_fpath, int start_idx, size_t memory_limit);
 #endif

src/nyx/output_writers.cpp

@@ -386,44 +386,30 @@ arrow::Status ArrowIPCWriter::close () {
 }
 
 
-std::shared_ptr<ApacheArrowWriter> WriterFactory::create_writer(const std::string &output_file, const std::vector<std::string> &header) {
+std::tuple<std::unique_ptr<ApacheArrowWriter>, std::optional<std::string>> WriterFactory::create_writer(const std::string &output_file, const std::vector<std::string> &header) {
 
     if (Nyxus::ends_with_substr(output_file, ".parquet")) {
 
-        return std::make_shared<ParquetWriter>(output_file, header);
+        return {std::make_unique<ParquetWriter>(output_file, header), std::nullopt};
 
     } else if (Nyxus::ends_with_substr(output_file, ".arrow") || Nyxus::ends_with_substr(output_file, ".feather")) {
 
-        return std::make_shared<ArrowIPCWriter>(output_file, header);
+        return {std::make_unique<ArrowIPCWriter>(output_file, header), std::nullopt};
 
     } else {
 
         std::filesystem::path path(output_file);
 
-        if (path.has_extension()) {
-            std::string file_extension = path.extension().string();
-
-            throw std::invalid_argument("No writer option for extension \"" + file_extension + "\". Valid options are \".parquet\" or \".arrow\".");
-
-        } else {
-
-            throw std::invalid_argument("No extension type was provided in the path. ");
+        auto error_msg = [&path](){        
+            if (path.has_extension()) 
+            {
+                return "No writer option for extension \"" + path.extension().string() + "\". Valid options are \".parquet\" or \".arrow\".";
+            } else {
+                return std::string{"No extension type was provided in the path."};
+            }
+        }; 
 
-        }
+        return {nullptr, error_msg()};
     }
 }
-#else
-
-    std::shared_ptr<arrow::Table> ApacheArrowWriter::get_arrow_table(const std::string& file_path) {
-        return nullptr;
-    }
-
-    arrow::Status ApacheArrowWriter::write (const std::vector<std::tuple<std::vector<std::string>, int, std::vector<double>>>& features) {
-        return arrow::Status();
-    }
-
-    arrow::Status ApacheArrowWriter::close () {
-        return arrow::Status();
-    }
-
 #endif