added version info for oceaneyes; restored arrow functionality

SlideRuleEarth · Sep 28, 2024 · a5b0ac0 · a5b0ac0
1 parent c7fee02
commit a5b0ac0
Show file tree

Hide file tree

Showing 6 changed files with 31 additions and 10 deletions.
diff --git a/datasets/bathy/CMakeLists.txt b/datasets/bathy/CMakeLists.txt
@@ -27,9 +27,9 @@ target_compile_definitions (slideruleLib PUBLIC QTREESINFO="${QTREESINFO}")
 # OpenOceansPP
 set (OPENOCEANSPPDIR ${PROJECT_SOURCE_DIR}/../ut-ATL24-oopp CACHE STRING "OpenOceans++ application source directory")
 target_compile_definitions (slideruleLib PUBLIC OPENOCEANSPP_INCLUDE="${OPENOCEANSPPDIR}/apps/classify_inc.h")
-execute_process (COMMAND git --work-tree ${OPENOCEANSPPDIR} --git-dir ${OPENOCEANSPPDIR}/.git describe --abbrev --dirty --always --tags --long OUTPUT_VARIABLE OPENOCEANSINFO)
-string(REGEX REPLACE "\n$" "" OPENOCEANSINFO "${OPENOCEANSINFO}")
-target_compile_definitions (slideruleLib PUBLIC OPENOCEANSINFO="${OPENOCEANSINFO}")
+execute_process (COMMAND git --work-tree ${OPENOCEANSPPDIR} --git-dir ${OPENOCEANSPPDIR}/.git describe --abbrev --dirty --always --tags --long OUTPUT_VARIABLE OPENOCEANSPPINFO)
+string(REGEX REPLACE "\n$" "" OPENOCEANSPPINFO "${OPENOCEANSPPINFO}")
+target_compile_definitions (slideruleLib PUBLIC OPENOCEANSPPINFO="${OPENOCEANSPPINFO}")
 
 target_sources (slideruleLib
     PRIVATE

diff --git a/datasets/bathy/docker/oceaneyes/runner.py b/datasets/bathy/docker/oceaneyes/runner.py
@@ -225,8 +225,7 @@ def openoceans(spot, df):
 def ensemble(spot, df):
     ensemble_model_filename = settings['ensemble']['ensemble_model_filename']
     print(f'loading ensemble model: {ensemble_model_filename}')
-#    df = df[['geoid_corr_h', 'surface_h', 'qtrees', 'cshelph', 'medianfilter', 'bathypathfinder', 'openoceanspp', 'coastnet', 'pointnet']]    
-    df = df[['ortho_h', 'qtrees', 'cshelph', 'medianfilter', 'bathypathfinder', 'openoceans', 'openoceanspp', 'coastnet', 'pointnet']]    
+    df = df[['ortho_h', 'surface_h', 'qtrees', 'cshelph', 'medianfilter', 'bathypathfinder', 'openoceanspp', 'coastnet', 'pointnet']]    
     clf = xgb.XGBClassifier(device='cpu')
     clf.load_model(ensemble_model_filename)
     x = df.to_numpy()
@@ -283,6 +282,12 @@ def runClassifier(classifier, classifier_func, num_processes=6):
     "subaqueous_photons": len(df[df["ortho_h"] < df["surface_h"]])
 }
 
+# read versions
+with open("cshelph/cshelph_version.txt") as file:
+    cshelph_version = file.read()
+with open("medianfilter/medianfilter_version.txt") as file:
+    medianfilter_version = file.read()
+
 # update profile
 profile["total_duration"] = time.time() - settings["latch"]
 print(f'ATL24 total duration is {profile["total_duration"]:.03f} seconds')
@@ -291,7 +296,9 @@ def runClassifier(classifier, classifier_func, num_processes=6):
 metadata = {
     "sliderule": json.dumps(rqst_parms),
     "profile": json.dumps(profile),
-    "stats": json.dumps(stats)
+    "stats": json.dumps(stats),
+    "cshelph": cshelph_version,
+    "medianfilter": medianfilter_version
 }
 
 # #####################

diff --git a/datasets/bathy/package/BathyFields.cpp b/datasets/bathy/package/BathyFields.cpp
@@ -104,7 +104,10 @@ BathyFields::BathyFields(lua_State* L):
                         {"spots",               &spots},
                         {"surface",             &surface},
                         {"refraction",          &refraction},
-                        {"uncertainty",         &uncertainty} })
+                        {"uncertainty",         &uncertainty},
+                        {"coastnet_version",    &coastnetVersion},
+                        {"qtrees_version",      &qtreesVersion},
+                        {"openoceanspp_version",&openoceansppVersion} })
 {
     LuaEngine::setAttrFunc(L, "classifier", luaClassifier);
 }

diff --git a/datasets/bathy/package/BathyFields.h b/datasets/bathy/package/BathyFields.h
@@ -47,7 +47,7 @@
 
 #define COASTNET_MODEL  "coastnet_model-20240917.json"
 #define QTREES_MODEL    "qtrees_model-20240916.json"
-#define ENSEMBLE_MODEL  "track_stacker_model.json" // "ensemble_model-20240919.json"
+#define ENSEMBLE_MODEL  "ensemble_model-20240919.json"
 #define POINTNET_MODEL  "pointnet2_model.pth"
 
 /******************************************************************************
@@ -300,6 +300,9 @@ class BathyFields: public Icesat2Fields
         CoastnetFields                                  coastnet;                   // coastnet fields
         OpenOceansPPFields                              openoceanspp;               // openoceans++ fields
         QtreesFields                                    qtrees;                     // qtrees fields
+        FieldElement<string>                            coastnetVersion {COASTNETINFO}; // git commit information for coastnet repo
+        FieldElement<string>                            qtreesVersion {QTREESINFO}; // git commit information for qtrees repo
+        FieldElement<string>                            openoceansppVersion {OPENOCEANSPPINFO}; // git commit information for openoceans repo
 
     protected:
 

diff --git a/packages/arrow/arrow.cpp b/packages/arrow/arrow.cpp
@@ -57,6 +57,7 @@
 int arrow_open (lua_State* L)
 {
     static const struct luaL_Reg arrow_functions[] = {
+        {"builder",     ArrowBuilder::luaCreate},
         {"dataframe",   ArrowDataFrame::luaCreate},
         {"sampler",     ArrowSampler::luaCreate},
         {"send2user",   ArrowCommon::luaSend2User},
@@ -68,6 +69,7 @@ int arrow_open (lua_State* L)
     luaL_newlib(L, arrow_functions);
 
     /* Set Attributes */
+    LuaEngine::setAttrStr(L, "PARMS",       ArrowParms::SELF);
     LuaEngine::setAttrInt(L, "PARQUET",     ArrowFields::PARQUET);
     LuaEngine::setAttrInt(L, "GEOPARQUET",  ArrowFields::GEOPARQUET);
     LuaEngine::setAttrInt(L, "FEATHER",     ArrowFields::FEATHER);

diff --git a/targets/slideruleearth-aws/Makefile b/targets/slideruleearth-aws/Makefile
@@ -56,8 +56,12 @@ STATIC_WEB_EXPORT_DIR = $(STATIC_WEB_STAGE_DIR)/dist
 OCEANEYES_SRC_DIR = $(ROOT)/datasets/bathy/docker/oceaneyes
 OCEANEYES_STAGE_DIR = $(STAGE)/oceaneyes
 MAGIC_PYTORCH_DIR = $(ROOT)/../../miniconda3/envs/pointnet2
-CSHELPH_SOURCE_FILE ?= $(ROOT)/../ut-ATL24-C-shelph/c_shelph.py
-MEDIANFILTER_SOURCE_FILE ?= $(ROOT)/../ut-ATL24-medianfilter/medianmodel.py
+CSHELPH_SOURCE_DIR ?= $(ROOT)/../ut-ATL24-C-shelph
+CSHELPH_SOURCE_FILE = $(CSHELPH_SOURCE_DIR)/c_shelph.py
+CSHELPH_VERSION = $(shell git --work-tree ${CSHELPH_SOURCE_DIR} --git-dir ${CSHELPH_SOURCE_DIR}/.git describe --abbrev --dirty --always --tags --long)
+MEDIANFILTER_SOURCE_DIR ?= $(ROOT)/../ut-ATL24-medianfilter
+MEDIANFILTER_SOURCE_FILE = $(MEDIANFILTER_SOURCE_DIR)/medianmodel.py
+MEDIANFILTER_VERSION = $(shell git --work-tree ${MEDIANFILTER_SOURCE_DIR} --git-dir ${MEDIANFILTER_SOURCE_DIR}/.git describe --abbrev --dirty --always --tags --long)
 
 INSTALLDIR ?= $(SLIDERULE_STAGE_DIR)
 
@@ -176,8 +180,10 @@ oceaneyes-docker:
 	cp $(OCEANEYES_SRC_DIR)/runner.* $(OCEANEYES_STAGE_DIR)
 	mkdir $(OCEANEYES_STAGE_DIR)/cshelph
 	cp $(CSHELPH_SOURCE_FILE) $(OCEANEYES_STAGE_DIR)/cshelph
+	@echo $(CSHELPH_VERSION) > $(OCEANEYES_STAGE_DIR)/cshelph/cshelph_version.txt
 	mkdir $(OCEANEYES_STAGE_DIR)/medianfilter
 	cp $(MEDIANFILTER_SOURCE_FILE) $(OCEANEYES_STAGE_DIR)/medianfilter
+	@echo $(MEDIANFILTER_VERSION) > $(OCEANEYES_STAGE_DIR)/medianfilter/medianfilter_version.txt
 	cp -R $(OCEANEYES_SRC_DIR)/bathypathfinder $(OCEANEYES_STAGE_DIR)
 	cp -R $(OCEANEYES_SRC_DIR)/pointnet $(OCEANEYES_STAGE_DIR)
 	cp -R $(OCEANEYES_SRC_DIR)/openoceans $(OCEANEYES_STAGE_DIR)