datamodel3 merge

SlideRuleEarth · Sep 28, 2024 · def9d75 · def9d75
2 parents 1a3d923 + a5b0ac0
commit def9d75
Show file tree

Hide file tree

Showing 377 changed files with 29,774 additions and 2,769 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -47,6 +47,7 @@ if(CMAKE_BUILD_TYPE MATCHES "Debug")
         "-readability-suspicious-call-argument,"
         "-readability-isolate-declaration,"
         "-readability-misleading-indentation,"
+        "-readability-container-contains,"
 
         "misc-*,"
         "-misc-non-private-member-variables-in-classes,"
@@ -102,12 +103,7 @@ if(CMAKE_BUILD_TYPE MATCHES "Debug")
     # Join checks into a single string parameter
     list(JOIN CLANG_TIDY_CHECKS "" CLANG_TIDY_CHECKS_PARM)
 
-    set (CMAKE_CXX_CLANG_TIDY
-       clang-tidy;
-       -header-filter=.;
-       -checks=${CLANG_TIDY_CHECKS_PARM};
-       -warnings-as-errors=*;
-    )
+    set (CMAKE_CXX_CLANG_TIDY "clang-tidy;-header-filter='^((?!ATL24_coastnet/).)*$';-checks=${CLANG_TIDY_CHECKS_PARM};-warnings-as-errors=*")
 
     # cppcheck, tested with version 2.13.0
     find_program (CMAKE_CXX_CPPCHECK NAMES cppcheck)
@@ -140,8 +136,13 @@ if(CMAKE_BUILD_TYPE MATCHES "Debug")
         "--suppress=constParameterReference:*/BathyRefractionCorrector.cpp" # in run() param extent cannot be const
         "--suppress=constParameterPointer:*/CcsdsPayloadDispatch.cpp"   # Not trivial to fix, would have to change DispachObject class as well.
         "--suppress=knownConditionTrueFalse"                            # -1 < 0 etc
+<<<<<<< HEAD
         "--suppress=constVariableReference:*/RasterObject.cpp"          # List [] const issue
         "--suppress=constVariableReference:*/GeoIndexedRaster.cpp"      # List [] const issue
+=======
+
+        "--suppress=*:*/coastnet.h"                                     # suppress all checks for coastnet
+>>>>>>> datamodel3
     )
 
 endif()
@@ -169,10 +170,11 @@ option (ENABLE_GDAL_ERROR_REPORTING "Log GDAL errors to message log" OFF)
 option (USE_ARROW_PACKAGE "Include the Apache Arrow package" ON)
 option (USE_AWS_PACKAGE "Include the AWS package" ON)
 option (USE_CCSDS_PACKAGE "Include the CCSDS package" OFF)
-option (USE_CRE_PACKAGE "Include the CRE package" ON)
-option (USE_GEO_PACKAGE "Include the GEO package" ON)
-option (USE_H5_PACKAGE "Include the HDF5 package" ON)
-option (USE_NETSVC_PACKAGE "Include the Network Services package" ON)
+option (USE_CRE_PACKAGE "Include the container runtime environment package" ON)
+option (USE_GEO_PACKAGE "Include the geospatial library (GDAL) package" ON)
+option (USE_H5_PACKAGE "Include the HDF5 (H5Coro) package" ON)
+option (USE_LEGACY_PACKAGE "Include the legacy (CommandProcessor) package" OFF)
+option (USE_STREAMING_PACKAGE "Include the streaming (RecordDispatcher) package" ON)
 
 # Platform Options #
 
@@ -191,14 +193,19 @@ option (USE_PGC_DATASET "Include the PGC dataset" ON)
 option (USE_SWOT_DATASET "Include the SWOT dataset" ON)
 option (USE_USGS3DEP_DATASET "Include the USDS 3DEP dataset" ON)
 
+# Target Options #
+
+option (USE_SLIDERULEEARTH_TARGET "Include slideruleearth.io target configuration" ON)
+
 # Version Information #
 
 file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/version.txt TGTVER)
 string(REPLACE "v" "" LIBVER ${TGTVER})
 
 # C++ Version #
 
-set(CXX_VERSION 17) # required if using pistache package
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_STANDARD 20)
 
 # Platform #
 
@@ -211,14 +218,13 @@ if(CMAKE_BUILD_PLATFORM MATCHES "Linux")
     LINK_DIRECTORIES(/usr/local/lib /usr/local/lib64)
 
     # Set Environment Variables
-    set (INSTALLDIR /usr/local CACHE STRING "Installation directory for library and executables")
+    set (INSTALLDIR /usr/local CACHE STRING "Installation prefix directory for sliderule application")
     set (CONFDIR ${INSTALLDIR}/etc/sliderule)
     set (INCDIR ${INSTALLDIR}/include/sliderule)
+    set (PLUGINDIR ${INSTALLDIR}/lib/sliderule)
 
 endif()
 
-set (RUNTIMEDIR ${CONFDIR} CACHE STRING "Runtime directory for plugins and configuration scripts")
-
 #####################
 # Target: SlideRule #
 #####################
@@ -242,7 +248,6 @@ endif()
 set_target_properties (slideruleLib PROPERTIES VERSION ${LIBVER})
 set_target_properties (slideruleLib PROPERTIES OUTPUT_NAME sliderule)
 set_target_properties (slideruleLib PROPERTIES ENABLE_EXPORTS true)
-set_target_properties (slideruleLib PROPERTIES CXX_STANDARD ${CXX_VERSION})
 
 # Compile Definitions #
 
@@ -284,7 +289,8 @@ endif ()
 # Platform File #
 
 set_property(GLOBAL APPEND PROPERTY platform "#define LIBID \"${TGTVER}\"\n")
-set_property(GLOBAL APPEND PROPERTY platform "#define CONFDIR \"${RUNTIMEDIR}\"\n")
+set_property(GLOBAL APPEND PROPERTY platform "#define CONFDIR \"${CONFDIR}\"\n")
+set_property(GLOBAL APPEND PROPERTY platform "#define PLUGINDIR \"${PLUGINDIR}\"\n")
 
 if(CMAKE_BUILD_TYPE STREQUAL "Debug")
     set_property(GLOBAL APPEND PROPERTY platform "#define __unittesting__\n")
@@ -345,8 +351,12 @@ if(${USE_H5_PACKAGE})
     add_subdirectory (packages/h5)
 endif()
 
-if(${USE_NETSVC_PACKAGE})
-    add_subdirectory (packages/netsvc)
+if(${USE_LEGACY_PACKAGE})
+    add_subdirectory (packages/legacy)
+endif()
+
+if(${USE_STREAMING_PACKAGE})
+    add_subdirectory (packages/streaming)
 endif()
 
 # Datasets #
@@ -387,6 +397,12 @@ if(${USE_USGS3DEP_DATASET})
     add_subdirectory (datasets/usgs3dep)
 endif()
 
+# Targets #
+
+if(${USE_SLIDERULEEARTH_TARGET})
+    add_subdirectory (targets/slideruleearth-aws)
+endif()
+
 # Scripts #
 
 add_subdirectory (scripts)

diff --git a/Makefile b/Makefile
@@ -1,6 +1,22 @@
 ROOT = $(shell pwd)
 BUILD = $(ROOT)/build
 
+USERCFG ?=
+
+CLANG_VER ?= ""
+
+FULL_CFG := -DENABLE_TRACING=ON
+FULL_CFG += -DH5CORO_MAXIMUM_NAME_SIZE=208
+FULL_CFG += -DENABLE_ADDRESS_SANITIZER=ON
+FULL_CFG += -DUSE_ARROW_PACKAGE=ON
+FULL_CFG += -DUSE_AWS_PACKAGE=ON
+FULL_CFG += -DUSE_CCSDS_PACKAGE=ON
+FULL_CFG += -DUSE_CRE_PACKAGE=ON
+FULL_CFG += -DUSE_GEO_PACKAGE=ON
+FULL_CFG += -DUSE_H5_PACKAGE=ON
+FULL_CFG += -DUSE_LEGACY_PACKAGE=ON
+FULL_CFG += -DUSE_STREAMING_PACKAGE=ON
+
 ########################
 # Application Targets
 ########################
@@ -9,20 +25,29 @@ all: ## build code
 	make -j8 -C $(BUILD)
 
 config: prep ## configure make for release version of sliderule
-	cd $(BUILD); cmake -DCMAKE_BUILD_TYPE=Release $(CFG) $(ROOT)
-
-config-debug: prep
-	cd $(BUILD); cmake -DCMAKE_BUILD_TYPE=Debug -DH5CORO_MAXIMUM_NAME_SIZE=208 $(CFG) $(ROOT)
+	cd $(BUILD) && \
+	cmake -DCMAKE_BUILD_TYPE=Release -DSHARED_LIBRARY=ON $(USERCFG) $(ROOT)
 
-config-library: prep ## configure make for shared library libsliderule.so
-	cd $(BUILD); cmake -DCMAKE_BUILD_TYPE=Release -DSHARED_LIBRARY=ON $(CFG) $(ROOT)
+config-debug: prep ## configure make for static analysis and debug runs
+	cd $(BUILD) && \
+	export CC=clang$(CLANG_VER) && export CXX=clang++$(CLANG_VER) && \
+	cmake -DCMAKE_BUILD_TYPE=Debug -DSHARED_LIBRARY=ON -DCMAKE_USER_MAKE_RULES_OVERRIDE=$(ROOT)/platforms/linux/ClangOverrides.txt -D_CMAKE_TOOLCHAIN_PREFIX=llvm- $(FULL_CFG) $(USERCFG) $(ROOT)
 
 install: ## install sliderule to system
 	make -C $(BUILD) install
 
 uninstall: ## uninstall most recent install of sliderule from system
 	xargs rm < $(BUILD)/install_manifest.txt
 
+prep: ## create necessary build directories
+	mkdir -p $(BUILD)
+
+clean: ## clean last build
+	- make -C $(BUILD) clean
+
+distclean: ## fully remove all non-version controlled files and directories
+	- rm -Rf $(BUILD)
+
 ########################
 # Development Targets
 ########################
@@ -49,24 +74,11 @@ testcov: ## analyze results of test coverage report
 	genhtml $(BUILD)/coverage.info --output-directory $(BUILD)/coverage_html
 	# firefox $(BUILD)/coverage_html/index.html
 
-########################
-# Global Targets
-########################
-
 ctags: ## generate ctags
 	if [ -d ".clangd/index/" ]; then rm -f .clangd/index/*; fi             ## clear clagnd index (before clangd-11)
 	if [ -d ".cache/clangd/index/" ]; then rm -f .cache/clangd/index/*; fi ## clear clagnd index (clangd-11)
 	/usr/bin/jq -s 'add' $(SLIDERULE_BUILD)/compile_commands.json $(PGC_BUILD)/compile_commands.json $(ICESAT2_BUILD)/compile_commands.json $(GEDI_BUILD)/compile_commands.json $(LANDSAT_BUILD)/compile_commands.json $(USGS3DEP_BUILD)/compile_commands.json $(OPENDATA_BUILD)/compile_commands.json $(SWOT_BUILD)/compile_commands.json > compile_commands.json
 
-prep: ## create necessary build directories
-	mkdir -p $(BUILD)
-
-clean: ## clean last build
-	- make -C $(BUILD) clean
-
-distclean: ## fully remove all non-version controlled files and directories
-	- rm -Rf $(BUILD)
-
 help: ## that's me!
 	@printf "\033[37m%-30s\033[0m %s\n" "#-----------------------------------------------------------------------------------------"
 	@printf "\033[37m%-30s\033[0m %s\n" "# Makefile Help                                                                          |"

diff --git a/README.md b/README.md
@@ -68,9 +68,6 @@ Options include:
    -DINSTALLDIR=[prefix]               location to install sliderule
                                        default: /usr/local
 
-   -DRUNTIMEDIR=[directory]            location for run-time files like plugins, configuration files, and lua scripts
-                                       default: /usr/local/etc/sliderule
-
    -DSHARED_LIBRARY=[ON|OFF]           build sliderule as a shared library (overrides all other targets)
                                        default: OFF
 
@@ -173,7 +170,7 @@ Contains a project or mission specific extension to the SlideRule framework that
 
 In order to build a plugin for SlideRule, the plugin code must compile down to a shared object that exposes a single function defined as `void init{plugin}(void)` where _{plugin}_ is the name of the plugin.  Note that if developing the plugin in C++ the initialization function must be externed as C in order to prevent the mangling of the exported symbol.
 
-Once the shared object is built, the build system must copy the shared object into the SlideRule configuration directory (specified by the `RUNTIMEDIR` option in the CMakeLists.txt file) with the name _{plugin}.so_.  On startup, the _sliderule_ application scans the configuration directory and loads all plugins present.
+Once the shared object is built, the build system must copy the shared object into the SlideRule plugin directory (specified by the `PLUGINDIR` option in the CMakeLists.txt file) with the name _{plugin}.so_.  On startup, the _sliderule_ application scans the configuration directory and loads all plugins present.
 
 
 ## VI. Delivering the Code

diff --git a/clients/python/tests/test_defaults.py b/clients/python/tests/test_defaults.py
@@ -14,6 +14,6 @@ def test_parms(self, init):
         assert d["output"]["as_geo"]
         assert d["cre"]["timeout"] == 600
         assert d["samples"]["sampling_algo"] == "NearestNeighbour"
-        assert d["netsvc"]["cluster_size_hint"] == 0
+        assert d["rqst"]["cluster_size_hint"] == 0
         assert d["icesat2"]["surface_type"] == "SRT_LAND_ICE"
         assert d["gedi"]["projection"] == "AUTOMATIC"
diff --git a/clients/python/utils/utils.py b/clients/python/utils/utils.py
@@ -81,9 +81,9 @@ def initialize_client(args):
         "profile":                  True,
         "verbose":                  True,
         "timeout":                  0,
-        "rqst-timeout":             0,
-        "node-timeout":             0,
-        "read-timeout":             0,
+        "rqst_timeout":             0,
+        "node_timeout":             0,
+        "read_timeout":             0,
         "output.path":              None,
         "output.format":            "native",
         "output.open_on_complete":  False
@@ -135,15 +135,15 @@ def initialize_client(args):
     # Provide Timeouts
     if cfg["timeout"] > 0:
         parms["timeout"] = cfg["timeout"]
-        parms["rqst-timeout"] = cfg["timeout"]
-        parms["node-timeout"] = cfg["timeout"]
-        parms["read-timeout"] = cfg["timeout"]
-    if cfg["rqst-timeout"] > 0:
-        parms["rqst-timeout"] = cfg["rqst-timeout"]
-    if cfg["node-timeout"] > 0:
-        parms["node-timeout"] = cfg["node-timeout"]
-    if cfg["read-timeout"] > 0:
-        parms["read-timeout"] = cfg["read-timeout"]
+        parms["rqst_timeout"] = cfg["timeout"]
+        parms["node_timeout"] = cfg["timeout"]
+        parms["read_timeout"] = cfg["timeout"]
+    if cfg["rqst_timeout"] > 0:
+        parms["rqst_timeout"] = cfg["rqst_timeout"]
+    if cfg["node_timeout"] > 0:
+        parms["node_timeout"] = cfg["node_timeout"]
+    if cfg["read_timeout"] > 0:
+        parms["read_timeout"] = cfg["read_timeout"]
 
     # Add Output Options
     if cfg["output.path"]:

diff --git a/datasets/bathy/CMakeLists.txt b/datasets/bathy/CMakeLists.txt
@@ -1,16 +1,52 @@
 message (STATUS "Including bathy dataset")
 
+find_package(OpenMP REQUIRED)
+find_package(xgboost REQUIRED)
+
 target_compile_definitions (slideruleLib PUBLIC __bathy__)
 
+target_compile_options (slideruleLib PUBLIC ${OpenMP_CXX_FLAGS})
+target_link_libraries(slideruleLib PUBLIC xgboost::xgboost)
+target_link_libraries(slideruleLib PUBLIC OpenMP::OpenMP_CXX)
+
+# CoastNet
+set (COASTNETDIR ${PROJECT_SOURCE_DIR}/../ut-ATL24-coastnet CACHE STRING "Coastnet application source directory")
+target_link_libraries(slideruleLib PUBLIC xgboost::xgboost)
+target_compile_definitions (slideruleLib PUBLIC COASTNET_INCLUDE="${COASTNETDIR}/apps/classify_inc.h")
+execute_process (COMMAND git --work-tree ${COASTNETDIR} --git-dir ${COASTNETDIR}/.git describe --abbrev --dirty --always --tags --long OUTPUT_VARIABLE COASTNETINFO)
+string(REGEX REPLACE "\n$" "" COASTNETINFO "${COASTNETINFO}")
+target_compile_definitions (slideruleLib PUBLIC COASTNETINFO="${COASTNETINFO}")
+
+# Qtrees
+set (QTREESDIR ${PROJECT_SOURCE_DIR}/../ut-ATL24-qtrees CACHE STRING "Qtrees application source directory")
+target_compile_definitions (slideruleLib PUBLIC QTREES_INCLUDE="${QTREESDIR}/apps/classify_inc.h")
+execute_process (COMMAND git --work-tree ${QTREESDIR} --git-dir ${QTREESDIR}/.git describe --abbrev --dirty --always --tags --long OUTPUT_VARIABLE QTREESINFO)
+string(REGEX REPLACE "\n$" "" QTREESINFO "${QTREESINFO}")
+target_compile_definitions (slideruleLib PUBLIC QTREESINFO="${QTREESINFO}")
+
+# OpenOceansPP
+set (OPENOCEANSPPDIR ${PROJECT_SOURCE_DIR}/../ut-ATL24-oopp CACHE STRING "OpenOceans++ application source directory")
+target_compile_definitions (slideruleLib PUBLIC OPENOCEANSPP_INCLUDE="${OPENOCEANSPPDIR}/apps/classify_inc.h")
+execute_process (COMMAND git --work-tree ${OPENOCEANSPPDIR} --git-dir ${OPENOCEANSPPDIR}/.git describe --abbrev --dirty --always --tags --long OUTPUT_VARIABLE OPENOCEANSPPINFO)
+string(REGEX REPLACE "\n$" "" OPENOCEANSPPINFO "${OPENOCEANSPPINFO}")
+target_compile_definitions (slideruleLib PUBLIC OPENOCEANSPPINFO="${OPENOCEANSPPINFO}")
+
 target_sources (slideruleLib
     PRIVATE
         ${CMAKE_CURRENT_LIST_DIR}/package/bathy.cpp
-        ${CMAKE_CURRENT_LIST_DIR}/package/BathyClassifier.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/package/BathyCoastnetClassifier.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/package/BathyDataFrame.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/package/BathyFields.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/package/BathyGranule.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/package/BathyKd.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/package/BathyMask.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/package/BathyOpenOceansPPClassifier.cpp
         ${CMAKE_CURRENT_LIST_DIR}/package/BathyParms.cpp
-        ${CMAKE_CURRENT_LIST_DIR}/package/BathyReader.cpp
-        ${CMAKE_CURRENT_LIST_DIR}/package/BathyViewer.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/package/BathyQtreesClassifier.cpp
         ${CMAKE_CURRENT_LIST_DIR}/package/BathyRefractionCorrector.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/package/BathySeaSurfaceFinder.cpp
         ${CMAKE_CURRENT_LIST_DIR}/package/BathyUncertaintyCalculator.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/package/BathyViewer.cpp
 )
 
 target_include_directories (slideruleLib
@@ -22,12 +58,19 @@ target_include_directories (slideruleLib
 install (
     FILES
         ${CMAKE_CURRENT_LIST_DIR}/package/bathy.h
-        ${CMAKE_CURRENT_LIST_DIR}/package/BathyClassifier.h
+        ${CMAKE_CURRENT_LIST_DIR}/package/BathyCoastnetClassifier.h
+        ${CMAKE_CURRENT_LIST_DIR}/package/BathyDataFrame.h
+        ${CMAKE_CURRENT_LIST_DIR}/package/BathyFields.h
+        ${CMAKE_CURRENT_LIST_DIR}/package/BathyGranule.h
+        ${CMAKE_CURRENT_LIST_DIR}/package/BathyKd.h
+        ${CMAKE_CURRENT_LIST_DIR}/package/BathyMask.h
+        ${CMAKE_CURRENT_LIST_DIR}/package/BathyOpenOceansPPClassifier.h
         ${CMAKE_CURRENT_LIST_DIR}/package/BathyParms.h
-        ${CMAKE_CURRENT_LIST_DIR}/package/BathyReader.h
-        ${CMAKE_CURRENT_LIST_DIR}/package/BathyViewer.h
+        ${CMAKE_CURRENT_LIST_DIR}/package/BathyQtreesClassifier.h
         ${CMAKE_CURRENT_LIST_DIR}/package/BathyRefractionCorrector.h
+        ${CMAKE_CURRENT_LIST_DIR}/package/BathySeaSurfaceFinder.h
         ${CMAKE_CURRENT_LIST_DIR}/package/BathyUncertaintyCalculator.h
+        ${CMAKE_CURRENT_LIST_DIR}/package/BathyViewer.h
     DESTINATION
         ${INCDIR}/bathy
 )

diff --git a/datasets/bathy/docker/oceaneyes/.gitignore b/datasets/bathy/docker/oceaneyes/.gitignore
@@ -0,0 +1 @@
+__pycache__/