Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

atl24 cleanup for latest tools #434

Open
wants to merge 23 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
3029136
atl24 cleanup for latest tools
elidwa Oct 2, 2024
2ca5974
created perf tests for different datasets
elidwa Oct 5, 2024
3e89a75
fixed debug statement
elidwa Oct 5, 2024
e1a1452
Fixed bug in poi time in arrow sampler, bug fixes
elidwa Oct 5, 2024
22dc8db
disabled debug stuff
elidwa Oct 7, 2024
551d9e7
using GEOS Rtree for features
elidwa Oct 7, 2024
1a55f6f
Implemented GeoRtree class
elidwa Oct 8, 2024
680a5fb
Rtree completly replaced vector of features, sorting added
elidwa Oct 8, 2024
0922103
fixed comment
elidwa Oct 8, 2024
39ca549
implemented sort_by_index geoparm, updated all lua tests
elidwa Oct 9, 2024
4980d43
removed index file bbox check
elidwa Oct 9, 2024
78b2756
reverting a few of the changes associated with the static analysis cl…
jpswinski Oct 9, 2024
04e306f
added tolerance for depth to gebco test, projlib issue
elidwa Oct 11, 2024
1200da1
aws per path region bug fix, global canopy on us-east-1
elidwa Oct 11, 2024
42cf2f8
fixed debug leftovers
elidwa Oct 11, 2024
d723f98
collecting samples uses all cpus, removed union code
elidwa Oct 15, 2024
46479ab
fixed broken cmr search, used cmr-search-after since scrolling no lon…
elidwa Oct 16, 2024
02d2349
fixed cmr_stac_search, uses 'next' in links, added test for it
elidwa Oct 16, 2024
f27dc6e
fixed bug in batch landsat code
elidwa Oct 16, 2024
81d0aa2
enabled info statement
elidwa Oct 16, 2024
fca672a
fixed coredump in landsat
elidwa Oct 17, 2024
403e92e
fixed gps time bugs in 3dep, one deadlock removed, unittests added
elidwa Oct 18, 2024
ca68256
optimized finding unique rasters
elidwa Oct 22, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions datasets/bathy/package/BathyDataFrame.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ int BathyDataFrame::luaCreate (lua_State* L)
*----------------------------------------------------------------------------*/
BathyDataFrame::BathyDataFrame (lua_State* L, const char* beam_str, BathyFields* _parms, H5Object* _hdf03, H5Object* _hdf09, const char* rqstq_name, BathyMask* _mask):
GeoDataFrame(L, LUA_META_NAME, LUA_META_TABLE,
{
{
{"time_ns", &time_ns},
{"index_ph", &index_ph},
{"index_seg", &index_seg},
Expand All @@ -124,7 +124,7 @@ BathyDataFrame::BathyDataFrame (lua_State* L, const char* beam_str, BathyFields*
{"class_ph", &class_ph},
{"predictions", &predictions},
{"geoid_corr_h", &geoid_corr_h},
},
},
{
{"spot", &spot},
{"beam", &beam},
Expand Down Expand Up @@ -190,7 +190,7 @@ BathyDataFrame::~BathyDataFrame (void)
{
active = false;
delete pid;

delete rqstQ;

hdf03->releaseLuaObject();
Expand Down Expand Up @@ -530,7 +530,7 @@ void* BathyDataFrame::subsettingThread (void* parm)
GeoLib::UTMTransform utm_transform(region.segment_lat[0], region.segment_lon[0]);
dataframe.utm_zone = utm_transform.zone;
dataframe.utm_is_north = region.segment_lat[0] >= 0.0;

/* Traverse All Photons In Dataset */
while(dataframe.active && (current_photon < atl03.dist_ph_along.size))
{
Expand Down Expand Up @@ -762,7 +762,7 @@ int BathyDataFrame::luaIsValid (lua_State* L)
bool status = false;
try
{
BathyDataFrame* lua_obj = dynamic_cast<BathyDataFrame*>(getLuaSelf(L, 1));
const BathyDataFrame* lua_obj = dynamic_cast<BathyDataFrame*>(getLuaSelf(L, 1));
status = lua_obj->valid;
}
catch(const RunTimeException& e)
Expand All @@ -780,7 +780,7 @@ int BathyDataFrame::luaLength (lua_State* L)
{
try
{
BathyDataFrame* lua_obj = dynamic_cast<BathyDataFrame*>(getLuaSelf(L, 1));
const BathyDataFrame* lua_obj = dynamic_cast<BathyDataFrame*>(getLuaSelf(L, 1));
lua_pushinteger(L, lua_obj->length());
}
catch(const RunTimeException& e)
Expand Down
82 changes: 41 additions & 41 deletions datasets/bathy/package/BathyFields.h
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
/******************/
/* Surface Fields */
/******************/
struct SurfaceFields: public FieldDictionary
struct SurfaceFields: public FieldDictionary
{
FieldElement<double> binSize {0.5}; // meters
FieldElement<double> maxRange {1000.0}; // meters
Expand All @@ -67,10 +67,10 @@ struct SurfaceFields: public FieldDictionary
FieldElement<double> highestPeakRatio {1.2};
FieldElement<double> surfaceWidth {3.0}; // standard deviations
FieldElement<bool> modelAsPoisson {true};
SurfaceFields(void):

SurfaceFields(void):
FieldDictionary({ {"bin_size", &binSize},
{"max_range", &maxRange},
{"max_range", &maxRange},
{"max_bins", &maxBins},
{"signal_threshold", &signalThreshold},
{"min_peak_separation", &minPeakSeparation},
Expand All @@ -85,15 +85,15 @@ struct SurfaceFields: public FieldDictionary
/*********************/
/* Refraction Fields */
/*********************/
struct RefractionFields: public FieldDictionary
struct RefractionFields: public FieldDictionary
{
FieldElement<bool> useWaterRIMask {true}; // global water refractive index mask downloaded in atl24 init lua routine
FieldElement<double> RIAir {1.00029}; // refraction index of air
FieldElement<double> RIWater {1.34116}; // refraction index of water
RefractionFields(void):

RefractionFields(void):
FieldDictionary({ {"use_water_ri_mask", &useWaterRIMask},
{"ri_air", &RIAir},
{"ri_air", &RIAir},
{"ri_water", &RIAir} }) {
};

Expand All @@ -103,13 +103,13 @@ struct RefractionFields: public FieldDictionary
/**********************/
/* Uncertainty Fields */
/**********************/
struct UncertaintyFields: public FieldDictionary
struct UncertaintyFields: public FieldDictionary
{
FieldElement<string> assetKdName {"viirsj1-s3"}; // global water refractive index mask downloaded in atl24 init lua routine

Asset* assetKd {NULL};

UncertaintyFields(void):
UncertaintyFields(void):
FieldDictionary({ {"asset_kd", &assetKdName} }) {
assetKd = dynamic_cast<Asset*>(LuaObject::getLuaObjectByName(assetKdName.value.c_str(), Asset::OBJECT_TYPE));
if(!assetKd) throw RunTimeException(CRITICAL, RTE_ERROR, "unable to find asset %s", assetKdName.value.c_str());
Expand All @@ -123,17 +123,17 @@ struct UncertaintyFields: public FieldDictionary
/*******************/
/* Coastnet Fields */
/*******************/
struct CoastnetFields: public FieldDictionary
struct CoastnetFields: public FieldDictionary
{
FieldElement<string> model {COASTNET_MODEL};
FieldElement<bool> setClass {true};
FieldElement<bool> usePredictions {false};
FieldElement<bool> verbose {true};
CoastnetFields(void):

CoastnetFields(void):
FieldDictionary({ {"model", &model},
{"set_class", &setClass},
{"use_predictions", &usePredictions},
{"set_class", &setClass},
{"use_predictions", &usePredictions},
{"verbose", &verbose} }) {
};

Expand All @@ -143,7 +143,7 @@ struct CoastnetFields: public FieldDictionary
/***********************/
/* OpenOceansPP Fields */
/***********************/
struct OpenOceansPPFields: public FieldDictionary
struct OpenOceansPPFields: public FieldDictionary
{
FieldElement<bool> setClass {false};
FieldElement<bool> setSurface {false};
Expand All @@ -161,10 +161,10 @@ struct OpenOceansPPFields: public FieldDictionary
FieldElement<double> bathySmoothingSigma {100.0};
FieldElement<double> minPeakProminence {0.01};
FieldElement<size_t> minPeakDistance {2};

size_t minSurfacePhotonsPerWindow;
size_t minBathyPhotonsPerWindow;

void updatePhotonsPerWindow(void) {
minSurfacePhotonsPerWindow = 0.25 * (xResolution.value / 0.7);
minBathyPhotonsPerWindow = 0.25 * (xResolution.value / 0.7);
Expand All @@ -175,43 +175,43 @@ struct OpenOceansPPFields: public FieldDictionary
updatePhotonsPerWindow();
}

OpenOceansPPFields(void):
OpenOceansPPFields(void):
FieldDictionary({ {"set_class", &setClass},
{"set_surface", &setSurface},
{"use_predictions", &usePredictions},
{"verbose", &verbose},
{"x_resolution", &xResolution},
{"z_resolution", &zResolution},
{"z_min", &zMin},
{"z_max", &zMax},
{"surface_z_min", &surfaceZMin},
{"surface_z_max", &surfaceZMax},
{"bathy_min_depth", &bathyMinDepth},
{"vertical_smoothing_sigma", &verticalSmoothingSigma},
{"surface_smoothing_sigma", &surfaceSmoothingSigma},
{"bathy_smoothing_sigma", &bathySmoothingSigma},
{"min_peak_prominence", &minPeakProminence},
{"set_surface", &setSurface},
{"use_predictions", &usePredictions},
{"verbose", &verbose},
{"x_resolution", &xResolution},
{"z_resolution", &zResolution},
{"z_min", &zMin},
{"z_max", &zMax},
{"surface_z_min", &surfaceZMin},
{"surface_z_max", &surfaceZMax},
{"bathy_min_depth", &bathyMinDepth},
{"vertical_smoothing_sigma", &verticalSmoothingSigma},
{"surface_smoothing_sigma", &surfaceSmoothingSigma},
{"bathy_smoothing_sigma", &bathySmoothingSigma},
{"min_peak_prominence", &minPeakProminence},
{"min_peak_distance", &minPeakDistance} }) {
updatePhotonsPerWindow();
};

virtual ~OpenOceansPPFields(void) override = default;
virtual ~OpenOceansPPFields(void) override = default;
};

/*****************/
/* Qtrees Fields */
/*****************/
struct QtreesFields: public FieldDictionary
struct QtreesFields: public FieldDictionary
{
FieldElement<string> model {QTREES_MODEL};
FieldElement<bool> setClass {false};
FieldElement<bool> setSurface {true};
FieldElement<bool> verbose {true};
QtreesFields(void):

QtreesFields(void):
FieldDictionary({ {"model", &model},
{"set_class", &setClass},
{"set_surface", &setSurface},
{"set_class", &setClass},
{"set_surface", &setSurface},
{"verbose", &verbose} }) {
};

Expand Down Expand Up @@ -319,7 +319,7 @@ class BathyFields: public Icesat2Fields
* Methods
*--------------------------------------------------------------------*/

BathyFields (lua_State* L);
explicit BathyFields (lua_State* L);
virtual ~BathyFields (void) override = default;
};

Expand Down
8 changes: 4 additions & 4 deletions datasets/bathy/package/BathyGranule.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -89,13 +89,13 @@ int BathyGranule::luaCreate (lua_State* L)
}

/*----------------------------------------------------------------------------
* luaExport - export() --> lua table
* luaExport - export() --> lua table
*----------------------------------------------------------------------------*/
int BathyGranule::luaExport (lua_State* L)
{
try
{
BathyGranule* lua_obj = dynamic_cast<BathyGranule*>(getLuaSelf(L, 1));
const BathyGranule* lua_obj = dynamic_cast<BathyGranule*>(getLuaSelf(L, 1));
return lua_obj->toLua(L);
}
catch(const RunTimeException& e)
Expand Down Expand Up @@ -146,7 +146,7 @@ BathyGranule::BathyGranule (lua_State* L, BathyFields* _parms, H5Object* _hdf03,
{
/* Set Thread Specific Trace ID for H5Coro */
EventLib::stashId (traceId);

/* Start Reader Thread */
active = true;
pid = new Thread(readingThread, this);
Expand Down Expand Up @@ -207,7 +207,7 @@ void* BathyGranule::readingThread (void* parm)
H5Element<const char*> version (granule.hdf03, "/ancillary_data/version");

H5Element<double> crossing_time (granule.hdf03, "/orbit_info/crossing_time");
H5Element<double> lan (granule.hdf03, "/orbit_info/lan");
H5Element<double> lan (granule.hdf03, "/orbit_info/lan");
H5Element<int16_t> orbit_number (granule.hdf03, "/orbit_info/orbit_number");
H5Element<int8_t> sc_orient (granule.hdf03, "/orbit_info/sc_orient");
H5Element<double> sc_orient_time (granule.hdf03, "/orbit_info/sc_orient_time");
Expand Down
2 changes: 1 addition & 1 deletion datasets/gebco/selftests/gebco_reader.lua
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ local expDepth = { -64, -4933, -4072}
local expFlags = { 70, 40, 44}

print(string.format("\n--------------------------------\nTest: GEBCO Correct Values\n--------------------------------"))
local dem = geo.raster(geo.parms({ asset = "gebco-bathy", algorithm = "NearestNeighbour", with_flags=true}))
local dem = geo.raster(geo.parms({ asset = "gebco-bathy", algorithm = "NearestNeighbour", with_flags=true, sort_by_index = true }))
runner.check(dem ~= nil)

for j, lon in ipairs(lons) do
Expand Down
4 changes: 1 addition & 3 deletions datasets/landsat/package/LandsatHlsRaster.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -219,12 +219,10 @@ bool LandsatHlsRaster::findRasters(finder_t* finder)
}
}
}
rgroup->infovect.shrink_to_fit();

// mlog(DEBUG, "Added group: %s with %ld rasters", rgroup->id.c_str(), rgroup->infovect.size());
finder->rasterGroups.push_back(rgroup);
}
finder->rasterGroups.shrink_to_fit();
// mlog(DEBUG, "Found %ld raster groups", finder->rasterGroups.size());
}
catch (const RunTimeException &e)
Expand Down Expand Up @@ -361,7 +359,7 @@ uint32_t LandsatHlsRaster::_getGroupSamples(sample_mode_t mode, const rasters_gr
/* Get the sample for this point from unique raster */
for(const point_sample_t& ps : ur->pointSamples)
{
if(ps.pointInfo.index == pointIndx)
if(ps.pointIndex == pointIndx)
{
/* sample can be NULL if raster read failed, (e.g. point out of bounds) */
if(ps.sample == NULL) break;
Expand Down
Loading
Loading