Skip to content

Commit

Permalink
migrated ICESat-2 and GEDI CMR providers to cloud data centers; retur…
Browse files Browse the repository at this point in the history 
…n error on truncation of CMR results
  • Loading branch information
@jpswinski
jpswinski committed Sep 6, 2024
1 parent 4e4ef08 commit 54cf7a4
Show file tree
Hide file tree
Showing 3 changed files with 24 additions and 22 deletions.
14 changes: 7 additions & 7 deletions clients/python/sliderule/earthdata.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,13 +57,13 @@

# best effort match of datasets to providers and versions for earthdata
DATASETS = {
"ATL03": {"provider": "NSIDC_ECS", "version": "006", "api": "cmr", "formats": [".h5"], "collections": []},
"ATL06": {"provider": "NSIDC_ECS", "version": "006", "api": "cmr", "formats": [".h5"], "collections": []},
"ATL08": {"provider": "NSIDC_ECS", "version": "006", "api": "cmr", "formats": [".h5"], "collections": []},
"ATL09": {"provider": "NSIDC_ECS", "version": "006", "api": "cmr", "formats": [".h5"], "collections": []},
"GEDI01_B": {"provider": "LPDAAC_ECS", "version": "002", "api": "cmr", "formats": [".h5"], "collections": []},
"GEDI02_A": {"provider": "LPDAAC_ECS", "version": "002", "api": "cmr", "formats": [".h5"], "collections": []},
"GEDI02_B": {"provider": "LPDAAC_ECS", "version": "002", "api": "cmr", "formats": [".tiff"], "collections": []},
"ATL03": {"provider": "NSIDC_CPRD", "version": "006", "api": "cmr", "formats": [".h5"], "collections": []},
"ATL06": {"provider": "NSIDC_CPRD", "version": "006", "api": "cmr", "formats": [".h5"], "collections": []},
"ATL08": {"provider": "NSIDC_CPRD", "version": "006", "api": "cmr", "formats": [".h5"], "collections": []},
"ATL09": {"provider": "NSIDC_CPRD", "version": "006", "api": "cmr", "formats": [".h5"], "collections": []},
"GEDI01_B": {"provider": "LPCLOUD", "version": "002", "api": "cmr", "formats": [".h5"], "collections": []},
"GEDI02_A": {"provider": "LPCLOUD", "version": "002", "api": "cmr", "formats": [".h5"], "collections": []},
"GEDI02_B": {"provider": "LPCLOUD", "version": "002", "api": "cmr", "formats": [".tiff"], "collections": []},
"GEDI_L3_LandSurface_Metrics_V2_1952": {"provider": "ORNL_CLOUD", "version": None, "api": "cmr", "formats": [".h5"], "collections": []},
"GEDI_L4A_AGB_Density_V2_1_2056": {"provider": "ORNL_CLOUD", "version": None, "api": "cmr", "formats": [".h5"], "collections": []},
"GEDI_L4B_Gridded_Biomass_2017": {"provider": "ORNL_CLOUD", "version": None, "api": "cmr", "formats": [".tiff"], "collections": []},
Expand Down
28 changes: 15 additions & 13 deletions scripts/extensions/earth_data_query.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,14 @@ DEFAULT_MAX_REQUESTED_RESOURCES = 300

-- best effort match of datasets to providers and versions for earthdata
DATASETS = {
ATL03 = {provider = "NSIDC_ECS", version = "006", api = "cmr", formats = {".h5"}, collections = {}},
ATL06 = {provider = "NSIDC_ECS", version = "006", api = "cmr", formats = {".h5"}, collections = {}},
ATL08 = {provider = "NSIDC_ECS", version = "006", api = "cmr", formats = {".h5"}, collections = {}},
ATL09 = {provider = "NSIDC_ECS", version = "006", api = "cmr", formats = {".h5"}, collections = {}},
ATL13 = {provider = "NSIDC_ECS", version = "006", api = "cmr", formats = {".h5"}, collections = {}},
GEDI01_B = {provider = "LPDAAC_ECS", version = "002", api = "cmr", formats = {".h5"}, collections = {}},
GEDI02_A = {provider = "LPDAAC_ECS", version = "002", api = "cmr", formats = {".h5"}, collections = {}},
GEDI02_B = {provider = "LPDAAC_ECS", version = "002", api = "cmr", formats = {".tiff"}, collections = {}},
ATL03 = {provider = "NSIDC_CPRD", version = "006", api = "cmr", formats = {".h5"}, collections = {}},
ATL06 = {provider = "NSIDC_CPRD", version = "006", api = "cmr", formats = {".h5"}, collections = {}},
ATL08 = {provider = "NSIDC_CPRD", version = "006", api = "cmr", formats = {".h5"}, collections = {}},
ATL09 = {provider = "NSIDC_CPRD", version = "006", api = "cmr", formats = {".h5"}, collections = {}},
ATL13 = {provider = "NSIDC_CPRD", version = "006", api = "cmr", formats = {".h5"}, collections = {}},
GEDI01_B = {provider = "LPCLOUD", version = "002", api = "cmr", formats = {".h5"}, collections = {}},
GEDI02_A = {provider = "LPCLOUD", version = "002", api = "cmr", formats = {".h5"}, collections = {}},
GEDI02_B = {provider = "LPCLOUD", version = "002", api = "cmr", formats = {".tiff"}, collections = {}},
GEDI_L3_LandSurface_Metrics_V2_1952 = {provider = "ORNL_CLOUD", version = nil, api = "cmr", formats = {".h5"}, collections = {}},
GEDI_L4A_AGB_Density_V2_1_2056 = {provider = "ORNL_CLOUD", version = nil, api = "cmr", formats = {".h5"}, collections = {}},
GEDI_L4B_Gridded_Biomass_2017 = {provider = "ORNL_CLOUD", version = nil, api = "cmr", formats = {".tiff"}, collections = {}},
Expand Down Expand Up @@ -72,6 +72,8 @@ RC_FAILURE = -1
RC_RQST_FAILED = -2
RC_RSPS_UNPARSEABLE = -3
RC_RSPS_UNEXPECTED = -4
RC_RSPS_TRUNCATED = -5
RC_UNSUPPORTED = -6

--
-- Build GeoJSON
Expand Down Expand Up @@ -260,8 +262,7 @@ local function cmr (parms, poly, with_meta)
if num_links == 0 then
break
elseif total_links >= max_resources then
sys.log(core.WARNING, string.format("Number of matched resources truncated to %d for %s", total_links, short_name))
break
return RC_RSPS_TRUNCATED, string.format("number of matched resources at %d exceeded maximum allowed for %s", total_links, short_name)
end
end

Expand Down Expand Up @@ -332,8 +333,7 @@ local function stac (parms, poly)
local num_returned = geotable["context"]["returned"]
local num_matched = geotable["context"]["matched"]
if num_matched > max_resources then
sys.log(core.WARNING, string.format("Number of matched resources truncated from %d to %d",num_matched, max_resources))
num_matched = max_resources
return RC_RSPS_TRUNCATED, string.format("number of matched resources truncated from %d to %d", num_matched, max_resources)
end
local num_pages = math.floor((num_matched + (num_returned - 1)) / num_returned)
for page = 2, num_pages do
Expand Down Expand Up @@ -496,7 +496,7 @@ local function search (parms, poly)
elseif api == "tnm" then
return tnm(parms, poly)
else
return RC_FAILURE, string.format("unsupport api: %s", api)
return RC_UNSUPPORTED, string.format("unsupported api: %s", api)
end

end
Expand All @@ -514,4 +514,6 @@ return {
RQST_FAILED = RC_RQST_FAILED,
RSPS_UNPARSEABLE = RC_RSPS_UNPARSEABLE,
RSPS_UNEXPECTED = RC_RSPS_UNEXPECTED,
RSPS_TRUNCATED = RC_RSPS_TRUNCATED,
UNSUPPORTED = RC_UNSUPPORTED
}
4 changes: 2 additions & 2 deletions scripts/extensions/proxy.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,8 @@ local function proxy(resources, parms, endpoint, rec)
parms[geo.PARMS][dataset]["catalog"] = json.encode(rsps)
local num_features = parms[geo.PARMS][dataset]["catalog"]["features"] and #parms[geo.PARMS][dataset]["catalog"]["features"] or 0
userlog:alert(core.INFO, core.RTE_INFO, string.format("proxy request <%s> returned %d resources for %s", rspq, num_features, dataset))
else
userlog:alert(core.WARNING, core.RTE_INFO, string.format("request <%s> failed to get catalog for %s: %d", rspq, dataset, rc))
elseif rc ~= earthdata.UNSUPPORTED then
userlog:alert(core.ERROR, core.RTE_ERROR, string.format("request <%s> failed to get catalog for %s <%d>: %s", rspq, dataset, rc, rsps))
end
end
end
Expand Down

0 comments on commit 54cf7a4

Please sign in to comment.