From 54cf7a4856c4ce4fec4846eaab5e4cd40cb0b060 Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Fri, 6 Sep 2024 12:19:53 +0000 Subject: [PATCH] migrated ICESat-2 and GEDI CMR providers to cloud data centers; return error on truncation of CMR results --- clients/python/sliderule/earthdata.py | 14 ++++++------- scripts/extensions/earth_data_query.lua | 28 +++++++++++++------------ scripts/extensions/proxy.lua | 4 ++-- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/clients/python/sliderule/earthdata.py b/clients/python/sliderule/earthdata.py index b34651efa..2799de643 100644 --- a/clients/python/sliderule/earthdata.py +++ b/clients/python/sliderule/earthdata.py @@ -57,13 +57,13 @@ # best effort match of datasets to providers and versions for earthdata DATASETS = { - "ATL03": {"provider": "NSIDC_ECS", "version": "006", "api": "cmr", "formats": [".h5"], "collections": []}, - "ATL06": {"provider": "NSIDC_ECS", "version": "006", "api": "cmr", "formats": [".h5"], "collections": []}, - "ATL08": {"provider": "NSIDC_ECS", "version": "006", "api": "cmr", "formats": [".h5"], "collections": []}, - "ATL09": {"provider": "NSIDC_ECS", "version": "006", "api": "cmr", "formats": [".h5"], "collections": []}, - "GEDI01_B": {"provider": "LPDAAC_ECS", "version": "002", "api": "cmr", "formats": [".h5"], "collections": []}, - "GEDI02_A": {"provider": "LPDAAC_ECS", "version": "002", "api": "cmr", "formats": [".h5"], "collections": []}, - "GEDI02_B": {"provider": "LPDAAC_ECS", "version": "002", "api": "cmr", "formats": [".tiff"], "collections": []}, + "ATL03": {"provider": "NSIDC_CPRD", "version": "006", "api": "cmr", "formats": [".h5"], "collections": []}, + "ATL06": {"provider": "NSIDC_CPRD", "version": "006", "api": "cmr", "formats": [".h5"], "collections": []}, + "ATL08": {"provider": "NSIDC_CPRD", "version": "006", "api": "cmr", "formats": [".h5"], "collections": []}, + "ATL09": {"provider": "NSIDC_CPRD", "version": "006", "api": "cmr", "formats": [".h5"], "collections": []}, + "GEDI01_B": {"provider": "LPCLOUD", "version": "002", "api": "cmr", "formats": [".h5"], "collections": []}, + "GEDI02_A": {"provider": "LPCLOUD", "version": "002", "api": "cmr", "formats": [".h5"], "collections": []}, + "GEDI02_B": {"provider": "LPCLOUD", "version": "002", "api": "cmr", "formats": [".tiff"], "collections": []}, "GEDI_L3_LandSurface_Metrics_V2_1952": {"provider": "ORNL_CLOUD", "version": None, "api": "cmr", "formats": [".h5"], "collections": []}, "GEDI_L4A_AGB_Density_V2_1_2056": {"provider": "ORNL_CLOUD", "version": None, "api": "cmr", "formats": [".h5"], "collections": []}, "GEDI_L4B_Gridded_Biomass_2017": {"provider": "ORNL_CLOUD", "version": None, "api": "cmr", "formats": [".tiff"], "collections": []}, diff --git a/scripts/extensions/earth_data_query.lua b/scripts/extensions/earth_data_query.lua index 6896d90a7..ae51ff38b 100644 --- a/scripts/extensions/earth_data_query.lua +++ b/scripts/extensions/earth_data_query.lua @@ -15,14 +15,14 @@ DEFAULT_MAX_REQUESTED_RESOURCES = 300 -- best effort match of datasets to providers and versions for earthdata DATASETS = { - ATL03 = {provider = "NSIDC_ECS", version = "006", api = "cmr", formats = {".h5"}, collections = {}}, - ATL06 = {provider = "NSIDC_ECS", version = "006", api = "cmr", formats = {".h5"}, collections = {}}, - ATL08 = {provider = "NSIDC_ECS", version = "006", api = "cmr", formats = {".h5"}, collections = {}}, - ATL09 = {provider = "NSIDC_ECS", version = "006", api = "cmr", formats = {".h5"}, collections = {}}, - ATL13 = {provider = "NSIDC_ECS", version = "006", api = "cmr", formats = {".h5"}, collections = {}}, - GEDI01_B = {provider = "LPDAAC_ECS", version = "002", api = "cmr", formats = {".h5"}, collections = {}}, - GEDI02_A = {provider = "LPDAAC_ECS", version = "002", api = "cmr", formats = {".h5"}, collections = {}}, - GEDI02_B = {provider = "LPDAAC_ECS", version = "002", api = "cmr", formats = {".tiff"}, collections = {}}, + ATL03 = {provider = "NSIDC_CPRD", version = "006", api = "cmr", formats = {".h5"}, collections = {}}, + ATL06 = {provider = "NSIDC_CPRD", version = "006", api = "cmr", formats = {".h5"}, collections = {}}, + ATL08 = {provider = "NSIDC_CPRD", version = "006", api = "cmr", formats = {".h5"}, collections = {}}, + ATL09 = {provider = "NSIDC_CPRD", version = "006", api = "cmr", formats = {".h5"}, collections = {}}, + ATL13 = {provider = "NSIDC_CPRD", version = "006", api = "cmr", formats = {".h5"}, collections = {}}, + GEDI01_B = {provider = "LPCLOUD", version = "002", api = "cmr", formats = {".h5"}, collections = {}}, + GEDI02_A = {provider = "LPCLOUD", version = "002", api = "cmr", formats = {".h5"}, collections = {}}, + GEDI02_B = {provider = "LPCLOUD", version = "002", api = "cmr", formats = {".tiff"}, collections = {}}, GEDI_L3_LandSurface_Metrics_V2_1952 = {provider = "ORNL_CLOUD", version = nil, api = "cmr", formats = {".h5"}, collections = {}}, GEDI_L4A_AGB_Density_V2_1_2056 = {provider = "ORNL_CLOUD", version = nil, api = "cmr", formats = {".h5"}, collections = {}}, GEDI_L4B_Gridded_Biomass_2017 = {provider = "ORNL_CLOUD", version = nil, api = "cmr", formats = {".tiff"}, collections = {}}, @@ -72,6 +72,8 @@ RC_FAILURE = -1 RC_RQST_FAILED = -2 RC_RSPS_UNPARSEABLE = -3 RC_RSPS_UNEXPECTED = -4 +RC_RSPS_TRUNCATED = -5 +RC_UNSUPPORTED = -6 -- -- Build GeoJSON @@ -260,8 +262,7 @@ local function cmr (parms, poly, with_meta) if num_links == 0 then break elseif total_links >= max_resources then - sys.log(core.WARNING, string.format("Number of matched resources truncated to %d for %s", total_links, short_name)) - break + return RC_RSPS_TRUNCATED, string.format("number of matched resources at %d exceeded maximum allowed for %s", total_links, short_name) end end @@ -332,8 +333,7 @@ local function stac (parms, poly) local num_returned = geotable["context"]["returned"] local num_matched = geotable["context"]["matched"] if num_matched > max_resources then - sys.log(core.WARNING, string.format("Number of matched resources truncated from %d to %d",num_matched, max_resources)) - num_matched = max_resources + return RC_RSPS_TRUNCATED, string.format("number of matched resources truncated from %d to %d", num_matched, max_resources) end local num_pages = math.floor((num_matched + (num_returned - 1)) / num_returned) for page = 2, num_pages do @@ -496,7 +496,7 @@ local function search (parms, poly) elseif api == "tnm" then return tnm(parms, poly) else - return RC_FAILURE, string.format("unsupport api: %s", api) + return RC_UNSUPPORTED, string.format("unsupported api: %s", api) end end @@ -514,4 +514,6 @@ return { RQST_FAILED = RC_RQST_FAILED, RSPS_UNPARSEABLE = RC_RSPS_UNPARSEABLE, RSPS_UNEXPECTED = RC_RSPS_UNEXPECTED, + RSPS_TRUNCATED = RC_RSPS_TRUNCATED, + UNSUPPORTED = RC_UNSUPPORTED } diff --git a/scripts/extensions/proxy.lua b/scripts/extensions/proxy.lua index e23e8c2f0..66b205252 100644 --- a/scripts/extensions/proxy.lua +++ b/scripts/extensions/proxy.lua @@ -79,8 +79,8 @@ local function proxy(resources, parms, endpoint, rec) parms[geo.PARMS][dataset]["catalog"] = json.encode(rsps) local num_features = parms[geo.PARMS][dataset]["catalog"]["features"] and #parms[geo.PARMS][dataset]["catalog"]["features"] or 0 userlog:alert(core.INFO, core.RTE_INFO, string.format("proxy request <%s> returned %d resources for %s", rspq, num_features, dataset)) - else - userlog:alert(core.WARNING, core.RTE_INFO, string.format("request <%s> failed to get catalog for %s: %d", rspq, dataset, rc)) + elseif rc ~= earthdata.UNSUPPORTED then + userlog:alert(core.ERROR, core.RTE_ERROR, string.format("request <%s> failed to get catalog for %s <%d>: %s", rspq, dataset, rc, rsps)) end end end