diff --git a/src/downloads/cdsretrieve.jl b/src/downloads/cdsretrieve.jl index ff34fbd7..11030d71 100644 --- a/src/downloads/cdsretrieve.jl +++ b/src/downloads/cdsretrieve.jl @@ -2,6 +2,7 @@ function cdsretrieve( e5ds :: ERA5CDStore, evar :: ERA5Variable, ereg :: ERA5Region, + grib :: Bool, overwrite :: Bool ) @@ -12,9 +13,17 @@ function cdsretrieve( for dtii in dtvec - fnc = e5dfnc(e5ds,evar,ereg,dtii) + if !grib + format = "netcdf" + fnc = e5dfnc(e5ds,evar,ereg,dtii) + else + format = "grib" + fnc = e5dgrib(e5ds,evar,ereg,dtii) + end fol = dirname(fnc); if !isdir(fol); mkpath(fol) end + + e5dkey = Dict( "product_type" => e5ds.ptype, "year" => year(dtii), @@ -22,7 +31,7 @@ function cdsretrieve( "variable" => evar.long, "grid" => [ereg.resolution, ereg.resolution], "time" => cdsretrieve_time(e5ds), - "format" => "netcdf", + "format" => format, ) if typeof(e5ds) <: ERA5Hourly @@ -126,11 +135,71 @@ function cdsretrieve( end +function cdsretrievegrib( + e5ds :: ERA5CDStore, + evar :: PressureVariable, + ereg :: ERA5Region, + pvec :: Vector{Int}, + overwrite :: Bool +) + + dtvec = cdsretrieve_dtvec(e5ds) + ckeys = cdskey() + + @info "$(modulelog()) - Using CDSAPI in Julia to download $(uppercase(e5ds.name)) $(evar.name) data in $(ereg.geo.name) (Horizontal Resolution: $(ereg.resolution)) from $(e5ds.start) to $(e5ds.stop)." + + for dtii in dtvec + + fnc = e5dgrib(e5ds,evar,ereg,dtii,pvec) + fol = dirname(fnc); if !isdir(fol); mkpath(fol) end + + e5dkey = Dict( + "product_type" => e5ds.ptype, + "year" => year(dtii), + "month" => cdsretrieve_month(dtii,e5ds), + "variable" => evar.long, + "grid" => [ereg.resolution, ereg.resolution], + "time" => cdsretrieve_time(e5ds), + "format" => "grib", + ) + + if typeof(e5ds) <: ERA5Hourly + e5dkey["day"] = collect(1:31) + end + + if typeof(evar) <: PressureVariable + e5dkey["pressure_level"] = pvec + end + + cdsretrieve_area!(e5dkey,ereg) + + if !isfile(fnc) || overwrite + tryretrieve = 0 + while isinteger(tryretrieve) && (tryretrieve < 20) + try + retrieve(cdsretrieve_dataset(evar,e5ds),e5dkey,fnc,ckeys) + tryretrieve += 0.5 + catch + tryretrieve += 1 + @info "$(modulelog()) - Failed to retrieve/request data from CDSAPI on Attempt $(tryretrieve) of 20" + end + end + if tryretrieve == 20 + @warn "$(modulelog()) - Failed to retrieve/request data, skipping to next set of requests" + end + end + + flush(stderr) + + end + +end + function cdsretrieve( e5ds :: ERA5CDStore, evar :: Vector{SingleVariable{ST}}, ereg :: ERA5Region, - overwrite :: Bool + overwrite :: Bool, ) where ST <: AbstractString dtvec = cdsretrieve_dtvec(e5ds) diff --git a/src/downloads/downloads.jl b/src/downloads/downloads.jl index 97cb7d3d..fcc5b86b 100644 --- a/src/downloads/downloads.jl +++ b/src/downloads/downloads.jl @@ -28,14 +28,13 @@ function download( evar :: SingleVariable, ereg :: ERA5Region; ispy :: Bool = false, + grib :: Bool = false, overwrite :: Bool = false ) - downloadcheckevar(evar) - if ispy pythonprint(e5ds,evar,ereg) - else; cdsretrieve(e5ds,evar,ereg,overwrite) + else; cdsretrieve(e5ds,evar,ereg,grib,overwrite) end end @@ -70,10 +69,6 @@ function download( overwrite :: Bool = false ) - for evarii in evar - downloadcheckevar(evarii) - end - cdsretrieve(e5ds,evar,ereg,overwrite) end @@ -120,39 +115,36 @@ function download( ptop :: Int = 0, pbot :: Int = 0, pvec :: Vector{Int} = [0], + grib :: Bool, overwrite :: Bool = false ) - downloadcheckevar(evar) - if ispy pythonprint(e5ds,evar,ereg) else - if pvec == [0] + if pvec == [0] || iszero(evar.hPa) pvec = downloadcheckplvl(pall,ptop,pbot) end if pall - cdsretrieve(e5ds,evar,ereg,pvec,overwrite) + if !grib + cdsretrieve(e5ds,evar,ereg,pvec,overwrite) + else + cdsretrievegrib(e5ds,evar,ereg,pvec,overwrite) + end else - cdsretrieve(e5ds,evar,ereg,overwrite) + cdsretrieve(e5ds,evar,ereg,grib,overwrite) end end end -function downloadcheckevar( - evar :: ERA5Variable +function downloadcheckhPa( + evar :: PressureVariable ) - if typeof(evar) <: Union{SingleVariable,PressureVariable} - - @info "$(modulelog()) - The ERA5Variable ID \"$(evar.ID)\" is a valid ERA5 Variable identifier and therefore can be used to download data from the Climate Data Store" - - flush(stderr) - - else + if iszero(evar.hPa) - error("$(modulelog()) - The ERA5Variable ID \"$(evar.ID)\" is a custom ERA5 Variable identifier and therefore cannot be used to download data from the Climate Data Store, and can only be calculated/analyzed from preexisting valid ERA5 data") + error("$(modulelog()) - The PressureVariable Level is set to 0, so \"pall\" is set to `true` (i.e., we are downloading all pressure levels, or a range specified by the keyword arguments `ptop`, `pbot` and `pvec`).") end