From 3abb3c81a71353d9b5c7d6e3c3a351d21b5617cd Mon Sep 17 00:00:00 2001 From: Nathanael Wong Date: Mon, 4 Mar 2024 10:27:29 -0500 Subject: [PATCH] Adding option to download grib files --- src/downloads/cdsretrieve.jl | 75 ++++++++++++++++++++++++++++-- src/downloads/downloads.jl | 36 ++++++--------- src/filesystem/raw.jl | 90 ++++++++++++++++++++++++++++++++++++ 3 files changed, 176 insertions(+), 25 deletions(-) diff --git a/src/downloads/cdsretrieve.jl b/src/downloads/cdsretrieve.jl index ff34fbd7..11030d71 100644 --- a/src/downloads/cdsretrieve.jl +++ b/src/downloads/cdsretrieve.jl @@ -2,6 +2,7 @@ function cdsretrieve( e5ds :: ERA5CDStore, evar :: ERA5Variable, ereg :: ERA5Region, + grib :: Bool, overwrite :: Bool ) @@ -12,9 +13,17 @@ function cdsretrieve( for dtii in dtvec - fnc = e5dfnc(e5ds,evar,ereg,dtii) + if !grib + format = "netcdf" + fnc = e5dfnc(e5ds,evar,ereg,dtii) + else + format = "grib" + fnc = e5dgrib(e5ds,evar,ereg,dtii) + end fol = dirname(fnc); if !isdir(fol); mkpath(fol) end + + e5dkey = Dict( "product_type" => e5ds.ptype, "year" => year(dtii), @@ -22,7 +31,7 @@ function cdsretrieve( "variable" => evar.long, "grid" => [ereg.resolution, ereg.resolution], "time" => cdsretrieve_time(e5ds), - "format" => "netcdf", + "format" => format, ) if typeof(e5ds) <: ERA5Hourly @@ -126,11 +135,71 @@ function cdsretrieve( end +function cdsretrievegrib( + e5ds :: ERA5CDStore, + evar :: PressureVariable, + ereg :: ERA5Region, + pvec :: Vector{Int}, + overwrite :: Bool +) + + dtvec = cdsretrieve_dtvec(e5ds) + ckeys = cdskey() + + @info "$(modulelog()) - Using CDSAPI in Julia to download $(uppercase(e5ds.name)) $(evar.name) data in $(ereg.geo.name) (Horizontal Resolution: $(ereg.resolution)) from $(e5ds.start) to $(e5ds.stop)." + + for dtii in dtvec + + fnc = e5dgrib(e5ds,evar,ereg,dtii,pvec) + fol = dirname(fnc); if !isdir(fol); mkpath(fol) end + + e5dkey = Dict( + "product_type" => e5ds.ptype, + "year" => year(dtii), + "month" => cdsretrieve_month(dtii,e5ds), + "variable" => evar.long, + "grid" => [ereg.resolution, ereg.resolution], + "time" => cdsretrieve_time(e5ds), + "format" => "grib", + ) + + if typeof(e5ds) <: ERA5Hourly + e5dkey["day"] = collect(1:31) + end + + if typeof(evar) <: PressureVariable + e5dkey["pressure_level"] = pvec + end + + cdsretrieve_area!(e5dkey,ereg) + + if !isfile(fnc) || overwrite + tryretrieve = 0 + while isinteger(tryretrieve) && (tryretrieve < 20) + try + retrieve(cdsretrieve_dataset(evar,e5ds),e5dkey,fnc,ckeys) + tryretrieve += 0.5 + catch + tryretrieve += 1 + @info "$(modulelog()) - Failed to retrieve/request data from CDSAPI on Attempt $(tryretrieve) of 20" + end + end + if tryretrieve == 20 + @warn "$(modulelog()) - Failed to retrieve/request data, skipping to next set of requests" + end + end + + flush(stderr) + + end + +end + function cdsretrieve( e5ds :: ERA5CDStore, evar :: Vector{SingleVariable{ST}}, ereg :: ERA5Region, - overwrite :: Bool + overwrite :: Bool, ) where ST <: AbstractString dtvec = cdsretrieve_dtvec(e5ds) diff --git a/src/downloads/downloads.jl b/src/downloads/downloads.jl index 97cb7d3d..fcc5b86b 100644 --- a/src/downloads/downloads.jl +++ b/src/downloads/downloads.jl @@ -28,14 +28,13 @@ function download( evar :: SingleVariable, ereg :: ERA5Region; ispy :: Bool = false, + grib :: Bool = false, overwrite :: Bool = false ) - downloadcheckevar(evar) - if ispy pythonprint(e5ds,evar,ereg) - else; cdsretrieve(e5ds,evar,ereg,overwrite) + else; cdsretrieve(e5ds,evar,ereg,grib,overwrite) end end @@ -70,10 +69,6 @@ function download( overwrite :: Bool = false ) - for evarii in evar - downloadcheckevar(evarii) - end - cdsretrieve(e5ds,evar,ereg,overwrite) end @@ -120,39 +115,36 @@ function download( ptop :: Int = 0, pbot :: Int = 0, pvec :: Vector{Int} = [0], + grib :: Bool, overwrite :: Bool = false ) - downloadcheckevar(evar) - if ispy pythonprint(e5ds,evar,ereg) else - if pvec == [0] + if pvec == [0] || iszero(evar.hPa) pvec = downloadcheckplvl(pall,ptop,pbot) end if pall - cdsretrieve(e5ds,evar,ereg,pvec,overwrite) + if !grib + cdsretrieve(e5ds,evar,ereg,pvec,overwrite) + else + cdsretrievegrib(e5ds,evar,ereg,pvec,overwrite) + end else - cdsretrieve(e5ds,evar,ereg,overwrite) + cdsretrieve(e5ds,evar,ereg,grib,overwrite) end end end -function downloadcheckevar( - evar :: ERA5Variable +function downloadcheckhPa( + evar :: PressureVariable ) - if typeof(evar) <: Union{SingleVariable,PressureVariable} - - @info "$(modulelog()) - The ERA5Variable ID \"$(evar.ID)\" is a valid ERA5 Variable identifier and therefore can be used to download data from the Climate Data Store" - - flush(stderr) - - else + if iszero(evar.hPa) - error("$(modulelog()) - The ERA5Variable ID \"$(evar.ID)\" is a custom ERA5 Variable identifier and therefore cannot be used to download data from the Climate Data Store, and can only be calculated/analyzed from preexisting valid ERA5 data") + error("$(modulelog()) - The PressureVariable Level is set to 0, so \"pall\" is set to `true` (i.e., we are downloading all pressure levels, or a range specified by the keyword arguments `ptop`, `pbot` and `pvec`).") end diff --git a/src/filesystem/raw.jl b/src/filesystem/raw.jl index d2791677..e7237c48 100644 --- a/src/filesystem/raw.jl +++ b/src/filesystem/raw.jl @@ -95,4 +95,94 @@ function e5dfnc( fnc = e5ds.ID * "-" * ereg.string * "-" * evp return fol, fnc +end + +function e5dgrib( + e5ds :: ERA5Hourly, + evar :: SingleLevel, + ereg :: ERA5Region, + dt :: TimeType +) + + dts = yr2str(dt) + fol = joinpath(e5ds.path,ereg.string,evar.ID,dts) + fnc = e5ds.ID * "-" * ereg.string * "-" * evar.ID * "-" * yrmo2str(dt) * ".grb" + return joinpath(fol,fnc) + +end + +function e5dgrib( + e5ds :: ERA5Hourly, + evar :: PressureLevel, + ereg :: ERA5Region, + dt :: TimeType +) + + evp = evar.ID * "-$(evar.hPa)hPa" + dts = yr2str(dt) + fol = joinpath(e5ds.path,ereg.string,evar.ID,evp,dts) + fnc = e5ds.ID * "-" * ereg.string * "-" * evp * "-" * yrmo2str(dt) * ".grb" + return joinpath(fol,fnc) + +end + +function e5dgrib( + e5ds :: ERA5Hourly, + evar :: PressureLevel, + ereg :: ERA5Region, + dt :: TimeType, + pvec :: Vector{Int}, +) + + evp = evar.ID * "-$(pvec[1])hPa_$(pvec[end])hPa" + dts = yr2str(dt) + fol = joinpath(e5ds.path,ereg.string,evar.ID,evp,dts) + fnc = e5ds.ID * "-" * ereg.string * "-" * evp * "-" * yrmo2str(dt) * ".grb" + return joinpath(fol,fnc) + +end + +function e5dgrib( + e5ds :: ERA5Monthly, + evar :: SingleLevel, + ereg :: ERA5Region, + dt :: TimeType +) + + dts = yr2str(dt) + fol = joinpath(e5ds.path,ereg.string,evar.ID) + fnc = e5ds.ID * "-" * ereg.string * "-" * evar.ID * "-" * dts * ".grb" + return joinpath(fol,fnc) + +end + +function e5dgrib( + e5ds :: ERA5Monthly, + evar :: PressureLevel, + ereg :: ERA5Region, + dt :: TimeType +) + + evp = evar.ID * "-$(evar.hPa)hPa" + dts = yr2str(dt) + fol = joinpath(e5ds.path,ereg.string,evar.ID,evp) + fnc = e5ds.ID * "-" * ereg.string * "-" * evp * "-" * dts * ".grb" + return joinpath(fol,fnc) + +end + +function e5dgrib( + e5ds :: ERA5Monthly, + evar :: PressureLevel, + ereg :: ERA5Region, + dt :: TimeType, + pvec :: Vector{Int} +) + + evp = evar.ID * "-$(pvec[1])hPa_$(pvec[end])hPa" + dts = yr2str(dt) + fol = joinpath(e5ds.path,ereg.string,evar.ID,evp) + fnc = e5ds.ID * "-" * ereg.string * "-" * evp * "-" * dts * ".grb" + return joinpath(fol,fnc) + end \ No newline at end of file