Skip to content

Commit

Permalink
Adding option to download grib files
Browse files Browse the repository at this point in the history
  • Loading branch information
@natgeo-wong
natgeo-wong committed Mar 4, 2024
1 parent bee2653 commit 3abb3c8
Show file tree
Hide file tree
Showing 3 changed files with 176 additions and 25 deletions.
75 changes: 72 additions & 3 deletions src/downloads/cdsretrieve.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ function cdsretrieve(
e5ds :: ERA5CDStore,
evar :: ERA5Variable,
ereg :: ERA5Region,
grib :: Bool,
overwrite :: Bool
)

Expand All @@ -12,17 +13,25 @@ function cdsretrieve(

for dtii in dtvec

fnc = e5dfnc(e5ds,evar,ereg,dtii)
if !grib
format = "netcdf"
fnc = e5dfnc(e5ds,evar,ereg,dtii)
else
format = "grib"
fnc = e5dgrib(e5ds,evar,ereg,dtii)
end
fol = dirname(fnc); if !isdir(fol); mkpath(fol) end



e5dkey = Dict(
"product_type" => e5ds.ptype,
"year" => year(dtii),
"month" => cdsretrieve_month(dtii,e5ds),
"variable" => evar.long,
"grid" => [ereg.resolution, ereg.resolution],
"time" => cdsretrieve_time(e5ds),
"format" => "netcdf",
"format" => format,
)

if typeof(e5ds) <: ERA5Hourly
Expand Down Expand Up @@ -126,11 +135,71 @@ function cdsretrieve(

end

function cdsretrievegrib(
e5ds :: ERA5CDStore,
evar :: PressureVariable,
ereg :: ERA5Region,
pvec :: Vector{Int},
overwrite :: Bool
)

dtvec = cdsretrieve_dtvec(e5ds)
ckeys = cdskey()

@info "$(modulelog()) - Using CDSAPI in Julia to download $(uppercase(e5ds.name)) $(evar.name) data in $(ereg.geo.name) (Horizontal Resolution: $(ereg.resolution)) from $(e5ds.start) to $(e5ds.stop)."

for dtii in dtvec

fnc = e5dgrib(e5ds,evar,ereg,dtii,pvec)
fol = dirname(fnc); if !isdir(fol); mkpath(fol) end

e5dkey = Dict(
"product_type" => e5ds.ptype,
"year" => year(dtii),
"month" => cdsretrieve_month(dtii,e5ds),
"variable" => evar.long,
"grid" => [ereg.resolution, ereg.resolution],
"time" => cdsretrieve_time(e5ds),
"format" => "grib",
)

if typeof(e5ds) <: ERA5Hourly
e5dkey["day"] = collect(1:31)
end

if typeof(evar) <: PressureVariable
e5dkey["pressure_level"] = pvec
end

cdsretrieve_area!(e5dkey,ereg)

if !isfile(fnc) || overwrite
tryretrieve = 0
while isinteger(tryretrieve) && (tryretrieve < 20)
try
retrieve(cdsretrieve_dataset(evar,e5ds),e5dkey,fnc,ckeys)
tryretrieve += 0.5
catch
tryretrieve += 1
@info "$(modulelog()) - Failed to retrieve/request data from CDSAPI on Attempt $(tryretrieve) of 20"
end
end
if tryretrieve == 20
@warn "$(modulelog()) - Failed to retrieve/request data, skipping to next set of requests"
end
end

flush(stderr)

end

end

function cdsretrieve(
e5ds :: ERA5CDStore,
evar :: Vector{SingleVariable{ST}},
ereg :: ERA5Region,
overwrite :: Bool
overwrite :: Bool,
) where ST <: AbstractString

dtvec = cdsretrieve_dtvec(e5ds)
Expand Down
36 changes: 14 additions & 22 deletions src/downloads/downloads.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,13 @@ function download(
evar :: SingleVariable,
ereg :: ERA5Region;
ispy :: Bool = false,
grib :: Bool = false,
overwrite :: Bool = false
)

downloadcheckevar(evar)

if ispy
pythonprint(e5ds,evar,ereg)
else; cdsretrieve(e5ds,evar,ereg,overwrite)
else; cdsretrieve(e5ds,evar,ereg,grib,overwrite)
end

end
Expand Down Expand Up @@ -70,10 +69,6 @@ function download(
overwrite :: Bool = false
)

for evarii in evar
downloadcheckevar(evarii)
end

cdsretrieve(e5ds,evar,ereg,overwrite)

end
Expand Down Expand Up @@ -120,39 +115,36 @@ function download(
ptop :: Int = 0,
pbot :: Int = 0,
pvec :: Vector{Int} = [0],
grib :: Bool,
overwrite :: Bool = false
)

downloadcheckevar(evar)

if ispy
pythonprint(e5ds,evar,ereg)
else
if pvec == [0]
if pvec == [0] || iszero(evar.hPa)
pvec = downloadcheckplvl(pall,ptop,pbot)
end
if pall
cdsretrieve(e5ds,evar,ereg,pvec,overwrite)
if !grib
cdsretrieve(e5ds,evar,ereg,pvec,overwrite)
else
cdsretrievegrib(e5ds,evar,ereg,pvec,overwrite)
end
else
cdsretrieve(e5ds,evar,ereg,overwrite)
cdsretrieve(e5ds,evar,ereg,grib,overwrite)
end
end

end

function downloadcheckevar(
evar :: ERA5Variable
function downloadcheckhPa(
evar :: PressureVariable
)

if typeof(evar) <: Union{SingleVariable,PressureVariable}

@info "$(modulelog()) - The ERA5Variable ID \"$(evar.ID)\" is a valid ERA5 Variable identifier and therefore can be used to download data from the Climate Data Store"

flush(stderr)

else
if iszero(evar.hPa)

error("$(modulelog()) - The ERA5Variable ID \"$(evar.ID)\" is a custom ERA5 Variable identifier and therefore cannot be used to download data from the Climate Data Store, and can only be calculated/analyzed from preexisting valid ERA5 data")
error("$(modulelog()) - The PressureVariable Level is set to 0, so \"pall\" is set to `true` (i.e., we are downloading all pressure levels, or a range specified by the keyword arguments `ptop`, `pbot` and `pvec`).")

end

Expand Down
90 changes: 90 additions & 0 deletions src/filesystem/raw.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,4 +95,94 @@ function e5dfnc(
fnc = e5ds.ID * "-" * ereg.string * "-" * evp
return fol, fnc

end

function e5dgrib(
e5ds :: ERA5Hourly,
evar :: SingleLevel,
ereg :: ERA5Region,
dt :: TimeType
)

dts = yr2str(dt)
fol = joinpath(e5ds.path,ereg.string,evar.ID,dts)
fnc = e5ds.ID * "-" * ereg.string * "-" * evar.ID * "-" * yrmo2str(dt) * ".grb"
return joinpath(fol,fnc)

end

function e5dgrib(
e5ds :: ERA5Hourly,
evar :: PressureLevel,
ereg :: ERA5Region,
dt :: TimeType
)

evp = evar.ID * "-$(evar.hPa)hPa"
dts = yr2str(dt)
fol = joinpath(e5ds.path,ereg.string,evar.ID,evp,dts)
fnc = e5ds.ID * "-" * ereg.string * "-" * evp * "-" * yrmo2str(dt) * ".grb"
return joinpath(fol,fnc)

end

function e5dgrib(
e5ds :: ERA5Hourly,
evar :: PressureLevel,
ereg :: ERA5Region,
dt :: TimeType,
pvec :: Vector{Int},
)

evp = evar.ID * "-$(pvec[1])hPa_$(pvec[end])hPa"
dts = yr2str(dt)
fol = joinpath(e5ds.path,ereg.string,evar.ID,evp,dts)
fnc = e5ds.ID * "-" * ereg.string * "-" * evp * "-" * yrmo2str(dt) * ".grb"
return joinpath(fol,fnc)

end

function e5dgrib(
e5ds :: ERA5Monthly,
evar :: SingleLevel,
ereg :: ERA5Region,
dt :: TimeType
)

dts = yr2str(dt)
fol = joinpath(e5ds.path,ereg.string,evar.ID)
fnc = e5ds.ID * "-" * ereg.string * "-" * evar.ID * "-" * dts * ".grb"
return joinpath(fol,fnc)

end

function e5dgrib(
e5ds :: ERA5Monthly,
evar :: PressureLevel,
ereg :: ERA5Region,
dt :: TimeType
)

evp = evar.ID * "-$(evar.hPa)hPa"
dts = yr2str(dt)
fol = joinpath(e5ds.path,ereg.string,evar.ID,evp)
fnc = e5ds.ID * "-" * ereg.string * "-" * evp * "-" * dts * ".grb"
return joinpath(fol,fnc)

end

function e5dgrib(
e5ds :: ERA5Monthly,
evar :: PressureLevel,
ereg :: ERA5Region,
dt :: TimeType,
pvec :: Vector{Int}
)

evp = evar.ID * "-$(pvec[1])hPa_$(pvec[end])hPa"
dts = yr2str(dt)
fol = joinpath(e5ds.path,ereg.string,evar.ID,evp)
fnc = e5ds.ID * "-" * ereg.string * "-" * evp * "-" * dts * ".grb"
return joinpath(fol,fnc)

end

0 comments on commit 3abb3c8

Please sign in to comment.