From c4c9315595b9289bf12c7bc30da65e6abd3b2868 Mon Sep 17 00:00:00 2001 From: Guido Kraemer Date: Tue, 26 Sep 2023 13:40:25 +0200 Subject: [PATCH 1/3] make esdc() not load a wrong fallback cube --- src/esdc.jl | 66 +++++++++++++++++++++++++++++--------------------- test/access.jl | 21 +++++++++++++++- 2 files changed, 58 insertions(+), 29 deletions(-) diff --git a/src/esdc.jl b/src/esdc.jl index 816b746..45663e2 100644 --- a/src/esdc.jl +++ b/src/esdc.jl @@ -7,21 +7,19 @@ global cubesdict function __init__() - global cubesdict, cubesdict3 + global cubesdict cubesdict = Dict( - ("low","ts","global") => ("esdl-esdc-v2.1.1","esdc-8d-0.25deg-184x90x90-2.1.1.zarr"), - ("low","map","global") => ("esdl-esdc-v2.1.1","esdc-8d-0.25deg-1x720x1440-2.1.1.zarr"), - ("high","ts","global") => ("esdl-esdc-v2.1.1","esdc-8d-0.083deg-184x270x270-2.1.1.zarr"), - ("high","map","global") => ("esdl-esdc-v2.1.1","esdc-8d-0.083deg-1x2160x4320-2.1.1.zarr"), - ("low","ts","Colombia") => ("esdl-esdc-v2.0.1","Cube_2019lowColombiaCube_184x60x60.zarr"), - ("low","map","Colombia") => ("esdl-esdc-v2.0.1","Cube_2019lowColombiaCube_1x336x276.zarr/"), - ("high","ts","Colombia") => ("esdl-esdc-v2.0.1","Cube_2019highColombiaCube_184x120x120.zarr"), - ("high","map","Colombia") => ("esdl-esdc-v2.0.1","Cube_2019highColombiaCube_1x3360x2760.zarr"), - ) - cubesdict3 = Dict( - "ts" => "esdc-8d-0.25deg-256x128x128-3.0.2.zarr", - "map" => "esdc-8d-0.25deg-1x720x1440-3.0.2.zarr", - "tiny" => "esdc-16d-2.5deg-46x72x1440-3.0.2.zarr", + ("low", "ts", "global", 2) => ("esdl-esdc-v2.1.1", "esdc-8d-0.25deg-184x90x90-2.1.1.zarr"), + ("low", "map", "global", 2) => ("esdl-esdc-v2.1.1", "esdc-8d-0.25deg-1x720x1440-2.1.1.zarr"), + ("high", "ts", "global", 2) => ("esdl-esdc-v2.1.1", "esdc-8d-0.083deg-184x270x270-2.1.1.zarr"), + ("high", "map", "global", 2) => ("esdl-esdc-v2.1.1", "esdc-8d-0.083deg-1x2160x4320-2.1.1.zarr"), + ("low", "ts", "Colombia", 2) => ("esdl-esdc-v2.0.1", "Cube_2019lowColombiaCube_184x60x60.zarr"), + ("low", "map", "Colombia", 2) => ("esdl-esdc-v2.0.1", "Cube_2019lowColombiaCube_1x336x276.zarr"), + ("high", "ts", "Colombia", 2) => ("esdl-esdc-v2.0.1", "Cube_2019highColombiaCube_184x120x120.zarr"), + ("high", "map", "Colombia", 2) => ("esdl-esdc-v2.0.1", "Cube_2019highColombiaCube_1x3360x2760.zarr"), + ("low", "ts", "global", 3) => ("esdl-esdc-v3.0.2", "esdc-8d-0.25deg-256x128x128-3.0.2.zarr"), + ("low", "map", "global", 3) => ("esdl-esdc-v3.0.2", "esdc-8d-0.25deg-1x720x1440-3.0.2.zarr"), + ("tiny", "ts", "global", 3) => ("esdl-esdc-v3.0.2", "esdc-16d-2.5deg-46x72x1440-3.0.2.zarr") ) end @@ -41,23 +39,35 @@ and `store` or pick a resolution, chunking and cube region. * `region="global"` choose a datacube (either `"global"` or `"Colombia"`), works only for esdc v2 * `version=3` """ -function esdd(;bucket=nothing, store="", res="low", chunks="ts", region="global", version=3) - if version == 2 - if bucket===nothing - bucket, store = cubesdict[(res,chunks,region)] - end - elseif version == 3 - if bucket===nothing - bucket = "esdl-esdc-v3.0.2" - if res=="tiny" - store = cubesdict3[res] - else - store = cubesdict3[chunks] - end +function esdd(; bucket = nothing, store = nothing, res = "low", chunks = "ts", region = "global", version = nothing) + + if version === nothing + if region == "global" + version = 3 + elseif region == "Colombia" + version = 2 end end + + k = (res, chunks, region, version) + + # give a sensible error if key does not exist + if !haskey(cubesdict, k) + @info "possible keys are" keys(cubesdict) + error("key $(k) does not exist") + end + + # switch out defaults + b, s = cubesdict[k] + if bucket === nothing + bucket = b + end + if store === nothing + store = s + end + path = "https://s3.bgc-jena.mpg.de:9000/" * bucket * "/" * store - open_dataset(zopen(path,consolidated=true,fill_as_missing=true)) + open_dataset(zopen(path, consolidated = true, fill_as_missing = true)) end """ diff --git a/test/access.jl b/test/access.jl index 9777cca..ac8f380 100644 --- a/test/access.jl +++ b/test/access.jl @@ -1,4 +1,5 @@ using Dates +import DimensionalData c=Cube() @@ -126,4 +127,22 @@ end c = esdd(res="tiny") c.gross_primary_productivity[time=DD.Near(DateTime(2005))].data[44,14] == 2.3713999f0 -end \ No newline at end of file +end + +@testset "regions work" begin + c1 = esdc(region = "Colombia") + @test DimensionalData.bounds(c1, :lon) == (-82.9587215, -60.0421465) + @test DimensionalData.bounds(c1, :lat) == (-13.957917500000002, 13.9586375) + + c2 = esdc() # global low resolution + @test DimensionalData.bounds(c2, :lon) == (-179.875, 179.875) + @test DimensionalData.span(c2, :lon).step == 0.25 + + c3 = esdc(res = "high", version = 2) # global high resolution + @test DimensionalData.bounds(c3, :lon) == (-179.95833333333331, 179.95833333333331) + @test DimensionalData.span(c3, :lon).step == 0.08333333333333333 + + c4 = esdc(res = "tiny") # global tiny cube + @test DimensionalData.span(c4, :lon).step == 2.5 + @test DimensionalData.bounds(c4, :lon) == (-178.75, 178.75) +end From 549355a90e17b6494afd43ae3538f3306111c4c5 Mon Sep 17 00:00:00 2001 From: Guido Kraemer Date: Tue, 26 Sep 2023 14:20:49 +0200 Subject: [PATCH 2/3] make esdd() a bit smarter --- src/esdc.jl | 12 ++++++++---- test/access.jl | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/esdc.jl b/src/esdc.jl index 45663e2..fb55aed 100644 --- a/src/esdc.jl +++ b/src/esdc.jl @@ -43,7 +43,11 @@ function esdd(; bucket = nothing, store = nothing, res = "low", chunks = "ts", r if version === nothing if region == "global" - version = 3 + if res == "high" + version = 2 + else + version = 3 + end elseif region == "Colombia" version = 2 end @@ -53,12 +57,12 @@ function esdd(; bucket = nothing, store = nothing, res = "low", chunks = "ts", r # give a sensible error if key does not exist if !haskey(cubesdict, k) - @info "possible keys are" keys(cubesdict) - error("key $(k) does not exist") + @info "possible keys are" keys(cubesdict) + error("key $(k) does not exist") end + b, s = cubesdict[k] # switch out defaults - b, s = cubesdict[k] if bucket === nothing bucket = b end diff --git a/test/access.jl b/test/access.jl index ac8f380..19bc218 100644 --- a/test/access.jl +++ b/test/access.jl @@ -129,7 +129,7 @@ end c.gross_primary_productivity[time=DD.Near(DateTime(2005))].data[44,14] == 2.3713999f0 end -@testset "regions work" begin +@testset "esdd smarts" begin c1 = esdc(region = "Colombia") @test DimensionalData.bounds(c1, :lon) == (-82.9587215, -60.0421465) @test DimensionalData.bounds(c1, :lat) == (-13.957917500000002, 13.9586375) @@ -138,7 +138,7 @@ end @test DimensionalData.bounds(c2, :lon) == (-179.875, 179.875) @test DimensionalData.span(c2, :lon).step == 0.25 - c3 = esdc(res = "high", version = 2) # global high resolution + c3 = esdc(res = "high") # global high resolution @test DimensionalData.bounds(c3, :lon) == (-179.95833333333331, 179.95833333333331) @test DimensionalData.span(c3, :lon).step == 0.08333333333333333 From 7200e386192e752e92aef0fb19b6e11c0b36e120 Mon Sep 17 00:00:00 2001 From: Guido Kraemer Date: Tue, 26 Sep 2023 14:24:07 +0200 Subject: [PATCH 3/3] fix documentation --- src/esdc.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/esdc.jl b/src/esdc.jl index fb55aed..a154b53 100644 --- a/src/esdc.jl +++ b/src/esdc.jl @@ -33,11 +33,11 @@ and `store` or pick a resolution, chunking and cube region. ### Keyword arguments * `bucket=nothing` specify an OBS bucket for example "obs-esdc-v2.0.0" - * `store=""` specify the root path of the cube, for example "esdc-8d-0.25deg-184x90x90-2.0.0.zarr" + * `store=nothing` specify the root path of the cube, for example "esdc-8d-0.25deg-184x90x90-2.0.0.zarr" * `res="low"` pick a datacube resolution (`"low"` or `"high"` for v2 or `"low"` or `"tiny"` for v3) * `chunks="ts"` choose a chunking (`"ts"` for time series access or `"map"` for spatial analyses) * `region="global"` choose a datacube (either `"global"` or `"Colombia"`), works only for esdc v2 - * `version=3` + * `version=nothing` """ function esdd(; bucket = nothing, store = nothing, res = "low", chunks = "ts", region = "global", version = nothing) @@ -84,10 +84,11 @@ and `store` or pick a resolution, chunking and cube region. ### Keyword arguments * `bucket=nothing` specify an OBS bucket for example "obs-esdc-v2.0.0" - * `store=""` specify the root path of the cube, for example "esdc-8d-0.25deg-184x90x90-2.0.0.zarr" + * `store=nothing` specify the root path of the cube, for example "esdc-8d-0.25deg-184x90x90-2.0.0.zarr" * `res="low"` pick a datacube resolution (`"low"` or `"high"`) * `chunks="ts"` choose a chunking (`"ts"` for time series access or `"map"` for spatial analyses) * `region="global"` choose a datacube (either `"global"` or `"Colombia"`) + * `version=nothing` """ esdc(;kwargs...) = Cube(esdd(;kwargs...),target_type = Union{Float32,Missing})