Merge pull request #439 from JuliaDataCubes/la/outdims

La/outdims

.gitignore

@@ -29,4 +29,5 @@ docs/node_modules
 docs/.vitepress/cache
 docs/.vitepress/dist
 docs/.DS_Store
-docs/package-lock.json
+docs/package-lock.json
+*.tif

Project.toml

@@ -36,7 +36,7 @@ CFTime = "0.0, 0.1"
 DataStructures = "0.17, 0.18"
 DimensionalData = "0.27, 0.28"
 DiskArrayTools = "0.1"
-DiskArrays = "0.3,0.4"
+DiskArrays = "0.3, 0.4"
 DocStringExtensions = "0.8, 0.9"
 Glob = "1.3"
 Interpolations = "0.12, 0.13, 0.14, 0.15"

docs/src/UserGuide/faq.md

@@ -313,6 +313,12 @@ nothing # hide
 ds = YAXArrays.Dataset(; (keylist .=> varlist)...)
 ````
 
+::: warning
+
+You will not be able to save this dataset, first you will need to rename those `dimensions` with the `same name` but different values.
+
+:::
+
 ## Ho do I construct a `Dataset` from a TimeArray
 
 In this section we will use `MarketData.jl` and `TimeSeries.jl` to simulate some stocks.

docs/src/UserGuide/read.md

@@ -54,5 +54,4 @@ using Downloads: download
 
 path = download("https://github.com/yeesian/ArchGDALDatasets/raw/307f8f0e584a39a050c042849004e6a2bd674f99/gdalworkshop/world.tif", "world.tif")
 ds = open_dataset(path)
-nothing
 ````

src/DAT/DAT.jl

@@ -796,14 +796,19 @@ function getbackend(oc, ispar, max_cache)
     outsize =
         sizeof(elementtype) * (length(oc.allAxes) > 0 ? prod(map(length, oc.allAxes)) : 1)
     rt = oc.desc.backend
-    if rt == :auto
-        if ispar[] || outsize > max_cache
-            rt = :zarr
+    ispath =  get(oc.desc.backendargs, :path, nothing)
+
+    b = if rt == :auto && !isnothing(ispath)
+            YAXArrayBase.backendfrompath(ispath)
+        elseif rt == :auto 
+            if ispar[] || outsize > max_cache
+                YAXArrayBase.backendlist[:zarr]
+            else
+                YAXArrayBase.backendlist[:array]
+            end
         else
-            rt = :array
-        end
+            YAXArrayBase.backendlist[Symbol(rt)]  # Handle non-auto rt case
     end
-    b = YAXArrayBase.backendlist[Symbol(rt)]
     if !allow_parallel_write(b)
         ispar[] = false
     end
@@ -849,7 +854,10 @@ function generateOutCube(
     newsize = map(length, oc.allAxes)
     outar = Array{elementtype}(undef, newsize...)
     fill!(outar,_zero(elementtype))
-    oc.cube = YAXArray(tuple(oc.allAxes...), outar)
+    # @show oc.desc.backendargs[:properties]
+    properties = get(oc.desc.backendargs, :properties, Dict{String, Any}())
+    # @show properties
+    oc.cube = YAXArray(tuple(oc.allAxes...), outar, properties) #  properties ? include properties!
     oc.cube_unpermuted = oc.cube
 end
 _zero(T) = zero(T)

src/DatasetAPI/Datasets.jl

@@ -678,7 +678,7 @@ function savecube(
     if chunks !== nothing
         error("Setting chunks in savecube is not supported anymore. Rechunk using `setchunks` before saving. ")
     end
-
+    
     ds = to_dataset(c; layername, datasetaxis)
     ds = savedataset(ds; path, max_cache, driver, overwrite, append,skeleton, writefac, kwargs...)
     Cube(ds, joinname = datasetaxis)
@@ -718,7 +718,7 @@ function createdataset(
     properties = Dict{String,Any}(),
     globalproperties = Dict{String,Any}(),
     datasetaxis = "Variable",
-    layername = "layer",
+    layername = get(properties, "name", "layer"),
     kwargs...,
 )
     if persist === nothing

test/Datasets/datasets.jl

@@ -410,6 +410,76 @@ end
 
 end
 
+@testset "Saving, OutDims" begin
+    using YAXArrays, Zarr, NetCDF, ArchGDAL
+    using Dates
+
+    flolat(lo, la, t) = (lo + la + Dates.dayofyear(t))
+    flola(lo, la) = lo + la + 1
+
+    function g(xout, lo, la, t)
+        xout .= flolat.(lo, la, t)
+    end
+
+    function g2d(xout, lo, la)
+        xout .= flola.(lo, la)
+    end
+
+    lon = YAXArray(Dim{:lon}(range(1, 15)))
+    lat = YAXArray(Dim{:lat}(range(1, 10)))
+    tspan = Date("2022-01-01"):Day(1):Date("2022-01-30")
+    time = YAXArray(Dim{:time}(tspan))
+
+    properties = Dict{String, Any}("name" => "out_array")
+
+    gen_cube = mapCube(g, (lon, lat, time);
+            indims = (InDims(), InDims(), InDims("time")),
+            outdims = OutDims("time"; properties,
+            outtype = Float32)
+            # max_cache=1e9
+        )
+
+    gen_cube2d = mapCube(g2d, (lon, lat);
+        indims = (InDims(), InDims()),
+        outdims = OutDims(; outtype = Float32)
+        # max_cache=1e9
+    )
+    properties = Dict{String, Any}("name" => "out_zarr")
+    # test saves, zarr
+    mapCube(g, (lon, lat, time);
+            indims = (InDims(), InDims(), InDims("time")),
+            outdims = OutDims("time"; overwrite=true, path="my_gen_cube.zarr",
+            properties,
+            outtype = Float32)
+            # max_cache=1e9
+        )
+    ds_zarr = open_dataset("my_gen_cube.zarr")
+    # test saves, nc
+    properties = Dict{String, Any}("name" => "out_nc")
+    mapCube(g, (lon, lat, time);
+            indims = (InDims(), InDims(), InDims("time")),
+            outdims = OutDims("time"; overwrite=true, path="my_gen_cube.nc",
+            properties,
+            outtype = Float32)
+            # max_cache=1e9
+        )
+    mapCube(g, (lon, lat, time);
+        indims = (InDims(), InDims(), InDims("time")),
+        outdims = OutDims("time"; overwrite=true, path="my_gen_cube_no_p.nc",
+        outtype = Float32)
+        # max_cache=1e9
+    )
+    ds_nc = open_dataset("my_gen_cube.nc")
+    ds_nc_no_p = open_dataset("my_gen_cube_no_p.nc")
+
+    # TODO: fix tif for general inputs, so that writing also works.
+
+    @test gen_cube.properties["name"] == "out_array"
+    @test gen_cube.data[:,:,:] == ds_zarr["out_zarr"].data[:,:,:]
+    @test gen_cube.data[:,:,:] == ds_nc["out_nc"].data[:,:,:]
+    @test gen_cube.data[:,:,:] == ds_nc_no_p["layer"].data[:,:,:]
+end
+
 @testset "Caching" begin
     using YAXArrays.Cubes.DiskArrays.TestTypes
     using YAXArrays.Cubes: DiskArrays

test/Project.toml

@@ -1,4 +1,5 @@
 [deps]
+ArchGDAL = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3"
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"