diff --git a/Project.toml b/Project.toml index 461f93f..5bdf8b1 100644 --- a/Project.toml +++ b/Project.toml @@ -9,6 +9,7 @@ EncodedArrays = "97725368-735e-11e9-0dd7-6bf34c3b13a8" HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" LegendDataTypes = "99e09c13-5545-5ee2-bfa2-77f358fb75d8" RadiationDetectorSignals = "bf2c0563-65cf-5db2-a620-ceb7de82658c" +Requires = "ae029012-a4dd-5104-9daa-d747884805df" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" @@ -27,9 +28,10 @@ ArraysOfArrays = "0.5, 0.6" ElasticArrays = "1" EncodedArrays = "0.4" HDF5 = "0.15, 0.16, 0.17" -LegendDataManagement = "0.2.8" +LegendDataManagement = "0.2" LegendDataTypes = "0.1" RadiationDetectorSignals = "0.3" +Requires = "0.5, 1" StaticArrays = "0.12, 1.0" StatsBase = "0.32, 0.33, 0.34" StructArrays = "0.4, 0.5, 0.6" diff --git a/ext/LegendHDF5IOLegendDataManagementExt.jl b/ext/LegendHDF5IOLegendDataManagementExt.jl index 70a58c9..4998147 100644 --- a/ext/LegendHDF5IOLegendDataManagementExt.jl +++ b/ext/LegendHDF5IOLegendDataManagementExt.jl @@ -45,35 +45,31 @@ LegendHDF5IO.LH5Array(ds::LegendHDF5IO.HDF5.Dataset, ::Type{<:T} T(s) end -function LegendHDF5IO.LH5Array(ds::LegendHDF5IO.HDF5.Dataset, ::Type{<:AbstractArray{<:T, N}} - ) where {T <: DataSelector, N} +function LegendHDF5IO.LH5Array(ds::LegendHDF5IO.HDF5.Dataset, + ::Type{<:AbstractArray{<:T, N}}) where {T <: DataSelector, N} s = read(ds) T.(s) end function __init__() - LegendHDF5IO._datatype_dict["expsetup"] = ExpSetup - LegendHDF5IO._datatype_dict["datatier"] = DataTier - LegendHDF5IO._datatype_dict["datapartition"] = DataPartition - LegendHDF5IO._datatype_dict["dataperiod"] = DataPeriod - LegendHDF5IO._datatype_dict["datarun"] = DataRun - LegendHDF5IO._datatype_dict["datacategory"] = DataCategory - LegendHDF5IO._datatype_dict["timestamp"] = Timestamp - LegendHDF5IO._datatype_dict["filekey"] = FileKey - LegendHDF5IO._datatype_dict["channelid"] = ChannelId - LegendHDF5IO._datatype_dict["detectorid"] = DetectorId + function extend_datatype_dict(::Type{T}, key::String + ) where {T <: DataSelector} + + LegendHDF5IO._datatype_dict[key] = T + dataselector_bytypes[T] = key + end - dataselector_bytypes[ExpSetup] = "expsetup" - dataselector_bytypes[DataTier] = "datatier" - dataselector_bytypes[DataPartition] = "datapartition" - dataselector_bytypes[DataPeriod] = "dataperiod" - dataselector_bytypes[DataRun] = "datarun" - dataselector_bytypes[DataCategory] = "datacategory" - dataselector_bytypes[Timestamp] = "timestamp" - dataselector_bytypes[FileKey] = "filekey" - dataselector_bytypes[ChannelId] = "channelid" - dataselector_bytypes[DetectorId] = "detectorid" + (@isdefined ExpSetup) && extend_datatype_dict(ExpSetup, "expsetup") + (@isdefined DataTier) && extend_datatype_dict(DataTier, "datatier") + (@isdefined DataRun) && extend_datatype_dict(DataRun, "datarun") + (@isdefined DataPeriod) && extend_datatype_dict(DataPeriod, "dataperiod") + (@isdefined DataCategory) && extend_datatype_dict(DataCategory, "datacategory") + (@isdefined Timestamp) && extend_datatype_dict(Timestamp, "timestamp") + (@isdefined FileKey) && extend_datatype_dict(FileKey, "filekey") + (@isdefined ChannelId) && extend_datatype_dict(ChannelId, "channelid") + (@isdefined DetectorId) && extend_datatype_dict(DetectorId, "detectorid") + (@isdefined DataPartition) && extend_datatype_dict(DataPartition, "datapartition") end end \ No newline at end of file diff --git a/src/LegendHDF5IO.jl b/src/LegendHDF5IO.jl index 25d63d9..8365cf0 100644 --- a/src/LegendHDF5IO.jl +++ b/src/LegendHDF5IO.jl @@ -31,11 +31,20 @@ include("types.jl") const _datatype_dict = Dict{String,Type}() +@static if !isdefined(Base, :get_extension) + using Requires +end function __init__() _datatype_dict[datatype_to_string(EventType)] = EventType _datatype_dict["table{t0,dt,values}"] = Vector{<:RDWaveform} _datatype_dict["struct{binning,weights,isdensity}"] = Histogram + + @static if !isdefined(Base, :get_extension) + @require LegendDataManagement = "9feedd95-f0e0-423f-a8dc-de0970eae6b3" begin + include("../ext/LegendHDF5IOLegendDataManagementExt.jl") + end + end end end # module diff --git a/src/types.jl b/src/types.jl index 8ce3342..c0b536b 100644 --- a/src/types.jl +++ b/src/types.jl @@ -417,7 +417,7 @@ Base.show(io::IO, lh::LHDataStore) = show(io, MIME"text/plain"(), lh) function Base.setindex!(lh::LHDataStore, v, i::AbstractString) create_entry(lh, i, v, usechunks=lh.usechunks) - return v + nothing end Base.setindex!(lh::LHDataStore, v, i::Any...) = @@ -686,6 +686,7 @@ function add_entries!(lhd::LHDataStore, i::AbstractString, HDF5.rename_attribute(lhd.data_store[i], "datatype", "datatype_old") HDF5.delete_attribute(lhd.data_store[i], "datatype_old") setdatatype!(lhd.data_store[i], typeof(tbl)) + nothing end """ @@ -704,6 +705,7 @@ function add_entries!(lhd::LHDataStore, i::AbstractString, src::NamedTuple, HDF5.rename_attribute(lhd.data_store[i], "datatype", "datatype_old") HDF5.delete_attribute(lhd.data_store[i], "datatype_old") setdatatype!(lhd.data_store[i], typeof(new_nt)) + nothing end export add_entries! @@ -736,6 +738,7 @@ function _delete_entry(lhd::LHDataStore, nt::NamedTuple, setdatatype!(lhd.data_store[parent], typeof(new_nt)) end HDF5.delete_object(lhd.data_store["$(parent)/$(child)"]) + nothing end function _delete_entry(lhd::LHDataStore, tbl::Table, parent::AbstractString, @@ -747,4 +750,5 @@ function _delete_entry(lhd::LHDataStore, tbl::Table, parent::AbstractString, HDF5.rename_attribute(lhd.data_store[parent], "datatype", "datatype_old") HDF5.delete_attribute(lhd.data_store[parent], "datatype_old") setdatatype!(lhd.data_store[parent], typeof(new_tbl)) + nothing end \ No newline at end of file diff --git a/test/arrays/arrays_io.jl b/test/arrays/arrays_io.jl deleted file mode 100644 index f58bb48..0000000 --- a/test/arrays/arrays_io.jl +++ /dev/null @@ -1,26 +0,0 @@ -using Unitful -using LegendHDF5IO: readdata, writedata, lh5open -using HDF5 - -@testset "Arrays I/O" begin - - a = rand(typeof(1.0u"keV"), 40) - h5open("array_test.h5", "w") do h5f - writedata(h5f, "array_in_keV", a) - writedata(h5f, "array_no_units", ustrip.(a)) - end - - @testset "readdata" begin - h5open("array_test.h5", "r") do h5f - @test readdata(h5f, "array_in_keV") == a - @test readdata(h5f, "array_no_units") == ustrip.(a) - end - end - - @testset "LHDataStore" begin - lh5open("array_test.h5") do h5f - @test h5f["array_in_keV"][:] == a - @test h5f["array_no_units"][:] == ustrip.(a) - end - end -end diff --git a/test/runtests.jl b/test/runtests.jl index cb586aa..72e75b5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,8 +2,7 @@ using Test -Test.@testset "Package LegendHDF5IO" begin - include("arrays/arrays_io.jl") +Test.@testset verbose=true "Package LegendHDF5IO" begin include("ranges/range_to_namedtuple.jl") include("histograms/histogram_io.jl") include("test_wrappers.jl") diff --git a/test/test_wrappers.jl b/test/test_wrappers.jl index fd27c6f..2407b0b 100644 --- a/test/test_wrappers.jl +++ b/test/test_wrappers.jl @@ -7,153 +7,184 @@ using StatsBase using EncodedArrays using LegendDataManagement -@testset "test wrapper" begin - @testset "reading and writing" begin - codec = VarlenDiffArrayCodec() - v = rand() - w = v * u"W" - z = rand(Bool) - s = "Test String" - fk = FileKey("l200-p03-r006-cal-20221226T200846Z") - filekey = fill(fk, 50) - h = fit(Histogram, (rand(10), rand(10)), (0:0.2:1, Float64[0, 0.5, 1])) - data3 = (v=v, w=w, z=z, s=s, h=h, i=filekey, j=fk) - boolarray = rand(Bool, 50) - x = rand(50) - x_enc = rand(-5:5, 50) |> codec - vofvec = VectorOfVectors([rand(-5:5, rand(1:100)) for _ in 1:50]) - vofsimvec = VectorOfSimilarVectors([rand(-5:5, 100) for _ in 1:50]) - vofsimvec_enc = broadcast(|>, vofsimvec, codec) - vofvec_enc = broadcast(|>, vofvec, codec) - y = x*u"J" - vofv1 = VectorOfVectors(fill(x, 50)) - vofv2 = VectorOfVectors(fill(y, 50)) - aofa1 = nestedview(rand(UInt16, 50, 50)) - aofa2 = nestedview(rand(UInt16, 50, 50)*u"m") - trng = range(0.0u"μs", 10.0u"μs"; length=50) - waveform = ArrayOfRDWaveforms((fill(trng, 50), aofa2)) - waveform2 = ArrayOfRDWaveforms((fill(trng, 50), vofv1)) - tbl = Table( - ( - a=x, - b=y, - c=vofv1, - d=vofv2, - e=aofa1, - f=waveform, - g=waveform2, - h=boolarray, - i=x_enc, - j=vofvec_enc, - k=vofsimvec_enc)) - nt = (data1=aofa2, data2=tbl, data3=data3) +@testset verbose=true "test wrapper" begin + @testset verbose=true "reading and writing" begin mktempdir(pwd()) do tmp path = joinpath(tmp, "tmp.lh5") - lh5open(path, "cw") do f - f["tmp"] = nt - end - # now check if datatypes and values are equal to the original - # data, that was written to tmp.lh5 - lh5open(path) do f - NT = f["tmp"] - @test keys(NT) == keys(nt) - @test NT.data1 == nt.data1 - @testset "test first named tuple" begin - for k=keys(data3) @test nt.data3[k] == NT.data3[k] end - end - for (col1, col2) in zip(columns(NT.data2), columns(nt.data2)) - if isa(col1, ArrayOfRDWaveforms) - @testset "check if RDWaveforms are equal" begin - @test col1[:].signal == col2.signal - @test col1[:].time == col2.time - end - else - @test col1 == col2 - end + lh5open(path, "cw") do lhd + @testset "IO of real value" begin + w = rand() + @test setindex!(lhd, w, "w") |> isnothing + @test lhd["w"] == w end - end - end - end - @testset "test append functionality" begin - x = rand(50) - y = x*u"J" - vofv1 = VectorOfVectors(collect(eachcol(rand(55, 50)))) - vofv2 = VectorOfVectors(collect(eachcol(rand(55, 50)*u"m"))) - aofa1 = nestedview(rand(UInt16, 55, 50)) - aofa2 = nestedview(rand(UInt16, 55, 50)*u"m") - trng = range(0.0u"μs", 10.0u"μs"; length=55) - waveform = ArrayOfRDWaveforms((fill(trng, 50), aofa2)) - tbl = Table((a=x, b=y, c=vofv1, d=vofv2, e=aofa1, f=waveform)) - nt = (data1=aofa2, data2=tbl) - - mktempdir(pwd()) do tmp - path = joinpath(tmp, "tmp.lh5") - lh5open(path, "cw"; usechunks=true) do f - f["tmp"] = nt - keys(f) - end - - # first append data to in-memory-data - nt2 = (data1=vcat(aofa2, aofa2), data2=vcat(tbl, tbl)) - - # append data to on-disk-data - lh5open(path, "cw") do f - append!(f["tmp/data1"], aofa2) - append!(f["tmp/data2"], tbl) - end - - # now check if datatypes and values are equal to the data - # that was extended in memory - lh5open(path) do f - NT = f["tmp"] - @test keys(NT) == keys(nt2) - - @test haskey(f, "tmp") - @test haskey(f, "tmp/data1") - @test haskey(f, "tmp/data2") - @test !haskey(f, "tmp/data3") - @test !haskey(f, "data1") - @test !haskey(f, "data2") - - @test NT.data1[:] == nt2.data1 - for (col1, col2) in zip(columns(NT.data2), columns(nt2.data2)) - if isa(col1, ArrayOfRDWaveforms) - @testset "check if RDWaveforms are equal" begin - @test col1.signal == col2.signal - @test col1.time == col2.time + @testset "IO of real quantity" begin + v = rand()*u"keV" + @test setindex!(lhd, v, "v") |> isnothing + @test lhd["v"] == v + end + @testset "IO of Bool value" begin + z = rand(Bool) + @test setindex!(lhd, z, "z") |> isnothing + @test lhd["z"] == z + end + @testset "IO of String" begin + s = "Test String" + @test setindex!(lhd, s, "s") |> isnothing + @test lhd["s"] == s + end + @testset "IO of Array{<:Real}" begin + x = rand(10, 10, 10) + @test setindex!(lhd, x, "x") |> isnothing + @test lhd["x"][:, :, :] == x + end + @testset "IO of Array{<:Bool}" begin + boolarray = rand(Bool, 10) + @test setindex!(lhd, boolarray, "boolarray") |> isnothing + @test lhd["boolarray"][:] == boolarray + end + @testset "IO of Array{<:Quantity}" begin + y = rand(10)*u"mm" + @test setindex!(lhd, y, "y") |> isnothing + @test lhd["y"][:] == y + end + @testset "IO of VectorOfVectors" begin + vofvec = VectorOfVectors( + [rand(-5:5, rand(1:100)) for _ in 1:50]) + @test setindex!(lhd, vofvec, "vofvec") |> isnothing + @test lhd["vofvec"][:] == vofvec + end + @testset "IO of VectorOfSimilarVectors" begin + vofsimvec = VectorOfSimilarVectors( + [rand(-5:5, 100) for _ in 1:50]) + @test setindex!(lhd, vofsimvec, "vofsimvec") |> isnothing + @test lhd["vofsimvec"][:] == vofsimvec + end + @testset "IO of NamedTuple" begin + nt = (a=10, b=10.0u"mm") + @test setindex!(lhd, nt, "nt") |> isnothing + @test lhd["nt"] == nt + end + @testset "IO of Histogram" begin + h = fit( + Histogram, + (rand(10), rand(10)), (0:0.2:1, Float64[0, 0.5, 1])) + @test setindex!(lhd, h, "h") |> isnothing + @test lhd["h"] == h + end + @testset "IO of EncodedArray" begin + codec = VarlenDiffArrayCodec() + x_enc = rand(-5:5, 50) |> codec + @test setindex!(lhd, x_enc, "x_enc") |> isnothing + @test lhd["x_enc"] == x_enc + end + @testset "IO of VectorOfEncodedArrays" begin + codec = VarlenDiffArrayCodec() + data = [rand(-5:5, rand(1:100)) for _ in 1:50] + vofvec = VectorOfVectors(data) + vofvec_enc = broadcast(|>, vofvec, codec) + @test setindex!(lhd, vofvec_enc, "vofvec_enc") |> isnothing + @test lhd["vofvec_enc"] == vofvec_enc + end + @testset "IO of VectorOfEncodedSimilarArrays" begin + codec = VarlenDiffArrayCodec() + data = [rand(-5:5, 100) for _ in 1:50] + vofsimvec = VectorOfSimilarVectors(data) + vofsimvec_enc = broadcast(|>, vofsimvec, codec) + @test setindex!(lhd, vofsimvec_enc, "vofsimvec_enc") |> isnothing + @test lhd["vofsimvec_enc"] == vofsimvec_enc + end + @testset "IO of ArrayOfRDWaveforms" begin + data = nestedview(rand(UInt16, 50, 50)*u"m") + trng = range(0.0u"μs", 10.0u"μs"; length=50) + waveform = ArrayOfRDWaveforms((fill(trng, 50), data)) + @test setindex!(lhd, waveform, "waveform") |> isnothing + @test lhd["waveform"][:] == waveform + end + @testset "IO of Table" begin + tbl = Table(a=rand(10), b=rand(10)) + @test setindex!(lhd, tbl, "tbl") |> isnothing + @test lhd["tbl"][:] == tbl + end + if @isdefined FileKey + @testset "IO of FileKey" begin + fk = FileKey("l200-p03-r006-cal-20221226T200846Z") + @test setindex!(lhd, fk, "fk") |> isnothing + @test lhd["fk"] == fk end - else - @test col1 == col2 + @testset "IO of Array{FileKey}" begin + fk = FileKey("l200-p03-r006-cal-20221226T200846Z") + filekey = fill(fk, 50) + @test setindex!(lhd, filekey, "filekey") |> isnothing + @test lhd["filekey"] == filekey end end end end end - @testset "test adding and removing columns from Tables" begin - x, y = rand(10), rand(10) - tbl = Table(x=x, y=y) + @testset verbose=true "test append functionality" begin mktempdir(pwd()) do tmp path = joinpath(tmp, "tmp.lh5") - lh5open(path, "cw") do lhd - lhd["tbl"] = tbl - delete_entry!(lhd, "tbl/y") - @test lhd["tbl"][:] == Table(tbl, y=nothing) - add_entries!(lhd, "tbl", Table(y=y)) - @test lhd["tbl"][:] == tbl + lh5open(path, "cw"; usechunks=true) do lhd + @testset "append Tables" begin + tbl = Table(x=rand(10), y=rand(10)) + newtbl = vcat(tbl, tbl) + lhd["tbl"] = tbl + @test append!(lhd["tbl"], tbl)[:] == newtbl + end + @testset "append VectorOfVectors" begin + data = collect(eachcol(rand(55, 50)*u"m")) + vofv = VectorOfVectors(data) + newvofv = vcat(vofv, vofv) + lhd["vofv"] = vofv + @test append!(lhd["vofv"], vofv)[:] == newvofv + end + @testset "append VectorOfSimilarVectors" begin + aofa = nestedview(rand(UInt16, 55, 50)*u"m") + newaofa = vcat(aofa, aofa) + lhd["aofa"] = aofa + @test append!(lhd["aofa"], aofa)[:] == newaofa + end + @testset "append VectorOfRDWaveforms" begin + aofa = nestedview(rand(UInt16, 55, 50)*u"m") + trng = range(0.0u"μs", 10.0u"μs"; length=55) + waveform = ArrayOfRDWaveforms((fill(trng, 50), aofa)) + new_waveform = vcat(waveform, waveform) + lhd["waveform"] = waveform + @test append!(lhd["waveform"], waveform)[:] == new_waveform + end end end end - @testset "test adding and removing columns from NamedTuples" begin - x, y = rand(10), rand(10) - nt = (x=x, y=y) + @testset verbose=true "test adding and removing entries" begin mktempdir(pwd()) do tmp path = joinpath(tmp, "tmp.lh5") lh5open(path, "cw") do lhd - lhd["nt"] = nt - delete_entry!(lhd, "nt/y") - @test lhd["nt"] == (x=x,) - add_entries!(lhd, "nt", (y=y,)) - @test lhd["nt"] == nt + @testset verbose=true "Tables" begin + x, y = rand(10), rand(10)*u"mm" + tbl = Table(x=x, y=y) + lhd["tbl"] = tbl + @testset verbose=true "deleting entry" begin + @test delete_entry!(lhd, "tbl/y") |> isnothing + @test lhd["tbl"][:] == Table(tbl, y=nothing) + end + @testset "adding entry" begin + @test add_entries!(lhd, "tbl", Table(y=y)) |> isnothing + @test lhd["tbl"][:] == tbl + end + end + @testset verbose=true "NamedTuples" begin + x, y = 1.0, 1.0 + nt = (x=x, y=y) + lhd["nt"] = nt + @testset "deleting entry" begin + @test delete_entry!(lhd, "nt/y") |> isnothing + @test lhd["nt"] == (x=x,) + end + @testset "add entry" begin + @test add_entries!(lhd, "nt", (y=y,)) |> isnothing + @test lhd["nt"] == nt + end + end end end end