From 83304243bf19d42e956fb3fb39c5b881719936d7 Mon Sep 17 00:00:00 2001 From: Rafael Schouten Date: Sun, 1 Sep 2024 18:01:06 +0200 Subject: [PATCH] Breaking: Broadcast performance and correctness (#777) * Improve GPU functionality and fix performnce combined commit * fix broadcast type stability * Improve GPU functionality * Add missing weakdeps * Update src/array/broadcast.jl Co-authored-by: Rafael Schouten * Update src/array/broadcast.jl Co-authored-by: Rafael Schouten * Push materialize fix * Clean up mapreduce and add a bunch of tests for JLArray broadcast * Add some more JLArray tests * Just return dest in broadcast * Update src/array/methods.jl Co-authored-by: Rafael Schouten * Format --------- Co-authored-by: Rafael Schouten * bugfixes * more bugfix * skip broken plot * use GPUArrays.allowscalar(false) in tests * add GPUArrays to test targets * use generated function for comparedims and promotedims * fix _remove * fix promotedims * test and bugfix promotedims * bugfix * remove @d --------- Co-authored-by: Paul Tiede --- Project.toml | 6 +- src/Dimensions/Dimensions.jl | 3 +- src/Dimensions/dimension.jl | 5 + src/Dimensions/format.jl | 2 + src/Dimensions/primitives.jl | 301 +++++++++++++++++++++-------------- src/Dimensions/set.jl | 1 + src/Dimensions/show.jl | 4 +- src/Lookups/Lookups.jl | 11 +- src/Lookups/lookup_arrays.jl | 149 ++++++++++++++--- src/Lookups/show.jl | 3 +- src/Lookups/utils.jl | 5 + src/array/broadcast.jl | 45 +++--- src/array/indexing.jl | 7 +- src/array/methods.jl | 38 ++--- src/set.jl | 2 + test/array.jl | 33 ++-- test/broadcast.jl | 209 +++++++++++++++++++++--- test/indexing.jl | 20 +-- test/matmul.jl | 4 +- test/methods.jl | 238 +++++++++++++++++++++++++-- test/plotrecipes.jl | 8 +- test/primitives.jl | 76 ++++++--- test/stack.jl | 4 +- 23 files changed, 876 insertions(+), 298 deletions(-) diff --git a/Project.toml b/Project.toml index 4a8e71869..11f46b0ee 100644 --- a/Project.toml +++ b/Project.toml @@ -48,12 +48,14 @@ Dates = "1" Distributions = "0.25" Documenter = "1" Extents = "0.1" +GPUArrays = "10" ImageFiltering = "0.7" ImageTransformations = "0.10" Interfaces = "0.3" IntervalSets = "0.5, 0.6, 0.7" InvertedIndices = "1" IteratorInterfaceExtensions = "1" +JLArrays = "0.1" LinearAlgebra = "1" Makie = "0.19, 0.20, 0.21" OffsetArrays = "1" @@ -83,8 +85,10 @@ CoordinateTransformations = "150eb455-5306-5404-9cee-2592286d6298" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +GPUArrays = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" ImageFiltering = "6a3955dd-da59-5b1f-98d4-e7296123deb5" ImageTransformations = "02fcd773-0e25-5acc-982a-7f6622650795" +JLArrays = "27aeb0d3-9eb9-45fb-866b-73c2ecf80fcb" Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" @@ -95,4 +99,4 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" [targets] -test = ["Aqua", "ArrayInterface", "BenchmarkTools", "CategoricalArrays", "ColorTypes", "Combinatorics", "CoordinateTransformations", "DataFrames", "Distributions", "Documenter", "ImageFiltering", "ImageTransformations", "CairoMakie", "OffsetArrays", "Plots", "Random", "SafeTestsets", "StatsPlots", "Test", "Unitful"] +test = ["Aqua", "ArrayInterface", "BenchmarkTools", "CategoricalArrays", "ColorTypes", "Combinatorics", "CoordinateTransformations", "DataFrames", "Distributions", "Documenter", "GPUArrays", "ImageFiltering", "ImageTransformations", "JLArrays", "CairoMakie", "OffsetArrays", "Plots", "Random", "SafeTestsets", "StatsPlots", "Test", "Unitful"] diff --git a/src/Dimensions/Dimensions.jl b/src/Dimensions/Dimensions.jl index 480d79e0d..aec431d40 100644 --- a/src/Dimensions/Dimensions.jl +++ b/src/Dimensions/Dimensions.jl @@ -25,7 +25,8 @@ const LookupArrays = Lookups import .Lookups: rebuild, order, span, sampling, locus, val, index, set, _set, metadata, bounds, intervalbounds, units, basetypeof, unwrap, selectindices, hasselection, - shiftlocus, maybeshiftlocus + shiftlocus, maybeshiftlocus, ordered_first, ordered_last, ordered_firstindex, ordered_lastindex, + promote_first, _remove using .Lookups: StandardIndices, SelTuple, CategoricalEltypes, LookupTrait, AllMetadata, LookupSetters, AbstractBeginEndRange, SelectorOrInterval, Interval diff --git a/src/Dimensions/dimension.jl b/src/Dimensions/dimension.jl index a103b274d..b1c6ca973 100644 --- a/src/Dimensions/dimension.jl +++ b/src/Dimensions/dimension.jl @@ -296,6 +296,11 @@ function Base.:(==)(d1::Dimension, d2::Dimension) basetypeof(d1) == basetypeof(d2) && val(d1) == val(d2) end +LookupArrays.ordered_first(d::Dimension{<:AbstractArray}) = ordered_first(lookup(d)) +LookupArrays.ordered_last(d::Dimension{<:AbstractArray}) = ordered_last(lookup(d)) +LookupArrays.ordered_firstindex(d::Dimension{<:AbstractArray}) = ordered_firstindex(lookup(d)) +LookupArrays.ordered_lastindex(d::Dimension{<:AbstractArray}) = ordered_lastindex(lookup(d)) + Base.size(dims::DimTuple) = map(length, dims) Base.CartesianIndices(dims::DimTuple) = CartesianIndices(map(d -> axes(d, 1), dims)) diff --git a/src/Dimensions/format.jl b/src/Dimensions/format.jl index 6b5a0b6c5..1616b4ce3 100644 --- a/src/Dimensions/format.jl +++ b/src/Dimensions/format.jl @@ -38,6 +38,7 @@ format(dims::Tuple{Vararg{Any,N}}, A::AbstractArray{<:Any,N}) where N = format(d format(dims::Tuple{Vararg{Any,N}}, axes::Tuple{Vararg{Any,N}}) where N = map(_format, dims, axes) format(d::Dimension{<:AbstractArray}) = _format(d, axes(val(d), 1)) format(d::Dimension, axis::AbstractRange) = _format(d, axis) +format(l::Lookup) = parent(format(AnonDim(l))) _format(dimname::Symbol, axis::AbstractRange) = Dim{dimname}(NoLookup(axes(axis, 1))) _format(::Type{D}, axis::AbstractRange) where D<:Dimension = D(NoLookup(axes(axis, 1))) @@ -55,6 +56,7 @@ format(v, D::Type, axis::AbstractRange) = _valformaterror(v, D) # Format Lookups # No more identification required for NoLookup +format(m::Lookups.Length1NoLookup, D::Type, values, axis::AbstractRange) = m format(m::NoLookup, D::Type, values, axis::AbstractRange) = m format(m::NoLookup, D::Type, values::AutoValues, axis::AbstractRange) = NoLookup(axis) # # AutoLookup diff --git a/src/Dimensions/primitives.jl b/src/Dimensions/primitives.jl index 5fede080f..bb936f9d2 100644 --- a/src/Dimensions/primitives.jl +++ b/src/Dimensions/primitives.jl @@ -79,18 +79,18 @@ function sortdims end _asfunc(::Type{typeof(<:)}) = <: _asfunc(::Type{typeof(>:)}) = >: -@inline function _sortdims(f, tosort, order::Tuple{<:Integer,Vararg}) +@inline function _sortdims(f, tosort::Tuple, order::Tuple{<:Integer,Vararg}) map(order) do i - if i in 1:length(tosort) + if i in 1:length(tosort) tosort[i] else nothing end end end -@inline _sortdims(f, tosort, order) = _sortdims_gen(f, tosort, order) +@inline _sortdims(f, tosort::Tuple, order::Tuple) = _sortdims_gen(f, tosort, order) -@generated _sortdims_gen(f, tosort::Tuple, order::Tuple) = begin +@generated function _sortdims_gen(f, tosort::Tuple, order::Tuple) expr = Expr(:tuple) allreadyfound = Int[] for (i, ord) in enumerate(order.parameters) @@ -149,8 +149,8 @@ function dims end @inline dims(a1, args...) = _dim_query(_dims, MaybeFirst(), a1, args...) @inline dims(::Tuple{}, ::Tuple{}) = () -@inline _dims(f, dims, query) = _remove_nothing(_sortdims(f, dims, query)) -@inline _dims(f, dims, query...) = _remove_nothing(_sortdims(f, dims, query)) +@inline _dims(f, dims, query) = _remove(Nothing, _sortdims(f, dims, query)...) +@inline _dims(f, dims, query...) = _remove(Nothing, _sortdims(f, dims, query)...) """ commondims([f], x, query) => Tuple{Vararg{Dimension}} @@ -305,7 +305,7 @@ end (_dimifnothing(f, first(ds), first(query))..., _otherdims_from_nothing(f, tail(ds), tail(query))...) @inline _otherdims_from_nothing(f, ::Tuple{}, ::Tuple{}) = () -@inline _dimifnothing(f, dim, query) = () +@inline _dimifnothing(f, dim, query) = () @inline _dimifnothing(f, dim, query::Nothing) = (dim,) @@ -499,8 +499,6 @@ function reducedims end const DimTupleOrEmpty = Union{DimTuple,Tuple{}} -struct _Throw end - """ comparedims(A::AbstractDimArray...; kw...) comparedims(A::Tuple...; kw...) @@ -529,99 +527,179 @@ These are all `Bool` flags: - `ignore_length_one`: ignore length `1` in comparisons, and return whichever dimension is not length 1, if any. This is useful in e.g. broadcasting comparisons. `false` by default. -- `warn`: a `String` or `nothing`. Used only for `Bool` methods, - to give a warning for `false` values and include `warn` in the warning text. +- `msg`: DimensionalData.Warn or DimensionalData.Throw. Both may contain string, + which will be added to error or warning mesages. """ function comparedims end +@inline comparedims(args...; kw...)::Bool = + _comparedims(args...; msg=Throw(), kw...) +@inline comparedims(::Type{Bool}, args...; kw...)::Bool = + _comparedims(args...; msg=nothing, kw...) -@inline comparedims(args...; kw...) = _comparedims(_Throw, args...; kw...) -@inline comparedims(T::Type, args...; kw...) = _comparedims(T, args...; kw...) - -@inline _comparedims(T::Type, d1::Dimension, ds::Dimension...; kw...) = - map(d -> _comparedims(T, d1, d; kw...), (d1, ds...)) -@inline _comparedims(T::Type, A::Tuple; kw...) = _comparedims(T, map(dims, A)...; kw...) -@inline _comparedims(T::Type, A...; kw...) = _comparedims(T, map(dims, A)...; kw...) -@inline _comparedims(T::Type, dims::Vararg{Tuple{Vararg{Dimension}}}; kw...) = - map(d -> _comparedims(T, first(dims), d; kw...), dims) - -@inline _comparedims(T::Type{_Throw}; kw...) = () -@inline _comparedims(T::Type{_Throw}, a::DimTuple, b::DimTuple; kw...) = - (_comparedims(T, first(a), first(b); kw...), _comparedims(T, tail(a), tail(b); kw...)...) -@inline _comparedims(::Type{_Throw}, a::DimTupleOrEmpty, ::Nothing; kw...) = a -@inline _comparedims(::Type{_Throw}, ::Nothing, b::DimTupleOrEmpty; kw...) = b -@inline _comparedims(::Type{_Throw}, a::DimTuple, b::Tuple{}; kw...) = a -@inline _comparedims(::Type{_Throw}, a::Tuple{}, b::DimTuple; kw...) = b -@inline _comparedims(::Type{_Throw}, a::Tuple{}, b::Tuple{}; kw...) = () -@inline _comparedims(::Type{_Throw}, ::Nothing, ::Nothing; kw...) = nothing -@inline _comparedims(::Type{_Throw}, a::AnonDim, b::AnonDim; kw...) = nothing -@inline _comparedims(::Type{_Throw}, a::Dimension, b::AnonDim; kw...) = a -@inline _comparedims(::Type{_Throw}, a::AnonDim, b::Dimension; kw...) = b -@inline function _comparedims(::Type{_Throw}, a::Dimension, b::Dimension; - type=true, valtype=false, val=false, length=true, order=false, ignore_length_one=false, -) - type && basetypeof(a) != basetypeof(b) && _dimsmismatcherror(a, b) - valtype && typeof(parent(a)) != typeof(parent(b)) && _valtypeerror(a, b) - val && parent(lookup(a)) != parent(lookup(b)) && _valerror(a, b) - if order - (isnolookup(a) || isnolookup(b) || LU.order(a) == LU.order(b)) || _ordererror(a, b) - end - if ignore_length_one && (Base.length(a) == 1 || Base.length(b) == 1) - return Base.length(b) == 1 ? a : b +abstract type AbstractMessage{M} end + +msg(t::AbstractMessage) = t.msg +msg(::AbstractMessage{Nothing}) = "" + +Base.string(m::AbstractMessage) = msg(m) + +struct Warn{M<:Union{AbstractString,Nothing}} <: AbstractMessage{M} + msg::M +end +Warn() = Warn(nothing) + +struct Throw{M<:Union{AbstractString,Nothing}} <: AbstractMessage{M} + msg::M +end +Throw() = Throw(nothing) + +_dimsmismatchmsg(a, b) = "$(basetypeof(a)) and $(basetypeof(b)) dims on the same axis." +_valmsg(a, b) = "Lookup values for $(basetypeof(a)) of $(parent(a)) and $(parent(b)) do not match." +_dimsizemsg(a, b) = "Found both lengths $(length(a)) and $(length(b)) for $(basetypeof(a))." +_valtypemsg(a, b) = "Lookup for $(basetypeof(a)) of $(lookup(a)) and $(lookup(b)) do not match." +_ordermsg(a, b) = "Lookups do not all have the same order: $(order(a)), $(order(b))." + +@noinline _dimsmismatchaction(err, a, b) = _failed_comparedims(err, _dimsmismatchmsg(a, b)) +@noinline _valaction(err, a, b) = _failed_comparedims(err, _valmsg(a, b)) +@noinline _dimsizeaction(err, a, b) = _failed_comparedims(err, _dimsizemsg(a, b)) +@noinline _valtypeaction(err, a, b) = _failed_comparedims(err, _valtypemsg(a, b)) +@noinline _orderaction(err, a, b) = _failed_comparedims(err, _ordermsg(a, b)) + +_failed_comparedims(w::Warn, msg_intro) = @warn string(msg_intro, msg(w)) +_failed_comparedims(t::Throw, msg_intro) = throw(DimensionMismatch(string(msg_intro, msg(t)))) + +@inline _comparedims(; kw...) = true +@inline _comparedims(xs...; kw...) = _comparedims(map(dims, xs)...; kw...) +@inline _comparedims(dt1::DimTupleOrEmpty; kw...) = true +@inline _comparedims(dt1::DimTupleOrEmpty, dts::DimTupleOrEmpty...; kw...) = + _comparedims_gen(dt1, dts; kw...) +@generated function _comparedims_gen(dt1::Tuple, dts::Tuple; kw...) + n = length(dts.parameters) + exprs = map(1:n) do i + :(_comparedims2(dt1, dts[$i]; kw...)) end - length && Base.length(a) != Base.length(b) && _dimsizeerror(a, b) - return a + return Expr(:call, :all, Expr(:tuple, exprs...)) end -@inline _comparedims(T::Type; kw...) = true -@inline _comparedims(T::Type{Bool}, d1::Dimension, ds::Dimension...; kw...) = - all(map(d -> _comparedims(T, d1, d; kw...), ds)) -@inline _comparedims(T::Type{Bool}, dims::Vararg{Tuple{Vararg{Dimension}}}; kw...) = - all(map(d -> _comparedims(T, first(dims), d; kw...), dims)) -@inline _comparedims(T::Type{Bool}, a::DimTuple, b::DimTuple; kw...) = - all((_comparedims(T, first(a), first(b); kw...), _comparedims(T, tail(a), tail(b); kw...)...)) -@inline _comparedims(T::Type{Bool}, a::DimTupleOrEmpty, ::Nothing; kw...) = true -@inline _comparedims(T::Type{Bool}, ::Nothing, b::DimTupleOrEmpty; kw...) = true -@inline _comparedims(T::Type{Bool}, ::Nothing, ::Nothing; kw...) = true -@inline _comparedims(T::Type{Bool}, a::DimTuple, b::Tuple{}; kw...) = true -@inline _comparedims(T::Type{Bool}, a::Tuple{}, b::DimTuple; kw...) = true -@inline _comparedims(T::Type{Bool}, a::Tuple{}, b::Tuple{}; kw...) = true -@inline _comparedims(T::Type{Bool}, a::AnonDim, b::AnonDim; kw...) = true -@inline _comparedims(T::Type{Bool}, a::Dimension, b::AnonDim; kw...) = true -@inline _comparedims(T::Type{Bool}, a::AnonDim, b::Dimension; kw...) = true -@inline function _comparedims(::Type{Bool}, a::Dimension, b::Dimension; - type=true, valtype=false, val=false, length=true, order=false, ignore_length_one=false, - warn::Union{Nothing,String}=nothing, +@inline _comparedims(d1::Union{Dimension,Nothing}, d2::Union{Dimension,Nothing}; kw...) = + _comparedims2(d1, d2; kw...) +@inline _comparedims(d1::Union{Dimension,Nothing}, d2::Union{Dimension,Nothing}, ds::Vararg{Union{Dimension,Nothing}}; kw...) = + all((_comparedims2(d1, d2; kw...), _comparedims(d1, ds...; kw...)...)) +@inline _comparedims(d1::Dimension; kw...) = true +@inline _comparedims(d1::Nothing; kw...) = true +@inline _comparedims(dt::Tuple; kw...) = true + +@inline _comparedims2(as::Tuple{<:Dimension}, bs::Tuple{<:Dimension}; kw...) = + _comparedims2(as[1], bs[1]; kw...) +@inline _comparedims2((a1, as...)::DimTuple, (b1, bs...)::DimTuple; kw...) = + _comparedims2(a1, b1; kw...) && _comparedims2(as, bs; kw...) +@inline _comparedims2(::Nothing, b::DimTupleOrEmpty; kw...) = true +@inline _comparedims2(::Nothing, ::Nothing; kw...) = true +@inline _comparedims2(a::DimTuple, b::Tuple{}; kw...) = true +@inline _comparedims2(a::Tuple{}, b::DimTuple; kw...) = true +@inline _comparedims2(a::Tuple{}, b::Tuple{}; kw...) = true +@inline _comparedims2(a::AnonDim, b::AnonDim; kw...) = true +@inline _comparedims2(a::Dimension, b::AnonDim; kw...) = true +@inline _comparedims2(a::AnonDim, b::Dimension; kw...) = true +@inline function _comparedims2(a::Dimension, b::Dimension; + type=true, valtype=false, val=false, length=true, order=false, + ignore_length_one=false, msg ) if type && basetypeof(a) != basetypeof(b) - isnothing(warn) || _dimsmismatchwarn(a, b, warn) + isnothing(msg) || _dimsmismatchaction(msg, a, b) return false end - if valtype && typeof(parent(a)) != typeof(parent(b)) - isnothing(warn) || _valtypewarn(a, b, warn) - return false + if ignore_length_one && (Base.length(a) == 1 || Base.length(b) == 1) + return true end - if val && parent(lookup(a)) != parent(lookup(b)) - isnothing(warn) || _valwarn(a, b, warn) + if order && !(isnolookup(a) || isnolookup(b) || Lookups.order(a) == Lookups.order(b)) + isnothing(msg) || _orderaction(msg, a, b) return false end - if order && !(isnolookup(a) || isnolookup(b) || LU.order(a) == LU.order(b)) - isnothing(warn) || _orderwarn(a, b, warn) + if valtype && typeof(parent(a)) != typeof(parent(b)) + isnothing(msg) || _valtypeaction(msg, a, b) return false end - if ignore_length_one && (Base.length(a) == 1 || Base.length(b) == 1) - return true + pa, pb = parent(lookup(a)), parent(lookup(b)) + if val && !(isnolookup(a) || isnolookup(b)) && pa != pb + if eltype(pa) <: Number && eltype(pb) <: Number + if !all(((a, b),) -> a ≈ b, zip(pa, pb)) + isnothing(msg) || _valaction(msg, a, b) + return false + end + else + isnothing(msg) || _valaction(msg, a, b) + return false + end end if length && Base.length(a) != Base.length(b) - isnothing(warn) || _dimsizewarn(a, b, warn) + isnothing(msg) || _dimsizeaction(msg, a, b) return false end return true end """ - combinedims(xs; check=true) + promotedims(dts::DimTuple...; skip_length_one=false) + promotedims(dts::Dimensions...; skip_length_one=false) + +Promote the types of dimensions and contained lookups so that +the returned type is always the same independent of order, but +that the lookup values come from the first argument. + +# Keywords + +- `skip_length_one`: the returned type will remain the same, + but the values from the first lookup with length larger than + one will be used. +""" + +@inline function promotedims(d1::Dimension, ds::Dimension...; skip_length_one=false) + vs = map(val, ds) + v = promote_first(val(d1), vs...) + promoted_v = if skip_length_one + _promote_non_length_one(v, v, vs...) + else + v + end + + return rebuild(d1, promoted_v) +end +promotedims(dts::DimTupleOrEmpty...; kw...) = _promotedims_gen(dts; kw...) + +# Hard to get this stable without @generated +@generated function _promotedims_gen(dts::Tuple; kw...) + maxlen = maximum(Tuple(dts.parameters)) do p + length(p.parameters) + end + exprs = map(1:maxlen) do j + expr = Expr(:call, :promotedims, Expr(:parameters, Expr(:..., :kw))) + for (i, p) in enumerate(dts.parameters) + if length(p.parameters) >= j + push!(expr.args, :(dts[$i][$j])) + end + end + expr + end + return Expr(:tuple, exprs...) +end + +@inline function _promote_non_length_one(template, v1, vs...) + if length(v1) > 1 + promote_first(v1, template) + else + _promote_non_length_one(template, vs...) + end +end +@inline _promote_non_length_one(template, v1) = promote_first(v1, template) + + +""" + combinedims(xs; check=true, kw...) Combine the dimensions of each object in `xs`, in the order they are found. + +Keywords are passed to [`comparedims`](@ref). """ function combinedims end function combinedims(xs::Vector; kw...) @@ -660,10 +738,12 @@ See [`basetypeof`](@ref) """ function basedims end @inline basedims(x) = basedims(dims(x)) +@inline basedims(::Nothing) = nothing @inline basedims(ds::Tuple) = map(basedims, ds) @inline basedims(d::Dimension) = basetypeof(d)() @inline basedims(d::Symbol) = name2dim(d) @inline basedims(T::Type{<:Dimension}) = basetypeof(T)() +@inline basedims(x, y) = basedims(dims(x, dims(y))) @inline pairs2dims(pairs::Pair...) = map(p -> basetypeof(name2dim(first(p)))(last(p)), pairs) @@ -690,48 +770,46 @@ struct AlwaysTuple <: QueryMode end # Call the function f with standardised args # This looks like HELL, but it removes this complexity # from every other method and makes sure they all behave the same way. -@inline _dim_query(f::Function, t::QueryMode, args...) = +@inline _dim_query(f::F, t::QueryMode, args...) where F<:Function = _dim_query(f, <:, t::QueryMode, args...) -@inline _dim_query(f::Function, t::QueryMode, op::Union{typeof(<:),typeof(>:)}, args...) = +@inline _dim_query(f::F, t::QueryMode, op::O, args...) where {F<:Function,O<:Union{typeof(<:),typeof(>:)}} = _dim_query(f, op, t::QueryMode, args...) -@inline _dim_query(f::Function, t::QueryMode, op::Union{typeof(<:),typeof(>:)}, a1, args::Tuple) = +@inline _dim_query(f::F, t::QueryMode, op::O, a1, args::Tuple) where {F<:Function,O<:Union{typeof(<:),typeof(>:)}} = _dim_query(f, op, t::QueryMode, a1, args...) -@inline _dim_query(f::Function, op::Function, t::QueryMode, a1, args...) = +@inline _dim_query(f::F, op::O, t::QueryMode, a1, args...) where {F<:Function,O<:Union{typeof(<:),typeof(>:)}} = _dim_query1(f, op, t, _wraparg(a1, args...)...) -@inline _dim_query(f::Function, op::Function, t::QueryMode, a1, args::Tuple) = +@inline _dim_query(f::F, op::O, t::QueryMode, a1, args::Tuple) where {F<:Function,O<:Union{typeof(<:),typeof(>:)}} = _dim_query1(f, op, t::QueryMode, _wraparg(a1)..., _wraparg(args...)) -@inline _dim_query1(f, op::Function, t, x, l1, l2, ls...) = _dim_query1(f, op, t, x, (l1, l2, ls...)) -@inline _dim_query1(f, op::Function, t, x) = _dim_query1(f, op, t, dims(x)) -@inline _dim_query1(f, op::Function, t, x, query) = _dim_query1(f, op, t, dims(x), query) -@inline _dim_query1(f, op::Function, t, x::Nothing) = _dimsnotdefinederror() -@inline _dim_query1(f, op::Function, t, x::Nothing, query) = _dimsnotdefinederror() -@inline _dim_query1(f, op::Function, t, ds::Tuple, query::Colon) = +@inline _dim_query1(f::F, op::O, t, x, l1, l2, ls...) where {F,O} = + _dim_query1(f, op, t, x, (l1, l2, ls...)) +@inline _dim_query1(f::F, op::O, t, x) where {F,O} = + _dim_query1(f, op, t, dims(x)) +@inline _dim_query1(f::F, op::O, t, x, query::Q) where {F,O,Q} = + _dim_query1(f, op, t, dims(x), query) +@inline _dim_query1(f::F, op::O, t, x::Nothing) where {F,O} = + _dimsnotdefinederror() +@inline _dim_query1(f::F, op::O, t, x::Nothing, query::Q) where {F,O,Q} = + _dimsnotdefinederror() +@inline _dim_query1(f::F, op::O, t, ds::Tuple, query::Colon) where {F,O} = _dim_query1(f, op, t, ds, basedims(ds)) -@inline function _dim_query1(f, op::Function, t, ds::Tuple, query::Function) +@inline function _dim_query1(f::F, op::O, t, ds::Tuple, query::Q) where {F,O,Q<:Function} selection = foldl(ds; init=()) do acc, d query(d) ? (acc..., d) : acc end _dim_query1(f, op, t, ds, selection) end -@inline function _dim_query1(f, op::Function, t, d::Tuple, query) +@inline function _dim_query1(f::F, op::O, t, d::Tuple, query::Q) where {F,O,Q} ds = dims(query) isnothing(ds) && _dims_are_not_dims() _dim_query1(f, op, t, d, ds) end -@inline _dim_query1(f, op::Function, t::QueryMode, d::Tuple, query::Union{Dimension,DimType,Val,Integer}) = +@inline _dim_query1(f::F, op::O, t::QueryMode, d::Tuple, query::Union{Dimension,DimType,Val,Integer}) where {F,O} = _dim_query1(f, op, t, d, (query,)) |> t -@inline _dim_query1(f, op::Function, ::QueryMode, d::Tuple, query::Tuple) = map(unwrap, f(op, d, query)) -@inline _dim_query1(f, op::Function, ::QueryMode, d::Tuple) = map(unwrap, f(op, d)) - - -# Utils - -# Remove `nothing` from a `Tuple` -@inline _remove_nothing(xs::Tuple) = _remove_nothing(xs...) -@inline _remove_nothing(x, xs...) = (x, _remove_nothing(xs...)...) -@inline _remove_nothing(::Nothing, xs...) = _remove_nothing(xs...) -@inline _remove_nothing() = () +@inline _dim_query1(f::F, op::O, ::QueryMode, d::Tuple, query::Tuple) where {F,O} = + map(unwrap, f(op, d, query)) +@inline _dim_query1(f::F, op::O, ::QueryMode, d::Tuple) where {F,O} = + map(unwrap, f(op, d)) # This looks ridiculous, but gives seven arguments with constant-propagation, # which means type stability using Symbols/types instead of objects. @@ -759,30 +837,15 @@ _astuple(x) = (x,) # Warnings and Error methods. -_dimsmismatchmsg(a, b) = "$(basetypeof(a)) and $(basetypeof(b)) dims on the same axis." -_valmsg(a, b) = "Lookup values for $(basetypeof(a)) of $(parent(a)) and $(parent(b)) do not match." -_dimsizemsg(a, b) = "Found both lengths $(length(a)) and $(length(b)) for $(basetypeof(a))." -_valtypemsg(a, b) = "Lookup for $(basetypeof(a)) of $(lookup(a)) and $(lookup(b)) do not match." -_extradimsmsg(extradims) = "$(map(basetypeof, extradims)) dims were not found in object." _extradimsmsg(::Tuple{}) = "Some dims were not found in object." +_extradimsmsg(extradims) = "$(map(basetypeof, extradims)) dims were not found in object." _metadatamsg(a, b) = "Metadata $(metadata(a)) and $(metadata(b)) do not match." -_ordermsg(a, b) = "Lookups do not all have the same order: $(order(a)), $(order(b))." _typemsg(a, b) = "Lookups do not all have the same type: $(order(a)), $(order(b))." -# Warning: @noinline to avoid allocations when it isn't used -@noinline _dimsmismatchwarn(a, b, msg="") = @warn string(_dimsmismatchmsg(a, b), msg) -@noinline _valwarn(a, b, msg="") = @warn string(_valmsg(a, b), msg) -@noinline _dimsizewarn(a, b, msg="") = @warn string(_dimsizemsg(a, b), msg) -@noinline _valtypewarn(a, b, msg="") = @warn string(_valtypemsg(a, b), msg) +# Warn @noinline _extradimswarn(dims, msg="") = @warn string(_extradimsmsg(dims), msg) -@noinline _orderwarn(a, b, msg="") = @warn string(_ordermsg(a, b), msg) # Error -@noinline _dimsmismatcherror(a, b) = throw(DimensionMismatch(_dimsmismatchmsg(a, b))) -@noinline _dimsizeerror(a, b) = throw(DimensionMismatch(_dimsizemsg(a, b))) -@noinline _valtypeerror(a, b) = throw(DimensionMismatch(_valtypemsg(a, b))) -@noinline _valerror(a, b) = throw(DimensionMismatch(_valmsg(a, b))) -@noinline _ordererror(a, b) = throw(DimensionMismatch(_ordermsg(a, b))) @noinline _metadataerror(a, b) = throw(DimensionMismatch(_metadatamsg(a, b))) @noinline _extradimserror(args) = throw(ArgumentError(_extradimsmsg(args))) @noinline _dimsnotdefinederror() = throw(ArgumentError("Object does not define a `dims` method")) diff --git a/src/Dimensions/set.jl b/src/Dimensions/set.jl index f3852c140..2b3b78913 100644 --- a/src/Dimensions/set.jl +++ b/src/Dimensions/set.jl @@ -4,6 +4,7 @@ set(dim::Dimension, ::Type{T}) where T = set(dim, T()) set(dims::DimTuple, ::Type{T}) where T = set(dims, T()) set(dim::Dimension, x::DimSetters) = _set(dim, x) set(dims_::DimTuple, args::Union{Dimension,DimTuple,Pair}...; kw...) = + _set(dims_, args...; kw...) set(dims::DimTuple, l::Lookup) = set(dims, map(d -> basedims(d) => l, dims)...) set(dims::DimTuple, l::LookupTrait) = set(dims, map(d -> basedims(d) => l, dims)...) diff --git a/src/Dimensions/show.jl b/src/Dimensions/show.jl index 7d485e6a9..a05e8735f 100644 --- a/src/Dimensions/show.jl +++ b/src/Dimensions/show.jl @@ -21,7 +21,7 @@ function show_dims(io::IO, mime::MIME"text/plain", dims::DimTuple; brackets = get(io, :dim_brackets, true) print(io, inset) brackets && print(io, '(') - if all(map(d -> !(parent(d) isa AbstractArray) || (parent(d) isa NoLookup), dims)) + if all(map(d -> !(parent(d) isa AbstractArray) || (parent(d) isa AbstractNoLookup), dims)) dc = colors[1] printstyled(io, dimsymbols(1), ' '; color=dc) show(IOContext(ctx, :dimcolor => dc, :dimname_len => 0), mime, first(dims)) @@ -120,7 +120,7 @@ end function print_dimval(io, mime, lookup::AbstractArray, nchars=0) Lookups.print_index(io, mime, lookup, nchars) end -print_dimval(io, mime, lookup::Union{AutoLookup,NoLookup}, nchars=0) = print(io, "") +print_dimval(io, mime, lookup::Union{AutoLookup,AbstractNoLookup}, nchars=0) = print(io, "") function print_dimval(io, mime, lookup::Lookup, nchars=0) print(io, " ") ctx = IOContext(io, :nchars=>nchars) diff --git a/src/Lookups/Lookups.jl b/src/Lookups/Lookups.jl index 4fee63197..e5f718208 100644 --- a/src/Lookups/Lookups.jl +++ b/src/Lookups/Lookups.jl @@ -25,9 +25,12 @@ import InvertedIndices using InvertedIndices: Not using Base: tail, OneTo, @propagate_inbounds -export order, sampling, span, bounds, hasselection, dim, - metadata, units, sort, selectindices, val, reducelookup, - locus, shiftlocus, maybeshiftlocus, intervalbounds +export order, sampling, span, bounds, dim, + metadata, units, sort, val, locus, intervalbounds + +export hasselection, selectindices + +export reducelookup, shiftlocus, maybeshiftlocus, promote_first # Deprecated export index @@ -49,7 +52,7 @@ export Metadata, NoMetadata export AutoStep, AutoBounds, AutoValues export Lookup -export AutoLookup, NoLookup +export AutoLookup, AbstractNoLookup, NoLookup export Aligned, AbstractSampled, Sampled, AbstractCyclic, Cyclic, AbstractCategorical, Categorical export Unaligned, Transformed diff --git a/src/Lookups/lookup_arrays.jl b/src/Lookups/lookup_arrays.jl index 2507c5fae..a2b745fda 100644 --- a/src/Lookups/lookup_arrays.jl +++ b/src/Lookups/lookup_arrays.jl @@ -23,7 +23,6 @@ locus(l::Lookup) = Center() # Deprecated index(l::Lookup) = parent(l) -@deprecate locus locus Base.eltype(l::Lookup{T}) where T = T Base.parent(l::Lookup) = l.data @@ -37,17 +36,20 @@ function Base.:(==)(l1::Lookup, l2::Lookup) basetypeof(l1) == basetypeof(l2) && parent(l1) == parent(l2) end -ordered_first(l::Lookup) = l[ordered_firstindex(l)] -ordered_last(l::Lookup) = l[ordered_lastindex(l)] +ordered_first(l::AbstractArray) = l[ordered_firstindex(l)] +ordered_last(l::AbstractArray) = l[ordered_lastindex(l)] +ordered_firstindex(l::AbstractArray) = firstindex(l) ordered_firstindex(l::Lookup) = ordered_firstindex(order(l), l) +ordered_firstindex(::ForwardOrdered, l::Lookup) = firstindex(parent(l)) +ordered_firstindex(::ReverseOrdered, l::Lookup) = lastindex(parent(l)) +ordered_firstindex(::Unordered, l::Lookup) = firstindex(parent(l)) + +ordered_lastindex(l::AbstractArray) = lastindex(l) ordered_lastindex(l::Lookup) = ordered_lastindex(order(l), l) -ordered_firstindex(o::ForwardOrdered, l::Lookup) = firstindex(parent(l)) -ordered_firstindex(o::ReverseOrdered, l::Lookup) = lastindex(parent(l)) -ordered_firstindex(o::Unordered, l::Lookup) = firstindex(parent(l)) -ordered_lastindex(o::ForwardOrdered, l::Lookup) = lastindex(parent(l)) -ordered_lastindex(o::ReverseOrdered, l::Lookup) = firstindex(parent(l)) -ordered_lastindex(o::Unordered, l::Lookup) = lastindex(parent(l)) +ordered_lastindex(::ForwardOrdered, l::Lookup) = lastindex(parent(l)) +ordered_lastindex(::ReverseOrdered, l::Lookup) = firstindex(parent(l)) +ordered_lastindex(::Unordered, l::Lookup) = lastindex(parent(l)) function Base.searchsortedfirst(lookup::Lookup, val; lt=<, kw...) searchsortedfirst(parent(lookup), unwrap(val); order=ordering(order(lookup)), lt=lt, kw...) @@ -105,6 +107,14 @@ abstract type Aligned{T,O} <: Lookup{T,1} end order(lookup::Aligned) = lookup.order + +abstract type AbstractNoLookup <: Aligned{Int,Order} end + +order(::AbstractNoLookup) = ForwardOrdered() +span(::AbstractNoLookup) = Regular(1) + +Base.step(lookup::AbstractNoLookup) = 1 + """ NoLookup <: Lookup @@ -141,17 +151,19 @@ Dimensions.lookup(A) NoLookup, NoLookup ``` """ -struct NoLookup{A<:AbstractVector{Int}} <: Aligned{Int,Order} +struct NoLookup{A<:AbstractVector{Int}} <: AbstractNoLookup data::A end NoLookup() = NoLookup(AutoValues()) -order(lookup::NoLookup) = ForwardOrdered() -span(lookup::NoLookup) = Regular(1) - rebuild(l::NoLookup; data=parent(l), kw...) = NoLookup(data) -Base.step(lookup::NoLookup) = 1 +# Used in @d broadcasts +struct Length1NoLookup <: AbstractNoLookup end +Length1NoLookup(::AbstractVector) = Length1NoLookup() + +rebuild(l::Length1NoLookup; kw...) = Length1NoLookup() +Base.parent(::Length1NoLookup) = Base.OneTo(1) """ AbstractSampled <: Aligned @@ -686,11 +698,8 @@ function _slicespan(span::Irregular, l::Lookup, i::InvertedIndices.InvertedIndex i1 = collect(i) # We could do something more efficient here, but I'm not sure what _slicespan(sampling(l), span, l, i1) end -function _slicespan(::Points, span::Irregular, l::Lookup, i::AbstractArray) - length(i) == 0 && return Irregular(nothing, nothing) - fi, la = first(i), last(i) - return Irregular(_maybeflipbounds(l, (l[fi], l[la]))) -end +_slicespan(::Points, span::Irregular, l::Lookup, i::AbstractArray) = + Irregular(nothing, nothing) _slicespan(::Intervals, span::Irregular, l::Lookup, i::AbstractArray) = Irregular(_slicebounds(span, l, i)) @@ -820,3 +829,105 @@ end ordering(::ForwardOrdered) = Base.Order.ForwardOrdering() ordering(::ReverseOrdered) = Base.Order.ReverseOrdering() + + +# Promotion + +# General case +promote_first(x) = x +promote_first(x1, x2, xs...) = + convert(promote_type(typeof(x1), typeof(x2), map(typeof, xs)...), x1) +# Fallback NoLookup if not identical type +promote_first(l1::Lookup) = l1 +promote_first(l1::L, ls::L...) where L<:Lookup = rebuild(l1; metadata=NoMetadata) +function promote_first(l1::L, ls::Lookup...) where {L<:Lookup} + ls = _remove(Length1NoLookup, l1, ls...) + if length(ls) > 1 + l1, ls... = ls + else + return first(ls) + end + if all(map(l -> typeof(l) == L, ls)) + if length(ls) > 0 + rebuild(l1; metadata=NoMetadata()) + else + l1 # Keep metadata if there is only one lookup + end + else + NoLookup(Base.OneTo(length(l1))) + end +end +# Categorical lookups +promote_first(l1::AbstractCategorical) = l1 +promote_first(l1::C, ls::C...) where C<:AbstractCategorical = l1 +promote_first(l1::C, ::C, ::C...) where C<:AbstractCategorical = rebuild(l1; metadata=NoMetadata()) +function promote_first(l1::AbstractCategorical, l2::AbstractCategorical, ls::AbstractCategorical...) + ls = (l2, ls...) + all(map(l -> order(l) == order(l1), ls)) || return NoLookup(Base.OneTo(length(l1))) + data = promote_first(parent(l1), map(parent, ls)...) + if all(map(l -> basetypeof(l) == basetypeof(l1), ls)) + return rebuild(l1; data, metadata=NoMetadata()) + else # Fall back to standard Categorical + return Categorical(data; order=order(l1), metadata=NoMetadata()) + end +end +promote_first(l1::AbstractSampled) = l1 +promote_first(l1::S, ::S, ::S...) where S<:AbstractSampled = l1 +function promote_first(l1::AbstractSampled, l2::AbstractSampled, ls::AbstractSampled...) + ls = (l2, ls...) + all(map(l -> order(l) == order(l1), ls)) && + all(map(l -> typeof(sampling(l)) == typeof(sampling(l1)), ls)) && + all(map(l -> basetypeof(span(l)) == basetypeof(span(l1)), ls)) || + return NoLookup(Base.OneTo(length(l1))) + + data = promote_first(parent(l1), map(parent, ls)...) + kw = (; + order=order(l1), + span=promote_first(span(l1), map(span, ls)...), + sampling=sampling(l1), + metadata=NoMetadata(), + ) + if all(map(l -> basetypeof(l) == basetypeof(l1), ls)) + return rebuild(l1; data, kw...) + else + return Sampled(data; kw...) + end +end +# Span +function promote_first(s::Regular, ss::Regular...) + T = promote_type(typeof(val(s)), map(typeof ∘ val, ss)...) + Regular(convert(T, val(s))) +end +promote_first(a::T, b::T...) where T<:Irregular = a +for E in (Base.Number, Dates.AbstractTime) + @eval function promote_first( + s::Irregular{Tuple{<:$E,<:$E}}, ss::Irregular{Tuple{<:$E,<:$E}}... + ) + T = promote_type(maps(s -> promote_type(typeof(val(s)[1]), typeof(val(s)[2])), (s, ss...))...) + return Irregular(convert(T, val(a)[1]), convert(T, val(a)[2])) + end +end +promote_first(::Irregular, ::Irregular...) = Irregular((nothing, nothing)) +# Data +promote_first(a1::A) where A<:AbstractArray = a1 +promote_first(a1::A, ::A, ::A...) where A<:AbstractArray = a1 +promote_first(a1::AbstractArray{<:AbstractString}, as::AbstractArray{<:AbstractString}...) = String.(a1) +function promote_first(a1::AbstractArray, as::AbstractArray...) + T = promote_type(eltype(a1), map(eltype, as)...) + C = if a1 isa AbstractRange && all(map(a -> a isa AbstractRange, as)) + if a1 isa AbstractUnitRange && all(map(a -> a isa AbstractUnitRange, as)) + UnitRange + elseif a1 isa OrdinalRange && all(map(a -> a isa OrdinalRange, as)) + S = promote_type(typeof(step(a1)), map(typeof ∘ step, as)...) + StepRange{T,S} + elseif a1 isa LinRange || any(map(a -> a isa LinRange, as)) + LinRange{T} + else + StepRangeLen{T} + end + else + Vector{T} + end + + return convert(C, a1) +end diff --git a/src/Lookups/show.jl b/src/Lookups/show.jl index 39f81142d..1e35bc876 100644 --- a/src/Lookups/show.jl +++ b/src/Lookups/show.jl @@ -2,6 +2,7 @@ Base.show(io::IO, mime::MIME"text/plain", lookup::AutoLookup) = nothing Base.show(io::IO, mime::MIME"text/plain", lookup::NoLookup) = print(io, "NoLookup") +Base.show(io::IO, mime::MIME"text/plain", lookup::Length1NoLookup) = print(io, "Length1NoLookup") function Base.show(io::IO, mime::MIME"text/plain", lookup::Transformed) show_compact(io, mime, lookup) @@ -40,7 +41,7 @@ end function Base.show(io::IO, mime::MIME"text/plain", lookups::Tuple{Lookup,Vararg{Lookup}}) length(lookups) > 0 || return 0 ctx = IOContext(io, :compact=>true) - if all(l -> l isa NoLookup, lookups) + if all(l -> l isa AbstractNoLookup, lookups) for l in lookups[begin:end-1] show(ctx, mime, l) print(io, ", ") diff --git a/src/Lookups/utils.jl b/src/Lookups/utils.jl index e9825bbda..93e54fe9b 100644 --- a/src/Lookups/utils.jl +++ b/src/Lookups/utils.jl @@ -113,3 +113,8 @@ _order(A::AbstractArray{<:IntervalSets.Interval}) = first(A).left <= last(A).lef @deprecate maybeshiftlocus maybeshiftlocus @deprecate shiftlocus shiftlocus + +# Remove objects of type T from a +Base.@assume_effects :foldable _remove(::Type{T}, x, xs...) where T = (x, _remove(T, xs...)...) +Base.@assume_effects :foldable _remove(::Type{T}, ::T, xs...) where T = _remove(T, xs...) +Base.@assume_effects :foldable _remove(::Type) = () diff --git a/src/array/broadcast.jl b/src/array/broadcast.jl index de6dec328..556326335 100644 --- a/src/array/broadcast.jl +++ b/src/array/broadcast.jl @@ -34,39 +34,37 @@ BroadcastStyle(a::Style{Tuple}, ::DimensionalStyle{B}) where {B} = DimensionalSt # We need to implement copy because if the wrapper array type does not # support setindex then the `similar` based default method will not work function Broadcast.copy(bc::Broadcasted{DimensionalStyle{S}}) where S - _dims = _broadcasted_dims(bc) + bdims = _broadcasted_dims(bc) + comparedims(bdims...; ignore_length_one=true, order=true, val=true, msg=Dimensions.Throw()) + dims = Dimensions.promotedims(bdims...; skip_length_one=true) A = _firstdimarray(bc) data = copy(_unwrap_broadcasted(bc)) - return if A isa Nothing || _dims isa Nothing || ndims(A) == 0 + return if A isa Nothing || dims isa Nothing || !(data isa AbstractArray) data elseif data isa AbstractDimArray - rebuild(A, parent(data), _dims, refdims(A), Symbol("")) + rebuild(A, parent(data), format(dims, data), refdims(A), Symbol("")) else - rebuild(A, data, _dims, refdims(A), Symbol("")) + rebuild(A, data, format(dims, data), refdims(A), Symbol("")) end end function Base.copyto!(dest::AbstractArray, bc::Broadcasted{DimensionalStyle{S}}) where S - _dims = comparedims(dims(dest), _broadcasted_dims(bc); ignore_length_one=true, order=true) + comparedims(_broadcasted_dims(bc)...; ignore_length_one=true, order=true) copyto!(dest, _unwrap_broadcasted(bc)) - A = _firstdimarray(bc) - if A isa Nothing || _dims isa Nothing - dest - else - rebuild(A, dest, _dims, refdims(A)) - end + return dest end -function Base.copyto!(dest::AbstractDimArray, bc::Broadcasted{DimensionalStyle{S}}) where S - _dims = comparedims(dims(dest), _broadcasted_dims(bc); ignore_length_one=true, order=true) - copyto!(parent(dest), _unwrap_broadcasted(bc)) - A = _firstdimarray(bc) - if A isa Nothing || _dims isa Nothing - dest - else - rebuild(A, parent(dest), _dims, refdims(A)) - end + +@inline function Base.Broadcast.materialize!(dest::AbstractDimArray, bc::Base.Broadcast.Broadcasted{<:Any}) + # Need to check whether the dims are compatible in dest, + # which are already stripped when sent to copyto! + comparedims(dims(dest), _broadcasted_dims(bc)...; ignore_length_one=true, order=true) + style = DimensionalData.DimensionalStyle(Base.Broadcast.combine_styles(parent(dest), bc)) + Base.Broadcast.materialize!(style, parent(dest), bc) + return dest end + + function Base.similar(bc::Broadcast.Broadcasted{DimensionalStyle{S}}, ::Type{T}) where {S,T} A = _firstdimarray(bc) rebuildsliced(A, similar(_unwrap_broadcasted(bc), T, axes(bc)...), axes(bc), Symbol("")) @@ -99,7 +97,6 @@ _firstdimarray(x::Tuple{}) = nothing # Make sure all arrays have the same dims, and return them _broadcasted_dims(bc::Broadcasted) = _broadcasted_dims(bc.args...) -_broadcasted_dims(a, bs...) = - comparedims(_broadcasted_dims(a), _broadcasted_dims(bs...); ignore_length_one=true, order=true) -_broadcasted_dims(a::AbstractBasicDimArray) = dims(a) -_broadcasted_dims(a) = nothing +_broadcasted_dims(a, bs...) = (_broadcasted_dims(a)..., _broadcasted_dims(bs...)...) +_broadcasted_dims(a::AbstractBasicDimArray) = (dims(a),) +_broadcasted_dims(a) = () diff --git a/src/array/indexing.jl b/src/array/indexing.jl index e1d579761..762d5c4cd 100644 --- a/src/array/indexing.jl +++ b/src/array/indexing.jl @@ -256,13 +256,12 @@ Base.@assume_effects :foldable @inline _simplify_dim_indices() = () setindex!(A, x, dims2indices(A, (i, I...))...) @propagate_inbounds Base.setindex!(A::AbstractDimArray, x, I::DimensionalIndices...; kw...) = setindex!(A, x, dims2indices(A, _simplify_dim_indices(I..., kw2dims(values(kw))...))...) -@propagate_inbounds Base.setindex!(::DimensionalData.AbstractDimArray, x, ::_DimIndicesAmb, ::_DimIndicesAmb...; kw...) = - setindex!(A, x, dims2indices(A, _simplify_dim_indices(I..., kw2dims(values(kw))...))...) +@propagate_inbounds Base.setindex!(::DimensionalData.AbstractDimArray, x, ::_DimIndicesAmb, ::_DimIndicesAmb...; kw...) = setindex!(A, x, dims2indices(A, _simplify_dim_indices(I..., kw2dims(values(kw))...))...) @propagate_inbounds Base.setindex!(A::AbstractDimArray, x, i1::StandardIndices, I::StandardIndices...) = setindex!(parent(A), x, i1, I...) # For @views macro to work with keywords -Base.maybeview(A::AbstractDimArray, args...; kw...) = +@propagate_inbounds Base.maybeview(A::AbstractDimArray, args...; kw...) = view(A, args...; kw...) -Base.maybeview(A::AbstractDimArray, args::Vararg{Union{Number,Base.AbstractCartesianIndex}}; kw...) = +@propagate_inbounds Base.maybeview(A::AbstractDimArray, args::Vararg{Union{Number,Base.AbstractCartesianIndex}}; kw...) = view(A, args...; kw...) diff --git a/src/array/methods.jl b/src/array/methods.jl index b6ce99b14..9987e5e45 100644 --- a/src/array/methods.jl +++ b/src/array/methods.jl @@ -53,25 +53,9 @@ for (m, f) in ((:Statistics, :median), (:Base, :any), (:Base, :all)) end end -# These are not exported but it makes a lot of things easier using them -function Base._mapreduce_dim(f, op, nt::NamedTuple{(),<:Tuple}, A::AbstractDimArray, dims) - rebuild(A, Base._mapreduce_dim(f, op, nt, parent(A), dimnum(A, _astuple(dims))), reducedims(A, dims)) -end -function Base._mapreduce_dim(f, op, nt::NamedTuple{(),<:Tuple}, A::AbstractDimArray, dims::Colon) - Base._mapreduce_dim(f, op, nt, parent(A), dims) -end -function Base._mapreduce_dim(f, op, nt, A::AbstractDimArray, dims) - rebuild(A, Base._mapreduce_dim(f, op, nt, parent(A), dimnum(A, dims)), reducedims(A, dims)) -end -function Base._mapreduce_dim(f, op, nt, A::AbstractDimArray, dims::Colon) - rebuild(A, Base._mapreduce_dim(f, op, nt, parent(A), dimnum(A, dims)), reducedims(A, dims)) -end - -function Base._mapreduce_dim(f, op, nt::Base._InitialValue, A::AbstractDimArray, dims) - rebuild(A, Base._mapreduce_dim(f, op, nt, parent(A), dimnum(A, dims)), reducedims(A, dims)) -end -function Base._mapreduce_dim(f, op, nt::Base._InitialValue, A::AbstractDimArray, dims::Colon) - Base._mapreduce_dim(f, op, nt, parent(A), dims) +function Base.mapreduce(f, op, A::AbstractDimArray; dims=Base.Colon(), kw...) + dims === Colon() && return mapreduce(f, op, parent(A); kw...) + rebuild(A, mapreduce(f, op, parent(A); dims=dimnum(A, dims), kw...), reducedims(A, dims)) end @@ -239,7 +223,7 @@ for (pkg, fname) in [(:Base, :permutedims), (:Base, :adjoint), @inline $pkg.$fname(A::AbstractDimArray{<:Any,2}) = rebuild(A, $pkg.$fname(parent(A)), reverse(dims(A))) @inline $pkg.$fname(A::AbstractDimArray{<:Any,1}) = - rebuild(A, $pkg.$fname(parent(A)), (AnonDim(Base.OneTo(1)), dims(A)...)) + rebuild(A, $pkg.$fname(parent(A)), (AnonDim(NoLookup(Base.OneTo(1))), dims(A)...)) end end @inline function Base.permutedims(A::AbstractDimArray, perm) @@ -314,8 +298,8 @@ function _cat(catdims::Tuple, A1::AbstractDimArray, As::AbstractDimArray...) cat_dnums = (inserted_dnums..., appended_dnums...) # Warn if dims or val do not match, and cat the parent - if !comparedims(Bool, map(x -> otherdims(x, newcatdims), Xin)...; - order=true, val=true, warn=" Can't `cat` AbstractDimArray, applying to `parent` object." + if !comparedims(map(x -> otherdims(x, newcatdims), Xin)...; + order=true, val=true, msg=Dimensions.Warn(" Can't `cat` AbstractDimArray, applying to `parent` object.") ) return Base.cat(map(parent, Xin)...; dims=cat_dnums) end @@ -331,7 +315,7 @@ function Base.hcat(As::Union{AbstractDimVector,AbstractDimMatrix}...) Base.cat(As; dims=2) A1 = first(As) catdim = if A1 isa AbstractDimVector - AnonDim() + AnonDim(NoLookup()) else joindims = map(last ∘ dims, As) check_cat_lookups(joindims...) || return Base.hcat(map(parent, As)...) @@ -339,8 +323,8 @@ function Base.hcat(As::Union{AbstractDimVector,AbstractDimMatrix}...) end noncatdim = dims(A1, 1) # Make sure this is the same dimension for all arrays - if !comparedims(Bool, map(x -> dims(x, 1), As)...; - val=true, warn=" Can't `hcat` AbstractDimArray, applying to `parent` object." + if !comparedims(map(x -> dims(x, 1), As)...; + val=true, msg=Dimensions.Warn(" Can't `hcat` AbstractDimArray, applying to `parent` object.") ) return Base.hcat(map(parent, As)...) end @@ -358,8 +342,8 @@ function Base.vcat(As::Union{AbstractDimVector,AbstractDimMatrix}...) (catdim,) else # Make sure this is the same dimension for all arrays - if !comparedims(Bool, map(x -> dims(x, 2), As)...; - val=true, warn = " Can't `vcat` AbstractDimArray, applying to `parent` object." + if !comparedims(map(x -> dims(x, 2), As)...; + val=true, msg=Dimensions.Warn(" Can't `vcat` AbstractDimArray, applying to `parent` object.") ) return Base.vcat(map(parent, As)...) end diff --git a/src/set.jl b/src/set.jl index bcbe5106c..39eb37428 100644 --- a/src/set.jl +++ b/src/set.jl @@ -117,6 +117,7 @@ julia> set(da, :custom => DD.Irregular(10, 12), Z => DD.Regular(9.9)) ``` """ function set end + # Types are constructed Base.@assume_effects :effect_free set(x::DimArrayOrStack, ::Type{T}) where T = set(x, T()) @@ -136,6 +137,7 @@ Base.@assume_effects :effect_free set(A::AbstractDimArray, newdata::AbstractArra axes(A) == axes(newdata) || _axiserr(A, newdata) rebuild(A; data=newdata) end + # NamedTuples are set as data for AbstractDimStack Base.@assume_effects :effect_free set(s::AbstractDimStack, newdata::NamedTuple) = begin dat = data(s) diff --git a/test/array.jl b/test/array.jl index 100ec82e1..9da392a08 100644 --- a/test/array.jl +++ b/test/array.jl @@ -22,8 +22,8 @@ val(dims(da, 1)) |> typeof da2 = DimArray(a2, dimz2; refdims=refdimz, name=:test2) @testset "checkbounds" begin - @test checkbounds(Bool, da, X(2), Y(1)) == true - @test checkbounds(Bool, da, X(10), Y(1)) == false + @test @inferred checkbounds(Bool, da, X(2), Y(1)) == true + @test @inferred checkbounds(Bool, da, X(10), Y(1)) == false checkbounds(da, X(2), Y(1)) @test_throws BoundsError checkbounds(da, X(10), Y(20)) @test_throws BoundsError checkbounds(da, X(1:10), Y(2:20)) @@ -37,8 +37,8 @@ end end @testset "rebuild" begin - @test rebuild(da2, parent(da2)) === da2 - @test rebuild(da2; dims=dims(da2)) === da2 + @test @inferred rebuild(da2, parent(da2)) === da2 + @test @inferred rebuild(da2; dims=dims(da2)) === da2 @test_throws ArgumentError rebuild(da2; dims=dims(da2, (Y,))) === da2 end @@ -81,12 +81,13 @@ end @test bounds(da) == ((143.0, 145.0), (-38.0, -36.0)) @test layerdims(da) == (X(), Y()) @test index(da, Y) == LinRange(-38.0, -36.0, 2) - da_intervals = set(da, X => Intervals, Y => Intervals) + @test_broken @inferred set(da, X => Intervals(), Y => Intervals()) + da_intervals = set(da, X => Intervals(), Y => Intervals()) @test intervalbounds(da_intervals) == ([(142.0, 144.0), (144.0, 146.0)], [(-39.0, -37.0), (-37.0, -35.0)]) end @testset "copy and friends" begin - dac = copy(da2) + dac = @inferred copy(da2) @test dac == da2 @test dims(dac) == dims(da2) @test refdims(dac) == refdims(da2) == (Ti(1:1),) @@ -135,7 +136,7 @@ end end @testset "similar with a type" begin - da_float = similar(da, Float64) + da_float = @inferred similar(da, Float64) @test eltype(da_float) == Float64 @test size(da_float) == size(da) @test dims(da_float) === dims(da) @@ -146,7 +147,7 @@ end @testset "similar with a size" begin # Changing the axis size removes dims. # TODO we can keep dims, but with NoLookup? - da_size = similar(da2, (5, 5)) + da_size = @inferred similar(da2, (5, 5)) @test eltype(da_size) == Int @test size(da_size) == (5, 5) da_size_splat = similar(da2, 5, 5) @@ -162,7 +163,7 @@ end end @testset "similar with sparse arrays" begin - sda = DimArray(sprand(Float64, 10, 10, 0.5), (X, Y)) + sda = @inferred DimArray(sprand(Float64, 10, 10, 0.5), (X(), Y())) sparse_size_int = similar(sda, Int64, (5, 5)) @test eltype(sparse_size_int) == Int64 != eltype(sda) @test size(sparse_size_int) == (5, 5) @@ -170,16 +171,16 @@ end end @testset "similar with dims" begin - da_sim_dims = similar(da, dims(da)) - da_sim_dims_splat = similar(da, dims(da)) + da_sim_dims = @inferred similar(da, dims(da)) + da_sim_dims_splat = @inferred similar(da, dims(da)) for A in (da_sim_dims, da_sim_dims_splat) @test eltype(A) == eltype(da) @test size(A) == size(da) @test dims(A) === dims(da) @test refdims(A) == () end - da_sim_type_dims = similar(da2, Bool, dims(da)) - da_sim_type_dims_splat = similar(da2, Bool, dims(da)...) + da_sim_type_dims = @inferred similar(da2, Bool, dims(da)) + da_sim_type_dims_splat = @inferred similar(da2, Bool, dims(da)...) for A in (da_sim_type_dims, da_sim_type_dims_splat) @test eltype(A) == Bool @test size(A) == size(da) @@ -456,7 +457,7 @@ end end @testset "ones, zeros, trues, falses constructors" begin - da = zeros(X(4), Y(40.0:10.0:80.0); metadata=(a=1, b=2)) + da = @inferred zeros(X(4), Y(40.0:10.0:80.0; order=ForwardOrdered()); metadata=(a=1, b=2)) @test eltype(da) <: Float64 @test metadata(da) == (a=1, b=2) @test all(==(0), da) @@ -506,7 +507,7 @@ end end @testset "rand constructors" begin - da = rand(1:10, X(8), Y(11:20); metadata=(a=1, b=2)) + da = @inferred rand(1:10, X(8), Y(11:20); metadata=(a=1, b=2)) @test size(da) == (8, 10) @test eltype(da) <: Int @test metadata(da) == (a=1, b=2) @@ -514,7 +515,7 @@ end X(NoLookup(Base.OneTo(8))), Y(Sampled(11:20, ForwardOrdered(), Regular(1), Points(), NoMetadata())) ) - da = rand(X([:a, :b]), Y(3)) + da = @inferred rand(X([:a, :b]; order=ForwardOrdered()), Y(3)) @test size(da) == (2, 3) @test eltype(da) <: Float64 da = rand(Bool, X([:a, :b]), Y(3)) diff --git a/test/broadcast.jl b/test/broadcast.jl index fd1586bc6..facbff544 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -1,33 +1,103 @@ -using DimensionalData, Test +using DimensionalData +using Test +using Dates +using JLArrays +using GPUArrays + +using DimensionalData.Lookups +using DimensionalData.Dimensions using DimensionalData: NoLookup +GPUArrays.allowscalar(false) + # Tests taken from NamedDims. Thanks @oxinabox da = ones(X(3)) +dajl = rebuild(da, JLArray(parent(da))); @test Base.BroadcastStyle(typeof(da)) isa DimensionalData.DimensionalStyle @testset "standard case" begin - @test da .+ da == 2ones(3) + @test (@inferred da .+ da) == 2ones(3) @test dims(da .+ da) == dims(da) - @test da .+ da .+ da == 3ones(3) + @test (@inferred da .+ da .+ da) == 3ones(3) @test dims(da .+ da .+ da) == dims(da) end @testset "broadcast over length one dimension" begin - da2 = DimArray((1:4) * (1:2:8)', (X, Y)) - @test da2 .* da2[:, 1:1] == [1, 4, 9, 16] * (1:2:8)' + da2 = DimArray((1:4) * (1:2:8)', (X, Y)); + @test (@inferred da2 .* da2[:, 1:1]) == [1, 4, 9, 16] * (1:2:8)' + @test (@inferred da2[:, 1:1] .* da2) == [1, 4, 9, 16] * (1:2:8)' +end + +@testset "JLArray broadcast over length one dimension" begin + da2 = DimArray(JLArray((1:4) * (1:2:8)'), (X, Y)) + @test Array(da2 .* da2[:, 1:1]) == [1, 4, 9, 16] * (1:2:8)' end @testset "in place" begin - @test parent(da .= 1 .* da .+ 7) == 8 * ones(3) - @test dims(da .= 1 .* da .+ 7) == dims(da) + @test parent(copy(da) .= 1 .* da .+ 7) == 8 * ones(3) + @test (@inferred dims(copy(da .= 1 .* da .+ 7))) == dims(da) +end + +@testset "JLArray in place" begin + @test Array(parent(dajl .= 1 .* dajl .+ 7)) == 8 * ones(3) + @test dims(dajl .= 1 .* dajl .+ 7) == dims(da) end @testset "Dimension disagreement" begin + @test_throws DimensionMismatch zeros(X(3), Y(3), Z(3)) .+ ones(Y(3), Z(3), X(3)) + @test_throws DimensionMismatch zeros(X(3), Y(3), Z(3)) .+ ones(X(3), Z(3)) +end + +@testset "Lookup promotion" begin + @testset "NoLookup resolves conflicts" begin + @test isnolookup(zeros(X([:a, :b, :c]), Y(1.0:2.0:10.0)) .* zeros(X(3), Y(5))) + @test_throws DimensionMismatch zeros(X(1.0:3.0), Y('a':'e')) .* zeros(X(3), Y(DateTime.(2001:2005))) + end + # TODO test the rest + @testset "Categorical" begin + @test_throws DimensionMismatch lookup(zeros(X([:a, :b, :c]),) .* zeros(X([:x, :y, :z]),), X) + ls = ( + Sampled([10, 20, 30]; span=Irregular((nothing, nothing)), sampling=Points(), order=ForwardOrdered()), + Categorical([:a, :b, :c]; order=ForwardOrdered()), + Categorical(["foo", "bar", "foobar"]; order=Unordered()), + Sampled(1.0:1:3.0; span=Regular(1.0), sampling=Points(), order=ForwardOrdered()), + Sampled(1.0:1:3.0; span=Regular(1.0), sampling=Intervals(Start()), order=ForwardOrdered()), + ) + l = first(ls) + for l in ls + @test (@inferred lookup(zeros(X(l),) .* zeros(X(3),), X)) == NoLookup(Base.OneTo(3)) + @test (@inferred lookup(zeros(X(l),) .* zeros(X(1),), X)) == NoLookup(Base.OneTo(3)) + @test (@inferred lookup(zeros(X(l),) .* zeros(X(l),), X)) === l + @test (@inferred lookup(zeros(X(l[1:1]),) .* zeros(X(l),), X)) == l + @test (@inferred lookup(zeros(X(l),) .* zeros(X(l[1:1]),), X)) == l + end + @testset "Lookup types are promoted" begin + a = zeros(Y((Int8(1):Int8(2):Int8(9)))) + b = zeros(Y(1:2:9)) + c = @inferred a .+ b + @test lookup(c) === lookup(b) + a = zeros(Y(LinRange(1.0, 10.0, 10))) + b = zeros(Y(1:1:10)) + c = @inferred a .+ b + @test lookup(c) === lookup(a) + a = zeros(Y((Float16(1):Float16(2):Float16(9)))) + b = zeros(Y(1:2:9)) + c = @inferred a .+ b + @test lookup(c) === lookup(a) + a = zeros(Y(DateTime(2000):Year(1):DateTime(2003))) + b = zeros(Y(Date(2000):Year(1):Date(2003))) + c = @inferred a .+ b + @test lookup(c) === lookup(a) + end + end +end + +@testset "JLArray Dimension disagreement" begin @test_throws DimensionMismatch begin - DimArray(zeros(3, 3, 3), (X, Y, Z)) .+ - DimArray(ones(3, 3, 3), (Y, Z, X)) + DimArray(JLArray(zeros(3, 3, 3)), (X, Y, Z)) .+ + DimArray(JLArray(ones(3, 3, 3)), (Y, Z, X)) end end @@ -41,6 +111,16 @@ end @test dims(right_sum) == dims(da) end +@testset "JLArray dims and regular" begin + da = DimArray(JLArray(ones(3, 3, 3)), (X, Y, Z)) + left_sum = da .+ ones(3, 3, 3) + @test Array(left_sum) == fill(2, 3, 3, 3) + @test dims(left_sum) == dims(da) + right_sum = ones(3, 3, 3) .+ da + @test Array(right_sum) == fill(2, 3, 3, 3) + @test dims(right_sum) == dims(da) +end + @testset "changing type" begin @test (da .> 0) isa DimArray @test (da .* da .> 0) isa DimArray @@ -51,20 +131,30 @@ end @test (rand(3) .> 1 .> 0 .* da) isa DimArray end -@testset "trailng dimensions" begin - @test zeros(X(10), Y(5)) .* zeros(X(10), Y(1)) == - zeros(X(10), Y(5)) .* zeros(X(1), Y(1)) == - zeros(X(1), Y(1)) .* zeros(X(10), Y(5)) == - zeros(X(10), Y(5)) .* zeros(X(1), Y(5)) == - zeros(X(10), Y(1)) .* zeros(X(1), Y(5)) == - zeros(X(10), Y(5)) .* zeros(X(1)) == - zeros(X(1), Y(5)) .* zeros(X(10)) +@testset "JLArray changing type" begin + @test (dajl .> 0) isa DimArray + @test (dajl .* dajl .> 0) isa DimArray + @test (dajl .> 0 .> rand(3)) isa DimArray + @test (dajl .* rand(3) .> 0.0) isa DimArray + @test (0 .> dajl .> 0 .> rand(3)) isa DimArray + @test (rand(3) .> dajl .> 0 .* rand(3)) isa DimArray + @test (rand(3) .> 1 .> 0 .* dajl) isa DimArray +end + +@testset "trailing dimensions" begin + a = @inferred zeros(X(10), Y(5)) .* zeros(X(10), Y(1)) + b = @inferred zeros(X(10), Y(5)) .* zeros(X(1), Y(1)) + c = @inferred zeros(X(1), Y(1)) .* zeros(X(10), Y(5)) + d = @inferred zeros(X(10), Y(5)) .* zeros(X(1), Y(5)) + e = @inferred zeros(X(10), Y(1)) .* zeros(X(1), Y(5)) + f = @inferred zeros(X(10), Y(5)) .* zeros(X(1)) + g = @inferred zeros(X(1), Y(5)) .* zeros(X(10)) + @test a == b == c == d == e == f == g end @testset "mixed order fails" begin @test_throws DimensionMismatch zeros(X(1:3), Y(5)) .* zeros(X(3:-1:1), Y(5)) @test_throws DimensionMismatch zeros(X([1, 3, 2]), Y(5)) .* zeros(X(3:-1:1), Y(5)) - zeros(X([1, 3, 2]), Y(5)) .* zeros(X(3), Y(5)) end @testset "broadcasting" begin @@ -79,6 +169,18 @@ end @test dims(s .+ v .+ m) == dims(m .+ s .+ v) end +@testset "JLArray broadcasting" begin + v = DimArray(JLArray(zeros(3,)), X) + m = DimArray(JLArray(ones(3, 3)), (X, Y)) + s = 0 + @test Array(v .+ m) == ones(3, 3) == Array(m .+ v) + @test Array(s .+ m) == ones(3, 3) == Array(m .+ s) + @test Array(s .+ v .+ m) == ones(3, 3) == Array(m .+ s .+ v) + @test dims(v .+ m) == dims(m .+ v) + @test dims(s .+ m) == dims(m .+ s) + @test dims(s .+ v .+ m) == dims(m .+ s .+ v) +end + @testset "adjoint broadcasting" begin a = DimArray(reshape(1:12, (4, 3)), (X, Y)) b = DimArray(1:3, Y) @@ -88,6 +190,16 @@ end @test dims(a .* b') == dims(a) end +@testset "JLArray adjoint broadcasting" begin + a = DimArray(JLArray(reshape(1:12, (4, 3))), (X, Y)) + b = DimArray(JLArray(1:3), Y) + @test_throws DimensionMismatch a .* b + @test_throws DimensionMismatch parent(a) .* parent(b) + @test_nowarn Array(parent(a) .* parent(b)') == Array(parent(a .* b')) + @test_nowarn dims(a .* b') == dims(a) +end + + @testset "Mixed array types" begin casts = ( A -> DimArray(A, (X, Y)), # Named Matrix @@ -112,22 +224,37 @@ end ba = DimArray(rand(2,2), (Y, X)) ac = DimArray(rand(2,2), (X, Z)) a_ = DimArray(rand(2,2), (X(), DimensionalData.AnonDim())) - z = zeros(2,2) + z = zeros(2, 2) @test_throws DimensionMismatch z .= ab .+ ba @test_throws DimensionMismatch z .= ab .+ ac - @test_throws DimensionMismatch a_ .= ab .+ ac + # Maybe this should work... + # @test_throws DimensionMismatch a_ .= ab .+ ac @test_throws DimensionMismatch ab .= a_ .+ ac @test_throws DimensionMismatch ac .= ab .+ ba # check that dest is written into: - @test dims(z .= ab .+ ba') == dims(ab .+ ba') + z .= ab .+ ba' @test z == (ab.data .+ ba.data') +end - @test dims(z .= ab .+ a_) == - (X(NoLookup(Base.OneTo(2))), Y(NoLookup(Base.OneTo(2)))) - @test dims(a_ .= ba' .+ ab) == - (X(NoLookup(Base.OneTo(2))), Y(NoLookup(Base.OneTo(2)))) +@testset "JLArray in-place assignment .=" begin + ab = DimArray(JLArray(rand(2,2)), (X, Y)) + ba = DimArray(JLArray(rand(2,2)), (Y, X)) + ac = DimArray(JLArray(rand(2,2)), (X, Z)) + a_ = DimArray(JLArray(rand(2,2)), (X(), DimensionalData.AnonDim())) + z = JLArray(zeros(2,2)) + + @test_throws DimensionMismatch z .= ab .+ ba + @test_throws DimensionMismatch z .= ab .+ ac + @test_throws DimensionMismatch a_ .= ab .+ ac + @test_throws DimensionMismatch ab .= a_ .+ ac + @test_throws DimensionMismatch ac .= ab .+ ba + + # check that dest is written into: + z .= ab .+ ba' + @test z == (ab.data .+ ba.data') + @test z == (ab.data .+ ba.data') end @testset "assign using named indexing and dotview" begin @@ -137,6 +264,13 @@ end @test A == [1.0 1.0; 2.0 2.0; 7.0 7.0] end +@testset "JLArray assign using named indexing and dotview" begin + A = DimArray(JLArray(zeros(3,2)), (X, Y)) + A[X=1:2] .= JLArray([1, 2]) + A[X=3] .= 7 + @test Array(A) == [1.0 1.0; 2.0 2.0; 7.0 7.0] +end + @testset "0-dimensional array broadcasting" begin x = DimArray(fill(3), ()) y = DimArray(fill(4), ()) @@ -168,6 +302,31 @@ end @test A[DimSelectors(sub)] == C[DimSelectors(sub)] end +@testset "JLArray DimIndices broadcasting" begin + ds = X(1.0:0.2:2.0), Y(10:2:20) + _A = (rand(ds)) + _B = (zeros(ds)) + _C = (zeros(ds)) + + A = rebuild(_A, JLArray(parent(_A))) + B = rebuild(_B, JLArray(parent(_B))) + C = rebuild(_C, JLArray(parent(_C))) + + B[DimIndices(B)] .+= A + C[DimSelectors(C)] .+= A + @test Array(A) == Array(B) == Array(C) + sub = A[1:4, 1:3] + B .= 0 + C .= 0 + B[DimIndices(sub)] .+= sub + C[DimSelectors(sub)] .+= sub + @test Array(A[DimIndices(sub)]) == Array(B[DimIndices(sub)]) == Array(C[DimIndices(sub)]) + sub = A[2:4, 2:5] + C .= 0 + C[DimSelectors(sub)] .+= sub + @test Array(A[DimSelectors(sub)]) == Array(C[DimSelectors(sub)]) +end + # @testset "Competing Wrappers" begin # da = DimArray(ones(4), X) # ta = TrackedArray(5 * ones(4)) diff --git a/test/indexing.jl b/test/indexing.jl index 6b7b1e72b..e1152c446 100644 --- a/test/indexing.jl +++ b/test/indexing.jl @@ -37,7 +37,7 @@ end @test l[1:5] isa typeof(l) @test l[[1, 3, 4]] == view(l, [1, 3, 4]) == Base.dotview(l, [1, 3, 4]) == - Sampled([2.0, 6.0, 8.0], ForwardOrdered(), Irregular(2.0, 8.0), Points(), NoMetadata()) + Sampled([2.0, 6.0, 8.0], ForwardOrdered(), Irregular(nothing, nothing), Points(), NoMetadata()) @test l[Int[]] == view(l, Int[]) == Base.dotview(l, Int[]) == Sampled(Float64[], ForwardOrdered(), Irregular(nothing, nothing), Points(), nothing) @test l[Near(2.1)] == Base.dotview(l, Near(2.1)) == 2.0 @@ -45,7 +45,7 @@ end @test l[[false, true, true, false, true]] == view(l, [false, true, true, false, true]) == Base.dotview(l, [false, true, true, false, true]) == - Sampled([4.0, 6.0, 10.0], ForwardOrdered(), Irregular(4.0, 10.0), Points(), nothing) + Sampled([4.0, 6.0, 10.0], ForwardOrdered(), Irregular(nothing, nothing), Points(), nothing) @test l[2] == Base.dotview(l, 2) === 4.0 @test view(l, 2) == fill(4.0) @test l[CartesianIndex((4,))] === Base.dotview(l, CartesianIndex((4,))) === 8.0 @@ -61,25 +61,25 @@ end # End Locus l = Sampled(2.0:2.0:10.0, ForwardOrdered(), Regular(2.0), Points(), nothing) @test l[[false, true, true, false, true]] == - Sampled([4.0, 6.0, 10.0], ForwardOrdered(), Irregular(4.0, 10.0), Points(), nothing) + Sampled([4.0, 6.0, 10.0], ForwardOrdered(), Irregular(nothing, nothing), Points(), nothing) # Center Locus l = Sampled(2.0:2.0:10.0, ForwardOrdered(), Regular(2.0), Points(), nothing) @test l[[false, true, true, false, true]] == - Sampled([4.0, 6.0, 10.0], ForwardOrdered(), Irregular(4.0, 10.0), Points(), nothing) + Sampled([4.0, 6.0, 10.0], ForwardOrdered(), Irregular(nothing, nothing), Points(), nothing) # Center Locus DateTime l = Sampled(DateTime(2001, 1, 1):Day(1):DateTime(2001, 1, 5), ForwardOrdered(), Regular(Day(1)), Points(), nothing) @test l[[false, true, true, false, true]] == - Sampled([DateTime(2001, 1, 2), DateTime(2001, 1, 3), DateTime(2001, 1, 5)], ForwardOrdered(), Irregular(DateTime(2001, 1, 2), DateTime(2001, 1, 5)), Points(), nothing) + Sampled([DateTime(2001, 1, 2), DateTime(2001, 1, 3), DateTime(2001, 1, 5)], ForwardOrdered(), Irregular(nothing, nothing), Points(), nothing) # Reverse l = Sampled(10.0:-2.0:2.0, ReverseOrdered(), Regular(-2.0), Points(), nothing) @test l[:] == l @test l[1:5] == l @test l[1:5] isa typeof(l) - @test l[[1, 3, 4]] == Sampled([10.0, 6.0, 4.0], ReverseOrdered(), Irregular(4.0, 10.0), Points(), nothing) + @test l[[1, 3, 4]] == Sampled([10.0, 6.0, 4.0], ReverseOrdered(), Irregular(nothing, nothing), Points(), nothing) @test l[Int[]] == Sampled(Float64[], ReverseOrdered(), Irregular(nothing, nothing), Points(), nothing) @test l[Near(2.1)] == 2.0 @test l[[false, true, true, false, true]] == - Sampled([8.0, 6.0, 2.0], ReverseOrdered(), Irregular(2.0, 8.0), Points(), nothing) + Sampled([8.0, 6.0, 2.0], ReverseOrdered(), Irregular(nothing, nothing), Points(), nothing) @test l[2] === 8.0 @test l[CartesianIndex((4,))] == 4.0 @test l[CartesianIndices((2:4,))] == Sampled(8.0:-2.0:4.0, ReverseOrdered(), Regular(-2.0), Points(), nothing) @@ -87,15 +87,15 @@ end # End Locus l = Sampled(10.0:-2.0:2.0, ReverseOrdered(), Regular(-2.0), Points(), nothing) @test l[[false, true, true, false, true]] == - Sampled([8.0, 6.0, 2.0], ReverseOrdered(), Irregular(2.0, 8.0), Points(), nothing) + Sampled([8.0, 6.0, 2.0], ReverseOrdered(), Irregular(nothing, nothing), Points(), nothing) # Center Locus l = Sampled(10.0:-2.0:2.0, ReverseOrdered(), Regular(-2.0), Points(), nothing) @test l[[false, true, true, false, true]] == - Sampled([8.0, 6.0, 2.0], ReverseOrdered(), Irregular(2.0, 8.0), Points(), nothing) + Sampled([8.0, 6.0, 2.0], ReverseOrdered(), Irregular(nothing, nothing), Points(), nothing) # Center Locus DateTime l = Sampled(DateTime(2001, 1, 5):Day(-1):DateTime(2001, 1, 1), ReverseOrdered(), Regular(Day(-1)), Points(), nothing) @test l[[false, true, true, false, true]] == - Sampled([DateTime(2001, 1, 4), DateTime(2001, 1, 3), DateTime(2001, 1, 1)], ReverseOrdered(), Irregular(DateTime(2001, 1, 1), DateTime(2001, 1, 4)), Points(), nothing) + Sampled([DateTime(2001, 1, 4), DateTime(2001, 1, 3), DateTime(2001, 1, 1)], ReverseOrdered(), Irregular(nothing, nothing), Points(), nothing) end @testset "Intervals" begin diff --git a/test/matmul.jl b/test/matmul.jl index c8e177970..c1c7707b9 100644 --- a/test/matmul.jl +++ b/test/matmul.jl @@ -47,9 +47,11 @@ using Combinatorics: combinations # Test where results have an empty dim true_result = (parent(b1)' * parent(B1)) + flip = adjoint for flip in (adjoint, transpose, permutedims) result = flip(b1) * B1 - @test result ≈ true_result # Permute dims is not exactly transpose + @test result ≈ true_result + # Permute dims is not exactly transpose @test dims(result) isa Tuple{<:AnonDim, <:X} @test length.(dims(result)) == (1, 6) end diff --git a/test/methods.jl b/test/methods.jl index e5b6dff1e..d874ec245 100644 --- a/test/methods.jl +++ b/test/methods.jl @@ -1,7 +1,10 @@ using DimensionalData, Statistics, Test, Unitful, SparseArrays, Dates using DimensionalData.Lookups, DimensionalData.Dimensions - +using JLArrays using LinearAlgebra: Transpose +using GPUArrays + +GPUArrays.allowscalar(false) xs = (1, X, X(), :X) ys = (2, Y, Y(), :Y) @@ -17,6 +20,16 @@ xys = ((1, 2), (X, Y), (X(), Y()), (:X, :Y)) @test map(*, da, da) isa DimArray{Int64,2} end +@testset "JLArray map" begin + a = JLArray([1 2; 3 4]) + dimz = X(143:2:145), Y(Sampled(-38:2:-36; span=Explicit([-38 -36; -36 -34]))) + da = DimArray(a, dimz) + @test Array(map(x -> 2x, da)) == [2 4; 6 8] + @test map(x -> 2x, da) isa DimArray{Int64,2} + @test Array(map(*, da, da)) == [1 4; 9 16] + @test map(*, da, da) isa DimArray{Int64,2} +end + @testset "dimension reducing methods" begin # Test some reducing methods with Explicit spans @@ -127,6 +140,117 @@ end end end +@testset "JLArray dimension reducing methods" begin + + # Test some reducing methods with Explicit spans + a = JLArray([1 2; 3 4]) + dimz = X(143:2:145), Y(Sampled(-38:2:-36; span=Explicit([-38 -36; -36 -34]))) + da = DimArray(a, dimz) + + # Test all dime combinations with maxium + for dims in xs + @test Array(sum(da; dims)) == [4 6] + @test Array(minimum(da; dims)) == [1 2] + + testdims = (X(Sampled(144.0:2.0:144.0, ForwardOrdered(), Regular(4.0), Points(), NoMetadata())), + Y(Sampled(-38:2:-36, ForwardOrdered(), Explicit([-38 -36; -36 -34]), Intervals(Center()), NoMetadata()))) + @test typeof(DimensionalData.dims(minimum(da; dims))) == typeof(testdims) + # @test val.(span(minimum(da; dims))) == val.(span(testdims)) + end + for dims in ys + @test Array(minimum(da; dims)) == [1 3]' + @test Array(maximum(da; dims)) == [2 4]' + @test Array(maximum(x -> 2x, da; dims)) == [4 8]' + testdims = (X(Sampled(143:2:145, ForwardOrdered(), Regular(2), Points(), NoMetadata())), + Y(Sampled(-37.0:4.0:-37.0, ForwardOrdered(), Explicit(reshape([-38, -34], 2, 1)), Intervals(Center()), NoMetadata()))) + @test typeof(DimensionalData.dims(sum(da; dims))) == typeof(testdims) + @test index(sum(da; dims)) == index.(testdims) + # @test val.(span(sum(da; dims))) == val.(span(testdims)) + end + for dims in xys + @test Array(maximum(da; dims)) == [4]' + @test Array(maximum(x -> 2x, da; dims)) == [8]' + end + + @test minimum(da; dims=:) == 1 + @test maximum(da; dims=:) == 4 + @test sum(da; dims=:) == 10 + @test sum(x -> 2x, da; dims=:) == 20 + + a = JLArray([1 2; 3 4]) + dimz = X(143:2:145), Y(-38:2:-36) + da = DimArray(a, dimz) + + @test reduce(+, da) == reduce(+, a) + @test mapreduce(x -> x > 3, +, da; dims=:) == 1 + @test std(da) === std(a) + + for dims in xs + @test Array(prod(da; dims)) == [3 8] + @test Array(mean(da; dims)) == [2.0 3.0] + @test Array(mean(x -> 2x, da; dims)) == [4.0 6.0] + @test Array(reduce(+, da; dims)) == [4 6] + @test Array(mapreduce(x -> x > 3, +, da; dims)) == [0 1] + @test Array(var(da; dims)) == [2.0 2.0] + @test Array(std(da; dims)) == [1.4142135623730951 1.4142135623730951] + @test Array(extrema(da; dims)) == [(1, 3) (2, 4)] + resultdimz = + (X(Sampled(144.0:2.0:144.0, ForwardOrdered(), Regular(4.0), Points(), NoMetadata())), + Y(Sampled(-38:2:-36, ForwardOrdered(), Regular(2), Points(), NoMetadata()))) + @test typeof(DimensionalData.dims(prod(da; dims))) == typeof(resultdimz) + @test bounds(DimensionalData.dims(prod(da; dims))) == bounds(resultdimz) + end + for dims in ys + @test Array(prod(da; dims=2)) == [2 12]' + @test Array(mean(da; dims)) == [1.5 3.5]' + @test Array(mean(x -> 2x, da; dims)) == [3.0 7.0]' + @test DimensionalData.dims(mean(da; dims)) == + (X(Sampled(143:2:145, ForwardOrdered(), Regular(2), Points(), NoMetadata())), + Y(Sampled(-37.0:4.0:-37.0, ForwardOrdered(), Regular(4.0), Points(), NoMetadata()))) + @test DimensionalData.dims(reduce(+, da; dims)) == + (X(Sampled(143:2:145, ForwardOrdered(), Regular(2), Points(), NoMetadata())), + Y(Sampled(-37.0:2.0:-37.0, ForwardOrdered(), Regular(4.0), Points(), NoMetadata()))) + @test DimensionalData.dims(mapreduce(x -> x > 3, +, da; dims)) == + (X(Sampled(143:2:145, ForwardOrdered(), Regular(2), Points(), NoMetadata())), + Y(Sampled(-37.0:2:-37.0, ForwardOrdered(), Regular(4.0), Points(), NoMetadata()))) + @test Array(std(da; dims)) == [0.7071067811865476 0.7071067811865476]' + @test Array(var(da; dims)) == [0.5 0.5]' + @test DimensionalData.dims(var(da; dims)) == + (X(Sampled(143:2:145, ForwardOrdered(), Regular(2), Points(), NoMetadata())), + Y(Sampled(-37.0:4.0:-37.0, ForwardOrdered(), Regular(4.0), Points(), NoMetadata()))) + @test Array(extrema(da; dims)) == permutedims([(1, 2) (3, 4)]) + end + for dims in xys + @test Array(mean(da; dims=dims)) == [2.5]' + @test Array(mean(x -> 2x, da; dims=dims)) == [5.0]' + @test Array(reduce(+, da; dims)) == [10]' + @test Array(mapreduce(x -> x > 3, +, da; dims)) == [1]' + @test Array(extrema(da; dims)) == reshape([(1, 4)], 1, 1) + end + + a = JLArray([1 2 3; 4 5 6]) + dimz = X(143:2:145), Y(-38:-36) + da = DimArray(a, dimz) + @test @inferred median(da) == 3.5 + # median along a dimension doesn't have a gpu implementation + @test_broken @inferred Array(median(da; dims=X())) == [2.5 3.5 4.5] + @test_broken @inferred Array(median(da; dims=2)) == [2.0 5.0]' + + a = JLArray(Bool[0 1 1; 0 0 0]) + da = DimArray(a, dimz); + @test_broken any(da) === true # This is broken because GPUArrays didn't support any(da, dims=:) only any(da) + @test_broken any(da; dims=Y) == reshape([true, false], 2, 1) + @test_broken all(da) === false + @test_broken all(da; dims=Y) == reshape([false, false], 2, 1) + @test_broken all(da; dims=(X, Y)) == reshape([false], 1, 1) + + @testset "inference" begin + x = DimArray(randn(2, 3, 4), (X, Y, Z)); + foo(x) = maximum(x; dims=(1, 2)) + @inferred foo(x) + end +end + @testset "dimension dropping methods" begin a = [1 2 3; 4 5 6] dimz = X(143:2:145), Y(-38:-36) @@ -142,6 +266,21 @@ end @test length.(dims(dropped[1:2])) == size(dropped[1:2]) end +@testset "JLArray dimension dropping methods" begin + a = JLArray([1 2 3; 4 5 6]) + dimz = X(143:2:145), Y(-38:-36) + da = DimArray(a, dimz); + # Dimensions must have length 1 to be dropped + @test Array(dropdims(da[X(1:1)]; dims=X)) == [1, 2, 3] + @test Array(dropdims(da[2:2, 1:1]; dims=(X(), Y())))[] == 4 + @test typeof(dropdims(da[2:2, 1:1]; dims=(X(), Y()))) <: DimArray{Int,0,Tuple{}} + @test refdims(dropdims(da[X(1:1)]; dims=X)) == + (X(Sampled(143:2:143, ForwardOrdered(), Regular(2), Points(), NoMetadata())),) + dropped = dropdims(da[X(1:1)]; dims=X) + @test Array(dropped[1:2]) == [1, 2] + @test length.(dims(dropped[1:2])) == size(dropped[1:2]) +end + @testset "eachslice" begin a = [1 2 3 4 3 4 5 6 @@ -233,7 +372,7 @@ end @testset "simple dimension permuting methods" begin da = DimArray(zeros(5, 4), (Y(LinRange(10, 20, 5)), X(1:4))) - tda = transpose(da) + tda = @inferred transpose(da) @test tda == transpose(parent(da)) resultdims = (X(Sampled(1:4, ForwardOrdered(), Regular(1), Points(), NoMetadata())), Y(Sampled(LinRange(10.0, 20.0, 5), ForwardOrdered(), Regular(2.5), Points(), NoMetadata()))) @@ -241,31 +380,30 @@ end @test dims(tda) == resultdims @test size(tda) == (4, 5) - tda = Transpose(da) + tda = @inferred Transpose(da) @test tda == Transpose(parent(da)) @test dims(tda) == (X(Sampled(1:4, ForwardOrdered(), Regular(1), Points(), NoMetadata())), Y(Sampled(LinRange(10.0, 20.0, 5), ForwardOrdered(), Regular(2.5), Points(), NoMetadata()))) @test size(tda) == (4, 5) @test typeof(tda) <: DimArray - ada = adjoint(da) + ada = @inferred adjoint(da) @test ada == adjoint(parent(da)) @test dims(ada) == (X(Sampled(1:4, ForwardOrdered(), Regular(1), Points(), NoMetadata())), Y(Sampled(LinRange(10.0, 20.0, 5), ForwardOrdered(), Regular(2.5), Points(), NoMetadata()))) @test size(ada) == (4, 5) - dsp = permutedims(da) + dsp = @inferred permutedims(da) @test permutedims(parent(da)) == parent(dsp) @test dims(dsp) == reverse(dims(da)) end - @testset "dimension permuting methods with specified permutation" begin da = DimArray(ones(5, 2, 4), (Y(LinRange(10, 20, 5)), Ti(10:11), X(1:4))) dsp = permutedims(da, [3, 1, 2]) @test permutedims(da, [X, Y, Ti]) == permutedims(da, (X, Y, Ti)) @test permutedims(da, [X(), Y(), Ti()]) == permutedims(da, (X(), Y(), Ti())) - dsp = permutedims(da, (X(), Y(), Ti())) + dsp = @inferred permutedims(da, (X(), Y(), Ti())) @test dsp == permutedims(parent(da), (3, 1, 2)) @test dims(dsp) == (X(Sampled(1:4, ForwardOrdered(), Regular(1), Points(), NoMetadata())), Y(Sampled(LinRange(10.0, 20.0, 5), ForwardOrdered(), Regular(2.5), Points(), NoMetadata())), @@ -281,13 +419,12 @@ end @test typeof(dsp2) <: DimArray end - @testset "dimension rotating methods" begin da = DimArray([1 2; 3 4], (X([:a, :b]), Y([1.0, 2.0]))) - l90 = rotl90(da) - r90 = rotr90(da) - r180_1 = rot180(da) + l90 = @inferred rotl90(da) + r90 = @inferred rotr90(da) + r180_1 = @inferred rot180(da) r180_2 = rotl90(da, 2) r180_3 = rotr90(da, 2) r270 = rotl90(da, 3) @@ -314,7 +451,7 @@ end xs = (X, X(), :X) ys = (Y, Y(), :Y) for dims in xs - cvda = cov(da; dims=X) + cvda = cov(da; dims) @test cvda == cov(a; dims=2) @test DimensionalData.dims(cvda) == (Y(Sampled(LinRange(10.0, 20.0, 5), ForwardOrdered(), Regular(2.5), Points(), NoMetadata())), @@ -709,3 +846,80 @@ end end end end + +@testset "mapreduce" begin + @testset "Array 2D" begin + y = Y(['a', 'b', 'c']) + ti = Ti(DateTime(2021, 1):Month(1):DateTime(2021, 4)) + ys = (1, Y, Y(), :Y, y) + tis = (2, Ti, Ti(), :Ti, ti) + data = [-87 -49 107 -18 + 24 44 -62 124 + 122 -11 48 -7] + A = DimArray(data, (y, ti)) + + + @test mapreduce(identity, +, A) ≈ mapreduce(identity, +, parent(A)) + @test mapreduce(x->x^3+5, +, A) ≈ mapreduce(x->x^3+5, +, parent(A)) + + for dims in ys + @test mapreduce(identity, +, A; dims) ≈ mapreduce(identity, +, parent(A); dims=1) + end + + for dims in tis + @test mapreduce(identity, +, A; dims) ≈ mapreduce(identity, +, parent(A); dims=2) + end + + @test mapreduce(identity, +, A; dims=Y) ≈ mapreduce(identity, +, parent(A); dims=1) + @test mapreduce(identity, +, A; dims=Ti) ≈ mapreduce(identity, +, parent(A); dims=2) + @test mapreduce(identity, +, A; dims=(Y, Ti)) ≈ mapreduce(identity, +, parent(A); dims=(1, 2)) + + init = 5.0 + @test mapreduce(identity, +, A; init) ≈ mapreduce(identity, +, parent(A); init) + @test mapreduce(x->x^3+5, +, A; init) ≈ mapreduce(x->x^3+5, +, parent(A); init) + @test mapreduce(identity, +, A; dims=Y, init) ≈ mapreduce(identity, +, parent(A); dims=1, init) + @test mapreduce(identity, +, A; dims=Ti, init) ≈ mapreduce(identity, +, parent(A); dims=2, init) + @test mapreduce(identity, +, A; dims=(Y, Ti), init) ≈ mapreduce(identity, +, parent(A); dims=(1, 2), init) + end + @testset "Vector" begin + x = DimArray([56, -123, -60, -44, -64, 70, 52, -48, -74, 86], X(2:2:20)) + @test mapreduce(x->x^2, +, x) ≈ mapreduce(x->x^2, +, parent(x)) + @test mapreduce(identity, +, x) ≈ mapreduce(identity, +, parent(x)) + @test mapreduce(identity, +, x; dims=X) ≈ mapreduce(identity, +, parent(x); dims=1) + @test mapreduce(x->x^2, +, x; dims=X) ≈ mapreduce(x->x^2, +, parent(x); dims=1) + @test mapreduce(identity, +, x; init=5.0) ≈ mapreduce(identity, +, parent(x); init=5.0) + end + + @testset "JLArray" begin + y = Y(['a', 'b', 'c']) + ti = Ti(DateTime(2021, 1):Month(1):DateTime(2021, 4)) + ys = (1, Y, Y(), :Y, y) + tis = (2, Ti, Ti(), :Ti, ti) + data = JLArray([-87 -49 107 -18 + 24 44 -62 124 + 122 -11 48 -7]) + A = DimArray(data, (y, ti)) + + @test mapreduce(identity, +, A) ≈ mapreduce(identity, +, parent(A)) + @test mapreduce(x->x^3+5, +, A) ≈ mapreduce(x->x^3+5, +, parent(A)) + # Using parent since JLArray errors + for dims in ys + @test parent(mapreduce(identity, +, A; dims)) ≈ mapreduce(identity, +, parent(A); dims=1) + end + + for dims in tis + @test parent(mapreduce(identity, +, A; dims)) ≈ mapreduce(identity, +, parent(A); dims=2) + end + + @test parent(mapreduce(identity, +, A; dims=Y)) ≈ mapreduce(identity, +, parent(A); dims=1) + @test parent(mapreduce(identity, +, A; dims=Ti)) ≈ mapreduce(identity, +, parent(A); dims=2) + @test parent(mapreduce(identity, +, A; dims=(Y, Ti))) ≈ mapreduce(identity, +, parent(A); dims=(1, 2)) + + init = 5.0 + @test mapreduce(identity, +, A; init) ≈ mapreduce(identity, +, parent(A); init) + @test mapreduce(x->x^3+5, +, A; init) ≈ mapreduce(x->x^3+5, +, parent(A); init) + @test parent(mapreduce(identity, +, A; dims=Y, init)) ≈ mapreduce(identity, +, parent(A); dims=1, init) + @test parent(mapreduce(identity, +, A; dims=Ti, init)) ≈ mapreduce(identity, +, parent(A); dims=2, init) + @test parent(mapreduce(identity, +, A; dims=(Y, Ti), init)) ≈ mapreduce(identity, +, parent(A); dims=(1, 2), init) + end +end diff --git a/test/plotrecipes.jl b/test/plotrecipes.jl index ae209ed9b..13787b151 100644 --- a/test/plotrecipes.jl +++ b/test/plotrecipes.jl @@ -235,6 +235,7 @@ end M.waterfall!(ax, A1u) fig, ax, _ = M.waterfall(A1m) M.waterfall!(ax, A1m) + # 2d A2 = rand(X(10:10:100), Y(['a', 'b', 'c'])) A2r = rand(Y(10:10:100), X(['a', 'b', 'c'])) @@ -255,8 +256,6 @@ end M.convert_arguments(M.VertexGrid(), A2u) M.convert_arguments(M.ImageLike(), A2u) - - fig, ax, _ = M.plot(A2) M.plot!(ax, A2) fig, ax, _ = M.plot(A2m) @@ -296,8 +295,9 @@ end M.surface!(ax, A2u) fig, ax, _ = M.surface(A2ui) M.surface!(ax, A2ui) - fig, ax, _ = M.surface(A2m) - M.surface!(ax, A2m) + # Broken with missing + # fig, ax, _ = M.surface(A2m) + # M.surface!(ax, A2m) # Series also puts Categories in the legend no matter where they are # TODO: method series! is incomplete, we need to include the colors logic, as in series. There should not be any issue if the correct amount of colours is provided. fig, ax, _ = M.series(A2) diff --git a/test/primitives.jl b/test/primitives.jl index 9bfb9c02d..16cad735b 100644 --- a/test/primitives.jl +++ b/test/primitives.jl @@ -1,7 +1,7 @@ using DimensionalData, Dates, Test , BenchmarkTools using DimensionalData.Lookups, DimensionalData.Dimensions -using .Dimensions: _dim_query, _wraparg, _reducedims, AlwaysTuple, MaybeFirst, comparedims +using .Dimensions: _dim_query, _wraparg, _reducedims, AlwaysTuple, MaybeFirst, comparedims, promotedims @dim Tst @@ -55,7 +55,7 @@ end end @testset "_wraparg" begin - @test _wraparg(X()) == (X(),) + @test @inferred _wraparg(X()) == (X(),) @test _wraparg((X, Y,), X,) == ((Val(X), Val(Y),), Val(X),) @test _wraparg((X, X(), :x), Y, Y(), :y) == ((Val(X), X(), Dim{:x}()), Val(Y), Y(), Dim{:y}()) @@ -320,51 +320,51 @@ end end @testset "comparedims" begin + + @testset "single dimensions" begin + z = Z(NoLookup(Base.OneTo(10))) + @test comparedims(z) == true + @test comparedims(z, z) == true + @test comparedims(z, z, z) == true + end + @testset "default keywords" begin - @test comparedims(Bool, X(1:2), X(1:2)) - @test !comparedims(Bool, X(1:2), Y(1:2)) - @test !comparedims(Bool, X(1:2), X(1:3)) - @test_warn "Found both lengths 2 and 3" comparedims(Bool, X(1:2), X(1:3); warn="") - @test_warn "X and Y dims on the same axis" comparedims(Bool, X(1:2), Y(1:2); warn="") - @test comparedims(X(1:2), X(1:2)) == X(1:2) + @test @inferred comparedims(Bool, X(1:2), X(1:2)) + @test @inferred !comparedims(Bool, X(1:2), Y(1:2)) + @test @inferred !comparedims(Bool, X(1:2), X(1:3)) + @test_warn "Found both lengths 2 and 3" comparedims(Bool, X(1:2), X(1:3); msg=Dimensions.Warn()) + @test_warn "X and Y dims on the same axis" comparedims(Bool, X(1:2), Y(1:2); msg=Dimensions.Warn()) @test_throws DimensionMismatch comparedims(X(1:2), Y(1:2)) @test_throws DimensionMismatch comparedims(X(1:2), X(1:3)) end @testset "compare type" begin @test comparedims(Bool, X(1:2), Y(1:2); type=false) @test !comparedims(Bool, X(1:2), Y(1:2); type=true) - @test_warn "X and Y dims on the same axis" comparedims(Bool, X(1:2), Y(1:2); type=true, warn="") - @test comparedims(X(1:2), Y(1:2); type=false) == X(Sampled(1:2)) + @test_warn "X and Y dims on the same axis" comparedims(Bool, X(1:2), Y(1:2); type=true, msg=Dimensions.Warn()) @test_throws DimensionMismatch comparedims(X(Sampled(1:2)), Y(Sampled(1:2)); type=true) end @testset "compare val type" begin - @test comparedims(Bool, X(Sampled(1:2)), X(Categorical(1:2)); valtype=false) - @test !comparedims(Bool, X(Sampled(1:2)), X(Categorical(1:2)); valtype=true) - @test comparedims(X(Sampled(1:2)), X(Categorical(1:2)); valtype=false) == X(Sampled(1:2)) + @test @inferred !comparedims(Bool, X(Sampled(1:2)), X(Categorical(1:2)); valtype=true) @test_throws DimensionMismatch comparedims(X(Sampled(1:2)), X(Categorical(1:2)); valtype=true) - @test comparedims(Bool, X(Sampled(1:2)), X(Sampled([1, 2])); valtype=false) + @test @inferred comparedims(Bool, X(Sampled(1:2)), X(Sampled([1, 2])); valtype=false) @test !comparedims(Bool, X(Sampled(1:2)), X(Sampled([1, 2])); valtype=true) - @test comparedims(X(Sampled(1:2)), X(Sampled([1, 2])); valtype=false) == X(Sampled(1:2)) @test_throws DimensionMismatch comparedims(X(Sampled([1, 2])), X(Sampled(1:2)); valtype=true) end @testset "compare values" begin @test comparedims(Bool, X(1:2), X(2:3); val=false) @test !comparedims(Bool, X(1:2), X(2:3); val=true) - @test_warn "do not match" comparedims(Bool, X(1:2), X(2:3); val=true, warn="") - @test comparedims(Bool, X(Sampled(1:2)), X(Sampled(2:3)); val=false) + @test_warn "do not match" comparedims(Bool, X(1:2), X(2:3); val=true, msg=Dimensions.Warn()) + @test_nowarn comparedims(Bool, X(Sampled(1:2)), X(Sampled(2:3)); val=false) @test !comparedims(Bool, X(Sampled(1:2)), X(Sampled(2:3)); val=true) - @test comparedims(X(Sampled(1:2)), X(Sampled(2:3)); val=false) == X(Sampled(1:2)) @test_throws DimensionMismatch comparedims(X(Sampled(1:2)), X(Sampled(2:3)); val=true) end @testset "compare length" begin @test comparedims(Bool, X(1:2), X(1:3); length=false) @test !comparedims(Bool, X(1:2), X(1:3); length=true) - @test_warn "Found both lengths" comparedims(Bool, X(1:2), X(1:3); length=true, warn="") - @test comparedims(X(1:2), X(1:3); length=false) == X(1:2) + @test_warn "Found both lengths" comparedims(Bool, X(1:2), X(1:3); length=true, msg=Dimensions.Warn()) @test_throws DimensionMismatch comparedims(X(1:2), X(1:3); length=true) @test comparedims(Bool, X(1:2), X(1:1); length=true, ignore_length_one=true) @test !comparedims(Bool, X(1:2), X(1:1); length=true, ignore_length_one=false) - @test comparedims(X(1:2), X(1:1); length=false, ignore_length_one=true) == X(1:2) @test_throws DimensionMismatch comparedims(X(1:2), X(1:1); length=true, ignore_length_one=false) end @testset "compare order" begin @@ -373,7 +373,7 @@ end @test !comparedims(Bool, a, b; order=true) @test comparedims(Bool, a, b; order=false) @test_nowarn comparedims(Bool, a, b; order=true) - @test_warn "Lookups do not all have the same order" comparedims(Bool, a, b; order=true, warn="") + @test_warn "Lookups do not all have the same order" comparedims(Bool, a, b; order=true, msg=Dimensions.Warn()) @test_throws DimensionMismatch comparedims(a, b; order=true) end end @@ -462,11 +462,11 @@ end Y(Sampled([40.0, 20.0, 10.0], ReverseOrdered(), Irregular(), Points(), NoMetadata())), )) irreg_dimz = dims(irreg) @test slicedims(irreg, (1:2, 3)) == slicedims(irreg, 1:2, 3) == - ((X(Sampled([140.0, 142.0], ForwardOrdered(), Irregular(140, 142), Points(), NoMetadata())),), - (Y(Sampled([10.0], ReverseOrdered(), Irregular(10.0, 10.0), Points(), NoMetadata())),)) + ((X(Sampled([140.0, 142.0], ForwardOrdered(), Irregular(nothing, nothing), Points(), NoMetadata())),), + (Y(Sampled([10.0], ReverseOrdered(), Irregular(nothing, nothing), Points(), NoMetadata())),)) @test slicedims(irreg, (2:2, 1:2)) == slicedims(irreg, 2:2, 1:2) == - ((X(Sampled([142.0], ForwardOrdered(), Irregular(142, 142), Points(), NoMetadata())), - Y(Sampled([40.0, 20.0], ReverseOrdered(), Irregular(20.0, 40.0), Points(), NoMetadata()))), ()) + ((X(Sampled([142.0], ForwardOrdered(), Irregular(nothing, nothing), Points(), NoMetadata())), + Y(Sampled([40.0, 20.0], ReverseOrdered(), Irregular(nothing, nothing), Points(), NoMetadata()))), ()) @test_broken slicedims((), (1:2, 3)) == slicedims((), (), (1:2, 3)) == ((), ()) @test slicedims((), (1, 1)) == slicedims((), (), (1, 1)) == ((), ()) end @@ -544,3 +544,27 @@ end @test step(testdim) == step(reduceddim) end end + +@testset "promotedims" begin + nl = NoLookup(Base.OneTo(2)) + c = Categorical(["a", "b"]; order=ForwardOrdered()) + c_sub = Categorical([view("a", 1:1), view("b", 1:1)]; order=ForwardOrdered()) + s = Sampled(1.0:1.0:2.0; span=Regular(1.0), sampling=Points(), order=ForwardOrdered()) + s_int = Sampled(1:1:2; span=Regular(1), sampling=Points(), order=ForwardOrdered()) + @test promotedims(X(nl), X(c)) === promotedims(X(c), X(nl)) === X(nl) + @test promotedims(X(nl), X(s)) === promotedims(X(s), X(nl)) === X(nl) + @test promotedims(X(c), X(s)) === promotedims(X(s), X(c)) === X(nl) + @test promotedims(X(s), X(s)) === X(s) + @test promotedims(X(s_int), X(s_int)) === X(s_int) + @test promotedims(X(s), X(s_int)) === promotedims(X(s_int), X(s)) === X(s) + @test promotedims(X(c), X(c_sub)) == promotedims(X(c_sub), X(c)) == X(c) + @test eltype(promotedims(X(c_sub), X(c))) == String + @test promotedims((X(c), Y(s)), (X(c), Y(s))) == (X(c), Y(s)) + @test promotedims((X(c), Y(nl)), (X(nl), Y(s))) == (X(nl), Y(nl)) + @test promotedims((X(c), Y(s_int)), (X(c_sub), Y(s))) == (X(c), Y(s)) + + @test promotedims(X(1), X(1.0)) == promotedims(X(1.0), X(1)) == X(1.0) + @test promotedims(X("a"), X(view("a", 1:1))) == + promotedims(X(view("a", 1:1)), X("a")) == + X("a") +end diff --git a/test/stack.jl b/test/stack.jl index 81f0d2001..95acfc1fc 100644 --- a/test/stack.jl +++ b/test/stack.jl @@ -56,8 +56,8 @@ end end @testset "broadcast over layer" begin - s[:one] .*= 2 - s[:one] ./= 2 + s[:one] .*= 2; + s[:one] ./= 2; end @testset "low level base methods" begin