From 45947bb371ab02bc3b28c67af1a5288ab8cf11b2 Mon Sep 17 00:00:00 2001 From: Christof Stocker Date: Sun, 4 Jun 2017 00:05:28 +0200 Subject: [PATCH 1/5] outline util operations for array handling --- src/Augmentor.jl | 8 ++++++++ src/operations/channels.jl | 42 ++++++++++++++++++++++++++++++++++++++ src/utils.jl | 4 +++- 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 src/operations/channels.jl diff --git a/src/Augmentor.jl b/src/Augmentor.jl index 37c3af5a..f8552346 100644 --- a/src/Augmentor.jl +++ b/src/Augmentor.jl @@ -1,6 +1,7 @@ __precompile__() module Augmentor +using ColorTypes using ImageCore using ImageTransformations using ImageFiltering @@ -17,6 +18,11 @@ using Base.PermutedDimsArrays: PermutedDimsArray export + SplitChannels, + CombineChannels, + PermuteDims, + Reshape, + Rotate90, Rotate180, Rotate270, @@ -53,6 +59,8 @@ include("utils.jl") include("types.jl") include("operation.jl") +include("operations/channels.jl") + include("operations/noop.jl") include("operations/cache.jl") include("operations/rotation.jl") diff --git a/src/operations/channels.jl b/src/operations/channels.jl new file mode 100644 index 00000000..c673a2fe --- /dev/null +++ b/src/operations/channels.jl @@ -0,0 +1,42 @@ +immutable SplitChannels <: Operation end + +@inline supports_eager(::Type{SplitChannels}) = false +@inline supports_lazy(::Type{SplitChannels}) = true + +applylazy(op::SplitChannels, img) = channelview(img) + +# -------------------------------------------------------------------- + +immutable CombineChannels{T<:Colorant} <: Operation + colortype::Type{T} +end + +@inline supports_eager{T<:CombineChannels}(::Type{T}) = false +@inline supports_lazy{T<:CombineChannels}(::Type{T}) = true + +applylazy(op::CombineChannels, img) = colorview(op.colortype, img) + +# -------------------------------------------------------------------- + +immutable PermuteDims{N,perm,iperm} <: Operation end +PermuteDims{N}(perm::Vararg{Int,N}) = PermuteDims{N,perm,invperm(perm)}() + +@inline supports_eager{T<:PermuteDims}(::Type{T}) = true +@inline supports_lazy{T<:PermuteDims}(::Type{T}) = true + +applyeager{N,perm}(op::PermuteDims{N,perm}, img) = permutedims(img, perm) +function applylazy{T,N,perm,iperm}(op::PermuteDims{N,perm,iperm}, img::AbstractArray{T,N}) + PermutedDimsArray{T,N,perm,iperm,typeof(img)}(img) +end + +# -------------------------------------------------------------------- + +immutable Reshape{T<:Tuple{Vararg{Int}}} <: Operation + dims::T +end +Reshape(dims::Int...) = Reshape(dims) + +@inline supports_eager{T<:Reshape}(::Type{T}) = false +@inline supports_lazy{T<:Reshape}(::Type{T}) = true + +applylazy(op::Reshape, img) = reshape(img, op.dims) diff --git a/src/utils.jl b/src/utils.jl index 29b47f2d..7093c67e 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -22,7 +22,9 @@ end @inline _plain_array(A::Array) = A @inline plain_array(A::OffsetArray) = parent(A) @inline plain_array(A::Array) = A -@inline plain_array(A::AbstractArray) = _plain_array(copy(A)) # avoid recursion +# avoid recursion +@inline plain_array(A::SubArray) = _plain_array(copy(A)) +@inline plain_array(A::AbstractArray) = _plain_array(collect(A)) # -------------------------------------------------------------------- From 9918fdc5b9ce7f1fc70f7a46c5abc9dd3aed0751 Mon Sep 17 00:00:00 2001 From: Christof Stocker Date: Tue, 13 Jun 2017 19:15:38 +0200 Subject: [PATCH 2/5] add docstring for array handling ops --- src/Augmentor.jl | 1 + src/operations/channels.jl | 217 +++++++++++++++++++++++++++++++++++++ 2 files changed, 218 insertions(+) diff --git a/src/Augmentor.jl b/src/Augmentor.jl index f8552346..f8fc5e82 100644 --- a/src/Augmentor.jl +++ b/src/Augmentor.jl @@ -2,6 +2,7 @@ __precompile__() module Augmentor using ColorTypes +using ColorTypes: AbstractGray using ImageCore using ImageTransformations using ImageFiltering diff --git a/src/operations/channels.jl b/src/operations/channels.jl index c673a2fe..fc9dd0b2 100644 --- a/src/operations/channels.jl +++ b/src/operations/channels.jl @@ -1,12 +1,118 @@ +""" + SplitChannels <: Augmentor.Operation + +Description +-------------- + +Splits out the color channels of the given image using the +function `ImageCore.channelview`. This will effectively create a +new array dimension for the colors in the front. In contrast to +`ImageCore.channelview` it will also result in a new dimension +for Gray images. + +This operation is mainly useful at the end of a pipeline in +combination with [`PermuteDims`](@ref) in order to prepare the +image for the training algorithm, which often requires the color +channels to be separate. + +Usage +-------------- + + SplitChannels() + +Examples +-------------- + +```julia +julia> using Augmentor + +julia> img = testpattern() +300×400 Array{RGBA{N0f8},2}: +[...] + +julia> augment(img, SplitChannels()) +4×300×400 Array{N0f8,3}: +[...] + +julia> augment(img, SplitChannels() |> PermuteDims(3,2,1)) +400×300×4 Array{N0f8,3}: +[...] +``` + +see also +-------------- + +[`PermuteDims`](@ref), [`CombineChannels`](@ref), [`augment`](@ref) +""" immutable SplitChannels <: Operation end @inline supports_eager(::Type{SplitChannels}) = false @inline supports_lazy(::Type{SplitChannels}) = true applylazy(op::SplitChannels, img) = channelview(img) +function applylazy{T<:AbstractGray}(op::SplitChannels, img::AbstractArray{T}) + ns = (1, map(length, indices(img))...) + reshape(channelview(img), ns) +end # -------------------------------------------------------------------- +""" + CombineChannels <: Augmentor.Operation + +Description +-------------- + +Combines the first dimension of a given array into a colorant of +type `colortype` using the function `ImageCore.colorview`. The +main difference is that a separate color channel is also expected +for Gray images. + +The shape of the input image has to be appropriate for the given +`colortype`, which also means that the separated color channel +has to be the first dimension of the array. See +[`PermuteDims`](@ref) if that is not the case. + +Usage +-------------- + + CombineChannels(colortype) + +Arguments +-------------- + +- **`colortype`** : The color type of the resulting image. Must + be a subtype of `ColorTypes.Colorant` and match the color + channel of the given image. + +Examples +-------------- + +```julia +julia> using Augmentor, Colors + +julia> A = rand(3, 10, 10) # three color channels +3×10×10 Array{Float64,3}: +[...] + +julia> augment(A, CombineChannels(RGB)) +10×10 Array{RGB{Float64},2}: +[...] + +julia> B = rand(1, 10, 10) # singleton color channel +1×10×10 Array{Float64,3}: +[...] + +julia> augment(B, CombineChannels(Gray)) +10×10 Array{Gray{Float64},2}: +[...] +``` + +see also +-------------- + +[`SplitChannels`](@ref), [`PermuteDims`](@ref), [`augment`](@ref) +""" immutable CombineChannels{T<:Colorant} <: Operation colortype::Type{T} end @@ -15,10 +121,76 @@ end @inline supports_lazy{T<:CombineChannels}(::Type{T}) = true applylazy(op::CombineChannels, img) = colorview(op.colortype, img) +function applylazy{T<:AbstractGray}(op::CombineChannels{T}, img) + length(indices(img,1)) == 1 || throw(ArgumentError("The given image must have a singleton colorchannel in the first dimension in order to combine the channels to a AbstractGray colorant")) + ns = Base.tail(map(length, indices(img))) + colorview(op.colortype, reshape(img, ns)) +end # -------------------------------------------------------------------- +""" + PermuteDims <: Augmentor.Operation + +Description +-------------- + +Permute the dimensions of the given array with the predefined +permutation `perm`. This operation is particularly useful if the +order of the dimensions needs to be different than the default +julian layout. + +Augmentor expects the given images to be in vertical-major layout +for which the colors are encoded in the element type itself. Many +deep learning frameworks however require their input in a +different order. For example it is not untypical that the color +channels are expected to be encoded in the third dimension. + +Usage +-------------- + + PermuteDims(perm) + + PermuteDims(perm...) + +Arguments +-------------- + +- **`perm`** : The concrete dimension permutation that should be + used. Has to be specified as a `Vararg{Int}` or as a `NTuple` + of `Int`. The length of `perm` has to match the number of + dimensions of the expected input image to that operation. + +Examples +-------------- + +```julia +julia> using Augmentor, Colors + +julia> A = rand(10, 5, 3) # width=10, height=5, and 3 color channels +10×5×3 Array{Float64,3}: +[...] + +julia> img = augment(A, PermuteDims(3,2,1) |> CombineChannels(RGB)) +5×10 Array{RGB{Float64},2}: +[...] + +julia> img2 = testpattern() +300×400 Array{RGBA{N0f8},2}: +[...] + +julia> B = augment(img2, SplitChannels() |> PermuteDims(3,2,1)) +400×300×4 Array{N0f8,3}: +[...] +``` + +see also +-------------- + +[`SplitChannels`](@ref), [`CombineChannels`](@ref), [`augment`](@ref) +""" immutable PermuteDims{N,perm,iperm} <: Operation end +PermuteDims{N}(perm::NTuple{N,Int}) = PermuteDims{N,perm,invperm(perm)}() PermuteDims{N}(perm::Vararg{Int,N}) = PermuteDims{N,perm,invperm(perm)}() @inline supports_eager{T<:PermuteDims}(::Type{T}) = true @@ -31,6 +203,51 @@ end # -------------------------------------------------------------------- +""" + Reshape <: Augmentor.Operation + +Description +-------------- + +Reinterpret the shape of the given array of numbers or colorants. +This is useful for example to create singleton dimensions that +deep learning frameworks may need for colorless images, or for +converting an image to a feature vector and vice versa. + +Usage +-------------- + + PermuteDims(dims) + + PermuteDims(dims...) + +Arguments +-------------- + +- **`dims`** : The new sizes for each dimension of the output + image. Has to be specified as a `Vararg{Int}` or as a + `NTuple` of `Int`. + +Examples +-------------- + +```julia +julia> using Augmentor, Colors + +julia> A = rand(10,10) +10×10 Array{Float64,2}: +[...] + +julia> augment(A, Reshape(10,10,1)) # add trailing singleton dimension +10×10×1 Array{Float64,3}: +[...] +``` + +see also +-------------- + +[`CombineChannels`](@ref), [`augment`](@ref) +""" immutable Reshape{T<:Tuple{Vararg{Int}}} <: Operation dims::T end From 77ffe7e6bdf0b61a92b8e284ba751662ec80a9bb Mon Sep 17 00:00:00 2001 From: Christof Stocker Date: Tue, 13 Jun 2017 19:19:12 +0200 Subject: [PATCH 3/5] update plain_array tests --- test/tst_utils.jl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/tst_utils.jl b/test/tst_utils.jl index b10c5d96..7cc61ae6 100644 --- a/test/tst_utils.jl +++ b/test/tst_utils.jl @@ -10,9 +10,14 @@ end @testset "plain_array" begin A = [1 2 3; 4 5 6; 7 8 9] As = sparse(A) + Ar = reshape(As, 3, 3, 1) Ast = @SMatrix [1 2 3; 4 5 6; 7 8 9] - @test_throws MethodError Augmentor.plain_array(As) - @test_throws MethodError Augmentor.plain_array(Ast) + @test @inferred(Augmentor.plain_array(As)) == A + @test typeof(Augmentor.plain_array(As)) == typeof(A) + @test @inferred(Augmentor.plain_array(Ar)) == reshape(A,3,3,1) + @test typeof(Augmentor.plain_array(Ar)) <: Array + @test @inferred(Augmentor.plain_array(Ast)) == A + @test typeof(Augmentor.plain_array(Ast)) == typeof(A) @test @inferred(Augmentor.plain_array(A)) === A @test @inferred(Augmentor.plain_array(OffsetArray(A, (-2,-1)))) === A v = view(A, 2:3, 1:2) From 07d8f828176e6fe76d76f3a463740b46e51cbd85 Mon Sep 17 00:00:00 2001 From: Christof Stocker Date: Wed, 14 Jun 2017 21:58:19 +0200 Subject: [PATCH 4/5] add tests for array handling ops --- src/operations/cache.jl | 2 +- src/operations/channels.jl | 80 +++++++++- src/operations/either.jl | 4 +- test/operations/tst_channels.jl | 253 ++++++++++++++++++++++++++++++++ test/operations/tst_either.jl | 9 +- test/runtests.jl | 2 + 6 files changed, 336 insertions(+), 14 deletions(-) create mode 100644 test/operations/tst_channels.jl diff --git a/src/operations/cache.jl b/src/operations/cache.jl index dae2fda3..99fa4793 100644 --- a/src/operations/cache.jl +++ b/src/operations/cache.jl @@ -55,7 +55,7 @@ immutable CacheImage <: ImageOperation end applyeager(op::CacheImage, img::Array) = img applyeager(op::CacheImage, img::OffsetArray) = img -applyeager(op::CacheImage, img) = copy(img) +applyeager(op::CacheImage, img) = copy(img) # FIXME: collect function showconstruction(io::IO, op::CacheImage) print(io, typeof(op).name.name, "()") diff --git a/src/operations/channels.jl b/src/operations/channels.jl index fc9dd0b2..f2000c38 100644 --- a/src/operations/channels.jl +++ b/src/operations/channels.jl @@ -49,12 +49,25 @@ immutable SplitChannels <: Operation end @inline supports_eager(::Type{SplitChannels}) = false @inline supports_lazy(::Type{SplitChannels}) = true -applylazy(op::SplitChannels, img) = channelview(img) +applylazy{T<:Colorant}(op::SplitChannels, img::AbstractArray{T}) = channelview(img) function applylazy{T<:AbstractGray}(op::SplitChannels, img::AbstractArray{T}) ns = (1, map(length, indices(img))...) reshape(channelview(img), ns) end +function showconstruction(io::IO, op::SplitChannels) + print(io, typeof(op).name.name, "()") +end + +function Base.show(io::IO, op::SplitChannels) + if get(io, :compact, false) + print(io, "Split colorant into its color channels") + else + print(io, "Augmentor.") + showconstruction(io, op) + end +end + # -------------------------------------------------------------------- """ @@ -120,13 +133,32 @@ end @inline supports_eager{T<:CombineChannels}(::Type{T}) = false @inline supports_lazy{T<:CombineChannels}(::Type{T}) = true -applylazy(op::CombineChannels, img) = colorview(op.colortype, img) -function applylazy{T<:AbstractGray}(op::CombineChannels{T}, img) +function applylazy{S<:Number}(op::CombineChannels, img::AbstractArray{S}) + colorview(op.colortype, img) +end + +function applylazy{T<:AbstractGray,S<:Number}(op::CombineChannels{T}, img::AbstractArray{S}) length(indices(img,1)) == 1 || throw(ArgumentError("The given image must have a singleton colorchannel in the first dimension in order to combine the channels to a AbstractGray colorant")) ns = Base.tail(map(length, indices(img))) colorview(op.colortype, reshape(img, ns)) end +function showconstruction(io::IO, op::CombineChannels) + print(io, typeof(op).name.name, '(') + ImageCore.showcoloranttype(io, op.colortype) + print(io, ')') +end + +function Base.show(io::IO, op::CombineChannels) + if get(io, :compact, false) + print(io, "Combine color channels into colorant ") + ImageCore.showcoloranttype(io, op.colortype) + else + print(io, "Augmentor.") + showconstruction(io, op) + end +end + # -------------------------------------------------------------------- """ @@ -190,17 +222,31 @@ see also [`SplitChannels`](@ref), [`CombineChannels`](@ref), [`augment`](@ref) """ immutable PermuteDims{N,perm,iperm} <: Operation end +PermuteDims() = throw(MethodError(PermuteDims, ())) +PermuteDims(perm::Tuple{}) = throw(MethodError(PermuteDims, (perm,))) PermuteDims{N}(perm::NTuple{N,Int}) = PermuteDims{N,perm,invperm(perm)}() PermuteDims{N}(perm::Vararg{Int,N}) = PermuteDims{N,perm,invperm(perm)}() @inline supports_eager{T<:PermuteDims}(::Type{T}) = true @inline supports_lazy{T<:PermuteDims}(::Type{T}) = true -applyeager{N,perm}(op::PermuteDims{N,perm}, img) = permutedims(img, perm) +applyeager{T,N,perm}(op::PermuteDims{N,perm}, img::AbstractArray{T,N}) = permutedims(img, perm) function applylazy{T,N,perm,iperm}(op::PermuteDims{N,perm,iperm}, img::AbstractArray{T,N}) PermutedDimsArray{T,N,perm,iperm,typeof(img)}(img) end +function showconstruction{N,perm}(io::IO, op::PermuteDims{N,perm}) + print(io, typeof(op).name.name, '(', join(map(string, perm),", "), ')') +end + +function Base.show{N,perm}(io::IO, op::PermuteDims{N,perm}) + if get(io, :compact, false) + print(io, "Permute dimension order to ", perm) + else + print(io, typeof(op).name, '(', perm, ')') + end +end + # -------------------------------------------------------------------- """ @@ -217,9 +263,9 @@ converting an image to a feature vector and vice versa. Usage -------------- - PermuteDims(dims) + Reshape(dims) - PermuteDims(dims...) + Reshape(dims...) Arguments -------------- @@ -248,12 +294,30 @@ see also [`CombineChannels`](@ref), [`augment`](@ref) """ -immutable Reshape{T<:Tuple{Vararg{Int}}} <: Operation - dims::T +immutable Reshape{N} <: Operation + dims::NTuple{N,Int} end +Reshape() = throw(MethodError(Reshape, ())) +Reshape(dims::Tuple{}) = throw(MethodError(Reshape, (dims,))) Reshape(dims::Int...) = Reshape(dims) @inline supports_eager{T<:Reshape}(::Type{T}) = false @inline supports_lazy{T<:Reshape}(::Type{T}) = true applylazy(op::Reshape, img) = reshape(img, op.dims) + +function showconstruction(io::IO, op::Reshape) + print(io, typeof(op).name.name, '(', join(map(string, op.dims),", "), ')') +end + +function Base.show{N}(io::IO, op::Reshape{N}) + if get(io, :compact, false) + if N == 1 + print(io, "Reshape array to ", first(op.dims), "-element vector") + else + print(io, "Reshape array to ", join(op.dims,"×")) + end + else + print(io, typeof(op), '(', op.dims, ')') + end +end diff --git a/src/operations/either.jl b/src/operations/either.jl index 932f4804..01c926c0 100644 --- a/src/operations/either.jl +++ b/src/operations/either.jl @@ -113,9 +113,9 @@ function Either(op::ImageOperation, p::Real = .5) end -Base.:*{T<:Number,O<:ImageOperation}(op1::Pair{T,O}, ops::Pair...) = +Base.:*{T<:Number,O<:Operation}(op1::Pair{T,O}, ops::Pair...) = Either(op1, ops...) -Base.:*(op1::ImageOperation, ops::ImageOperation...) = Either((op1, ops...)) +Base.:*(op1::Operation, ops::Operation...) = Either((op1, ops...)) @inline supports_permute{N,T}(::Type{Either{N,T}}) = all(map(supports_permute, T.types)) @inline supports_view{N,T}(::Type{Either{N,T}}) = all(map(supports_view, T.types)) diff --git a/test/operations/tst_channels.jl b/test/operations/tst_channels.jl new file mode 100644 index 00000000..787c45d8 --- /dev/null +++ b/test/operations/tst_channels.jl @@ -0,0 +1,253 @@ +@testset "SplitChannels" begin + @test (SplitChannels <: Augmentor.AffineOperation) == false + @test (SplitChannels <: Augmentor.ImageOperation) == false + @test (SplitChannels <: Augmentor.Operation) == true + + @testset "constructor" begin + @test typeof(@inferred(SplitChannels())) <: SplitChannels <: Augmentor.Operation + @test str_show(SplitChannels()) == "Augmentor.SplitChannels()" + @test str_showconst(SplitChannels()) == "SplitChannels()" + @test str_showcompact(SplitChannels()) == "Split colorant into its color channels" + end + @testset "eager" begin + @test @inferred(Augmentor.supports_eager(SplitChannels)) === false + @test_throws MethodError Augmentor.applyeager(SplitChannels(), rand(2,2)) + for img in (Augmentor.prepareaffine(rect), rect, OffsetArray(rect, -2, -1), view(rect, IdentityRange(1:2), IdentityRange(1:3))) + @test @inferred(Augmentor.applyeager(SplitChannels(), img)) == reshape(rect, 1, 2, 3) + @test typeof(Augmentor.applyeager(SplitChannels(), img)) <: Array{N0f8} + end + for img in (Augmentor.prepareaffine(rgb_rect), rgb_rect, OffsetArray(rgb_rect, -2, -1), view(rgb_rect, IdentityRange(1:2), IdentityRange(1:3))) + @test @inferred(Augmentor.applyeager(SplitChannels(), img)) == channelview(rgb_rect) + @test typeof(Augmentor.applyeager(SplitChannels(), img)) <: Array{N0f8} + end + end + @testset "affine" begin + @test @inferred(Augmentor.isaffine(SplitChannels)) === false + @test @inferred(Augmentor.supports_affine(SplitChannels)) === false + end + @testset "lazy" begin + @test @inferred(Augmentor.supports_lazy(SplitChannels)) === true + @test @inferred(Augmentor.supports_lazy(typeof(SplitChannels()))) === true + @test_throws MethodError Augmentor.applylazy(SplitChannels(), rand(2,2)) + for img in (Augmentor.prepareaffine(rect), rect, OffsetArray(rect, -2, -1), view(rect, IdentityRange(1:2), IdentityRange(1:3))) + @test @inferred(Augmentor.applylazy(SplitChannels(), img)) == reshape(rect, 1, 2, 3) + if typeof(img) <: Array + @test typeof(Augmentor.applylazy(SplitChannels(), img)) <: Array{N0f8} + else + @test typeof(Augmentor.applylazy(SplitChannels(), img)) <: Base.ReshapedArray + end + end + for img in (Augmentor.prepareaffine(rgb_rect), rgb_rect, OffsetArray(rgb_rect, -2, -1), view(rgb_rect, IdentityRange(1:2), IdentityRange(1:3))) + @test @inferred(Augmentor.applylazy(SplitChannels(), img)) == channelview(img) + if typeof(img) <: Array + @test typeof(Augmentor.applylazy(SplitChannels(), img)) <: Array{N0f8} + else + @test typeof(Augmentor.applylazy(SplitChannels(), img)) <: ChannelView + end + end + end + @testset "view" begin + @test @inferred(Augmentor.supports_view(SplitChannels)) === false + end + @testset "stepview" begin + @test @inferred(Augmentor.supports_stepview(SplitChannels)) === false + end + @testset "permute" begin + @test @inferred(Augmentor.supports_permute(SplitChannels)) === false + end +end + +# -------------------------------------------------------------------- + +@testset "CombineChannels" begin + @test (CombineChannels <: Augmentor.AffineOperation) == false + @test (CombineChannels <: Augmentor.ImageOperation) == false + @test (CombineChannels <: Augmentor.Operation) == true + + @testset "constructor" begin + @test_throws MethodError CombineChannels() + @test_throws MethodError CombineChannels(Float64) + @test typeof(@inferred(CombineChannels(RGB))) <: CombineChannels <: Augmentor.Operation + @test typeof(@inferred(CombineChannels(RGB{N0f8}))) <: CombineChannels <: Augmentor.Operation + @test str_show(CombineChannels(RGB)) == "Augmentor.CombineChannels(RGB{Any})" + @test str_show(CombineChannels(Gray{N0f8})) == "Augmentor.CombineChannels(Gray{N0f8})" + @test str_showconst(CombineChannels(RGB{N0f8})) == "CombineChannels(RGB{N0f8})" + @test str_showcompact(CombineChannels(Gray)) == "Combine color channels into colorant Gray{Any}" + end + @testset "eager" begin + @test @inferred(Augmentor.supports_eager(CombineChannels)) === false + @test_throws MethodError Augmentor.applyeager(CombineChannels(RGB), rand(RGB{N0f8},4,4)) + @test_throws MethodError Augmentor.applyeager(CombineChannels(Gray), rand(Gray{N0f8},4,4)) + @test_throws ArgumentError Augmentor.applyeager(CombineChannels(Gray), rand(4,4)) + rect_split = reshape(channelview(rect), 1, 2, 3) + for img in (rect_split, OffsetArray(rect_split, 0, -2, -1), view(rect_split, 1:1, IdentityRange(1:2), IdentityRange(1:3))) + @test @inferred(Augmentor.applyeager(CombineChannels(Gray), img)) == rect + @test typeof(Augmentor.applyeager(CombineChannels(Gray), img)) <: Array{Gray{N0f8}} + @test @inferred(Augmentor.applyeager(CombineChannels(Gray{N0f8}), img)) == rect + @test typeof(Augmentor.applyeager(CombineChannels(Gray{N0f8}), img)) <: Array{Gray{N0f8}} + end + rgb_rect_split = channelview(rgb_rect) + for img in (rgb_rect_split, OffsetArray(rgb_rect_split, 0, -2, -1), view(rgb_rect_split, 1:3, IdentityRange(1:2), IdentityRange(1:3))) + @test @inferred(Augmentor.applyeager(CombineChannels(RGB), img)) == rgb_rect + @test typeof(Augmentor.applyeager(CombineChannels(RGB), img)) <: Array{RGB{N0f8}} + @test @inferred(Augmentor.applyeager(CombineChannels(RGB{N0f8}), img)) == rgb_rect + @test typeof(Augmentor.applyeager(CombineChannels(RGB{N0f8}), img)) <: Array{RGB{N0f8}} + end + end + @testset "affine" begin + @test @inferred(Augmentor.isaffine(CombineChannels)) === false + @test @inferred(Augmentor.supports_affine(CombineChannels)) === false + end + @testset "lazy" begin + @test @inferred(Augmentor.supports_lazy(CombineChannels)) === true + @test @inferred(Augmentor.supports_lazy(typeof(CombineChannels(Gray)))) === true + @test_throws MethodError Augmentor.applylazy(CombineChannels(RGB), rand(RGB{N0f8},4,4)) + @test_throws MethodError Augmentor.applylazy(CombineChannels(Gray), rand(Gray{N0f8},4,4)) + @test_throws ArgumentError Augmentor.applylazy(CombineChannels(Gray), rand(4,4)) + rect_split = reshape(channelview(rect), 1, 2, 3) + for img in (rect_split, OffsetArray(rect_split, 0, -2, -1), view(rect_split, 1:1, IdentityRange(1:2), IdentityRange(1:3))) + @test @inferred(Augmentor.applylazy(CombineChannels(Gray), img)) == rect + if typeof(img) <: Array + @test typeof(Augmentor.applylazy(CombineChannels(Gray), img)) <: Array{Gray{N0f8}} + else + @test typeof(Augmentor.applylazy(CombineChannels(Gray), img)) <: ColorView{Gray{N0f8}} + end + end + rgb_rect_split = channelview(rgb_rect) + for img in (rgb_rect_split, OffsetArray(rgb_rect_split, 0, -2, -1), view(rgb_rect_split, 1:3, IdentityRange(1:2), IdentityRange(1:3))) + @test @inferred(Augmentor.applylazy(CombineChannels(RGB), img)) == colorview(RGB, img) + if typeof(img) <: Array + @test typeof(Augmentor.applylazy(CombineChannels(RGB), img)) <: Array{RGB{N0f8}} + else + @test typeof(Augmentor.applylazy(CombineChannels(RGB), img)) <: ColorView{RGB{N0f8}} + end + end + end + @testset "view" begin + @test @inferred(Augmentor.supports_view(CombineChannels)) === false + end + @testset "stepview" begin + @test @inferred(Augmentor.supports_stepview(CombineChannels)) === false + end + @testset "permute" begin + @test @inferred(Augmentor.supports_permute(CombineChannels)) === false + end +end + +# -------------------------------------------------------------------- + +@testset "PermuteDims" begin + @test (PermuteDims <: Augmentor.AffineOperation) == false + @test (PermuteDims <: Augmentor.ImageOperation) == false + @test (PermuteDims <: Augmentor.Operation) == true + + @testset "constructor" begin + @test_throws MethodError PermuteDims() + @test_throws MethodError PermuteDims(()) + @test_throws MethodError PermuteDims(Float64) + @test_throws ArgumentError PermuteDims((2,2)) + @test_throws ArgumentError PermuteDims(2,2) + @test typeof(PermuteDims(1,2)) <: PermuteDims{2} <: Augmentor.Operation + @test typeof(PermuteDims((1,2))) <: PermuteDims{2} <: Augmentor.Operation + @test typeof(PermuteDims(1)) <: PermuteDims{1} <: Augmentor.Operation + @test typeof(PermuteDims((1,))) <: PermuteDims{1} <: Augmentor.Operation + @test typeof(PermuteDims((1,2))) <: PermuteDims{2} <: Augmentor.Operation + @test typeof(PermuteDims((3,1,2))) <: PermuteDims{3} <: Augmentor.Operation + @test str_show(PermuteDims((1,))) == "Augmentor.PermuteDims((1,))" + @test str_show(PermuteDims((1,2))) == "Augmentor.PermuteDims((1,$(SPACE)2))" + @test str_showconst(PermuteDims((1,3,2))) == "PermuteDims(1, 3, 2)" + @test str_showcompact(PermuteDims((3,2,1))) == "Permute dimension order to (3,$(SPACE)2,$(SPACE)1)" + end + @testset "eager" begin + @test @inferred(Augmentor.supports_eager(PermuteDims)) === true + @test @inferred(Augmentor.supports_eager(typeof(PermuteDims(2,1)))) === true + for img in (Augmentor.prepareaffine(rect), rect, OffsetArray(rect, -2, -1), view(rect, IdentityRange(1:2), IdentityRange(1:3))) + @test_throws MethodError Augmentor.applyeager(PermuteDims(3,2,1), img) + @test @inferred(Augmentor.applyeager(PermuteDims(2,1), img)) == permutedims(img, (2,1)) + @test typeof(Augmentor.applyeager(PermuteDims(2,1), img)) == typeof(permutedims(img, (2,1))) + end + end + @testset "affine" begin + @test @inferred(Augmentor.isaffine(PermuteDims)) === false + @test @inferred(Augmentor.supports_affine(PermuteDims)) === false + end + @testset "lazy" begin + @test @inferred(Augmentor.supports_lazy(PermuteDims)) === true + @test @inferred(Augmentor.supports_lazy(typeof(PermuteDims(2,1)))) === true + for img in (Augmentor.prepareaffine(rect), rect, OffsetArray(rect, -2, -1), view(rect, IdentityRange(1:2), IdentityRange(1:3))) + @test_throws MethodError Augmentor.applylazy(PermuteDims(3,2,1), img) + @test @inferred(Augmentor.applylazy(PermuteDims(2,1), img)) === permuteddimsview(img, (2,1)) + end + end + @testset "view" begin + @test @inferred(Augmentor.supports_view(PermuteDims)) === false + end + @testset "stepview" begin + @test @inferred(Augmentor.supports_stepview(PermuteDims)) === false + end + @testset "permute" begin + @test @inferred(Augmentor.supports_permute(PermuteDims)) === false + end +end + +# -------------------------------------------------------------------- + +@testset "Reshape" begin + @test (Reshape <: Augmentor.AffineOperation) == false + @test (Reshape <: Augmentor.ImageOperation) == false + @test (Reshape <: Augmentor.Operation) == true + + @testset "constructor" begin + @test_throws MethodError Reshape() + @test_throws MethodError Reshape(()) + @test_throws MethodError Reshape(Float64) + @test_throws MethodError Reshape(1.,2.) + @test typeof(Reshape(2,2)) <: Reshape{2} <: Augmentor.Operation + @test typeof(Reshape(1,2)) <: Reshape{2} <: Augmentor.Operation + @test typeof(Reshape((1,2))) <: Reshape{2} <: Augmentor.Operation + @test typeof(Reshape(1)) <: Reshape{1} <: Augmentor.Operation + @test typeof(Reshape((1,))) <: Reshape{1} <: Augmentor.Operation + @test typeof(Reshape((1,2))) <: Reshape{2} <: Augmentor.Operation + @test typeof(Reshape((3,1,2))) <: Reshape{3} <: Augmentor.Operation + @test str_show(Reshape((1,))) == "Augmentor.Reshape{1}((1,))" + @test str_show(Reshape((1,2))) == "Augmentor.Reshape{2}((1,$(SPACE)2))" + @test str_showconst(Reshape((1,3,2))) == "Reshape(1, 3, 2)" + @test str_showcompact(Reshape((3,2,1))) == "Reshape array to 3×2×1" + @test str_showcompact(Reshape(10)) == "Reshape array to 10-element vector" + end + @testset "eager" begin + @test @inferred(Augmentor.supports_eager(Reshape)) === false + @test @inferred(Augmentor.supports_eager(typeof(Reshape(2,1)))) === false + # FIXME: reintroduce Augmentor.prepareaffine(rect) in 0.6 + for img in (rect, OffsetArray(rect, -2, -1), view(rect, IdentityRange(1:2), IdentityRange(1:3))) + @test @inferred(Augmentor.applyeager(Reshape(3,2,1), img)) == reshape(img, (3,2,1)) + @test typeof(Augmentor.applyeager(Reshape(3,2,1), img)) <: Array{Gray{N0f8}} + end + end + @testset "affine" begin + @test @inferred(Augmentor.isaffine(Reshape)) === false + @test @inferred(Augmentor.supports_affine(Reshape)) === false + end + @testset "lazy" begin + @test @inferred(Augmentor.supports_lazy(Reshape)) === true + @test @inferred(Augmentor.supports_lazy(typeof(Reshape(2,1)))) === true + # FIXME: reintroduce Augmentor.prepareaffine(rect) in 0.6 + for img in (rect, OffsetArray(rect, -2, -1), view(rect, IdentityRange(1:2), IdentityRange(1:3))) + @test @inferred(Augmentor.applylazy(Reshape(3,2,1), img)) == reshape(img, (3,2,1)) + if typeof(img) <: Array + @test typeof(Augmentor.applylazy(Reshape(3,2,1), img)) <: Array{Gray{N0f8}} + else + @test typeof(Augmentor.applylazy(Reshape(3,2,1), img)) <: Base.ReshapedArray + end + end + end + @testset "view" begin + @test @inferred(Augmentor.supports_view(Reshape)) === false + end + @testset "stepview" begin + @test @inferred(Augmentor.supports_stepview(Reshape)) === false + end + @testset "permute" begin + @test @inferred(Augmentor.supports_permute(Reshape)) === false + end +end diff --git a/test/operations/tst_either.jl b/test/operations/tst_either.jl index 6fe39d2e..78ee085c 100644 --- a/test/operations/tst_either.jl +++ b/test/operations/tst_either.jl @@ -7,6 +7,9 @@ @test_throws ArgumentError Either((NoOp(),),(0,)) @test_throws ArgumentError Either((NoOp(),),(-1,)) @test_throws MethodError Either((NoOp(),),(1,1)) + @test_throws MethodError Either(SplitChannels(), NoOp()) + @test_throws MethodError SplitChannels() * NoOp() + @test_throws MethodError SplitChannels() * SplitChannels() let op = @inferred Either(Rotate90(), 0.3) @test op.operations === (Rotate90(), NoOp()) @test op.chances === @SVector([0.3,0.7]) @@ -76,9 +79,9 @@ end - 4.5% chance to: No operation""" @test str_showcompact(Either((Rotate90(),Rotate270(),NoOp()), (0.155,0.8,0.045))) == "Either: (16%) Rotate 90 degree. (80%) Rotate 270 degree. (4%) No operation." - @test str_showconst(Either(Rotate90(), Rotate270(), NoOp())) == "Rotate90() * Rotate270() * NoOp()" - @test str_showconst(Either((Rotate90(), Rotate270(), NoOp()),(1,1,2))) == "(0.25=>Rotate90()) * (0.25=>Rotate270()) * (0.5=>NoOp())" - @test str_showconst(Either((Rotate90(), NoOp()),(1,2))) == "(0.333=>Rotate90()) * (0.667=>NoOp())" + @test str_showconst(Either(Rotate90(), Rotate270(), NoOp())) == "Rotate90() * Rotate270() * NoOp()" + @test str_showconst(Either((Rotate90(), Rotate270(), NoOp()),(1,1,2))) == "(0.25=>Rotate90()) * (0.25=>Rotate270()) * (0.5=>NoOp())" + @test str_showconst(Either((Rotate90(), NoOp()),(1,2))) == "(0.333=>Rotate90()) * (0.667=>NoOp())" end @testset "eager" begin diff --git a/test/runtests.jl b/test/runtests.jl index 2f397f5c..b87ca40d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -38,9 +38,11 @@ square = Gray{N0f8}[0.1 0.2 0.3; 0.4 0.5 0.6; 0.7 0.6 0.9] square2 = rand(Gray{N0f8}, 4, 4) rect = Gray{N0f8}[0.1 0.2 0.3; 0.4 0.5 0.6] checkers = Gray{N0f8}[1 0 1 0 1; 0 1 0 1 0; 1 0 1 0 1] +rgb_rect = rand(RGB{N0f8}, 2, 3) tests = [ "tst_utils.jl", + "operations/tst_channels.jl", "operations/tst_noop.jl", "operations/tst_cache.jl", "operations/tst_rotation.jl", From 403787376bcc58f304c50fe7b4c781ac96bb3b2f Mon Sep 17 00:00:00 2001 From: Christof Stocker Date: Thu, 15 Jun 2017 13:47:18 +0200 Subject: [PATCH 5/5] add documentation for array handling ops --- NEWS.md | 12 ++- README.md | 6 +- docs/usersguide/operations.rst | 129 +++++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index 765da7d7..2af7427d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,14 @@ New operations: -- `CropRatio`: crop to the specified aspect ratio around the center. +- `CropRatio`: Crop to the specified aspect ratio around the center. -- `RCropRatio`: crop random window with the specified aspect ratio. +- `RCropRatio`: Crop random window with the specified aspect ratio. + +- `SplitChannels`: Separate the color channels into a dedicated array dimension. + +- `CombineChannels`: Collapse the first dimension into a specific colorant. + +- `PermuteDims`: Reorganize the array dimensions into a specific order. + +- `Reshape`: Change or reinterpret the shape of the array. diff --git a/README.md b/README.md index 69beff7c..8186e16a 100644 --- a/README.md +++ b/README.md @@ -202,7 +202,7 @@ look at the corresponding section of the [documentation](http://augmentorjl.readthedocs.io/en/latest/usersguide/operations.html). | Category | Operation | Description -|--------------:|:--------------------|:----------------------------------------------------- +|--------------:|:--------------------|:----------------------------------------------------------------- | *Mirroring:* | `FlipX` | Reverse the order of each pixel row. | | `FlipY` | Reverse the order of each pixel column. | *Rotating:* | `Rotate90` | Rotate upwards 90 degree. @@ -220,6 +220,10 @@ look at the corresponding section of the | | `CropSize` | Crop area around the center with specified size. | | `CropRatio` | Crop to specified aspect ratio. | | `RCropRatio` | Crop random window of specified aspect ratio. +| *Layout:* | `SplitChannels` | Separate the color channels into a dedicated array dimension. +| | `CombineChannels` | Collapse the first dimension into a specific colorant. +| | `PermuteDims` | Reorganize the array dimensions into a specific order. +| | `Reshape` | Change or reinterpret the shape of the array. | *Utilities:* | `NoOp` | Identity function. Pass image along unchanged. | | `CacheImage` | Buffer the current image into (preallocated) memory. | | `Either` | Apply one of the given operations at random. diff --git a/docs/usersguide/operations.rst b/docs/usersguide/operations.rst index b6cbb0b3..ac2a037c 100644 --- a/docs/usersguide/operations.rst +++ b/docs/usersguide/operations.rst @@ -21,6 +21,9 @@ functionality. | Cropping | :class:`Crop` :class:`CropNative` :class:`CropSize` :class:`CropRatio` | | | :class:`RCropRatio` | +-----------------------+----------------------------------------------------------------------------+ +| Information Layout | :class:`SplitChannels` :class:`CombineChannels` :class:`PermuteDims` | +| | :class:`Reshape` | ++-----------------------+----------------------------------------------------------------------------+ | Utility Operations | :class:`NoOp` :class:`CacheImage` :class:`Either` | +-----------------------+----------------------------------------------------------------------------+ @@ -551,6 +554,132 @@ Resizing | .. image:: https://raw.githubusercontent.com/JuliaML/FileStorage/master/Augmentor/testpattern_small.png | .. image:: https://raw.githubusercontent.com/JuliaML/FileStorage/master/Augmentor/operations/Resize.png | +---------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------+ +Information Layout +-------------------- + +It is not uncommon that machine learning frameworks require the +data in a specific form and layout. For example many deep +learning frameworks expect the colorchannel of the images to be +encoded in the third dimension of a 4-dimensional array. + +Augmentor allows to convert from (and to) these different layouts +using special operations that are mainly useful in the beginning +or end of a augmentation pipeline. + +Color Channels +******************** + +.. class:: SplitChannels + + Separate the color channels of the given image into a + dedicated array dimension. This will effectively create a new + array dimension for the colors as the first dimension. In the + case of greyscale images a singleton dimension will be created + + This operation is mainly useful at the end of a pipeline in + combination with :class:`PermuteDims` in order to prepare the + image for the training algorithm, which often requires the + color channels to be separate. + +.. code-block:: jlcon + + julia> op = SplitChannels() + Split colorant into its color channels + + julia> img = testpattern() + 300×400 Array{RGBA{N0f8},2}: + [...] + + julia> augment(img, op)) + 4×300×400 Array{N0f8,3}: + [...] + +.. class:: CombineChannels + + Combines the first dimension of a given array into a colorant + of the specified type ``colortype``. A separate color channel + is also expected for Gray images. + + The shape of the input image has to be appropriate for the + given ``colortype``, which also means that the separated color + channel has to be the first dimension of the array. Use + :class:`PermuteDims` and/or :class:`Reshape` if that is not + the case. + + This operation is mainly useful at the beginning of the + pipline, if the colorchannels of the input images are + separated. + +.. code-block:: jlcon + + julia> op = CombineChannels(RGB) + Combine color channels into colorant RGB{Any} + + julia> A = rand(3, 10, 10) # random 10x10 RGB image + 3×10×10 Array{Float64,3}: + [...] + + julia> augment(A, op) + 10×10 Array{RGB{Float64},2}: + [...] + +Array Shape +******************** + +.. class:: PermuteDims + + Permute the dimensions of the given array with the predefined + permutation ``perm``. This operation is particularly useful if + the order of the dimensions needs to be different than the + default "julian" layout. + + More concretely, Augmentor expects the given images to be in + vertical-major layout for which the colors are encoded in the + element type itself. Many deep learning frameworks however + require their input in a different order. For example it is + not untypical that the color channels are expected to be + encoded in the third dimension. + +.. code-block:: jlcon + + julia> op = PermuteDims(3,2,1) + Permute dimension order to (3,2,1) + + julia> img = testpattern() + 300×400 Array{RGBA{N0f8},2}: + [...] + + julia> augment(img, PermuteDims(2,1)) + 400×300 Array{RGBA{N0f8},2}: + [...] + +.. class:: Reshape + + Reinterpret the shape of the given array of numbers or + colorants. This is useful for example to create singleton + dimensions that deep learning frameworks may need for + colorless images, or for converting an image to a feature + vector and vice versa. + + Note that this operation has nothing to do with image + resizing, but instead is strictly concerned with changing + the shape of the array. + +.. code-block:: jlcon + + julia> Reshape(10,15) + Reshape array to 10×15 + + julia> op = Reshape(25) + Reshape array to 25-element vector + + julia> A = rand(5,5) + 5×5 Array{Float64,2}: + [...] + + julia> augment(A, op) + 25-element Array{Float64,1}: + [...] Utility Operations --------------------