diff --git a/stdlib/LinearAlgebra/src/adjtrans.jl b/stdlib/LinearAlgebra/src/adjtrans.jl index 6003339735fcf..f6d8b33eb7639 100644 --- a/stdlib/LinearAlgebra/src/adjtrans.jl +++ b/stdlib/LinearAlgebra/src/adjtrans.jl @@ -64,6 +64,41 @@ end Adjoint(A) = Adjoint{Base.promote_op(adjoint,eltype(A)),typeof(A)}(A) Transpose(A) = Transpose{Base.promote_op(transpose,eltype(A)),typeof(A)}(A) +""" + adj_or_trans(::AbstractArray) -> adjoint|transpose|identity + adj_or_trans(::Type{<:AbstractArray}) -> adjoint|transpose|identity + +Return [`adjoint`](@ref) from an `Adjoint` type or object and +[`transpose`](@ref) from a `Transpose` type or object. Otherwise, +return [`identity`](@ref). Note that `Adjoint` and `Transpose` have +to be the outer-most wrapper object for a non-`identity` function to be +returned. +""" +adj_or_trans(::T) where {T<:AbstractArray} = adj_or_trans(T) +adj_or_trans(::Type{<:AbstractArray}) = identity +adj_or_trans(::Type{<:Adjoint}) = adjoint +adj_or_trans(::Type{<:Transpose}) = transpose + +""" + inplace_adj_or_trans(::AbstractArray) -> adjoint!|transpose!|copyto! + inplace_adj_or_trans(::Type{<:AbstractArray}) -> adjoint!|transpose!|copyto! + +Return [`adjoint!`](@ref) from an `Adjoint` type or object and +[`transpose!`](@ref) from a `Transpose` type or object. Otherwise, +return [`copyto!`](@ref). Note that `Adjoint` and `Transpose` have +to be the outer-most wrapper object for a non-`identity` function to be +returned. +""" +inplace_adj_or_trans(::T) where {T <: AbstractArray} = inplace_adj_or_trans(T) +inplace_adj_or_trans(::Type{<:AbstractArray}) = copyto! +inplace_adj_or_trans(::Type{<:Adjoint}) = adjoint! +inplace_adj_or_trans(::Type{<:Transpose}) = transpose! + +adj_or_trans_char(::T) where {T<:AbstractArray} = adj_or_trans_char(T) +adj_or_trans_char(::Type{<:AbstractArray}) = 'N' +adj_or_trans_char(::Type{<:Adjoint}) = 'C' +adj_or_trans_char(::Type{<:Transpose}) = 'T' + Base.dataids(A::Union{Adjoint, Transpose}) = Base.dataids(A.parent) Base.unaliascopy(A::Union{Adjoint,Transpose}) = typeof(A)(Base.unaliascopy(A.parent)) diff --git a/stdlib/LinearAlgebra/src/bidiag.jl b/stdlib/LinearAlgebra/src/bidiag.jl index 90f5c03f7fcfb..d136bf351b134 100644 --- a/stdlib/LinearAlgebra/src/bidiag.jl +++ b/stdlib/LinearAlgebra/src/bidiag.jl @@ -743,17 +743,13 @@ function ldiv!(c::AbstractVecOrMat, A::Bidiagonal, b::AbstractVecOrMat) end return c end -ldiv!(A::Transpose{<:Any,<:Bidiagonal}, b::AbstractVecOrMat) = @inline ldiv!(b, A, b) -ldiv!(A::Adjoint{<:Any,<:Bidiagonal}, b::AbstractVecOrMat) = @inline ldiv!(b, A, b) -ldiv!(c::AbstractVecOrMat, A::Transpose{<:Any,<:Bidiagonal}, b::AbstractVecOrMat) = - (_rdiv!(transpose(c), transpose(b), transpose(A)); return c) -ldiv!(c::AbstractVecOrMat, A::Adjoint{<:Any,<:Bidiagonal}, b::AbstractVecOrMat) = - (_rdiv!(adjoint(c), adjoint(b), adjoint(A)); return c) +ldiv!(A::AdjOrTrans{<:Any,<:Bidiagonal}, b::AbstractVecOrMat) = @inline ldiv!(b, A, b) +ldiv!(c::AbstractVecOrMat, A::AdjOrTrans{<:Any,<:Bidiagonal}, b::AbstractVecOrMat) = + (t = adj_or_trans(A); _rdiv!(t(c), t(b), t(A)); return c) ### Generic promotion methods and fallbacks \(A::Bidiagonal, B::AbstractVecOrMat) = ldiv!(_initarray(\, eltype(A), eltype(B), B), A, B) -\(tA::Transpose{<:Any,<:Bidiagonal}, B::AbstractVecOrMat) = copy(tA) \ B -\(adjA::Adjoint{<:Any,<:Bidiagonal}, B::AbstractVecOrMat) = copy(adjA) \ B +\(xA::AdjOrTrans{<:Any,<:Bidiagonal}, B::AbstractVecOrMat) = copy(xA) \ B ### Triangular specializations function \(B::Bidiagonal, U::UpperTriangular) @@ -837,12 +833,9 @@ function _rdiv!(C::AbstractMatrix, A::AbstractMatrix, B::Bidiagonal) C end rdiv!(A::AbstractMatrix, B::Bidiagonal) = @inline _rdiv!(A, A, B) -rdiv!(A::AbstractMatrix, B::Adjoint{<:Any,<:Bidiagonal}) = @inline _rdiv!(A, A, B) -rdiv!(A::AbstractMatrix, B::Transpose{<:Any,<:Bidiagonal}) = @inline _rdiv!(A, A, B) -_rdiv!(C::AbstractMatrix, A::AbstractMatrix, B::Adjoint{<:Any,<:Bidiagonal}) = - (ldiv!(adjoint(C), adjoint(B), adjoint(A)); return C) -_rdiv!(C::AbstractMatrix, A::AbstractMatrix, B::Transpose{<:Any,<:Bidiagonal}) = - (ldiv!(transpose(C), transpose(B), transpose(A)); return C) +rdiv!(A::AbstractMatrix, B::AdjOrTrans{<:Any,<:Bidiagonal}) = @inline _rdiv!(A, A, B) +_rdiv!(C::AbstractMatrix, A::AbstractMatrix, B::AdjOrTrans{<:Any,<:Bidiagonal}) = + (t = adj_or_trans(B); ldiv!(t(C), t(B), t(A)); return C) /(A::AbstractMatrix, B::Bidiagonal) = _rdiv!(_initarray(/, eltype(A), eltype(B), A), A, B) diff --git a/stdlib/LinearAlgebra/src/diagonal.jl b/stdlib/LinearAlgebra/src/diagonal.jl index ec1bca909ce7b..466501e72cd4f 100644 --- a/stdlib/LinearAlgebra/src/diagonal.jl +++ b/stdlib/LinearAlgebra/src/diagonal.jl @@ -396,16 +396,12 @@ end _muldiag!(out, D, V, alpha, beta) @inline mul!(out::AbstractMatrix, D::Diagonal, B::AbstractMatrix, alpha::Number, beta::Number) = _muldiag!(out, D, B, alpha, beta) -@inline mul!(out::AbstractMatrix, D::Diagonal, B::Adjoint{<:Any,<:AbstractVecOrMat}, - alpha::Number, beta::Number) = _muldiag!(out, D, B, alpha, beta) -@inline mul!(out::AbstractMatrix, D::Diagonal, B::Transpose{<:Any,<:AbstractVecOrMat}, +@inline mul!(out::AbstractMatrix, D::Diagonal, B::AdjOrTrans{<:Any,<:AbstractVecOrMat}, alpha::Number, beta::Number) = _muldiag!(out, D, B, alpha, beta) @inline mul!(out::AbstractMatrix, A::AbstractMatrix, D::Diagonal, alpha::Number, beta::Number) = _muldiag!(out, A, D, alpha, beta) -@inline mul!(out::AbstractMatrix, A::Adjoint{<:Any,<:AbstractVecOrMat}, D::Diagonal, - alpha::Number, beta::Number) = _muldiag!(out, A, D, alpha, beta) -@inline mul!(out::AbstractMatrix, A::Transpose{<:Any,<:AbstractVecOrMat}, D::Diagonal, +@inline mul!(out::AbstractMatrix, A::AdjOrTrans{<:Any,<:AbstractVecOrMat}, D::Diagonal, alpha::Number, beta::Number) = _muldiag!(out, A, D, alpha, beta) @inline mul!(C::Diagonal, Da::Diagonal, Db::Diagonal, alpha::Number, beta::Number) = _muldiag!(C, Da, Db, alpha, beta) diff --git a/stdlib/LinearAlgebra/src/matmul.jl b/stdlib/LinearAlgebra/src/matmul.jl index 6d00b950525e6..8b71db6dc328d 100644 --- a/stdlib/LinearAlgebra/src/matmul.jl +++ b/stdlib/LinearAlgebra/src/matmul.jl @@ -1,11 +1,16 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +# matmul.jl: Everything to do with dense matrix multiplication + # Matrix-matrix multiplication -AdjOrTransStridedMat{T} = Union{Adjoint{T, <:StridedMatrix}, Transpose{T, <:StridedMatrix}} -StridedMaybeAdjOrTransMat{T} = Union{StridedMatrix{T}, Adjoint{T, <:StridedMatrix}, Transpose{T, <:StridedMatrix}} +AdjOrTransStridedMat{T} = Union{Adjoint{<:Any, <:StridedMatrix{T}}, Transpose{<:Any, <:StridedMatrix{T}}} +StridedMaybeAdjOrTransMat{T} = Union{StridedMatrix{T}, Adjoint{<:Any, <:StridedMatrix{T}}, Transpose{<:Any, <:StridedMatrix{T}}} +StridedMaybeAdjOrTransVecOrMat{T} = Union{StridedVecOrMat{T}, AdjOrTrans{<:Any, <:StridedVecOrMat{T}}} -# matmul.jl: Everything to do with dense matrix multiplication +_parent(A) = A +_parent(A::Adjoint) = parent(A) +_parent(A::Transpose) = parent(A) matprod(x, y) = x*y + x*y @@ -46,14 +51,14 @@ function *(transx::Transpose{<:Any,<:StridedVector{T}}, y::StridedVector{T}) whe end # Matrix-vector multiplication -function (*)(A::StridedMatrix{T}, x::StridedVector{S}) where {T<:BlasFloat,S<:Real} +function (*)(A::StridedMaybeAdjOrTransMat{T}, x::StridedVector{S}) where {T<:BlasFloat,S<:Real} TS = promote_op(matprod, T, S) y = isconcretetype(TS) ? convert(AbstractVector{TS}, x) : x mul!(similar(x, TS, size(A,1)), A, y) end function (*)(A::AbstractMatrix{T}, x::AbstractVector{S}) where {T,S} TS = promote_op(matprod, T, S) - mul!(similar(x,TS,axes(A,1)),A,x) + mul!(similar(x, TS, axes(A,1)), A, x) end # these will throw a DimensionMismatch unless B has 1 row (or 1 col for transposed case): @@ -61,68 +66,33 @@ end (*)(a::AbstractVector, adjB::AdjointAbsMat) = reshape(a, length(a), 1) * adjB (*)(a::AbstractVector, B::AbstractMatrix) = reshape(a, length(a), 1) * B -@inline mul!(y::StridedVector{T}, A::StridedVecOrMat{T}, x::StridedVector{T}, - alpha::Number, beta::Number) where {T<:BlasFloat} = - gemv!(y, 'N', A, x, alpha, beta) - +@inline mul!(y::AbstractVector, A::AbstractVecOrMat, x::AbstractVector, + alpha::Number, beta::Number) = + generic_matvecmul!(y, adj_or_trans_char(A), _parent(A), x, MulAddMul(alpha, beta)) +# BLAS cases +@inline mul!(y::StridedVector{T}, A::StridedMaybeAdjOrTransVecOrMat{T}, x::StridedVector{T}, + alpha::Number, beta::Number) where {T<:BlasFloat} = + gemv!(y, adj_or_trans_char(A), _parent(A), x, alpha, beta) +# catch the real adjoint case and rewrap to transpose +@inline mul!(y::StridedVector{T}, adjA::Adjoint{<:Any,<:StridedVecOrMat{T}}, x::StridedVector{T}, + alpha::Number, beta::Number) where {T<:BlasReal} = + mul!(y, transpose(adjA.parent), x, alpha, beta) # Complex matrix times real vector. # Reinterpret the matrix as a real matrix and do real matvec computation. @inline mul!(y::StridedVector{Complex{T}}, A::StridedVecOrMat{Complex{T}}, x::StridedVector{T}, alpha::Number, beta::Number) where {T<:BlasReal} = gemv!(y, 'N', A, x, alpha, beta) - # Real matrix times complex vector. # Multiply the matrix with the real and imaginary parts separately @inline mul!(y::StridedVector{Complex{T}}, A::StridedMaybeAdjOrTransMat{T}, x::StridedVector{Complex{T}}, alpha::Number, beta::Number) where {T<:BlasReal} = - gemv!(y, A isa StridedArray ? 'N' : 'T', A isa StridedArray ? A : parent(A), x, alpha, beta) - -@inline mul!(y::AbstractVector, A::AbstractVecOrMat, x::AbstractVector, - alpha::Number, beta::Number) = - generic_matvecmul!(y, 'N', A, x, MulAddMul(alpha, beta)) - -function *(tA::Transpose{<:Any,<:StridedMatrix{T}}, x::StridedVector{S}) where {T<:BlasFloat,S} - TS = promote_op(matprod, T, S) - mul!(similar(x, TS, size(tA, 1)), tA, convert(AbstractVector{TS}, x)) -end -function *(tA::Transpose{<:Any,<:AbstractMatrix{T}}, x::AbstractVector{S}) where {T,S} - TS = promote_op(matprod, T, S) - mul!(similar(x, TS, size(tA, 1)), tA, x) -end -@inline mul!(y::StridedVector{T}, tA::Transpose{<:Any,<:StridedVecOrMat{T}}, x::StridedVector{T}, - alpha::Number, beta::Number) where {T<:BlasFloat} = - gemv!(y, 'T', tA.parent, x, alpha, beta) -@inline mul!(y::AbstractVector, tA::Transpose{<:Any,<:AbstractVecOrMat}, x::AbstractVector, - alpha::Number, beta::Number) = - generic_matvecmul!(y, 'T', tA.parent, x, MulAddMul(alpha, beta)) - -function *(adjA::Adjoint{<:Any,<:StridedMatrix{T}}, x::StridedVector{S}) where {T<:BlasFloat,S} - TS = promote_op(matprod, T, S) - mul!(similar(x, TS, size(adjA, 1)), adjA, convert(AbstractVector{TS}, x)) -end -function *(adjA::Adjoint{<:Any,<:AbstractMatrix{T}}, x::AbstractVector{S}) where {T,S} - TS = promote_op(matprod, T, S) - mul!(similar(x, TS, size(adjA, 1)), adjA, x) -end - -@inline mul!(y::StridedVector{T}, adjA::Adjoint{<:Any,<:StridedVecOrMat{T}}, x::StridedVector{T}, - alpha::Number, beta::Number) where {T<:BlasReal} = - mul!(y, transpose(adjA.parent), x, alpha, beta) -@inline mul!(y::StridedVector{T}, adjA::Adjoint{<:Any,<:StridedVecOrMat{T}}, x::StridedVector{T}, - alpha::Number, beta::Number) where {T<:BlasComplex} = - gemv!(y, 'C', adjA.parent, x, alpha, beta) -@inline mul!(y::AbstractVector, adjA::Adjoint{<:Any,<:AbstractVecOrMat}, x::AbstractVector, - alpha::Number, beta::Number) = - generic_matvecmul!(y, 'C', adjA.parent, x, MulAddMul(alpha, beta)) + gemv!(y, A isa StridedArray ? 'N' : 'T', _parent(A), x, alpha, beta) # Vector-Matrix multiplication (*)(x::AdjointAbsVec, A::AbstractMatrix) = (A'*x')' (*)(x::TransposeAbsVec, A::AbstractMatrix) = transpose(transpose(A)*transpose(x)) -_parent(A) = A -_parent(A::Adjoint) = parent(A) -_parent(A::Transpose) = parent(A) - +# Matrix-matrix multiplication """ *(A::AbstractMatrix, B::AbstractMatrix) @@ -156,10 +126,6 @@ function (*)(A::StridedMaybeAdjOrTransMat{<:BlasComplex}, B::StridedMaybeAdjOrTr wrapperop(B)(convert(AbstractArray{TS}, _parent(B)))) end -@inline function mul!(C::StridedMatrix{T}, A::StridedVecOrMat{T}, B::StridedVecOrMat{T}, - alpha::Number, beta::Number) where {T<:BlasFloat} - return gemm_wrapper!(C, 'N', 'N', A, B, MulAddMul(alpha, beta)) -end # Complex Matrix times real matrix: We use that it is generally faster to reinterpret the # first matrix as a real matrix and carry out real matrix matrix multiply function (*)(A::StridedMatrix{<:BlasComplex}, B::StridedMaybeAdjOrTransMat{<:BlasReal}) @@ -301,7 +267,14 @@ julia> C """ @inline mul!(C::AbstractMatrix, A::AbstractVecOrMat, B::AbstractVecOrMat, alpha::Number, beta::Number) = - generic_matmatmul!(C, 'N', 'N', A, B, MulAddMul(alpha, beta)) + generic_matmatmul!( + C, + adj_or_trans_char(A), + adj_or_trans_char(B), + _parent(A), + _parent(B), + MulAddMul(alpha, beta) + ) """ rmul!(A, B) @@ -369,6 +342,12 @@ julia> lmul!(F.Q, B) """ lmul!(A, B) +# generic case +@inline mul!(C::StridedMatrix{T}, A::StridedMaybeAdjOrTransVecOrMat{T}, B::StridedMaybeAdjOrTransVecOrMat{T}, + alpha::Number, beta::Number) where {T<:BlasFloat} = + gemm_wrapper!(C, adj_or_trans_char(A), adj_or_trans_char(B), _parent(A), _parent(B), MulAddMul(alpha, beta)) + +# AtB & ABt (including B === A) @inline function mul!(C::StridedMatrix{T}, tA::Transpose{<:Any,<:StridedVecOrMat{T}}, B::StridedVecOrMat{T}, alpha::Number, beta::Number) where {T<:BlasFloat} A = tA.parent @@ -378,10 +357,6 @@ lmul!(A, B) return gemm_wrapper!(C, 'T', 'N', A, B, MulAddMul(alpha, beta)) end end -@inline mul!(C::AbstractMatrix, tA::Transpose{<:Any,<:AbstractVecOrMat}, B::AbstractVecOrMat, - alpha::Number, beta::Number) = - generic_matmatmul!(C, 'T', 'N', tA.parent, B, MulAddMul(alpha, beta)) - @inline function mul!(C::StridedMatrix{T}, A::StridedVecOrMat{T}, tB::Transpose{<:Any,<:StridedVecOrMat{T}}, alpha::Number, beta::Number) where {T<:BlasFloat} B = tB.parent @@ -391,39 +366,15 @@ end return gemm_wrapper!(C, 'N', 'T', A, B, MulAddMul(alpha, beta)) end end -# Complex matrix times (transposed) real matrix. Reinterpret the first matrix to real for efficiency. -@inline mul!(C::StridedMatrix{Complex{T}}, A::StridedVecOrMat{Complex{T}}, B::StridedVecOrMat{T}, - alpha::Number, beta::Number) where {T<:BlasReal} = - gemm_wrapper!(C, 'N', 'N', A, B, MulAddMul(alpha, beta)) -@inline mul!(C::StridedMatrix{Complex{T}}, A::StridedVecOrMat{Complex{T}}, tB::Transpose{<:Any,<:StridedVecOrMat{T}}, - alpha::Number, beta::Number) where {T<:BlasReal} = - gemm_wrapper!(C, 'N', 'T', A, parent(tB), MulAddMul(alpha, beta)) - -# collapsing the following two defs with C::AbstractVecOrMat yields ambiguities -@inline mul!(C::AbstractVector, A::AbstractVecOrMat, tB::Transpose{<:Any,<:AbstractVecOrMat}, - alpha::Number, beta::Number) = - generic_matmatmul!(C, 'N', 'T', A, tB.parent, MulAddMul(alpha, beta)) -@inline mul!(C::AbstractMatrix, A::AbstractVecOrMat, tB::Transpose{<:Any,<:AbstractVecOrMat}, - alpha::Number, beta::Number) = - generic_matmatmul!(C, 'N', 'T', A, tB.parent, MulAddMul(alpha, beta)) - -@inline mul!(C::StridedMatrix{T}, tA::Transpose{<:Any,<:StridedVecOrMat{T}}, tB::Transpose{<:Any,<:StridedVecOrMat{T}}, - alpha::Number, beta::Number) where {T<:BlasFloat} = - gemm_wrapper!(C, 'T', 'T', tA.parent, tB.parent, MulAddMul(alpha, beta)) -@inline mul!(C::AbstractMatrix, tA::Transpose{<:Any,<:AbstractVecOrMat}, tB::Transpose{<:Any,<:AbstractVecOrMat}, - alpha::Number, beta::Number) = - generic_matmatmul!(C, 'T', 'T', tA.parent, tB.parent, MulAddMul(alpha, beta)) - -@inline mul!(C::StridedMatrix{T}, tA::Transpose{<:Any,<:StridedVecOrMat{T}}, adjB::Adjoint{<:Any,<:StridedVecOrMat{T}}, - alpha::Number, beta::Number) where {T<:BlasFloat} = - gemm_wrapper!(C, 'T', 'C', tA.parent, adjB.parent, MulAddMul(alpha, beta)) -@inline mul!(C::AbstractMatrix, tA::Transpose{<:Any,<:AbstractVecOrMat}, tB::Adjoint{<:Any,<:AbstractVecOrMat}, - alpha::Number, beta::Number) = - generic_matmatmul!(C, 'T', 'C', tA.parent, tB.parent, MulAddMul(alpha, beta)) - +# real adjoint cases, also needed for disambiguation +@inline mul!(C::StridedMatrix{T}, A::StridedVecOrMat{T}, adjB::Adjoint{<:Any,<:StridedVecOrMat{T}}, + alpha::Number, beta::Number) where {T<:BlasReal} = + mul!(C, A, transpose(adjB.parent), alpha, beta) @inline mul!(C::StridedMatrix{T}, adjA::Adjoint{<:Any,<:StridedVecOrMat{T}}, B::StridedVecOrMat{T}, - alpha::Real, beta::Real) where {T<:BlasReal} = + alpha::Real, beta::Real) where {T<:BlasReal} = mul!(C, transpose(adjA.parent), B, alpha, beta) + +# AcB & ABc (including B === A) @inline function mul!(C::StridedMatrix{T}, adjA::Adjoint{<:Any,<:StridedVecOrMat{T}}, B::StridedVecOrMat{T}, alpha::Number, beta::Number) where {T<:BlasComplex} A = adjA.parent @@ -433,13 +384,6 @@ end return gemm_wrapper!(C, 'C', 'N', A, B, MulAddMul(alpha, beta)) end end -@inline mul!(C::AbstractMatrix, adjA::Adjoint{<:Any,<:AbstractVecOrMat}, B::AbstractVecOrMat, - alpha::Number, beta::Number) = - generic_matmatmul!(C, 'C', 'N', adjA.parent, B, MulAddMul(alpha, beta)) - -@inline mul!(C::StridedMatrix{T}, A::StridedVecOrMat{T}, adjB::Adjoint{<:Any,<:StridedVecOrMat{<:BlasReal}}, - alpha::Number, beta::Number) where {T<:BlasFloat} = - mul!(C, A, transpose(adjB.parent), alpha, beta) @inline function mul!(C::StridedMatrix{T}, A::StridedVecOrMat{T}, adjB::Adjoint{<:Any,<:StridedVecOrMat{T}}, alpha::Number, beta::Number) where {T<:BlasComplex} B = adjB.parent @@ -449,23 +393,16 @@ end return gemm_wrapper!(C, 'N', 'C', A, B, MulAddMul(alpha, beta)) end end -@inline mul!(C::AbstractMatrix, A::AbstractVecOrMat, adjB::Adjoint{<:Any,<:AbstractVecOrMat}, - alpha::Number, beta::Number) = - generic_matmatmul!(C, 'N', 'C', A, adjB.parent, MulAddMul(alpha, beta)) - -@inline mul!(C::StridedMatrix{T}, adjA::Adjoint{<:Any,<:StridedVecOrMat{T}}, adjB::Adjoint{<:Any,<:StridedVecOrMat{T}}, - alpha::Number, beta::Number) where {T<:BlasFloat} = - gemm_wrapper!(C, 'C', 'C', adjA.parent, adjB.parent, MulAddMul(alpha, beta)) -@inline mul!(C::AbstractMatrix, adjA::Adjoint{<:Any,<:AbstractVecOrMat}, adjB::Adjoint{<:Any,<:AbstractVecOrMat}, - alpha::Number, beta::Number) = - generic_matmatmul!(C, 'C', 'C', adjA.parent, adjB.parent, MulAddMul(alpha, beta)) - -@inline mul!(C::StridedMatrix{T}, adjA::Adjoint{<:Any,<:StridedVecOrMat{T}}, tB::Transpose{<:Any,<:StridedVecOrMat{T}}, - alpha::Number, beta::Number) where {T<:BlasFloat} = - gemm_wrapper!(C, 'C', 'T', adjA.parent, tB.parent, MulAddMul(alpha, beta)) -@inline mul!(C::AbstractMatrix, adjA::Adjoint{<:Any,<:AbstractVecOrMat}, tB::Transpose{<:Any,<:AbstractVecOrMat}, - alpha::Number, beta::Number) = - generic_matmatmul!(C, 'C', 'T', adjA.parent, tB.parent, MulAddMul(alpha, beta)) + +# Complex matrix times (transposed) real matrix. Reinterpret the first matrix to real for efficiency. +@inline mul!(C::StridedMatrix{Complex{T}}, A::StridedMaybeAdjOrTransVecOrMat{Complex{T}}, B::StridedMaybeAdjOrTransVecOrMat{T}, + alpha::Number, beta::Number) where {T<:BlasReal} = + gemm_wrapper!(C, adj_or_trans_char(A), adj_or_trans_char(B), _parent(A), _parent(B), MulAddMul(alpha, beta)) +# catch the real adjoint case and interpret it as a transpose +@inline mul!(C::StridedMatrix{Complex{T}}, A::StridedVecOrMat{Complex{T}}, adjB::Adjoint{<:Any,<:StridedVecOrMat{T}}, + alpha::Number, beta::Number) where {T<:BlasReal} = + mul!(C, A, transpose(adjB.parent), alpha, beta) + # Supporting functions for matrix multiplication diff --git a/stdlib/LinearAlgebra/src/triangular.jl b/stdlib/LinearAlgebra/src/triangular.jl index 248fc048612c8..557516d84811b 100644 --- a/stdlib/LinearAlgebra/src/triangular.jl +++ b/stdlib/LinearAlgebra/src/triangular.jl @@ -670,23 +670,9 @@ fillstored!(A::UnitUpperTriangular, x) = (fillband!(A.data, x, 1, size(A,2)-1); lmul!(A::Tridiagonal, B::AbstractTriangular) = A*full!(B) # is this necessary? -mul!(C::AbstractVector, A::AbstractTriangular, transB::Transpose{<:Any,<:AbstractVecOrMat}) = - (B = transB.parent; lmul!(A, transpose!(C, B))) -mul!(C::AbstractMatrix, A::AbstractTriangular, transB::Transpose{<:Any,<:AbstractVecOrMat}) = - (B = transB.parent; lmul!(A, transpose!(C, B))) -mul!(C::AbstractMatrix, A::AbstractTriangular, adjB::Adjoint{<:Any,<:AbstractVecOrMat}) = - (B = adjB.parent; lmul!(A, adjoint!(C, B))) -mul!(C::AbstractVecOrMat, A::AbstractTriangular, adjB::Adjoint{<:Any,<:AbstractVecOrMat}) = - (B = adjB.parent; lmul!(A, adjoint!(C, B))) - -# The three methods are necessary to avoid ambiguities with definitions in matmul.jl -mul!(C::AbstractVector , A::AbstractTriangular, B::AbstractVector) = lmul!(A, copyto!(C, B)) -mul!(C::AbstractMatrix , A::AbstractTriangular, B::AbstractVecOrMat) = lmul!(A, copyto!(C, B)) -mul!(C::AbstractVecOrMat, A::AbstractTriangular, B::AbstractVecOrMat) = lmul!(A, copyto!(C, B)) - -@inline mul!(C::AbstractMatrix, A::AbstractTriangular, B::Adjoint{<:Any,<:AbstractVecOrMat}, alpha::Number, beta::Number) = - mul!(C, A, copy(B), alpha, beta) -@inline mul!(C::AbstractMatrix, A::AbstractTriangular, B::Transpose{<:Any,<:AbstractVecOrMat}, alpha::Number, beta::Number) = +mul!(C::AbstractMatrix, A::AbstractTriangular, B::AbstractVecOrMat) = + lmul!(A, inplace_adj_or_trans(B)(C, _parent(B))) +@inline mul!(C::AbstractMatrix, A::AbstractTriangular, B::AdjOrTrans{<:Any,<:AbstractVecOrMat}, alpha::Number, beta::Number) = mul!(C, A, copy(B), alpha, beta) # preserve triangular structure in in-place multiplication @@ -1116,78 +1102,78 @@ end # in the following transpose and conjugate transpose naive substitution variants, # accumulating in z rather than b[j,k] significantly improves performance as of Dec 2015 -for (t, tfun) in ((:Adjoint, :adjoint), (:Transpose, :transpose)) - @eval begin - function ldiv!(xA::UpperTriangular{<:Any,<:$t}, b::AbstractVector) - require_one_based_indexing(xA, b) - A = parent(parent(xA)) - n = size(A, 1) - if !(n == length(b)) - throw(DimensionMismatch("first dimension of left hand side A, $n, and length of right hand side b, $(length(b)), must be equal")) - end - @inbounds for j in n:-1:1 - z = b[j] - for i in n:-1:j+1 - z -= $tfun(A[i,j]) * b[i] - end - iszero(A[j,j]) && throw(SingularException(j)) - b[j] = $tfun(A[j,j]) \ z - end - return b +function ldiv!(xA::UpperTriangular{<:Any,<:AdjOrTrans}, b::AbstractVector) + require_one_based_indexing(xA, b) + tfun = adj_or_trans(parent(xA)) + A = parent(parent(xA)) + n = size(A, 1) + if !(n == length(b)) + throw(DimensionMismatch("first dimension of left hand side A, $n, and length of right hand side b, $(length(b)), must be equal")) + end + @inbounds for j in n:-1:1 + z = b[j] + for i in n:-1:j+1 + z -= tfun(A[i,j]) * b[i] end + iszero(A[j,j]) && throw(SingularException(j)) + b[j] = tfun(A[j,j]) \ z + end + return b +end - function ldiv!(xA::UnitUpperTriangular{<:Any,<:$t}, b::AbstractVector) - require_one_based_indexing(xA, b) - A = parent(parent(xA)) - n = size(A, 1) - if !(n == length(b)) - throw(DimensionMismatch("first dimension of left hand side A, $n, and length of right hand side b, $(length(b)), must be equal")) - end - @inbounds for j in n:-1:1 - z = b[j] - for i in n:-1:j+1 - z -= $tfun(A[i,j]) * b[i] - end - b[j] = z - end - return b +function ldiv!(xA::UnitUpperTriangular{<:Any,<:AdjOrTrans}, b::AbstractVector) + require_one_based_indexing(xA, b) + tfun = adj_or_trans(parent(xA)) + A = parent(parent(xA)) + n = size(A, 1) + if !(n == length(b)) + throw(DimensionMismatch("first dimension of left hand side A, $n, and length of right hand side b, $(length(b)), must be equal")) + end + @inbounds for j in n:-1:1 + z = b[j] + for i in n:-1:j+1 + z -= tfun(A[i,j]) * b[i] end + b[j] = z + end + return b +end - function ldiv!(xA::LowerTriangular{<:Any,<:$t}, b::AbstractVector) - require_one_based_indexing(xA, b) - A = parent(parent(xA)) - n = size(A, 1) - if !(n == length(b)) - throw(DimensionMismatch("first dimension of left hand side A, $n, and length of right hand side b, $(length(b)), must be equal")) - end - @inbounds for j in 1:n - z = b[j] - for i in 1:j-1 - z -= $tfun(A[i,j]) * b[i] - end - iszero(A[j,j]) && throw(SingularException(j)) - b[j] = $tfun(A[j,j]) \ z - end - return b +function ldiv!(xA::LowerTriangular{<:Any,<:AdjOrTrans}, b::AbstractVector) + require_one_based_indexing(xA, b) + tfun = adj_or_trans(parent(xA)) + A = parent(parent(xA)) + n = size(A, 1) + if !(n == length(b)) + throw(DimensionMismatch("first dimension of left hand side A, $n, and length of right hand side b, $(length(b)), must be equal")) + end + @inbounds for j in 1:n + z = b[j] + for i in 1:j-1 + z -= tfun(A[i,j]) * b[i] end + iszero(A[j,j]) && throw(SingularException(j)) + b[j] = tfun(A[j,j]) \ z + end + return b +end - function ldiv!(xA::UnitLowerTriangular{<:Any,<:$t}, b::AbstractVector) - require_one_based_indexing(xA, b) - A = parent(parent(xA)) - n = size(A, 1) - if !(n == length(b)) - throw(DimensionMismatch("first dimension of left hand side A, $n, and length of right hand side b, $(length(b)), must be equal")) - end - @inbounds for j in 1:n - z = b[j] - for i in 1:j-1 - z -= $tfun(A[i,j]) * b[i] - end - b[j] = z - end - return b +function ldiv!(xA::UnitLowerTriangular{<:Any,<:AdjOrTrans}, b::AbstractVector) + require_one_based_indexing(xA, b) + tfun = adj_or_trans(parent(xA)) + A = parent(parent(xA)) + n = size(A, 1) + if !(n == length(b)) + throw(DimensionMismatch("first dimension of left hand side A, $n, and length of right hand side b, $(length(b)), must be equal")) + end + @inbounds for j in 1:n + z = b[j] + for i in 1:j-1 + z -= tfun(A[i,j]) * b[i] end + b[j] = z end + return b end function rdiv!(A::AbstractMatrix, B::UpperTriangular) diff --git a/stdlib/LinearAlgebra/test/matmul.jl b/stdlib/LinearAlgebra/test/matmul.jl index 0150c4c2efdc8..2d99856a2667b 100644 --- a/stdlib/LinearAlgebra/test/matmul.jl +++ b/stdlib/LinearAlgebra/test/matmul.jl @@ -655,10 +655,10 @@ Transpose(x::RootInt) = x @testset "#14293" begin a = [RootInt(3)] - C = [0] + C = [0;;] mul!(C, a, transpose(a)) @test C[1] == 9 - C = [1] + C = [1;;] mul!(C, a, transpose(a), 2, 3) @test C[1] == 21 a = [RootInt(2), RootInt(10)]