Skip to content

Commit

Permalink
[Backport to 18][SPIR-V 1.4] Allow OpCopyMemorySized to have Memory O…
Browse files Browse the repository at this point in the history 
…perands for both Source and Target

This change addresses p.4 of KhronosGroup#2460.
No changes for OpCopyMemory as we currently don't use it in translation.
  • Loading branch information
@vmaksimo
vmaksimo committed Apr 3, 2024
1 parent 1745c78 commit 3a32237
Show file tree
Hide file tree
Showing 4 changed files with 70 additions and 18 deletions.
4 changes: 3 additions & 1 deletion lib/SPIRV/SPIRVReader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1840,6 +1840,8 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F,
CallInst *CI = nullptr;
llvm::Value *Dst = transValue(BC->getTarget(), F, BB);
MaybeAlign Align(BC->getAlignment());
MaybeAlign SrcAlign =
BC->getSrcAlignment() ? MaybeAlign(BC->getSrcAlignment()) : Align;
llvm::Value *Size = transValue(BC->getSize(), F, BB);
bool IsVolatile = BC->SPIRVMemoryAccess::isVolatile();
IRBuilder<> Builder(BB);
Expand All @@ -1852,7 +1854,7 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F,

// A ptr.annotation may have been generated for the source variable.
replaceOperandWithAnnotationIntrinsicCallResult(F, Src);
CI = Builder.CreateMemCpy(Dst, Align, Src, Align, Size, IsVolatile);
CI = Builder.CreateMemCpy(Dst, Align, Src, SrcAlign, Size, IsVolatile);
}
if (isFuncNoUnwind())
CI->getFunction()->addFnAttr(Attribute::NoUnwind);
Expand Down
30 changes: 23 additions & 7 deletions lib/SPIRV/SPIRVWriter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3919,19 +3919,30 @@ bool allowDecorateWithBufferLocationOrLatencyControlINTEL(IntrinsicInst *II) {

SPIRVValue *LLVMToSPIRVBase::transIntrinsicInst(IntrinsicInst *II,
SPIRVBasicBlock *BB) {
auto GetMemoryAccess = [](MemIntrinsic *MI) -> std::vector<SPIRVWord> {
auto GetMemoryAccess =
[](MemIntrinsic *MI,
bool AllowTwoMemAccessMasks) -> std::vector<SPIRVWord> {
std::vector<SPIRVWord> MemoryAccess(1, MemoryAccessMaskNone);
MaybeAlign DestAlignVal = MI->getDestAlign();
if (DestAlignVal) {
Align AlignVal = *DestAlignVal;
MemoryAccess[0] |= MemoryAccessAlignedMask;
if (auto *MTI = dyn_cast<MemTransferInst>(MI)) {
if (auto *MTI = dyn_cast<MemCpyInst>(MI)) {
MaybeAlign SourceAlignVal = MTI->getSourceAlign();
assert(SourceAlignVal && "Missed Source alignment!");

// In a case when alignment of source differs from dest one
// least value is guaranteed anyway.
AlignVal = std::min(*DestAlignVal, *SourceAlignVal);
// we either preserve both (allowed since SPIR-V 1.4), or the least
// value is guaranteed anyway.
if (AllowTwoMemAccessMasks) {
if (*DestAlignVal != *SourceAlignVal) {
MemoryAccess.push_back(DestAlignVal.valueOrOne().value());
MemoryAccess.push_back(MemoryAccessAlignedMask);
AlignVal = *SourceAlignVal;
}
} else {
AlignVal = std::min(*DestAlignVal, *SourceAlignVal);
}
}
MemoryAccess.push_back(AlignVal.value());
}
Expand Down Expand Up @@ -4425,14 +4436,19 @@ SPIRVValue *LLVMToSPIRVBase::transIntrinsicInst(IntrinsicInst *II,
transPointerType(Val->getType(), SPIRV::SPIRAS_Constant);
SPIRVValue *Source = BM->addUnaryInst(OpBitcast, SourceTy, Var, BB);
SPIRVValue *Target = transValue(MSI->getRawDest(), BB);
return BM->addCopyMemorySizedInst(Target, Source, CompositeTy->getLength(),
GetMemoryAccess(MSI), BB);
return BM->addCopyMemorySizedInst(
Target, Source, CompositeTy->getLength(),
GetMemoryAccess(MSI,
BM->isAllowedToUseVersion(VersionNumber::SPIRV_1_4)),
BB);
} break;
case Intrinsic::memcpy:
return BM->addCopyMemorySizedInst(
transValue(II->getOperand(0), BB), transValue(II->getOperand(1), BB),
transValue(II->getOperand(2), BB),
GetMemoryAccess(cast<MemIntrinsic>(II)), BB);
GetMemoryAccess(cast<MemIntrinsic>(II),
BM->isAllowedToUseVersion(VersionNumber::SPIRV_1_4)),
BB);
case Intrinsic::lifetime_start:
case Intrinsic::lifetime_end: {
Op OC = (II->getIntrinsicID() == Intrinsic::lifetime_start)
Expand Down
31 changes: 26 additions & 5 deletions lib/SPIRV/libSPIRV/SPIRVInstruction.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -378,14 +378,14 @@ class SPIRVInstTemplate : public BT {
class SPIRVMemoryAccess {
public:
SPIRVMemoryAccess(const std::vector<SPIRVWord> &TheMemoryAccess)
: TheMemoryAccessMask(0), Alignment(0), AliasScopeInstID(0),
NoAliasInstID(0) {
: TheMemoryAccessMask(0), Alignment(0), SrcAlignment(0),
AliasScopeInstID(0), NoAliasInstID(0) {
memoryAccessUpdate(TheMemoryAccess);
}

SPIRVMemoryAccess()
: TheMemoryAccessMask(0), Alignment(0), AliasScopeInstID(0),
NoAliasInstID(0) {}
: TheMemoryAccessMask(0), Alignment(0), SrcAlignment(0),
AliasScopeInstID(0), NoAliasInstID(0) {}

void memoryAccessUpdate(const std::vector<SPIRVWord> &MemoryAccess) {
if (!MemoryAccess.size())
Expand All @@ -406,7 +406,18 @@ class SPIRVMemoryAccess {
if (MemoryAccess[0] & MemoryAccessNoAliasINTELMaskMask) {
assert(MemoryAccess.size() > MemAccessNumParam &&
"Aliasing operand is missing");
NoAliasInstID = MemoryAccess[MemAccessNumParam];
NoAliasInstID = MemoryAccess[MemAccessNumParam++];
}

// Exit if there is no second memory operand mask
if (MemoryAccess.size() == MemAccessNumParam)
return;

size_t SecondMaskId = MemAccessNumParam++;
if (MemoryAccess[SecondMaskId] & MemoryAccessAlignedMask) {
assert(MemoryAccess.size() > MemAccessNumParam &&
"Alignment operand is missing");
SrcAlignment = MemoryAccess[MemAccessNumParam];
}
}
SPIRVWord isVolatile() const {
Expand All @@ -423,12 +434,14 @@ class SPIRVMemoryAccess {
}
SPIRVWord getMemoryAccessMask() const { return TheMemoryAccessMask; }
SPIRVWord getAlignment() const { return Alignment; }
SPIRVWord getSrcAlignment() const { return SrcAlignment; }
SPIRVWord getAliasScopeInstID() const { return AliasScopeInstID; }
SPIRVWord getNoAliasInstID() const { return NoAliasInstID; }

protected:
SPIRVWord TheMemoryAccessMask;
SPIRVWord Alignment;
SPIRVWord SrcAlignment;
SPIRVId AliasScopeInstID;
SPIRVId NoAliasInstID;
};
Expand Down Expand Up @@ -2088,13 +2101,21 @@ class SPIRVCopyMemorySized : public SPIRVInstruction, public SPIRVMemoryAccess {
Size(TheSize->getId()) {
validate();
assert(TheBB && "Invalid BB");
updateModuleVersion();
}
// Incomplete constructor
SPIRVCopyMemorySized()
: SPIRVInstruction(OC), SPIRVMemoryAccess(), Target(SPIRVID_INVALID),
Source(SPIRVID_INVALID), Size(0) {
setHasNoId();
setHasNoType();
updateModuleVersion();
}

SPIRVWord getRequiredSPIRVVersion() const override {
if (getSrcAlignment())
return static_cast<SPIRVWord>(VersionNumber::SPIRV_1_4);
return static_cast<SPIRVWord>(VersionNumber::SPIRV_1_0);
}

SPIRVValue *getSource() { return getValue(Source); }
Expand Down
23 changes: 18 additions & 5 deletions test/llvm-intrinsics/memcpy.align.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,21 @@
; clang -cc1 -triple spir -disable-llvm-passes t.cl -emit-llvm -o t.ll

; RUN: llvm-as %s -o %t.bc
; RUN: llvm-spirv %t.bc -spirv-text -o %t.txt
; RUN: llvm-spirv %t.bc --spirv-max-version=1.3 -spirv-text -o %t.txt
; RUN: FileCheck < %t.txt %s --check-prefix=CHECK-SPIRV
; RUN: llvm-spirv %t.bc -o %t.spv
; RUN: llvm-spirv %t.bc --spirv-max-version=1.3 -o %t.spv
; RUN: spirv-val %t.spv
; RUN: llvm-spirv -r %t.spv -o %t.rev.bc
; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM
; RUN: llvm-dis %t.rev.bc
; RUN: FileCheck < %t.rev.ll %s --check-prefix=CHECK-LLVM

; RUN: llvm-spirv %t.bc --spirv-max-version=1.4 -spirv-text -o %t.txt
; RUN: FileCheck < %t.txt %s --check-prefix=ALIGN-MATCH-SPIRV
; RUN: llvm-spirv %t.bc --spirv-max-version=1.4 -o %t.spv
; RUN: spirv-val %t.spv
; RUN: llvm-spirv -r %t.spv -o %t.rev.bc
; RUN: llvm-dis %t.rev.bc
; RUN: FileCheck < %t.rev.ll %s --check-prefix=ALIGN-MATCH-LLVM

target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
target triple = "spir"
Expand All @@ -53,8 +62,10 @@ entry:
%b1 = getelementptr inbounds %struct.A, %struct.A* %agg.result, i32 0, i32 1
%2 = bitcast %struct.B* %b1 to i8*
%3 = bitcast %struct.B* %b to i8*
; CHECK-SPIRV: CopyMemorySized {{[0-9]+}} {{[0-9]+}} {{[0-9]+}} {{[0-9]+}} 4
; CHECK-SPIRV: CopyMemorySized {{[0-9]+}} {{[0-9]+}} {{[0-9]+}} 2 4
; CHECK-LLVM: call void @llvm.memcpy.p0.p0.i32(ptr align 4 {{%[0-9]+}}, ptr align 4 {{%[0-9]+}}, i32 8, i1 false)
; ALIGN-MATCH-SPIRV: CopyMemorySized {{[0-9]+}} {{[0-9]+}} {{[0-9]+}} 2 8 2 4
; ALIGN-MATCH-LLVM: call void @llvm.memcpy.p0.p0.i32(ptr align 8 {{%[0-9]+}}, ptr align 4 {{%[0-9]+}}, i32 8, i1 false)
call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 8 %2, i8* align 4 %3, i32 8, i1 false), !tbaa.struct !4
%4 = bitcast %struct.B* %b to i8*
call void @llvm.lifetime.end.p0i8(i64 8, i8* %4) #2
Expand Down Expand Up @@ -85,8 +96,10 @@ entry:
%b = getelementptr inbounds %struct.A, %struct.A* %a, i32 0, i32 1
%2 = bitcast %struct.B* %agg.result to i8*
%3 = bitcast %struct.B* %b to i8*
; CHECK-SPIRV: CopyMemorySized {{[0-9]+}} {{[0-9]+}} {{[0-9]+}} {{[0-9]+}} 4
; CHECK-SPIRV: CopyMemorySized {{[0-9]+}} {{[0-9]+}} {{[0-9]+}} 2 4
; CHECK-LLVM: call void @llvm.memcpy.p0.p0.i32(ptr align 4 {{%[0-9]+}}, ptr align 4 {{%[0-9]+}}, i32 8, i1 false)
; ALIGN-MATCH-SPIRV: CopyMemorySized {{[0-9]+}} {{[0-9]+}} {{[0-9]+}} 2 4 2 8
; ALIGN-MATCH-LLVM: call void @llvm.memcpy.p0.p0.i32(ptr align 4 {{%[0-9]+}}, ptr align 8 {{%[0-9]+}}, i32 8, i1 false)
call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 4 %2, i8* align 8 %3, i32 8, i1 false), !tbaa.struct !4
%4 = bitcast %struct.A* %a to i8*
call void @llvm.lifetime.end.p0i8(i64 16, i8* %4) #2
Expand Down

0 comments on commit 3a32237

Please sign in to comment.