SPIRVInternal.h

//===- LLVMSPIRVInternal.h -  SPIR-V internal header file -------*- C++ -*-===//
//
//                     The LLVM/SPIRV Translator
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
// Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal with the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimers.
// Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimers in the documentation
// and/or other materials provided with the distribution.
// Neither the names of Advanced Micro Devices, Inc., nor the names of its
// contributors may be used to endorse or promote products derived from this
// Software without specific prior written permission.
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH
// THE SOFTWARE.
//
//===----------------------------------------------------------------------===//
/// \file
///
/// This file declares classes and functions shared by SPIR-V reader/writer.
///
//===----------------------------------------------------------------------===//
#ifndef SPIRV_SPIRVINTERNAL_H
#define SPIRV_SPIRVINTERNAL_H

#include "NameMangleAPI.h"
#include "libSPIRV/SPIRVEnum.h"
#include "libSPIRV/SPIRVError.h"
#include "libSPIRV/SPIRVNameMapEnum.h"
#include "libSPIRV/SPIRVType.h"
#include "libSPIRV/SPIRVUtil.h"

#include "LLVMSPIRVLib.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/TypedPointerType.h"

#include <functional>
#include <utility>

using namespace SPIRV;
using namespace llvm;

namespace llvm {
class IntrinsicInst;
class IRBuilderBase;
}

namespace SPIRV {

/// The LLVM/SPIR-V translator version used to fill the lower 16 bits of the
/// generator's magic number in the generated SPIR-V module.
/// This number should be bumped up whenever the generated SPIR-V changes.
const static unsigned short KTranslatorVer = 14;

class SPIRVOpaqueType;
typedef SPIRVMap<std::string, Op, SPIRVOpaqueType> SPIRVOpaqueTypeOpCodeMap;

// Ad hoc function used by LLVM/SPIRV converter for type casting
#define SPCV_CAST "spcv.cast"
#define LLVM_MEMCPY "llvm.memcpy"

// The name of function generated by Clang to initialize sampler(which is
// opaqueue type) by 32-bit integer. The name is taken from
// CodeGenModule::createOpenCLIntToSamplerConversion().
#define SAMPLER_INIT "__translate_sampler_initializer"

template <> inline void SPIRVMap<unsigned, Op>::init() {
#define _SPIRV_OP(x, y) add(Instruction::x, Op##y);
  /* Casts */
  _SPIRV_OP(ZExt, UConvert)
  _SPIRV_OP(SExt, SConvert)
  _SPIRV_OP(Trunc, UConvert)
  _SPIRV_OP(FPToUI, ConvertFToU)
  _SPIRV_OP(FPToSI, ConvertFToS)
  _SPIRV_OP(UIToFP, ConvertUToF)
  _SPIRV_OP(SIToFP, ConvertSToF)
  _SPIRV_OP(FPTrunc, FConvert)
  _SPIRV_OP(FPExt, FConvert)
  _SPIRV_OP(PtrToInt, ConvertPtrToU)
  _SPIRV_OP(IntToPtr, ConvertUToPtr)
  _SPIRV_OP(BitCast, Bitcast)
  _SPIRV_OP(AddrSpaceCast, GenericCastToPtr)
  _SPIRV_OP(GetElementPtr, AccessChain)
  _SPIRV_OP(FNeg, FNegate)
  /*Binary*/
  _SPIRV_OP(And, BitwiseAnd)
  _SPIRV_OP(Or, BitwiseOr)
  _SPIRV_OP(Xor, BitwiseXor)
  _SPIRV_OP(Add, IAdd)
  _SPIRV_OP(FAdd, FAdd)
  _SPIRV_OP(Sub, ISub)
  _SPIRV_OP(FSub, FSub)
  _SPIRV_OP(Mul, IMul)
  _SPIRV_OP(FMul, FMul)
  _SPIRV_OP(UDiv, UDiv)
  _SPIRV_OP(SDiv, SDiv)
  _SPIRV_OP(FDiv, FDiv)
  _SPIRV_OP(SRem, SRem)
  _SPIRV_OP(FRem, FRem)
  _SPIRV_OP(URem, UMod)
  _SPIRV_OP(Shl, ShiftLeftLogical)
  _SPIRV_OP(LShr, ShiftRightLogical)
  _SPIRV_OP(AShr, ShiftRightArithmetic)
#undef _SPIRV_OP
}
typedef SPIRVMap<unsigned, Op> OpCodeMap;

template <> inline void SPIRVMap<CmpInst::Predicate, Op>::init() {
#define _SPIRV_OP(x, y) add(CmpInst::x, Op##y);
  _SPIRV_OP(FCMP_OEQ, FOrdEqual)
  _SPIRV_OP(FCMP_OGT, FOrdGreaterThan)
  _SPIRV_OP(FCMP_OGE, FOrdGreaterThanEqual)
  _SPIRV_OP(FCMP_OLT, FOrdLessThan)
  _SPIRV_OP(FCMP_OLE, FOrdLessThanEqual)
  _SPIRV_OP(FCMP_ONE, FOrdNotEqual)
  _SPIRV_OP(FCMP_ORD, Ordered)
  _SPIRV_OP(FCMP_UNO, Unordered)
  _SPIRV_OP(FCMP_UEQ, FUnordEqual)
  _SPIRV_OP(FCMP_UGT, FUnordGreaterThan)
  _SPIRV_OP(FCMP_UGE, FUnordGreaterThanEqual)
  _SPIRV_OP(FCMP_ULT, FUnordLessThan)
  _SPIRV_OP(FCMP_ULE, FUnordLessThanEqual)
  _SPIRV_OP(FCMP_UNE, FUnordNotEqual)
  _SPIRV_OP(ICMP_EQ, IEqual)
  _SPIRV_OP(ICMP_NE, INotEqual)
  _SPIRV_OP(ICMP_UGT, UGreaterThan)
  _SPIRV_OP(ICMP_UGE, UGreaterThanEqual)
  _SPIRV_OP(ICMP_ULT, ULessThan)
  _SPIRV_OP(ICMP_ULE, ULessThanEqual)
  _SPIRV_OP(ICMP_SGT, SGreaterThan)
  _SPIRV_OP(ICMP_SGE, SGreaterThanEqual)
  _SPIRV_OP(ICMP_SLT, SLessThan)
  _SPIRV_OP(ICMP_SLE, SLessThanEqual)
#undef _SPIRV_OP
}
typedef SPIRVMap<CmpInst::Predicate, Op> CmpMap;

class IntBoolOpMapId;
template <> inline void SPIRVMap<Op, Op, IntBoolOpMapId>::init() {
  add(OpNot, OpLogicalNot);
  add(OpBitwiseAnd, OpLogicalAnd);
  add(OpBitwiseOr, OpLogicalOr);
  add(OpBitwiseXor, OpLogicalNotEqual);
  add(OpIEqual, OpLogicalEqual);
  add(OpINotEqual, OpLogicalNotEqual);
}
typedef SPIRVMap<Op, Op, IntBoolOpMapId> IntBoolOpMap;

#define SPIR_TARGETTRIPLE32 "spir-unknown-unknown"
#define SPIR_TARGETTRIPLE64 "spir64-unknown-unknown"
#define SPIR_DATALAYOUT32                                                      \
  "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32"                             \
  "-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32"                         \
  "-v32:32:32-v48:64:64-v64:64:64-v96:128:128"                                 \
  "-v128:128:128-v192:256:256-v256:256:256"                                    \
  "-v512:512:512-v1024:1024:1024"
#define SPIR_DATALAYOUT64                                                      \
  "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32"                             \
  "-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32"                         \
  "-v32:32:32-v48:64:64-v64:64:64-v96:128:128"                                 \
  "-v128:128:128-v192:256:256-v256:256:256"                                    \
  "-v512:512:512-v1024:1024:1024"

enum SPIRAddressSpace {
  SPIRAS_Private,
  SPIRAS_Global,
  SPIRAS_Constant,
  SPIRAS_Local,
  SPIRAS_Generic,
  SPIRAS_GlobalDevice,
  SPIRAS_GlobalHost,
  SPIRAS_Input,
  SPIRAS_Output,
  SPIRAS_Count,
};

template <> inline void SPIRVMap<SPIRAddressSpace, std::string>::init() {
  add(SPIRAS_Private, "Private");
  add(SPIRAS_Global, "Global");
  add(SPIRAS_Constant, "Constant");
  add(SPIRAS_Local, "Local");
  add(SPIRAS_Generic, "Generic");
  add(SPIRAS_Input, "Input");
  add(SPIRAS_GlobalDevice, "GlobalDevice");
  add(SPIRAS_GlobalHost, "GlobalHost");
}
typedef SPIRVMap<SPIRAddressSpace, SPIRVStorageClassKind>
    SPIRAddrSpaceCapitalizedNameMap;

template <>
inline void SPIRVMap<SPIRAddressSpace, SPIRVStorageClassKind>::init() {
  add(SPIRAS_Private, StorageClassFunction);
  add(SPIRAS_Global, StorageClassCrossWorkgroup);
  add(SPIRAS_Constant, StorageClassUniformConstant);
  add(SPIRAS_Local, StorageClassWorkgroup);
  add(SPIRAS_Generic, StorageClassGeneric);
  add(SPIRAS_Input, StorageClassInput);
  add(SPIRAS_GlobalDevice, StorageClassDeviceOnlyINTEL);
  add(SPIRAS_GlobalHost, StorageClassHostOnlyINTEL);
}
typedef SPIRVMap<SPIRAddressSpace, SPIRVStorageClassKind> SPIRSPIRVAddrSpaceMap;

// Maps OCL builtin function to SPIRV builtin variable.
template <>
inline void SPIRVMap<std::string, SPIRVAccessQualifierKind>::init() {
  add("read_only", AccessQualifierReadOnly);
  add("write_only", AccessQualifierWriteOnly);
  add("read_write", AccessQualifierReadWrite);
}
typedef SPIRVMap<std::string, SPIRVAccessQualifierKind>
    SPIRSPIRVAccessQualifierMap;

template <>
inline void SPIRVMap<Attribute::AttrKind, SPIRVFuncParamAttrKind>::init() {
  add(Attribute::ZExt, FunctionParameterAttributeZext);
  add(Attribute::SExt, FunctionParameterAttributeSext);
  add(Attribute::ByVal, FunctionParameterAttributeByVal);
  add(Attribute::StructRet, FunctionParameterAttributeSret);
  add(Attribute::NoAlias, FunctionParameterAttributeNoAlias);
  add(Attribute::NoCapture, FunctionParameterAttributeNoCapture);
  add(Attribute::ReadOnly, FunctionParameterAttributeNoWrite);
  add(Attribute::ReadNone, FunctionParameterAttributeNoReadWrite);
}
typedef SPIRVMap<Attribute::AttrKind, SPIRVFuncParamAttrKind>
    SPIRSPIRVFuncParamAttrMap;

template <>
inline void
SPIRVMap<Attribute::AttrKind, SPIRVFunctionControlMaskKind>::init() {
  add(Attribute::AlwaysInline, FunctionControlInlineMask);
  add(Attribute::NoInline, FunctionControlDontInlineMask);
  add(Attribute::OptimizeNone, internal::FunctionControlOptNoneINTELMask);
}
typedef SPIRVMap<Attribute::AttrKind, SPIRVFunctionControlMaskKind>
    SPIRSPIRVFuncCtlMaskMap;

class SPIRVExtSetShortName;
template <>
inline void
SPIRVMap<SPIRVExtInstSetKind, std::string, SPIRVExtSetShortName>::init() {
  add(SPIRVEIS_OpenCL, "ocl");
}
typedef SPIRVMap<SPIRVExtInstSetKind, std::string, SPIRVExtSetShortName>
    SPIRVExtSetShortNameMap;

#define SPIR_MD_COMPILER_OPTIONS "opencl.compiler.options"
#define SPIR_MD_KERNEL_ARG_ADDR_SPACE "kernel_arg_addr_space"
#define SPIR_MD_KERNEL_ARG_ACCESS_QUAL "kernel_arg_access_qual"
#define SPIR_MD_KERNEL_ARG_TYPE "kernel_arg_type"
#define SPIR_MD_KERNEL_ARG_BASE_TYPE "kernel_arg_base_type"
#define SPIR_MD_KERNEL_ARG_TYPE_QUAL "kernel_arg_type_qual"
#define SPIR_MD_KERNEL_ARG_NAME "kernel_arg_name"

#define SPIRV_MD_PARAMETER_DECORATIONS "spirv.ParameterDecorations"
#define SPIRV_MD_DECORATIONS "spirv.Decorations"
#define SPIRV_MD_INTEL_CACHE_DECORATIONS "spirv.DecorationCacheControlINTEL"

#define OCL_TYPE_NAME_SAMPLER_T "sampler_t"
#define SPIR_TYPE_NAME_EVENT_T "opencl.event_t"
#define SPIR_TYPE_NAME_CLK_EVENT_T "opencl.clk_event_t"
#define SPIR_TYPE_NAME_BLOCK_T "opencl.block"
#define SPIR_INTRINSIC_BLOCK_BIND "spir_block_bind"
#define SPIR_INTRINSIC_GET_BLOCK_INVOKE "spir_get_block_invoke"
#define SPIR_INTRINSIC_GET_BLOCK_CONTEXT "spir_get_block_context"
#define SPIR_TEMP_NAME_PREFIX_BLOCK "block"
#define SPIR_TEMP_NAME_PREFIX_CALL "call"

namespace kLLVMTypeName {
const static char StructPrefix[] = "struct.";
} // namespace kLLVMTypeName

namespace kSPIRVImageSampledTypeName {
const static char Float[] = "float";
const static char Half[] = "half";
const static char Int[] = "int";
const static char UInt[] = "uint";
const static char Long[] = "long";
const static char ULong[] = "ulong";
const static char Void[] = "void";
} // namespace kSPIRVImageSampledTypeName

namespace kSPIRVTypeName {
const static char Delimiter = '.';
const static char DeviceEvent[] = "DeviceEvent";
const static char Event[] = "Event";
const static char Image[] = "Image";
const static char Pipe[] = "Pipe";
const static char PostfixDelim = '_';
const static char Prefix[] = "spirv";
const static char PrefixAndDelim[] = "spirv.";
const static char Queue[] = "Queue";
const static char ReserveId[] = "ReserveId";
const static char SampledImg[] = "SampledImage";
const static char Sampler[] = "Sampler";
const static char ConstantSampler[] = "ConstantSampler";
const static char PipeStorage[] = "PipeStorage";
const static char ConstantPipeStorage[] = "ConstantPipeStorage";
const static char VmeImageINTEL[] = "VmeImageINTEL";
const static char JointMatrixINTEL[] = "JointMatrixINTEL";
const static char CooperativeMatrixKHR[] = "CooperativeMatrixKHR";
const static char BufferSurfaceINTEL[] = "BufferSurfaceINTEL";
} // namespace kSPIRVTypeName

namespace kSPR2TypeName {
const static char Delimiter = '.';
const static char OCLPrefix[] = "opencl.";
const static char ImagePrefix[] = "opencl.image";
const static char PipeRO[] = "opencl.pipe_ro_t";
const static char PipeWO[] = "opencl.pipe_wo_t";
const static char Sampler[] = "opencl.sampler_t";
const static char Event[] = "opencl.event_t";
} // namespace kSPR2TypeName

namespace kAccessQualName {
const static char ReadOnly[] = "read_only";
const static char WriteOnly[] = "write_only";
const static char ReadWrite[] = "read_write";
} // namespace kAccessQualName

namespace kAccessQualPostfix {
const static char ReadOnly[] = "_ro";
const static char WriteOnly[] = "_wo";
const static char ReadWrite[] = "_rw";
const static char Type[] = "_t";
} // namespace kAccessQualPostfix

namespace kMangledName {
const static char Sampler[] = "11ocl_sampler";
const static char AtomicPrefixIncoming[] = "U7_Atomic";
const static char AtomicPrefixInternal[] = "atomic_";
} // namespace kMangledName

namespace kSPIRVName {
const static char GroupPrefix[] = "group_";
const static char GroupNonUniformPrefix[] = "group_non_uniform_";
const static char ClusteredPrefix[] = "clustered_";
const static char Prefix[] = "__spirv_";
const static char Postfix[] = "__";
const static char ImageQuerySize[] = "ImageQuerySize";
const static char ImageQuerySizeLod[] = "ImageQuerySizeLod";
const static char ImageSampleExplicitLod[] = "ImageSampleExplicitLod";
const static char ReservedPrefix[] = "reserved_";
const static char SampledImage[] = "SampledImage";
const static char TempSampledImage[] = "TempSampledImage";
const static char TranslateOCLMemOrder[] = "__translate_ocl_memory_order";
const static char TranslateOCLMemScope[] = "__translate_ocl_memory_scope";
const static char TranslateSPIRVMemOrder[] = "__translate_spirv_memory_order";
const static char TranslateSPIRVMemScope[] = "__translate_spirv_memory_scope";
const static char TranslateSPIRVMemFence[] = "__translate_spirv_memory_fence";
const static char EntrypointPrefix[] = "__spirv_entry_";
const static char ConvertHandleToImageINTEL[] = "ConvertHandleToImageINTEL";
const static char ConvertHandleToSamplerINTEL[] = "ConvertHandleToSamplerINTEL";
const static char ConvertHandleToSampledImageINTEL[] =
    "ConvertHandleToSampledImageINTEL";
} // namespace kSPIRVName

namespace kSPIRVPostfix {
const static char ToGlobal[] = "ToGlobal";
const static char ToLocal[] = "ToLocal";
const static char ToPrivate[] = "ToPrivate";
const static char Sat[] = "sat";
const static char Rtz[] = "rtz";
const static char Rte[] = "rte";
const static char Rtp[] = "rtp";
const static char Rtn[] = "rtn";
const static char Rt[] = "rt";
const static char Return[] = "R";
const static char Divider[] = "_";
/// Divider between extended instruction name and postfix
const static char ExtDivider[] = "__";
} // namespace kSPIRVPostfix

namespace kSPIRVMD {
const static char Capability[] = "spirv.Capability";
const static char EntryPoint[] = "spirv.EntryPoint";
const static char ExecutionMode[] = "spirv.ExecutionMode";
const static char Extension[] = "spirv.Extension";
const static char Generator[] = "spirv.Generator";
const static char Source[] = "spirv.Source";
const static char SourceExtension[] = "spirv.SourceExtension";
const static char MemoryModel[] = "spirv.MemoryModel";
} // namespace kSPIRVMD

namespace kSPIR2MD {
const static char Extensions[] = "opencl.used.extensions";
const static char FPContract[] = "opencl.enable.FP_CONTRACT";
const static char OCLVer[] = "opencl.ocl.version";
const static char OCLCXXVer[] = "opencl.cxx.version";
const static char OptFeatures[] = "opencl.used.optional.core.features";
const static char SPIRVer[] = "opencl.spir.version";
const static char VecTyHint[] = "vec_type_hint";
const static char WGSize[] = "reqd_work_group_size";
const static char WGSizeHint[] = "work_group_size_hint";
const static char SubgroupSize[] = "intel_reqd_sub_group_size";
const static char MaxWGSize[] = "max_work_group_size";
const static char NoGlobalOffset[] = "no_global_work_offset";
const static char MaxWGDim[] = "max_global_work_dim";
const static char NumSIMD[] = "num_simd_work_items";
const static char StallEnable[] = "stall_enable";
const static char StallFree[] = "stall_free";
const static char FmaxMhz[] = "scheduler_target_fmax_mhz";
const static char LoopFuse[] = "loop_fuse";
const static char PreferDSP[] = "prefer_dsp";
const static char PropDSPPref[] = "propagate_dsp_preference";
const static char InitiationInterval[] = "initiation_interval";
const static char MaxConcurrency[] = "max_concurrency";
const static char PipelineKernel[] = "pipeline_kernel";
const static char IntelFPGAIPInterface[] = "ip_interface";
} // namespace kSPIR2MD

enum Spir2SamplerKind {
  CLK_ADDRESS_NONE = 0x0000,
  CLK_ADDRESS_CLAMP = 0x0004,
  CLK_ADDRESS_CLAMP_TO_EDGE = 0x0002,
  CLK_ADDRESS_REPEAT = 0x0006,
  CLK_ADDRESS_MIRRORED_REPEAT = 0x0008,
  CLK_NORMALIZED_COORDS_FALSE = 0x0000,
  CLK_NORMALIZED_COORDS_TRUE = 0x0001,
  CLK_FILTER_NEAREST = 0x0010,
  CLK_FILTER_LINEAR = 0x0020,
};

/// Additional information for mangling a function argument type.
struct BuiltinArgTypeMangleInfo {
  bool IsSigned;
  bool IsVoidPtr;
  bool IsEnum;
  bool IsSampler;
  bool IsAtomic;
  bool IsLocalArgBlock;
  SPIR::TypePrimitiveEnum Enum;
  unsigned Attr;
  Type *PointerTy;
  BuiltinArgTypeMangleInfo()
      : IsSigned(true), IsVoidPtr(false), IsEnum(false), IsSampler(false),
        IsAtomic(false), IsLocalArgBlock(false), Enum(SPIR::PRIMITIVE_NONE),
        Attr(0), PointerTy(nullptr) {}
};

/// Information for mangling builtin function.
class BuiltinFuncMangleInfo {
public:
  /// Translate builtin function name and set
  /// argument attributes and unsigned args.
  BuiltinFuncMangleInfo(const std::string &UniqName = "")
      : VarArgIdx(-1), DontMangle(false) {
    if (!UniqName.empty())
      init(UniqName);
  }
  virtual ~BuiltinFuncMangleInfo() {}
  const std::string &getUnmangledName() const { return UnmangledName; }
  void addUnsignedArg(int Ndx) {
    if (Ndx == -1)
      return addUnsignedArgs(0, 10); // 10 is enough for everybody, right?
    getTypeMangleInfo(Ndx).IsSigned = false;
  }
  void addUnsignedArgs(int StartNdx, int StopNdx) {
    assert(StartNdx < StopNdx && "wrong parameters");
    for (int I = StartNdx; I <= StopNdx; ++I)
      addUnsignedArg(I);
  }
  void addVoidPtrArg(unsigned Ndx) { getTypeMangleInfo(Ndx).IsVoidPtr = true; }
  void addSamplerArg(unsigned Ndx) { getTypeMangleInfo(Ndx).IsSampler = true; }
  void addAtomicArg(unsigned Ndx) { getTypeMangleInfo(Ndx).IsAtomic = true; }
  void setLocalArgBlock(unsigned Ndx) {
    getTypeMangleInfo(Ndx).IsLocalArgBlock = true;
  }
  void setEnumArg(unsigned Ndx, SPIR::TypePrimitiveEnum Enum) {
    auto &Info = getTypeMangleInfo(Ndx);
    Info.IsEnum = true;
    Info.Enum = Enum;
  }
  void setArgAttr(unsigned Ndx, unsigned Attr) {
    getTypeMangleInfo(Ndx).Attr = Attr;
  }
  void setVarArg(int Ndx) {
    assert(0 <= Ndx && "it is not allowed to set less than zero index");
    VarArgIdx = Ndx;
  }
  void setAsDontMangle() { DontMangle = true; }
  bool avoidMangling() { return DontMangle; }
  // get ellipsis index, single ellipsis at the end of the function is possible
  // only return value < 0 if none
  int getVarArg() const { return VarArgIdx; }
  BuiltinArgTypeMangleInfo &getTypeMangleInfo(unsigned Ndx) {
    while (Ndx >= ArgInfo.size())
      ArgInfo.emplace_back();
    BuiltinArgTypeMangleInfo &Info = ArgInfo[Ndx];
    return Info;
  }
  virtual void init(StringRef UniqUnmangledName) {
    UnmangledName = UniqUnmangledName.str();
  }

protected:
  std::string UnmangledName;
  std::vector<BuiltinArgTypeMangleInfo> ArgInfo;
  int VarArgIdx; // index of ellipsis argument, idx < 0 if none
private:
  bool DontMangle; // clang doesn't apply mangling for some builtin functions
                   // (i.e. enqueue_kernel)
};

/// \returns a vector of types for a collection of values.
template <class T> std::vector<Type *> getTypes(llvm::ArrayRef<T> V) {
  std::vector<Type *> Tys;
  for (auto &I : V)
    Tys.push_back(I->getType());
  return Tys;
}

/// Move elements of std::vector from [begin, end) to target.
template <typename T>
void move(std::vector<T> &V, size_t Begin, size_t End, size_t Target) {
  assert(Begin < End && End <= V.size() && Target <= V.size() &&
         !(Begin < Target && Target < End));
  if (Begin <= Target && Target <= End)
    return;
  auto B = V.begin() + Begin, E = V.begin() + End;
  if (Target > V.size())
    Target = V.size();
  if (Target > End)
    Target -= (End - Begin);
  std::vector<T> Segment(B, E);
  V.erase(B, E);
  V.insert(V.begin() + Target, Segment.begin(), Segment.end());
}

/// Find position of first pointer type value in a vector.
template <typename Container>
inline unsigned findFirstPtr(const Container &Args) {
  auto PtArg = std::find_if(Args.begin(), Args.end(), [](Value *V) {
    return V->getType()->isPointerTy();
  });
  return PtArg - Args.begin();
}

bool isSupportedTriple(Triple T);
void removeFnAttr(CallInst *Call, Attribute::AttrKind Attr);
void addFnAttr(CallInst *Call, Attribute::AttrKind Attr);
void saveLLVMModule(Module *M, const std::string &OutputFile);
std::string mapLLVMTypeToOCLType(const Type *Ty, bool Signed,
                                 Type *PointerElementType = nullptr);
SPIRVDecorate *mapPostfixToDecorate(StringRef Postfix, SPIRVEntry *Target);

/// Return vector V extended with poison elements to match the number of
/// components of NewType.
Value *extendVector(Value *V, FixedVectorType *NewType, IRBuilderBase &Builder);

/// Add decorations to a SPIR-V entry.
/// \param Decs Each string is a postfix without _ at the beginning.
SPIRVValue *addDecorations(SPIRVValue *Target,
                           const SmallVectorImpl<std::string> &Decs);

StructType *getOrCreateOpaqueStructType(Module *M, StringRef Name);
void getFunctionTypeParameterTypes(llvm::FunctionType *FT,
                                   std::vector<Type *> &ArgTys);
Function *getOrCreateFunction(Module *M, Type *RetTy, ArrayRef<Type *> ArgTypes,
                              StringRef Name,
                              BuiltinFuncMangleInfo *Mangle = nullptr,
                              AttributeList *Attrs = nullptr,
                              bool TakeName = true);

/// Get function call arguments.
/// \param Start Starting index.
/// \param End Ending index.
std::vector<Value *> getArguments(CallInst *CI, unsigned Start = 0,
                                  unsigned End = 0);

/// Get constant function call argument as an integer.
/// \param I argument index.
uint64_t getArgAsInt(CallInst *CI, unsigned I);

/// Get constant function call argument as type \param T.
/// \param I argument index.
template <typename T> T getArgAs(CallInst *CI, unsigned I) {
  return static_cast<T>(getArgAsInt(CI, I));
}

/// Get constant function call argument as a Scope enum.
/// \param I argument index.
Scope getArgAsScope(CallInst *CI, unsigned I);

/// Check if a type is OCL image type.
/// \return type name without "opencl." prefix.
bool isOCLImageType(llvm::Type *Ty, StringRef *Name = nullptr);

/// \param BaseTyName is the type name as in spirv.BaseTyName.Postfixes
/// \param Postfix contains postfixes extracted from the SPIR-V image
///   type name as spirv.BaseTyName.Postfixes.
bool isSPIRVStructType(llvm::Type *Ty, StringRef BaseTyName,
                       StringRef *Postfix = 0);

bool isSYCLHalfType(llvm::Type *Ty);

bool isSYCLBfloat16Type(llvm::Type *Ty);

/// Check if a function has decorated name as __spirv_{Name}_
/// and get the original name.
bool isDecoratedSPIRVFunc(const Function *F, StringRef &UndecName);

std::string prefixSPIRVName(const std::string &S);

StringRef dePrefixSPIRVName(StringRef R, SmallVectorImpl<StringRef> &Postfix);

/// Get a canonical function name for a SPIR-V op code.
std::string getSPIRVFuncName(Op OC, StringRef PostFix = "");

std::string getSPIRVFuncName(Op OC, const Type *PRetTy, bool IsSigned = false,
                             Type *PointerElementType = nullptr);

std::string getSPIRVFuncName(SPIRVBuiltinVariableKind BVKind);

/// Get a canonical function name for a SPIR-V extended instruction
std::string getSPIRVExtFuncName(SPIRVExtInstSetKind Set, unsigned ExtOp,
                                StringRef PostFix = "");

/// Get SPIR-V op code given the canonical function name.
/// Assume \param Name is either IA64 mangled or unmangled, and the unmangled
/// name takes the __spirv_{OpName}_{Postfixes} format.
/// \return op code if the unmangled function name is a valid op code name,
///   otherwise return OpNop.
/// \param Dec contains decorations decoded from function name if it is
///   not nullptr.
Op getSPIRVFuncOC(StringRef Name, SmallVectorImpl<std::string> *Dec = nullptr);

/// Get SPIR-V builtin variable enum given the canonical builtin name
/// Assume \param Name is in format __spirv_BuiltIn{Name}
/// \return false if \param Name is not a valid builtin name.
bool getSPIRVBuiltin(const std::string &Name, spv::BuiltIn &Builtin);

/// \param Name LLVM function name
/// \param DemangledName demanged name of the OpenCL built-in function
/// \returns true if Name is the name of the OpenCL built-in function,
/// false for other functions
bool oclIsBuiltin(StringRef Name, StringRef &DemangledName, bool IsCpp = false);

/// Check if a function returns void
bool isVoidFuncTy(FunctionType *FT);

/// \returns true if function \p F has array type argument.
bool hasArrayArg(Function *F);

/// Mutates function call instruction by changing the arguments.
/// \param ArgMutate mutates the function arguments.
/// \return mutated call instruction.
CallInst *mutateCallInst(
    Module *M, CallInst *CI,
    std::function<std::string(CallInst *, std::vector<Value *> &)> ArgMutate,
    BuiltinFuncMangleInfo *Mangle = nullptr, AttributeList *Attrs = nullptr,
    bool TakeName = false);

/// Mutates function call instruction by changing the arguments and return
/// value.
/// \param ArgMutate mutates the function arguments.
/// \param RetMutate mutates the return value.
/// \return mutated instruction.
Instruction *mutateCallInst(
    Module *M, CallInst *CI,
    std::function<std::string(CallInst *, std::vector<Value *> &, Type *&RetTy)>
        ArgMutate,
    std::function<Instruction *(CallInst *)> RetMutate,
    BuiltinFuncMangleInfo *Mangle = nullptr, AttributeList *Attrs = nullptr,
    bool TakeName = false);

/// Mutate function by change the arguments.
/// \param ArgMutate mutates the function arguments.
/// \param TakeName Take the original function's name if a new function with
///   different type needs to be created.
void mutateFunction(
    Function *F,
    std::function<std::string(CallInst *, std::vector<Value *> &)> ArgMutate,
    BuiltinFuncMangleInfo *Mangle = nullptr, AttributeList *Attrs = nullptr,
    bool TakeName = true);

/// Mutate function by change the arguments & the return type.
/// \param ArgMutate mutates the function arguments.
/// \param RetMutate mutates the function return value.
/// \param TakeName Take the original function's name if a new function with
///   different type needs to be created.
void mutateFunction(
    Function *F,
    std::function<std::string(CallInst *, std::vector<Value *> &, Type *&RetTy)>
        ArgMutate,
    std::function<Instruction *(CallInst *)> RetMutate,
    BuiltinFuncMangleInfo *Mangle = nullptr, AttributeList *Attrs = nullptr,
    bool TakeName = true);

/// Add a call instruction at \p Pos.
CallInst *addCallInst(Module *M, StringRef FuncName, Type *RetTy,
                      ArrayRef<Value *> Args, AttributeList *Attrs,
                      Instruction *Pos, BuiltinFuncMangleInfo *Mangle = nullptr,
                      StringRef InstName = SPIR_TEMP_NAME_PREFIX_CALL,
                      bool TakeFuncName = true);

/// Add a call instruction for SPIR-V builtin function.
CallInst *addCallInstSPIRV(Module *M, StringRef FuncName, Type *RetTy,
                           ArrayRef<Value *> Args, AttributeList *Attrs,
                           ArrayRef<Type *> PointerElementTypes,
                           Instruction *Pos, StringRef InstName);

typedef std::pair<std::vector<Value *>::iterator,
                  std::vector<Value *>::iterator>
    ValueVecRange;

/// Add a vector at \param InsPos.
Value *addVector(Instruction *InsPos, ValueVecRange Range);

/// Replace scalar values with a vector created at \param InsPos.
void makeVector(Instruction *InsPos, std::vector<Value *> &Ops,
                ValueVecRange Range);

/// Get size_t type.
IntegerType *getSizetType(Module *M);

/// Get a 64 bit integer constant.
ConstantInt *getInt64(Module *M, int64_t Value);

/// Get a 32 bit integer constant.
ConstantInt *getInt32(Module *M, int Value);

/// Get a 32 bit unsigned integer constant.
ConstantInt *getUInt32(Module *M, unsigned Value);

/// Get 32 bit integer constant if the value fits in 32 bits,
/// return 64 bit integer constant otherwise
ConstantInt *getInt(Module *M, int64_t Value);

/// Get 32 bit unsigned integer constant if the value fits in 32 bits,
/// return 64 bit unsigned integer constant otherwise
ConstantInt *getUInt(Module *M, uint64_t Value);

/// Get a 16 bit unsigned integer constant.
ConstantInt *getUInt16(Module *M, unsigned short Value);

// Get a 32 bit floating point constant.
Constant *getFloat32(Module *M, float Value);

/// Get a 32 bit integer constant vector.
std::vector<Value *> getInt32(Module *M, const std::vector<int> &Value);

/// Get a size_t type constant.
ConstantInt *getSizet(Module *M, uint64_t Value);

/// Get metadata operand as int.
int64_t getMDOperandAsInt(MDNode *N, unsigned I);

/// Get metadata operand as string.
StringRef getMDOperandAsString(MDNode *N, unsigned I);

/// Get metadata operand as another metadata node
MDNode *getMDOperandAsMDNode(MDNode *N, unsigned I);

/// Get metadata operand as type.
Type *getMDOperandAsType(MDNode *N, unsigned I);

/// Get a named metadata as a set of string.
/// Assume the named metadata has one or more operands each of which might
/// contain set of strings. For instance:
/// !opencl.used.optional.core.features = !{!0}
/// !0 = !{!"cl_doubles", !"cl_images"}
/// or if we linked two modules we may have
/// !opencl.used.optional.core.features = !{!0, !1}
/// !0 = !{!"cl_doubles"}
/// !1 = !{!"cl_images"}
std::set<std::string> getNamedMDAsStringSet(Module *M,
                                            const std::string &MDName);

/// Get SPIR-V language by SPIR-V metadata spirv.Source
std::tuple<unsigned, unsigned, std::string> getSPIRVSource(Module *M);

/// Get postfix _R{ReturnType} for return type
/// The returned postfix does not includ "_" at the beginning
std::string getPostfixForReturnType(CallInst *CI, bool IsSigned = false);
std::string getPostfixForReturnType(const Type *PRetTy, bool IsSigned = false,
                                    Type *PointerElementType = nullptr);

Constant *getScalarOrVectorConstantInt(Type *T, uint64_t V,
                                       bool IsSigned = false);

/// Get a constant int or a constant int array.
/// \param T is the type of the constant. It should be an integer type or
//  an integer pointer type.
/// \param Len is the length of the array.
/// \param V is the value to fill the array.
Value *getScalarOrArrayConstantInt(Instruction *P, Type *T, unsigned Len,
                                   uint64_t V, bool IsSigned = false);

/// Get the array from GEP.
/// \param V is a GEP whose pointer operand is a pointer to an array of size
/// \param Size.
Value *getScalarOrArray(Value *V, unsigned Size, Instruction *Pos);

void dumpUsers(Value *V, StringRef Prompt = "");

/// Get SPIR-V type name as spirv.BaseTyName.Postfixes.
std::string getSPIRVTypeName(StringRef BaseTyName, StringRef Postfixes = "");

/// Checks if given type name is either ConstantSampler or ConsantPipeStorage.
bool isSPIRVConstantName(StringRef TyName);

/// Get LLVM type for sampled type of SPIR-V image type by postfix.
Type *getLLVMTypeForSPIRVImageSampledTypePostfix(StringRef Postfix,
                                                 LLVMContext &Ctx);

/// Convert an LLVM type to a string postfix name.
std::string convertTypeToPostfix(Type *T);

/// Return the unqualified and unsuffixed base name of an image type.
/// E.g. opencl.image2d_ro_t.3 -> image2d_t
std::string getImageBaseTypeName(StringRef Name);

/// Extract the image type descriptor from the given image type.
SPIRVTypeImageDescriptor getImageDescriptor(Type *Ty);

/// Return the index of image operands given an image op.
size_t getImageOperandsIndex(Op OpCode);

/// Check if access qualifier is encoded in the type name.
bool hasAccessQualifiedName(StringRef TyName);

/// Get access qualifier from the type name.
SPIRVAccessQualifierKind getAccessQualifier(StringRef TyName);

/// Get access qualifier from the type name.
StringRef getAccessQualifierPostfix(SPIRVAccessQualifierKind Access);

bool eraseUselessFunctions(Module *M);

/// Erase a function if it is declaration, has internal linkage and has no use.
bool eraseIfNoUse(Function *F);

// Check if a mangled type name is unsigned
bool isMangledTypeUnsigned(char Mangled);

// Check if a mangled type name is signed
bool isMangledTypeSigned(char Mangled);

// Check if a mangled type name is floating point (except half)
bool isMangledTypeFP(char Mangled);

// Check if a mangled type name is half
bool isMangledTypeHalf(std::string Mangled);

// Check if \param I is valid vector size: 2, 3, 4, 8, 16.
bool isValidVectorSize(unsigned I);

enum class ParamType { FLOAT = 0, SIGNED = 1, UNSIGNED = 2, UNKNOWN = 3 };

ParamType lastFuncParamType(StringRef MangledName);

// Check if the last function parameter is signed
bool isLastFuncParamSigned(StringRef MangledName);

// Check if a mangled function name contains unsigned atomic type
bool containsUnsignedAtomicType(StringRef Name);

/// Mangle builtin function name.
/// \return \param UniqName if \param BtnInfo is null pointer, otherwise
///    return IA64 mangled name.
std::string mangleBuiltin(StringRef UniqName, ArrayRef<Type *> ArgTypes,
                          BuiltinFuncMangleInfo *BtnInfo);

/// Extract the true pointer types, expressed as a TypedPointerType, of
/// arguments from a mangled function name. If the corresponding type is not a
/// pointer type, its value will be the argument's actual type instead. Returns
/// true if the function name was successfully demangled.
bool getParameterTypes(
    Function *F, SmallVectorImpl<Type *> &ArgTys,
    std::function<std::string(StringRef)> StructNameMapFn = nullptr);
inline bool getParameterTypes(CallInst *CI, SmallVectorImpl<Type *> &ArgTys) {
  return getParameterTypes(CI->getCalledFunction(), ArgTys);
}

enum class ParamSignedness { Signed = 0, Unsigned, Unknown };

/// Extract signedness of return type and parameter types from a mangled
/// function name.
bool getRetParamSignedness(Function *F, ParamSignedness &RetSignedness,
                           SmallVectorImpl<ParamSignedness> &ArgSignedness);

/// Mangle a function from OpenCL extended instruction set in SPIR-V friendly IR
/// manner
std::string getSPIRVFriendlyIRFunctionName(OCLExtOpKind ExtOpId,
                                           ArrayRef<Type *> ArgTys,
                                           Type *RetTy = nullptr);

/// Mangle a function in SPIR-V friendly IR manner
/// \param UniqName full unmangled name of the SPIR-V built-in function that
/// contains possible postfixes that depend not on opcode but on decorations or
/// return type, for example __spirv_UConvert_Rint_sat.
/// \param OC opcode of corresponding built-in instruction. Used to gather info
/// for unsigned/constant arguments.
/// \param Types of arguments of SPIR-V built-in function
/// \param Ops Operands of SPIRVInstruction
/// \return IA64 mangled name.
std::string getSPIRVFriendlyIRFunctionName(const std::string &UniqName,
                                           spv::Op OC, ArrayRef<Type *> ArgTys,
                                           ArrayRef<SPIRVValue *> Ops);

/// Get i8* with the same address space.
PointerType *getInt8PtrTy(PointerType *T);

/// Cast a value to a i8* by inserting a cast instruction.
Value *castToInt8Ptr(Value *V, Instruction *Pos);

template <> inline void SPIRVMap<std::string, Op, SPIRVOpaqueType>::init() {
#define _SPIRV_OP(x) add(#x, OpType##x);
  _SPIRV_OP(DeviceEvent)
  _SPIRV_OP(Event)
  _SPIRV_OP(Image)
  _SPIRV_OP(Pipe)
  _SPIRV_OP(Queue)
  _SPIRV_OP(ReserveId)
  _SPIRV_OP(Sampler)
  _SPIRV_OP(SampledImage)
  _SPIRV_OP(PipeStorage)
  // SPV_INTEL_device_side_avc_motion_estimation types
  _SPIRV_OP(AvcMcePayloadINTEL)
  _SPIRV_OP(AvcImePayloadINTEL)
  _SPIRV_OP(AvcRefPayloadINTEL)
  _SPIRV_OP(AvcSicPayloadINTEL)
  _SPIRV_OP(AvcMceResultINTEL)
  _SPIRV_OP(AvcImeResultINTEL)
  _SPIRV_OP(AvcImeResultSingleReferenceStreamoutINTEL)
  _SPIRV_OP(AvcImeResultDualReferenceStreamoutINTEL)
  _SPIRV_OP(AvcImeSingleReferenceStreaminINTEL)
  _SPIRV_OP(AvcImeDualReferenceStreaminINTEL)
  _SPIRV_OP(AvcRefResultINTEL)
  _SPIRV_OP(AvcSicResultINTEL)
  _SPIRV_OP(VmeImageINTEL)
  _SPIRV_OP(BufferSurfaceINTEL)
  _SPIRV_OP(CooperativeMatrixKHR)
#undef _SPIRV_OP
  add("JointMatrixINTEL", internal::OpTypeJointMatrixINTEL);
  add("TaskSequenceINTEL", internal::OpTypeTaskSequenceINTEL);
}

// Check if the module contains llvm.loop.* metadata
bool hasLoopMetadata(const Module *M);

// Check if CI is a call to instruction from OpenCL Extended Instruction Set.
// If so, return it's extended opcode in ExtOp.
bool isSPIRVOCLExtInst(const CallInst *CI, OCLExtOpKind *ExtOp);

/// Returns true if a function name corresponds to an OpenCL builtin that is not
/// expected to have name mangling.
bool isNonMangledOCLBuiltin(StringRef Name);

// check LLVM Intrinsics type(s) for validity
bool checkTypeForSPIRVExtendedInstLowering(IntrinsicInst *II, SPIRVModule *BM);

/// Decode SPIR-V type name in the format spirv.{TypeName}._{Postfixes}
/// where Postfixes are strings separated by underscores.
/// \return TypeName.
/// \param Strs contains the integers decoded from postfixes.
std::string decodeSPIRVTypeName(StringRef Name,
                                SmallVectorImpl<std::string> &Strs);

// Copy attributes from function to call site.
CallInst *setAttrByCalledFunc(CallInst *Call);
bool isSPIRVBuiltinVariable(GlobalVariable *GV, SPIRVBuiltinVariableKind *Kind);
// Transform builtin variable from GlobalVariable to builtin call.
// e.g.
// - GlobalInvolcationId[x] -> _Z33__spirv_BuiltInGlobalInvocationIdi(x)
// - WorkDim -> _Z22__spirv_BuiltInWorkDimv()
bool lowerBuiltinVariableToCall(GlobalVariable *GV,
                                SPIRVBuiltinVariableKind Kind);
// Transform all builtin variables into calls
bool lowerBuiltinVariablesToCalls(Module *M);

// Transform all builtin calls into variables
bool lowerBuiltinCallsToVariables(Module *M);

//  Transform all builtins into variables or calls
//  depending on user specification
bool lowerBuiltins(SPIRVModule *BM, Module *M);

/// \brief Post-process OpenCL or SPIRV builtin function returning struct type.
///
/// Some builtin functions are translated to SPIR-V instructions with
/// struct type result, e.g. NDRange creation functions. Such functions
/// need to be post-processed to return the struct through sret argument.
bool postProcessBuiltinReturningStruct(Function *F);

/// \brief Post-process OpenCL or SPIRV builtin function having array argument.
///
/// These functions are translated to functions with array type argument
/// first, then post-processed to have pointer arguments.
bool postProcessBuiltinWithArrayArguments(Function *F, StringRef DemangledName);

bool postProcessBuiltinsReturningStruct(Module *M, bool IsCpp = false);

bool postProcessBuiltinsWithArrayArguments(Module *M, bool IsCpp = false);

} // namespace SPIRV

#endif // SPIRV_SPIRVINTERNAL_H