contrib/llvm/tools/clang/lib/CodeGen/CGBuiltin.cpp

   1 //===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This contains code to emit Builtin calls as LLVM code.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "CGCXXABI.h"
  15 #include "CGObjCRuntime.h"
  16 #include "CGOpenCLRuntime.h"
  17 #include "CodeGenFunction.h"
  18 #include "CodeGenModule.h"
  19 #include "TargetInfo.h"
  20 #include "clang/AST/ASTContext.h"
  21 #include "clang/AST/Decl.h"
  22 #include "clang/Analysis/Analyses/OSLog.h"
  23 #include "clang/Basic/TargetBuiltins.h"
  24 #include "clang/Basic/TargetInfo.h"
  25 #include "clang/CodeGen/CGFunctionInfo.h"
  26 #include "llvm/ADT/StringExtras.h"
  27 #include "llvm/IR/CallSite.h"
  28 #include "llvm/IR/DataLayout.h"
  29 #include "llvm/IR/InlineAsm.h"
  30 #include "llvm/IR/Intrinsics.h"
  31 #include "llvm/IR/MDBuilder.h"
  32 #include <sstream>
  33
  34 using namespace clang;
  35 using namespace CodeGen;
  36 using namespace llvm;
  37
  38 /// getBuiltinLibFunction - Given a builtin id for a function like
  39 /// "__builtin_fabsf", return a Function* for "fabsf".
  40 llvm::Constant *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD,
  41                                                      unsigned BuiltinID) {
  42   assert(Context.BuiltinInfo.isLibFunction(BuiltinID));
  43
  44   // Get the name, skip over the __builtin_ prefix (if necessary).
  45   StringRef Name;
  46   GlobalDecl D(FD);
  47
  48   // If the builtin has been declared explicitly with an assembler label,
  49   // use the mangled name. This differs from the plain label on platforms
  50   // that prefix labels.
  51   if (FD->hasAttr<AsmLabelAttr>())
  52     Name = getMangledName(D);
  53   else
  54     Name = Context.BuiltinInfo.getName(BuiltinID) + 10;
  55
  56   llvm::FunctionType *Ty =
  57     cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType()));
  58
  59   return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false);
  60 }
  61
  62 /// Emit the conversions required to turn the given value into an
  63 /// integer of the given size.
  64 static Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V,
  65                         QualType T, llvm::IntegerType *IntType) {
  66   V = CGF.EmitToMemory(V, T);
  67
  68   if (V->getType()->isPointerTy())
  69     return CGF.Builder.CreatePtrToInt(V, IntType);
  70
  71   assert(V->getType() == IntType);
  72   return V;
  73 }
  74
  75 static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,
  76                           QualType T, llvm::Type *ResultType) {
  77   V = CGF.EmitFromMemory(V, T);
  78
  79   if (ResultType->isPointerTy())
  80     return CGF.Builder.CreateIntToPtr(V, ResultType);
  81
  82   assert(V->getType() == ResultType);
  83   return V;
  84 }
  85
  86 /// Utility to insert an atomic instruction based on Instrinsic::ID
  87 /// and the expression node.
  88 static Value *MakeBinaryAtomicValue(CodeGenFunction &CGF,
  89                                     llvm::AtomicRMWInst::BinOp Kind,
  90                                     const CallExpr *E) {
  91   QualType T = E->getType();
  92   assert(E->getArg(0)->getType()->isPointerType());
  93   assert(CGF.getContext().hasSameUnqualifiedType(T,
  94                                   E->getArg(0)->getType()->getPointeeType()));
  95   assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
  96
  97   llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
  98   unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
  99
 100   llvm::IntegerType *IntType =
 101     llvm::IntegerType::get(CGF.getLLVMContext(),
 102                            CGF.getContext().getTypeSize(T));
 103   llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
 104
 105   llvm::Value *Args[2];
 106   Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
 107   Args[1] = CGF.EmitScalarExpr(E->getArg(1));
 108   llvm::Type *ValueType = Args[1]->getType();
 109   Args[1] = EmitToInt(CGF, Args[1], T, IntType);
 110
 111   llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
 112       Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);
 113   return EmitFromInt(CGF, Result, T, ValueType);
 114 }
 115
 116 static Value *EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E) {
 117   Value *Val = CGF.EmitScalarExpr(E->getArg(0));
 118   Value *Address = CGF.EmitScalarExpr(E->getArg(1));
 119
 120   // Convert the type of the pointer to a pointer to the stored type.
 121   Val = CGF.EmitToMemory(Val, E->getArg(0)->getType());
 122   Value *BC = CGF.Builder.CreateBitCast(
 123       Address, llvm::PointerType::getUnqual(Val->getType()), "cast");
 124   LValue LV = CGF.MakeNaturalAlignAddrLValue(BC, E->getArg(0)->getType());
 125   LV.setNontemporal(true);
 126   CGF.EmitStoreOfScalar(Val, LV, false);
 127   return nullptr;
 128 }
 129
 130 static Value *EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E) {
 131   Value *Address = CGF.EmitScalarExpr(E->getArg(0));
 132
 133   LValue LV = CGF.MakeNaturalAlignAddrLValue(Address, E->getType());
 134   LV.setNontemporal(true);
 135   return CGF.EmitLoadOfScalar(LV, E->getExprLoc());
 136 }
 137
 138 static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
 139                                llvm::AtomicRMWInst::BinOp Kind,
 140                                const CallExpr *E) {
 141   return RValue::get(MakeBinaryAtomicValue(CGF, Kind, E));
 142 }
 143
 144 /// Utility to insert an atomic instruction based Instrinsic::ID and
 145 /// the expression node, where the return value is the result of the
 146 /// operation.
 147 static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
 148                                    llvm::AtomicRMWInst::BinOp Kind,
 149                                    const CallExpr *E,
 150                                    Instruction::BinaryOps Op,
 151                                    bool Invert = false) {
 152   QualType T = E->getType();
 153   assert(E->getArg(0)->getType()->isPointerType());
 154   assert(CGF.getContext().hasSameUnqualifiedType(T,
 155                                   E->getArg(0)->getType()->getPointeeType()));
 156   assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
 157
 158   llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
 159   unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
 160
 161   llvm::IntegerType *IntType =
 162     llvm::IntegerType::get(CGF.getLLVMContext(),
 163                            CGF.getContext().getTypeSize(T));
 164   llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
 165
 166   llvm::Value *Args[2];
 167   Args[1] = CGF.EmitScalarExpr(E->getArg(1));
 168   llvm::Type *ValueType = Args[1]->getType();
 169   Args[1] = EmitToInt(CGF, Args[1], T, IntType);
 170   Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
 171
 172   llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
 173       Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);
 174   Result = CGF.Builder.CreateBinOp(Op, Result, Args[1]);
 175   if (Invert)
 176     Result = CGF.Builder.CreateBinOp(llvm::Instruction::Xor, Result,
 177                                      llvm::ConstantInt::get(IntType, -1));
 178   Result = EmitFromInt(CGF, Result, T, ValueType);
 179   return RValue::get(Result);
 180 }
 181
 182 /// @brief Utility to insert an atomic cmpxchg instruction.
 183 ///
 184 /// @param CGF The current codegen function.
 185 /// @param E   Builtin call expression to convert to cmpxchg.
 186 ///            arg0 - address to operate on
 187 ///            arg1 - value to compare with
 188 ///            arg2 - new value
 189 /// @param ReturnBool Specifies whether to return success flag of
 190 ///                   cmpxchg result or the old value.
 191 ///
 192 /// @returns result of cmpxchg, according to ReturnBool
 193 static Value *MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E,
 194                                      bool ReturnBool) {
 195   QualType T = ReturnBool ? E->getArg(1)->getType() : E->getType();
 196   llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
 197   unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
 198
 199   llvm::IntegerType *IntType = llvm::IntegerType::get(
 200       CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));
 201   llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
 202
 203   Value *Args[3];
 204   Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
 205   Args[1] = CGF.EmitScalarExpr(E->getArg(1));
 206   llvm::Type *ValueType = Args[1]->getType();
 207   Args[1] = EmitToInt(CGF, Args[1], T, IntType);
 208   Args[2] = EmitToInt(CGF, CGF.EmitScalarExpr(E->getArg(2)), T, IntType);
 209
 210   Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
 211       Args[0], Args[1], Args[2], llvm::AtomicOrdering::SequentiallyConsistent,
 212       llvm::AtomicOrdering::SequentiallyConsistent);
 213   if (ReturnBool)
 214     // Extract boolean success flag and zext it to int.
 215     return CGF.Builder.CreateZExt(CGF.Builder.CreateExtractValue(Pair, 1),
 216                                   CGF.ConvertType(E->getType()));
 217   else
 218     // Extract old value and emit it using the same type as compare value.
 219     return EmitFromInt(CGF, CGF.Builder.CreateExtractValue(Pair, 0), T,
 220                        ValueType);
 221 }
 222
 223 // Emit a simple mangled intrinsic that has 1 argument and a return type
 224 // matching the argument type.
 225 static Value *emitUnaryBuiltin(CodeGenFunction &CGF,
 226                                const CallExpr *E,
 227                                unsigned IntrinsicID) {
 228   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
 229
 230   Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
 231   return CGF.Builder.CreateCall(F, Src0);
 232 }
 233
 234 // Emit an intrinsic that has 2 operands of the same type as its result.
 235 static Value *emitBinaryBuiltin(CodeGenFunction &CGF,
 236                                 const CallExpr *E,
 237                                 unsigned IntrinsicID) {
 238   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
 239   llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
 240
 241   Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
 242   return CGF.Builder.CreateCall(F, { Src0, Src1 });
 243 }
 244
 245 // Emit an intrinsic that has 3 operands of the same type as its result.
 246 static Value *emitTernaryBuiltin(CodeGenFunction &CGF,
 247                                  const CallExpr *E,
 248                                  unsigned IntrinsicID) {
 249   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
 250   llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
 251   llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2));
 252
 253   Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
 254   return CGF.Builder.CreateCall(F, { Src0, Src1, Src2 });
 255 }
 256
 257 // Emit an intrinsic that has 1 float or double operand, and 1 integer.
 258 static Value *emitFPIntBuiltin(CodeGenFunction &CGF,
 259                                const CallExpr *E,
 260                                unsigned IntrinsicID) {
 261   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
 262   llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
 263
 264   Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
 265   return CGF.Builder.CreateCall(F, {Src0, Src1});
 266 }
 267
 268 /// EmitFAbs - Emit a call to @llvm.fabs().
 269 static Value *EmitFAbs(CodeGenFunction &CGF, Value *V) {
 270   Value *F = CGF.CGM.getIntrinsic(Intrinsic::fabs, V->getType());
 271   llvm::CallInst *Call = CGF.Builder.CreateCall(F, V);
 272   Call->setDoesNotAccessMemory();
 273   return Call;
 274 }
 275
 276 /// Emit the computation of the sign bit for a floating point value. Returns
 277 /// the i1 sign bit value.
 278 static Value *EmitSignBit(CodeGenFunction &CGF, Value *V) {
 279   LLVMContext &C = CGF.CGM.getLLVMContext();
 280
 281   llvm::Type *Ty = V->getType();
 282   int Width = Ty->getPrimitiveSizeInBits();
 283   llvm::Type *IntTy = llvm::IntegerType::get(C, Width);
 284   V = CGF.Builder.CreateBitCast(V, IntTy);
 285   if (Ty->isPPC_FP128Ty()) {
 286     // We want the sign bit of the higher-order double. The bitcast we just
 287     // did works as if the double-double was stored to memory and then
 288     // read as an i128. The "store" will put the higher-order double in the
 289     // lower address in both little- and big-Endian modes, but the "load"
 290     // will treat those bits as a different part of the i128: the low bits in
 291     // little-Endian, the high bits in big-Endian. Therefore, on big-Endian
 292     // we need to shift the high bits down to the low before truncating.
 293     Width >>= 1;
 294     if (CGF.getTarget().isBigEndian()) {
 295       Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
 296       V = CGF.Builder.CreateLShr(V, ShiftCst);
 297     }
 298     // We are truncating value in order to extract the higher-order
 299     // double, which we will be using to extract the sign from.
 300     IntTy = llvm::IntegerType::get(C, Width);
 301     V = CGF.Builder.CreateTrunc(V, IntTy);
 302   }
 303   Value *Zero = llvm::Constant::getNullValue(IntTy);
 304   return CGF.Builder.CreateICmpSLT(V, Zero);
 305 }
 306
 307 static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD,
 308                               const CallExpr *E, llvm::Constant *calleeValue) {
 309   CGCallee callee = CGCallee::forDirect(calleeValue, FD);
 310   return CGF.EmitCall(E->getCallee()->getType(), callee, E, ReturnValueSlot());
 311 }
 312
 313 /// \brief Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.*
 314 /// depending on IntrinsicID.
 315 ///
 316 /// \arg CGF The current codegen function.
 317 /// \arg IntrinsicID The ID for the Intrinsic we wish to generate.
 318 /// \arg X The first argument to the llvm.*.with.overflow.*.
 319 /// \arg Y The second argument to the llvm.*.with.overflow.*.
 320 /// \arg Carry The carry returned by the llvm.*.with.overflow.*.
 321 /// \returns The result (i.e. sum/product) returned by the intrinsic.
 322 static llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF,
 323                                           const llvm::Intrinsic::ID IntrinsicID,
 324                                           llvm::Value *X, llvm::Value *Y,
 325                                           llvm::Value *&Carry) {
 326   // Make sure we have integers of the same width.
 327   assert(X->getType() == Y->getType() &&
 328          "Arguments must be the same type. (Did you forget to make sure both "
 329          "arguments have the same integer width?)");
 330
 331   llvm::Value *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType());
 332   llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, {X, Y});
 333   Carry = CGF.Builder.CreateExtractValue(Tmp, 1);
 334   return CGF.Builder.CreateExtractValue(Tmp, 0);
 335 }
 336
 337 static Value *emitRangedBuiltin(CodeGenFunction &CGF,
 338                                 unsigned IntrinsicID,
 339                                 int low, int high) {
 340     llvm::MDBuilder MDHelper(CGF.getLLVMContext());
 341     llvm::MDNode *RNode = MDHelper.createRange(APInt(32, low), APInt(32, high));
 342     Value *F = CGF.CGM.getIntrinsic(IntrinsicID, {});
 343     llvm::Instruction *Call = CGF.Builder.CreateCall(F);
 344     Call->setMetadata(llvm::LLVMContext::MD_range, RNode);
 345     return Call;
 346 }
 347
 348 namespace {
 349   struct WidthAndSignedness {
 350     unsigned Width;
 351     bool Signed;
 352   };
 353 }
 354
 355 static WidthAndSignedness
 356 getIntegerWidthAndSignedness(const clang::ASTContext &context,
 357                              const clang::QualType Type) {
 358   assert(Type->isIntegerType() && "Given type is not an integer.");
 359   unsigned Width = Type->isBooleanType() ? 1 : context.getTypeInfo(Type).Width;
 360   bool Signed = Type->isSignedIntegerType();
 361   return {Width, Signed};
 362 }
 363
 364 // Given one or more integer types, this function produces an integer type that
 365 // encompasses them: any value in one of the given types could be expressed in
 366 // the encompassing type.
 367 static struct WidthAndSignedness
 368 EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) {
 369   assert(Types.size() > 0 && "Empty list of types.");
 370
 371   // If any of the given types is signed, we must return a signed type.
 372   bool Signed = false;
 373   for (const auto &Type : Types) {
 374     Signed |= Type.Signed;
 375   }
 376
 377   // The encompassing type must have a width greater than or equal to the width
 378   // of the specified types.  Aditionally, if the encompassing type is signed,
 379   // its width must be strictly greater than the width of any unsigned types
 380   // given.
 381   unsigned Width = 0;
 382   for (const auto &Type : Types) {
 383     unsigned MinWidth = Type.Width + (Signed && !Type.Signed);
 384     if (Width < MinWidth) {
 385       Width = MinWidth;
 386     }
 387   }
 388
 389   return {Width, Signed};
 390 }
 391
 392 Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) {
 393   llvm::Type *DestType = Int8PtrTy;
 394   if (ArgValue->getType() != DestType)
 395     ArgValue =
 396         Builder.CreateBitCast(ArgValue, DestType, ArgValue->getName().data());
 397
 398   Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
 399   return Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue);
 400 }
 401
 402 /// Checks if using the result of __builtin_object_size(p, @p From) in place of
 403 /// __builtin_object_size(p, @p To) is correct
 404 static bool areBOSTypesCompatible(int From, int To) {
 405   // Note: Our __builtin_object_size implementation currently treats Type=0 and
 406   // Type=2 identically. Encoding this implementation detail here may make
 407   // improving __builtin_object_size difficult in the future, so it's omitted.
 408   return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
 409 }
 410
 411 static llvm::Value *
 412 getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType) {
 413   return ConstantInt::get(ResType, (Type & 2) ? 0 : -1, /*isSigned=*/true);
 414 }
 415
 416 llvm::Value *
 417 CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
 418                                                  llvm::IntegerType *ResType) {
 419   uint64_t ObjectSize;
 420   if (!E->tryEvaluateObjectSize(ObjectSize, getContext(), Type))
 421     return emitBuiltinObjectSize(E, Type, ResType);
 422   return ConstantInt::get(ResType, ObjectSize, /*isSigned=*/true);
 423 }
 424
 425 /// Returns a Value corresponding to the size of the given expression.
 426 /// This Value may be either of the following:
 427 ///   - A llvm::Argument (if E is a param with the pass_object_size attribute on
 428 ///     it)
 429 ///   - A call to the @llvm.objectsize intrinsic
 430 llvm::Value *
 431 CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type,
 432                                        llvm::IntegerType *ResType) {
 433   // We need to reference an argument if the pointer is a parameter with the
 434   // pass_object_size attribute.
 435   if (auto *D = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts())) {
 436     auto *Param = dyn_cast<ParmVarDecl>(D->getDecl());
 437     auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
 438     if (Param != nullptr && PS != nullptr &&
 439         areBOSTypesCompatible(PS->getType(), Type)) {
 440       auto Iter = SizeArguments.find(Param);
 441       assert(Iter != SizeArguments.end());
 442
 443       const ImplicitParamDecl *D = Iter->second;
 444       auto DIter = LocalDeclMap.find(D);
 445       assert(DIter != LocalDeclMap.end());
 446
 447       return EmitLoadOfScalar(DIter->second, /*volatile=*/false,
 448                               getContext().getSizeType(), E->getLocStart());
 449     }
 450   }
 451
 452   // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't
 453   // evaluate E for side-effects. In either case, we shouldn't lower to
 454   // @llvm.objectsize.
 455   if (Type == 3 || E->HasSideEffects(getContext()))
 456     return getDefaultBuiltinObjectSizeResult(Type, ResType);
 457
 458   // LLVM only supports 0 and 2, make sure that we pass along that
 459   // as a boolean.
 460   auto *CI = ConstantInt::get(Builder.getInt1Ty(), (Type & 2) >> 1);
 461   // FIXME: Get right address space.
 462   llvm::Type *Tys[] = {ResType, Builder.getInt8PtrTy(0)};
 463   Value *F = CGM.getIntrinsic(Intrinsic::objectsize, Tys);
 464   return Builder.CreateCall(F, {EmitScalarExpr(E), CI});
 465 }
 466
 467 // Many of MSVC builtins are on both x64 and ARM; to avoid repeating code, we
 468 // handle them here.
 469 enum class CodeGenFunction::MSVCIntrin {
 470   _BitScanForward,
 471   _BitScanReverse,
 472   _InterlockedAnd,
 473   _InterlockedDecrement,
 474   _InterlockedExchange,
 475   _InterlockedExchangeAdd,
 476   _InterlockedExchangeSub,
 477   _InterlockedIncrement,
 478   _InterlockedOr,
 479   _InterlockedXor,
 480 };
 481
 482 Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID,
 483   const CallExpr *E) {
 484   switch (BuiltinID) {
 485   case MSVCIntrin::_BitScanForward:
 486   case MSVCIntrin::_BitScanReverse: {
 487     Value *ArgValue = EmitScalarExpr(E->getArg(1));
 488
 489     llvm::Type *ArgType = ArgValue->getType();
 490     llvm::Type *IndexType =
 491       EmitScalarExpr(E->getArg(0))->getType()->getPointerElementType();
 492     llvm::Type *ResultType = ConvertType(E->getType());
 493
 494     Value *ArgZero = llvm::Constant::getNullValue(ArgType);
 495     Value *ResZero = llvm::Constant::getNullValue(ResultType);
 496     Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
 497
 498     BasicBlock *Begin = Builder.GetInsertBlock();
 499     BasicBlock *End = createBasicBlock("bitscan_end", this->CurFn);
 500     Builder.SetInsertPoint(End);
 501     PHINode *Result = Builder.CreatePHI(ResultType, 2, "bitscan_result");
 502
 503     Builder.SetInsertPoint(Begin);
 504     Value *IsZero = Builder.CreateICmpEQ(ArgValue, ArgZero);
 505     BasicBlock *NotZero = createBasicBlock("bitscan_not_zero", this->CurFn);
 506     Builder.CreateCondBr(IsZero, End, NotZero);
 507     Result->addIncoming(ResZero, Begin);
 508
 509     Builder.SetInsertPoint(NotZero);
 510     Address IndexAddress = EmitPointerWithAlignment(E->getArg(0));
 511
 512     if (BuiltinID == MSVCIntrin::_BitScanForward) {
 513       Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
 514       Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
 515       ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType, false);
 516       Builder.CreateStore(ZeroCount, IndexAddress, false);
 517     } else {
 518       unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
 519       Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
 520
 521       Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
 522       Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
 523       ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType, false);
 524       Value *Index = Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
 525       Builder.CreateStore(Index, IndexAddress, false);
 526     }
 527     Builder.CreateBr(End);
 528     Result->addIncoming(ResOne, NotZero);
 529
 530     Builder.SetInsertPoint(End);
 531     return Result;
 532   }
 533   case MSVCIntrin::_InterlockedAnd:
 534     return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E);
 535   case MSVCIntrin::_InterlockedExchange:
 536     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E);
 537   case MSVCIntrin::_InterlockedExchangeAdd:
 538     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E);
 539   case MSVCIntrin::_InterlockedExchangeSub:
 540     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Sub, E);
 541   case MSVCIntrin::_InterlockedOr:
 542     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E);
 543   case MSVCIntrin::_InterlockedXor:
 544     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E);
 545
 546   case MSVCIntrin::_InterlockedDecrement: {
 547     llvm::Type *IntTy = ConvertType(E->getType());
 548     AtomicRMWInst *RMWI = Builder.CreateAtomicRMW(
 549       AtomicRMWInst::Sub,
 550       EmitScalarExpr(E->getArg(0)),
 551       ConstantInt::get(IntTy, 1),
 552       llvm::AtomicOrdering::SequentiallyConsistent);
 553     return Builder.CreateSub(RMWI, ConstantInt::get(IntTy, 1));
 554   }
 555   case MSVCIntrin::_InterlockedIncrement: {
 556     llvm::Type *IntTy = ConvertType(E->getType());
 557     AtomicRMWInst *RMWI = Builder.CreateAtomicRMW(
 558       AtomicRMWInst::Add,
 559       EmitScalarExpr(E->getArg(0)),
 560       ConstantInt::get(IntTy, 1),
 561       llvm::AtomicOrdering::SequentiallyConsistent);
 562     return Builder.CreateAdd(RMWI, ConstantInt::get(IntTy, 1));
 563   }
 564   }
 565   llvm_unreachable("Incorrect MSVC intrinsic!");
 566 }
 567
 568 namespace {
 569 // ARC cleanup for __builtin_os_log_format
 570 struct CallObjCArcUse final : EHScopeStack::Cleanup {
 571   CallObjCArcUse(llvm::Value *object) : object(object) {}
 572   llvm::Value *object;
 573
 574   void Emit(CodeGenFunction &CGF, Flags flags) override {
 575     CGF.EmitARCIntrinsicUse(object);
 576   }
 577 };
 578 }
 579
 580 RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
 581                                         unsigned BuiltinID, const CallExpr *E,
 582                                         ReturnValueSlot ReturnValue) {
 583   // See if we can constant fold this builtin.  If so, don't emit it at all.
 584   Expr::EvalResult Result;
 585   if (E->EvaluateAsRValue(Result, CGM.getContext()) &&
 586       !Result.hasSideEffects()) {
 587     if (Result.Val.isInt())
 588       return RValue::get(llvm::ConstantInt::get(getLLVMContext(),
 589                                                 Result.Val.getInt()));
 590     if (Result.Val.isFloat())
 591       return RValue::get(llvm::ConstantFP::get(getLLVMContext(),
 592                                                Result.Val.getFloat()));
 593   }
 594
 595   switch (BuiltinID) {
 596   default: break;  // Handle intrinsics and libm functions below.
 597   case Builtin::BI__builtin___CFStringMakeConstantString:
 598   case Builtin::BI__builtin___NSStringMakeConstantString:
 599     return RValue::get(CGM.EmitConstantExpr(E, E->getType(), nullptr));
 600   case Builtin::BI__builtin_stdarg_start:
 601   case Builtin::BI__builtin_va_start:
 602   case Builtin::BI__va_start:
 603   case Builtin::BI__builtin_va_end:
 604     return RValue::get(
 605         EmitVAStartEnd(BuiltinID == Builtin::BI__va_start
 606                            ? EmitScalarExpr(E->getArg(0))
 607                            : EmitVAListRef(E->getArg(0)).getPointer(),
 608                        BuiltinID != Builtin::BI__builtin_va_end));
 609   case Builtin::BI__builtin_va_copy: {
 610     Value *DstPtr = EmitVAListRef(E->getArg(0)).getPointer();
 611     Value *SrcPtr = EmitVAListRef(E->getArg(1)).getPointer();
 612
 613     llvm::Type *Type = Int8PtrTy;
 614
 615     DstPtr = Builder.CreateBitCast(DstPtr, Type);
 616     SrcPtr = Builder.CreateBitCast(SrcPtr, Type);
 617     return RValue::get(Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy),
 618                                           {DstPtr, SrcPtr}));
 619   }
 620   case Builtin::BI__builtin_abs:
 621   case Builtin::BI__builtin_labs:
 622   case Builtin::BI__builtin_llabs: {
 623     Value *ArgValue = EmitScalarExpr(E->getArg(0));
 624
 625     Value *NegOp = Builder.CreateNeg(ArgValue, "neg");
 626     Value *CmpResult =
 627     Builder.CreateICmpSGE(ArgValue,
 628                           llvm::Constant::getNullValue(ArgValue->getType()),
 629                                                             "abscond");
 630     Value *Result =
 631       Builder.CreateSelect(CmpResult, ArgValue, NegOp, "abs");
 632
 633     return RValue::get(Result);
 634   }
 635   case Builtin::BI__builtin_fabs:
 636   case Builtin::BI__builtin_fabsf:
 637   case Builtin::BI__builtin_fabsl: {
 638     return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::fabs));
 639   }
 640   case Builtin::BI__builtin_fmod:
 641   case Builtin::BI__builtin_fmodf:
 642   case Builtin::BI__builtin_fmodl: {
 643     Value *Arg1 = EmitScalarExpr(E->getArg(0));
 644     Value *Arg2 = EmitScalarExpr(E->getArg(1));
 645     Value *Result = Builder.CreateFRem(Arg1, Arg2, "fmod");
 646     return RValue::get(Result);
 647   }
 648   case Builtin::BI__builtin_copysign:
 649   case Builtin::BI__builtin_copysignf:
 650   case Builtin::BI__builtin_copysignl: {
 651     return RValue::get(emitBinaryBuiltin(*this, E, Intrinsic::copysign));
 652   }
 653   case Builtin::BI__builtin_ceil:
 654   case Builtin::BI__builtin_ceilf:
 655   case Builtin::BI__builtin_ceill: {
 656     return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::ceil));
 657   }
 658   case Builtin::BI__builtin_floor:
 659   case Builtin::BI__builtin_floorf:
 660   case Builtin::BI__builtin_floorl: {
 661     return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::floor));
 662   }
 663   case Builtin::BI__builtin_trunc:
 664   case Builtin::BI__builtin_truncf:
 665   case Builtin::BI__builtin_truncl: {
 666     return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::trunc));
 667   }
 668   case Builtin::BI__builtin_rint:
 669   case Builtin::BI__builtin_rintf:
 670   case Builtin::BI__builtin_rintl: {
 671     return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::rint));
 672   }
 673   case Builtin::BI__builtin_nearbyint:
 674   case Builtin::BI__builtin_nearbyintf:
 675   case Builtin::BI__builtin_nearbyintl: {
 676     return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::nearbyint));
 677   }
 678   case Builtin::BI__builtin_round:
 679   case Builtin::BI__builtin_roundf:
 680   case Builtin::BI__builtin_roundl: {
 681     return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::round));
 682   }
 683   case Builtin::BI__builtin_fmin:
 684   case Builtin::BI__builtin_fminf:
 685   case Builtin::BI__builtin_fminl: {
 686     return RValue::get(emitBinaryBuiltin(*this, E, Intrinsic::minnum));
 687   }
 688   case Builtin::BI__builtin_fmax:
 689   case Builtin::BI__builtin_fmaxf:
 690   case Builtin::BI__builtin_fmaxl: {
 691     return RValue::get(emitBinaryBuiltin(*this, E, Intrinsic::maxnum));
 692   }
 693   case Builtin::BI__builtin_conj:
 694   case Builtin::BI__builtin_conjf:
 695   case Builtin::BI__builtin_conjl: {
 696     ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
 697     Value *Real = ComplexVal.first;
 698     Value *Imag = ComplexVal.second;
 699     Value *Zero =
 700       Imag->getType()->isFPOrFPVectorTy()
 701         ? llvm::ConstantFP::getZeroValueForNegation(Imag->getType())
 702         : llvm::Constant::getNullValue(Imag->getType());
 703
 704     Imag = Builder.CreateFSub(Zero, Imag, "sub");
 705     return RValue::getComplex(std::make_pair(Real, Imag));
 706   }
 707   case Builtin::BI__builtin_creal:
 708   case Builtin::BI__builtin_crealf:
 709   case Builtin::BI__builtin_creall:
 710   case Builtin::BIcreal:
 711   case Builtin::BIcrealf:
 712   case Builtin::BIcreall: {
 713     ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
 714     return RValue::get(ComplexVal.first);
 715   }
 716
 717   case Builtin::BI__builtin_cimag:
 718   case Builtin::BI__builtin_cimagf:
 719   case Builtin::BI__builtin_cimagl:
 720   case Builtin::BIcimag:
 721   case Builtin::BIcimagf:
 722   case Builtin::BIcimagl: {
 723     ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
 724     return RValue::get(ComplexVal.second);
 725   }
 726
 727   case Builtin::BI__builtin_ctzs:
 728   case Builtin::BI__builtin_ctz:
 729   case Builtin::BI__builtin_ctzl:
 730   case Builtin::BI__builtin_ctzll: {
 731     Value *ArgValue = EmitScalarExpr(E->getArg(0));
 732
 733     llvm::Type *ArgType = ArgValue->getType();
 734     Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
 735
 736     llvm::Type *ResultType = ConvertType(E->getType());
 737     Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
 738     Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
 739     if (Result->getType() != ResultType)
 740       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
 741                                      "cast");
 742     return RValue::get(Result);
 743   }
 744   case Builtin::BI__builtin_clzs:
 745   case Builtin::BI__builtin_clz:
 746   case Builtin::BI__builtin_clzl:
 747   case Builtin::BI__builtin_clzll: {
 748     Value *ArgValue = EmitScalarExpr(E->getArg(0));
 749
 750     llvm::Type *ArgType = ArgValue->getType();
 751     Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
 752
 753     llvm::Type *ResultType = ConvertType(E->getType());
 754     Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
 755     Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
 756     if (Result->getType() != ResultType)
 757       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
 758                                      "cast");
 759     return RValue::get(Result);
 760   }
 761   case Builtin::BI__builtin_ffs:
 762   case Builtin::BI__builtin_ffsl:
 763   case Builtin::BI__builtin_ffsll: {
 764     // ffs(x) -> x ? cttz(x) + 1 : 0
 765     Value *ArgValue = EmitScalarExpr(E->getArg(0));
 766
 767     llvm::Type *ArgType = ArgValue->getType();
 768     Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
 769
 770     llvm::Type *ResultType = ConvertType(E->getType());
 771     Value *Tmp =
 772         Builder.CreateAdd(Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
 773                           llvm::ConstantInt::get(ArgType, 1));
 774     Value *Zero = llvm::Constant::getNullValue(ArgType);
 775     Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero");
 776     Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs");
 777     if (Result->getType() != ResultType)
 778       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
 779                                      "cast");
 780     return RValue::get(Result);
 781   }
 782   case Builtin::BI__builtin_parity:
 783   case Builtin::BI__builtin_parityl:
 784   case Builtin::BI__builtin_parityll: {
 785     // parity(x) -> ctpop(x) & 1
 786     Value *ArgValue = EmitScalarExpr(E->getArg(0));
 787
 788     llvm::Type *ArgType = ArgValue->getType();
 789     Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
 790
 791     llvm::Type *ResultType = ConvertType(E->getType());
 792     Value *Tmp = Builder.CreateCall(F, ArgValue);
 793     Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
 794     if (Result->getType() != ResultType)
 795       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
 796                                      "cast");
 797     return RValue::get(Result);
 798   }
 799   case Builtin::BI__popcnt16:
 800   case Builtin::BI__popcnt:
 801   case Builtin::BI__popcnt64:
 802   case Builtin::BI__builtin_popcount:
 803   case Builtin::BI__builtin_popcountl:
 804   case Builtin::BI__builtin_popcountll: {
 805     Value *ArgValue = EmitScalarExpr(E->getArg(0));
 806
 807     llvm::Type *ArgType = ArgValue->getType();
 808     Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
 809
 810     llvm::Type *ResultType = ConvertType(E->getType());
 811     Value *Result = Builder.CreateCall(F, ArgValue);
 812     if (Result->getType() != ResultType)
 813       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
 814                                      "cast");
 815     return RValue::get(Result);
 816   }
 817   case Builtin::BI_rotr8:
 818   case Builtin::BI_rotr16:
 819   case Builtin::BI_rotr:
 820   case Builtin::BI_lrotr:
 821   case Builtin::BI_rotr64: {
 822     Value *Val = EmitScalarExpr(E->getArg(0));
 823     Value *Shift = EmitScalarExpr(E->getArg(1));
 824
 825     llvm::Type *ArgType = Val->getType();
 826     Shift = Builder.CreateIntCast(Shift, ArgType, false);
 827     unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
 828     Value *ArgTypeSize = llvm::ConstantInt::get(ArgType, ArgWidth);
 829     Value *ArgZero = llvm::Constant::getNullValue(ArgType);
 830
 831     Value *Mask = llvm::ConstantInt::get(ArgType, ArgWidth - 1);
 832     Shift = Builder.CreateAnd(Shift, Mask);
 833     Value *LeftShift = Builder.CreateSub(ArgTypeSize, Shift);
 834
 835     Value *RightShifted = Builder.CreateLShr(Val, Shift);
 836     Value *LeftShifted = Builder.CreateShl(Val, LeftShift);
 837     Value *Rotated = Builder.CreateOr(LeftShifted, RightShifted);
 838
 839     Value *ShiftIsZero = Builder.CreateICmpEQ(Shift, ArgZero);
 840     Value *Result = Builder.CreateSelect(ShiftIsZero, Val, Rotated);
 841     return RValue::get(Result);
 842   }
 843   case Builtin::BI_rotl8:
 844   case Builtin::BI_rotl16:
 845   case Builtin::BI_rotl:
 846   case Builtin::BI_lrotl:
 847   case Builtin::BI_rotl64: {
 848     Value *Val = EmitScalarExpr(E->getArg(0));
 849     Value *Shift = EmitScalarExpr(E->getArg(1));
 850
 851     llvm::Type *ArgType = Val->getType();
 852     Shift = Builder.CreateIntCast(Shift, ArgType, false);
 853     unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
 854     Value *ArgTypeSize = llvm::ConstantInt::get(ArgType, ArgWidth);
 855     Value *ArgZero = llvm::Constant::getNullValue(ArgType);
 856
 857     Value *Mask = llvm::ConstantInt::get(ArgType, ArgWidth - 1);
 858     Shift = Builder.CreateAnd(Shift, Mask);
 859     Value *RightShift = Builder.CreateSub(ArgTypeSize, Shift);
 860
 861     Value *LeftShifted = Builder.CreateShl(Val, Shift);
 862     Value *RightShifted = Builder.CreateLShr(Val, RightShift);
 863     Value *Rotated = Builder.CreateOr(LeftShifted, RightShifted);
 864
 865     Value *ShiftIsZero = Builder.CreateICmpEQ(Shift, ArgZero);
 866     Value *Result = Builder.CreateSelect(ShiftIsZero, Val, Rotated);
 867     return RValue::get(Result);
 868   }
 869   case Builtin::BI__builtin_unpredictable: {
 870     // Always return the argument of __builtin_unpredictable. LLVM does not
 871     // handle this builtin. Metadata for this builtin should be added directly
 872     // to instructions such as branches or switches that use it.
 873     return RValue::get(EmitScalarExpr(E->getArg(0)));
 874   }
 875   case Builtin::BI__builtin_expect: {
 876     Value *ArgValue = EmitScalarExpr(E->getArg(0));
 877     llvm::Type *ArgType = ArgValue->getType();
 878
 879     Value *ExpectedValue = EmitScalarExpr(E->getArg(1));
 880     // Don't generate llvm.expect on -O0 as the backend won't use it for
 881     // anything.
 882     // Note, we still IRGen ExpectedValue because it could have side-effects.
 883     if (CGM.getCodeGenOpts().OptimizationLevel == 0)
 884       return RValue::get(ArgValue);
 885
 886     Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType);
 887     Value *Result =
 888         Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue}, "expval");
 889     return RValue::get(Result);
 890   }
 891   case Builtin::BI__builtin_assume_aligned: {
 892     Value *PtrValue = EmitScalarExpr(E->getArg(0));
 893     Value *OffsetValue =
 894       (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : nullptr;
 895
 896     Value *AlignmentValue = EmitScalarExpr(E->getArg(1));
 897     ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
 898     unsigned Alignment = (unsigned) AlignmentCI->getZExtValue();
 899
 900     EmitAlignmentAssumption(PtrValue, Alignment, OffsetValue);
 901     return RValue::get(PtrValue);
 902   }
 903   case Builtin::BI__assume:
 904   case Builtin::BI__builtin_assume: {
 905     if (E->getArg(0)->HasSideEffects(getContext()))
 906       return RValue::get(nullptr);
 907
 908     Value *ArgValue = EmitScalarExpr(E->getArg(0));
 909     Value *FnAssume = CGM.getIntrinsic(Intrinsic::assume);
 910     return RValue::get(Builder.CreateCall(FnAssume, ArgValue));
 911   }
 912   case Builtin::BI__builtin_bswap16:
 913   case Builtin::BI__builtin_bswap32:
 914   case Builtin::BI__builtin_bswap64: {
 915     return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bswap));
 916   }
 917   case Builtin::BI__builtin_bitreverse8:
 918   case Builtin::BI__builtin_bitreverse16:
 919   case Builtin::BI__builtin_bitreverse32:
 920   case Builtin::BI__builtin_bitreverse64: {
 921     return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bitreverse));
 922   }
 923   case Builtin::BI__builtin_object_size: {
 924     unsigned Type =
 925         E->getArg(1)->EvaluateKnownConstInt(getContext()).getZExtValue();
 926     auto *ResType = cast<llvm::IntegerType>(ConvertType(E->getType()));
 927
 928     // We pass this builtin onto the optimizer so that it can figure out the
 929     // object size in more complex cases.
 930     return RValue::get(emitBuiltinObjectSize(E->getArg(0), Type, ResType));
 931   }
 932   case Builtin::BI__builtin_prefetch: {
 933     Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0));
 934     // FIXME: Technically these constants should of type 'int', yes?
 935     RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E->getArg(1)) :
 936       llvm::ConstantInt::get(Int32Ty, 0);
 937     Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) :
 938       llvm::ConstantInt::get(Int32Ty, 3);
 939     Value *Data = llvm::ConstantInt::get(Int32Ty, 1);
 940     Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
 941     return RValue::get(Builder.CreateCall(F, {Address, RW, Locality, Data}));
 942   }
 943   case Builtin::BI__builtin_readcyclecounter: {
 944     Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter);
 945     return RValue::get(Builder.CreateCall(F));
 946   }
 947   case Builtin::BI__builtin___clear_cache: {
 948     Value *Begin = EmitScalarExpr(E->getArg(0));
 949     Value *End = EmitScalarExpr(E->getArg(1));
 950     Value *F = CGM.getIntrinsic(Intrinsic::clear_cache);
 951     return RValue::get(Builder.CreateCall(F, {Begin, End}));
 952   }
 953   case Builtin::BI__builtin_trap:
 954     return RValue::get(EmitTrapCall(Intrinsic::trap));
 955   case Builtin::BI__debugbreak:
 956     return RValue::get(EmitTrapCall(Intrinsic::debugtrap));
 957   case Builtin::BI__builtin_unreachable: {
 958     if (SanOpts.has(SanitizerKind::Unreachable)) {
 959       SanitizerScope SanScope(this);
 960       EmitCheck(std::make_pair(static_cast<llvm::Value *>(Builder.getFalse()),
 961                                SanitizerKind::Unreachable),
 962                 SanitizerHandler::BuiltinUnreachable,
 963                 EmitCheckSourceLocation(E->getExprLoc()), None);
 964     } else
 965       Builder.CreateUnreachable();
 966
 967     // We do need to preserve an insertion point.
 968     EmitBlock(createBasicBlock("unreachable.cont"));
 969
 970     return RValue::get(nullptr);
 971   }
 972
 973   case Builtin::BI__builtin_powi:
 974   case Builtin::BI__builtin_powif:
 975   case Builtin::BI__builtin_powil: {
 976     Value *Base = EmitScalarExpr(E->getArg(0));
 977     Value *Exponent = EmitScalarExpr(E->getArg(1));
 978     llvm::Type *ArgType = Base->getType();
 979     Value *F = CGM.getIntrinsic(Intrinsic::powi, ArgType);
 980     return RValue::get(Builder.CreateCall(F, {Base, Exponent}));
 981   }
 982
 983   case Builtin::BI__builtin_isgreater:
 984   case Builtin::BI__builtin_isgreaterequal:
 985   case Builtin::BI__builtin_isless:
 986   case Builtin::BI__builtin_islessequal:
 987   case Builtin::BI__builtin_islessgreater:
 988   case Builtin::BI__builtin_isunordered: {
 989     // Ordered comparisons: we know the arguments to these are matching scalar
 990     // floating point values.
 991     Value *LHS = EmitScalarExpr(E->getArg(0));
 992     Value *RHS = EmitScalarExpr(E->getArg(1));
 993
 994     switch (BuiltinID) {
 995     default: llvm_unreachable("Unknown ordered comparison");
 996     case Builtin::BI__builtin_isgreater:
 997       LHS = Builder.CreateFCmpOGT(LHS, RHS, "cmp");
 998       break;
 999     case Builtin::BI__builtin_isgreaterequal:
1000       LHS = Builder.CreateFCmpOGE(LHS, RHS, "cmp");
1001       break;
1002     case Builtin::BI__builtin_isless:
1003       LHS = Builder.CreateFCmpOLT(LHS, RHS, "cmp");
1004       break;
1005     case Builtin::BI__builtin_islessequal:
1006       LHS = Builder.CreateFCmpOLE(LHS, RHS, "cmp");
1007       break;
1008     case Builtin::BI__builtin_islessgreater:
1009       LHS = Builder.CreateFCmpONE(LHS, RHS, "cmp");
1010       break;
1011     case Builtin::BI__builtin_isunordered:
1012       LHS = Builder.CreateFCmpUNO(LHS, RHS, "cmp");
1013       break;
1014     }
1015     // ZExt bool to int type.
1016     return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType())));
1017   }
1018   case Builtin::BI__builtin_isnan: {
1019     Value *V = EmitScalarExpr(E->getArg(0));
1020     V = Builder.CreateFCmpUNO(V, V, "cmp");
1021     return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
1022   }
1023
1024   case Builtin::BIfinite:
1025   case Builtin::BI__finite:
1026   case Builtin::BIfinitef:
1027   case Builtin::BI__finitef:
1028   case Builtin::BIfinitel:
1029   case Builtin::BI__finitel:
1030   case Builtin::BI__builtin_isinf:
1031   case Builtin::BI__builtin_isfinite: {
1032     // isinf(x)    --> fabs(x) == infinity
1033     // isfinite(x) --> fabs(x) != infinity
1034     // x != NaN via the ordered compare in either case.
1035     Value *V = EmitScalarExpr(E->getArg(0));
1036     Value *Fabs = EmitFAbs(*this, V);
1037     Constant *Infinity = ConstantFP::getInfinity(V->getType());
1038     CmpInst::Predicate Pred = (BuiltinID == Builtin::BI__builtin_isinf)
1039                                   ? CmpInst::FCMP_OEQ
1040                                   : CmpInst::FCMP_ONE;
1041     Value *FCmp = Builder.CreateFCmp(Pred, Fabs, Infinity, "cmpinf");
1042     return RValue::get(Builder.CreateZExt(FCmp, ConvertType(E->getType())));
1043   }
1044
1045   case Builtin::BI__builtin_isinf_sign: {
1046     // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0
1047     Value *Arg = EmitScalarExpr(E->getArg(0));
1048     Value *AbsArg = EmitFAbs(*this, Arg);
1049     Value *IsInf = Builder.CreateFCmpOEQ(
1050         AbsArg, ConstantFP::getInfinity(Arg->getType()), "isinf");
1051     Value *IsNeg = EmitSignBit(*this, Arg);
1052
1053     llvm::Type *IntTy = ConvertType(E->getType());
1054     Value *Zero = Constant::getNullValue(IntTy);
1055     Value *One = ConstantInt::get(IntTy, 1);
1056     Value *NegativeOne = ConstantInt::get(IntTy, -1);
1057     Value *SignResult = Builder.CreateSelect(IsNeg, NegativeOne, One);
1058     Value *Result = Builder.CreateSelect(IsInf, SignResult, Zero);
1059     return RValue::get(Result);
1060   }
1061
1062   case Builtin::BI__builtin_isnormal: {
1063     // isnormal(x) --> x == x && fabsf(x) < infinity && fabsf(x) >= float_min
1064     Value *V = EmitScalarExpr(E->getArg(0));
1065     Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq");
1066
1067     Value *Abs = EmitFAbs(*this, V);
1068     Value *IsLessThanInf =
1069       Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(V->getType()),"isinf");
1070     APFloat Smallest = APFloat::getSmallestNormalized(
1071                    getContext().getFloatTypeSemantics(E->getArg(0)->getType()));
1072     Value *IsNormal =
1073       Builder.CreateFCmpUGE(Abs, ConstantFP::get(V->getContext(), Smallest),
1074                             "isnormal");
1075     V = Builder.CreateAnd(Eq, IsLessThanInf, "and");
1076     V = Builder.CreateAnd(V, IsNormal, "and");
1077     return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
1078   }
1079
1080   case Builtin::BI__builtin_fpclassify: {
1081     Value *V = EmitScalarExpr(E->getArg(5));
1082     llvm::Type *Ty = ConvertType(E->getArg(5)->getType());
1083
1084     // Create Result
1085     BasicBlock *Begin = Builder.GetInsertBlock();
1086     BasicBlock *End = createBasicBlock("fpclassify_end", this->CurFn);
1087     Builder.SetInsertPoint(End);
1088     PHINode *Result =
1089       Builder.CreatePHI(ConvertType(E->getArg(0)->getType()), 4,
1090                         "fpclassify_result");
1091
1092     // if (V==0) return FP_ZERO
1093     Builder.SetInsertPoint(Begin);
1094     Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty),
1095                                           "iszero");
1096     Value *ZeroLiteral = EmitScalarExpr(E->getArg(4));
1097     BasicBlock *NotZero = createBasicBlock("fpclassify_not_zero", this->CurFn);
1098     Builder.CreateCondBr(IsZero, End, NotZero);
1099     Result->addIncoming(ZeroLiteral, Begin);
1100
1101     // if (V != V) return FP_NAN
1102     Builder.SetInsertPoint(NotZero);
1103     Value *IsNan = Builder.CreateFCmpUNO(V, V, "cmp");
1104     Value *NanLiteral = EmitScalarExpr(E->getArg(0));
1105     BasicBlock *NotNan = createBasicBlock("fpclassify_not_nan", this->CurFn);
1106     Builder.CreateCondBr(IsNan, End, NotNan);
1107     Result->addIncoming(NanLiteral, NotZero);
1108
1109     // if (fabs(V) == infinity) return FP_INFINITY
1110     Builder.SetInsertPoint(NotNan);
1111     Value *VAbs = EmitFAbs(*this, V);
1112     Value *IsInf =
1113       Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()),
1114                             "isinf");
1115     Value *InfLiteral = EmitScalarExpr(E->getArg(1));
1116     BasicBlock *NotInf = createBasicBlock("fpclassify_not_inf", this->CurFn);
1117     Builder.CreateCondBr(IsInf, End, NotInf);
1118     Result->addIncoming(InfLiteral, NotNan);
1119
1120     // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL
1121     Builder.SetInsertPoint(NotInf);
1122     APFloat Smallest = APFloat::getSmallestNormalized(
1123         getContext().getFloatTypeSemantics(E->getArg(5)->getType()));
1124     Value *IsNormal =
1125       Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest),
1126                             "isnormal");
1127     Value *NormalResult =
1128       Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)),
1129                            EmitScalarExpr(E->getArg(3)));
1130     Builder.CreateBr(End);
1131     Result->addIncoming(NormalResult, NotInf);
1132
1133     // return Result
1134     Builder.SetInsertPoint(End);
1135     return RValue::get(Result);
1136   }
1137
1138   case Builtin::BIalloca:
1139   case Builtin::BI_alloca:
1140   case Builtin::BI__builtin_alloca: {
1141     Value *Size = EmitScalarExpr(E->getArg(0));
1142     const TargetInfo &TI = getContext().getTargetInfo();
1143     // The alignment of the alloca should correspond to __BIGGEST_ALIGNMENT__.
1144     unsigned SuitableAlignmentInBytes =
1145         CGM.getContext()
1146             .toCharUnitsFromBits(TI.getSuitableAlign())
1147             .getQuantity();
1148     AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
1149     AI->setAlignment(SuitableAlignmentInBytes);
1150     return RValue::get(AI);
1151   }
1152
1153   case Builtin::BI__builtin_alloca_with_align: {
1154     Value *Size = EmitScalarExpr(E->getArg(0));
1155     Value *AlignmentInBitsValue = EmitScalarExpr(E->getArg(1));
1156     auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
1157     unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
1158     unsigned AlignmentInBytes =
1159         CGM.getContext().toCharUnitsFromBits(AlignmentInBits).getQuantity();
1160     AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
1161     AI->setAlignment(AlignmentInBytes);
1162     return RValue::get(AI);
1163   }
1164
1165   case Builtin::BIbzero:
1166   case Builtin::BI__builtin_bzero: {
1167     Address Dest = EmitPointerWithAlignment(E->getArg(0));
1168     Value *SizeVal = EmitScalarExpr(E->getArg(1));
1169     EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
1170                         E->getArg(0)->getExprLoc(), FD, 0);
1171     Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false);
1172     return RValue::get(Dest.getPointer());
1173   }
1174   case Builtin::BImemcpy:
1175   case Builtin::BI__builtin_memcpy: {
1176     Address Dest = EmitPointerWithAlignment(E->getArg(0));
1177     Address Src = EmitPointerWithAlignment(E->getArg(1));
1178     Value *SizeVal = EmitScalarExpr(E->getArg(2));
1179     EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
1180                         E->getArg(0)->getExprLoc(), FD, 0);
1181     EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(),
1182                         E->getArg(1)->getExprLoc(), FD, 1);
1183     Builder.CreateMemCpy(Dest, Src, SizeVal, false);
1184     return RValue::get(Dest.getPointer());
1185   }
1186
1187   case Builtin::BI__builtin___memcpy_chk: {
1188     // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2.
1189     llvm::APSInt Size, DstSize;
1190     if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
1191         !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
1192       break;
1193     if (Size.ugt(DstSize))
1194       break;
1195     Address Dest = EmitPointerWithAlignment(E->getArg(0));
1196     Address Src = EmitPointerWithAlignment(E->getArg(1));
1197     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
1198     Builder.CreateMemCpy(Dest, Src, SizeVal, false);
1199     return RValue::get(Dest.getPointer());
1200   }
1201
1202   case Builtin::BI__builtin_objc_memmove_collectable: {
1203     Address DestAddr = EmitPointerWithAlignment(E->getArg(0));
1204     Address SrcAddr = EmitPointerWithAlignment(E->getArg(1));
1205     Value *SizeVal = EmitScalarExpr(E->getArg(2));
1206     CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this,
1207                                                   DestAddr, SrcAddr, SizeVal);
1208     return RValue::get(DestAddr.getPointer());
1209   }
1210
1211   case Builtin::BI__builtin___memmove_chk: {
1212     // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2.
1213     llvm::APSInt Size, DstSize;
1214     if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
1215         !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
1216       break;
1217     if (Size.ugt(DstSize))
1218       break;
1219     Address Dest = EmitPointerWithAlignment(E->getArg(0));
1220     Address Src = EmitPointerWithAlignment(E->getArg(1));
1221     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
1222     Builder.CreateMemMove(Dest, Src, SizeVal, false);
1223     return RValue::get(Dest.getPointer());
1224   }
1225
1226   case Builtin::BImemmove:
1227   case Builtin::BI__builtin_memmove: {
1228     Address Dest = EmitPointerWithAlignment(E->getArg(0));
1229     Address Src = EmitPointerWithAlignment(E->getArg(1));
1230     Value *SizeVal = EmitScalarExpr(E->getArg(2));
1231     EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
1232                         E->getArg(0)->getExprLoc(), FD, 0);
1233     EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(),
1234                         E->getArg(1)->getExprLoc(), FD, 1);
1235     Builder.CreateMemMove(Dest, Src, SizeVal, false);
1236     return RValue::get(Dest.getPointer());
1237   }
1238   case Builtin::BImemset:
1239   case Builtin::BI__builtin_memset: {
1240     Address Dest = EmitPointerWithAlignment(E->getArg(0));
1241     Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
1242                                          Builder.getInt8Ty());
1243     Value *SizeVal = EmitScalarExpr(E->getArg(2));
1244     EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
1245                         E->getArg(0)->getExprLoc(), FD, 0);
1246     Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
1247     return RValue::get(Dest.getPointer());
1248   }
1249   case Builtin::BI__builtin___memset_chk: {
1250     // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
1251     llvm::APSInt Size, DstSize;
1252     if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
1253         !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
1254       break;
1255     if (Size.ugt(DstSize))
1256       break;
1257     Address Dest = EmitPointerWithAlignment(E->getArg(0));
1258     Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
1259                                          Builder.getInt8Ty());
1260     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
1261     Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
1262     return RValue::get(Dest.getPointer());
1263   }
1264   case Builtin::BI__builtin_dwarf_cfa: {
1265     // The offset in bytes from the first argument to the CFA.
1266     //
1267     // Why on earth is this in the frontend?  Is there any reason at
1268     // all that the backend can't reasonably determine this while
1269     // lowering llvm.eh.dwarf.cfa()?
1270     //
1271     // TODO: If there's a satisfactory reason, add a target hook for
1272     // this instead of hard-coding 0, which is correct for most targets.
1273     int32_t Offset = 0;
1274
1275     Value *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa);
1276     return RValue::get(Builder.CreateCall(F,
1277                                       llvm::ConstantInt::get(Int32Ty, Offset)));
1278   }
1279   case Builtin::BI__builtin_return_address: {
1280     Value *Depth =
1281         CGM.EmitConstantExpr(E->getArg(0), getContext().UnsignedIntTy, this);
1282     Value *F = CGM.getIntrinsic(Intrinsic::returnaddress);
1283     return RValue::get(Builder.CreateCall(F, Depth));
1284   }
1285   case Builtin::BI_ReturnAddress: {
1286     Value *F = CGM.getIntrinsic(Intrinsic::returnaddress);
1287     return RValue::get(Builder.CreateCall(F, Builder.getInt32(0)));
1288   }
1289   case Builtin::BI__builtin_frame_address: {
1290     Value *Depth =
1291         CGM.EmitConstantExpr(E->getArg(0), getContext().UnsignedIntTy, this);
1292     Value *F = CGM.getIntrinsic(Intrinsic::frameaddress);
1293     return RValue::get(Builder.CreateCall(F, Depth));
1294   }
1295   case Builtin::BI__builtin_extract_return_addr: {
1296     Value *Address = EmitScalarExpr(E->getArg(0));
1297     Value *Result = getTargetHooks().decodeReturnAddress(*this, Address);
1298     return RValue::get(Result);
1299   }
1300   case Builtin::BI__builtin_frob_return_addr: {
1301     Value *Address = EmitScalarExpr(E->getArg(0));
1302     Value *Result = getTargetHooks().encodeReturnAddress(*this, Address);
1303     return RValue::get(Result);
1304   }
1305   case Builtin::BI__builtin_dwarf_sp_column: {
1306     llvm::IntegerType *Ty
1307       = cast<llvm::IntegerType>(ConvertType(E->getType()));
1308     int Column = getTargetHooks().getDwarfEHStackPointer(CGM);
1309     if (Column == -1) {
1310       CGM.ErrorUnsupported(E, "__builtin_dwarf_sp_column");
1311       return RValue::get(llvm::UndefValue::get(Ty));
1312     }
1313     return RValue::get(llvm::ConstantInt::get(Ty, Column, true));
1314   }
1315   case Builtin::BI__builtin_init_dwarf_reg_size_table: {
1316     Value *Address = EmitScalarExpr(E->getArg(0));
1317     if (getTargetHooks().initDwarfEHRegSizeTable(*this, Address))
1318       CGM.ErrorUnsupported(E, "__builtin_init_dwarf_reg_size_table");
1319     return RValue::get(llvm::UndefValue::get(ConvertType(E->getType())));
1320   }
1321   case Builtin::BI__builtin_eh_return: {
1322     Value *Int = EmitScalarExpr(E->getArg(0));
1323     Value *Ptr = EmitScalarExpr(E->getArg(1));
1324
1325     llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType());
1326     assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
1327            "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
1328     Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32
1329                                   ? Intrinsic::eh_return_i32
1330                                   : Intrinsic::eh_return_i64);
1331     Builder.CreateCall(F, {Int, Ptr});
1332     Builder.CreateUnreachable();
1333
1334     // We do need to preserve an insertion point.
1335     EmitBlock(createBasicBlock("builtin_eh_return.cont"));
1336
1337     return RValue::get(nullptr);
1338   }
1339   case Builtin::BI__builtin_unwind_init: {
1340     Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init);
1341     return RValue::get(Builder.CreateCall(F));
1342   }
1343   case Builtin::BI__builtin_extend_pointer: {
1344     // Extends a pointer to the size of an _Unwind_Word, which is
1345     // uint64_t on all platforms.  Generally this gets poked into a
1346     // register and eventually used as an address, so if the
1347     // addressing registers are wider than pointers and the platform
1348     // doesn't implicitly ignore high-order bits when doing
1349     // addressing, we need to make sure we zext / sext based on
1350     // the platform's expectations.
1351     //
1352     // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html
1353
1354     // Cast the pointer to intptr_t.
1355     Value *Ptr = EmitScalarExpr(E->getArg(0));
1356     Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy, "extend.cast");
1357
1358     // If that's 64 bits, we're done.
1359     if (IntPtrTy->getBitWidth() == 64)
1360       return RValue::get(Result);
1361
1362     // Otherwise, ask the codegen data what to do.
1363     if (getTargetHooks().extendPointerWithSExt())
1364       return RValue::get(Builder.CreateSExt(Result, Int64Ty, "extend.sext"));
1365     else
1366       return RValue::get(Builder.CreateZExt(Result, Int64Ty, "extend.zext"));
1367   }
1368   case Builtin::BI__builtin_setjmp: {
1369     // Buffer is a void**.
1370     Address Buf = EmitPointerWithAlignment(E->getArg(0));
1371
1372     // Store the frame pointer to the setjmp buffer.
1373     Value *FrameAddr =
1374       Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
1375                          ConstantInt::get(Int32Ty, 0));
1376     Builder.CreateStore(FrameAddr, Buf);
1377
1378     // Store the stack pointer to the setjmp buffer.
1379     Value *StackAddr =
1380         Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave));
1381     Address StackSaveSlot =
1382       Builder.CreateConstInBoundsGEP(Buf, 2, getPointerSize());
1383     Builder.CreateStore(StackAddr, StackSaveSlot);
1384
1385     // Call LLVM's EH setjmp, which is lightweight.
1386     Value *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
1387     Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
1388     return RValue::get(Builder.CreateCall(F, Buf.getPointer()));
1389   }
1390   case Builtin::BI__builtin_longjmp: {
1391     Value *Buf = EmitScalarExpr(E->getArg(0));
1392     Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
1393
1394     // Call LLVM's EH longjmp, which is lightweight.
1395     Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf);
1396
1397     // longjmp doesn't return; mark this as unreachable.
1398     Builder.CreateUnreachable();
1399
1400     // We do need to preserve an insertion point.
1401     EmitBlock(createBasicBlock("longjmp.cont"));
1402
1403     return RValue::get(nullptr);
1404   }
1405   case Builtin::BI__sync_fetch_and_add:
1406   case Builtin::BI__sync_fetch_and_sub:
1407   case Builtin::BI__sync_fetch_and_or:
1408   case Builtin::BI__sync_fetch_and_and:
1409   case Builtin::BI__sync_fetch_and_xor:
1410   case Builtin::BI__sync_fetch_and_nand:
1411   case Builtin::BI__sync_add_and_fetch:
1412   case Builtin::BI__sync_sub_and_fetch:
1413   case Builtin::BI__sync_and_and_fetch:
1414   case Builtin::BI__sync_or_and_fetch:
1415   case Builtin::BI__sync_xor_and_fetch:
1416   case Builtin::BI__sync_nand_and_fetch:
1417   case Builtin::BI__sync_val_compare_and_swap:
1418   case Builtin::BI__sync_bool_compare_and_swap:
1419   case Builtin::BI__sync_lock_test_and_set:
1420   case Builtin::BI__sync_lock_release:
1421   case Builtin::BI__sync_swap:
1422     llvm_unreachable("Shouldn't make it through sema");
1423   case Builtin::BI__sync_fetch_and_add_1:
1424   case Builtin::BI__sync_fetch_and_add_2:
1425   case Builtin::BI__sync_fetch_and_add_4:
1426   case Builtin::BI__sync_fetch_and_add_8:
1427   case Builtin::BI__sync_fetch_and_add_16:
1428     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Add, E);
1429   case Builtin::BI__sync_fetch_and_sub_1:
1430   case Builtin::BI__sync_fetch_and_sub_2:
1431   case Builtin::BI__sync_fetch_and_sub_4:
1432   case Builtin::BI__sync_fetch_and_sub_8:
1433   case Builtin::BI__sync_fetch_and_sub_16:
1434     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Sub, E);
1435   case Builtin::BI__sync_fetch_and_or_1:
1436   case Builtin::BI__sync_fetch_and_or_2:
1437   case Builtin::BI__sync_fetch_and_or_4:
1438   case Builtin::BI__sync_fetch_and_or_8:
1439   case Builtin::BI__sync_fetch_and_or_16:
1440     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Or, E);
1441   case Builtin::BI__sync_fetch_and_and_1:
1442   case Builtin::BI__sync_fetch_and_and_2:
1443   case Builtin::BI__sync_fetch_and_and_4:
1444   case Builtin::BI__sync_fetch_and_and_8:
1445   case Builtin::BI__sync_fetch_and_and_16:
1446     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::And, E);
1447   case Builtin::BI__sync_fetch_and_xor_1:
1448   case Builtin::BI__sync_fetch_and_xor_2:
1449   case Builtin::BI__sync_fetch_and_xor_4:
1450   case Builtin::BI__sync_fetch_and_xor_8:
1451   case Builtin::BI__sync_fetch_and_xor_16:
1452     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xor, E);
1453   case Builtin::BI__sync_fetch_and_nand_1:
1454   case Builtin::BI__sync_fetch_and_nand_2:
1455   case Builtin::BI__sync_fetch_and_nand_4:
1456   case Builtin::BI__sync_fetch_and_nand_8:
1457   case Builtin::BI__sync_fetch_and_nand_16:
1458     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Nand, E);
1459
1460   // Clang extensions: not overloaded yet.
1461   case Builtin::BI__sync_fetch_and_min:
1462     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Min, E);
1463   case Builtin::BI__sync_fetch_and_max:
1464     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Max, E);
1465   case Builtin::BI__sync_fetch_and_umin:
1466     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMin, E);
1467   case Builtin::BI__sync_fetch_and_umax:
1468     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMax, E);
1469
1470   case Builtin::BI__sync_add_and_fetch_1:
1471   case Builtin::BI__sync_add_and_fetch_2:
1472   case Builtin::BI__sync_add_and_fetch_4:
1473   case Builtin::BI__sync_add_and_fetch_8:
1474   case Builtin::BI__sync_add_and_fetch_16:
1475     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Add, E,
1476                                 llvm::Instruction::Add);
1477   case Builtin::BI__sync_sub_and_fetch_1:
1478   case Builtin::BI__sync_sub_and_fetch_2:
1479   case Builtin::BI__sync_sub_and_fetch_4:
1480   case Builtin::BI__sync_sub_and_fetch_8:
1481   case Builtin::BI__sync_sub_and_fetch_16:
1482     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Sub, E,
1483                                 llvm::Instruction::Sub);
1484   case Builtin::BI__sync_and_and_fetch_1:
1485   case Builtin::BI__sync_and_and_fetch_2:
1486   case Builtin::BI__sync_and_and_fetch_4:
1487   case Builtin::BI__sync_and_and_fetch_8:
1488   case Builtin::BI__sync_and_and_fetch_16:
1489     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::And, E,
1490                                 llvm::Instruction::And);
1491   case Builtin::BI__sync_or_and_fetch_1:
1492   case Builtin::BI__sync_or_and_fetch_2:
1493   case Builtin::BI__sync_or_and_fetch_4:
1494   case Builtin::BI__sync_or_and_fetch_8:
1495   case Builtin::BI__sync_or_and_fetch_16:
1496     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Or, E,
1497                                 llvm::Instruction::Or);
1498   case Builtin::BI__sync_xor_and_fetch_1:
1499   case Builtin::BI__sync_xor_and_fetch_2:
1500   case Builtin::BI__sync_xor_and_fetch_4:
1501   case Builtin::BI__sync_xor_and_fetch_8:
1502   case Builtin::BI__sync_xor_and_fetch_16:
1503     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Xor, E,
1504                                 llvm::Instruction::Xor);
1505   case Builtin::BI__sync_nand_and_fetch_1:
1506   case Builtin::BI__sync_nand_and_fetch_2:
1507   case Builtin::BI__sync_nand_and_fetch_4:
1508   case Builtin::BI__sync_nand_and_fetch_8:
1509   case Builtin::BI__sync_nand_and_fetch_16:
1510     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Nand, E,
1511                                 llvm::Instruction::And, true);
1512
1513   case Builtin::BI__sync_val_compare_and_swap_1:
1514   case Builtin::BI__sync_val_compare_and_swap_2:
1515   case Builtin::BI__sync_val_compare_and_swap_4:
1516   case Builtin::BI__sync_val_compare_and_swap_8:
1517   case Builtin::BI__sync_val_compare_and_swap_16:
1518     return RValue::get(MakeAtomicCmpXchgValue(*this, E, false));
1519
1520   case Builtin::BI__sync_bool_compare_and_swap_1:
1521   case Builtin::BI__sync_bool_compare_and_swap_2:
1522   case Builtin::BI__sync_bool_compare_and_swap_4:
1523   case Builtin::BI__sync_bool_compare_and_swap_8:
1524   case Builtin::BI__sync_bool_compare_and_swap_16:
1525     return RValue::get(MakeAtomicCmpXchgValue(*this, E, true));
1526
1527   case Builtin::BI__sync_swap_1:
1528   case Builtin::BI__sync_swap_2:
1529   case Builtin::BI__sync_swap_4:
1530   case Builtin::BI__sync_swap_8:
1531   case Builtin::BI__sync_swap_16:
1532     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
1533
1534   case Builtin::BI__sync_lock_test_and_set_1:
1535   case Builtin::BI__sync_lock_test_and_set_2:
1536   case Builtin::BI__sync_lock_test_and_set_4:
1537   case Builtin::BI__sync_lock_test_and_set_8:
1538   case Builtin::BI__sync_lock_test_and_set_16:
1539     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
1540
1541   case Builtin::BI__sync_lock_release_1:
1542   case Builtin::BI__sync_lock_release_2:
1543   case Builtin::BI__sync_lock_release_4:
1544   case Builtin::BI__sync_lock_release_8:
1545   case Builtin::BI__sync_lock_release_16: {
1546     Value *Ptr = EmitScalarExpr(E->getArg(0));
1547     QualType ElTy = E->getArg(0)->getType()->getPointeeType();
1548     CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy);
1549     llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
1550                                              StoreSize.getQuantity() * 8);
1551     Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
1552     llvm::StoreInst *Store =
1553       Builder.CreateAlignedStore(llvm::Constant::getNullValue(ITy), Ptr,
1554                                  StoreSize);
1555     Store->setAtomic(llvm::AtomicOrdering::Release);
1556     return RValue::get(nullptr);
1557   }
1558
1559   case Builtin::BI__sync_synchronize: {
1560     // We assume this is supposed to correspond to a C++0x-style
1561     // sequentially-consistent fence (i.e. this is only usable for
1562     // synchonization, not device I/O or anything like that). This intrinsic
1563     // is really badly designed in the sense that in theory, there isn't
1564     // any way to safely use it... but in practice, it mostly works
1565     // to use it with non-atomic loads and stores to get acquire/release
1566     // semantics.
1567     Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
1568     return RValue::get(nullptr);
1569   }
1570
1571   case Builtin::BI__builtin_nontemporal_load:
1572     return RValue::get(EmitNontemporalLoad(*this, E));
1573   case Builtin::BI__builtin_nontemporal_store:
1574     return RValue::get(EmitNontemporalStore(*this, E));
1575   case Builtin::BI__c11_atomic_is_lock_free:
1576   case Builtin::BI__atomic_is_lock_free: {
1577     // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the
1578     // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since
1579     // _Atomic(T) is always properly-aligned.
1580     const char *LibCallName = "__atomic_is_lock_free";
1581     CallArgList Args;
1582     Args.add(RValue::get(EmitScalarExpr(E->getArg(0))),
1583              getContext().getSizeType());
1584     if (BuiltinID == Builtin::BI__atomic_is_lock_free)
1585       Args.add(RValue::get(EmitScalarExpr(E->getArg(1))),
1586                getContext().VoidPtrTy);
1587     else
1588       Args.add(RValue::get(llvm::Constant::getNullValue(VoidPtrTy)),
1589                getContext().VoidPtrTy);
1590     const CGFunctionInfo &FuncInfo =
1591         CGM.getTypes().arrangeBuiltinFunctionCall(E->getType(), Args);
1592     llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo);
1593     llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
1594     return EmitCall(FuncInfo, CGCallee::forDirect(Func),
1595                     ReturnValueSlot(), Args);
1596   }
1597
1598   case Builtin::BI__atomic_test_and_set: {
1599     // Look at the argument type to determine whether this is a volatile
1600     // operation. The parameter type is always volatile.
1601     QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType();
1602     bool Volatile =
1603         PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
1604
1605     Value *Ptr = EmitScalarExpr(E->getArg(0));
1606     unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace();
1607     Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
1608     Value *NewVal = Builder.getInt8(1);
1609     Value *Order = EmitScalarExpr(E->getArg(1));
1610     if (isa<llvm::ConstantInt>(Order)) {
1611       int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1612       AtomicRMWInst *Result = nullptr;
1613       switch (ord) {
1614       case 0:  // memory_order_relaxed
1615       default: // invalid order
1616         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
1617                                          llvm::AtomicOrdering::Monotonic);
1618         break;
1619       case 1: // memory_order_consume
1620       case 2: // memory_order_acquire
1621         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
1622                                          llvm::AtomicOrdering::Acquire);
1623         break;
1624       case 3: // memory_order_release
1625         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
1626                                          llvm::AtomicOrdering::Release);
1627         break;
1628       case 4: // memory_order_acq_rel
1629
1630         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
1631                                          llvm::AtomicOrdering::AcquireRelease);
1632         break;
1633       case 5: // memory_order_seq_cst
1634         Result = Builder.CreateAtomicRMW(
1635             llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
1636             llvm::AtomicOrdering::SequentiallyConsistent);
1637         break;
1638       }
1639       Result->setVolatile(Volatile);
1640       return RValue::get(Builder.CreateIsNotNull(Result, "tobool"));
1641     }
1642
1643     llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1644
1645     llvm::BasicBlock *BBs[5] = {
1646       createBasicBlock("monotonic", CurFn),
1647       createBasicBlock("acquire", CurFn),
1648       createBasicBlock("release", CurFn),
1649       createBasicBlock("acqrel", CurFn),
1650       createBasicBlock("seqcst", CurFn)
1651     };
1652     llvm::AtomicOrdering Orders[5] = {
1653         llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire,
1654         llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease,
1655         llvm::AtomicOrdering::SequentiallyConsistent};
1656
1657     Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1658     llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
1659
1660     Builder.SetInsertPoint(ContBB);
1661     PHINode *Result = Builder.CreatePHI(Int8Ty, 5, "was_set");
1662
1663     for (unsigned i = 0; i < 5; ++i) {
1664       Builder.SetInsertPoint(BBs[i]);
1665       AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
1666                                                    Ptr, NewVal, Orders[i]);
1667       RMW->setVolatile(Volatile);
1668       Result->addIncoming(RMW, BBs[i]);
1669       Builder.CreateBr(ContBB);
1670     }
1671
1672     SI->addCase(Builder.getInt32(0), BBs[0]);
1673     SI->addCase(Builder.getInt32(1), BBs[1]);
1674     SI->addCase(Builder.getInt32(2), BBs[1]);
1675     SI->addCase(Builder.getInt32(3), BBs[2]);
1676     SI->addCase(Builder.getInt32(4), BBs[3]);
1677     SI->addCase(Builder.getInt32(5), BBs[4]);
1678
1679     Builder.SetInsertPoint(ContBB);
1680     return RValue::get(Builder.CreateIsNotNull(Result, "tobool"));
1681   }
1682
1683   case Builtin::BI__atomic_clear: {
1684     QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType();
1685     bool Volatile =
1686         PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
1687
1688     Address Ptr = EmitPointerWithAlignment(E->getArg(0));
1689     unsigned AddrSpace = Ptr.getPointer()->getType()->getPointerAddressSpace();
1690     Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
1691     Value *NewVal = Builder.getInt8(0);
1692     Value *Order = EmitScalarExpr(E->getArg(1));
1693     if (isa<llvm::ConstantInt>(Order)) {
1694       int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1695       StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile);
1696       switch (ord) {
1697       case 0:  // memory_order_relaxed
1698       default: // invalid order
1699         Store->setOrdering(llvm::AtomicOrdering::Monotonic);
1700         break;
1701       case 3:  // memory_order_release
1702         Store->setOrdering(llvm::AtomicOrdering::Release);
1703         break;
1704       case 5:  // memory_order_seq_cst
1705         Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent);
1706         break;
1707       }
1708       return RValue::get(nullptr);
1709     }
1710
1711     llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1712
1713     llvm::BasicBlock *BBs[3] = {
1714       createBasicBlock("monotonic", CurFn),
1715       createBasicBlock("release", CurFn),
1716       createBasicBlock("seqcst", CurFn)
1717     };
1718     llvm::AtomicOrdering Orders[3] = {
1719         llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release,
1720         llvm::AtomicOrdering::SequentiallyConsistent};
1721
1722     Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1723     llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
1724
1725     for (unsigned i = 0; i < 3; ++i) {
1726       Builder.SetInsertPoint(BBs[i]);
1727       StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile);
1728       Store->setOrdering(Orders[i]);
1729       Builder.CreateBr(ContBB);
1730     }
1731
1732     SI->addCase(Builder.getInt32(0), BBs[0]);
1733     SI->addCase(Builder.getInt32(3), BBs[1]);
1734     SI->addCase(Builder.getInt32(5), BBs[2]);
1735
1736     Builder.SetInsertPoint(ContBB);
1737     return RValue::get(nullptr);
1738   }
1739
1740   case Builtin::BI__atomic_thread_fence:
1741   case Builtin::BI__atomic_signal_fence:
1742   case Builtin::BI__c11_atomic_thread_fence:
1743   case Builtin::BI__c11_atomic_signal_fence: {
1744     llvm::SynchronizationScope Scope;
1745     if (BuiltinID == Builtin::BI__atomic_signal_fence ||
1746         BuiltinID == Builtin::BI__c11_atomic_signal_fence)
1747       Scope = llvm::SingleThread;
1748     else
1749       Scope = llvm::CrossThread;
1750     Value *Order = EmitScalarExpr(E->getArg(0));
1751     if (isa<llvm::ConstantInt>(Order)) {
1752       int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1753       switch (ord) {
1754       case 0:  // memory_order_relaxed
1755       default: // invalid order
1756         break;
1757       case 1:  // memory_order_consume
1758       case 2:  // memory_order_acquire
1759         Builder.CreateFence(llvm::AtomicOrdering::Acquire, Scope);
1760         break;
1761       case 3:  // memory_order_release
1762         Builder.CreateFence(llvm::AtomicOrdering::Release, Scope);
1763         break;
1764       case 4:  // memory_order_acq_rel
1765         Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, Scope);
1766         break;
1767       case 5:  // memory_order_seq_cst
1768         Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
1769                             Scope);
1770         break;
1771       }
1772       return RValue::get(nullptr);
1773     }
1774
1775     llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
1776     AcquireBB = createBasicBlock("acquire", CurFn);
1777     ReleaseBB = createBasicBlock("release", CurFn);
1778     AcqRelBB = createBasicBlock("acqrel", CurFn);
1779     SeqCstBB = createBasicBlock("seqcst", CurFn);
1780     llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1781
1782     Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1783     llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB);
1784
1785     Builder.SetInsertPoint(AcquireBB);
1786     Builder.CreateFence(llvm::AtomicOrdering::Acquire, Scope);
1787     Builder.CreateBr(ContBB);
1788     SI->addCase(Builder.getInt32(1), AcquireBB);
1789     SI->addCase(Builder.getInt32(2), AcquireBB);
1790
1791     Builder.SetInsertPoint(ReleaseBB);
1792     Builder.CreateFence(llvm::AtomicOrdering::Release, Scope);
1793     Builder.CreateBr(ContBB);
1794     SI->addCase(Builder.getInt32(3), ReleaseBB);
1795
1796     Builder.SetInsertPoint(AcqRelBB);
1797     Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, Scope);
1798     Builder.CreateBr(ContBB);
1799     SI->addCase(Builder.getInt32(4), AcqRelBB);
1800
1801     Builder.SetInsertPoint(SeqCstBB);
1802     Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1803     Builder.CreateBr(ContBB);
1804     SI->addCase(Builder.getInt32(5), SeqCstBB);
1805
1806     Builder.SetInsertPoint(ContBB);
1807     return RValue::get(nullptr);
1808   }
1809
1810     // Library functions with special handling.
1811   case Builtin::BIsqrt:
1812   case Builtin::BIsqrtf:
1813   case Builtin::BIsqrtl: {
1814     // Transform a call to sqrt* into a @llvm.sqrt.* intrinsic call, but only
1815     // in finite- or unsafe-math mode (the intrinsic has different semantics
1816     // for handling negative numbers compared to the library function, so
1817     // -fmath-errno=0 is not enough).
1818     if (!FD->hasAttr<ConstAttr>())
1819       break;
1820     if (!(CGM.getCodeGenOpts().UnsafeFPMath ||
1821           CGM.getCodeGenOpts().NoNaNsFPMath))
1822       break;
1823     Value *Arg0 = EmitScalarExpr(E->getArg(0));
1824     llvm::Type *ArgType = Arg0->getType();
1825     Value *F = CGM.getIntrinsic(Intrinsic::sqrt, ArgType);
1826     return RValue::get(Builder.CreateCall(F, Arg0));
1827   }
1828
1829   case Builtin::BI__builtin_pow:
1830   case Builtin::BI__builtin_powf:
1831   case Builtin::BI__builtin_powl:
1832   case Builtin::BIpow:
1833   case Builtin::BIpowf:
1834   case Builtin::BIpowl: {
1835     // Transform a call to pow* into a @llvm.pow.* intrinsic call.
1836     if (!FD->hasAttr<ConstAttr>())
1837       break;
1838     Value *Base = EmitScalarExpr(E->getArg(0));
1839     Value *Exponent = EmitScalarExpr(E->getArg(1));
1840     llvm::Type *ArgType = Base->getType();
1841     Value *F = CGM.getIntrinsic(Intrinsic::pow, ArgType);
1842     return RValue::get(Builder.CreateCall(F, {Base, Exponent}));
1843   }
1844
1845   case Builtin::BIfma:
1846   case Builtin::BIfmaf:
1847   case Builtin::BIfmal:
1848   case Builtin::BI__builtin_fma:
1849   case Builtin::BI__builtin_fmaf:
1850   case Builtin::BI__builtin_fmal: {
1851     // Rewrite fma to intrinsic.
1852     Value *FirstArg = EmitScalarExpr(E->getArg(0));
1853     llvm::Type *ArgType = FirstArg->getType();
1854     Value *F = CGM.getIntrinsic(Intrinsic::fma, ArgType);
1855     return RValue::get(
1856         Builder.CreateCall(F, {FirstArg, EmitScalarExpr(E->getArg(1)),
1857                                EmitScalarExpr(E->getArg(2))}));
1858   }
1859
1860   case Builtin::BI__builtin_signbit:
1861   case Builtin::BI__builtin_signbitf:
1862   case Builtin::BI__builtin_signbitl: {
1863     return RValue::get(
1864         Builder.CreateZExt(EmitSignBit(*this, EmitScalarExpr(E->getArg(0))),
1865                            ConvertType(E->getType())));
1866   }
1867   case Builtin::BI__builtin_annotation: {
1868     llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0));
1869     llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::annotation,
1870                                       AnnVal->getType());
1871
1872     // Get the annotation string, go through casts. Sema requires this to be a
1873     // non-wide string literal, potentially casted, so the cast<> is safe.
1874     const Expr *AnnotationStrExpr = E->getArg(1)->IgnoreParenCasts();
1875     StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
1876     return RValue::get(EmitAnnotationCall(F, AnnVal, Str, E->getExprLoc()));
1877   }
1878   case Builtin::BI__builtin_addcb:
1879   case Builtin::BI__builtin_addcs:
1880   case Builtin::BI__builtin_addc:
1881   case Builtin::BI__builtin_addcl:
1882   case Builtin::BI__builtin_addcll:
1883   case Builtin::BI__builtin_subcb:
1884   case Builtin::BI__builtin_subcs:
1885   case Builtin::BI__builtin_subc:
1886   case Builtin::BI__builtin_subcl:
1887   case Builtin::BI__builtin_subcll: {
1888
1889     // We translate all of these builtins from expressions of the form:
1890     //   int x = ..., y = ..., carryin = ..., carryout, result;
1891     //   result = __builtin_addc(x, y, carryin, &carryout);
1892     //
1893     // to LLVM IR of the form:
1894     //
1895     //   %tmp1 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
1896     //   %tmpsum1 = extractvalue {i32, i1} %tmp1, 0
1897     //   %carry1 = extractvalue {i32, i1} %tmp1, 1
1898     //   %tmp2 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %tmpsum1,
1899     //                                                       i32 %carryin)
1900     //   %result = extractvalue {i32, i1} %tmp2, 0
1901     //   %carry2 = extractvalue {i32, i1} %tmp2, 1
1902     //   %tmp3 = or i1 %carry1, %carry2
1903     //   %tmp4 = zext i1 %tmp3 to i32
1904     //   store i32 %tmp4, i32* %carryout
1905
1906     // Scalarize our inputs.
1907     llvm::Value *X = EmitScalarExpr(E->getArg(0));
1908     llvm::Value *Y = EmitScalarExpr(E->getArg(1));
1909     llvm::Value *Carryin = EmitScalarExpr(E->getArg(2));
1910     Address CarryOutPtr = EmitPointerWithAlignment(E->getArg(3));
1911
1912     // Decide if we are lowering to a uadd.with.overflow or usub.with.overflow.
1913     llvm::Intrinsic::ID IntrinsicId;
1914     switch (BuiltinID) {
1915     default: llvm_unreachable("Unknown multiprecision builtin id.");
1916     case Builtin::BI__builtin_addcb:
1917     case Builtin::BI__builtin_addcs:
1918     case Builtin::BI__builtin_addc:
1919     case Builtin::BI__builtin_addcl:
1920     case Builtin::BI__builtin_addcll:
1921       IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
1922       break;
1923     case Builtin::BI__builtin_subcb:
1924     case Builtin::BI__builtin_subcs:
1925     case Builtin::BI__builtin_subc:
1926     case Builtin::BI__builtin_subcl:
1927     case Builtin::BI__builtin_subcll:
1928       IntrinsicId = llvm::Intrinsic::usub_with_overflow;
1929       break;
1930     }
1931
1932     // Construct our resulting LLVM IR expression.
1933     llvm::Value *Carry1;
1934     llvm::Value *Sum1 = EmitOverflowIntrinsic(*this, IntrinsicId,
1935                                               X, Y, Carry1);
1936     llvm::Value *Carry2;
1937     llvm::Value *Sum2 = EmitOverflowIntrinsic(*this, IntrinsicId,
1938                                               Sum1, Carryin, Carry2);
1939     llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2),
1940                                                X->getType());
1941     Builder.CreateStore(CarryOut, CarryOutPtr);
1942     return RValue::get(Sum2);
1943   }
1944
1945   case Builtin::BI__builtin_add_overflow:
1946   case Builtin::BI__builtin_sub_overflow:
1947   case Builtin::BI__builtin_mul_overflow: {
1948     const clang::Expr *LeftArg = E->getArg(0);
1949     const clang::Expr *RightArg = E->getArg(1);
1950     const clang::Expr *ResultArg = E->getArg(2);
1951
1952     clang::QualType ResultQTy =
1953         ResultArg->getType()->castAs<PointerType>()->getPointeeType();
1954
1955     WidthAndSignedness LeftInfo =
1956         getIntegerWidthAndSignedness(CGM.getContext(), LeftArg->getType());
1957     WidthAndSignedness RightInfo =
1958         getIntegerWidthAndSignedness(CGM.getContext(), RightArg->getType());
1959     WidthAndSignedness ResultInfo =
1960         getIntegerWidthAndSignedness(CGM.getContext(), ResultQTy);
1961     WidthAndSignedness EncompassingInfo =
1962         EncompassingIntegerType({LeftInfo, RightInfo, ResultInfo});
1963
1964     llvm::Type *EncompassingLLVMTy =
1965         llvm::IntegerType::get(CGM.getLLVMContext(), EncompassingInfo.Width);
1966
1967     llvm::Type *ResultLLVMTy = CGM.getTypes().ConvertType(ResultQTy);
1968
1969     llvm::Intrinsic::ID IntrinsicId;
1970     switch (BuiltinID) {
1971     default:
1972       llvm_unreachable("Unknown overflow builtin id.");
1973     case Builtin::BI__builtin_add_overflow:
1974       IntrinsicId = EncompassingInfo.Signed
1975                         ? llvm::Intrinsic::sadd_with_overflow
1976                         : llvm::Intrinsic::uadd_with_overflow;
1977       break;
1978     case Builtin::BI__builtin_sub_overflow:
1979       IntrinsicId = EncompassingInfo.Signed
1980                         ? llvm::Intrinsic::ssub_with_overflow
1981                         : llvm::Intrinsic::usub_with_overflow;
1982       break;
1983     case Builtin::BI__builtin_mul_overflow:
1984       IntrinsicId = EncompassingInfo.Signed
1985                         ? llvm::Intrinsic::smul_with_overflow
1986                         : llvm::Intrinsic::umul_with_overflow;
1987       break;
1988     }
1989
1990     llvm::Value *Left = EmitScalarExpr(LeftArg);
1991     llvm::Value *Right = EmitScalarExpr(RightArg);
1992     Address ResultPtr = EmitPointerWithAlignment(ResultArg);
1993
1994     // Extend each operand to the encompassing type.
1995     Left = Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed);
1996     Right = Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed);
1997
1998     // Perform the operation on the extended values.
1999     llvm::Value *Overflow, *Result;
2000     Result = EmitOverflowIntrinsic(*this, IntrinsicId, Left, Right, Overflow);
2001
2002     if (EncompassingInfo.Width > ResultInfo.Width) {
2003       // The encompassing type is wider than the result type, so we need to
2004       // truncate it.
2005       llvm::Value *ResultTrunc = Builder.CreateTrunc(Result, ResultLLVMTy);
2006
2007       // To see if the truncation caused an overflow, we will extend
2008       // the result and then compare it to the original result.
2009       llvm::Value *ResultTruncExt = Builder.CreateIntCast(
2010           ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed);
2011       llvm::Value *TruncationOverflow =
2012           Builder.CreateICmpNE(Result, ResultTruncExt);
2013
2014       Overflow = Builder.CreateOr(Overflow, TruncationOverflow);
2015       Result = ResultTrunc;
2016     }
2017
2018     // Finally, store the result using the pointer.
2019     bool isVolatile =
2020       ResultArg->getType()->getPointeeType().isVolatileQualified();
2021     Builder.CreateStore(EmitToMemory(Result, ResultQTy), ResultPtr, isVolatile);
2022
2023     return RValue::get(Overflow);
2024   }
2025
2026   case Builtin::BI__builtin_uadd_overflow:
2027   case Builtin::BI__builtin_uaddl_overflow:
2028   case Builtin::BI__builtin_uaddll_overflow:
2029   case Builtin::BI__builtin_usub_overflow:
2030   case Builtin::BI__builtin_usubl_overflow:
2031   case Builtin::BI__builtin_usubll_overflow:
2032   case Builtin::BI__builtin_umul_overflow:
2033   case Builtin::BI__builtin_umull_overflow:
2034   case Builtin::BI__builtin_umulll_overflow:
2035   case Builtin::BI__builtin_sadd_overflow:
2036   case Builtin::BI__builtin_saddl_overflow:
2037   case Builtin::BI__builtin_saddll_overflow:
2038   case Builtin::BI__builtin_ssub_overflow:
2039   case Builtin::BI__builtin_ssubl_overflow:
2040   case Builtin::BI__builtin_ssubll_overflow:
2041   case Builtin::BI__builtin_smul_overflow:
2042   case Builtin::BI__builtin_smull_overflow:
2043   case Builtin::BI__builtin_smulll_overflow: {
2044
2045     // We translate all of these builtins directly to the relevant llvm IR node.
2046
2047     // Scalarize our inputs.
2048     llvm::Value *X = EmitScalarExpr(E->getArg(0));
2049     llvm::Value *Y = EmitScalarExpr(E->getArg(1));
2050     Address SumOutPtr = EmitPointerWithAlignment(E->getArg(2));
2051
2052     // Decide which of the overflow intrinsics we are lowering to:
2053     llvm::Intrinsic::ID IntrinsicId;
2054     switch (BuiltinID) {
2055     default: llvm_unreachable("Unknown overflow builtin id.");
2056     case Builtin::BI__builtin_uadd_overflow:
2057     case Builtin::BI__builtin_uaddl_overflow:
2058     case Builtin::BI__builtin_uaddll_overflow:
2059       IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
2060       break;
2061     case Builtin::BI__builtin_usub_overflow:
2062     case Builtin::BI__builtin_usubl_overflow:
2063     case Builtin::BI__builtin_usubll_overflow:
2064       IntrinsicId = llvm::Intrinsic::usub_with_overflow;
2065       break;
2066     case Builtin::BI__builtin_umul_overflow:
2067     case Builtin::BI__builtin_umull_overflow:
2068     case Builtin::BI__builtin_umulll_overflow:
2069       IntrinsicId = llvm::Intrinsic::umul_with_overflow;
2070       break;
2071     case Builtin::BI__builtin_sadd_overflow:
2072     case Builtin::BI__builtin_saddl_overflow:
2073     case Builtin::BI__builtin_saddll_overflow:
2074       IntrinsicId = llvm::Intrinsic::sadd_with_overflow;
2075       break;
2076     case Builtin::BI__builtin_ssub_overflow:
2077     case Builtin::BI__builtin_ssubl_overflow:
2078     case Builtin::BI__builtin_ssubll_overflow:
2079       IntrinsicId = llvm::Intrinsic::ssub_with_overflow;
2080       break;
2081     case Builtin::BI__builtin_smul_overflow:
2082     case Builtin::BI__builtin_smull_overflow:
2083     case Builtin::BI__builtin_smulll_overflow:
2084       IntrinsicId = llvm::Intrinsic::smul_with_overflow;
2085       break;
2086     }
2087
2088
2089     llvm::Value *Carry;
2090     llvm::Value *Sum = EmitOverflowIntrinsic(*this, IntrinsicId, X, Y, Carry);
2091     Builder.CreateStore(Sum, SumOutPtr);
2092
2093     return RValue::get(Carry);
2094   }
2095   case Builtin::BI__builtin_addressof:
2096     return RValue::get(EmitLValue(E->getArg(0)).getPointer());
2097   case Builtin::BI__builtin_operator_new:
2098     return EmitBuiltinNewDeleteCall(FD->getType()->castAs<FunctionProtoType>(),
2099                                     E->getArg(0), false);
2100   case Builtin::BI__builtin_operator_delete:
2101     return EmitBuiltinNewDeleteCall(FD->getType()->castAs<FunctionProtoType>(),
2102                                     E->getArg(0), true);
2103   case Builtin::BI__noop:
2104     // __noop always evaluates to an integer literal zero.
2105     return RValue::get(ConstantInt::get(IntTy, 0));
2106   case Builtin::BI__builtin_call_with_static_chain: {
2107     const CallExpr *Call = cast<CallExpr>(E->getArg(0));
2108     const Expr *Chain = E->getArg(1);
2109     return EmitCall(Call->getCallee()->getType(),
2110                     EmitCallee(Call->getCallee()), Call, ReturnValue,
2111                     EmitScalarExpr(Chain));
2112   }
2113   case Builtin::BI_InterlockedExchange8:
2114   case Builtin::BI_InterlockedExchange16:
2115   case Builtin::BI_InterlockedExchange:
2116   case Builtin::BI_InterlockedExchangePointer:
2117     return RValue::get(
2118         EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E));
2119   case Builtin::BI_InterlockedCompareExchangePointer: {
2120     llvm::Type *RTy;
2121     llvm::IntegerType *IntType =
2122       IntegerType::get(getLLVMContext(),
2123                        getContext().getTypeSize(E->getType()));
2124     llvm::Type *IntPtrType = IntType->getPointerTo();
2125
2126     llvm::Value *Destination =
2127       Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), IntPtrType);
2128
2129     llvm::Value *Exchange = EmitScalarExpr(E->getArg(1));
2130     RTy = Exchange->getType();
2131     Exchange = Builder.CreatePtrToInt(Exchange, IntType);
2132
2133     llvm::Value *Comparand =
2134       Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType);
2135
2136     auto Result =
2137         Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange,
2138                                     AtomicOrdering::SequentiallyConsistent,
2139                                     AtomicOrdering::SequentiallyConsistent);
2140     Result->setVolatile(true);
2141
2142     return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result,
2143                                                                          0),
2144                                               RTy));
2145   }
2146   case Builtin::BI_InterlockedCompareExchange8:
2147   case Builtin::BI_InterlockedCompareExchange16:
2148   case Builtin::BI_InterlockedCompareExchange:
2149   case Builtin::BI_InterlockedCompareExchange64: {
2150     AtomicCmpXchgInst *CXI = Builder.CreateAtomicCmpXchg(
2151         EmitScalarExpr(E->getArg(0)),
2152         EmitScalarExpr(E->getArg(2)),
2153         EmitScalarExpr(E->getArg(1)),
2154         AtomicOrdering::SequentiallyConsistent,
2155         AtomicOrdering::SequentiallyConsistent);
2156       CXI->setVolatile(true);
2157       return RValue::get(Builder.CreateExtractValue(CXI, 0));
2158   }
2159   case Builtin::BI_InterlockedIncrement16:
2160   case Builtin::BI_InterlockedIncrement:
2161     return RValue::get(
2162         EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E));
2163   case Builtin::BI_InterlockedDecrement16:
2164   case Builtin::BI_InterlockedDecrement:
2165     return RValue::get(
2166         EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E));
2167   case Builtin::BI_InterlockedAnd8:
2168   case Builtin::BI_InterlockedAnd16:
2169   case Builtin::BI_InterlockedAnd:
2170     return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E));
2171   case Builtin::BI_InterlockedExchangeAdd8:
2172   case Builtin::BI_InterlockedExchangeAdd16:
2173   case Builtin::BI_InterlockedExchangeAdd:
2174     return RValue::get(
2175         EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E));
2176   case Builtin::BI_InterlockedExchangeSub8:
2177   case Builtin::BI_InterlockedExchangeSub16:
2178   case Builtin::BI_InterlockedExchangeSub:
2179     return RValue::get(
2180         EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E));
2181   case Builtin::BI_InterlockedOr8:
2182   case Builtin::BI_InterlockedOr16:
2183   case Builtin::BI_InterlockedOr:
2184     return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E));
2185   case Builtin::BI_InterlockedXor8:
2186   case Builtin::BI_InterlockedXor16:
2187   case Builtin::BI_InterlockedXor:
2188     return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E));
2189   case Builtin::BI__readfsdword: {
2190     llvm::Type *IntTy = ConvertType(E->getType());
2191     Value *IntToPtr =
2192       Builder.CreateIntToPtr(EmitScalarExpr(E->getArg(0)),
2193                              llvm::PointerType::get(IntTy, 257));
2194     LoadInst *Load = Builder.CreateAlignedLoad(
2195         IntTy, IntToPtr, getContext().getTypeAlignInChars(E->getType()));
2196     Load->setVolatile(true);
2197     return RValue::get(Load);
2198   }
2199
2200   case Builtin::BI__exception_code:
2201   case Builtin::BI_exception_code:
2202     return RValue::get(EmitSEHExceptionCode());
2203   case Builtin::BI__exception_info:
2204   case Builtin::BI_exception_info:
2205     return RValue::get(EmitSEHExceptionInfo());
2206   case Builtin::BI__abnormal_termination:
2207   case Builtin::BI_abnormal_termination:
2208     return RValue::get(EmitSEHAbnormalTermination());
2209   case Builtin::BI_setjmpex: {
2210     if (getTarget().getTriple().isOSMSVCRT()) {
2211       llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy};
2212       llvm::AttributeSet ReturnsTwiceAttr =
2213           AttributeSet::get(getLLVMContext(), llvm::AttributeSet::FunctionIndex,
2214                             llvm::Attribute::ReturnsTwice);
2215       llvm::Constant *SetJmpEx = CGM.CreateRuntimeFunction(
2216           llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/false),
2217           "_setjmpex", ReturnsTwiceAttr, /*Local=*/true);
2218       llvm::Value *Buf = Builder.CreateBitOrPointerCast(
2219           EmitScalarExpr(E->getArg(0)), Int8PtrTy);
2220       llvm::Value *FrameAddr =
2221           Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
2222                              ConstantInt::get(Int32Ty, 0));
2223       llvm::Value *Args[] = {Buf, FrameAddr};
2224       llvm::CallSite CS = EmitRuntimeCallOrInvoke(SetJmpEx, Args);
2225       CS.setAttributes(ReturnsTwiceAttr);
2226       return RValue::get(CS.getInstruction());
2227     }
2228     break;
2229   }
2230   case Builtin::BI_setjmp: {
2231     if (getTarget().getTriple().isOSMSVCRT()) {
2232       llvm::AttributeSet ReturnsTwiceAttr =
2233           AttributeSet::get(getLLVMContext(), llvm::AttributeSet::FunctionIndex,
2234                             llvm::Attribute::ReturnsTwice);
2235       llvm::Value *Buf = Builder.CreateBitOrPointerCast(
2236           EmitScalarExpr(E->getArg(0)), Int8PtrTy);
2237       llvm::CallSite CS;
2238       if (getTarget().getTriple().getArch() == llvm::Triple::x86) {
2239         llvm::Type *ArgTypes[] = {Int8PtrTy, IntTy};
2240         llvm::Constant *SetJmp3 = CGM.CreateRuntimeFunction(
2241             llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/true),
2242             "_setjmp3", ReturnsTwiceAttr, /*Local=*/true);
2243         llvm::Value *Count = ConstantInt::get(IntTy, 0);
2244         llvm::Value *Args[] = {Buf, Count};
2245         CS = EmitRuntimeCallOrInvoke(SetJmp3, Args);
2246       } else {
2247         llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy};
2248         llvm::Constant *SetJmp = CGM.CreateRuntimeFunction(
2249             llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/false),
2250             "_setjmp", ReturnsTwiceAttr, /*Local=*/true);
2251         llvm::Value *FrameAddr =
2252             Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
2253                                ConstantInt::get(Int32Ty, 0));
2254         llvm::Value *Args[] = {Buf, FrameAddr};
2255         CS = EmitRuntimeCallOrInvoke(SetJmp, Args);
2256       }
2257       CS.setAttributes(ReturnsTwiceAttr);
2258       return RValue::get(CS.getInstruction());
2259     }
2260     break;
2261   }
2262
2263   case Builtin::BI__GetExceptionInfo: {
2264     if (llvm::GlobalVariable *GV =
2265             CGM.getCXXABI().getThrowInfo(FD->getParamDecl(0)->getType()))
2266       return RValue::get(llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy));
2267     break;
2268   }
2269
2270   case Builtin::BI__builtin_coro_size: {
2271     auto & Context = getContext();
2272     auto SizeTy = Context.getSizeType();
2273     auto T = Builder.getIntNTy(Context.getTypeSize(SizeTy));
2274     Value *F = CGM.getIntrinsic(Intrinsic::coro_size, T);
2275     return RValue::get(Builder.CreateCall(F));
2276   }
2277
2278   case Builtin::BI__builtin_coro_id:
2279     return EmitCoroutineIntrinsic(E, Intrinsic::coro_id);
2280   case Builtin::BI__builtin_coro_promise:
2281     return EmitCoroutineIntrinsic(E, Intrinsic::coro_promise);
2282   case Builtin::BI__builtin_coro_resume:
2283     return EmitCoroutineIntrinsic(E, Intrinsic::coro_resume);
2284   case Builtin::BI__builtin_coro_frame:
2285     return EmitCoroutineIntrinsic(E, Intrinsic::coro_frame);
2286   case Builtin::BI__builtin_coro_free:
2287     return EmitCoroutineIntrinsic(E, Intrinsic::coro_free);
2288   case Builtin::BI__builtin_coro_destroy:
2289     return EmitCoroutineIntrinsic(E, Intrinsic::coro_destroy);
2290   case Builtin::BI__builtin_coro_done:
2291     return EmitCoroutineIntrinsic(E, Intrinsic::coro_done);
2292   case Builtin::BI__builtin_coro_alloc:
2293     return EmitCoroutineIntrinsic(E, Intrinsic::coro_alloc);
2294   case Builtin::BI__builtin_coro_begin:
2295     return EmitCoroutineIntrinsic(E, Intrinsic::coro_begin);
2296   case Builtin::BI__builtin_coro_end:
2297     return EmitCoroutineIntrinsic(E, Intrinsic::coro_end);
2298   case Builtin::BI__builtin_coro_suspend:
2299     return EmitCoroutineIntrinsic(E, Intrinsic::coro_suspend);
2300   case Builtin::BI__builtin_coro_param:
2301     return EmitCoroutineIntrinsic(E, Intrinsic::coro_param);
2302
2303   // OpenCL v2.0 s6.13.16.2, Built-in pipe read and write functions
2304   case Builtin::BIread_pipe:
2305   case Builtin::BIwrite_pipe: {
2306     Value *Arg0 = EmitScalarExpr(E->getArg(0)),
2307           *Arg1 = EmitScalarExpr(E->getArg(1));
2308     CGOpenCLRuntime OpenCLRT(CGM);
2309     Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
2310     Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
2311
2312     // Type of the generic packet parameter.
2313     unsigned GenericAS =
2314         getContext().getTargetAddressSpace(LangAS::opencl_generic);
2315     llvm::Type *I8PTy = llvm::PointerType::get(
2316         llvm::Type::getInt8Ty(getLLVMContext()), GenericAS);
2317
2318     // Testing which overloaded version we should generate the call for.
2319     if (2U == E->getNumArgs()) {
2320       const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_2"
2321                                                              : "__write_pipe_2";
2322       // Creating a generic function type to be able to call with any builtin or
2323       // user defined type.
2324       llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy, Int32Ty, Int32Ty};
2325       llvm::FunctionType *FTy = llvm::FunctionType::get(
2326           Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
2327       Value *BCast = Builder.CreatePointerCast(Arg1, I8PTy);
2328       return RValue::get(
2329           Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2330                              {Arg0, BCast, PacketSize, PacketAlign}));
2331     } else {
2332       assert(4 == E->getNumArgs() &&
2333              "Illegal number of parameters to pipe function");
2334       const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_4"
2335                                                              : "__write_pipe_4";
2336
2337       llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, I8PTy,
2338                               Int32Ty, Int32Ty};
2339       Value *Arg2 = EmitScalarExpr(E->getArg(2)),
2340             *Arg3 = EmitScalarExpr(E->getArg(3));
2341       llvm::FunctionType *FTy = llvm::FunctionType::get(
2342           Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
2343       Value *BCast = Builder.CreatePointerCast(Arg3, I8PTy);
2344       // We know the third argument is an integer type, but we may need to cast
2345       // it to i32.
2346       if (Arg2->getType() != Int32Ty)
2347         Arg2 = Builder.CreateZExtOrTrunc(Arg2, Int32Ty);
2348       return RValue::get(Builder.CreateCall(
2349           CGM.CreateRuntimeFunction(FTy, Name),
2350           {Arg0, Arg1, Arg2, BCast, PacketSize, PacketAlign}));
2351     }
2352   }
2353   // OpenCL v2.0 s6.13.16 ,s9.17.3.5 - Built-in pipe reserve read and write
2354   // functions
2355   case Builtin::BIreserve_read_pipe:
2356   case Builtin::BIreserve_write_pipe:
2357   case Builtin::BIwork_group_reserve_read_pipe:
2358   case Builtin::BIwork_group_reserve_write_pipe:
2359   case Builtin::BIsub_group_reserve_read_pipe:
2360   case Builtin::BIsub_group_reserve_write_pipe: {
2361     // Composing the mangled name for the function.
2362     const char *Name;
2363     if (BuiltinID == Builtin::BIreserve_read_pipe)
2364       Name = "__reserve_read_pipe";
2365     else if (BuiltinID == Builtin::BIreserve_write_pipe)
2366       Name = "__reserve_write_pipe";
2367     else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
2368       Name = "__work_group_reserve_read_pipe";
2369     else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
2370       Name = "__work_group_reserve_write_pipe";
2371     else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
2372       Name = "__sub_group_reserve_read_pipe";
2373     else
2374       Name = "__sub_group_reserve_write_pipe";
2375
2376     Value *Arg0 = EmitScalarExpr(E->getArg(0)),
2377           *Arg1 = EmitScalarExpr(E->getArg(1));
2378     llvm::Type *ReservedIDTy = ConvertType(getContext().OCLReserveIDTy);
2379     CGOpenCLRuntime OpenCLRT(CGM);
2380     Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
2381     Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
2382
2383     // Building the generic function prototype.
2384     llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty, Int32Ty};
2385     llvm::FunctionType *FTy = llvm::FunctionType::get(
2386         ReservedIDTy, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
2387     // We know the second argument is an integer type, but we may need to cast
2388     // it to i32.
2389     if (Arg1->getType() != Int32Ty)
2390       Arg1 = Builder.CreateZExtOrTrunc(Arg1, Int32Ty);
2391     return RValue::get(
2392         Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2393                            {Arg0, Arg1, PacketSize, PacketAlign}));
2394   }
2395   // OpenCL v2.0 s6.13.16, s9.17.3.5 - Built-in pipe commit read and write
2396   // functions
2397   case Builtin::BIcommit_read_pipe:
2398   case Builtin::BIcommit_write_pipe:
2399   case Builtin::BIwork_group_commit_read_pipe:
2400   case Builtin::BIwork_group_commit_write_pipe:
2401   case Builtin::BIsub_group_commit_read_pipe:
2402   case Builtin::BIsub_group_commit_write_pipe: {
2403     const char *Name;
2404     if (BuiltinID == Builtin::BIcommit_read_pipe)
2405       Name = "__commit_read_pipe";
2406     else if (BuiltinID == Builtin::BIcommit_write_pipe)
2407       Name = "__commit_write_pipe";
2408     else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
2409       Name = "__work_group_commit_read_pipe";
2410     else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
2411       Name = "__work_group_commit_write_pipe";
2412     else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
2413       Name = "__sub_group_commit_read_pipe";
2414     else
2415       Name = "__sub_group_commit_write_pipe";
2416
2417     Value *Arg0 = EmitScalarExpr(E->getArg(0)),
2418           *Arg1 = EmitScalarExpr(E->getArg(1));
2419     CGOpenCLRuntime OpenCLRT(CGM);
2420     Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
2421     Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
2422
2423     // Building the generic function prototype.
2424     llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, Int32Ty};
2425     llvm::FunctionType *FTy =
2426         llvm::FunctionType::get(llvm::Type::getVoidTy(getLLVMContext()),
2427                                 llvm::ArrayRef<llvm::Type *>(ArgTys), false);
2428
2429     return RValue::get(
2430         Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2431                            {Arg0, Arg1, PacketSize, PacketAlign}));
2432   }
2433   // OpenCL v2.0 s6.13.16.4 Built-in pipe query functions
2434   case Builtin::BIget_pipe_num_packets:
2435   case Builtin::BIget_pipe_max_packets: {
2436     const char *Name;
2437     if (BuiltinID == Builtin::BIget_pipe_num_packets)
2438       Name = "__get_pipe_num_packets";
2439     else
2440       Name = "__get_pipe_max_packets";
2441
2442     // Building the generic function prototype.
2443     Value *Arg0 = EmitScalarExpr(E->getArg(0));
2444     CGOpenCLRuntime OpenCLRT(CGM);
2445     Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
2446     Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
2447     llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty};
2448     llvm::FunctionType *FTy = llvm::FunctionType::get(
2449         Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
2450
2451     return RValue::get(Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2452                                           {Arg0, PacketSize, PacketAlign}));
2453   }
2454
2455   // OpenCL v2.0 s6.13.9 - Address space qualifier functions.
2456   case Builtin::BIto_global:
2457   case Builtin::BIto_local:
2458   case Builtin::BIto_private: {
2459     auto Arg0 = EmitScalarExpr(E->getArg(0));
2460     auto NewArgT = llvm::PointerType::get(Int8Ty,
2461       CGM.getContext().getTargetAddressSpace(LangAS::opencl_generic));
2462     auto NewRetT = llvm::PointerType::get(Int8Ty,
2463       CGM.getContext().getTargetAddressSpace(
2464         E->getType()->getPointeeType().getAddressSpace()));
2465     auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT}, false);
2466     llvm::Value *NewArg;
2467     if (Arg0->getType()->getPointerAddressSpace() !=
2468         NewArgT->getPointerAddressSpace())
2469       NewArg = Builder.CreateAddrSpaceCast(Arg0, NewArgT);
2470     else
2471       NewArg = Builder.CreateBitOrPointerCast(Arg0, NewArgT);
2472     auto NewName = std::string("__") + E->getDirectCallee()->getName().str();
2473     auto NewCall =
2474         Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, NewName), {NewArg});
2475     return RValue::get(Builder.CreateBitOrPointerCast(NewCall,
2476       ConvertType(E->getType())));
2477   }
2478
2479   // OpenCL v2.0, s6.13.17 - Enqueue kernel function.
2480   // It contains four different overload formats specified in Table 6.13.17.1.
2481   case Builtin::BIenqueue_kernel: {
2482     StringRef Name; // Generated function call name
2483     unsigned NumArgs = E->getNumArgs();
2484
2485     llvm::Type *QueueTy = ConvertType(getContext().OCLQueueTy);
2486     llvm::Type *RangeTy = ConvertType(getContext().OCLNDRangeTy);
2487
2488     llvm::Value *Queue = EmitScalarExpr(E->getArg(0));
2489     llvm::Value *Flags = EmitScalarExpr(E->getArg(1));
2490     llvm::Value *Range = EmitScalarExpr(E->getArg(2));
2491
2492     if (NumArgs == 4) {
2493       // The most basic form of the call with parameters:
2494       // queue_t, kernel_enqueue_flags_t, ndrange_t, block(void)
2495       Name = "__enqueue_kernel_basic";
2496       llvm::Type *ArgTys[] = {QueueTy, Int32Ty, RangeTy, Int8PtrTy};
2497       llvm::FunctionType *FTy = llvm::FunctionType::get(
2498           Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys, 4), false);
2499
2500       llvm::Value *Block =
2501           Builder.CreateBitCast(EmitScalarExpr(E->getArg(3)), Int8PtrTy);
2502
2503       return RValue::get(Builder.CreateCall(
2504           CGM.CreateRuntimeFunction(FTy, Name), {Queue, Flags, Range, Block}));
2505     }
2506     assert(NumArgs >= 5 && "Invalid enqueue_kernel signature");
2507
2508     // Could have events and/or vaargs.
2509     if (E->getArg(3)->getType()->isBlockPointerType()) {
2510       // No events passed, but has variadic arguments.
2511       Name = "__enqueue_kernel_vaargs";
2512       llvm::Value *Block =
2513           Builder.CreateBitCast(EmitScalarExpr(E->getArg(3)), Int8PtrTy);
2514       // Create a vector of the arguments, as well as a constant value to
2515       // express to the runtime the number of variadic arguments.
2516       std::vector<llvm::Value *> Args = {Queue, Flags, Range, Block,
2517                                          ConstantInt::get(IntTy, NumArgs - 4)};
2518       std::vector<llvm::Type *> ArgTys = {QueueTy, IntTy, RangeTy, Int8PtrTy,
2519                                           IntTy};
2520
2521       // Each of the following arguments specifies the size of the corresponding
2522       // argument passed to the enqueued block.
2523       for (unsigned I = 4/*Position of the first size arg*/; I < NumArgs; ++I)
2524         Args.push_back(
2525             Builder.CreateZExtOrTrunc(EmitScalarExpr(E->getArg(I)), SizeTy));
2526
2527       llvm::FunctionType *FTy = llvm::FunctionType::get(
2528           Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), true);
2529       return RValue::get(
2530           Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2531                              llvm::ArrayRef<llvm::Value *>(Args)));
2532     }
2533     // Any calls now have event arguments passed.
2534     if (NumArgs >= 7) {
2535       llvm::Type *EventTy = ConvertType(getContext().OCLClkEventTy);
2536       llvm::Type *EventPtrTy = EventTy->getPointerTo(
2537           CGM.getContext().getTargetAddressSpace(LangAS::opencl_generic));
2538
2539       llvm::Value *NumEvents =
2540           Builder.CreateZExtOrTrunc(EmitScalarExpr(E->getArg(3)), Int32Ty);
2541       llvm::Value *EventList =
2542           E->getArg(4)->getType()->isArrayType()
2543               ? EmitArrayToPointerDecay(E->getArg(4)).getPointer()
2544               : EmitScalarExpr(E->getArg(4));
2545       llvm::Value *ClkEvent = EmitScalarExpr(E->getArg(5));
2546       // Convert to generic address space.
2547       EventList = Builder.CreatePointerCast(EventList, EventPtrTy);
2548       ClkEvent = Builder.CreatePointerCast(ClkEvent, EventPtrTy);
2549       llvm::Value *Block =
2550           Builder.CreateBitCast(EmitScalarExpr(E->getArg(6)), Int8PtrTy);
2551
2552       std::vector<llvm::Type *> ArgTys = {QueueTy,  Int32Ty,    RangeTy,
2553                                           Int32Ty,  EventPtrTy, EventPtrTy,
2554                                           Int8PtrTy};
2555
2556       std::vector<llvm::Value *> Args = {Queue,     Flags,    Range, NumEvents,
2557                                          EventList, ClkEvent, Block};
2558
2559       if (NumArgs == 7) {
2560         // Has events but no variadics.
2561         Name = "__enqueue_kernel_basic_events";
2562         llvm::FunctionType *FTy = llvm::FunctionType::get(
2563             Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
2564         return RValue::get(
2565             Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2566                                llvm::ArrayRef<llvm::Value *>(Args)));
2567       }
2568       // Has event info and variadics
2569       // Pass the number of variadics to the runtime function too.
2570       Args.push_back(ConstantInt::get(Int32Ty, NumArgs - 7));
2571       ArgTys.push_back(Int32Ty);
2572       Name = "__enqueue_kernel_events_vaargs";
2573
2574       // Each of the following arguments specifies the size of the corresponding
2575       // argument passed to the enqueued block.
2576       for (unsigned I = 7/*Position of the first size arg*/; I < NumArgs; ++I)
2577         Args.push_back(
2578             Builder.CreateZExtOrTrunc(EmitScalarExpr(E->getArg(I)), SizeTy));
2579
2580       llvm::FunctionType *FTy = llvm::FunctionType::get(
2581           Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), true);
2582       return RValue::get(
2583           Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
2584                              llvm::ArrayRef<llvm::Value *>(Args)));
2585     }
2586   }
2587   // OpenCL v2.0 s6.13.17.6 - Kernel query functions need bitcast of block
2588   // parameter.
2589   case Builtin::BIget_kernel_work_group_size: {
2590     Value *Arg = EmitScalarExpr(E->getArg(0));
2591     Arg = Builder.CreateBitCast(Arg, Int8PtrTy);
2592     return RValue::get(
2593         Builder.CreateCall(CGM.CreateRuntimeFunction(
2594                                llvm::FunctionType::get(IntTy, Int8PtrTy, false),
2595                                "__get_kernel_work_group_size_impl"),
2596                            Arg));
2597   }
2598   case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
2599     Value *Arg = EmitScalarExpr(E->getArg(0));
2600     Arg = Builder.CreateBitCast(Arg, Int8PtrTy);
2601     return RValue::get(Builder.CreateCall(
2602         CGM.CreateRuntimeFunction(
2603             llvm::FunctionType::get(IntTy, Int8PtrTy, false),
2604             "__get_kernel_preferred_work_group_multiple_impl"),
2605         Arg));
2606   }
2607   case Builtin::BIprintf:
2608     if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice)
2609       return EmitCUDADevicePrintfCallExpr(E, ReturnValue);
2610     break;
2611   case Builtin::BI__builtin_canonicalize:
2612   case Builtin::BI__builtin_canonicalizef:
2613   case Builtin::BI__builtin_canonicalizel:
2614     return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::canonicalize));
2615
2616   case Builtin::BI__builtin_thread_pointer: {
2617     if (!getContext().getTargetInfo().isTLSSupported())
2618       CGM.ErrorUnsupported(E, "__builtin_thread_pointer");
2619     // Fall through - it's already mapped to the intrinsic by GCCBuiltin.
2620     break;
2621   }
2622   case Builtin::BI__builtin_os_log_format: {
2623     assert(E->getNumArgs() >= 2 &&
2624            "__builtin_os_log_format takes at least 2 arguments");
2625     analyze_os_log::OSLogBufferLayout Layout;
2626     analyze_os_log::computeOSLogBufferLayout(CGM.getContext(), E, Layout);
2627     Address BufAddr = EmitPointerWithAlignment(E->getArg(0));
2628     // Ignore argument 1, the format string. It is not currently used.
2629     CharUnits Offset;
2630     Builder.CreateStore(
2631         Builder.getInt8(Layout.getSummaryByte()),
2632         Builder.CreateConstByteGEP(BufAddr, Offset++, "summary"));
2633     Builder.CreateStore(
2634         Builder.getInt8(Layout.getNumArgsByte()),
2635         Builder.CreateConstByteGEP(BufAddr, Offset++, "numArgs"));
2636
2637     llvm::SmallVector<llvm::Value *, 4> RetainableOperands;
2638     for (const auto &Item : Layout.Items) {
2639       Builder.CreateStore(
2640           Builder.getInt8(Item.getDescriptorByte()),
2641           Builder.CreateConstByteGEP(BufAddr, Offset++, "argDescriptor"));
2642       Builder.CreateStore(
2643           Builder.getInt8(Item.getSizeByte()),
2644           Builder.CreateConstByteGEP(BufAddr, Offset++, "argSize"));
2645       Address Addr = Builder.CreateConstByteGEP(BufAddr, Offset);
2646       if (const Expr *TheExpr = Item.getExpr()) {
2647         Addr = Builder.CreateElementBitCast(
2648             Addr, ConvertTypeForMem(TheExpr->getType()));
2649         // Check if this is a retainable type.
2650         if (TheExpr->getType()->isObjCRetainableType()) {
2651           assert(getEvaluationKind(TheExpr->getType()) == TEK_Scalar &&
2652                  "Only scalar can be a ObjC retainable type");
2653           llvm::Value *SV = EmitScalarExpr(TheExpr, /*Ignore*/ false);
2654           RValue RV = RValue::get(SV);
2655           LValue LV = MakeAddrLValue(Addr, TheExpr->getType());
2656           EmitStoreThroughLValue(RV, LV);
2657           // Check if the object is constant, if not, save it in
2658           // RetainableOperands.
2659           if (!isa<Constant>(SV))
2660             RetainableOperands.push_back(SV);
2661         } else {
2662           EmitAnyExprToMem(TheExpr, Addr, Qualifiers(), /*isInit*/ true);
2663         }
2664       } else {
2665         Addr = Builder.CreateElementBitCast(Addr, Int32Ty);
2666         Builder.CreateStore(
2667             Builder.getInt32(Item.getConstValue().getQuantity()), Addr);
2668       }
2669       Offset += Item.size();
2670     }
2671
2672     // Push a clang.arc.use cleanup for each object in RetainableOperands. The
2673     // cleanup will cause the use to appear after the final log call, keeping
2674     // the object valid while it’s held in the log buffer.  Note that if there’s
2675     // a release cleanup on the object, it will already be active; since
2676     // cleanups are emitted in reverse order, the use will occur before the
2677     // object is released.
2678     if (!RetainableOperands.empty() && getLangOpts().ObjCAutoRefCount &&
2679         CGM.getCodeGenOpts().OptimizationLevel != 0)
2680       for (llvm::Value *object : RetainableOperands)
2681         pushFullExprCleanup<CallObjCArcUse>(getARCCleanupKind(), object);
2682
2683     return RValue::get(BufAddr.getPointer());
2684   }
2685
2686   case Builtin::BI__builtin_os_log_format_buffer_size: {
2687     analyze_os_log::OSLogBufferLayout Layout;
2688     analyze_os_log::computeOSLogBufferLayout(CGM.getContext(), E, Layout);
2689     return RValue::get(ConstantInt::get(ConvertType(E->getType()),
2690                                         Layout.size().getQuantity()));
2691   }
2692   }
2693
2694   // If this is an alias for a lib function (e.g. __builtin_sin), emit
2695   // the call using the normal call path, but using the unmangled
2696   // version of the function name.
2697   if (getContext().BuiltinInfo.isLibFunction(BuiltinID))
2698     return emitLibraryCall(*this, FD, E,
2699                            CGM.getBuiltinLibFunction(FD, BuiltinID));
2700
2701   // If this is a predefined lib function (e.g. malloc), emit the call
2702   // using exactly the normal call path.
2703   if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID))
2704     return emitLibraryCall(*this, FD, E,
2705                       cast<llvm::Constant>(EmitScalarExpr(E->getCallee())));
2706
2707   // Check that a call to a target specific builtin has the correct target
2708   // features.
2709   // This is down here to avoid non-target specific builtins, however, if
2710   // generic builtins start to require generic target features then we
2711   // can move this up to the beginning of the function.
2712   checkTargetFeatures(E, FD);
2713
2714   // See if we have a target specific intrinsic.
2715   const char *Name = getContext().BuiltinInfo.getName(BuiltinID);
2716   Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
2717   StringRef Prefix =
2718       llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch());
2719   if (!Prefix.empty()) {
2720     IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix.data(), Name);
2721     // NOTE we dont need to perform a compatibility flag check here since the
2722     // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the
2723     // MS builtins via ALL_MS_LANGUAGES and are filtered earlier.
2724     if (IntrinsicID == Intrinsic::not_intrinsic)
2725       IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix.data(), Name);
2726   }
2727
2728   if (IntrinsicID != Intrinsic::not_intrinsic) {
2729     SmallVector<Value*, 16> Args;
2730
2731     // Find out if any arguments are required to be integer constant
2732     // expressions.
2733     unsigned ICEArguments = 0;
2734     ASTContext::GetBuiltinTypeError Error;
2735     getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
2736     assert(Error == ASTContext::GE_None && "Should not codegen an error");
2737
2738     Function *F = CGM.getIntrinsic(IntrinsicID);
2739     llvm::FunctionType *FTy = F->getFunctionType();
2740
2741     for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
2742       Value *ArgValue;
2743       // If this is a normal argument, just emit it as a scalar.
2744       if ((ICEArguments & (1 << i)) == 0) {
2745         ArgValue = EmitScalarExpr(E->getArg(i));
2746       } else {
2747         // If this is required to be a constant, constant fold it so that we
2748         // know that the generated intrinsic gets a ConstantInt.
2749         llvm::APSInt Result;
2750         bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result,getContext());
2751         assert(IsConst && "Constant arg isn't actually constant?");
2752         (void)IsConst;
2753         ArgValue = llvm::ConstantInt::get(getLLVMContext(), Result);
2754       }
2755
2756       // If the intrinsic arg type is different from the builtin arg type
2757       // we need to do a bit cast.
2758       llvm::Type *PTy = FTy->getParamType(i);
2759       if (PTy != ArgValue->getType()) {
2760         assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) &&
2761                "Must be able to losslessly bit cast to param");
2762         ArgValue = Builder.CreateBitCast(ArgValue, PTy);
2763       }
2764
2765       Args.push_back(ArgValue);
2766     }
2767
2768     Value *V = Builder.CreateCall(F, Args);
2769     QualType BuiltinRetType = E->getType();
2770
2771     llvm::Type *RetTy = VoidTy;
2772     if (!BuiltinRetType->isVoidType())
2773       RetTy = ConvertType(BuiltinRetType);
2774
2775     if (RetTy != V->getType()) {
2776       assert(V->getType()->canLosslesslyBitCastTo(RetTy) &&
2777              "Must be able to losslessly bit cast result type");
2778       V = Builder.CreateBitCast(V, RetTy);
2779     }
2780
2781     return RValue::get(V);
2782   }
2783
2784   // See if we have a target specific builtin that needs to be lowered.
2785   if (Value *V = EmitTargetBuiltinExpr(BuiltinID, E))
2786     return RValue::get(V);
2787
2788   ErrorUnsupported(E, "builtin function");
2789
2790   // Unknown builtin, for now just dump it out and return undef.
2791   return GetUndefRValue(E->getType());
2792 }
2793
2794 static Value *EmitTargetArchBuiltinExpr(CodeGenFunction *CGF,
2795                                         unsigned BuiltinID, const CallExpr *E,
2796                                         llvm::Triple::ArchType Arch) {
2797   switch (Arch) {
2798   case llvm::Triple::arm:
2799   case llvm::Triple::armeb:
2800   case llvm::Triple::thumb:
2801   case llvm::Triple::thumbeb:
2802     return CGF->EmitARMBuiltinExpr(BuiltinID, E);
2803   case llvm::Triple::aarch64:
2804   case llvm::Triple::aarch64_be:
2805     return CGF->EmitAArch64BuiltinExpr(BuiltinID, E);
2806   case llvm::Triple::x86:
2807   case llvm::Triple::x86_64:
2808     return CGF->EmitX86BuiltinExpr(BuiltinID, E);
2809   case llvm::Triple::ppc:
2810   case llvm::Triple::ppc64:
2811   case llvm::Triple::ppc64le:
2812     return CGF->EmitPPCBuiltinExpr(BuiltinID, E);
2813   case llvm::Triple::r600:
2814   case llvm::Triple::amdgcn:
2815     return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E);
2816   case llvm::Triple::systemz:
2817     return CGF->EmitSystemZBuiltinExpr(BuiltinID, E);
2818   case llvm::Triple::nvptx:
2819   case llvm::Triple::nvptx64:
2820     return CGF->EmitNVPTXBuiltinExpr(BuiltinID, E);
2821   case llvm::Triple::wasm32:
2822   case llvm::Triple::wasm64:
2823     return CGF->EmitWebAssemblyBuiltinExpr(BuiltinID, E);
2824   default:
2825     return nullptr;
2826   }
2827 }
2828
2829 Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID,
2830                                               const CallExpr *E) {
2831   if (getContext().BuiltinInfo.isAuxBuiltinID(BuiltinID)) {
2832     assert(getContext().getAuxTargetInfo() && "Missing aux target info");
2833     return EmitTargetArchBuiltinExpr(
2834         this, getContext().BuiltinInfo.getAuxBuiltinID(BuiltinID), E,
2835         getContext().getAuxTargetInfo()->getTriple().getArch());
2836   }
2837
2838   return EmitTargetArchBuiltinExpr(this, BuiltinID, E,
2839                                    getTarget().getTriple().getArch());
2840 }
2841
2842 static llvm::VectorType *GetNeonType(CodeGenFunction *CGF,
2843                                      NeonTypeFlags TypeFlags,
2844                                      bool V1Ty=false) {
2845   int IsQuad = TypeFlags.isQuad();
2846   switch (TypeFlags.getEltType()) {
2847   case NeonTypeFlags::Int8:
2848   case NeonTypeFlags::Poly8:
2849     return llvm::VectorType::get(CGF->Int8Ty, V1Ty ? 1 : (8 << IsQuad));
2850   case NeonTypeFlags::Int16:
2851   case NeonTypeFlags::Poly16:
2852   case NeonTypeFlags::Float16:
2853     return llvm::VectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
2854   case NeonTypeFlags::Int32:
2855     return llvm::VectorType::get(CGF->Int32Ty, V1Ty ? 1 : (2 << IsQuad));
2856   case NeonTypeFlags::Int64:
2857   case NeonTypeFlags::Poly64:
2858     return llvm::VectorType::get(CGF->Int64Ty, V1Ty ? 1 : (1 << IsQuad));
2859   case NeonTypeFlags::Poly128:
2860     // FIXME: i128 and f128 doesn't get fully support in Clang and llvm.
2861     // There is a lot of i128 and f128 API missing.
2862     // so we use v16i8 to represent poly128 and get pattern matched.
2863     return llvm::VectorType::get(CGF->Int8Ty, 16);
2864   case NeonTypeFlags::Float32:
2865     return llvm::VectorType::get(CGF->FloatTy, V1Ty ? 1 : (2 << IsQuad));
2866   case NeonTypeFlags::Float64:
2867     return llvm::VectorType::get(CGF->DoubleTy, V1Ty ? 1 : (1 << IsQuad));
2868   }
2869   llvm_unreachable("Unknown vector element type!");
2870 }
2871
2872 static llvm::VectorType *GetFloatNeonType(CodeGenFunction *CGF,
2873                                           NeonTypeFlags IntTypeFlags) {
2874   int IsQuad = IntTypeFlags.isQuad();
2875   switch (IntTypeFlags.getEltType()) {
2876   case NeonTypeFlags::Int32:
2877     return llvm::VectorType::get(CGF->FloatTy, (2 << IsQuad));
2878   case NeonTypeFlags::Int64:
2879     return llvm::VectorType::get(CGF->DoubleTy, (1 << IsQuad));
2880   default:
2881     llvm_unreachable("Type can't be converted to floating-point!");
2882   }
2883 }
2884
2885 Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) {
2886   unsigned nElts = V->getType()->getVectorNumElements();
2887   Value* SV = llvm::ConstantVector::getSplat(nElts, C);
2888   return Builder.CreateShuffleVector(V, V, SV, "lane");
2889 }
2890
2891 Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops,
2892                                      const char *name,
2893                                      unsigned shift, bool rightshift) {
2894   unsigned j = 0;
2895   for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
2896        ai != ae; ++ai, ++j)
2897     if (shift > 0 && shift == j)
2898       Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift);
2899     else
2900       Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name);
2901
2902   return Builder.CreateCall(F, Ops, name);
2903 }
2904
2905 Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty,
2906                                             bool neg) {
2907   int SV = cast<ConstantInt>(V)->getSExtValue();
2908   return ConstantInt::get(Ty, neg ? -SV : SV);
2909 }
2910
2911 // \brief Right-shift a vector by a constant.
2912 Value *CodeGenFunction::EmitNeonRShiftImm(Value *Vec, Value *Shift,
2913                                           llvm::Type *Ty, bool usgn,
2914                                           const char *name) {
2915   llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
2916
2917   int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue();
2918   int EltSize = VTy->getScalarSizeInBits();
2919
2920   Vec = Builder.CreateBitCast(Vec, Ty);
2921
2922   // lshr/ashr are undefined when the shift amount is equal to the vector
2923   // element size.
2924   if (ShiftAmt == EltSize) {
2925     if (usgn) {
2926       // Right-shifting an unsigned value by its size yields 0.
2927       return llvm::ConstantAggregateZero::get(VTy);
2928     } else {
2929       // Right-shifting a signed value by its size is equivalent
2930       // to a shift of size-1.
2931       --ShiftAmt;
2932       Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
2933     }
2934   }
2935
2936   Shift = EmitNeonShiftVector(Shift, Ty, false);
2937   if (usgn)
2938     return Builder.CreateLShr(Vec, Shift, name);
2939   else
2940     return Builder.CreateAShr(Vec, Shift, name);
2941 }
2942
2943 enum {
2944   AddRetType = (1 << 0),
2945   Add1ArgType = (1 << 1),
2946   Add2ArgTypes = (1 << 2),
2947
2948   VectorizeRetType = (1 << 3),
2949   VectorizeArgTypes = (1 << 4),
2950
2951   InventFloatType = (1 << 5),
2952   UnsignedAlts = (1 << 6),
2953
2954   Use64BitVectors = (1 << 7),
2955   Use128BitVectors = (1 << 8),
2956
2957   Vectorize1ArgType = Add1ArgType | VectorizeArgTypes,
2958   VectorRet = AddRetType | VectorizeRetType,
2959   VectorRetGetArgs01 =
2960       AddRetType | Add2ArgTypes | VectorizeRetType | VectorizeArgTypes,
2961   FpCmpzModifiers =
2962       AddRetType | VectorizeRetType | Add1ArgType | InventFloatType
2963 };
2964
2965 namespace {
2966 struct NeonIntrinsicInfo {
2967   const char *NameHint;
2968   unsigned BuiltinID;
2969   unsigned LLVMIntrinsic;
2970   unsigned AltLLVMIntrinsic;
2971   unsigned TypeModifier;
2972
2973   bool operator<(unsigned RHSBuiltinID) const {
2974     return BuiltinID < RHSBuiltinID;
2975   }
2976   bool operator<(const NeonIntrinsicInfo &TE) const {
2977     return BuiltinID < TE.BuiltinID;
2978   }
2979 };
2980 } // end anonymous namespace
2981
2982 #define NEONMAP0(NameBase) \
2983   { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 }
2984
2985 #define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
2986   { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
2987       Intrinsic::LLVMIntrinsic, 0, TypeModifier }
2988
2989 #define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
2990   { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
2991       Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \
2992       TypeModifier }
2993
2994 static const NeonIntrinsicInfo ARMSIMDIntrinsicMap [] = {
2995   NEONMAP2(vabd_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
2996   NEONMAP2(vabdq_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
2997   NEONMAP1(vabs_v, arm_neon_vabs, 0),
2998   NEONMAP1(vabsq_v, arm_neon_vabs, 0),
2999   NEONMAP0(vaddhn_v),
3000   NEONMAP1(vaesdq_v, arm_neon_aesd, 0),
3001   NEONMAP1(vaeseq_v, arm_neon_aese, 0),
3002   NEONMAP1(vaesimcq_v, arm_neon_aesimc, 0),
3003   NEONMAP1(vaesmcq_v, arm_neon_aesmc, 0),
3004   NEONMAP1(vbsl_v, arm_neon_vbsl, AddRetType),
3005   NEONMAP1(vbslq_v, arm_neon_vbsl, AddRetType),
3006   NEONMAP1(vcage_v, arm_neon_vacge, 0),
3007   NEONMAP1(vcageq_v, arm_neon_vacge, 0),
3008   NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
3009   NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
3010   NEONMAP1(vcale_v, arm_neon_vacge, 0),
3011   NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
3012   NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
3013   NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
3014   NEONMAP1(vcls_v, arm_neon_vcls, Add1ArgType),
3015   NEONMAP1(vclsq_v, arm_neon_vcls, Add1ArgType),
3016   NEONMAP1(vclz_v, ctlz, Add1ArgType),
3017   NEONMAP1(vclzq_v, ctlz, Add1ArgType),
3018   NEONMAP1(vcnt_v, ctpop, Add1ArgType),
3019   NEONMAP1(vcntq_v, ctpop, Add1ArgType),
3020   NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0),
3021   NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0),
3022   NEONMAP0(vcvt_f32_v),
3023   NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
3024   NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
3025   NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
3026   NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
3027   NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
3028   NEONMAP0(vcvt_s32_v),
3029   NEONMAP0(vcvt_s64_v),
3030   NEONMAP0(vcvt_u32_v),
3031   NEONMAP0(vcvt_u64_v),
3032   NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
3033   NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
3034   NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
3035   NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
3036   NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
3037   NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
3038   NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
3039   NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
3040   NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
3041   NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
3042   NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
3043   NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
3044   NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
3045   NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
3046   NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
3047   NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
3048   NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
3049   NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
3050   NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
3051   NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
3052   NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
3053   NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
3054   NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
3055   NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
3056   NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
3057   NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
3058   NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
3059   NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
3060   NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
3061   NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
3062   NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
3063   NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
3064   NEONMAP0(vcvtq_f32_v),
3065   NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
3066   NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
3067   NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
3068   NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
3069   NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
3070   NEONMAP0(vcvtq_s32_v),
3071   NEONMAP0(vcvtq_s64_v),
3072   NEONMAP0(vcvtq_u32_v),
3073   NEONMAP0(vcvtq_u64_v),
3074   NEONMAP0(vext_v),
3075   NEONMAP0(vextq_v),
3076   NEONMAP0(vfma_v),
3077   NEONMAP0(vfmaq_v),
3078   NEONMAP2(vhadd_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts),
3079   NEONMAP2(vhaddq_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts),
3080   NEONMAP2(vhsub_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts),
3081   NEONMAP2(vhsubq_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts),
3082   NEONMAP0(vld1_dup_v),
3083   NEONMAP1(vld1_v, arm_neon_vld1, 0),
3084   NEONMAP0(vld1q_dup_v),
3085   NEONMAP1(vld1q_v, arm_neon_vld1, 0),
3086   NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
3087   NEONMAP1(vld2_v, arm_neon_vld2, 0),
3088   NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
3089   NEONMAP1(vld2q_v, arm_neon_vld2, 0),
3090   NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
3091   NEONMAP1(vld3_v, arm_neon_vld3, 0),
3092   NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
3093   NEONMAP1(vld3q_v, arm_neon_vld3, 0),
3094   NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
3095   NEONMAP1(vld4_v, arm_neon_vld4, 0),
3096   NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
3097   NEONMAP1(vld4q_v, arm_neon_vld4, 0),
3098   NEONMAP2(vmax_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
3099   NEONMAP1(vmaxnm_v, arm_neon_vmaxnm, Add1ArgType),
3100   NEONMAP1(vmaxnmq_v, arm_neon_vmaxnm, Add1ArgType),
3101   NEONMAP2(vmaxq_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
3102   NEONMAP2(vmin_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
3103   NEONMAP1(vminnm_v, arm_neon_vminnm, Add1ArgType),
3104   NEONMAP1(vminnmq_v, arm_neon_vminnm, Add1ArgType),
3105   NEONMAP2(vminq_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
3106   NEONMAP0(vmovl_v),
3107   NEONMAP0(vmovn_v),
3108   NEONMAP1(vmul_v, arm_neon_vmulp, Add1ArgType),
3109   NEONMAP0(vmull_v),
3110   NEONMAP1(vmulq_v, arm_neon_vmulp, Add1ArgType),
3111   NEONMAP2(vpadal_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts),
3112   NEONMAP2(vpadalq_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts),
3113   NEONMAP1(vpadd_v, arm_neon_vpadd, Add1ArgType),
3114   NEONMAP2(vpaddl_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts),
3115   NEONMAP2(vpaddlq_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts),
3116   NEONMAP1(vpaddq_v, arm_neon_vpadd, Add1ArgType),
3117   NEONMAP2(vpmax_v, arm_neon_vpmaxu, arm_neon_vpmaxs, Add1ArgType | UnsignedAlts),
3118   NEONMAP2(vpmin_v, arm_neon_vpminu, arm_neon_vpmins, Add1ArgType | UnsignedAlts),
3119   NEONMAP1(vqabs_v, arm_neon_vqabs, Add1ArgType),
3120   NEONMAP1(vqabsq_v, arm_neon_vqabs, Add1ArgType),
3121   NEONMAP2(vqadd_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts),
3122   NEONMAP2(vqaddq_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts),
3123   NEONMAP2(vqdmlal_v, arm_neon_vqdmull, arm_neon_vqadds, 0),
3124   NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, arm_neon_vqsubs, 0),
3125   NEONMAP1(vqdmulh_v, arm_neon_vqdmulh, Add1ArgType),
3126   NEONMAP1(vqdmulhq_v, arm_neon_vqdmulh, Add1ArgType),
3127   NEONMAP1(vqdmull_v, arm_neon_vqdmull, Add1ArgType),
3128   NEONMAP2(vqmovn_v, arm_neon_vqmovnu, arm_neon_vqmovns, Add1ArgType | UnsignedAlts),
3129   NEONMAP1(vqmovun_v, arm_neon_vqmovnsu, Add1ArgType),
3130   NEONMAP1(vqneg_v, arm_neon_vqneg, Add1ArgType),
3131   NEONMAP1(vqnegq_v, arm_neon_vqneg, Add1ArgType),
3132   NEONMAP1(vqrdmulh_v, arm_neon_vqrdmulh, Add1ArgType),
3133   NEONMAP1(vqrdmulhq_v, arm_neon_vqrdmulh, Add1ArgType),
3134   NEONMAP2(vqrshl_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts),
3135   NEONMAP2(vqrshlq_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts),
3136   NEONMAP2(vqshl_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts),
3137   NEONMAP2(vqshl_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts),
3138   NEONMAP2(vqshlq_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts),
3139   NEONMAP2(vqshlq_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts),
3140   NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
3141   NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
3142   NEONMAP2(vqsub_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts),
3143   NEONMAP2(vqsubq_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts),
3144   NEONMAP1(vraddhn_v, arm_neon_vraddhn, Add1ArgType),
3145   NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
3146   NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
3147   NEONMAP1(vrecps_v, arm_neon_vrecps, Add1ArgType),
3148   NEONMAP1(vrecpsq_v, arm_neon_vrecps, Add1ArgType),
3149   NEONMAP2(vrhadd_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
3150   NEONMAP2(vrhaddq_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
3151   NEONMAP1(vrnd_v, arm_neon_vrintz, Add1ArgType),
3152   NEONMAP1(vrnda_v, arm_neon_vrinta, Add1ArgType),
3153   NEONMAP1(vrndaq_v, arm_neon_vrinta, Add1ArgType),
3154   NEONMAP1(vrndm_v, arm_neon_vrintm, Add1ArgType),
3155   NEONMAP1(vrndmq_v, arm_neon_vrintm, Add1ArgType),
3156   NEONMAP1(vrndn_v, arm_neon_vrintn, Add1ArgType),
3157   NEONMAP1(vrndnq_v, arm_neon_vrintn, Add1ArgType),
3158   NEONMAP1(vrndp_v, arm_neon_vrintp, Add1ArgType),
3159   NEONMAP1(vrndpq_v, arm_neon_vrintp, Add1ArgType),
3160   NEONMAP1(vrndq_v, arm_neon_vrintz, Add1ArgType),
3161   NEONMAP1(vrndx_v, arm_neon_vrintx, Add1ArgType),
3162   NEONMAP1(vrndxq_v, arm_neon_vrintx, Add1ArgType),
3163   NEONMAP2(vrshl_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
3164   NEONMAP2(vrshlq_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
3165   NEONMAP2(vrshr_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts),
3166   NEONMAP2(vrshrq_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts),
3167   NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
3168   NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
3169   NEONMAP1(vrsqrts_v, arm_neon_vrsqrts, Add1ArgType),
3170   NEONMAP1(vrsqrtsq_v, arm_neon_vrsqrts, Add1ArgType),
3171   NEONMAP1(vrsubhn_v, arm_neon_vrsubhn, Add1ArgType),
3172   NEONMAP1(vsha1su0q_v, arm_neon_sha1su0, 0),
3173   NEONMAP1(vsha1su1q_v, arm_neon_sha1su1, 0),
3174   NEONMAP1(vsha256h2q_v, arm_neon_sha256h2, 0),
3175   NEONMAP1(vsha256hq_v, arm_neon_sha256h, 0),
3176   NEONMAP1(vsha256su0q_v, arm_neon_sha256su0, 0),
3177   NEONMAP1(vsha256su1q_v, arm_neon_sha256su1, 0),
3178   NEONMAP0(vshl_n_v),
3179   NEONMAP2(vshl_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts),
3180   NEONMAP0(vshll_n_v),
3181   NEONMAP0(vshlq_n_v),
3182   NEONMAP2(vshlq_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts),
3183   NEONMAP0(vshr_n_v),
3184   NEONMAP0(vshrn_n_v),
3185   NEONMAP0(vshrq_n_v),
3186   NEONMAP1(vst1_v, arm_neon_vst1, 0),
3187   NEONMAP1(vst1q_v, arm_neon_vst1, 0),
3188   NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
3189   NEONMAP1(vst2_v, arm_neon_vst2, 0),
3190   NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
3191   NEONMAP1(vst2q_v, arm_neon_vst2, 0),
3192   NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
3193   NEONMAP1(vst3_v, arm_neon_vst3, 0),
3194   NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
3195   NEONMAP1(vst3q_v, arm_neon_vst3, 0),
3196   NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
3197   NEONMAP1(vst4_v, arm_neon_vst4, 0),
3198   NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
3199   NEONMAP1(vst4q_v, arm_neon_vst4, 0),
3200   NEONMAP0(vsubhn_v),
3201   NEONMAP0(vtrn_v),
3202   NEONMAP0(vtrnq_v),
3203   NEONMAP0(vtst_v),
3204   NEONMAP0(vtstq_v),
3205   NEONMAP0(vuzp_v),
3206   NEONMAP0(vuzpq_v),
3207   NEONMAP0(vzip_v),
3208   NEONMAP0(vzipq_v)
3209 };
3210
3211 static const NeonIntrinsicInfo AArch64SIMDIntrinsicMap[] = {
3212   NEONMAP1(vabs_v, aarch64_neon_abs, 0),
3213   NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
3214   NEONMAP0(vaddhn_v),
3215   NEONMAP1(vaesdq_v, aarch64_crypto_aesd, 0),
3216   NEONMAP1(vaeseq_v, aarch64_crypto_aese, 0),
3217   NEONMAP1(vaesimcq_v, aarch64_crypto_aesimc, 0),
3218   NEONMAP1(vaesmcq_v, aarch64_crypto_aesmc, 0),
3219   NEONMAP1(vcage_v, aarch64_neon_facge, 0),
3220   NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
3221   NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
3222   NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
3223   NEONMAP1(vcale_v, aarch64_neon_facge, 0),
3224   NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
3225   NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
3226   NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
3227   NEONMAP1(vcls_v, aarch64_neon_cls, Add1ArgType),
3228   NEONMAP1(vclsq_v, aarch64_neon_cls, Add1ArgType),
3229   NEONMAP1(vclz_v, ctlz, Add1ArgType),
3230   NEONMAP1(vclzq_v, ctlz, Add1ArgType),
3231   NEONMAP1(vcnt_v, ctpop, Add1ArgType),
3232   NEONMAP1(vcntq_v, ctpop, Add1ArgType),
3233   NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
3234   NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
3235   NEONMAP0(vcvt_f32_v),
3236   NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
3237   NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
3238   NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
3239   NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
3240   NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
3241   NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
3242   NEONMAP0(vcvtq_f32_v),
3243   NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
3244   NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
3245   NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
3246   NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
3247   NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
3248   NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
3249   NEONMAP1(vcvtx_f32_v, aarch64_neon_fcvtxn, AddRetType | Add1ArgType),
3250   NEONMAP0(vext_v),
3251   NEONMAP0(vextq_v),
3252   NEONMAP0(vfma_v),
3253   NEONMAP0(vfmaq_v),
3254   NEONMAP2(vhadd_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts),
3255   NEONMAP2(vhaddq_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts),
3256   NEONMAP2(vhsub_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts),
3257   NEONMAP2(vhsubq_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts),
3258   NEONMAP0(vmovl_v),
3259   NEONMAP0(vmovn_v),
3260   NEONMAP1(vmul_v, aarch64_neon_pmul, Add1ArgType),
3261   NEONMAP1(vmulq_v, aarch64_neon_pmul, Add1ArgType),
3262   NEONMAP1(vpadd_v, aarch64_neon_addp, Add1ArgType),
3263   NEONMAP2(vpaddl_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts),
3264   NEONMAP2(vpaddlq_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts),
3265   NEONMAP1(vpaddq_v, aarch64_neon_addp, Add1ArgType),
3266   NEONMAP1(vqabs_v, aarch64_neon_sqabs, Add1ArgType),
3267   NEONMAP1(vqabsq_v, aarch64_neon_sqabs, Add1ArgType),
3268   NEONMAP2(vqadd_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts),
3269   NEONMAP2(vqaddq_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts),
3270   NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
3271   NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
3272   NEONMAP1(vqdmulh_v, aarch64_neon_sqdmulh, Add1ArgType),
3273   NEONMAP1(vqdmulhq_v, aarch64_neon_sqdmulh, Add1ArgType),
3274   NEONMAP1(vqdmull_v, aarch64_neon_sqdmull, Add1ArgType),
3275   NEONMAP2(vqmovn_v, aarch64_neon_uqxtn, aarch64_neon_sqxtn, Add1ArgType | UnsignedAlts),
3276   NEONMAP1(vqmovun_v, aarch64_neon_sqxtun, Add1ArgType),
3277   NEONMAP1(vqneg_v, aarch64_neon_sqneg, Add1ArgType),
3278   NEONMAP1(vqnegq_v, aarch64_neon_sqneg, Add1ArgType),
3279   NEONMAP1(vqrdmulh_v, aarch64_neon_sqrdmulh, Add1ArgType),
3280   NEONMAP1(vqrdmulhq_v, aarch64_neon_sqrdmulh, Add1ArgType),
3281   NEONMAP2(vqrshl_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts),
3282   NEONMAP2(vqrshlq_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts),
3283   NEONMAP2(vqshl_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl, UnsignedAlts),
3284   NEONMAP2(vqshl_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts),
3285   NEONMAP2(vqshlq_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl,UnsignedAlts),
3286   NEONMAP2(vqshlq_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts),
3287   NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
3288   NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
3289   NEONMAP2(vqsub_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts),
3290   NEONMAP2(vqsubq_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts),
3291   NEONMAP1(vraddhn_v, aarch64_neon_raddhn, Add1ArgType),
3292   NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
3293   NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
3294   NEONMAP1(vrecps_v, aarch64_neon_frecps, Add1ArgType),
3295   NEONMAP1(vrecpsq_v, aarch64_neon_frecps, Add1ArgType),
3296   NEONMAP2(vrhadd_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts),
3297   NEONMAP2(vrhaddq_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts),
3298   NEONMAP2(vrshl_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts),
3299   NEONMAP2(vrshlq_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts),
3300   NEONMAP2(vrshr_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts),
3301   NEONMAP2(vrshrq_n_v, aarch64_neon_urshl, aarch64_neon_srshl, UnsignedAlts),
3302   NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
3303   NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
3304   NEONMAP1(vrsqrts_v, aarch64_neon_frsqrts, Add1ArgType),
3305   NEONMAP1(vrsqrtsq_v, aarch64_neon_frsqrts, Add1ArgType),
3306   NEONMAP1(vrsubhn_v, aarch64_neon_rsubhn, Add1ArgType),
3307   NEONMAP1(vsha1su0q_v, aarch64_crypto_sha1su0, 0),
3308   NEONMAP1(vsha1su1q_v, aarch64_crypto_sha1su1, 0),
3309   NEONMAP1(vsha256h2q_v, aarch64_crypto_sha256h2, 0),
3310   NEONMAP1(vsha256hq_v, aarch64_crypto_sha256h, 0),
3311   NEONMAP1(vsha256su0q_v, aarch64_crypto_sha256su0, 0),
3312   NEONMAP1(vsha256su1q_v, aarch64_crypto_sha256su1, 0),
3313   NEONMAP0(vshl_n_v),
3314   NEONMAP2(vshl_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts),
3315   NEONMAP0(vshll_n_v),
3316   NEONMAP0(vshlq_n_v),
3317   NEONMAP2(vshlq_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts),
3318   NEONMAP0(vshr_n_v),
3319   NEONMAP0(vshrn_n_v),
3320   NEONMAP0(vshrq_n_v),
3321   NEONMAP0(vsubhn_v),
3322   NEONMAP0(vtst_v),
3323   NEONMAP0(vtstq_v),
3324 };
3325
3326 static const NeonIntrinsicInfo AArch64SISDIntrinsicMap[] = {
3327   NEONMAP1(vabdd_f64, aarch64_sisd_fabd, Add1ArgType),
3328   NEONMAP1(vabds_f32, aarch64_sisd_fabd, Add1ArgType),
3329   NEONMAP1(vabsd_s64, aarch64_neon_abs, Add1ArgType),
3330   NEONMAP1(vaddlv_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType),
3331   NEONMAP1(vaddlv_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType),
3332   NEONMAP1(vaddlvq_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType),
3333   NEONMAP1(vaddlvq_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType),
3334   NEONMAP1(vaddv_f32, aarch64_neon_faddv, AddRetType | Add1ArgType),
3335   NEONMAP1(vaddv_s32, aarch64_neon_saddv, AddRetType | Add1ArgType),
3336   NEONMAP1(vaddv_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType),
3337   NEONMAP1(vaddvq_f32, aarch64_neon_faddv, AddRetType | Add1ArgType),
3338   NEONMAP1(vaddvq_f64, aarch64_neon_faddv, AddRetType | Add1ArgType),
3339   NEONMAP1(vaddvq_s32, aarch64_neon_saddv, AddRetType | Add1ArgType),
3340   NEONMAP1(vaddvq_s64, aarch64_neon_saddv, AddRetType | Add1ArgType),
3341   NEONMAP1(vaddvq_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType),
3342   NEONMAP1(vaddvq_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType),
3343   NEONMAP1(vcaged_f64, aarch64_neon_facge, AddRetType | Add1ArgType),
3344   NEONMAP1(vcages_f32, aarch64_neon_facge, AddRetType | Add1ArgType),
3345   NEONMAP1(vcagtd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType),
3346   NEONMAP1(vcagts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType),
3347   NEONMAP1(vcaled_f64, aarch64_neon_facge, AddRetType | Add1ArgType),
3348   NEONMAP1(vcales_f32, aarch64_neon_facge, AddRetType | Add1ArgType),
3349   NEONMAP1(vcaltd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType),
3350   NEONMAP1(vcalts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType),
3351   NEONMAP1(vcvtad_s64_f64, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
3352   NEONMAP1(vcvtad_u64_f64, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
3353   NEONMAP1(vcvtas_s32_f32, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
3354   NEONMAP1(vcvtas_u32_f32, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
3355   NEONMAP1(vcvtd_n_f64_s64, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
3356   NEONMAP1(vcvtd_n_f64_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
3357   NEONMAP1(vcvtd_n_s64_f64, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
3358   NEONMAP1(vcvtd_n_u64_f64, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
3359   NEONMAP1(vcvtmd_s64_f64, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
3360   NEONMAP1(vcvtmd_u64_f64, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
3361   NEONMAP1(vcvtms_s32_f32, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
3362   NEONMAP1(vcvtms_u32_f32, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
3363   NEONMAP1(vcvtnd_s64_f64, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
3364   NEONMAP1(vcvtnd_u64_f64, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
3365   NEONMAP1(vcvtns_s32_f32, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
3366   NEONMAP1(vcvtns_u32_f32, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
3367   NEONMAP1(vcvtpd_s64_f64, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
3368   NEONMAP1(vcvtpd_u64_f64, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
3369   NEONMAP1(vcvtps_s32_f32, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
3370   NEONMAP1(vcvtps_u32_f32, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
3371   NEONMAP1(vcvts_n_f32_s32, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
3372   NEONMAP1(vcvts_n_f32_u32, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
3373   NEONMAP1(vcvts_n_s32_f32, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
3374   NEONMAP1(vcvts_n_u32_f32, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
3375   NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
3376   NEONMAP1(vmaxnmv_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
3377   NEONMAP1(vmaxnmvq_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
3378   NEONMAP1(vmaxnmvq_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
3379   NEONMAP1(vmaxv_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
3380   NEONMAP1(vmaxv_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType),
3381   NEONMAP1(vmaxv_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType),
3382   NEONMAP1(vmaxvq_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
3383   NEONMAP1(vmaxvq_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
3384   NEONMAP1(vmaxvq_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType),
3385   NEONMAP1(vmaxvq_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType),
3386   NEONMAP1(vminnmv_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
3387   NEONMAP1(vminnmvq_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
3388   NEONMAP1(vminnmvq_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
3389   NEONMAP1(vminv_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
3390   NEONMAP1(vminv_s32, aarch64_neon_sminv, AddRetType | Add1ArgType),
3391   NEONMAP1(vminv_u32, aarch64_neon_uminv, AddRetType | Add1ArgType),
3392   NEONMAP1(vminvq_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
3393   NEONMAP1(vminvq_f64, aarch64_neon_fminv, AddRetType | Add1ArgType),
3394   NEONMAP1(vminvq_s32, aarch64_neon_sminv, AddRetType | Add1ArgType),
3395   NEONMAP1(vminvq_u32, aarch64_neon_uminv, AddRetType | Add1ArgType),
3396   NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
3397   NEONMAP1(vmulxd_f64, aarch64_neon_fmulx, Add1ArgType),
3398   NEONMAP1(vmulxs_f32, aarch64_neon_fmulx, Add1ArgType),
3399   NEONMAP1(vpaddd_s64, aarch64_neon_uaddv, AddRetType | Add1ArgType),
3400   NEONMAP1(vpaddd_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType),
3401   NEONMAP1(vpmaxnmqd_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
3402   NEONMAP1(vpmaxnms_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
3403   NEONMAP1(vpmaxqd_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
3404   NEONMAP1(vpmaxs_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
3405   NEONMAP1(vpminnmqd_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
3406   NEONMAP1(vpminnms_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
3407   NEONMAP1(vpminqd_f64, aarch64_neon_fminv, AddRetType | Add1ArgType),
3408   NEONMAP1(vpmins_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
3409   NEONMAP1(vqabsb_s8, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors),
3410   NEONMAP1(vqabsd_s64, aarch64_neon_sqabs, Add1ArgType),
3411   NEONMAP1(vqabsh_s16, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors),
3412   NEONMAP1(vqabss_s32, aarch64_neon_sqabs, Add1ArgType),
3413   NEONMAP1(vqaddb_s8, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors),
3414   NEONMAP1(vqaddb_u8, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors),
3415   NEONMAP1(vqaddd_s64, aarch64_neon_sqadd, Add1ArgType),
3416   NEONMAP1(vqaddd_u64, aarch64_neon_uqadd, Add1ArgType),
3417   NEONMAP1(vqaddh_s16, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors),
3418   NEONMAP1(vqaddh_u16, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors),
3419   NEONMAP1(vqadds_s32, aarch64_neon_sqadd, Add1ArgType),
3420   NEONMAP1(vqadds_u32, aarch64_neon_uqadd, Add1ArgType),
3421   NEONMAP1(vqdmulhh_s16, aarch64_neon_sqdmulh, Vectorize1ArgType | Use64BitVectors),
3422   NEONMAP1(vqdmulhs_s32, aarch64_neon_sqdmulh, Add1ArgType),
3423   NEONMAP1(vqdmullh_s16, aarch64_neon_sqdmull, VectorRet | Use128BitVectors),
3424   NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
3425   NEONMAP1(vqmovnd_s64, aarch64_neon_scalar_sqxtn, AddRetType | Add1ArgType),
3426   NEONMAP1(vqmovnd_u64, aarch64_neon_scalar_uqxtn, AddRetType | Add1ArgType),
3427   NEONMAP1(vqmovnh_s16, aarch64_neon_sqxtn, VectorRet | Use64BitVectors),
3428   NEONMAP1(vqmovnh_u16, aarch64_neon_uqxtn, VectorRet | Use64BitVectors),
3429   NEONMAP1(vqmovns_s32, aarch64_neon_sqxtn, VectorRet | Use64BitVectors),
3430   NEONMAP1(vqmovns_u32, aarch64_neon_uqxtn, VectorRet | Use64BitVectors),
3431   NEONMAP1(vqmovund_s64, aarch64_neon_scalar_sqxtun, AddRetType | Add1ArgType),
3432   NEONMAP1(vqmovunh_s16, aarch64_neon_sqxtun, VectorRet | Use64BitVectors),
3433   NEONMAP1(vqmovuns_s32, aarch64_neon_sqxtun, VectorRet | Use64BitVectors),
3434   NEONMAP1(vqnegb_s8, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors),
3435   NEONMAP1(vqnegd_s64, aarch64_neon_sqneg, Add1ArgType),
3436   NEONMAP1(vqnegh_s16, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors),
3437   NEONMAP1(vqnegs_s32, aarch64_neon_sqneg, Add1ArgType),
3438   NEONMAP1(vqrdmulhh_s16, aarch64_neon_sqrdmulh, Vectorize1ArgType | Use64BitVectors),
3439   NEONMAP1(vqrdmulhs_s32, aarch64_neon_sqrdmulh, Add1ArgType),
3440   NEONMAP1(vqrshlb_s8, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors),
3441   NEONMAP1(vqrshlb_u8, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors),
3442   NEONMAP1(vqrshld_s64, aarch64_neon_sqrshl, Add1ArgType),
3443   NEONMAP1(vqrshld_u64, aarch64_neon_uqrshl, Add1ArgType),
3444   NEONMAP1(vqrshlh_s16, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors),
3445   NEONMAP1(vqrshlh_u16, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors),
3446   NEONMAP1(vqrshls_s32, aarch64_neon_sqrshl, Add1ArgType),
3447   NEONMAP1(vqrshls_u32, aarch64_neon_uqrshl, Add1ArgType),
3448   NEONMAP1(vqrshrnd_n_s64, aarch64_neon_sqrshrn, AddRetType),
3449   NEONMAP1(vqrshrnd_n_u64, aarch64_neon_uqrshrn, AddRetType),
3450   NEONMAP1(vqrshrnh_n_s16, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors),
3451   NEONMAP1(vqrshrnh_n_u16, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors),
3452   NEONMAP1(vqrshrns_n_s32, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors),
3453   NEONMAP1(vqrshrns_n_u32, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors),
3454   NEONMAP1(vqrshrund_n_s64, aarch64_neon_sqrshrun, AddRetType),
3455   NEONMAP1(vqrshrunh_n_s16, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors),
3456   NEONMAP1(vqrshruns_n_s32, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors),
3457   NEONMAP1(vqshlb_n_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
3458   NEONMAP1(vqshlb_n_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
3459   NEONMAP1(vqshlb_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
3460   NEONMAP1(vqshlb_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
3461   NEONMAP1(vqshld_s64, aarch64_neon_sqshl, Add1ArgType),
3462   NEONMAP1(vqshld_u64, aarch64_neon_uqshl, Add1ArgType),
3463   NEONMAP1(vqshlh_n_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
3464   NEONMAP1(vqshlh_n_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
3465   NEONMAP1(vqshlh_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
3466   NEONMAP1(vqshlh_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
3467   NEONMAP1(vqshls_n_s32, aarch64_neon_sqshl, Add1ArgType),
3468   NEONMAP1(vqshls_n_u32, aarch64_neon_uqshl, Add1ArgType),
3469   NEONMAP1(vqshls_s32, aarch64_neon_sqshl, Add1ArgType),
3470   NEONMAP1(vqshls_u32, aarch64_neon_uqshl, Add1ArgType),
3471   NEONMAP1(vqshlub_n_s8, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors),
3472   NEONMAP1(vqshluh_n_s16, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors),
3473   NEONMAP1(vqshlus_n_s32, aarch64_neon_sqshlu, Add1ArgType),
3474   NEONMAP1(vqshrnd_n_s64, aarch64_neon_sqshrn, AddRetType),
3475   NEONMAP1(vqshrnd_n_u64, aarch64_neon_uqshrn, AddRetType),
3476   NEONMAP1(vqshrnh_n_s16, aarch64_neon_sqshrn, VectorRet | Use64BitVectors),
3477   NEONMAP1(vqshrnh_n_u16, aarch64_neon_uqshrn, VectorRet | Use64BitVectors),
3478   NEONMAP1(vqshrns_n_s32, aarch64_neon_sqshrn, VectorRet | Use64BitVectors),
3479   NEONMAP1(vqshrns_n_u32, aarch64_neon_uqshrn, VectorRet | Use64BitVectors),
3480   NEONMAP1(vqshrund_n_s64, aarch64_neon_sqshrun, AddRetType),
3481   NEONMAP1(vqshrunh_n_s16, aarch64_neon_sqshrun, VectorRet | Use64BitVectors),
3482   NEONMAP1(vqshruns_n_s32, aarch64_neon_sqshrun, VectorRet | Use64BitVectors),
3483   NEONMAP1(vqsubb_s8, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors),
3484   NEONMAP1(vqsubb_u8, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors),
3485   NEONMAP1(vqsubd_s64, aarch64_neon_sqsub, Add1ArgType),
3486   NEONMAP1(vqsubd_u64, aarch64_neon_uqsub, Add1ArgType),
3487   NEONMAP1(vqsubh_s16, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors),
3488   NEONMAP1(vqsubh_u16, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors),
3489   NEONMAP1(vqsubs_s32, aarch64_neon_sqsub, Add1ArgType),
3490   NEONMAP1(vqsubs_u32, aarch64_neon_uqsub, Add1ArgType),
3491   NEONMAP1(vrecped_f64, aarch64_neon_frecpe, Add1ArgType),
3492   NEONMAP1(vrecpes_f32, aarch64_neon_frecpe, Add1ArgType),
3493   NEONMAP1(vrecpxd_f64, aarch64_neon_frecpx, Add1ArgType),
3494   NEONMAP1(vrecpxs_f32, aarch64_neon_frecpx, Add1ArgType),
3495   NEONMAP1(vrshld_s64, aarch64_neon_srshl, Add1ArgType),
3496   NEONMAP1(vrshld_u64, aarch64_neon_urshl, Add1ArgType),
3497   NEONMAP1(vrsqrted_f64, aarch64_neon_frsqrte, Add1ArgType),
3498   NEONMAP1(vrsqrtes_f32, aarch64_neon_frsqrte, Add1ArgType),
3499   NEONMAP1(vrsqrtsd_f64, aarch64_neon_frsqrts, Add1ArgType),
3500   NEONMAP1(vrsqrtss_f32, aarch64_neon_frsqrts, Add1ArgType),
3501   NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
3502   NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
3503   NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
3504   NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
3505   NEONMAP1(vshld_s64, aarch64_neon_sshl, Add1ArgType),
3506   NEONMAP1(vshld_u64, aarch64_neon_ushl, Add1ArgType),
3507   NEONMAP1(vslid_n_s64, aarch64_neon_vsli, Vectorize1ArgType),
3508   NEONMAP1(vslid_n_u64, aarch64_neon_vsli, Vectorize1ArgType),
3509   NEONMAP1(vsqaddb_u8, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors),
3510   NEONMAP1(vsqaddd_u64, aarch64_neon_usqadd, Add1ArgType),
3511   NEONMAP1(vsqaddh_u16, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors),
3512   NEONMAP1(vsqadds_u32, aarch64_neon_usqadd, Add1ArgType),
3513   NEONMAP1(vsrid_n_s64, aarch64_neon_vsri, Vectorize1ArgType),
3514   NEONMAP1(vsrid_n_u64, aarch64_neon_vsri, Vectorize1ArgType),
3515   NEONMAP1(vuqaddb_s8, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors),
3516   NEONMAP1(vuqaddd_s64, aarch64_neon_suqadd, Add1ArgType),
3517   NEONMAP1(vuqaddh_s16, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors),
3518   NEONMAP1(vuqadds_s32, aarch64_neon_suqadd, Add1ArgType),
3519 };
3520
3521 #undef NEONMAP0
3522 #undef NEONMAP1
3523 #undef NEONMAP2
3524
3525 static bool NEONSIMDIntrinsicsProvenSorted = false;
3526
3527 static bool AArch64SIMDIntrinsicsProvenSorted = false;
3528 static bool AArch64SISDIntrinsicsProvenSorted = false;
3529
3530
3531 static const NeonIntrinsicInfo *
3532 findNeonIntrinsicInMap(ArrayRef<NeonIntrinsicInfo> IntrinsicMap,
3533                        unsigned BuiltinID, bool &MapProvenSorted) {
3534
3535 #ifndef NDEBUG
3536   if (!MapProvenSorted) {
3537     assert(std::is_sorted(std::begin(IntrinsicMap), std::end(IntrinsicMap)));
3538     MapProvenSorted = true;
3539   }
3540 #endif
3541
3542   const NeonIntrinsicInfo *Builtin =
3543       std::lower_bound(IntrinsicMap.begin(), IntrinsicMap.end(), BuiltinID);
3544
3545   if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
3546     return Builtin;
3547
3548   return nullptr;
3549 }
3550
3551 Function *CodeGenFunction::LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
3552                                                    unsigned Modifier,
3553                                                    llvm::Type *ArgType,
3554                                                    const CallExpr *E) {
3555   int VectorSize = 0;
3556   if (Modifier & Use64BitVectors)
3557     VectorSize = 64;
3558   else if (Modifier & Use128BitVectors)
3559     VectorSize = 128;
3560
3561   // Return type.
3562   SmallVector<llvm::Type *, 3> Tys;
3563   if (Modifier & AddRetType) {
3564     llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext()));
3565     if (Modifier & VectorizeRetType)
3566       Ty = llvm::VectorType::get(
3567           Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
3568
3569     Tys.push_back(Ty);
3570   }
3571
3572   // Arguments.
3573   if (Modifier & VectorizeArgTypes) {
3574     int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
3575     ArgType = llvm::VectorType::get(ArgType, Elts);
3576   }
3577
3578   if (Modifier & (Add1ArgType | Add2ArgTypes))
3579     Tys.push_back(ArgType);
3580
3581   if (Modifier & Add2ArgTypes)
3582     Tys.push_back(ArgType);
3583
3584   if (Modifier & InventFloatType)
3585     Tys.push_back(FloatTy);
3586
3587   return CGM.getIntrinsic(IntrinsicID, Tys);
3588 }
3589
3590 static Value *EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF,
3591                                             const NeonIntrinsicInfo &SISDInfo,
3592                                             SmallVectorImpl<Value *> &Ops,
3593                                             const CallExpr *E) {
3594   unsigned BuiltinID = SISDInfo.BuiltinID;
3595   unsigned int Int = SISDInfo.LLVMIntrinsic;
3596   unsigned Modifier = SISDInfo.TypeModifier;
3597   const char *s = SISDInfo.NameHint;
3598
3599   switch (BuiltinID) {
3600   case NEON::BI__builtin_neon_vcled_s64:
3601   case NEON::BI__builtin_neon_vcled_u64:
3602   case NEON::BI__builtin_neon_vcles_f32:
3603   case NEON::BI__builtin_neon_vcled_f64:
3604   case NEON::BI__builtin_neon_vcltd_s64:
3605   case NEON::BI__builtin_neon_vcltd_u64:
3606   case NEON::BI__builtin_neon_vclts_f32:
3607   case NEON::BI__builtin_neon_vcltd_f64:
3608   case NEON::BI__builtin_neon_vcales_f32:
3609   case NEON::BI__builtin_neon_vcaled_f64:
3610   case NEON::BI__builtin_neon_vcalts_f32:
3611   case NEON::BI__builtin_neon_vcaltd_f64:
3612     // Only one direction of comparisons actually exist, cmle is actually a cmge
3613     // with swapped operands. The table gives us the right intrinsic but we
3614     // still need to do the swap.
3615     std::swap(Ops[0], Ops[1]);
3616     break;
3617   }
3618
3619   assert(Int && "Generic code assumes a valid intrinsic");
3620
3621   // Determine the type(s) of this overloaded AArch64 intrinsic.
3622   const Expr *Arg = E->getArg(0);
3623   llvm::Type *ArgTy = CGF.ConvertType(Arg->getType());
3624   Function *F = CGF.LookupNeonLLVMIntrinsic(Int, Modifier, ArgTy, E);
3625
3626   int j = 0;
3627   ConstantInt *C0 = ConstantInt::get(CGF.SizeTy, 0);
3628   for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
3629        ai != ae; ++ai, ++j) {
3630     llvm::Type *ArgTy = ai->getType();
3631     if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
3632              ArgTy->getPrimitiveSizeInBits())
3633       continue;
3634
3635     assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy());
3636     // The constant argument to an _n_ intrinsic always has Int32Ty, so truncate
3637     // it before inserting.
3638     Ops[j] =
3639         CGF.Builder.CreateTruncOrBitCast(Ops[j], ArgTy->getVectorElementType());
3640     Ops[j] =
3641         CGF.Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0);
3642   }
3643
3644   Value *Result = CGF.EmitNeonCall(F, Ops, s);
3645   llvm::Type *ResultType = CGF.ConvertType(E->getType());
3646   if (ResultType->getPrimitiveSizeInBits() <
3647       Result->getType()->getPrimitiveSizeInBits())
3648     return CGF.Builder.CreateExtractElement(Result, C0);
3649
3650   return CGF.Builder.CreateBitCast(Result, ResultType, s);
3651 }
3652
3653 Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
3654     unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic,
3655     const char *NameHint, unsigned Modifier, const CallExpr *E,
3656     SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1) {
3657   // Get the last argument, which specifies the vector type.
3658   llvm::APSInt NeonTypeConst;
3659   const Expr *Arg = E->getArg(E->getNumArgs() - 1);
3660   if (!Arg->isIntegerConstantExpr(NeonTypeConst, getContext()))
3661     return nullptr;
3662
3663   // Determine the type of this overloaded NEON intrinsic.
3664   NeonTypeFlags Type(NeonTypeConst.getZExtValue());
3665   bool Usgn = Type.isUnsigned();
3666   bool Quad = Type.isQuad();
3667
3668   llvm::VectorType *VTy = GetNeonType(this, Type);
3669   llvm::Type *Ty = VTy;
3670   if (!Ty)
3671     return nullptr;
3672
3673   auto getAlignmentValue32 = [&](Address addr) -> Value* {
3674     return Builder.getInt32(addr.getAlignment().getQuantity());
3675   };
3676
3677   unsigned Int = LLVMIntrinsic;
3678   if ((Modifier & UnsignedAlts) && !Usgn)
3679     Int = AltLLVMIntrinsic;
3680
3681   switch (BuiltinID) {
3682   default: break;
3683   case NEON::BI__builtin_neon_vabs_v:
3684   case NEON::BI__builtin_neon_vabsq_v:
3685     if (VTy->getElementType()->isFloatingPointTy())
3686       return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs");
3687     return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs");
3688   case NEON::BI__builtin_neon_vaddhn_v: {
3689     llvm::VectorType *SrcTy =
3690         llvm::VectorType::getExtendedElementVectorType(VTy);
3691
3692     // %sum = add <4 x i32> %lhs, %rhs
3693     Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
3694     Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
3695     Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn");
3696
3697     // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
3698     Constant *ShiftAmt =
3699         ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
3700     Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn");
3701
3702     // %res = trunc <4 x i32> %high to <4 x i16>
3703     return Builder.CreateTrunc(Ops[0], VTy, "vaddhn");
3704   }
3705   case NEON::BI__builtin_neon_vcale_v:
3706   case NEON::BI__builtin_neon_vcaleq_v:
3707   case NEON::BI__builtin_neon_vcalt_v:
3708   case NEON::BI__builtin_neon_vcaltq_v:
3709     std::swap(Ops[0], Ops[1]);
3710   case NEON::BI__builtin_neon_vcage_v:
3711   case NEON::BI__builtin_neon_vcageq_v:
3712   case NEON::BI__builtin_neon_vcagt_v:
3713   case NEON::BI__builtin_neon_vcagtq_v: {
3714     llvm::Type *VecFlt = llvm::VectorType::get(
3715         VTy->getScalarSizeInBits() == 32 ? FloatTy : DoubleTy,
3716         VTy->getNumElements());
3717     llvm::Type *Tys[] = { VTy, VecFlt };
3718     Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
3719     return EmitNeonCall(F, Ops, NameHint);
3720   }
3721   case NEON::BI__builtin_neon_vclz_v:
3722   case NEON::BI__builtin_neon_vclzq_v:
3723     // We generate target-independent intrinsic, which needs a second argument
3724     // for whether or not clz of zero is undefined; on ARM it isn't.
3725     Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef()));
3726     break;
3727   case NEON::BI__builtin_neon_vcvt_f32_v:
3728   case NEON::BI__builtin_neon_vcvtq_f32_v:
3729     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3730     Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, Quad));
3731     return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
3732                 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
3733   case NEON::BI__builtin_neon_vcvt_n_f32_v:
3734   case NEON::BI__builtin_neon_vcvt_n_f64_v:
3735   case NEON::BI__builtin_neon_vcvtq_n_f32_v:
3736   case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
3737     llvm::Type *Tys[2] = { GetFloatNeonType(this, Type), Ty };
3738     Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
3739     Function *F = CGM.getIntrinsic(Int, Tys);
3740     return EmitNeonCall(F, Ops, "vcvt_n");
3741   }
3742   case NEON::BI__builtin_neon_vcvt_n_s32_v:
3743   case NEON::BI__builtin_neon_vcvt_n_u32_v:
3744   case NEON::BI__builtin_neon_vcvt_n_s64_v:
3745   case NEON::BI__builtin_neon_vcvt_n_u64_v:
3746   case NEON::BI__builtin_neon_vcvtq_n_s32_v:
3747   case NEON::BI__builtin_neon_vcvtq_n_u32_v:
3748   case NEON::BI__builtin_neon_vcvtq_n_s64_v:
3749   case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
3750     llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
3751     Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
3752     return EmitNeonCall(F, Ops, "vcvt_n");
3753   }
3754   case NEON::BI__builtin_neon_vcvt_s32_v:
3755   case NEON::BI__builtin_neon_vcvt_u32_v:
3756   case NEON::BI__builtin_neon_vcvt_s64_v:
3757   case NEON::BI__builtin_neon_vcvt_u64_v:
3758   case NEON::BI__builtin_neon_vcvtq_s32_v:
3759   case NEON::BI__builtin_neon_vcvtq_u32_v:
3760   case NEON::BI__builtin_neon_vcvtq_s64_v:
3761   case NEON::BI__builtin_neon_vcvtq_u64_v: {
3762     Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type));
3763     return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt")
3764                 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt");
3765   }
3766   case NEON::BI__builtin_neon_vcvta_s32_v:
3767   case NEON::BI__builtin_neon_vcvta_s64_v:
3768   case NEON::BI__builtin_neon_vcvta_u32_v:
3769   case NEON::BI__builtin_neon_vcvta_u64_v:
3770   case NEON::BI__builtin_neon_vcvtaq_s32_v:
3771   case NEON::BI__builtin_neon_vcvtaq_s64_v:
3772   case NEON::BI__builtin_neon_vcvtaq_u32_v:
3773   case NEON::BI__builtin_neon_vcvtaq_u64_v:
3774   case NEON::BI__builtin_neon_vcvtn_s32_v:
3775   case NEON::BI__builtin_neon_vcvtn_s64_v:
3776   case NEON::BI__builtin_neon_vcvtn_u32_v:
3777   case NEON::BI__builtin_neon_vcvtn_u64_v:
3778   case NEON::BI__builtin_neon_vcvtnq_s32_v:
3779   case NEON::BI__builtin_neon_vcvtnq_s64_v:
3780   case NEON::BI__builtin_neon_vcvtnq_u32_v:
3781   case NEON::BI__builtin_neon_vcvtnq_u64_v:
3782   case NEON::BI__builtin_neon_vcvtp_s32_v:
3783   case NEON::BI__builtin_neon_vcvtp_s64_v:
3784   case NEON::BI__builtin_neon_vcvtp_u32_v:
3785   case NEON::BI__builtin_neon_vcvtp_u64_v:
3786   case NEON::BI__builtin_neon_vcvtpq_s32_v:
3787   case NEON::BI__builtin_neon_vcvtpq_s64_v:
3788   case NEON::BI__builtin_neon_vcvtpq_u32_v:
3789   case NEON::BI__builtin_neon_vcvtpq_u64_v:
3790   case NEON::BI__builtin_neon_vcvtm_s32_v:
3791   case NEON::BI__builtin_neon_vcvtm_s64_v:
3792   case NEON::BI__builtin_neon_vcvtm_u32_v:
3793   case NEON::BI__builtin_neon_vcvtm_u64_v:
3794   case NEON::BI__builtin_neon_vcvtmq_s32_v:
3795   case NEON::BI__builtin_neon_vcvtmq_s64_v:
3796   case NEON::BI__builtin_neon_vcvtmq_u32_v:
3797   case NEON::BI__builtin_neon_vcvtmq_u64_v: {
3798     llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
3799     return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint);
3800   }
3801   case NEON::BI__builtin_neon_vext_v:
3802   case NEON::BI__builtin_neon_vextq_v: {
3803     int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
3804     SmallVector<uint32_t, 16> Indices;
3805     for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
3806       Indices.push_back(i+CV);
3807
3808     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3809     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3810     return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices, "vext");
3811   }
3812   case NEON::BI__builtin_neon_vfma_v:
3813   case NEON::BI__builtin_neon_vfmaq_v: {
3814     Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
3815     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3816     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3817     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
3818
3819     // NEON intrinsic puts accumulator first, unlike the LLVM fma.
3820     return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
3821   }
3822   case NEON::BI__builtin_neon_vld1_v:
3823   case NEON::BI__builtin_neon_vld1q_v: {
3824     llvm::Type *Tys[] = {Ty, Int8PtrTy};
3825     Ops.push_back(getAlignmentValue32(PtrOp0));
3826     return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vld1");
3827   }
3828   case NEON::BI__builtin_neon_vld2_v:
3829   case NEON::BI__builtin_neon_vld2q_v:
3830   case NEON::BI__builtin_neon_vld3_v:
3831   case NEON::BI__builtin_neon_vld3q_v:
3832   case NEON::BI__builtin_neon_vld4_v:
3833   case NEON::BI__builtin_neon_vld4q_v: {
3834     llvm::Type *Tys[] = {Ty, Int8PtrTy};
3835     Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
3836     Value *Align = getAlignmentValue32(PtrOp1);
3837     Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint);
3838     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
3839     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3840     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
3841   }
3842   case NEON::BI__builtin_neon_vld1_dup_v:
3843   case NEON::BI__builtin_neon_vld1q_dup_v: {
3844     Value *V = UndefValue::get(Ty);
3845     Ty = llvm::PointerType::getUnqual(VTy->getElementType());
3846     PtrOp0 = Builder.CreateBitCast(PtrOp0, Ty);
3847     LoadInst *Ld = Builder.CreateLoad(PtrOp0);
3848     llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
3849     Ops[0] = Builder.CreateInsertElement(V, Ld, CI);
3850     return EmitNeonSplat(Ops[0], CI);
3851   }
3852   case NEON::BI__builtin_neon_vld2_lane_v:
3853   case NEON::BI__builtin_neon_vld2q_lane_v:
3854   case NEON::BI__builtin_neon_vld3_lane_v:
3855   case NEON::BI__builtin_neon_vld3q_lane_v:
3856   case NEON::BI__builtin_neon_vld4_lane_v:
3857   case NEON::BI__builtin_neon_vld4q_lane_v: {
3858     llvm::Type *Tys[] = {Ty, Int8PtrTy};
3859     Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
3860     for (unsigned I = 2; I < Ops.size() - 1; ++I)
3861       Ops[I] = Builder.CreateBitCast(Ops[I], Ty);
3862     Ops.push_back(getAlignmentValue32(PtrOp1));
3863     Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint);
3864     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
3865     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3866     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
3867   }
3868   case NEON::BI__builtin_neon_vmovl_v: {
3869     llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
3870     Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
3871     if (Usgn)
3872       return Builder.CreateZExt(Ops[0], Ty, "vmovl");
3873     return Builder.CreateSExt(Ops[0], Ty, "vmovl");
3874   }
3875   case NEON::BI__builtin_neon_vmovn_v: {
3876     llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy);
3877     Ops[0] = Builder.CreateBitCast(Ops[0], QTy);
3878     return Builder.CreateTrunc(Ops[0], Ty, "vmovn");
3879   }
3880   case NEON::BI__builtin_neon_vmull_v:
3881     // FIXME: the integer vmull operations could be emitted in terms of pure
3882     // LLVM IR (2 exts followed by a mul). Unfortunately LLVM has a habit of
3883     // hoisting the exts outside loops. Until global ISel comes along that can
3884     // see through such movement this leads to bad CodeGen. So we need an
3885     // intrinsic for now.
3886     Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
3887     Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int;
3888     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
3889   case NEON::BI__builtin_neon_vpadal_v:
3890   case NEON::BI__builtin_neon_vpadalq_v: {
3891     // The source operand type has twice as many elements of half the size.
3892     unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
3893     llvm::Type *EltTy =
3894       llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
3895     llvm::Type *NarrowTy =
3896       llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
3897     llvm::Type *Tys[2] = { Ty, NarrowTy };
3898     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
3899   }
3900   case NEON::BI__builtin_neon_vpaddl_v:
3901   case NEON::BI__builtin_neon_vpaddlq_v: {
3902     // The source operand type has twice as many elements of half the size.
3903     unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
3904     llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
3905     llvm::Type *NarrowTy =
3906       llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
3907     llvm::Type *Tys[2] = { Ty, NarrowTy };
3908     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl");
3909   }
3910   case NEON::BI__builtin_neon_vqdmlal_v:
3911   case NEON::BI__builtin_neon_vqdmlsl_v: {
3912     SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end());
3913     Ops[1] =
3914         EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), MulOps, "vqdmlal");
3915     Ops.resize(2);
3916     return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint);
3917   }
3918   case NEON::BI__builtin_neon_vqshl_n_v:
3919   case NEON::BI__builtin_neon_vqshlq_n_v:
3920     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n",
3921                         1, false);
3922   case NEON::BI__builtin_neon_vqshlu_n_v:
3923   case NEON::BI__builtin_neon_vqshluq_n_v:
3924     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n",
3925                         1, false);
3926   case NEON::BI__builtin_neon_vrecpe_v:
3927   case NEON::BI__builtin_neon_vrecpeq_v:
3928   case NEON::BI__builtin_neon_vrsqrte_v:
3929   case NEON::BI__builtin_neon_vrsqrteq_v:
3930     Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
3931     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint);
3932
3933   case NEON::BI__builtin_neon_vrshr_n_v:
3934   case NEON::BI__builtin_neon_vrshrq_n_v:
3935     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n",
3936                         1, true);
3937   case NEON::BI__builtin_neon_vshl_n_v:
3938   case NEON::BI__builtin_neon_vshlq_n_v:
3939     Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false);
3940     return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1],
3941                              "vshl_n");
3942   case NEON::BI__builtin_neon_vshll_n_v: {
3943     llvm::Type *SrcTy = llvm::VectorType::getTruncatedElementVectorType(VTy);
3944     Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
3945     if (Usgn)
3946       Ops[0] = Builder.CreateZExt(Ops[0], VTy);
3947     else
3948       Ops[0] = Builder.CreateSExt(Ops[0], VTy);
3949     Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false);
3950     return Builder.CreateShl(Ops[0], Ops[1], "vshll_n");
3951   }
3952   case NEON::BI__builtin_neon_vshrn_n_v: {
3953     llvm::Type *SrcTy = llvm::VectorType::getExtendedElementVectorType(VTy);
3954     Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
3955     Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false);
3956     if (Usgn)
3957       Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]);
3958     else
3959       Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]);
3960     return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n");
3961   }
3962   case NEON::BI__builtin_neon_vshr_n_v:
3963   case NEON::BI__builtin_neon_vshrq_n_v:
3964     return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n");
3965   case NEON::BI__builtin_neon_vst1_v:
3966   case NEON::BI__builtin_neon_vst1q_v:
3967   case NEON::BI__builtin_neon_vst2_v:
3968   case NEON::BI__builtin_neon_vst2q_v:
3969   case NEON::BI__builtin_neon_vst3_v:
3970   case NEON::BI__builtin_neon_vst3q_v:
3971   case NEON::BI__builtin_neon_vst4_v:
3972   case NEON::BI__builtin_neon_vst4q_v:
3973   case NEON::BI__builtin_neon_vst2_lane_v:
3974   case NEON::BI__builtin_neon_vst2q_lane_v:
3975   case NEON::BI__builtin_neon_vst3_lane_v:
3976   case NEON::BI__builtin_neon_vst3q_lane_v:
3977   case NEON::BI__builtin_neon_vst4_lane_v:
3978   case NEON::BI__builtin_neon_vst4q_lane_v: {
3979     llvm::Type *Tys[] = {Int8PtrTy, Ty};
3980     Ops.push_back(getAlignmentValue32(PtrOp0));
3981     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "");
3982   }
3983   case NEON::BI__builtin_neon_vsubhn_v: {
3984     llvm::VectorType *SrcTy =
3985         llvm::VectorType::getExtendedElementVectorType(VTy);
3986
3987     // %sum = add <4 x i32> %lhs, %rhs
3988     Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
3989     Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
3990     Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn");
3991
3992     // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
3993     Constant *ShiftAmt =
3994         ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
3995     Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn");
3996
3997     // %res = trunc <4 x i32> %high to <4 x i16>
3998     return Builder.CreateTrunc(Ops[0], VTy, "vsubhn");
3999   }
4000   case NEON::BI__builtin_neon_vtrn_v:
4001   case NEON::BI__builtin_neon_vtrnq_v: {
4002     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
4003     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4004     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
4005     Value *SV = nullptr;
4006
4007     for (unsigned vi = 0; vi != 2; ++vi) {
4008       SmallVector<uint32_t, 16> Indices;
4009       for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
4010         Indices.push_back(i+vi);
4011         Indices.push_back(i+e+vi);
4012       }
4013       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
4014       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn");
4015       SV = Builder.CreateDefaultAlignedStore(SV, Addr);
4016     }
4017     return SV;
4018   }
4019   case NEON::BI__builtin_neon_vtst_v:
4020   case NEON::BI__builtin_neon_vtstq_v: {
4021     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4022     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4023     Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
4024     Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
4025                                 ConstantAggregateZero::get(Ty));
4026     return Builder.CreateSExt(Ops[0], Ty, "vtst");
4027   }
4028   case NEON::BI__builtin_neon_vuzp_v:
4029   case NEON::BI__builtin_neon_vuzpq_v: {
4030     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
4031     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4032     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
4033     Value *SV = nullptr;
4034
4035     for (unsigned vi = 0; vi != 2; ++vi) {
4036       SmallVector<uint32_t, 16> Indices;
4037       for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
4038         Indices.push_back(2*i+vi);
4039
4040       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
4041       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp");
4042       SV = Builder.CreateDefaultAlignedStore(SV, Addr);
4043     }
4044     return SV;
4045   }
4046   case NEON::BI__builtin_neon_vzip_v:
4047   case NEON::BI__builtin_neon_vzipq_v: {
4048     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
4049     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4050     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
4051     Value *SV = nullptr;
4052
4053     for (unsigned vi = 0; vi != 2; ++vi) {
4054       SmallVector<uint32_t, 16> Indices;
4055       for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
4056         Indices.push_back((i + vi*e) >> 1);
4057         Indices.push_back(((i + vi*e) >> 1)+e);
4058       }
4059       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
4060       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip");
4061       SV = Builder.CreateDefaultAlignedStore(SV, Addr);
4062     }
4063     return SV;
4064   }
4065   }
4066
4067   assert(Int && "Expected valid intrinsic number");
4068
4069   // Determine the type(s) of this overloaded AArch64 intrinsic.
4070   Function *F = LookupNeonLLVMIntrinsic(Int, Modifier, Ty, E);
4071
4072   Value *Result = EmitNeonCall(F, Ops, NameHint);
4073   llvm::Type *ResultType = ConvertType(E->getType());
4074   // AArch64 intrinsic one-element vector type cast to
4075   // scalar type expected by the builtin
4076   return Builder.CreateBitCast(Result, ResultType, NameHint);
4077 }
4078
4079 Value *CodeGenFunction::EmitAArch64CompareBuiltinExpr(
4080     Value *Op, llvm::Type *Ty, const CmpInst::Predicate Fp,
4081     const CmpInst::Predicate Ip, const Twine &Name) {
4082   llvm::Type *OTy = Op->getType();
4083
4084   // FIXME: this is utterly horrific. We should not be looking at previous
4085   // codegen context to find out what needs doing. Unfortunately TableGen
4086   // currently gives us exactly the same calls for vceqz_f32 and vceqz_s32
4087   // (etc).
4088   if (BitCastInst *BI = dyn_cast<BitCastInst>(Op))
4089     OTy = BI->getOperand(0)->getType();
4090
4091   Op = Builder.CreateBitCast(Op, OTy);
4092   if (OTy->getScalarType()->isFloatingPointTy()) {
4093     Op = Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
4094   } else {
4095     Op = Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy));
4096   }
4097   return Builder.CreateSExt(Op, Ty, Name);
4098 }
4099
4100 static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops,
4101                                  Value *ExtOp, Value *IndexOp,
4102                                  llvm::Type *ResTy, unsigned IntID,
4103                                  const char *Name) {
4104   SmallVector<Value *, 2> TblOps;
4105   if (ExtOp)
4106     TblOps.push_back(ExtOp);
4107
4108   // Build a vector containing sequential number like (0, 1, 2, ..., 15)
4109   SmallVector<uint32_t, 16> Indices;
4110   llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType());
4111   for (unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
4112     Indices.push_back(2*i);
4113     Indices.push_back(2*i+1);
4114   }
4115
4116   int PairPos = 0, End = Ops.size() - 1;
4117   while (PairPos < End) {
4118     TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
4119                                                      Ops[PairPos+1], Indices,
4120                                                      Name));
4121     PairPos += 2;
4122   }
4123
4124   // If there's an odd number of 64-bit lookup table, fill the high 64-bit
4125   // of the 128-bit lookup table with zero.
4126   if (PairPos == End) {
4127     Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
4128     TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
4129                                                      ZeroTbl, Indices, Name));
4130   }
4131
4132   Function *TblF;
4133   TblOps.push_back(IndexOp);
4134   TblF = CGF.CGM.getIntrinsic(IntID, ResTy);
4135
4136   return CGF.EmitNeonCall(TblF, TblOps, Name);
4137 }
4138
4139 Value *CodeGenFunction::GetValueForARMHint(unsigned BuiltinID) {
4140   unsigned Value;
4141   switch (BuiltinID) {
4142   default:
4143     return nullptr;
4144   case ARM::BI__builtin_arm_nop:
4145     Value = 0;
4146     break;
4147   case ARM::BI__builtin_arm_yield:
4148   case ARM::BI__yield:
4149     Value = 1;
4150     break;
4151   case ARM::BI__builtin_arm_wfe:
4152   case ARM::BI__wfe:
4153     Value = 2;
4154     break;
4155   case ARM::BI__builtin_arm_wfi:
4156   case ARM::BI__wfi:
4157     Value = 3;
4158     break;
4159   case ARM::BI__builtin_arm_sev:
4160   case ARM::BI__sev:
4161     Value = 4;
4162     break;
4163   case ARM::BI__builtin_arm_sevl:
4164   case ARM::BI__sevl:
4165     Value = 5;
4166     break;
4167   }
4168
4169   return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint),
4170                             llvm::ConstantInt::get(Int32Ty, Value));
4171 }
4172
4173 // Generates the IR for the read/write special register builtin,
4174 // ValueType is the type of the value that is to be written or read,
4175 // RegisterType is the type of the register being written to or read from.
4176 static Value *EmitSpecialRegisterBuiltin(CodeGenFunction &CGF,
4177                                          const CallExpr *E,
4178                                          llvm::Type *RegisterType,
4179                                          llvm::Type *ValueType,
4180                                          bool IsRead,
4181                                          StringRef SysReg = "") {
4182   // write and register intrinsics only support 32 and 64 bit operations.
4183   assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64))
4184           && "Unsupported size for register.");
4185
4186   CodeGen::CGBuilderTy &Builder = CGF.Builder;
4187   CodeGen::CodeGenModule &CGM = CGF.CGM;
4188   LLVMContext &Context = CGM.getLLVMContext();
4189
4190   if (SysReg.empty()) {
4191     const Expr *SysRegStrExpr = E->getArg(0)->IgnoreParenCasts();
4192     SysReg = cast<clang::StringLiteral>(SysRegStrExpr)->getString();
4193   }
4194
4195   llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) };
4196   llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
4197   llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
4198
4199   llvm::Type *Types[] = { RegisterType };
4200
4201   bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
4202   assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
4203             && "Can't fit 64-bit value in 32-bit register");
4204
4205   if (IsRead) {
4206     llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types);
4207     llvm::Value *Call = Builder.CreateCall(F, Metadata);
4208
4209     if (MixedTypes)
4210       // Read into 64 bit register and then truncate result to 32 bit.
4211       return Builder.CreateTrunc(Call, ValueType);
4212
4213     if (ValueType->isPointerTy())
4214       // Have i32/i64 result (Call) but want to return a VoidPtrTy (i8*).
4215       return Builder.CreateIntToPtr(Call, ValueType);
4216
4217     return Call;
4218   }
4219
4220   llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types);
4221   llvm::Value *ArgValue = CGF.EmitScalarExpr(E->getArg(1));
4222   if (MixedTypes) {
4223     // Extend 32 bit write value to 64 bit to pass to write.
4224     ArgValue = Builder.CreateZExt(ArgValue, RegisterType);
4225     return Builder.CreateCall(F, { Metadata, ArgValue });
4226   }
4227
4228   if (ValueType->isPointerTy()) {
4229     // Have VoidPtrTy ArgValue but want to return an i32/i64.
4230     ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType);
4231     return Builder.CreateCall(F, { Metadata, ArgValue });
4232   }
4233
4234   return Builder.CreateCall(F, { Metadata, ArgValue });
4235 }
4236
4237 /// Return true if BuiltinID is an overloaded Neon intrinsic with an extra
4238 /// argument that specifies the vector type.
4239 static bool HasExtraNeonArgument(unsigned BuiltinID) {
4240   switch (BuiltinID) {
4241   default: break;
4242   case NEON::BI__builtin_neon_vget_lane_i8:
4243   case NEON::BI__builtin_neon_vget_lane_i16:
4244   case NEON::BI__builtin_neon_vget_lane_i32:
4245   case NEON::BI__builtin_neon_vget_lane_i64:
4246   case NEON::BI__builtin_neon_vget_lane_f32:
4247   case NEON::BI__builtin_neon_vgetq_lane_i8:
4248   case NEON::BI__builtin_neon_vgetq_lane_i16:
4249   case NEON::BI__builtin_neon_vgetq_lane_i32:
4250   case NEON::BI__builtin_neon_vgetq_lane_i64:
4251   case NEON::BI__builtin_neon_vgetq_lane_f32:
4252   case NEON::BI__builtin_neon_vset_lane_i8:
4253   case NEON::BI__builtin_neon_vset_lane_i16:
4254   case NEON::BI__builtin_neon_vset_lane_i32:
4255   case NEON::BI__builtin_neon_vset_lane_i64:
4256   case NEON::BI__builtin_neon_vset_lane_f32:
4257   case NEON::BI__builtin_neon_vsetq_lane_i8:
4258   case NEON::BI__builtin_neon_vsetq_lane_i16:
4259   case NEON::BI__builtin_neon_vsetq_lane_i32:
4260   case NEON::BI__builtin_neon_vsetq_lane_i64:
4261   case NEON::BI__builtin_neon_vsetq_lane_f32:
4262   case NEON::BI__builtin_neon_vsha1h_u32:
4263   case NEON::BI__builtin_neon_vsha1cq_u32:
4264   case NEON::BI__builtin_neon_vsha1pq_u32:
4265   case NEON::BI__builtin_neon_vsha1mq_u32:
4266   case ARM::BI_MoveToCoprocessor:
4267   case ARM::BI_MoveToCoprocessor2:
4268     return false;
4269   }
4270   return true;
4271 }
4272
4273 Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
4274                                            const CallExpr *E) {
4275   if (auto Hint = GetValueForARMHint(BuiltinID))
4276     return Hint;
4277
4278   if (BuiltinID == ARM::BI__emit) {
4279     bool IsThumb = getTarget().getTriple().getArch() == llvm::Triple::thumb;
4280     llvm::FunctionType *FTy =
4281         llvm::FunctionType::get(VoidTy, /*Variadic=*/false);
4282
4283     APSInt Value;
4284     if (!E->getArg(0)->EvaluateAsInt(Value, CGM.getContext()))
4285       llvm_unreachable("Sema will ensure that the parameter is constant");
4286
4287     uint64_t ZExtValue = Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
4288
4289     llvm::InlineAsm *Emit =
4290         IsThumb ? InlineAsm::get(FTy, ".inst.n 0x" + utohexstr(ZExtValue), "",
4291                                  /*SideEffects=*/true)
4292                 : InlineAsm::get(FTy, ".inst 0x" + utohexstr(ZExtValue), "",
4293                                  /*SideEffects=*/true);
4294
4295     return Builder.CreateCall(Emit);
4296   }
4297
4298   if (BuiltinID == ARM::BI__builtin_arm_dbg) {
4299     Value *Option = EmitScalarExpr(E->getArg(0));
4300     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_dbg), Option);
4301   }
4302
4303   if (BuiltinID == ARM::BI__builtin_arm_prefetch) {
4304     Value *Address = EmitScalarExpr(E->getArg(0));
4305     Value *RW      = EmitScalarExpr(E->getArg(1));
4306     Value *IsData  = EmitScalarExpr(E->getArg(2));
4307
4308     // Locality is not supported on ARM target
4309     Value *Locality = llvm::ConstantInt::get(Int32Ty, 3);
4310
4311     Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
4312     return Builder.CreateCall(F, {Address, RW, Locality, IsData});
4313   }
4314
4315   if (BuiltinID == ARM::BI__builtin_arm_rbit) {
4316     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_rbit),
4317                                                EmitScalarExpr(E->getArg(0)),
4318                               "rbit");
4319   }
4320
4321   if (BuiltinID == ARM::BI__clear_cache) {
4322     assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments");
4323     const FunctionDecl *FD = E->getDirectCallee();
4324     Value *Ops[2];
4325     for (unsigned i = 0; i < 2; i++)
4326       Ops[i] = EmitScalarExpr(E->getArg(i));
4327     llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
4328     llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
4329     StringRef Name = FD->getName();
4330     return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
4331   }
4332
4333   if (BuiltinID == ARM::BI__builtin_arm_mcrr ||
4334       BuiltinID == ARM::BI__builtin_arm_mcrr2) {
4335     Function *F;
4336
4337     switch (BuiltinID) {
4338     default: llvm_unreachable("unexpected builtin");
4339     case ARM::BI__builtin_arm_mcrr:
4340       F = CGM.getIntrinsic(Intrinsic::arm_mcrr);
4341       break;
4342     case ARM::BI__builtin_arm_mcrr2:
4343       F = CGM.getIntrinsic(Intrinsic::arm_mcrr2);
4344       break;
4345     }
4346
4347     // MCRR{2} instruction has 5 operands but
4348     // the intrinsic has 4 because Rt and Rt2
4349     // are represented as a single unsigned 64
4350     // bit integer in the intrinsic definition
4351     // but internally it's represented as 2 32
4352     // bit integers.
4353
4354     Value *Coproc = EmitScalarExpr(E->getArg(0));
4355     Value *Opc1 = EmitScalarExpr(E->getArg(1));
4356     Value *RtAndRt2 = EmitScalarExpr(E->getArg(2));
4357     Value *CRm = EmitScalarExpr(E->getArg(3));
4358
4359     Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
4360     Value *Rt = Builder.CreateTruncOrBitCast(RtAndRt2, Int32Ty);
4361     Value *Rt2 = Builder.CreateLShr(RtAndRt2, C1);
4362     Rt2 = Builder.CreateTruncOrBitCast(Rt2, Int32Ty);
4363
4364     return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
4365   }
4366
4367   if (BuiltinID == ARM::BI__builtin_arm_mrrc ||
4368       BuiltinID == ARM::BI__builtin_arm_mrrc2) {
4369     Function *F;
4370
4371     switch (BuiltinID) {
4372     default: llvm_unreachable("unexpected builtin");
4373     case ARM::BI__builtin_arm_mrrc:
4374       F = CGM.getIntrinsic(Intrinsic::arm_mrrc);
4375       break;
4376     case ARM::BI__builtin_arm_mrrc2:
4377       F = CGM.getIntrinsic(Intrinsic::arm_mrrc2);
4378       break;
4379     }
4380
4381     Value *Coproc = EmitScalarExpr(E->getArg(0));
4382     Value *Opc1 = EmitScalarExpr(E->getArg(1));
4383     Value *CRm  = EmitScalarExpr(E->getArg(2));
4384     Value *RtAndRt2 = Builder.CreateCall(F, {Coproc, Opc1, CRm});
4385
4386     // Returns an unsigned 64 bit integer, represented
4387     // as two 32 bit integers.
4388
4389     Value *Rt = Builder.CreateExtractValue(RtAndRt2, 1);
4390     Value *Rt1 = Builder.CreateExtractValue(RtAndRt2, 0);
4391     Rt = Builder.CreateZExt(Rt, Int64Ty);
4392     Rt1 = Builder.CreateZExt(Rt1, Int64Ty);
4393
4394     Value *ShiftCast = llvm::ConstantInt::get(Int64Ty, 32);
4395     RtAndRt2 = Builder.CreateShl(Rt, ShiftCast, "shl", true);
4396     RtAndRt2 = Builder.CreateOr(RtAndRt2, Rt1);
4397
4398     return Builder.CreateBitCast(RtAndRt2, ConvertType(E->getType()));
4399   }
4400
4401   if (BuiltinID == ARM::BI__builtin_arm_ldrexd ||
4402       ((BuiltinID == ARM::BI__builtin_arm_ldrex ||
4403         BuiltinID == ARM::BI__builtin_arm_ldaex) &&
4404        getContext().getTypeSize(E->getType()) == 64) ||
4405       BuiltinID == ARM::BI__ldrexd) {
4406     Function *F;
4407
4408     switch (BuiltinID) {
4409     default: llvm_unreachable("unexpected builtin");
4410     case ARM::BI__builtin_arm_ldaex:
4411       F = CGM.getIntrinsic(Intrinsic::arm_ldaexd);
4412       break;
4413     case ARM::BI__builtin_arm_ldrexd:
4414     case ARM::BI__builtin_arm_ldrex:
4415     case ARM::BI__ldrexd:
4416       F = CGM.getIntrinsic(Intrinsic::arm_ldrexd);
4417       break;
4418     }
4419
4420     Value *LdPtr = EmitScalarExpr(E->getArg(0));
4421     Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy),
4422                                     "ldrexd");
4423
4424     Value *Val0 = Builder.CreateExtractValue(Val, 1);
4425     Value *Val1 = Builder.CreateExtractValue(Val, 0);
4426     Val0 = Builder.CreateZExt(Val0, Int64Ty);
4427     Val1 = Builder.CreateZExt(Val1, Int64Ty);
4428
4429     Value *ShiftCst = llvm::ConstantInt::get(Int64Ty, 32);
4430     Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */);
4431     Val = Builder.CreateOr(Val, Val1);
4432     return Builder.CreateBitCast(Val, ConvertType(E->getType()));
4433   }
4434
4435   if (BuiltinID == ARM::BI__builtin_arm_ldrex ||
4436       BuiltinID == ARM::BI__builtin_arm_ldaex) {
4437     Value *LoadAddr = EmitScalarExpr(E->getArg(0));
4438
4439     QualType Ty = E->getType();
4440     llvm::Type *RealResTy = ConvertType(Ty);
4441     llvm::Type *PtrTy = llvm::IntegerType::get(
4442         getLLVMContext(), getContext().getTypeSize(Ty))->getPointerTo();
4443     LoadAddr = Builder.CreateBitCast(LoadAddr, PtrTy);
4444
4445     Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_ldaex
4446                                        ? Intrinsic::arm_ldaex
4447                                        : Intrinsic::arm_ldrex,
4448                                    PtrTy);
4449     Value *Val = Builder.CreateCall(F, LoadAddr, "ldrex");
4450
4451     if (RealResTy->isPointerTy())
4452       return Builder.CreateIntToPtr(Val, RealResTy);
4453     else {
4454       llvm::Type *IntResTy = llvm::IntegerType::get(
4455           getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
4456       Val = Builder.CreateTruncOrBitCast(Val, IntResTy);
4457       return Builder.CreateBitCast(Val, RealResTy);
4458     }
4459   }
4460
4461   if (BuiltinID == ARM::BI__builtin_arm_strexd ||
4462       ((BuiltinID == ARM::BI__builtin_arm_stlex ||
4463         BuiltinID == ARM::BI__builtin_arm_strex) &&
4464        getContext().getTypeSize(E->getArg(0)->getType()) == 64)) {
4465     Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex
4466                                        ? Intrinsic::arm_stlexd
4467                                        : Intrinsic::arm_strexd);
4468     llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, nullptr);
4469
4470     Address Tmp = CreateMemTemp(E->getArg(0)->getType());
4471     Value *Val = EmitScalarExpr(E->getArg(0));
4472     Builder.CreateStore(Val, Tmp);
4473
4474     Address LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy));
4475     Val = Builder.CreateLoad(LdPtr);
4476
4477     Value *Arg0 = Builder.CreateExtractValue(Val, 0);
4478     Value *Arg1 = Builder.CreateExtractValue(Val, 1);
4479     Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), Int8PtrTy);
4480     return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "strexd");
4481   }
4482
4483   if (BuiltinID == ARM::BI__builtin_arm_strex ||
4484       BuiltinID == ARM::BI__builtin_arm_stlex) {
4485     Value *StoreVal = EmitScalarExpr(E->getArg(0));
4486     Value *StoreAddr = EmitScalarExpr(E->getArg(1));
4487
4488     QualType Ty = E->getArg(0)->getType();
4489     llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(),
4490                                                  getContext().getTypeSize(Ty));
4491     StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo());
4492
4493     if (StoreVal->getType()->isPointerTy())
4494       StoreVal = Builder.CreatePtrToInt(StoreVal, Int32Ty);
4495     else {
4496       llvm::Type *IntTy = llvm::IntegerType::get(
4497           getLLVMContext(),
4498           CGM.getDataLayout().getTypeSizeInBits(StoreVal->getType()));
4499       StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
4500       StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty);
4501     }
4502
4503     Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex
4504                                        ? Intrinsic::arm_stlex
4505                                        : Intrinsic::arm_strex,
4506                                    StoreAddr->getType());
4507     return Builder.CreateCall(F, {StoreVal, StoreAddr}, "strex");
4508   }
4509
4510   switch (BuiltinID) {
4511   case ARM::BI__iso_volatile_load8:
4512   case ARM::BI__iso_volatile_load16:
4513   case ARM::BI__iso_volatile_load32:
4514   case ARM::BI__iso_volatile_load64: {
4515     Value *Ptr = EmitScalarExpr(E->getArg(0));
4516     QualType ElTy = E->getArg(0)->getType()->getPointeeType();
4517     CharUnits LoadSize = getContext().getTypeSizeInChars(ElTy);
4518     llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
4519                                              LoadSize.getQuantity() * 8);
4520     Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
4521     llvm::LoadInst *Load =
4522       Builder.CreateAlignedLoad(Ptr, LoadSize);
4523     Load->setVolatile(true);
4524     return Load;
4525   }
4526   case ARM::BI__iso_volatile_store8:
4527   case ARM::BI__iso_volatile_store16:
4528   case ARM::BI__iso_volatile_store32:
4529   case ARM::BI__iso_volatile_store64: {
4530     Value *Ptr = EmitScalarExpr(E->getArg(0));
4531     Value *Value = EmitScalarExpr(E->getArg(1));
4532     QualType ElTy = E->getArg(0)->getType()->getPointeeType();
4533     CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy);
4534     llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
4535                                              StoreSize.getQuantity() * 8);
4536     Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
4537     llvm::StoreInst *Store =
4538       Builder.CreateAlignedStore(Value, Ptr,
4539                                  StoreSize);
4540     Store->setVolatile(true);
4541     return Store;
4542   }
4543   }
4544
4545   if (BuiltinID == ARM::BI__builtin_arm_clrex) {
4546     Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex);
4547     return Builder.CreateCall(F);
4548   }
4549
4550   // CRC32
4551   Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
4552   switch (BuiltinID) {
4553   case ARM::BI__builtin_arm_crc32b:
4554     CRCIntrinsicID = Intrinsic::arm_crc32b; break;
4555   case ARM::BI__builtin_arm_crc32cb:
4556     CRCIntrinsicID = Intrinsic::arm_crc32cb; break;
4557   case ARM::BI__builtin_arm_crc32h:
4558     CRCIntrinsicID = Intrinsic::arm_crc32h; break;
4559   case ARM::BI__builtin_arm_crc32ch:
4560     CRCIntrinsicID = Intrinsic::arm_crc32ch; break;
4561   case ARM::BI__builtin_arm_crc32w:
4562   case ARM::BI__builtin_arm_crc32d:
4563     CRCIntrinsicID = Intrinsic::arm_crc32w; break;
4564   case ARM::BI__builtin_arm_crc32cw:
4565   case ARM::BI__builtin_arm_crc32cd:
4566     CRCIntrinsicID = Intrinsic::arm_crc32cw; break;
4567   }
4568
4569   if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
4570     Value *Arg0 = EmitScalarExpr(E->getArg(0));
4571     Value *Arg1 = EmitScalarExpr(E->getArg(1));
4572
4573     // crc32{c,}d intrinsics are implemnted as two calls to crc32{c,}w
4574     // intrinsics, hence we need different codegen for these cases.
4575     if (BuiltinID == ARM::BI__builtin_arm_crc32d ||
4576         BuiltinID == ARM::BI__builtin_arm_crc32cd) {
4577       Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
4578       Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1, Int32Ty);
4579       Value *Arg1b = Builder.CreateLShr(Arg1, C1);
4580       Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty);
4581
4582       Function *F = CGM.getIntrinsic(CRCIntrinsicID);
4583       Value *Res = Builder.CreateCall(F, {Arg0, Arg1a});
4584       return Builder.CreateCall(F, {Res, Arg1b});
4585     } else {
4586       Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty);
4587
4588       Function *F = CGM.getIntrinsic(CRCIntrinsicID);
4589       return Builder.CreateCall(F, {Arg0, Arg1});
4590     }
4591   }
4592
4593   if (BuiltinID == ARM::BI__builtin_arm_rsr ||
4594       BuiltinID == ARM::BI__builtin_arm_rsr64 ||
4595       BuiltinID == ARM::BI__builtin_arm_rsrp ||
4596       BuiltinID == ARM::BI__builtin_arm_wsr ||
4597       BuiltinID == ARM::BI__builtin_arm_wsr64 ||
4598       BuiltinID == ARM::BI__builtin_arm_wsrp) {
4599
4600     bool IsRead = BuiltinID == ARM::BI__builtin_arm_rsr ||
4601                   BuiltinID == ARM::BI__builtin_arm_rsr64 ||
4602                   BuiltinID == ARM::BI__builtin_arm_rsrp;
4603
4604     bool IsPointerBuiltin = BuiltinID == ARM::BI__builtin_arm_rsrp ||
4605                             BuiltinID == ARM::BI__builtin_arm_wsrp;
4606
4607     bool Is64Bit = BuiltinID == ARM::BI__builtin_arm_rsr64 ||
4608                    BuiltinID == ARM::BI__builtin_arm_wsr64;
4609
4610     llvm::Type *ValueType;
4611     llvm::Type *RegisterType;
4612     if (IsPointerBuiltin) {
4613       ValueType = VoidPtrTy;
4614       RegisterType = Int32Ty;
4615     } else if (Is64Bit) {
4616       ValueType = RegisterType = Int64Ty;
4617     } else {
4618       ValueType = RegisterType = Int32Ty;
4619     }
4620
4621     return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType, IsRead);
4622   }
4623
4624   // Find out if any arguments are required to be integer constant
4625   // expressions.
4626   unsigned ICEArguments = 0;
4627   ASTContext::GetBuiltinTypeError Error;
4628   getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
4629   assert(Error == ASTContext::GE_None && "Should not codegen an error");
4630
4631   auto getAlignmentValue32 = [&](Address addr) -> Value* {
4632     return Builder.getInt32(addr.getAlignment().getQuantity());
4633   };
4634
4635   Address PtrOp0 = Address::invalid();
4636   Address PtrOp1 = Address::invalid();
4637   SmallVector<Value*, 4> Ops;
4638   bool HasExtraArg = HasExtraNeonArgument(BuiltinID);
4639   unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0);
4640   for (unsigned i = 0, e = NumArgs; i != e; i++) {
4641     if (i == 0) {
4642       switch (BuiltinID) {
4643       case NEON::BI__builtin_neon_vld1_v:
4644       case NEON::BI__builtin_neon_vld1q_v:
4645       case NEON::BI__builtin_neon_vld1q_lane_v:
4646       case NEON::BI__builtin_neon_vld1_lane_v:
4647       case NEON::BI__builtin_neon_vld1_dup_v:
4648       case NEON::BI__builtin_neon_vld1q_dup_v:
4649       case NEON::BI__builtin_neon_vst1_v:
4650       case NEON::BI__builtin_neon_vst1q_v:
4651       case NEON::BI__builtin_neon_vst1q_lane_v:
4652       case NEON::BI__builtin_neon_vst1_lane_v:
4653       case NEON::BI__builtin_neon_vst2_v:
4654       case NEON::BI__builtin_neon_vst2q_v:
4655       case NEON::BI__builtin_neon_vst2_lane_v:
4656       case NEON::BI__builtin_neon_vst2q_lane_v:
4657       case NEON::BI__builtin_neon_vst3_v:
4658       case NEON::BI__builtin_neon_vst3q_v:
4659       case NEON::BI__builtin_neon_vst3_lane_v:
4660       case NEON::BI__builtin_neon_vst3q_lane_v:
4661       case NEON::BI__builtin_neon_vst4_v:
4662       case NEON::BI__builtin_neon_vst4q_v:
4663       case NEON::BI__builtin_neon_vst4_lane_v:
4664       case NEON::BI__builtin_neon_vst4q_lane_v:
4665         // Get the alignment for the argument in addition to the value;
4666         // we'll use it later.
4667         PtrOp0 = EmitPointerWithAlignment(E->getArg(0));
4668         Ops.push_back(PtrOp0.getPointer());
4669         continue;
4670       }
4671     }
4672     if (i == 1) {
4673       switch (BuiltinID) {
4674       case NEON::BI__builtin_neon_vld2_v:
4675       case NEON::BI__builtin_neon_vld2q_v:
4676       case NEON::BI__builtin_neon_vld3_v:
4677       case NEON::BI__builtin_neon_vld3q_v:
4678       case NEON::BI__builtin_neon_vld4_v:
4679       case NEON::BI__builtin_neon_vld4q_v:
4680       case NEON::BI__builtin_neon_vld2_lane_v:
4681       case NEON::BI__builtin_neon_vld2q_lane_v:
4682       case NEON::BI__builtin_neon_vld3_lane_v:
4683       case NEON::BI__builtin_neon_vld3q_lane_v:
4684       case NEON::BI__builtin_neon_vld4_lane_v:
4685       case NEON::BI__builtin_neon_vld4q_lane_v:
4686       case NEON::BI__builtin_neon_vld2_dup_v:
4687       case NEON::BI__builtin_neon_vld3_dup_v:
4688       case NEON::BI__builtin_neon_vld4_dup_v:
4689         // Get the alignment for the argument in addition to the value;
4690         // we'll use it later.
4691         PtrOp1 = EmitPointerWithAlignment(E->getArg(1));
4692         Ops.push_back(PtrOp1.getPointer());
4693         continue;
4694       }
4695     }
4696
4697     if ((ICEArguments & (1 << i)) == 0) {
4698       Ops.push_back(EmitScalarExpr(E->getArg(i)));
4699     } else {
4700       // If this is required to be a constant, constant fold it so that we know
4701       // that the generated intrinsic gets a ConstantInt.
4702       llvm::APSInt Result;
4703       bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext());
4704       assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst;
4705       Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
4706     }
4707   }
4708
4709   switch (BuiltinID) {
4710   default: break;
4711
4712   case NEON::BI__builtin_neon_vget_lane_i8:
4713   case NEON::BI__builtin_neon_vget_lane_i16:
4714   case NEON::BI__builtin_neon_vget_lane_i32:
4715   case NEON::BI__builtin_neon_vget_lane_i64:
4716   case NEON::BI__builtin_neon_vget_lane_f32:
4717   case NEON::BI__builtin_neon_vgetq_lane_i8:
4718   case NEON::BI__builtin_neon_vgetq_lane_i16:
4719   case NEON::BI__builtin_neon_vgetq_lane_i32:
4720   case NEON::BI__builtin_neon_vgetq_lane_i64:
4721   case NEON::BI__builtin_neon_vgetq_lane_f32:
4722     return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
4723
4724   case NEON::BI__builtin_neon_vset_lane_i8:
4725   case NEON::BI__builtin_neon_vset_lane_i16:
4726   case NEON::BI__builtin_neon_vset_lane_i32:
4727   case NEON::BI__builtin_neon_vset_lane_i64:
4728   case NEON::BI__builtin_neon_vset_lane_f32:
4729   case NEON::BI__builtin_neon_vsetq_lane_i8:
4730   case NEON::BI__builtin_neon_vsetq_lane_i16:
4731   case NEON::BI__builtin_neon_vsetq_lane_i32:
4732   case NEON::BI__builtin_neon_vsetq_lane_i64:
4733   case NEON::BI__builtin_neon_vsetq_lane_f32:
4734     return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
4735
4736   case NEON::BI__builtin_neon_vsha1h_u32:
4737     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops,
4738                         "vsha1h");
4739   case NEON::BI__builtin_neon_vsha1cq_u32:
4740     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops,
4741                         "vsha1h");
4742   case NEON::BI__builtin_neon_vsha1pq_u32:
4743     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops,
4744                         "vsha1h");
4745   case NEON::BI__builtin_neon_vsha1mq_u32:
4746     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops,
4747                         "vsha1h");
4748
4749   // The ARM _MoveToCoprocessor builtins put the input register value as
4750   // the first argument, but the LLVM intrinsic expects it as the third one.
4751   case ARM::BI_MoveToCoprocessor:
4752   case ARM::BI_MoveToCoprocessor2: {
4753     Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI_MoveToCoprocessor ?
4754                                    Intrinsic::arm_mcr : Intrinsic::arm_mcr2);
4755     return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
4756                                   Ops[3], Ops[4], Ops[5]});
4757   }
4758   case ARM::BI_BitScanForward:
4759   case ARM::BI_BitScanForward64:
4760     return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanForward, E);
4761   case ARM::BI_BitScanReverse:
4762   case ARM::BI_BitScanReverse64:
4763     return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanReverse, E);
4764
4765   case ARM::BI_InterlockedAnd64:
4766     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E);
4767   case ARM::BI_InterlockedExchange64:
4768     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E);
4769   case ARM::BI_InterlockedExchangeAdd64:
4770     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E);
4771   case ARM::BI_InterlockedExchangeSub64:
4772     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E);
4773   case ARM::BI_InterlockedOr64:
4774     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E);
4775   case ARM::BI_InterlockedXor64:
4776     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E);
4777   case ARM::BI_InterlockedDecrement64:
4778     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E);
4779   case ARM::BI_InterlockedIncrement64:
4780     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E);
4781   }
4782
4783   // Get the last argument, which specifies the vector type.
4784   assert(HasExtraArg);
4785   llvm::APSInt Result;
4786   const Expr *Arg = E->getArg(E->getNumArgs()-1);
4787   if (!Arg->isIntegerConstantExpr(Result, getContext()))
4788     return nullptr;
4789
4790   if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f ||
4791       BuiltinID == ARM::BI__builtin_arm_vcvtr_d) {
4792     // Determine the overloaded type of this builtin.
4793     llvm::Type *Ty;
4794     if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f)
4795       Ty = FloatTy;
4796     else
4797       Ty = DoubleTy;
4798
4799     // Determine whether this is an unsigned conversion or not.
4800     bool usgn = Result.getZExtValue() == 1;
4801     unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
4802
4803     // Call the appropriate intrinsic.
4804     Function *F = CGM.getIntrinsic(Int, Ty);
4805     return Builder.CreateCall(F, Ops, "vcvtr");
4806   }
4807
4808   // Determine the type of this overloaded NEON intrinsic.
4809   NeonTypeFlags Type(Result.getZExtValue());
4810   bool usgn = Type.isUnsigned();
4811   bool rightShift = false;
4812
4813   llvm::VectorType *VTy = GetNeonType(this, Type);
4814   llvm::Type *Ty = VTy;
4815   if (!Ty)
4816     return nullptr;
4817
4818   // Many NEON builtins have identical semantics and uses in ARM and
4819   // AArch64. Emit these in a single function.
4820   auto IntrinsicMap = makeArrayRef(ARMSIMDIntrinsicMap);
4821   const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap(
4822       IntrinsicMap, BuiltinID, NEONSIMDIntrinsicsProvenSorted);
4823   if (Builtin)
4824     return EmitCommonNeonBuiltinExpr(
4825         Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
4826         Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1);
4827
4828   unsigned Int;
4829   switch (BuiltinID) {
4830   default: return nullptr;
4831   case NEON::BI__builtin_neon_vld1q_lane_v:
4832     // Handle 64-bit integer elements as a special case.  Use shuffles of
4833     // one-element vectors to avoid poor code for i64 in the backend.
4834     if (VTy->getElementType()->isIntegerTy(64)) {
4835       // Extract the other lane.
4836       Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4837       uint32_t Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
4838       Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane));
4839       Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
4840       // Load the value as a one-element vector.
4841       Ty = llvm::VectorType::get(VTy->getElementType(), 1);
4842       llvm::Type *Tys[] = {Ty, Int8PtrTy};
4843       Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Tys);
4844       Value *Align = getAlignmentValue32(PtrOp0);
4845       Value *Ld = Builder.CreateCall(F, {Ops[0], Align});
4846       // Combine them.
4847       uint32_t Indices[] = {1 - Lane, Lane};
4848       SV = llvm::ConstantDataVector::get(getLLVMContext(), Indices);
4849       return Builder.CreateShuffleVector(Ops[1], Ld, SV, "vld1q_lane");
4850     }
4851     // fall through
4852   case NEON::BI__builtin_neon_vld1_lane_v: {
4853     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4854     PtrOp0 = Builder.CreateElementBitCast(PtrOp0, VTy->getElementType());
4855     Value *Ld = Builder.CreateLoad(PtrOp0);
4856     return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane");
4857   }
4858   case NEON::BI__builtin_neon_vld2_dup_v:
4859   case NEON::BI__builtin_neon_vld3_dup_v:
4860   case NEON::BI__builtin_neon_vld4_dup_v: {
4861     // Handle 64-bit elements as a special-case.  There is no "dup" needed.
4862     if (VTy->getElementType()->getPrimitiveSizeInBits() == 64) {
4863       switch (BuiltinID) {
4864       case NEON::BI__builtin_neon_vld2_dup_v:
4865         Int = Intrinsic::arm_neon_vld2;
4866         break;
4867       case NEON::BI__builtin_neon_vld3_dup_v:
4868         Int = Intrinsic::arm_neon_vld3;
4869         break;
4870       case NEON::BI__builtin_neon_vld4_dup_v:
4871         Int = Intrinsic::arm_neon_vld4;
4872         break;
4873       default: llvm_unreachable("unknown vld_dup intrinsic?");
4874       }
4875       llvm::Type *Tys[] = {Ty, Int8PtrTy};
4876       Function *F = CGM.getIntrinsic(Int, Tys);
4877       llvm::Value *Align = getAlignmentValue32(PtrOp1);
4878       Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, "vld_dup");
4879       Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
4880       Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4881       return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
4882     }
4883     switch (BuiltinID) {
4884     case NEON::BI__builtin_neon_vld2_dup_v:
4885       Int = Intrinsic::arm_neon_vld2lane;
4886       break;
4887     case NEON::BI__builtin_neon_vld3_dup_v:
4888       Int = Intrinsic::arm_neon_vld3lane;
4889       break;
4890     case NEON::BI__builtin_neon_vld4_dup_v:
4891       Int = Intrinsic::arm_neon_vld4lane;
4892       break;
4893     default: llvm_unreachable("unknown vld_dup intrinsic?");
4894     }
4895     llvm::Type *Tys[] = {Ty, Int8PtrTy};
4896     Function *F = CGM.getIntrinsic(Int, Tys);
4897     llvm::StructType *STy = cast<llvm::StructType>(F->getReturnType());
4898
4899     SmallVector<Value*, 6> Args;
4900     Args.push_back(Ops[1]);
4901     Args.append(STy->getNumElements(), UndefValue::get(Ty));
4902
4903     llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
4904     Args.push_back(CI);
4905     Args.push_back(getAlignmentValue32(PtrOp1));
4906
4907     Ops[1] = Builder.CreateCall(F, Args, "vld_dup");
4908     // splat lane 0 to all elts in each vector of the result.
4909     for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
4910       Value *Val = Builder.CreateExtractValue(Ops[1], i);
4911       Value *Elt = Builder.CreateBitCast(Val, Ty);
4912       Elt = EmitNeonSplat(Elt, CI);
4913       Elt = Builder.CreateBitCast(Elt, Val->getType());
4914       Ops[1] = Builder.CreateInsertValue(Ops[1], Elt, i);
4915     }
4916     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
4917     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4918     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
4919   }
4920   case NEON::BI__builtin_neon_vqrshrn_n_v:
4921     Int =
4922       usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
4923     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n",
4924                         1, true);
4925   case NEON::BI__builtin_neon_vqrshrun_n_v:
4926     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty),
4927                         Ops, "vqrshrun_n", 1, true);
4928   case NEON::BI__builtin_neon_vqshrn_n_v:
4929     Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
4930     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n",
4931                         1, true);
4932   case NEON::BI__builtin_neon_vqshrun_n_v:
4933     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty),
4934                         Ops, "vqshrun_n", 1, true);
4935   case NEON::BI__builtin_neon_vrecpe_v:
4936   case NEON::BI__builtin_neon_vrecpeq_v:
4937     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecpe, Ty),
4938                         Ops, "vrecpe");
4939   case NEON::BI__builtin_neon_vrshrn_n_v:
4940     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty),
4941                         Ops, "vrshrn_n", 1, true);
4942   case NEON::BI__builtin_neon_vrsra_n_v:
4943   case NEON::BI__builtin_neon_vrsraq_n_v:
4944     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4945     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4946     Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true);
4947     Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
4948     Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]});
4949     return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n");
4950   case NEON::BI__builtin_neon_vsri_n_v:
4951   case NEON::BI__builtin_neon_vsriq_n_v:
4952     rightShift = true;
4953   case NEON::BI__builtin_neon_vsli_n_v:
4954   case NEON::BI__builtin_neon_vsliq_n_v:
4955     Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift);
4956     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty),
4957                         Ops, "vsli_n");
4958   case NEON::BI__builtin_neon_vsra_n_v:
4959   case NEON::BI__builtin_neon_vsraq_n_v:
4960     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4961     Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n");
4962     return Builder.CreateAdd(Ops[0], Ops[1]);
4963   case NEON::BI__builtin_neon_vst1q_lane_v:
4964     // Handle 64-bit integer elements as a special case.  Use a shuffle to get
4965     // a one-element vector and avoid poor code for i64 in the backend.
4966     if (VTy->getElementType()->isIntegerTy(64)) {
4967       Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4968       Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
4969       Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
4970       Ops[2] = getAlignmentValue32(PtrOp0);
4971       llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()};
4972       return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1,
4973                                                  Tys), Ops);
4974     }
4975     // fall through
4976   case NEON::BI__builtin_neon_vst1_lane_v: {
4977     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4978     Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
4979     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
4980     auto St = Builder.CreateStore(Ops[1], Builder.CreateBitCast(PtrOp0, Ty));
4981     return St;
4982   }
4983   case NEON::BI__builtin_neon_vtbl1_v:
4984     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1),
4985                         Ops, "vtbl1");
4986   case NEON::BI__builtin_neon_vtbl2_v:
4987     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl2),
4988                         Ops, "vtbl2");
4989   case NEON::BI__builtin_neon_vtbl3_v:
4990     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl3),
4991                         Ops, "vtbl3");
4992   case NEON::BI__builtin_neon_vtbl4_v:
4993     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl4),
4994                         Ops, "vtbl4");
4995   case NEON::BI__builtin_neon_vtbx1_v:
4996     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx1),
4997                         Ops, "vtbx1");
4998   case NEON::BI__builtin_neon_vtbx2_v:
4999     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx2),
5000                         Ops, "vtbx2");
5001   case NEON::BI__builtin_neon_vtbx3_v:
5002     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx3),
5003                         Ops, "vtbx3");
5004   case NEON::BI__builtin_neon_vtbx4_v:
5005     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx4),
5006                         Ops, "vtbx4");
5007   }
5008 }
5009
5010 static Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID,
5011                                       const CallExpr *E,
5012                                       SmallVectorImpl<Value *> &Ops) {
5013   unsigned int Int = 0;
5014   const char *s = nullptr;
5015
5016   switch (BuiltinID) {
5017   default:
5018     return nullptr;
5019   case NEON::BI__builtin_neon_vtbl1_v:
5020   case NEON::BI__builtin_neon_vqtbl1_v:
5021   case NEON::BI__builtin_neon_vqtbl1q_v:
5022   case NEON::BI__builtin_neon_vtbl2_v:
5023   case NEON::BI__builtin_neon_vqtbl2_v:
5024   case NEON::BI__builtin_neon_vqtbl2q_v:
5025   case NEON::BI__builtin_neon_vtbl3_v:
5026   case NEON::BI__builtin_neon_vqtbl3_v:
5027   case NEON::BI__builtin_neon_vqtbl3q_v:
5028   case NEON::BI__builtin_neon_vtbl4_v:
5029   case NEON::BI__builtin_neon_vqtbl4_v:
5030   case NEON::BI__builtin_neon_vqtbl4q_v:
5031     break;
5032   case NEON::BI__builtin_neon_vtbx1_v:
5033   case NEON::BI__builtin_neon_vqtbx1_v:
5034   case NEON::BI__builtin_neon_vqtbx1q_v:
5035   case NEON::BI__builtin_neon_vtbx2_v:
5036   case NEON::BI__builtin_neon_vqtbx2_v:
5037   case NEON::BI__builtin_neon_vqtbx2q_v:
5038   case NEON::BI__builtin_neon_vtbx3_v:
5039   case NEON::BI__builtin_neon_vqtbx3_v:
5040   case NEON::BI__builtin_neon_vqtbx3q_v:
5041   case NEON::BI__builtin_neon_vtbx4_v:
5042   case NEON::BI__builtin_neon_vqtbx4_v:
5043   case NEON::BI__builtin_neon_vqtbx4q_v:
5044     break;
5045   }
5046
5047   assert(E->getNumArgs() >= 3);
5048
5049   // Get the last argument, which specifies the vector type.
5050   llvm::APSInt Result;
5051   const Expr *Arg = E->getArg(E->getNumArgs() - 1);
5052   if (!Arg->isIntegerConstantExpr(Result, CGF.getContext()))
5053     return nullptr;
5054
5055   // Determine the type of this overloaded NEON intrinsic.
5056   NeonTypeFlags Type(Result.getZExtValue());
5057   llvm::VectorType *Ty = GetNeonType(&CGF, Type);
5058   if (!Ty)
5059     return nullptr;
5060
5061   CodeGen::CGBuilderTy &Builder = CGF.Builder;
5062
5063   // AArch64 scalar builtins are not overloaded, they do not have an extra
5064   // argument that specifies the vector type, need to handle each case.
5065   switch (BuiltinID) {
5066   case NEON::BI__builtin_neon_vtbl1_v: {
5067     return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 1), nullptr,
5068                               Ops[1], Ty, Intrinsic::aarch64_neon_tbl1,
5069                               "vtbl1");
5070   }
5071   case NEON::BI__builtin_neon_vtbl2_v: {
5072     return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 2), nullptr,
5073                               Ops[2], Ty, Intrinsic::aarch64_neon_tbl1,
5074                               "vtbl1");
5075   }
5076   case NEON::BI__builtin_neon_vtbl3_v: {
5077     return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 3), nullptr,
5078                               Ops[3], Ty, Intrinsic::aarch64_neon_tbl2,
5079                               "vtbl2");
5080   }
5081   case NEON::BI__builtin_neon_vtbl4_v: {
5082     return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 4), nullptr,
5083                               Ops[4], Ty, Intrinsic::aarch64_neon_tbl2,
5084                               "vtbl2");
5085   }
5086   case NEON::BI__builtin_neon_vtbx1_v: {
5087     Value *TblRes =
5088         packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 1), nullptr, Ops[2],
5089                            Ty, Intrinsic::aarch64_neon_tbl1, "vtbl1");
5090
5091     llvm::Constant *EightV = ConstantInt::get(Ty, 8);
5092     Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
5093     CmpRes = Builder.CreateSExt(CmpRes, Ty);
5094
5095     Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
5096     Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
5097     return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx");
5098   }
5099   case NEON::BI__builtin_neon_vtbx2_v: {
5100     return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 2), Ops[0],
5101                               Ops[3], Ty, Intrinsic::aarch64_neon_tbx1,
5102                               "vtbx1");
5103   }
5104   case NEON::BI__builtin_neon_vtbx3_v: {
5105     Value *TblRes =
5106         packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 3), nullptr, Ops[4],
5107                            Ty, Intrinsic::aarch64_neon_tbl2, "vtbl2");
5108
5109     llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24);
5110     Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
5111                                            TwentyFourV);
5112     CmpRes = Builder.CreateSExt(CmpRes, Ty);
5113
5114     Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
5115     Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
5116     return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx");
5117   }
5118   case NEON::BI__builtin_neon_vtbx4_v: {
5119     return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 4), Ops[0],
5120                               Ops[5], Ty, Intrinsic::aarch64_neon_tbx2,
5121                               "vtbx2");
5122   }
5123   case NEON::BI__builtin_neon_vqtbl1_v:
5124   case NEON::BI__builtin_neon_vqtbl1q_v:
5125     Int = Intrinsic::aarch64_neon_tbl1; s = "vtbl1"; break;
5126   case NEON::BI__builtin_neon_vqtbl2_v:
5127   case NEON::BI__builtin_neon_vqtbl2q_v: {
5128     Int = Intrinsic::aarch64_neon_tbl2; s = "vtbl2"; break;
5129   case NEON::BI__builtin_neon_vqtbl3_v:
5130   case NEON::BI__builtin_neon_vqtbl3q_v:
5131     Int = Intrinsic::aarch64_neon_tbl3; s = "vtbl3"; break;
5132   case NEON::BI__builtin_neon_vqtbl4_v:
5133   case NEON::BI__builtin_neon_vqtbl4q_v:
5134     Int = Intrinsic::aarch64_neon_tbl4; s = "vtbl4"; break;
5135   case NEON::BI__builtin_neon_vqtbx1_v:
5136   case NEON::BI__builtin_neon_vqtbx1q_v:
5137     Int = Intrinsic::aarch64_neon_tbx1; s = "vtbx1"; break;
5138   case NEON::BI__builtin_neon_vqtbx2_v:
5139   case NEON::BI__builtin_neon_vqtbx2q_v:
5140     Int = Intrinsic::aarch64_neon_tbx2; s = "vtbx2"; break;
5141   case NEON::BI__builtin_neon_vqtbx3_v:
5142   case NEON::BI__builtin_neon_vqtbx3q_v:
5143     Int = Intrinsic::aarch64_neon_tbx3; s = "vtbx3"; break;
5144   case NEON::BI__builtin_neon_vqtbx4_v:
5145   case NEON::BI__builtin_neon_vqtbx4q_v:
5146     Int = Intrinsic::aarch64_neon_tbx4; s = "vtbx4"; break;
5147   }
5148   }
5149
5150   if (!Int)
5151     return nullptr;
5152
5153   Function *F = CGF.CGM.getIntrinsic(Int, Ty);
5154   return CGF.EmitNeonCall(F, Ops, s);
5155 }
5156
5157 Value *CodeGenFunction::vectorWrapScalar16(Value *Op) {
5158   llvm::Type *VTy = llvm::VectorType::get(Int16Ty, 4);
5159   Op = Builder.CreateBitCast(Op, Int16Ty);
5160   Value *V = UndefValue::get(VTy);
5161   llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
5162   Op = Builder.CreateInsertElement(V, Op, CI);
5163   return Op;
5164 }
5165
5166 Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
5167                                                const CallExpr *E) {
5168   unsigned HintID = static_cast<unsigned>(-1);
5169   switch (BuiltinID) {
5170   default: break;
5171   case AArch64::BI__builtin_arm_nop:
5172     HintID = 0;
5173     break;
5174   case AArch64::BI__builtin_arm_yield:
5175     HintID = 1;
5176     break;
5177   case AArch64::BI__builtin_arm_wfe:
5178     HintID = 2;
5179     break;
5180   case AArch64::BI__builtin_arm_wfi:
5181     HintID = 3;
5182     break;
5183   case AArch64::BI__builtin_arm_sev:
5184     HintID = 4;
5185     break;
5186   case AArch64::BI__builtin_arm_sevl:
5187     HintID = 5;
5188     break;
5189   }
5190
5191   if (HintID != static_cast<unsigned>(-1)) {
5192     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hint);
5193     return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID));
5194   }
5195
5196   if (BuiltinID == AArch64::BI__builtin_arm_prefetch) {
5197     Value *Address         = EmitScalarExpr(E->getArg(0));
5198     Value *RW              = EmitScalarExpr(E->getArg(1));
5199     Value *CacheLevel      = EmitScalarExpr(E->getArg(2));
5200     Value *RetentionPolicy = EmitScalarExpr(E->getArg(3));
5201     Value *IsData          = EmitScalarExpr(E->getArg(4));
5202
5203     Value *Locality = nullptr;
5204     if (cast<llvm::ConstantInt>(RetentionPolicy)->isZero()) {
5205       // Temporal fetch, needs to convert cache level to locality.
5206       Locality = llvm::ConstantInt::get(Int32Ty,
5207         -cast<llvm::ConstantInt>(CacheLevel)->getValue() + 3);
5208     } else {
5209       // Streaming fetch.
5210       Locality = llvm::ConstantInt::get(Int32Ty, 0);
5211     }
5212
5213     // FIXME: We need AArch64 specific LLVM intrinsic if we want to specify
5214     // PLDL3STRM or PLDL2STRM.
5215     Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
5216     return Builder.CreateCall(F, {Address, RW, Locality, IsData});
5217   }
5218
5219   if (BuiltinID == AArch64::BI__builtin_arm_rbit) {
5220     assert((getContext().getTypeSize(E->getType()) == 32) &&
5221            "rbit of unusual size!");
5222     llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
5223     return Builder.CreateCall(
5224         CGM.getIntrinsic(Intrinsic::aarch64_rbit, Arg->getType()), Arg, "rbit");
5225   }
5226   if (BuiltinID == AArch64::BI__builtin_arm_rbit64) {
5227     assert((getContext().getTypeSize(E->getType()) == 64) &&
5228            "rbit of unusual size!");
5229     llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
5230     return Builder.CreateCall(
5231         CGM.getIntrinsic(Intrinsic::aarch64_rbit, Arg->getType()), Arg, "rbit");
5232   }
5233
5234   if (BuiltinID == AArch64::BI__clear_cache) {
5235     assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments");
5236     const FunctionDecl *FD = E->getDirectCallee();
5237     Value *Ops[2];
5238     for (unsigned i = 0; i < 2; i++)
5239       Ops[i] = EmitScalarExpr(E->getArg(i));
5240     llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
5241     llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
5242     StringRef Name = FD->getName();
5243     return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
5244   }
5245
5246   if ((BuiltinID == AArch64::BI__builtin_arm_ldrex ||
5247       BuiltinID == AArch64::BI__builtin_arm_ldaex) &&
5248       getContext().getTypeSize(E->getType()) == 128) {
5249     Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex
5250                                        ? Intrinsic::aarch64_ldaxp
5251                                        : Intrinsic::aarch64_ldxp);
5252
5253     Value *LdPtr = EmitScalarExpr(E->getArg(0));
5254     Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy),
5255                                     "ldxp");
5256
5257     Value *Val0 = Builder.CreateExtractValue(Val, 1);
5258     Value *Val1 = Builder.CreateExtractValue(Val, 0);
5259     llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
5260     Val0 = Builder.CreateZExt(Val0, Int128Ty);
5261     Val1 = Builder.CreateZExt(Val1, Int128Ty);
5262
5263     Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
5264     Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */);
5265     Val = Builder.CreateOr(Val, Val1);
5266     return Builder.CreateBitCast(Val, ConvertType(E->getType()));
5267   } else if (BuiltinID == AArch64::BI__builtin_arm_ldrex ||
5268              BuiltinID == AArch64::BI__builtin_arm_ldaex) {
5269     Value *LoadAddr = EmitScalarExpr(E->getArg(0));
5270
5271     QualType Ty = E->getType();
5272     llvm::Type *RealResTy = ConvertType(Ty);
5273     llvm::Type *PtrTy = llvm::IntegerType::get(
5274         getLLVMContext(), getContext().getTypeSize(Ty))->getPointerTo();
5275     LoadAddr = Builder.CreateBitCast(LoadAddr, PtrTy);
5276
5277     Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex
5278                                        ? Intrinsic::aarch64_ldaxr
5279                                        : Intrinsic::aarch64_ldxr,
5280                                    PtrTy);
5281     Value *Val = Builder.CreateCall(F, LoadAddr, "ldxr");
5282
5283     if (RealResTy->isPointerTy())
5284       return Builder.CreateIntToPtr(Val, RealResTy);
5285
5286     llvm::Type *IntResTy = llvm::IntegerType::get(
5287         getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
5288     Val = Builder.CreateTruncOrBitCast(Val, IntResTy);
5289     return Builder.CreateBitCast(Val, RealResTy);
5290   }
5291
5292   if ((BuiltinID == AArch64::BI__builtin_arm_strex ||
5293        BuiltinID == AArch64::BI__builtin_arm_stlex) &&
5294       getContext().getTypeSize(E->getArg(0)->getType()) == 128) {
5295     Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex
5296                                        ? Intrinsic::aarch64_stlxp
5297                                        : Intrinsic::aarch64_stxp);
5298     llvm::Type *STy = llvm::StructType::get(Int64Ty, Int64Ty, nullptr);
5299
5300     Address Tmp = CreateMemTemp(E->getArg(0)->getType());
5301     EmitAnyExprToMem(E->getArg(0), Tmp, Qualifiers(), /*init*/ true);
5302
5303     Tmp = Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(STy));
5304     llvm::Value *Val = Builder.CreateLoad(Tmp);
5305
5306     Value *Arg0 = Builder.CreateExtractValue(Val, 0);
5307     Value *Arg1 = Builder.CreateExtractValue(Val, 1);
5308     Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)),
5309                                          Int8PtrTy);
5310     return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "stxp");
5311   }
5312
5313   if (BuiltinID == AArch64::BI__builtin_arm_strex ||
5314       BuiltinID == AArch64::BI__builtin_arm_stlex) {
5315     Value *StoreVal = EmitScalarExpr(E->getArg(0));
5316     Value *StoreAddr = EmitScalarExpr(E->getArg(1));
5317
5318     QualType Ty = E->getArg(0)->getType();
5319     llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(),
5320                                                  getContext().getTypeSize(Ty));
5321     StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo());
5322
5323     if (StoreVal->getType()->isPointerTy())
5324       StoreVal = Builder.CreatePtrToInt(StoreVal, Int64Ty);
5325     else {
5326       llvm::Type *IntTy = llvm::IntegerType::get(
5327           getLLVMContext(),
5328           CGM.getDataLayout().getTypeSizeInBits(StoreVal->getType()));
5329       StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
5330       StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty);
5331     }
5332
5333     Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex
5334                                        ? Intrinsic::aarch64_stlxr
5335                                        : Intrinsic::aarch64_stxr,
5336                                    StoreAddr->getType());
5337     return Builder.CreateCall(F, {StoreVal, StoreAddr}, "stxr");
5338   }
5339
5340   if (BuiltinID == AArch64::BI__builtin_arm_clrex) {
5341     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex);
5342     return Builder.CreateCall(F);
5343   }
5344
5345   // CRC32
5346   Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
5347   switch (BuiltinID) {
5348   case AArch64::BI__builtin_arm_crc32b:
5349     CRCIntrinsicID = Intrinsic::aarch64_crc32b; break;
5350   case AArch64::BI__builtin_arm_crc32cb:
5351     CRCIntrinsicID = Intrinsic::aarch64_crc32cb; break;
5352   case AArch64::BI__builtin_arm_crc32h:
5353     CRCIntrinsicID = Intrinsic::aarch64_crc32h; break;
5354   case AArch64::BI__builtin_arm_crc32ch:
5355     CRCIntrinsicID = Intrinsic::aarch64_crc32ch; break;
5356   case AArch64::BI__builtin_arm_crc32w:
5357     CRCIntrinsicID = Intrinsic::aarch64_crc32w; break;
5358   case AArch64::BI__builtin_arm_crc32cw:
5359     CRCIntrinsicID = Intrinsic::aarch64_crc32cw; break;
5360   case AArch64::BI__builtin_arm_crc32d:
5361     CRCIntrinsicID = Intrinsic::aarch64_crc32x; break;
5362   case AArch64::BI__builtin_arm_crc32cd:
5363     CRCIntrinsicID = Intrinsic::aarch64_crc32cx; break;
5364   }
5365
5366   if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
5367     Value *Arg0 = EmitScalarExpr(E->getArg(0));
5368     Value *Arg1 = EmitScalarExpr(E->getArg(1));
5369     Function *F = CGM.getIntrinsic(CRCIntrinsicID);
5370
5371     llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
5372     Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy);
5373
5374     return Builder.CreateCall(F, {Arg0, Arg1});
5375   }
5376
5377   if (BuiltinID == AArch64::BI__builtin_arm_rsr ||
5378       BuiltinID == AArch64::BI__builtin_arm_rsr64 ||
5379       BuiltinID == AArch64::BI__builtin_arm_rsrp ||
5380       BuiltinID == AArch64::BI__builtin_arm_wsr ||
5381       BuiltinID == AArch64::BI__builtin_arm_wsr64 ||
5382       BuiltinID == AArch64::BI__builtin_arm_wsrp) {
5383
5384     bool IsRead = BuiltinID == AArch64::BI__builtin_arm_rsr ||
5385                   BuiltinID == AArch64::BI__builtin_arm_rsr64 ||
5386                   BuiltinID == AArch64::BI__builtin_arm_rsrp;
5387
5388     bool IsPointerBuiltin = BuiltinID == AArch64::BI__builtin_arm_rsrp ||
5389                             BuiltinID == AArch64::BI__builtin_arm_wsrp;
5390
5391     bool Is64Bit = BuiltinID != AArch64::BI__builtin_arm_rsr &&
5392                    BuiltinID != AArch64::BI__builtin_arm_wsr;
5393
5394     llvm::Type *ValueType;
5395     llvm::Type *RegisterType = Int64Ty;
5396     if (IsPointerBuiltin) {
5397       ValueType = VoidPtrTy;
5398     } else if (Is64Bit) {
5399       ValueType = Int64Ty;
5400     } else {
5401       ValueType = Int32Ty;
5402     }
5403
5404     return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType, IsRead);
5405   }
5406
5407   // Find out if any arguments are required to be integer constant
5408   // expressions.
5409   unsigned ICEArguments = 0;
5410   ASTContext::GetBuiltinTypeError Error;
5411   getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
5412   assert(Error == ASTContext::GE_None && "Should not codegen an error");
5413
5414   llvm::SmallVector<Value*, 4> Ops;
5415   for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) {
5416     if ((ICEArguments & (1 << i)) == 0) {
5417       Ops.push_back(EmitScalarExpr(E->getArg(i)));
5418     } else {
5419       // If this is required to be a constant, constant fold it so that we know
5420       // that the generated intrinsic gets a ConstantInt.
5421       llvm::APSInt Result;
5422       bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext());
5423       assert(IsConst && "Constant arg isn't actually constant?");
5424       (void)IsConst;
5425       Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
5426     }
5427   }
5428
5429   auto SISDMap = makeArrayRef(AArch64SISDIntrinsicMap);
5430   const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap(
5431       SISDMap, BuiltinID, AArch64SISDIntrinsicsProvenSorted);
5432
5433   if (Builtin) {
5434     Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1)));
5435     Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E);
5436     assert(Result && "SISD intrinsic should have been handled");
5437     return Result;
5438   }
5439
5440   llvm::APSInt Result;
5441   const Expr *Arg = E->getArg(E->getNumArgs()-1);
5442   NeonTypeFlags Type(0);
5443   if (Arg->isIntegerConstantExpr(Result, getContext()))
5444     // Determine the type of this overloaded NEON intrinsic.
5445     Type = NeonTypeFlags(Result.getZExtValue());
5446
5447   bool usgn = Type.isUnsigned();
5448   bool quad = Type.isQuad();
5449
5450   // Handle non-overloaded intrinsics first.
5451   switch (BuiltinID) {
5452   default: break;
5453   case NEON::BI__builtin_neon_vldrq_p128: {
5454     llvm::Type *Int128Ty = llvm::Type::getIntNTy(getLLVMContext(), 128);
5455     llvm::Type *Int128PTy = llvm::PointerType::get(Int128Ty, 0);
5456     Value *Ptr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), Int128PTy);
5457     return Builder.CreateAlignedLoad(Int128Ty, Ptr,
5458                                      CharUnits::fromQuantity(16));
5459   }
5460   case NEON::BI__builtin_neon_vstrq_p128: {
5461     llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128);
5462     Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy);
5463     return Builder.CreateDefaultAlignedStore(EmitScalarExpr(E->getArg(1)), Ptr);
5464   }
5465   case NEON::BI__builtin_neon_vcvts_u32_f32:
5466   case NEON::BI__builtin_neon_vcvtd_u64_f64:
5467     usgn = true;
5468     // FALL THROUGH
5469   case NEON::BI__builtin_neon_vcvts_s32_f32:
5470   case NEON::BI__builtin_neon_vcvtd_s64_f64: {
5471     Ops.push_back(EmitScalarExpr(E->getArg(0)));
5472     bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
5473     llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty;
5474     llvm::Type *FTy = Is64 ? DoubleTy : FloatTy;
5475     Ops[0] = Builder.CreateBitCast(Ops[0], FTy);
5476     if (usgn)
5477       return Builder.CreateFPToUI(Ops[0], InTy);
5478     return Builder.CreateFPToSI(Ops[0], InTy);
5479   }
5480   case NEON::BI__builtin_neon_vcvts_f32_u32:
5481   case NEON::BI__builtin_neon_vcvtd_f64_u64:
5482     usgn = true;
5483     // FALL THROUGH
5484   case NEON::BI__builtin_neon_vcvts_f32_s32:
5485   case NEON::BI__builtin_neon_vcvtd_f64_s64: {
5486     Ops.push_back(EmitScalarExpr(E->getArg(0)));
5487     bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
5488     llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty;
5489     llvm::Type *FTy = Is64 ? DoubleTy : FloatTy;
5490     Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
5491     if (usgn)
5492       return Builder.CreateUIToFP(Ops[0], FTy);
5493     return Builder.CreateSIToFP(Ops[0], FTy);
5494   }
5495   case NEON::BI__builtin_neon_vpaddd_s64: {
5496     llvm::Type *Ty = llvm::VectorType::get(Int64Ty, 2);
5497     Value *Vec = EmitScalarExpr(E->getArg(0));
5498     // The vector is v2f64, so make sure it's bitcast to that.
5499     Vec = Builder.CreateBitCast(Vec, Ty, "v2i64");
5500     llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
5501     llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
5502     Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0");
5503     Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1");
5504     // Pairwise addition of a v2f64 into a scalar f64.
5505     return Builder.CreateAdd(Op0, Op1, "vpaddd");
5506   }
5507   case NEON::BI__builtin_neon_vpaddd_f64: {
5508     llvm::Type *Ty =
5509       llvm::VectorType::get(DoubleTy, 2);
5510     Value *Vec = EmitScalarExpr(E->getArg(0));
5511     // The vector is v2f64, so make sure it's bitcast to that.
5512     Vec = Builder.CreateBitCast(Vec, Ty, "v2f64");
5513     llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
5514     llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
5515     Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0");
5516     Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1");
5517     // Pairwise addition of a v2f64 into a scalar f64.
5518     return Builder.CreateFAdd(Op0, Op1, "vpaddd");
5519   }
5520   case NEON::BI__builtin_neon_vpadds_f32: {
5521     llvm::Type *Ty =
5522       llvm::VectorType::get(FloatTy, 2);
5523     Value *Vec = EmitScalarExpr(E->getArg(0));
5524     // The vector is v2f32, so make sure it's bitcast to that.
5525     Vec = Builder.CreateBitCast(Vec, Ty, "v2f32");
5526     llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
5527     llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
5528     Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0");
5529     Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1");
5530     // Pairwise addition of a v2f32 into a scalar f32.
5531     return Builder.CreateFAdd(Op0, Op1, "vpaddd");
5532   }
5533   case NEON::BI__builtin_neon_vceqzd_s64:
5534   case NEON::BI__builtin_neon_vceqzd_f64:
5535   case NEON::BI__builtin_neon_vceqzs_f32:
5536     Ops.push_back(EmitScalarExpr(E->getArg(0)));
5537     return EmitAArch64CompareBuiltinExpr(
5538         Ops[0], ConvertType(E->getCallReturnType(getContext())),
5539         ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ, "vceqz");
5540   case NEON::BI__builtin_neon_vcgezd_s64:
5541   case NEON::BI__builtin_neon_vcgezd_f64:
5542   case NEON::BI__builtin_neon_vcgezs_f32:
5543     Ops.push_back(EmitScalarExpr(E->getArg(0)));
5544     return EmitAArch64CompareBuiltinExpr(
5545         Ops[0], ConvertType(E->getCallReturnType(getContext())),
5546         ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE, "vcgez");
5547   case NEON::BI__builtin_neon_vclezd_s64:
5548   case NEON::BI__builtin_neon_vclezd_f64:
5549   case NEON::BI__builtin_neon_vclezs_f32:
5550     Ops.push_back(EmitScalarExpr(E->getArg(0)));
5551     return EmitAArch64CompareBuiltinExpr(
5552         Ops[0], ConvertType(E->getCallReturnType(getContext())),
5553         ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE, "vclez");
5554   case NEON::BI__builtin_neon_vcgtzd_s64:
5555   case NEON::BI__builtin_neon_vcgtzd_f64:
5556   case NEON::BI__builtin_neon_vcgtzs_f32:
5557     Ops.push_back(EmitScalarExpr(E->getArg(0)));
5558     return EmitAArch64CompareBuiltinExpr(
5559         Ops[0], ConvertType(E->getCallReturnType(getContext())),
5560         ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT, "vcgtz");
5561   case NEON::BI__builtin_neon_vcltzd_s64:
5562   case NEON::BI__builtin_neon_vcltzd_f64:
5563   case NEON::BI__builtin_neon_vcltzs_f32:
5564     Ops.push_back(EmitScalarExpr(E->getArg(0)));
5565     return EmitAArch64CompareBuiltinExpr(
5566         Ops[0], ConvertType(E->getCallReturnType(getContext())),
5567         ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT, "vcltz");
5568
5569   case NEON::BI__builtin_neon_vceqzd_u64: {
5570     Ops.push_back(EmitScalarExpr(E->getArg(0)));
5571     Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
5572     Ops[0] =
5573         Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty));
5574     return Builder.CreateSExt(Ops[0], Int64Ty, "vceqzd");
5575   }
5576   case NEON::BI__builtin_neon_vceqd_f64:
5577   case NEON::BI__builtin_neon_vcled_f64:
5578   case NEON::BI__builtin_neon_vcltd_f64:
5579   case NEON::BI__builtin_neon_vcged_f64:
5580   case NEON::BI__builtin_neon_vcgtd_f64: {
5581     llvm::CmpInst::Predicate P;
5582     switch (BuiltinID) {
5583     default: llvm_unreachable("missing builtin ID in switch!");
5584     case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ; break;
5585     case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE; break;
5586     case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT; break;
5587     case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE; break;
5588     case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT; break;
5589     }
5590     Ops.push_back(EmitScalarExpr(E->getArg(1)));
5591     Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
5592     Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
5593     Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
5594     return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd");
5595   }
5596   case NEON::BI__builtin_neon_vceqs_f32:
5597   case NEON::BI__builtin_neon_vcles_f32:
5598   case NEON::BI__builtin_neon_vclts_f32:
5599   case NEON::BI__builtin_neon_vcges_f32:
5600   case NEON::BI__builtin_neon_vcgts_f32: {
5601     llvm::CmpInst::Predicate P;
5602     switch (BuiltinID) {
5603     default: llvm_unreachable("missing builtin ID in switch!");
5604     case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ; break;
5605     case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE; break;
5606     case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT; break;
5607     case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE; break;
5608     case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT; break;
5609     }
5610     Ops.push_back(EmitScalarExpr(E->getArg(1)));
5611     Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy);
5612     Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy);
5613     Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
5614     return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd");
5615   }
5616   case NEON::BI__builtin_neon_vceqd_s64:
5617   case NEON::BI__builtin_neon_vceqd_u64:
5618   case NEON::BI__builtin_neon_vcgtd_s64:
5619   case NEON::BI__builtin_neon_vcgtd_u64:
5620   case NEON::BI__builtin_neon_vcltd_s64:
5621   case NEON::BI__builtin_neon_vcltd_u64:
5622   case NEON::BI__builtin_neon_vcged_u64:
5623   case NEON::BI__builtin_neon_vcged_s64:
5624   case NEON::BI__builtin_neon_vcled_u64:
5625   case NEON::BI__builtin_neon_vcled_s64: {
5626     llvm::CmpInst::Predicate P;
5627     switch (BuiltinID) {
5628     default: llvm_unreachable("missing builtin ID in switch!");
5629     case NEON::BI__builtin_neon_vceqd_s64:
5630     case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;break;
5631     case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;break;
5632     case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;break;
5633     case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;break;
5634     case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;break;
5635     case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;break;
5636     case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;break;
5637     case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;break;
5638     case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;break;
5639     }
5640     Ops.push_back(EmitScalarExpr(E->getArg(1)));
5641     Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
5642     Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
5643     Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]);
5644     return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd");
5645   }
5646   case NEON::BI__builtin_neon_vtstd_s64:
5647   case NEON::BI__builtin_neon_vtstd_u64: {
5648     Ops.push_back(EmitScalarExpr(E->getArg(1)));
5649     Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
5650     Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
5651     Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
5652     Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
5653                                 llvm::Constant::getNullValue(Int64Ty));
5654     return Builder.CreateSExt(Ops[0], Int64Ty, "vtstd");
5655   }
5656   case NEON::BI__builtin_neon_vset_lane_i8:
5657   case NEON::BI__builtin_neon_vset_lane_i16:
5658   case NEON::BI__builtin_neon_vset_lane_i32:
5659   case NEON::BI__builtin_neon_vset_lane_i64:
5660   case NEON::BI__builtin_neon_vset_lane_f32:
5661   case NEON::BI__builtin_neon_vsetq_lane_i8:
5662   case NEON::BI__builtin_neon_vsetq_lane_i16:
5663   case NEON::BI__builtin_neon_vsetq_lane_i32:
5664   case NEON::BI__builtin_neon_vsetq_lane_i64:
5665   case NEON::BI__builtin_neon_vsetq_lane_f32:
5666     Ops.push_back(EmitScalarExpr(E->getArg(2)));
5667     return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
5668   case NEON::BI__builtin_neon_vset_lane_f64:
5669     // The vector type needs a cast for the v1f64 variant.
5670     Ops[1] = Builder.CreateBitCast(Ops[1],
5671                                    llvm::VectorType::get(DoubleTy, 1));
5672     Ops.push_back(EmitScalarExpr(E->getArg(2)));
5673     return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
5674   case NEON::BI__builtin_neon_vsetq_lane_f64:
5675     // The vector type needs a cast for the v2f64 variant.
5676     Ops[1] = Builder.CreateBitCast(Ops[1],
5677         llvm::VectorType::get(DoubleTy, 2));
5678     Ops.push_back(EmitScalarExpr(E->getArg(2)));
5679     return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
5680
5681   case NEON::BI__builtin_neon_vget_lane_i8:
5682   case NEON::BI__builtin_neon_vdupb_lane_i8:
5683     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 8));
5684     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
5685                                         "vget_lane");
5686   case NEON::BI__builtin_neon_vgetq_lane_i8:
5687   case NEON::BI__builtin_neon_vdupb_laneq_i8:
5688     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 16));
5689     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
5690                                         "vgetq_lane");
5691   case NEON::BI__builtin_neon_vget_lane_i16:
5692   case NEON::BI__builtin_neon_vduph_lane_i16:
5693     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 4));
5694     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
5695                                         "vget_lane");
5696   case NEON::BI__builtin_neon_vgetq_lane_i16:
5697   case NEON::BI__builtin_neon_vduph_laneq_i16:
5698     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 8));
5699     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
5700                                         "vgetq_lane");
5701   case NEON::BI__builtin_neon_vget_lane_i32:
5702   case NEON::BI__builtin_neon_vdups_lane_i32:
5703     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 2));
5704     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
5705                                         "vget_lane");
5706   case NEON::BI__builtin_neon_vdups_lane_f32:
5707     Ops[0] = Builder.CreateBitCast(Ops[0],
5708         llvm::VectorType::get(FloatTy, 2));
5709     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
5710                                         "vdups_lane");
5711   case NEON::BI__builtin_neon_vgetq_lane_i32:
5712   case NEON::BI__builtin_neon_vdups_laneq_i32:
5713     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4));
5714     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
5715                                         "vgetq_lane");
5716   case NEON::BI__builtin_neon_vget_lane_i64:
5717   case NEON::BI__builtin_neon_vdupd_lane_i64:
5718     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 1));
5719     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
5720                                         "vget_lane");
5721   case NEON::BI__builtin_neon_vdupd_lane_f64:
5722     Ops[0] = Builder.CreateBitCast(Ops[0],
5723         llvm::VectorType::get(DoubleTy, 1));
5724     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
5725                                         "vdupd_lane");
5726   case NEON::BI__builtin_neon_vgetq_lane_i64:
5727   case NEON::BI__builtin_neon_vdupd_laneq_i64:
5728     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2));
5729     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
5730                                         "vgetq_lane");
5731   case NEON::BI__builtin_neon_vget_lane_f32:
5732     Ops[0] = Builder.CreateBitCast(Ops[0],
5733         llvm::VectorType::get(FloatTy, 2));
5734     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
5735                                         "vget_lane");
5736   case NEON::BI__builtin_neon_vget_lane_f64:
5737     Ops[0] = Builder.CreateBitCast(Ops[0],
5738         llvm::VectorType::get(DoubleTy, 1));
5739     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
5740                                         "vget_lane");
5741   case NEON::BI__builtin_neon_vgetq_lane_f32:
5742   case NEON::BI__builtin_neon_vdups_laneq_f32:
5743     Ops[0] = Builder.CreateBitCast(Ops[0],
5744         llvm::VectorType::get(FloatTy, 4));
5745     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
5746                                         "vgetq_lane");
5747   case NEON::BI__builtin_neon_vgetq_lane_f64:
5748   case NEON::BI__builtin_neon_vdupd_laneq_f64:
5749     Ops[0] = Builder.CreateBitCast(Ops[0],
5750         llvm::VectorType::get(DoubleTy, 2));
5751     return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
5752                                         "vgetq_lane");
5753   case NEON::BI__builtin_neon_vaddd_s64:
5754   case NEON::BI__builtin_neon_vaddd_u64:
5755     return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd");
5756   case NEON::BI__builtin_neon_vsubd_s64:
5757   case NEON::BI__builtin_neon_vsubd_u64:
5758     return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd");
5759   case NEON::BI__builtin_neon_vqdmlalh_s16:
5760   case NEON::BI__builtin_neon_vqdmlslh_s16: {
5761     SmallVector<Value *, 2> ProductOps;
5762     ProductOps.push_back(vectorWrapScalar16(Ops[1]));
5763     ProductOps.push_back(vectorWrapScalar16(EmitScalarExpr(E->getArg(2))));
5764     llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4);
5765     Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
5766                           ProductOps, "vqdmlXl");
5767     Constant *CI = ConstantInt::get(SizeTy, 0);
5768     Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
5769
5770     unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
5771                                         ? Intrinsic::aarch64_neon_sqadd
5772                                         : Intrinsic::aarch64_neon_sqsub;
5773     return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl");
5774   }
5775   case NEON::BI__builtin_neon_vqshlud_n_s64: {
5776     Ops.push_back(EmitScalarExpr(E->getArg(1)));
5777     Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
5778     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqshlu, Int64Ty),
5779                         Ops, "vqshlu_n");
5780   }
5781   case NEON::BI__builtin_neon_vqshld_n_u64:
5782   case NEON::BI__builtin_neon_vqshld_n_s64: {
5783     unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
5784                                    ? Intrinsic::aarch64_neon_uqshl
5785                                    : Intrinsic::aarch64_neon_sqshl;
5786     Ops.push_back(EmitScalarExpr(E->getArg(1)));
5787     Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
5788     return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n");
5789   }
5790   case NEON::BI__builtin_neon_vrshrd_n_u64:
5791   case NEON::BI__builtin_neon_vrshrd_n_s64: {
5792     unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
5793                                    ? Intrinsic::aarch64_neon_urshl
5794                                    : Intrinsic::aarch64_neon_srshl;
5795     Ops.push_back(EmitScalarExpr(E->getArg(1)));
5796     int SV = cast<ConstantInt>(Ops[1])->getSExtValue();
5797     Ops[1] = ConstantInt::get(Int64Ty, -SV);
5798     return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n");
5799   }
5800   case NEON::BI__builtin_neon_vrsrad_n_u64:
5801   case NEON::BI__builtin_neon_vrsrad_n_s64: {
5802     unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
5803                                    ? Intrinsic::aarch64_neon_urshl
5804                                    : Intrinsic::aarch64_neon_srshl;
5805     Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
5806     Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2))));
5807     Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty),
5808                                 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
5809     return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty));
5810   }
5811   case NEON::BI__builtin_neon_vshld_n_s64:
5812   case NEON::BI__builtin_neon_vshld_n_u64: {
5813     llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
5814     return Builder.CreateShl(
5815         Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n");
5816   }
5817   case NEON::BI__builtin_neon_vshrd_n_s64: {
5818     llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
5819     return Builder.CreateAShr(
5820         Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63),
5821                                                    Amt->getZExtValue())),
5822         "shrd_n");
5823   }
5824   case NEON::BI__builtin_neon_vshrd_n_u64: {
5825     llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
5826     uint64_t ShiftAmt = Amt->getZExtValue();
5827     // Right-shifting an unsigned value by its size yields 0.
5828     if (ShiftAmt == 64)
5829       return ConstantInt::get(Int64Ty, 0);
5830     return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt),
5831                               "shrd_n");
5832   }
5833   case NEON::BI__builtin_neon_vsrad_n_s64: {
5834     llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2)));
5835     Ops[1] = Builder.CreateAShr(
5836         Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63),
5837                                                    Amt->getZExtValue())),
5838         "shrd_n");
5839     return Builder.CreateAdd(Ops[0], Ops[1]);
5840   }
5841   case NEON::BI__builtin_neon_vsrad_n_u64: {
5842     llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2)));
5843     uint64_t ShiftAmt = Amt->getZExtValue();
5844     // Right-shifting an unsigned value by its size yields 0.
5845     // As Op + 0 = Op, return Ops[0] directly.
5846     if (ShiftAmt == 64)
5847       return Ops[0];
5848     Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt),
5849                                 "shrd_n");
5850     return Builder.CreateAdd(Ops[0], Ops[1]);
5851   }
5852   case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
5853   case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
5854   case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
5855   case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
5856     Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)),
5857                                           "lane");
5858     SmallVector<Value *, 2> ProductOps;
5859     ProductOps.push_back(vectorWrapScalar16(Ops[1]));
5860     ProductOps.push_back(vectorWrapScalar16(Ops[2]));
5861     llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4);
5862     Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
5863                           ProductOps, "vqdmlXl");
5864     Constant *CI = ConstantInt::get(SizeTy, 0);
5865     Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
5866     Ops.pop_back();
5867
5868     unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
5869                        BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
5870                           ? Intrinsic::aarch64_neon_sqadd
5871                           : Intrinsic::aarch64_neon_sqsub;
5872     return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl");
5873   }
5874   case NEON::BI__builtin_neon_vqdmlals_s32:
5875   case NEON::BI__builtin_neon_vqdmlsls_s32: {
5876     SmallVector<Value *, 2> ProductOps;
5877     ProductOps.push_back(Ops[1]);
5878     ProductOps.push_back(EmitScalarExpr(E->getArg(2)));
5879     Ops[1] =
5880         EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
5881                      ProductOps, "vqdmlXl");
5882
5883     unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
5884                                         ? Intrinsic::aarch64_neon_sqadd
5885                                         : Intrinsic::aarch64_neon_sqsub;
5886     return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl");
5887   }
5888   case NEON::BI__builtin_neon_vqdmlals_lane_s32:
5889   case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
5890   case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
5891   case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
5892     Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)),
5893                                           "lane");
5894     SmallVector<Value *, 2> ProductOps;
5895     ProductOps.push_back(Ops[1]);
5896     ProductOps.push_back(Ops[2]);
5897     Ops[1] =
5898         EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
5899                      ProductOps, "vqdmlXl");
5900     Ops.pop_back();
5901
5902     unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
5903                        BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
5904                           ? Intrinsic::aarch64_neon_sqadd
5905                           : Intrinsic::aarch64_neon_sqsub;
5906     return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl");
5907   }
5908   }
5909
5910   llvm::VectorType *VTy = GetNeonType(this, Type);
5911   llvm::Type *Ty = VTy;
5912   if (!Ty)
5913     return nullptr;
5914
5915   // Not all intrinsics handled by the common case work for AArch64 yet, so only
5916   // defer to common code if it's been added to our special map.
5917   Builtin = findNeonIntrinsicInMap(AArch64SIMDIntrinsicMap, BuiltinID,
5918                                    AArch64SIMDIntrinsicsProvenSorted);
5919
5920   if (Builtin)
5921     return EmitCommonNeonBuiltinExpr(
5922         Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
5923         Builtin->NameHint, Builtin->TypeModifier, E, Ops,
5924         /*never use addresses*/ Address::invalid(), Address::invalid());
5925
5926   if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops))
5927     return V;
5928
5929   unsigned Int;
5930   switch (BuiltinID) {
5931   default: return nullptr;
5932   case NEON::BI__builtin_neon_vbsl_v:
5933   case NEON::BI__builtin_neon_vbslq_v: {
5934     llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
5935     Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl");
5936     Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl");
5937     Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl");
5938
5939     Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl");
5940     Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl");
5941     Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl");
5942     return Builder.CreateBitCast(Ops[0], Ty);
5943   }
5944   case NEON::BI__builtin_neon_vfma_lane_v:
5945   case NEON::BI__builtin_neon_vfmaq_lane_v: { // Only used for FP types
5946     // The ARM builtins (and instructions) have the addend as the first
5947     // operand, but the 'fma' intrinsics have it last. Swap it around here.
5948     Value *Addend = Ops[0];
5949     Value *Multiplicand = Ops[1];
5950     Value *LaneSource = Ops[2];
5951     Ops[0] = Multiplicand;
5952     Ops[1] = LaneSource;
5953     Ops[2] = Addend;
5954
5955     // Now adjust things to handle the lane access.
5956     llvm::Type *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v ?
5957       llvm::VectorType::get(VTy->getElementType(), VTy->getNumElements() / 2) :
5958       VTy;
5959     llvm::Constant *cst = cast<Constant>(Ops[3]);
5960     Value *SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), cst);
5961     Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy);
5962     Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane");
5963
5964     Ops.pop_back();
5965     Int = Intrinsic::fma;
5966     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla");
5967   }
5968   case NEON::BI__builtin_neon_vfma_laneq_v: {
5969     llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
5970     // v1f64 fma should be mapped to Neon scalar f64 fma
5971     if (VTy && VTy->getElementType() == DoubleTy) {
5972       Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
5973       Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
5974       llvm::Type *VTy = GetNeonType(this,
5975         NeonTypeFlags(NeonTypeFlags::Float64, false, true));
5976       Ops[2] = Builder.CreateBitCast(Ops[2], VTy);
5977       Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
5978       Value *F = CGM.getIntrinsic(Intrinsic::fma, DoubleTy);
5979       Value *Result = Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
5980       return Builder.CreateBitCast(Result, Ty);
5981     }
5982     Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
5983     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5984     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5985
5986     llvm::Type *STy = llvm::VectorType::get(VTy->getElementType(),
5987                                             VTy->getNumElements() * 2);
5988     Ops[2] = Builder.CreateBitCast(Ops[2], STy);
5989     Value* SV = llvm::ConstantVector::getSplat(VTy->getNumElements(),
5990                                                cast<ConstantInt>(Ops[3]));
5991     Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane");
5992
5993     return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
5994   }
5995   case NEON::BI__builtin_neon_vfmaq_laneq_v: {
5996     Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
5997     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5998     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5999
6000     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6001     Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3]));
6002     return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
6003   }
6004   case NEON::BI__builtin_neon_vfmas_lane_f32:
6005   case NEON::BI__builtin_neon_vfmas_laneq_f32:
6006   case NEON::BI__builtin_neon_vfmad_lane_f64:
6007   case NEON::BI__builtin_neon_vfmad_laneq_f64: {
6008     Ops.push_back(EmitScalarExpr(E->getArg(3)));
6009     llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext()));
6010     Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
6011     Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
6012     return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
6013   }
6014   case NEON::BI__builtin_neon_vmull_v:
6015     // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6016     Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
6017     if (Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull;
6018     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
6019   case NEON::BI__builtin_neon_vmax_v:
6020   case NEON::BI__builtin_neon_vmaxq_v:
6021     // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6022     Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax;
6023     if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax;
6024     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax");
6025   case NEON::BI__builtin_neon_vmin_v:
6026   case NEON::BI__builtin_neon_vminq_v:
6027     // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6028     Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin;
6029     if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin;
6030     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin");
6031   case NEON::BI__builtin_neon_vabd_v:
6032   case NEON::BI__builtin_neon_vabdq_v:
6033     // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6034     Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
6035     if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd;
6036     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd");
6037   case NEON::BI__builtin_neon_vpadal_v:
6038   case NEON::BI__builtin_neon_vpadalq_v: {
6039     unsigned ArgElts = VTy->getNumElements();
6040     llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType());
6041     unsigned BitWidth = EltTy->getBitWidth();
6042     llvm::Type *ArgTy = llvm::VectorType::get(
6043         llvm::IntegerType::get(getLLVMContext(), BitWidth/2), 2*ArgElts);
6044     llvm::Type* Tys[2] = { VTy, ArgTy };
6045     Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
6046     SmallVector<llvm::Value*, 1> TmpOps;
6047     TmpOps.push_back(Ops[1]);
6048     Function *F = CGM.getIntrinsic(Int, Tys);
6049     llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vpadal");
6050     llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType());
6051     return Builder.CreateAdd(tmp, addend);
6052   }
6053   case NEON::BI__builtin_neon_vpmin_v:
6054   case NEON::BI__builtin_neon_vpminq_v:
6055     // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6056     Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
6057     if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp;
6058     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin");
6059   case NEON::BI__builtin_neon_vpmax_v:
6060   case NEON::BI__builtin_neon_vpmaxq_v:
6061     // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
6062     Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
6063     if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp;
6064     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax");
6065   case NEON::BI__builtin_neon_vminnm_v:
6066   case NEON::BI__builtin_neon_vminnmq_v:
6067     Int = Intrinsic::aarch64_neon_fminnm;
6068     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm");
6069   case NEON::BI__builtin_neon_vmaxnm_v:
6070   case NEON::BI__builtin_neon_vmaxnmq_v:
6071     Int = Intrinsic::aarch64_neon_fmaxnm;
6072     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm");
6073   case NEON::BI__builtin_neon_vrecpss_f32: {
6074     Ops.push_back(EmitScalarExpr(E->getArg(1)));
6075     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, FloatTy),
6076                         Ops, "vrecps");
6077   }
6078   case NEON::BI__builtin_neon_vrecpsd_f64: {
6079     Ops.push_back(EmitScalarExpr(E->getArg(1)));
6080     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, DoubleTy),
6081                         Ops, "vrecps");
6082   }
6083   case NEON::BI__builtin_neon_vqshrun_n_v:
6084     Int = Intrinsic::aarch64_neon_sqshrun;
6085     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n");
6086   case NEON::BI__builtin_neon_vqrshrun_n_v:
6087     Int = Intrinsic::aarch64_neon_sqrshrun;
6088     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n");
6089   case NEON::BI__builtin_neon_vqshrn_n_v:
6090     Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
6091     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n");
6092   case NEON::BI__builtin_neon_vrshrn_n_v:
6093     Int = Intrinsic::aarch64_neon_rshrn;
6094     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n");
6095   case NEON::BI__builtin_neon_vqrshrn_n_v:
6096     Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
6097     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n");
6098   case NEON::BI__builtin_neon_vrnda_v:
6099   case NEON::BI__builtin_neon_vrndaq_v: {
6100     Int = Intrinsic::round;
6101     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda");
6102   }
6103   case NEON::BI__builtin_neon_vrndi_v:
6104   case NEON::BI__builtin_neon_vrndiq_v: {
6105     Int = Intrinsic::nearbyint;
6106     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndi");
6107   }
6108   case NEON::BI__builtin_neon_vrndm_v:
6109   case NEON::BI__builtin_neon_vrndmq_v: {
6110     Int = Intrinsic::floor;
6111     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm");
6112   }
6113   case NEON::BI__builtin_neon_vrndn_v:
6114   case NEON::BI__builtin_neon_vrndnq_v: {
6115     Int = Intrinsic::aarch64_neon_frintn;
6116     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn");
6117   }
6118   case NEON::BI__builtin_neon_vrndp_v:
6119   case NEON::BI__builtin_neon_vrndpq_v: {
6120     Int = Intrinsic::ceil;
6121     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp");
6122   }
6123   case NEON::BI__builtin_neon_vrndx_v:
6124   case NEON::BI__builtin_neon_vrndxq_v: {
6125     Int = Intrinsic::rint;
6126     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx");
6127   }
6128   case NEON::BI__builtin_neon_vrnd_v:
6129   case NEON::BI__builtin_neon_vrndq_v: {
6130     Int = Intrinsic::trunc;
6131     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz");
6132   }
6133   case NEON::BI__builtin_neon_vceqz_v:
6134   case NEON::BI__builtin_neon_vceqzq_v:
6135     return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ,
6136                                          ICmpInst::ICMP_EQ, "vceqz");
6137   case NEON::BI__builtin_neon_vcgez_v:
6138   case NEON::BI__builtin_neon_vcgezq_v:
6139     return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE,
6140                                          ICmpInst::ICMP_SGE, "vcgez");
6141   case NEON::BI__builtin_neon_vclez_v:
6142   case NEON::BI__builtin_neon_vclezq_v:
6143     return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE,
6144                                          ICmpInst::ICMP_SLE, "vclez");
6145   case NEON::BI__builtin_neon_vcgtz_v:
6146   case NEON::BI__builtin_neon_vcgtzq_v:
6147     return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT,
6148                                          ICmpInst::ICMP_SGT, "vcgtz");
6149   case NEON::BI__builtin_neon_vcltz_v:
6150   case NEON::BI__builtin_neon_vcltzq_v:
6151     return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT,
6152                                          ICmpInst::ICMP_SLT, "vcltz");
6153   case NEON::BI__builtin_neon_vcvt_f64_v:
6154   case NEON::BI__builtin_neon_vcvtq_f64_v:
6155     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6156     Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, quad));
6157     return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
6158                 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
6159   case NEON::BI__builtin_neon_vcvt_f64_f32: {
6160     assert(Type.getEltType() == NeonTypeFlags::Float64 && quad &&
6161            "unexpected vcvt_f64_f32 builtin");
6162     NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float32, false, false);
6163     Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
6164
6165     return Builder.CreateFPExt(Ops[0], Ty, "vcvt");
6166   }
6167   case NEON::BI__builtin_neon_vcvt_f32_f64: {
6168     assert(Type.getEltType() == NeonTypeFlags::Float32 &&
6169            "unexpected vcvt_f32_f64 builtin");
6170     NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float64, false, true);
6171     Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
6172
6173     return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt");
6174   }
6175   case NEON::BI__builtin_neon_vcvt_s32_v:
6176   case NEON::BI__builtin_neon_vcvt_u32_v:
6177   case NEON::BI__builtin_neon_vcvt_s64_v:
6178   case NEON::BI__builtin_neon_vcvt_u64_v:
6179   case NEON::BI__builtin_neon_vcvtq_s32_v:
6180   case NEON::BI__builtin_neon_vcvtq_u32_v:
6181   case NEON::BI__builtin_neon_vcvtq_s64_v:
6182   case NEON::BI__builtin_neon_vcvtq_u64_v: {
6183     Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type));
6184     if (usgn)
6185       return Builder.CreateFPToUI(Ops[0], Ty);
6186     return Builder.CreateFPToSI(Ops[0], Ty);
6187   }
6188   case NEON::BI__builtin_neon_vcvta_s32_v:
6189   case NEON::BI__builtin_neon_vcvtaq_s32_v:
6190   case NEON::BI__builtin_neon_vcvta_u32_v:
6191   case NEON::BI__builtin_neon_vcvtaq_u32_v:
6192   case NEON::BI__builtin_neon_vcvta_s64_v:
6193   case NEON::BI__builtin_neon_vcvtaq_s64_v:
6194   case NEON::BI__builtin_neon_vcvta_u64_v:
6195   case NEON::BI__builtin_neon_vcvtaq_u64_v: {
6196     Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
6197     llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
6198     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta");
6199   }
6200   case NEON::BI__builtin_neon_vcvtm_s32_v:
6201   case NEON::BI__builtin_neon_vcvtmq_s32_v:
6202   case NEON::BI__builtin_neon_vcvtm_u32_v:
6203   case NEON::BI__builtin_neon_vcvtmq_u32_v:
6204   case NEON::BI__builtin_neon_vcvtm_s64_v:
6205   case NEON::BI__builtin_neon_vcvtmq_s64_v:
6206   case NEON::BI__builtin_neon_vcvtm_u64_v:
6207   case NEON::BI__builtin_neon_vcvtmq_u64_v: {
6208     Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
6209     llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
6210     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm");
6211   }
6212   case NEON::BI__builtin_neon_vcvtn_s32_v:
6213   case NEON::BI__builtin_neon_vcvtnq_s32_v:
6214   case NEON::BI__builtin_neon_vcvtn_u32_v:
6215   case NEON::BI__builtin_neon_vcvtnq_u32_v:
6216   case NEON::BI__builtin_neon_vcvtn_s64_v:
6217   case NEON::BI__builtin_neon_vcvtnq_s64_v:
6218   case NEON::BI__builtin_neon_vcvtn_u64_v:
6219   case NEON::BI__builtin_neon_vcvtnq_u64_v: {
6220     Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
6221     llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
6222     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn");
6223   }
6224   case NEON::BI__builtin_neon_vcvtp_s32_v:
6225   case NEON::BI__builtin_neon_vcvtpq_s32_v:
6226   case NEON::BI__builtin_neon_vcvtp_u32_v:
6227   case NEON::BI__builtin_neon_vcvtpq_u32_v:
6228   case NEON::BI__builtin_neon_vcvtp_s64_v:
6229   case NEON::BI__builtin_neon_vcvtpq_s64_v:
6230   case NEON::BI__builtin_neon_vcvtp_u64_v:
6231   case NEON::BI__builtin_neon_vcvtpq_u64_v: {
6232     Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
6233     llvm::Type *Tys[2] = { Ty, GetFloatNeonType(this, Type) };
6234     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp");
6235   }
6236   case NEON::BI__builtin_neon_vmulx_v:
6237   case NEON::BI__builtin_neon_vmulxq_v: {
6238     Int = Intrinsic::aarch64_neon_fmulx;
6239     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx");
6240   }
6241   case NEON::BI__builtin_neon_vmul_lane_v:
6242   case NEON::BI__builtin_neon_vmul_laneq_v: {
6243     // v1f64 vmul_lane should be mapped to Neon scalar mul lane
6244     bool Quad = false;
6245     if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
6246       Quad = true;
6247     Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
6248     llvm::Type *VTy = GetNeonType(this,
6249       NeonTypeFlags(NeonTypeFlags::Float64, false, Quad));
6250     Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
6251     Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
6252     Value *Result = Builder.CreateFMul(Ops[0], Ops[1]);
6253     return Builder.CreateBitCast(Result, Ty);
6254   }
6255   case NEON::BI__builtin_neon_vnegd_s64:
6256     return Builder.CreateNeg(EmitScalarExpr(E->getArg(0)), "vnegd");
6257   case NEON::BI__builtin_neon_vpmaxnm_v:
6258   case NEON::BI__builtin_neon_vpmaxnmq_v: {
6259     Int = Intrinsic::aarch64_neon_fmaxnmp;
6260     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm");
6261   }
6262   case NEON::BI__builtin_neon_vpminnm_v:
6263   case NEON::BI__builtin_neon_vpminnmq_v: {
6264     Int = Intrinsic::aarch64_neon_fminnmp;
6265     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm");
6266   }
6267   case NEON::BI__builtin_neon_vsqrt_v:
6268   case NEON::BI__builtin_neon_vsqrtq_v: {
6269     Int = Intrinsic::sqrt;
6270     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6271     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt");
6272   }
6273   case NEON::BI__builtin_neon_vrbit_v:
6274   case NEON::BI__builtin_neon_vrbitq_v: {
6275     Int = Intrinsic::aarch64_neon_rbit;
6276     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit");
6277   }
6278   case NEON::BI__builtin_neon_vaddv_u8:
6279     // FIXME: These are handled by the AArch64 scalar code.
6280     usgn = true;
6281     // FALLTHROUGH
6282   case NEON::BI__builtin_neon_vaddv_s8: {
6283     Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
6284     Ty = Int32Ty;
6285     VTy = llvm::VectorType::get(Int8Ty, 8);
6286     llvm::Type *Tys[2] = { Ty, VTy };
6287     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6288     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
6289     return Builder.CreateTrunc(Ops[0], Int8Ty);
6290   }
6291   case NEON::BI__builtin_neon_vaddv_u16:
6292     usgn = true;
6293     // FALLTHROUGH
6294   case NEON::BI__builtin_neon_vaddv_s16: {
6295     Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
6296     Ty = Int32Ty;
6297     VTy = llvm::VectorType::get(Int16Ty, 4);
6298     llvm::Type *Tys[2] = { Ty, VTy };
6299     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6300     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
6301     return Builder.CreateTrunc(Ops[0], Int16Ty);
6302   }
6303   case NEON::BI__builtin_neon_vaddvq_u8:
6304     usgn = true;
6305     // FALLTHROUGH
6306   case NEON::BI__builtin_neon_vaddvq_s8: {
6307     Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
6308     Ty = Int32Ty;
6309     VTy = llvm::VectorType::get(Int8Ty, 16);
6310     llvm::Type *Tys[2] = { Ty, VTy };
6311     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6312     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
6313     return Builder.CreateTrunc(Ops[0], Int8Ty);
6314   }
6315   case NEON::BI__builtin_neon_vaddvq_u16:
6316     usgn = true;
6317     // FALLTHROUGH
6318   case NEON::BI__builtin_neon_vaddvq_s16: {
6319     Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
6320     Ty = Int32Ty;
6321     VTy = llvm::VectorType::get(Int16Ty, 8);
6322     llvm::Type *Tys[2] = { Ty, VTy };
6323     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6324     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
6325     return Builder.CreateTrunc(Ops[0], Int16Ty);
6326   }
6327   case NEON::BI__builtin_neon_vmaxv_u8: {
6328     Int = Intrinsic::aarch64_neon_umaxv;
6329     Ty = Int32Ty;
6330     VTy = llvm::VectorType::get(Int8Ty, 8);
6331     llvm::Type *Tys[2] = { Ty, VTy };
6332     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6333     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
6334     return Builder.CreateTrunc(Ops[0], Int8Ty);
6335   }
6336   case NEON::BI__builtin_neon_vmaxv_u16: {
6337     Int = Intrinsic::aarch64_neon_umaxv;
6338     Ty = Int32Ty;
6339     VTy = llvm::VectorType::get(Int16Ty, 4);
6340     llvm::Type *Tys[2] = { Ty, VTy };
6341     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6342     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
6343     return Builder.CreateTrunc(Ops[0], Int16Ty);
6344   }
6345   case NEON::BI__builtin_neon_vmaxvq_u8: {
6346     Int = Intrinsic::aarch64_neon_umaxv;
6347     Ty = Int32Ty;
6348     VTy = llvm::VectorType::get(Int8Ty, 16);
6349     llvm::Type *Tys[2] = { Ty, VTy };
6350     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6351     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
6352     return Builder.CreateTrunc(Ops[0], Int8Ty);
6353   }
6354   case NEON::BI__builtin_neon_vmaxvq_u16: {
6355     Int = Intrinsic::aarch64_neon_umaxv;
6356     Ty = Int32Ty;
6357     VTy = llvm::VectorType::get(Int16Ty, 8);
6358     llvm::Type *Tys[2] = { Ty, VTy };
6359     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6360     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
6361     return Builder.CreateTrunc(Ops[0], Int16Ty);
6362   }
6363   case NEON::BI__builtin_neon_vmaxv_s8: {
6364     Int = Intrinsic::aarch64_neon_smaxv;
6365     Ty = Int32Ty;
6366     VTy = llvm::VectorType::get(Int8Ty, 8);
6367     llvm::Type *Tys[2] = { Ty, VTy };
6368     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6369     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
6370     return Builder.CreateTrunc(Ops[0], Int8Ty);
6371   }
6372   case NEON::BI__builtin_neon_vmaxv_s16: {
6373     Int = Intrinsic::aarch64_neon_smaxv;
6374     Ty = Int32Ty;
6375     VTy = llvm::VectorType::get(Int16Ty, 4);
6376     llvm::Type *Tys[2] = { Ty, VTy };
6377     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6378     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
6379     return Builder.CreateTrunc(Ops[0], Int16Ty);
6380   }
6381   case NEON::BI__builtin_neon_vmaxvq_s8: {
6382     Int = Intrinsic::aarch64_neon_smaxv;
6383     Ty = Int32Ty;
6384     VTy = llvm::VectorType::get(Int8Ty, 16);
6385     llvm::Type *Tys[2] = { Ty, VTy };
6386     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6387     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
6388     return Builder.CreateTrunc(Ops[0], Int8Ty);
6389   }
6390   case NEON::BI__builtin_neon_vmaxvq_s16: {
6391     Int = Intrinsic::aarch64_neon_smaxv;
6392     Ty = Int32Ty;
6393     VTy = llvm::VectorType::get(Int16Ty, 8);
6394     llvm::Type *Tys[2] = { Ty, VTy };
6395     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6396     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
6397     return Builder.CreateTrunc(Ops[0], Int16Ty);
6398   }
6399   case NEON::BI__builtin_neon_vminv_u8: {
6400     Int = Intrinsic::aarch64_neon_uminv;
6401     Ty = Int32Ty;
6402     VTy = llvm::VectorType::get(Int8Ty, 8);
6403     llvm::Type *Tys[2] = { Ty, VTy };
6404     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6405     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
6406     return Builder.CreateTrunc(Ops[0], Int8Ty);
6407   }
6408   case NEON::BI__builtin_neon_vminv_u16: {
6409     Int = Intrinsic::aarch64_neon_uminv;
6410     Ty = Int32Ty;
6411     VTy = llvm::VectorType::get(Int16Ty, 4);
6412     llvm::Type *Tys[2] = { Ty, VTy };
6413     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6414     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
6415     return Builder.CreateTrunc(Ops[0], Int16Ty);
6416   }
6417   case NEON::BI__builtin_neon_vminvq_u8: {
6418     Int = Intrinsic::aarch64_neon_uminv;
6419     Ty = Int32Ty;
6420     VTy = llvm::VectorType::get(Int8Ty, 16);
6421     llvm::Type *Tys[2] = { Ty, VTy };
6422     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6423     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
6424     return Builder.CreateTrunc(Ops[0], Int8Ty);
6425   }
6426   case NEON::BI__builtin_neon_vminvq_u16: {
6427     Int = Intrinsic::aarch64_neon_uminv;
6428     Ty = Int32Ty;
6429     VTy = llvm::VectorType::get(Int16Ty, 8);
6430     llvm::Type *Tys[2] = { Ty, VTy };
6431     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6432     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
6433     return Builder.CreateTrunc(Ops[0], Int16Ty);
6434   }
6435   case NEON::BI__builtin_neon_vminv_s8: {
6436     Int = Intrinsic::aarch64_neon_sminv;
6437     Ty = Int32Ty;
6438     VTy = llvm::VectorType::get(Int8Ty, 8);
6439     llvm::Type *Tys[2] = { Ty, VTy };
6440     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6441     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
6442     return Builder.CreateTrunc(Ops[0], Int8Ty);
6443   }
6444   case NEON::BI__builtin_neon_vminv_s16: {
6445     Int = Intrinsic::aarch64_neon_sminv;
6446     Ty = Int32Ty;
6447     VTy = llvm::VectorType::get(Int16Ty, 4);
6448     llvm::Type *Tys[2] = { Ty, VTy };
6449     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6450     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
6451     return Builder.CreateTrunc(Ops[0], Int16Ty);
6452   }
6453   case NEON::BI__builtin_neon_vminvq_s8: {
6454     Int = Intrinsic::aarch64_neon_sminv;
6455     Ty = Int32Ty;
6456     VTy = llvm::VectorType::get(Int8Ty, 16);
6457     llvm::Type *Tys[2] = { Ty, VTy };
6458     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6459     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
6460     return Builder.CreateTrunc(Ops[0], Int8Ty);
6461   }
6462   case NEON::BI__builtin_neon_vminvq_s16: {
6463     Int = Intrinsic::aarch64_neon_sminv;
6464     Ty = Int32Ty;
6465     VTy = llvm::VectorType::get(Int16Ty, 8);
6466     llvm::Type *Tys[2] = { Ty, VTy };
6467     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6468     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
6469     return Builder.CreateTrunc(Ops[0], Int16Ty);
6470   }
6471   case NEON::BI__builtin_neon_vmul_n_f64: {
6472     Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
6473     Value *RHS = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), DoubleTy);
6474     return Builder.CreateFMul(Ops[0], RHS);
6475   }
6476   case NEON::BI__builtin_neon_vaddlv_u8: {
6477     Int = Intrinsic::aarch64_neon_uaddlv;
6478     Ty = Int32Ty;
6479     VTy = llvm::VectorType::get(Int8Ty, 8);
6480     llvm::Type *Tys[2] = { Ty, VTy };
6481     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6482     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
6483     return Builder.CreateTrunc(Ops[0], Int16Ty);
6484   }
6485   case NEON::BI__builtin_neon_vaddlv_u16: {
6486     Int = Intrinsic::aarch64_neon_uaddlv;
6487     Ty = Int32Ty;
6488     VTy = llvm::VectorType::get(Int16Ty, 4);
6489     llvm::Type *Tys[2] = { Ty, VTy };
6490     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6491     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
6492   }
6493   case NEON::BI__builtin_neon_vaddlvq_u8: {
6494     Int = Intrinsic::aarch64_neon_uaddlv;
6495     Ty = Int32Ty;
6496     VTy = llvm::VectorType::get(Int8Ty, 16);
6497     llvm::Type *Tys[2] = { Ty, VTy };
6498     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6499     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
6500     return Builder.CreateTrunc(Ops[0], Int16Ty);
6501   }
6502   case NEON::BI__builtin_neon_vaddlvq_u16: {
6503     Int = Intrinsic::aarch64_neon_uaddlv;
6504     Ty = Int32Ty;
6505     VTy = llvm::VectorType::get(Int16Ty, 8);
6506     llvm::Type *Tys[2] = { Ty, VTy };
6507     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6508     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
6509   }
6510   case NEON::BI__builtin_neon_vaddlv_s8: {
6511     Int = Intrinsic::aarch64_neon_saddlv;
6512     Ty = Int32Ty;
6513     VTy = llvm::VectorType::get(Int8Ty, 8);
6514     llvm::Type *Tys[2] = { Ty, VTy };
6515     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6516     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
6517     return Builder.CreateTrunc(Ops[0], Int16Ty);
6518   }
6519   case NEON::BI__builtin_neon_vaddlv_s16: {
6520     Int = Intrinsic::aarch64_neon_saddlv;
6521     Ty = Int32Ty;
6522     VTy = llvm::VectorType::get(Int16Ty, 4);
6523     llvm::Type *Tys[2] = { Ty, VTy };
6524     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6525     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
6526   }
6527   case NEON::BI__builtin_neon_vaddlvq_s8: {
6528     Int = Intrinsic::aarch64_neon_saddlv;
6529     Ty = Int32Ty;
6530     VTy = llvm::VectorType::get(Int8Ty, 16);
6531     llvm::Type *Tys[2] = { Ty, VTy };
6532     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6533     Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
6534     return Builder.CreateTrunc(Ops[0], Int16Ty);
6535   }
6536   case NEON::BI__builtin_neon_vaddlvq_s16: {
6537     Int = Intrinsic::aarch64_neon_saddlv;
6538     Ty = Int32Ty;
6539     VTy = llvm::VectorType::get(Int16Ty, 8);
6540     llvm::Type *Tys[2] = { Ty, VTy };
6541     Ops.push_back(EmitScalarExpr(E->getArg(0)));
6542     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
6543   }
6544   case NEON::BI__builtin_neon_vsri_n_v:
6545   case NEON::BI__builtin_neon_vsriq_n_v: {
6546     Int = Intrinsic::aarch64_neon_vsri;
6547     llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty);
6548     return EmitNeonCall(Intrin, Ops, "vsri_n");
6549   }
6550   case NEON::BI__builtin_neon_vsli_n_v:
6551   case NEON::BI__builtin_neon_vsliq_n_v: {
6552     Int = Intrinsic::aarch64_neon_vsli;
6553     llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty);
6554     return EmitNeonCall(Intrin, Ops, "vsli_n");
6555   }
6556   case NEON::BI__builtin_neon_vsra_n_v:
6557   case NEON::BI__builtin_neon_vsraq_n_v:
6558     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6559     Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n");
6560     return Builder.CreateAdd(Ops[0], Ops[1]);
6561   case NEON::BI__builtin_neon_vrsra_n_v:
6562   case NEON::BI__builtin_neon_vrsraq_n_v: {
6563     Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
6564     SmallVector<llvm::Value*,2> TmpOps;
6565     TmpOps.push_back(Ops[1]);
6566     TmpOps.push_back(Ops[2]);
6567     Function* F = CGM.getIntrinsic(Int, Ty);
6568     llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vrshr_n", 1, true);
6569     Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
6570     return Builder.CreateAdd(Ops[0], tmp);
6571   }
6572     // FIXME: Sharing loads & stores with 32-bit is complicated by the absence
6573     // of an Align parameter here.
6574   case NEON::BI__builtin_neon_vld1_x2_v:
6575   case NEON::BI__builtin_neon_vld1q_x2_v:
6576   case NEON::BI__builtin_neon_vld1_x3_v:
6577   case NEON::BI__builtin_neon_vld1q_x3_v:
6578   case NEON::BI__builtin_neon_vld1_x4_v:
6579   case NEON::BI__builtin_neon_vld1q_x4_v: {
6580     llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType());
6581     Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
6582     llvm::Type *Tys[2] = { VTy, PTy };
6583     unsigned Int;
6584     switch (BuiltinID) {
6585     case NEON::BI__builtin_neon_vld1_x2_v:
6586     case NEON::BI__builtin_neon_vld1q_x2_v:
6587       Int = Intrinsic::aarch64_neon_ld1x2;
6588       break;
6589     case NEON::BI__builtin_neon_vld1_x3_v:
6590     case NEON::BI__builtin_neon_vld1q_x3_v:
6591       Int = Intrinsic::aarch64_neon_ld1x3;
6592       break;
6593     case NEON::BI__builtin_neon_vld1_x4_v:
6594     case NEON::BI__builtin_neon_vld1q_x4_v:
6595       Int = Intrinsic::aarch64_neon_ld1x4;
6596       break;
6597     }
6598     Function *F = CGM.getIntrinsic(Int, Tys);
6599     Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN");
6600     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
6601     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6602     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6603   }
6604   case NEON::BI__builtin_neon_vst1_x2_v:
6605   case NEON::BI__builtin_neon_vst1q_x2_v:
6606   case NEON::BI__builtin_neon_vst1_x3_v:
6607   case NEON::BI__builtin_neon_vst1q_x3_v:
6608   case NEON::BI__builtin_neon_vst1_x4_v:
6609   case NEON::BI__builtin_neon_vst1q_x4_v: {
6610     llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType());
6611     llvm::Type *Tys[2] = { VTy, PTy };
6612     unsigned Int;
6613     switch (BuiltinID) {
6614     case NEON::BI__builtin_neon_vst1_x2_v:
6615     case NEON::BI__builtin_neon_vst1q_x2_v:
6616       Int = Intrinsic::aarch64_neon_st1x2;
6617       break;
6618     case NEON::BI__builtin_neon_vst1_x3_v:
6619     case NEON::BI__builtin_neon_vst1q_x3_v:
6620       Int = Intrinsic::aarch64_neon_st1x3;
6621       break;
6622     case NEON::BI__builtin_neon_vst1_x4_v:
6623     case NEON::BI__builtin_neon_vst1q_x4_v:
6624       Int = Intrinsic::aarch64_neon_st1x4;
6625       break;
6626     }
6627     std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
6628     return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "");
6629   }
6630   case NEON::BI__builtin_neon_vld1_v:
6631   case NEON::BI__builtin_neon_vld1q_v: {
6632     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
6633     auto Alignment = CharUnits::fromQuantity(
6634         BuiltinID == NEON::BI__builtin_neon_vld1_v ? 8 : 16);
6635     return Builder.CreateAlignedLoad(VTy, Ops[0], Alignment);
6636   }
6637   case NEON::BI__builtin_neon_vst1_v:
6638   case NEON::BI__builtin_neon_vst1q_v:
6639     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
6640     Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
6641     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6642   case NEON::BI__builtin_neon_vld1_lane_v:
6643   case NEON::BI__builtin_neon_vld1q_lane_v: {
6644     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6645     Ty = llvm::PointerType::getUnqual(VTy->getElementType());
6646     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6647     auto Alignment = CharUnits::fromQuantity(
6648         BuiltinID == NEON::BI__builtin_neon_vld1_lane_v ? 8 : 16);
6649     Ops[0] =
6650         Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], Alignment);
6651     return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane");
6652   }
6653   case NEON::BI__builtin_neon_vld1_dup_v:
6654   case NEON::BI__builtin_neon_vld1q_dup_v: {
6655     Value *V = UndefValue::get(Ty);
6656     Ty = llvm::PointerType::getUnqual(VTy->getElementType());
6657     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6658     auto Alignment = CharUnits::fromQuantity(
6659         BuiltinID == NEON::BI__builtin_neon_vld1_dup_v ? 8 : 16);
6660     Ops[0] =
6661         Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], Alignment);
6662     llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
6663     Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI);
6664     return EmitNeonSplat(Ops[0], CI);
6665   }
6666   case NEON::BI__builtin_neon_vst1_lane_v:
6667   case NEON::BI__builtin_neon_vst1q_lane_v:
6668     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6669     Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
6670     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
6671     return Builder.CreateDefaultAlignedStore(Ops[1],
6672                                              Builder.CreateBitCast(Ops[0], Ty));
6673   case NEON::BI__builtin_neon_vld2_v:
6674   case NEON::BI__builtin_neon_vld2q_v: {
6675     llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
6676     Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
6677     llvm::Type *Tys[2] = { VTy, PTy };
6678     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2, Tys);
6679     Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
6680     Ops[0] = Builder.CreateBitCast(Ops[0],
6681                 llvm::PointerType::getUnqual(Ops[1]->getType()));
6682     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6683   }
6684   case NEON::BI__builtin_neon_vld3_v:
6685   case NEON::BI__builtin_neon_vld3q_v: {
6686     llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
6687     Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
6688     llvm::Type *Tys[2] = { VTy, PTy };
6689     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3, Tys);
6690     Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
6691     Ops[0] = Builder.CreateBitCast(Ops[0],
6692                 llvm::PointerType::getUnqual(Ops[1]->getType()));
6693     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6694   }
6695   case NEON::BI__builtin_neon_vld4_v:
6696   case NEON::BI__builtin_neon_vld4q_v: {
6697     llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
6698     Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
6699     llvm::Type *Tys[2] = { VTy, PTy };
6700     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4, Tys);
6701     Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
6702     Ops[0] = Builder.CreateBitCast(Ops[0],
6703                 llvm::PointerType::getUnqual(Ops[1]->getType()));
6704     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6705   }
6706   case NEON::BI__builtin_neon_vld2_dup_v:
6707   case NEON::BI__builtin_neon_vld2q_dup_v: {
6708     llvm::Type *PTy =
6709       llvm::PointerType::getUnqual(VTy->getElementType());
6710     Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
6711     llvm::Type *Tys[2] = { VTy, PTy };
6712     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2r, Tys);
6713     Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
6714     Ops[0] = Builder.CreateBitCast(Ops[0],
6715                 llvm::PointerType::getUnqual(Ops[1]->getType()));
6716     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6717   }
6718   case NEON::BI__builtin_neon_vld3_dup_v:
6719   case NEON::BI__builtin_neon_vld3q_dup_v: {
6720     llvm::Type *PTy =
6721       llvm::PointerType::getUnqual(VTy->getElementType());
6722     Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
6723     llvm::Type *Tys[2] = { VTy, PTy };
6724     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3r, Tys);
6725     Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
6726     Ops[0] = Builder.CreateBitCast(Ops[0],
6727                 llvm::PointerType::getUnqual(Ops[1]->getType()));
6728     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6729   }
6730   case NEON::BI__builtin_neon_vld4_dup_v:
6731   case NEON::BI__builtin_neon_vld4q_dup_v: {
6732     llvm::Type *PTy =
6733       llvm::PointerType::getUnqual(VTy->getElementType());
6734     Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
6735     llvm::Type *Tys[2] = { VTy, PTy };
6736     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4r, Tys);
6737     Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
6738     Ops[0] = Builder.CreateBitCast(Ops[0],
6739                 llvm::PointerType::getUnqual(Ops[1]->getType()));
6740     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6741   }
6742   case NEON::BI__builtin_neon_vld2_lane_v:
6743   case NEON::BI__builtin_neon_vld2q_lane_v: {
6744     llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6745     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys);
6746     Ops.push_back(Ops[1]);
6747     Ops.erase(Ops.begin()+1);
6748     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6749     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6750     Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
6751     Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane");
6752     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
6753     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6754     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6755   }
6756   case NEON::BI__builtin_neon_vld3_lane_v:
6757   case NEON::BI__builtin_neon_vld3q_lane_v: {
6758     llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6759     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys);
6760     Ops.push_back(Ops[1]);
6761     Ops.erase(Ops.begin()+1);
6762     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6763     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6764     Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
6765     Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
6766     Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane");
6767     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
6768     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6769     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6770   }
6771   case NEON::BI__builtin_neon_vld4_lane_v:
6772   case NEON::BI__builtin_neon_vld4q_lane_v: {
6773     llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
6774     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys);
6775     Ops.push_back(Ops[1]);
6776     Ops.erase(Ops.begin()+1);
6777     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6778     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6779     Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
6780     Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
6781     Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty);
6782     Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld4_lane");
6783     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
6784     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6785     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
6786   }
6787   case NEON::BI__builtin_neon_vst2_v:
6788   case NEON::BI__builtin_neon_vst2q_v: {
6789     Ops.push_back(Ops[0]);
6790     Ops.erase(Ops.begin());
6791     llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
6792     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2, Tys),
6793                         Ops, "");
6794   }
6795   case NEON::BI__builtin_neon_vst2_lane_v:
6796   case NEON::BI__builtin_neon_vst2q_lane_v: {
6797     Ops.push_back(Ops[0]);
6798     Ops.erase(Ops.begin());
6799     Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty);
6800     llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
6801     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys),
6802                         Ops, "");
6803   }
6804   case NEON::BI__builtin_neon_vst3_v:
6805   case NEON::BI__builtin_neon_vst3q_v: {
6806     Ops.push_back(Ops[0]);
6807     Ops.erase(Ops.begin());
6808     llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
6809     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3, Tys),
6810                         Ops, "");
6811   }
6812   case NEON::BI__builtin_neon_vst3_lane_v:
6813   case NEON::BI__builtin_neon_vst3q_lane_v: {
6814     Ops.push_back(Ops[0]);
6815     Ops.erase(Ops.begin());
6816     Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
6817     llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
6818     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys),
6819                         Ops, "");
6820   }
6821   case NEON::BI__builtin_neon_vst4_v:
6822   case NEON::BI__builtin_neon_vst4q_v: {
6823     Ops.push_back(Ops[0]);
6824     Ops.erase(Ops.begin());
6825     llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
6826     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4, Tys),
6827                         Ops, "");
6828   }
6829   case NEON::BI__builtin_neon_vst4_lane_v:
6830   case NEON::BI__builtin_neon_vst4q_lane_v: {
6831     Ops.push_back(Ops[0]);
6832     Ops.erase(Ops.begin());
6833     Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
6834     llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
6835     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys),
6836                         Ops, "");
6837   }
6838   case NEON::BI__builtin_neon_vtrn_v:
6839   case NEON::BI__builtin_neon_vtrnq_v: {
6840     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
6841     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6842     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6843     Value *SV = nullptr;
6844
6845     for (unsigned vi = 0; vi != 2; ++vi) {
6846       SmallVector<uint32_t, 16> Indices;
6847       for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
6848         Indices.push_back(i+vi);
6849         Indices.push_back(i+e+vi);
6850       }
6851       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
6852       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn");
6853       SV = Builder.CreateDefaultAlignedStore(SV, Addr);
6854     }
6855     return SV;
6856   }
6857   case NEON::BI__builtin_neon_vuzp_v:
6858   case NEON::BI__builtin_neon_vuzpq_v: {
6859     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
6860     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6861     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6862     Value *SV = nullptr;
6863
6864     for (unsigned vi = 0; vi != 2; ++vi) {
6865       SmallVector<uint32_t, 16> Indices;
6866       for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
6867         Indices.push_back(2*i+vi);
6868
6869       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
6870       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp");
6871       SV = Builder.CreateDefaultAlignedStore(SV, Addr);
6872     }
6873     return SV;
6874   }
6875   case NEON::BI__builtin_neon_vzip_v:
6876   case NEON::BI__builtin_neon_vzipq_v: {
6877     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
6878     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6879     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
6880     Value *SV = nullptr;
6881
6882     for (unsigned vi = 0; vi != 2; ++vi) {
6883       SmallVector<uint32_t, 16> Indices;
6884       for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
6885         Indices.push_back((i + vi*e) >> 1);
6886         Indices.push_back(((i + vi*e) >> 1)+e);
6887       }
6888       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
6889       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip");
6890       SV = Builder.CreateDefaultAlignedStore(SV, Addr);
6891     }
6892     return SV;
6893   }
6894   case NEON::BI__builtin_neon_vqtbl1q_v: {
6895     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl1, Ty),
6896                         Ops, "vtbl1");
6897   }
6898   case NEON::BI__builtin_neon_vqtbl2q_v: {
6899     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl2, Ty),
6900                         Ops, "vtbl2");
6901   }
6902   case NEON::BI__builtin_neon_vqtbl3q_v: {
6903     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl3, Ty),
6904                         Ops, "vtbl3");
6905   }
6906   case NEON::BI__builtin_neon_vqtbl4q_v: {
6907     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl4, Ty),
6908                         Ops, "vtbl4");
6909   }
6910   case NEON::BI__builtin_neon_vqtbx1q_v: {
6911     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx1, Ty),
6912                         Ops, "vtbx1");
6913   }
6914   case NEON::BI__builtin_neon_vqtbx2q_v: {
6915     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx2, Ty),
6916                         Ops, "vtbx2");
6917   }
6918   case NEON::BI__builtin_neon_vqtbx3q_v: {
6919     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx3, Ty),
6920                         Ops, "vtbx3");
6921   }
6922   case NEON::BI__builtin_neon_vqtbx4q_v: {
6923     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx4, Ty),
6924                         Ops, "vtbx4");
6925   }
6926   case NEON::BI__builtin_neon_vsqadd_v:
6927   case NEON::BI__builtin_neon_vsqaddq_v: {
6928     Int = Intrinsic::aarch64_neon_usqadd;
6929     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd");
6930   }
6931   case NEON::BI__builtin_neon_vuqadd_v:
6932   case NEON::BI__builtin_neon_vuqaddq_v: {
6933     Int = Intrinsic::aarch64_neon_suqadd;
6934     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd");
6935   }
6936   }
6937 }
6938
6939 llvm::Value *CodeGenFunction::
6940 BuildVector(ArrayRef<llvm::Value*> Ops) {
6941   assert((Ops.size() & (Ops.size() - 1)) == 0 &&
6942          "Not a power-of-two sized vector!");
6943   bool AllConstants = true;
6944   for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
6945     AllConstants &= isa<Constant>(Ops[i]);
6946
6947   // If this is a constant vector, create a ConstantVector.
6948   if (AllConstants) {
6949     SmallVector<llvm::Constant*, 16> CstOps;
6950     for (unsigned i = 0, e = Ops.size(); i != e; ++i)
6951       CstOps.push_back(cast<Constant>(Ops[i]));
6952     return llvm::ConstantVector::get(CstOps);
6953   }
6954
6955   // Otherwise, insertelement the values to build the vector.
6956   Value *Result =
6957     llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size()));
6958
6959   for (unsigned i = 0, e = Ops.size(); i != e; ++i)
6960     Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i));
6961
6962   return Result;
6963 }
6964
6965 // Convert the mask from an integer type to a vector of i1.
6966 static Value *getMaskVecValue(CodeGenFunction &CGF, Value *Mask,
6967                               unsigned NumElts) {
6968
6969   llvm::VectorType *MaskTy = llvm::VectorType::get(CGF.Builder.getInt1Ty(),
6970                          cast<IntegerType>(Mask->getType())->getBitWidth());
6971   Value *MaskVec = CGF.Builder.CreateBitCast(Mask, MaskTy);
6972
6973   // If we have less than 8 elements, then the starting mask was an i8 and
6974   // we need to extract down to the right number of elements.
6975   if (NumElts < 8) {
6976     uint32_t Indices[4];
6977     for (unsigned i = 0; i != NumElts; ++i)
6978       Indices[i] = i;
6979     MaskVec = CGF.Builder.CreateShuffleVector(MaskVec, MaskVec,
6980                                              makeArrayRef(Indices, NumElts),
6981                                              "extract");
6982   }
6983   return MaskVec;
6984 }
6985
6986 static Value *EmitX86MaskedStore(CodeGenFunction &CGF,
6987                                  SmallVectorImpl<Value *> &Ops,
6988                                  unsigned Align) {
6989   // Cast the pointer to right type.
6990   Ops[0] = CGF.Builder.CreateBitCast(Ops[0],
6991                                llvm::PointerType::getUnqual(Ops[1]->getType()));
6992
6993   // If the mask is all ones just emit a regular store.
6994   if (const auto *C = dyn_cast<Constant>(Ops[2]))
6995     if (C->isAllOnesValue())
6996       return CGF.Builder.CreateAlignedStore(Ops[1], Ops[0], Align);
6997
6998   Value *MaskVec = getMaskVecValue(CGF, Ops[2],
6999                                    Ops[1]->getType()->getVectorNumElements());
7000
7001   return CGF.Builder.CreateMaskedStore(Ops[1], Ops[0], Align, MaskVec);
7002 }
7003
7004 static Value *EmitX86MaskedLoad(CodeGenFunction &CGF,
7005                                 SmallVectorImpl<Value *> &Ops, unsigned Align) {
7006   // Cast the pointer to right type.
7007   Ops[0] = CGF.Builder.CreateBitCast(Ops[0],
7008                                llvm::PointerType::getUnqual(Ops[1]->getType()));
7009
7010   // If the mask is all ones just emit a regular store.
7011   if (const auto *C = dyn_cast<Constant>(Ops[2]))
7012     if (C->isAllOnesValue())
7013       return CGF.Builder.CreateAlignedLoad(Ops[0], Align);
7014
7015   Value *MaskVec = getMaskVecValue(CGF, Ops[2],
7016                                    Ops[1]->getType()->getVectorNumElements());
7017
7018   return CGF.Builder.CreateMaskedLoad(Ops[0], Align, MaskVec, Ops[1]);
7019 }
7020
7021 static Value *EmitX86SubVectorBroadcast(CodeGenFunction &CGF,
7022                                         SmallVectorImpl<Value *> &Ops,
7023                                         llvm::Type *DstTy,
7024                                         unsigned SrcSizeInBits,
7025                                         unsigned Align) {
7026   // Load the subvector.
7027   Ops[0] = CGF.Builder.CreateAlignedLoad(Ops[0], Align);
7028
7029   // Create broadcast mask.
7030   unsigned NumDstElts = DstTy->getVectorNumElements();
7031   unsigned NumSrcElts = SrcSizeInBits / DstTy->getScalarSizeInBits();
7032
7033   SmallVector<uint32_t, 8> Mask;
7034   for (unsigned i = 0; i != NumDstElts; i += NumSrcElts)
7035     for (unsigned j = 0; j != NumSrcElts; ++j)
7036       Mask.push_back(j);
7037
7038   return CGF.Builder.CreateShuffleVector(Ops[0], Ops[0], Mask, "subvecbcst");
7039 }
7040
7041 static Value *EmitX86Select(CodeGenFunction &CGF,
7042                             Value *Mask, Value *Op0, Value *Op1) {
7043
7044   // If the mask is all ones just return first argument.
7045   if (const auto *C = dyn_cast<Constant>(Mask))
7046     if (C->isAllOnesValue())
7047       return Op0;
7048
7049   Mask = getMaskVecValue(CGF, Mask, Op0->getType()->getVectorNumElements());
7050
7051   return CGF.Builder.CreateSelect(Mask, Op0, Op1);
7052 }
7053
7054 static Value *EmitX86MaskedCompare(CodeGenFunction &CGF, unsigned CC,
7055                                    bool Signed, SmallVectorImpl<Value *> &Ops) {
7056   unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
7057   Value *Cmp;
7058
7059   if (CC == 3) {
7060     Cmp = Constant::getNullValue(
7061                        llvm::VectorType::get(CGF.Builder.getInt1Ty(), NumElts));
7062   } else if (CC == 7) {
7063     Cmp = Constant::getAllOnesValue(
7064                        llvm::VectorType::get(CGF.Builder.getInt1Ty(), NumElts));
7065   } else {
7066     ICmpInst::Predicate Pred;
7067     switch (CC) {
7068     default: llvm_unreachable("Unknown condition code");
7069     case 0: Pred = ICmpInst::ICMP_EQ;  break;
7070     case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
7071     case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
7072     case 4: Pred = ICmpInst::ICMP_NE;  break;
7073     case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
7074     case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
7075     }
7076     Cmp = CGF.Builder.CreateICmp(Pred, Ops[0], Ops[1]);
7077   }
7078
7079   const auto *C = dyn_cast<Constant>(Ops.back());
7080   if (!C || !C->isAllOnesValue())
7081     Cmp = CGF.Builder.CreateAnd(Cmp, getMaskVecValue(CGF, Ops.back(), NumElts));
7082
7083   if (NumElts < 8) {
7084     uint32_t Indices[8];
7085     for (unsigned i = 0; i != NumElts; ++i)
7086       Indices[i] = i;
7087     for (unsigned i = NumElts; i != 8; ++i)
7088       Indices[i] = i % NumElts + NumElts;
7089     Cmp = CGF.Builder.CreateShuffleVector(
7090         Cmp, llvm::Constant::getNullValue(Cmp->getType()), Indices);
7091   }
7092   return CGF.Builder.CreateBitCast(Cmp,
7093                                    IntegerType::get(CGF.getLLVMContext(),
7094                                                     std::max(NumElts, 8U)));
7095 }
7096
7097 static Value *EmitX86MinMax(CodeGenFunction &CGF, ICmpInst::Predicate Pred,
7098                             ArrayRef<Value *> Ops) {
7099   Value *Cmp = CGF.Builder.CreateICmp(Pred, Ops[0], Ops[1]);
7100   Value *Res = CGF.Builder.CreateSelect(Cmp, Ops[0], Ops[1]);
7101
7102   if (Ops.size() == 2)
7103     return Res;
7104
7105   assert(Ops.size() == 4);
7106   return EmitX86Select(CGF, Ops[3], Res, Ops[2]);
7107 }
7108
7109 Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
7110                                            const CallExpr *E) {
7111   if (BuiltinID == X86::BI__builtin_ms_va_start ||
7112       BuiltinID == X86::BI__builtin_ms_va_end)
7113     return EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).getPointer(),
7114                           BuiltinID == X86::BI__builtin_ms_va_start);
7115   if (BuiltinID == X86::BI__builtin_ms_va_copy) {
7116     // Lower this manually. We can't reliably determine whether or not any
7117     // given va_copy() is for a Win64 va_list from the calling convention
7118     // alone, because it's legal to do this from a System V ABI function.
7119     // With opaque pointer types, we won't have enough information in LLVM
7120     // IR to determine this from the argument types, either. Best to do it
7121     // now, while we have enough information.
7122     Address DestAddr = EmitMSVAListRef(E->getArg(0));
7123     Address SrcAddr = EmitMSVAListRef(E->getArg(1));
7124
7125     llvm::Type *BPP = Int8PtrPtrTy;
7126
7127     DestAddr = Address(Builder.CreateBitCast(DestAddr.getPointer(), BPP, "cp"),
7128                        DestAddr.getAlignment());
7129     SrcAddr = Address(Builder.CreateBitCast(SrcAddr.getPointer(), BPP, "ap"),
7130                       SrcAddr.getAlignment());
7131
7132     Value *ArgPtr = Builder.CreateLoad(SrcAddr, "ap.val");
7133     return Builder.CreateStore(ArgPtr, DestAddr);
7134   }
7135
7136   SmallVector<Value*, 4> Ops;
7137
7138   // Find out if any arguments are required to be integer constant expressions.
7139   unsigned ICEArguments = 0;
7140   ASTContext::GetBuiltinTypeError Error;
7141   getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
7142   assert(Error == ASTContext::GE_None && "Should not codegen an error");
7143
7144   for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) {
7145     // If this is a normal argument, just emit it as a scalar.
7146     if ((ICEArguments & (1 << i)) == 0) {
7147       Ops.push_back(EmitScalarExpr(E->getArg(i)));
7148       continue;
7149     }
7150
7151     // If this is required to be a constant, constant fold it so that we know
7152     // that the generated intrinsic gets a ConstantInt.
7153     llvm::APSInt Result;
7154     bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext());
7155     assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst;
7156     Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
7157   }
7158
7159   // These exist so that the builtin that takes an immediate can be bounds
7160   // checked by clang to avoid passing bad immediates to the backend. Since
7161   // AVX has a larger immediate than SSE we would need separate builtins to
7162   // do the different bounds checking. Rather than create a clang specific
7163   // SSE only builtin, this implements eight separate builtins to match gcc
7164   // implementation.
7165   auto getCmpIntrinsicCall = [this, &Ops](Intrinsic::ID ID, unsigned Imm) {
7166     Ops.push_back(llvm::ConstantInt::get(Int8Ty, Imm));
7167     llvm::Function *F = CGM.getIntrinsic(ID);
7168     return Builder.CreateCall(F, Ops);
7169   };
7170
7171   // For the vector forms of FP comparisons, translate the builtins directly to
7172   // IR.
7173   // TODO: The builtins could be removed if the SSE header files used vector
7174   // extension comparisons directly (vector ordered/unordered may need
7175   // additional support via __builtin_isnan()).
7176   auto getVectorFCmpIR = [this, &Ops](CmpInst::Predicate Pred) {
7177     Value *Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
7178     llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType());
7179     llvm::VectorType *IntVecTy = llvm::VectorType::getInteger(FPVecTy);
7180     Value *Sext = Builder.CreateSExt(Cmp, IntVecTy);
7181     return Builder.CreateBitCast(Sext, FPVecTy);
7182   };
7183
7184   switch (BuiltinID) {
7185   default: return nullptr;
7186   case X86::BI__builtin_cpu_supports: {
7187     const Expr *FeatureExpr = E->getArg(0)->IgnoreParenCasts();
7188     StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
7189
7190     // TODO: When/if this becomes more than x86 specific then use a TargetInfo
7191     // based mapping.
7192     // Processor features and mapping to processor feature value.
7193     enum X86Features {
7194       CMOV = 0,
7195       MMX,
7196       POPCNT,
7197       SSE,
7198       SSE2,
7199       SSE3,
7200       SSSE3,
7201       SSE4_1,
7202       SSE4_2,
7203       AVX,
7204       AVX2,
7205       SSE4_A,
7206       FMA4,
7207       XOP,
7208       FMA,
7209       AVX512F,
7210       BMI,
7211       BMI2,
7212       AES,
7213       PCLMUL,
7214       AVX512VL,
7215       AVX512BW,
7216       AVX512DQ,
7217       AVX512CD,
7218       AVX512ER,
7219       AVX512PF,
7220       AVX512VBMI,
7221       AVX512IFMA,
7222       MAX
7223     };
7224
7225     X86Features Feature = StringSwitch<X86Features>(FeatureStr)
7226                               .Case("cmov", X86Features::CMOV)
7227                               .Case("mmx", X86Features::MMX)
7228                               .Case("popcnt", X86Features::POPCNT)
7229                               .Case("sse", X86Features::SSE)
7230                               .Case("sse2", X86Features::SSE2)
7231                               .Case("sse3", X86Features::SSE3)
7232                               .Case("ssse3", X86Features::SSSE3)
7233                               .Case("sse4.1", X86Features::SSE4_1)
7234                               .Case("sse4.2", X86Features::SSE4_2)
7235                               .Case("avx", X86Features::AVX)
7236                               .Case("avx2", X86Features::AVX2)
7237                               .Case("sse4a", X86Features::SSE4_A)
7238                               .Case("fma4", X86Features::FMA4)
7239                               .Case("xop", X86Features::XOP)
7240                               .Case("fma", X86Features::FMA)
7241                               .Case("avx512f", X86Features::AVX512F)
7242                               .Case("bmi", X86Features::BMI)
7243                               .Case("bmi2", X86Features::BMI2)
7244                               .Case("aes", X86Features::AES)
7245                               .Case("pclmul", X86Features::PCLMUL)
7246                               .Case("avx512vl", X86Features::AVX512VL)
7247                               .Case("avx512bw", X86Features::AVX512BW)
7248                               .Case("avx512dq", X86Features::AVX512DQ)
7249                               .Case("avx512cd", X86Features::AVX512CD)
7250                               .Case("avx512er", X86Features::AVX512ER)
7251                               .Case("avx512pf", X86Features::AVX512PF)
7252                               .Case("avx512vbmi", X86Features::AVX512VBMI)
7253                               .Case("avx512ifma", X86Features::AVX512IFMA)
7254                               .Default(X86Features::MAX);
7255     assert(Feature != X86Features::MAX && "Invalid feature!");
7256
7257     // Matching the struct layout from the compiler-rt/libgcc structure that is
7258     // filled in:
7259     // unsigned int __cpu_vendor;
7260     // unsigned int __cpu_type;
7261     // unsigned int __cpu_subtype;
7262     // unsigned int __cpu_features[1];
7263     llvm::Type *STy = llvm::StructType::get(
7264         Int32Ty, Int32Ty, Int32Ty, llvm::ArrayType::get(Int32Ty, 1), nullptr);
7265
7266     // Grab the global __cpu_model.
7267     llvm::Constant *CpuModel = CGM.CreateRuntimeVariable(STy, "__cpu_model");
7268
7269     // Grab the first (0th) element from the field __cpu_features off of the
7270     // global in the struct STy.
7271     Value *Idxs[] = {
7272       ConstantInt::get(Int32Ty, 0),
7273       ConstantInt::get(Int32Ty, 3),
7274       ConstantInt::get(Int32Ty, 0)
7275     };
7276     Value *CpuFeatures = Builder.CreateGEP(STy, CpuModel, Idxs);
7277     Value *Features = Builder.CreateAlignedLoad(CpuFeatures,
7278                                                 CharUnits::fromQuantity(4));
7279
7280     // Check the value of the bit corresponding to the feature requested.
7281     Value *Bitset = Builder.CreateAnd(
7282         Features, llvm::ConstantInt::get(Int32Ty, 1ULL << Feature));
7283     return Builder.CreateICmpNE(Bitset, llvm::ConstantInt::get(Int32Ty, 0));
7284   }
7285   case X86::BI_mm_prefetch: {
7286     Value *Address = Ops[0];
7287     Value *RW = ConstantInt::get(Int32Ty, 0);
7288     Value *Locality = Ops[1];
7289     Value *Data = ConstantInt::get(Int32Ty, 1);
7290     Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
7291     return Builder.CreateCall(F, {Address, RW, Locality, Data});
7292   }
7293   case X86::BI_mm_clflush: {
7294     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_clflush),
7295                               Ops[0]);
7296   }
7297   case X86::BI_mm_lfence: {
7298     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_lfence));
7299   }
7300   case X86::BI_mm_mfence: {
7301     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_mfence));
7302   }
7303   case X86::BI_mm_sfence: {
7304     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_sfence));
7305   }
7306   case X86::BI_mm_pause: {
7307     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_pause));
7308   }
7309   case X86::BI__rdtsc: {
7310     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_rdtsc));
7311   }
7312   case X86::BI__builtin_ia32_undef128:
7313   case X86::BI__builtin_ia32_undef256:
7314   case X86::BI__builtin_ia32_undef512:
7315     return UndefValue::get(ConvertType(E->getType()));
7316   case X86::BI__builtin_ia32_vec_init_v8qi:
7317   case X86::BI__builtin_ia32_vec_init_v4hi:
7318   case X86::BI__builtin_ia32_vec_init_v2si:
7319     return Builder.CreateBitCast(BuildVector(Ops),
7320                                  llvm::Type::getX86_MMXTy(getLLVMContext()));
7321   case X86::BI__builtin_ia32_vec_ext_v2si:
7322     return Builder.CreateExtractElement(Ops[0],
7323                                   llvm::ConstantInt::get(Ops[1]->getType(), 0));
7324   case X86::BI_mm_setcsr:
7325   case X86::BI__builtin_ia32_ldmxcsr: {
7326     Address Tmp = CreateMemTemp(E->getArg(0)->getType());
7327     Builder.CreateStore(Ops[0], Tmp);
7328     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr),
7329                           Builder.CreateBitCast(Tmp.getPointer(), Int8PtrTy));
7330   }
7331   case X86::BI_mm_getcsr:
7332   case X86::BI__builtin_ia32_stmxcsr: {
7333     Address Tmp = CreateMemTemp(E->getType());
7334     Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr),
7335                        Builder.CreateBitCast(Tmp.getPointer(), Int8PtrTy));
7336     return Builder.CreateLoad(Tmp, "stmxcsr");
7337   }
7338   case X86::BI__builtin_ia32_xsave:
7339   case X86::BI__builtin_ia32_xsave64:
7340   case X86::BI__builtin_ia32_xrstor:
7341   case X86::BI__builtin_ia32_xrstor64:
7342   case X86::BI__builtin_ia32_xsaveopt:
7343   case X86::BI__builtin_ia32_xsaveopt64:
7344   case X86::BI__builtin_ia32_xrstors:
7345   case X86::BI__builtin_ia32_xrstors64:
7346   case X86::BI__builtin_ia32_xsavec:
7347   case X86::BI__builtin_ia32_xsavec64:
7348   case X86::BI__builtin_ia32_xsaves:
7349   case X86::BI__builtin_ia32_xsaves64: {
7350     Intrinsic::ID ID;
7351 #define INTRINSIC_X86_XSAVE_ID(NAME) \
7352     case X86::BI__builtin_ia32_##NAME: \
7353       ID = Intrinsic::x86_##NAME; \
7354       break
7355     switch (BuiltinID) {
7356     default: llvm_unreachable("Unsupported intrinsic!");
7357     INTRINSIC_X86_XSAVE_ID(xsave);
7358     INTRINSIC_X86_XSAVE_ID(xsave64);
7359     INTRINSIC_X86_XSAVE_ID(xrstor);
7360     INTRINSIC_X86_XSAVE_ID(xrstor64);
7361     INTRINSIC_X86_XSAVE_ID(xsaveopt);
7362     INTRINSIC_X86_XSAVE_ID(xsaveopt64);
7363     INTRINSIC_X86_XSAVE_ID(xrstors);
7364     INTRINSIC_X86_XSAVE_ID(xrstors64);
7365     INTRINSIC_X86_XSAVE_ID(xsavec);
7366     INTRINSIC_X86_XSAVE_ID(xsavec64);
7367     INTRINSIC_X86_XSAVE_ID(xsaves);
7368     INTRINSIC_X86_XSAVE_ID(xsaves64);
7369     }
7370 #undef INTRINSIC_X86_XSAVE_ID
7371     Value *Mhi = Builder.CreateTrunc(
7372       Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, 32)), Int32Ty);
7373     Value *Mlo = Builder.CreateTrunc(Ops[1], Int32Ty);
7374     Ops[1] = Mhi;
7375     Ops.push_back(Mlo);
7376     return Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
7377   }
7378   case X86::BI__builtin_ia32_storedqudi128_mask:
7379   case X86::BI__builtin_ia32_storedqusi128_mask:
7380   case X86::BI__builtin_ia32_storedquhi128_mask:
7381   case X86::BI__builtin_ia32_storedquqi128_mask:
7382   case X86::BI__builtin_ia32_storeupd128_mask:
7383   case X86::BI__builtin_ia32_storeups128_mask:
7384   case X86::BI__builtin_ia32_storedqudi256_mask:
7385   case X86::BI__builtin_ia32_storedqusi256_mask:
7386   case X86::BI__builtin_ia32_storedquhi256_mask:
7387   case X86::BI__builtin_ia32_storedquqi256_mask:
7388   case X86::BI__builtin_ia32_storeupd256_mask:
7389   case X86::BI__builtin_ia32_storeups256_mask:
7390   case X86::BI__builtin_ia32_storedqudi512_mask:
7391   case X86::BI__builtin_ia32_storedqusi512_mask:
7392   case X86::BI__builtin_ia32_storedquhi512_mask:
7393   case X86::BI__builtin_ia32_storedquqi512_mask:
7394   case X86::BI__builtin_ia32_storeupd512_mask:
7395   case X86::BI__builtin_ia32_storeups512_mask:
7396     return EmitX86MaskedStore(*this, Ops, 1);
7397
7398   case X86::BI__builtin_ia32_storess128_mask:
7399   case X86::BI__builtin_ia32_storesd128_mask: {
7400     return EmitX86MaskedStore(*this, Ops, 16);
7401   }
7402   case X86::BI__builtin_ia32_movdqa32store128_mask:
7403   case X86::BI__builtin_ia32_movdqa64store128_mask:
7404   case X86::BI__builtin_ia32_storeaps128_mask:
7405   case X86::BI__builtin_ia32_storeapd128_mask:
7406   case X86::BI__builtin_ia32_movdqa32store256_mask:
7407   case X86::BI__builtin_ia32_movdqa64store256_mask:
7408   case X86::BI__builtin_ia32_storeaps256_mask:
7409   case X86::BI__builtin_ia32_storeapd256_mask:
7410   case X86::BI__builtin_ia32_movdqa32store512_mask:
7411   case X86::BI__builtin_ia32_movdqa64store512_mask:
7412   case X86::BI__builtin_ia32_storeaps512_mask:
7413   case X86::BI__builtin_ia32_storeapd512_mask: {
7414     unsigned Align =
7415       getContext().getTypeAlignInChars(E->getArg(1)->getType()).getQuantity();
7416     return EmitX86MaskedStore(*this, Ops, Align);
7417   }
7418   case X86::BI__builtin_ia32_loadups128_mask:
7419   case X86::BI__builtin_ia32_loadups256_mask:
7420   case X86::BI__builtin_ia32_loadups512_mask:
7421   case X86::BI__builtin_ia32_loadupd128_mask:
7422   case X86::BI__builtin_ia32_loadupd256_mask:
7423   case X86::BI__builtin_ia32_loadupd512_mask:
7424   case X86::BI__builtin_ia32_loaddquqi128_mask:
7425   case X86::BI__builtin_ia32_loaddquqi256_mask:
7426   case X86::BI__builtin_ia32_loaddquqi512_mask:
7427   case X86::BI__builtin_ia32_loaddquhi128_mask:
7428   case X86::BI__builtin_ia32_loaddquhi256_mask:
7429   case X86::BI__builtin_ia32_loaddquhi512_mask:
7430   case X86::BI__builtin_ia32_loaddqusi128_mask:
7431   case X86::BI__builtin_ia32_loaddqusi256_mask:
7432   case X86::BI__builtin_ia32_loaddqusi512_mask:
7433   case X86::BI__builtin_ia32_loaddqudi128_mask:
7434   case X86::BI__builtin_ia32_loaddqudi256_mask:
7435   case X86::BI__builtin_ia32_loaddqudi512_mask:
7436     return EmitX86MaskedLoad(*this, Ops, 1);
7437
7438   case X86::BI__builtin_ia32_loadss128_mask:
7439   case X86::BI__builtin_ia32_loadsd128_mask:
7440     return EmitX86MaskedLoad(*this, Ops, 16);
7441
7442   case X86::BI__builtin_ia32_loadaps128_mask:
7443   case X86::BI__builtin_ia32_loadaps256_mask:
7444   case X86::BI__builtin_ia32_loadaps512_mask:
7445   case X86::BI__builtin_ia32_loadapd128_mask:
7446   case X86::BI__builtin_ia32_loadapd256_mask:
7447   case X86::BI__builtin_ia32_loadapd512_mask:
7448   case X86::BI__builtin_ia32_movdqa32load128_mask:
7449   case X86::BI__builtin_ia32_movdqa32load256_mask:
7450   case X86::BI__builtin_ia32_movdqa32load512_mask:
7451   case X86::BI__builtin_ia32_movdqa64load128_mask:
7452   case X86::BI__builtin_ia32_movdqa64load256_mask:
7453   case X86::BI__builtin_ia32_movdqa64load512_mask: {
7454     unsigned Align =
7455       getContext().getTypeAlignInChars(E->getArg(1)->getType()).getQuantity();
7456     return EmitX86MaskedLoad(*this, Ops, Align);
7457   }
7458
7459   case X86::BI__builtin_ia32_vbroadcastf128_pd256:
7460   case X86::BI__builtin_ia32_vbroadcastf128_ps256: {
7461     llvm::Type *DstTy = ConvertType(E->getType());
7462     return EmitX86SubVectorBroadcast(*this, Ops, DstTy, 128, 1);
7463   }
7464
7465   case X86::BI__builtin_ia32_storehps:
7466   case X86::BI__builtin_ia32_storelps: {
7467     llvm::Type *PtrTy = llvm::PointerType::getUnqual(Int64Ty);
7468     llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2);
7469
7470     // cast val v2i64
7471     Ops[1] = Builder.CreateBitCast(Ops[1], VecTy, "cast");
7472
7473     // extract (0, 1)
7474     unsigned Index = BuiltinID == X86::BI__builtin_ia32_storelps ? 0 : 1;
7475     llvm::Value *Idx = llvm::ConstantInt::get(SizeTy, Index);
7476     Ops[1] = Builder.CreateExtractElement(Ops[1], Idx, "extract");
7477
7478     // cast pointer to i64 & store
7479     Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy);
7480     return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7481   }
7482   case X86::BI__builtin_ia32_palignr128:
7483   case X86::BI__builtin_ia32_palignr256:
7484   case X86::BI__builtin_ia32_palignr512_mask: {
7485     unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
7486
7487     unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
7488     assert(NumElts % 16 == 0);
7489
7490     // If palignr is shifting the pair of vectors more than the size of two
7491     // lanes, emit zero.
7492     if (ShiftVal >= 32)
7493       return llvm::Constant::getNullValue(ConvertType(E->getType()));
7494
7495     // If palignr is shifting the pair of input vectors more than one lane,
7496     // but less than two lanes, convert to shifting in zeroes.
7497     if (ShiftVal > 16) {
7498       ShiftVal -= 16;
7499       Ops[1] = Ops[0];
7500       Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType());
7501     }
7502
7503     uint32_t Indices[64];
7504     // 256-bit palignr operates on 128-bit lanes so we need to handle that
7505     for (unsigned l = 0; l != NumElts; l += 16) {
7506       for (unsigned i = 0; i != 16; ++i) {
7507         unsigned Idx = ShiftVal + i;
7508         if (Idx >= 16)
7509           Idx += NumElts - 16; // End of lane, switch operand.
7510         Indices[l + i] = Idx + l;
7511       }
7512     }
7513
7514     Value *Align = Builder.CreateShuffleVector(Ops[1], Ops[0],
7515                                                makeArrayRef(Indices, NumElts),
7516                                                "palignr");
7517
7518     // If this isn't a masked builtin, just return the align operation.
7519     if (Ops.size() == 3)
7520       return Align;
7521
7522     return EmitX86Select(*this, Ops[4], Align, Ops[3]);
7523   }
7524
7525   case X86::BI__builtin_ia32_movnti:
7526   case X86::BI__builtin_ia32_movnti64:
7527   case X86::BI__builtin_ia32_movntsd:
7528   case X86::BI__builtin_ia32_movntss: {
7529     llvm::MDNode *Node = llvm::MDNode::get(
7530         getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
7531
7532     Value *Ptr = Ops[0];
7533     Value *Src = Ops[1];
7534
7535     // Extract the 0'th element of the source vector.
7536     if (BuiltinID == X86::BI__builtin_ia32_movntsd ||
7537         BuiltinID == X86::BI__builtin_ia32_movntss)
7538       Src = Builder.CreateExtractElement(Src, (uint64_t)0, "extract");
7539
7540     // Convert the type of the pointer to a pointer to the stored type.
7541     Value *BC = Builder.CreateBitCast(
7542         Ptr, llvm::PointerType::getUnqual(Src->getType()), "cast");
7543
7544     // Unaligned nontemporal store of the scalar value.
7545     StoreInst *SI = Builder.CreateDefaultAlignedStore(Src, BC);
7546     SI->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node);
7547     SI->setAlignment(1);
7548     return SI;
7549   }
7550
7551   case X86::BI__builtin_ia32_selectb_128:
7552   case X86::BI__builtin_ia32_selectb_256:
7553   case X86::BI__builtin_ia32_selectb_512:
7554   case X86::BI__builtin_ia32_selectw_128:
7555   case X86::BI__builtin_ia32_selectw_256:
7556   case X86::BI__builtin_ia32_selectw_512:
7557   case X86::BI__builtin_ia32_selectd_128:
7558   case X86::BI__builtin_ia32_selectd_256:
7559   case X86::BI__builtin_ia32_selectd_512:
7560   case X86::BI__builtin_ia32_selectq_128:
7561   case X86::BI__builtin_ia32_selectq_256:
7562   case X86::BI__builtin_ia32_selectq_512:
7563   case X86::BI__builtin_ia32_selectps_128:
7564   case X86::BI__builtin_ia32_selectps_256:
7565   case X86::BI__builtin_ia32_selectps_512:
7566   case X86::BI__builtin_ia32_selectpd_128:
7567   case X86::BI__builtin_ia32_selectpd_256:
7568   case X86::BI__builtin_ia32_selectpd_512:
7569     return EmitX86Select(*this, Ops[0], Ops[1], Ops[2]);
7570   case X86::BI__builtin_ia32_pcmpeqb128_mask:
7571   case X86::BI__builtin_ia32_pcmpeqb256_mask:
7572   case X86::BI__builtin_ia32_pcmpeqb512_mask:
7573   case X86::BI__builtin_ia32_pcmpeqw128_mask:
7574   case X86::BI__builtin_ia32_pcmpeqw256_mask:
7575   case X86::BI__builtin_ia32_pcmpeqw512_mask:
7576   case X86::BI__builtin_ia32_pcmpeqd128_mask:
7577   case X86::BI__builtin_ia32_pcmpeqd256_mask:
7578   case X86::BI__builtin_ia32_pcmpeqd512_mask:
7579   case X86::BI__builtin_ia32_pcmpeqq128_mask:
7580   case X86::BI__builtin_ia32_pcmpeqq256_mask:
7581   case X86::BI__builtin_ia32_pcmpeqq512_mask:
7582     return EmitX86MaskedCompare(*this, 0, false, Ops);
7583   case X86::BI__builtin_ia32_pcmpgtb128_mask:
7584   case X86::BI__builtin_ia32_pcmpgtb256_mask:
7585   case X86::BI__builtin_ia32_pcmpgtb512_mask:
7586   case X86::BI__builtin_ia32_pcmpgtw128_mask:
7587   case X86::BI__builtin_ia32_pcmpgtw256_mask:
7588   case X86::BI__builtin_ia32_pcmpgtw512_mask:
7589   case X86::BI__builtin_ia32_pcmpgtd128_mask:
7590   case X86::BI__builtin_ia32_pcmpgtd256_mask:
7591   case X86::BI__builtin_ia32_pcmpgtd512_mask:
7592   case X86::BI__builtin_ia32_pcmpgtq128_mask:
7593   case X86::BI__builtin_ia32_pcmpgtq256_mask:
7594   case X86::BI__builtin_ia32_pcmpgtq512_mask:
7595     return EmitX86MaskedCompare(*this, 6, true, Ops);
7596   case X86::BI__builtin_ia32_cmpb128_mask:
7597   case X86::BI__builtin_ia32_cmpb256_mask:
7598   case X86::BI__builtin_ia32_cmpb512_mask:
7599   case X86::BI__builtin_ia32_cmpw128_mask:
7600   case X86::BI__builtin_ia32_cmpw256_mask:
7601   case X86::BI__builtin_ia32_cmpw512_mask:
7602   case X86::BI__builtin_ia32_cmpd128_mask:
7603   case X86::BI__builtin_ia32_cmpd256_mask:
7604   case X86::BI__builtin_ia32_cmpd512_mask:
7605   case X86::BI__builtin_ia32_cmpq128_mask:
7606   case X86::BI__builtin_ia32_cmpq256_mask:
7607   case X86::BI__builtin_ia32_cmpq512_mask: {
7608     unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
7609     return EmitX86MaskedCompare(*this, CC, true, Ops);
7610   }
7611   case X86::BI__builtin_ia32_ucmpb128_mask:
7612   case X86::BI__builtin_ia32_ucmpb256_mask:
7613   case X86::BI__builtin_ia32_ucmpb512_mask:
7614   case X86::BI__builtin_ia32_ucmpw128_mask:
7615   case X86::BI__builtin_ia32_ucmpw256_mask:
7616   case X86::BI__builtin_ia32_ucmpw512_mask:
7617   case X86::BI__builtin_ia32_ucmpd128_mask:
7618   case X86::BI__builtin_ia32_ucmpd256_mask:
7619   case X86::BI__builtin_ia32_ucmpd512_mask:
7620   case X86::BI__builtin_ia32_ucmpq128_mask:
7621   case X86::BI__builtin_ia32_ucmpq256_mask:
7622   case X86::BI__builtin_ia32_ucmpq512_mask: {
7623     unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
7624     return EmitX86MaskedCompare(*this, CC, false, Ops);
7625   }
7626
7627   case X86::BI__builtin_ia32_vplzcntd_128_mask:
7628   case X86::BI__builtin_ia32_vplzcntd_256_mask:
7629   case X86::BI__builtin_ia32_vplzcntd_512_mask:
7630   case X86::BI__builtin_ia32_vplzcntq_128_mask:
7631   case X86::BI__builtin_ia32_vplzcntq_256_mask:
7632   case X86::BI__builtin_ia32_vplzcntq_512_mask: {
7633     Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType());
7634     return EmitX86Select(*this, Ops[2],
7635                          Builder.CreateCall(F, {Ops[0],Builder.getInt1(false)}),
7636                          Ops[1]);
7637   }
7638
7639   case X86::BI__builtin_ia32_pmaxsb128:
7640   case X86::BI__builtin_ia32_pmaxsw128:
7641   case X86::BI__builtin_ia32_pmaxsd128:
7642   case X86::BI__builtin_ia32_pmaxsq128_mask:
7643   case X86::BI__builtin_ia32_pmaxsb256:
7644   case X86::BI__builtin_ia32_pmaxsw256:
7645   case X86::BI__builtin_ia32_pmaxsd256:
7646   case X86::BI__builtin_ia32_pmaxsq256_mask:
7647   case X86::BI__builtin_ia32_pmaxsb512_mask:
7648   case X86::BI__builtin_ia32_pmaxsw512_mask:
7649   case X86::BI__builtin_ia32_pmaxsd512_mask:
7650   case X86::BI__builtin_ia32_pmaxsq512_mask:
7651     return EmitX86MinMax(*this, ICmpInst::ICMP_SGT, Ops);
7652   case X86::BI__builtin_ia32_pmaxub128:
7653   case X86::BI__builtin_ia32_pmaxuw128:
7654   case X86::BI__builtin_ia32_pmaxud128:
7655   case X86::BI__builtin_ia32_pmaxuq128_mask:
7656   case X86::BI__builtin_ia32_pmaxub256:
7657   case X86::BI__builtin_ia32_pmaxuw256:
7658   case X86::BI__builtin_ia32_pmaxud256:
7659   case X86::BI__builtin_ia32_pmaxuq256_mask:
7660   case X86::BI__builtin_ia32_pmaxub512_mask:
7661   case X86::BI__builtin_ia32_pmaxuw512_mask:
7662   case X86::BI__builtin_ia32_pmaxud512_mask:
7663   case X86::BI__builtin_ia32_pmaxuq512_mask:
7664     return EmitX86MinMax(*this, ICmpInst::ICMP_UGT, Ops);
7665   case X86::BI__builtin_ia32_pminsb128:
7666   case X86::BI__builtin_ia32_pminsw128:
7667   case X86::BI__builtin_ia32_pminsd128:
7668   case X86::BI__builtin_ia32_pminsq128_mask:
7669   case X86::BI__builtin_ia32_pminsb256:
7670   case X86::BI__builtin_ia32_pminsw256:
7671   case X86::BI__builtin_ia32_pminsd256:
7672   case X86::BI__builtin_ia32_pminsq256_mask:
7673   case X86::BI__builtin_ia32_pminsb512_mask:
7674   case X86::BI__builtin_ia32_pminsw512_mask:
7675   case X86::BI__builtin_ia32_pminsd512_mask:
7676   case X86::BI__builtin_ia32_pminsq512_mask:
7677     return EmitX86MinMax(*this, ICmpInst::ICMP_SLT, Ops);
7678   case X86::BI__builtin_ia32_pminub128:
7679   case X86::BI__builtin_ia32_pminuw128:
7680   case X86::BI__builtin_ia32_pminud128:
7681   case X86::BI__builtin_ia32_pminuq128_mask:
7682   case X86::BI__builtin_ia32_pminub256:
7683   case X86::BI__builtin_ia32_pminuw256:
7684   case X86::BI__builtin_ia32_pminud256:
7685   case X86::BI__builtin_ia32_pminuq256_mask:
7686   case X86::BI__builtin_ia32_pminub512_mask:
7687   case X86::BI__builtin_ia32_pminuw512_mask:
7688   case X86::BI__builtin_ia32_pminud512_mask:
7689   case X86::BI__builtin_ia32_pminuq512_mask:
7690     return EmitX86MinMax(*this, ICmpInst::ICMP_ULT, Ops);
7691
7692   // 3DNow!
7693   case X86::BI__builtin_ia32_pswapdsf:
7694   case X86::BI__builtin_ia32_pswapdsi: {
7695     llvm::Type *MMXTy = llvm::Type::getX86_MMXTy(getLLVMContext());
7696     Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast");
7697     llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_3dnowa_pswapd);
7698     return Builder.CreateCall(F, Ops, "pswapd");
7699   }
7700   case X86::BI__builtin_ia32_rdrand16_step:
7701   case X86::BI__builtin_ia32_rdrand32_step:
7702   case X86::BI__builtin_ia32_rdrand64_step:
7703   case X86::BI__builtin_ia32_rdseed16_step:
7704   case X86::BI__builtin_ia32_rdseed32_step:
7705   case X86::BI__builtin_ia32_rdseed64_step: {
7706     Intrinsic::ID ID;
7707     switch (BuiltinID) {
7708     default: llvm_unreachable("Unsupported intrinsic!");
7709     case X86::BI__builtin_ia32_rdrand16_step:
7710       ID = Intrinsic::x86_rdrand_16;
7711       break;
7712     case X86::BI__builtin_ia32_rdrand32_step:
7713       ID = Intrinsic::x86_rdrand_32;
7714       break;
7715     case X86::BI__builtin_ia32_rdrand64_step:
7716       ID = Intrinsic::x86_rdrand_64;
7717       break;
7718     case X86::BI__builtin_ia32_rdseed16_step:
7719       ID = Intrinsic::x86_rdseed_16;
7720       break;
7721     case X86::BI__builtin_ia32_rdseed32_step:
7722       ID = Intrinsic::x86_rdseed_32;
7723       break;
7724     case X86::BI__builtin_ia32_rdseed64_step:
7725       ID = Intrinsic::x86_rdseed_64;
7726       break;
7727     }
7728
7729     Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID));
7730     Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 0),
7731                                       Ops[0]);
7732     return Builder.CreateExtractValue(Call, 1);
7733   }
7734
7735   // SSE packed comparison intrinsics
7736   case X86::BI__builtin_ia32_cmpeqps:
7737   case X86::BI__builtin_ia32_cmpeqpd:
7738     return getVectorFCmpIR(CmpInst::FCMP_OEQ);
7739   case X86::BI__builtin_ia32_cmpltps:
7740   case X86::BI__builtin_ia32_cmpltpd:
7741     return getVectorFCmpIR(CmpInst::FCMP_OLT);
7742   case X86::BI__builtin_ia32_cmpleps:
7743   case X86::BI__builtin_ia32_cmplepd:
7744     return getVectorFCmpIR(CmpInst::FCMP_OLE);
7745   case X86::BI__builtin_ia32_cmpunordps:
7746   case X86::BI__builtin_ia32_cmpunordpd:
7747     return getVectorFCmpIR(CmpInst::FCMP_UNO);
7748   case X86::BI__builtin_ia32_cmpneqps:
7749   case X86::BI__builtin_ia32_cmpneqpd:
7750     return getVectorFCmpIR(CmpInst::FCMP_UNE);
7751   case X86::BI__builtin_ia32_cmpnltps:
7752   case X86::BI__builtin_ia32_cmpnltpd:
7753     return getVectorFCmpIR(CmpInst::FCMP_UGE);
7754   case X86::BI__builtin_ia32_cmpnleps:
7755   case X86::BI__builtin_ia32_cmpnlepd:
7756     return getVectorFCmpIR(CmpInst::FCMP_UGT);
7757   case X86::BI__builtin_ia32_cmpordps:
7758   case X86::BI__builtin_ia32_cmpordpd:
7759     return getVectorFCmpIR(CmpInst::FCMP_ORD);
7760   case X86::BI__builtin_ia32_cmpps:
7761   case X86::BI__builtin_ia32_cmpps256:
7762   case X86::BI__builtin_ia32_cmppd:
7763   case X86::BI__builtin_ia32_cmppd256: {
7764     unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
7765     // If this one of the SSE immediates, we can use native IR.
7766     if (CC < 8) {
7767       FCmpInst::Predicate Pred;
7768       switch (CC) {
7769       case 0: Pred = FCmpInst::FCMP_OEQ; break;
7770       case 1: Pred = FCmpInst::FCMP_OLT; break;
7771       case 2: Pred = FCmpInst::FCMP_OLE; break;
7772       case 3: Pred = FCmpInst::FCMP_UNO; break;
7773       case 4: Pred = FCmpInst::FCMP_UNE; break;
7774       case 5: Pred = FCmpInst::FCMP_UGE; break;
7775       case 6: Pred = FCmpInst::FCMP_UGT; break;
7776       case 7: Pred = FCmpInst::FCMP_ORD; break;
7777       }
7778       return getVectorFCmpIR(Pred);
7779     }
7780
7781     // We can't handle 8-31 immediates with native IR, use the intrinsic.
7782     Intrinsic::ID ID;
7783     switch (BuiltinID) {
7784     default: llvm_unreachable("Unsupported intrinsic!");
7785     case X86::BI__builtin_ia32_cmpps:
7786       ID = Intrinsic::x86_sse_cmp_ps;
7787       break;
7788     case X86::BI__builtin_ia32_cmpps256:
7789       ID = Intrinsic::x86_avx_cmp_ps_256;
7790       break;
7791     case X86::BI__builtin_ia32_cmppd:
7792       ID = Intrinsic::x86_sse2_cmp_pd;
7793       break;
7794     case X86::BI__builtin_ia32_cmppd256:
7795       ID = Intrinsic::x86_avx_cmp_pd_256;
7796       break;
7797     }
7798
7799     return Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
7800   }
7801
7802   // SSE scalar comparison intrinsics
7803   case X86::BI__builtin_ia32_cmpeqss:
7804     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 0);
7805   case X86::BI__builtin_ia32_cmpltss:
7806     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 1);
7807   case X86::BI__builtin_ia32_cmpless:
7808     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 2);
7809   case X86::BI__builtin_ia32_cmpunordss:
7810     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 3);
7811   case X86::BI__builtin_ia32_cmpneqss:
7812     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 4);
7813   case X86::BI__builtin_ia32_cmpnltss:
7814     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 5);
7815   case X86::BI__builtin_ia32_cmpnless:
7816     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 6);
7817   case X86::BI__builtin_ia32_cmpordss:
7818     return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 7);
7819   case X86::BI__builtin_ia32_cmpeqsd:
7820     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 0);
7821   case X86::BI__builtin_ia32_cmpltsd:
7822     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 1);
7823   case X86::BI__builtin_ia32_cmplesd:
7824     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 2);
7825   case X86::BI__builtin_ia32_cmpunordsd:
7826     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 3);
7827   case X86::BI__builtin_ia32_cmpneqsd:
7828     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 4);
7829   case X86::BI__builtin_ia32_cmpnltsd:
7830     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 5);
7831   case X86::BI__builtin_ia32_cmpnlesd:
7832     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 6);
7833   case X86::BI__builtin_ia32_cmpordsd:
7834     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 7);
7835
7836   case X86::BI__emul:
7837   case X86::BI__emulu: {
7838     llvm::Type *Int64Ty = llvm::IntegerType::get(getLLVMContext(), 64);
7839     bool isSigned = (BuiltinID == X86::BI__emul);
7840     Value *LHS = Builder.CreateIntCast(Ops[0], Int64Ty, isSigned);
7841     Value *RHS = Builder.CreateIntCast(Ops[1], Int64Ty, isSigned);
7842     return Builder.CreateMul(LHS, RHS, "", !isSigned, isSigned);
7843   }
7844   case X86::BI__mulh:
7845   case X86::BI__umulh:
7846   case X86::BI_mul128:
7847   case X86::BI_umul128: {
7848     llvm::Type *ResType = ConvertType(E->getType());
7849     llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
7850
7851     bool IsSigned = (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI_mul128);
7852     Value *LHS = Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned);
7853     Value *RHS = Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned);
7854
7855     Value *MulResult, *HigherBits;
7856     if (IsSigned) {
7857       MulResult = Builder.CreateNSWMul(LHS, RHS);
7858       HigherBits = Builder.CreateAShr(MulResult, 64);
7859     } else {
7860       MulResult = Builder.CreateNUWMul(LHS, RHS);
7861       HigherBits = Builder.CreateLShr(MulResult, 64);
7862     }
7863     HigherBits = Builder.CreateIntCast(HigherBits, ResType, IsSigned);
7864
7865     if (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI__umulh)
7866       return HigherBits;
7867
7868     Address HighBitsAddress = EmitPointerWithAlignment(E->getArg(2));
7869     Builder.CreateStore(HigherBits, HighBitsAddress);
7870     return Builder.CreateIntCast(MulResult, ResType, IsSigned);
7871   }
7872
7873   case X86::BI__faststorefence: {
7874     return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
7875                                llvm::CrossThread);
7876   }
7877   case X86::BI_ReadWriteBarrier:
7878   case X86::BI_ReadBarrier:
7879   case X86::BI_WriteBarrier: {
7880     return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
7881                                llvm::SingleThread);
7882   }
7883   case X86::BI_BitScanForward:
7884   case X86::BI_BitScanForward64:
7885     return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanForward, E);
7886   case X86::BI_BitScanReverse:
7887   case X86::BI_BitScanReverse64:
7888     return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanReverse, E);
7889
7890   case X86::BI_InterlockedAnd64:
7891     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E);
7892   case X86::BI_InterlockedExchange64:
7893     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E);
7894   case X86::BI_InterlockedExchangeAdd64:
7895     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E);
7896   case X86::BI_InterlockedExchangeSub64:
7897     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E);
7898   case X86::BI_InterlockedOr64:
7899     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E);
7900   case X86::BI_InterlockedXor64:
7901     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E);
7902   case X86::BI_InterlockedDecrement64:
7903     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E);
7904   case X86::BI_InterlockedIncrement64:
7905     return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E);
7906
7907   case X86::BI_AddressOfReturnAddress: {
7908     Value *F = CGM.getIntrinsic(Intrinsic::addressofreturnaddress);
7909     return Builder.CreateCall(F);
7910   }
7911   case X86::BI__stosb: {
7912     // We treat __stosb as a volatile memset - it may not generate "rep stosb"
7913     // instruction, but it will create a memset that won't be optimized away.
7914     return Builder.CreateMemSet(Ops[0], Ops[1], Ops[2], 1, true);
7915   }
7916   }
7917 }
7918
7919
7920 Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
7921                                            const CallExpr *E) {
7922   SmallVector<Value*, 4> Ops;
7923
7924   for (unsigned i = 0, e = E->getNumArgs(); i != e; i++)
7925     Ops.push_back(EmitScalarExpr(E->getArg(i)));
7926
7927   Intrinsic::ID ID = Intrinsic::not_intrinsic;
7928
7929   switch (BuiltinID) {
7930   default: return nullptr;
7931
7932   // __builtin_ppc_get_timebase is GCC 4.8+'s PowerPC-specific name for what we
7933   // call __builtin_readcyclecounter.
7934   case PPC::BI__builtin_ppc_get_timebase:
7935     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::readcyclecounter));
7936
7937   // vec_ld, vec_xl_be, vec_lvsl, vec_lvsr
7938   case PPC::BI__builtin_altivec_lvx:
7939   case PPC::BI__builtin_altivec_lvxl:
7940   case PPC::BI__builtin_altivec_lvebx:
7941   case PPC::BI__builtin_altivec_lvehx:
7942   case PPC::BI__builtin_altivec_lvewx:
7943   case PPC::BI__builtin_altivec_lvsl:
7944   case PPC::BI__builtin_altivec_lvsr:
7945   case PPC::BI__builtin_vsx_lxvd2x:
7946   case PPC::BI__builtin_vsx_lxvw4x:
7947   case PPC::BI__builtin_vsx_lxvd2x_be:
7948   case PPC::BI__builtin_vsx_lxvw4x_be:
7949   case PPC::BI__builtin_vsx_lxvl:
7950   case PPC::BI__builtin_vsx_lxvll:
7951   {
7952     if(BuiltinID == PPC::BI__builtin_vsx_lxvl ||
7953        BuiltinID == PPC::BI__builtin_vsx_lxvll){
7954       Ops[0] = Builder.CreateBitCast(Ops[0], Int8PtrTy);
7955     }else {
7956       Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy);
7957       Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]);
7958       Ops.pop_back();
7959     }
7960
7961     switch (BuiltinID) {
7962     default: llvm_unreachable("Unsupported ld/lvsl/lvsr intrinsic!");
7963     case PPC::BI__builtin_altivec_lvx:
7964       ID = Intrinsic::ppc_altivec_lvx;
7965       break;
7966     case PPC::BI__builtin_altivec_lvxl:
7967       ID = Intrinsic::ppc_altivec_lvxl;
7968       break;
7969     case PPC::BI__builtin_altivec_lvebx:
7970       ID = Intrinsic::ppc_altivec_lvebx;
7971       break;
7972     case PPC::BI__builtin_altivec_lvehx:
7973       ID = Intrinsic::ppc_altivec_lvehx;
7974       break;
7975     case PPC::BI__builtin_altivec_lvewx:
7976       ID = Intrinsic::ppc_altivec_lvewx;
7977       break;
7978     case PPC::BI__builtin_altivec_lvsl:
7979       ID = Intrinsic::ppc_altivec_lvsl;
7980       break;
7981     case PPC::BI__builtin_altivec_lvsr:
7982       ID = Intrinsic::ppc_altivec_lvsr;
7983       break;
7984     case PPC::BI__builtin_vsx_lxvd2x:
7985       ID = Intrinsic::ppc_vsx_lxvd2x;
7986       break;
7987     case PPC::BI__builtin_vsx_lxvw4x:
7988       ID = Intrinsic::ppc_vsx_lxvw4x;
7989       break;
7990     case PPC::BI__builtin_vsx_lxvd2x_be:
7991       ID = Intrinsic::ppc_vsx_lxvd2x_be;
7992       break;
7993     case PPC::BI__builtin_vsx_lxvw4x_be:
7994       ID = Intrinsic::ppc_vsx_lxvw4x_be;
7995       break;
7996     case PPC::BI__builtin_vsx_lxvl:
7997       ID = Intrinsic::ppc_vsx_lxvl;
7998       break;
7999     case PPC::BI__builtin_vsx_lxvll:
8000       ID = Intrinsic::ppc_vsx_lxvll;
8001       break;
8002     }
8003     llvm::Function *F = CGM.getIntrinsic(ID);
8004     return Builder.CreateCall(F, Ops, "");
8005   }
8006
8007   // vec_st, vec_xst_be
8008   case PPC::BI__builtin_altivec_stvx:
8009   case PPC::BI__builtin_altivec_stvxl:
8010   case PPC::BI__builtin_altivec_stvebx:
8011   case PPC::BI__builtin_altivec_stvehx:
8012   case PPC::BI__builtin_altivec_stvewx:
8013   case PPC::BI__builtin_vsx_stxvd2x:
8014   case PPC::BI__builtin_vsx_stxvw4x:
8015   case PPC::BI__builtin_vsx_stxvd2x_be:
8016   case PPC::BI__builtin_vsx_stxvw4x_be:
8017   case PPC::BI__builtin_vsx_stxvl:
8018   case PPC::BI__builtin_vsx_stxvll:
8019   {
8020     if(BuiltinID == PPC::BI__builtin_vsx_stxvl ||
8021       BuiltinID == PPC::BI__builtin_vsx_stxvll ){
8022       Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy);
8023     }else {
8024       Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy);
8025       Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]);
8026       Ops.pop_back();
8027     }
8028
8029     switch (BuiltinID) {
8030     default: llvm_unreachable("Unsupported st intrinsic!");
8031     case PPC::BI__builtin_altivec_stvx:
8032       ID = Intrinsic::ppc_altivec_stvx;
8033       break;
8034     case PPC::BI__builtin_altivec_stvxl:
8035       ID = Intrinsic::ppc_altivec_stvxl;
8036       break;
8037     case PPC::BI__builtin_altivec_stvebx:
8038       ID = Intrinsic::ppc_altivec_stvebx;
8039       break;
8040     case PPC::BI__builtin_altivec_stvehx:
8041       ID = Intrinsic::ppc_altivec_stvehx;
8042       break;
8043     case PPC::BI__builtin_altivec_stvewx:
8044       ID = Intrinsic::ppc_altivec_stvewx;
8045       break;
8046     case PPC::BI__builtin_vsx_stxvd2x:
8047       ID = Intrinsic::ppc_vsx_stxvd2x;
8048       break;
8049     case PPC::BI__builtin_vsx_stxvw4x:
8050       ID = Intrinsic::ppc_vsx_stxvw4x;
8051       break;
8052     case PPC::BI__builtin_vsx_stxvd2x_be:
8053       ID = Intrinsic::ppc_vsx_stxvd2x_be;
8054       break;
8055     case PPC::BI__builtin_vsx_stxvw4x_be:
8056       ID = Intrinsic::ppc_vsx_stxvw4x_be;
8057       break;
8058     case PPC::BI__builtin_vsx_stxvl:
8059       ID = Intrinsic::ppc_vsx_stxvl;
8060       break;
8061     case PPC::BI__builtin_vsx_stxvll:
8062       ID = Intrinsic::ppc_vsx_stxvll;
8063       break;
8064     }
8065     llvm::Function *F = CGM.getIntrinsic(ID);
8066     return Builder.CreateCall(F, Ops, "");
8067   }
8068   // Square root
8069   case PPC::BI__builtin_vsx_xvsqrtsp:
8070   case PPC::BI__builtin_vsx_xvsqrtdp: {
8071     llvm::Type *ResultType = ConvertType(E->getType());
8072     Value *X = EmitScalarExpr(E->getArg(0));
8073     ID = Intrinsic::sqrt;
8074     llvm::Function *F = CGM.getIntrinsic(ID, ResultType);
8075     return Builder.CreateCall(F, X);
8076   }
8077   // Count leading zeros
8078   case PPC::BI__builtin_altivec_vclzb:
8079   case PPC::BI__builtin_altivec_vclzh:
8080   case PPC::BI__builtin_altivec_vclzw:
8081   case PPC::BI__builtin_altivec_vclzd: {
8082     llvm::Type *ResultType = ConvertType(E->getType());
8083     Value *X = EmitScalarExpr(E->getArg(0));
8084     Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false);
8085     Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType);
8086     return Builder.CreateCall(F, {X, Undef});
8087   }
8088   case PPC::BI__builtin_altivec_vctzb:
8089   case PPC::BI__builtin_altivec_vctzh:
8090   case PPC::BI__builtin_altivec_vctzw:
8091   case PPC::BI__builtin_altivec_vctzd: {
8092     llvm::Type *ResultType = ConvertType(E->getType());
8093     Value *X = EmitScalarExpr(E->getArg(0));
8094     Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false);
8095     Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType);
8096     return Builder.CreateCall(F, {X, Undef});
8097   }
8098   case PPC::BI__builtin_altivec_vpopcntb:
8099   case PPC::BI__builtin_altivec_vpopcnth:
8100   case PPC::BI__builtin_altivec_vpopcntw:
8101   case PPC::BI__builtin_altivec_vpopcntd: {
8102     llvm::Type *ResultType = ConvertType(E->getType());
8103     Value *X = EmitScalarExpr(E->getArg(0));
8104     llvm::Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
8105     return Builder.CreateCall(F, X);
8106   }
8107   // Copy sign
8108   case PPC::BI__builtin_vsx_xvcpsgnsp:
8109   case PPC::BI__builtin_vsx_xvcpsgndp: {
8110     llvm::Type *ResultType = ConvertType(E->getType());
8111     Value *X = EmitScalarExpr(E->getArg(0));
8112     Value *Y = EmitScalarExpr(E->getArg(1));
8113     ID = Intrinsic::copysign;
8114     llvm::Function *F = CGM.getIntrinsic(ID, ResultType);
8115     return Builder.CreateCall(F, {X, Y});
8116   }
8117   // Rounding/truncation
8118   case PPC::BI__builtin_vsx_xvrspip:
8119   case PPC::BI__builtin_vsx_xvrdpip:
8120   case PPC::BI__builtin_vsx_xvrdpim:
8121   case PPC::BI__builtin_vsx_xvrspim:
8122   case PPC::BI__builtin_vsx_xvrdpi:
8123   case PPC::BI__builtin_vsx_xvrspi:
8124   case PPC::BI__builtin_vsx_xvrdpic:
8125   case PPC::BI__builtin_vsx_xvrspic:
8126   case PPC::BI__builtin_vsx_xvrdpiz:
8127   case PPC::BI__builtin_vsx_xvrspiz: {
8128     llvm::Type *ResultType = ConvertType(E->getType());
8129     Value *X = EmitScalarExpr(E->getArg(0));
8130     if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim ||
8131         BuiltinID == PPC::BI__builtin_vsx_xvrspim)
8132       ID = Intrinsic::floor;
8133     else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi ||
8134              BuiltinID == PPC::BI__builtin_vsx_xvrspi)
8135       ID = Intrinsic::round;
8136     else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic ||
8137              BuiltinID == PPC::BI__builtin_vsx_xvrspic)
8138       ID = Intrinsic::nearbyint;
8139     else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip ||
8140              BuiltinID == PPC::BI__builtin_vsx_xvrspip)
8141       ID = Intrinsic::ceil;
8142     else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz ||
8143              BuiltinID == PPC::BI__builtin_vsx_xvrspiz)
8144       ID = Intrinsic::trunc;
8145     llvm::Function *F = CGM.getIntrinsic(ID, ResultType);
8146     return Builder.CreateCall(F, X);
8147   }
8148
8149   // Absolute value
8150   case PPC::BI__builtin_vsx_xvabsdp:
8151   case PPC::BI__builtin_vsx_xvabssp: {
8152     llvm::Type *ResultType = ConvertType(E->getType());
8153     Value *X = EmitScalarExpr(E->getArg(0));
8154     llvm::Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType);
8155     return Builder.CreateCall(F, X);
8156   }
8157
8158   // FMA variations
8159   case PPC::BI__builtin_vsx_xvmaddadp:
8160   case PPC::BI__builtin_vsx_xvmaddasp:
8161   case PPC::BI__builtin_vsx_xvnmaddadp:
8162   case PPC::BI__builtin_vsx_xvnmaddasp:
8163   case PPC::BI__builtin_vsx_xvmsubadp:
8164   case PPC::BI__builtin_vsx_xvmsubasp:
8165   case PPC::BI__builtin_vsx_xvnmsubadp:
8166   case PPC::BI__builtin_vsx_xvnmsubasp: {
8167     llvm::Type *ResultType = ConvertType(E->getType());
8168     Value *X = EmitScalarExpr(E->getArg(0));
8169     Value *Y = EmitScalarExpr(E->getArg(1));
8170     Value *Z = EmitScalarExpr(E->getArg(2));
8171     Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
8172     llvm::Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
8173     switch (BuiltinID) {
8174       case PPC::BI__builtin_vsx_xvmaddadp:
8175       case PPC::BI__builtin_vsx_xvmaddasp:
8176         return Builder.CreateCall(F, {X, Y, Z});
8177       case PPC::BI__builtin_vsx_xvnmaddadp:
8178       case PPC::BI__builtin_vsx_xvnmaddasp:
8179         return Builder.CreateFSub(Zero,
8180                                   Builder.CreateCall(F, {X, Y, Z}), "sub");
8181       case PPC::BI__builtin_vsx_xvmsubadp:
8182       case PPC::BI__builtin_vsx_xvmsubasp:
8183         return Builder.CreateCall(F,
8184                                   {X, Y, Builder.CreateFSub(Zero, Z, "sub")});
8185       case PPC::BI__builtin_vsx_xvnmsubadp:
8186       case PPC::BI__builtin_vsx_xvnmsubasp:
8187         Value *FsubRes =
8188           Builder.CreateCall(F, {X, Y, Builder.CreateFSub(Zero, Z, "sub")});
8189         return Builder.CreateFSub(Zero, FsubRes, "sub");
8190     }
8191     llvm_unreachable("Unknown FMA operation");
8192     return nullptr; // Suppress no-return warning
8193   }
8194   }
8195 }
8196
8197 Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
8198                                               const CallExpr *E) {
8199   switch (BuiltinID) {
8200   case AMDGPU::BI__builtin_amdgcn_div_scale:
8201   case AMDGPU::BI__builtin_amdgcn_div_scalef: {
8202     // Translate from the intrinsics's struct return to the builtin's out
8203     // argument.
8204
8205     Address FlagOutPtr = EmitPointerWithAlignment(E->getArg(3));
8206
8207     llvm::Value *X = EmitScalarExpr(E->getArg(0));
8208     llvm::Value *Y = EmitScalarExpr(E->getArg(1));
8209     llvm::Value *Z = EmitScalarExpr(E->getArg(2));
8210
8211     llvm::Value *Callee = CGM.getIntrinsic(Intrinsic::amdgcn_div_scale,
8212                                            X->getType());
8213
8214     llvm::Value *Tmp = Builder.CreateCall(Callee, {X, Y, Z});
8215
8216     llvm::Value *Result = Builder.CreateExtractValue(Tmp, 0);
8217     llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1);
8218
8219     llvm::Type *RealFlagType
8220       = FlagOutPtr.getPointer()->getType()->getPointerElementType();
8221
8222     llvm::Value *FlagExt = Builder.CreateZExt(Flag, RealFlagType);
8223     Builder.CreateStore(FlagExt, FlagOutPtr);
8224     return Result;
8225   }
8226   case AMDGPU::BI__builtin_amdgcn_div_fmas:
8227   case AMDGPU::BI__builtin_amdgcn_div_fmasf: {
8228     llvm::Value *Src0 = EmitScalarExpr(E->getArg(0));
8229     llvm::Value *Src1 = EmitScalarExpr(E->getArg(1));
8230     llvm::Value *Src2 = EmitScalarExpr(E->getArg(2));
8231     llvm::Value *Src3 = EmitScalarExpr(E->getArg(3));
8232
8233     llvm::Value *F = CGM.getIntrinsic(Intrinsic::amdgcn_div_fmas,
8234                                       Src0->getType());
8235     llvm::Value *Src3ToBool = Builder.CreateIsNotNull(Src3);
8236     return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool});
8237   }
8238
8239   case AMDGPU::BI__builtin_amdgcn_ds_swizzle:
8240     return emitBinaryBuiltin(*this, E, Intrinsic::amdgcn_ds_swizzle);
8241   case AMDGPU::BI__builtin_amdgcn_div_fixup:
8242   case AMDGPU::BI__builtin_amdgcn_div_fixupf:
8243   case AMDGPU::BI__builtin_amdgcn_div_fixuph:
8244     return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_div_fixup);
8245   case AMDGPU::BI__builtin_amdgcn_trig_preop:
8246   case AMDGPU::BI__builtin_amdgcn_trig_preopf:
8247     return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_trig_preop);
8248   case AMDGPU::BI__builtin_amdgcn_rcp:
8249   case AMDGPU::BI__builtin_amdgcn_rcpf:
8250   case AMDGPU::BI__builtin_amdgcn_rcph:
8251     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rcp);
8252   case AMDGPU::BI__builtin_amdgcn_rsq:
8253   case AMDGPU::BI__builtin_amdgcn_rsqf:
8254   case AMDGPU::BI__builtin_amdgcn_rsqh:
8255     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rsq);
8256   case AMDGPU::BI__builtin_amdgcn_rsq_clamp:
8257   case AMDGPU::BI__builtin_amdgcn_rsq_clampf:
8258     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_rsq_clamp);
8259   case AMDGPU::BI__builtin_amdgcn_sinf:
8260   case AMDGPU::BI__builtin_amdgcn_sinh:
8261     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_sin);
8262   case AMDGPU::BI__builtin_amdgcn_cosf:
8263   case AMDGPU::BI__builtin_amdgcn_cosh:
8264     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_cos);
8265   case AMDGPU::BI__builtin_amdgcn_log_clampf:
8266     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_log_clamp);
8267   case AMDGPU::BI__builtin_amdgcn_ldexp:
8268   case AMDGPU::BI__builtin_amdgcn_ldexpf:
8269   case AMDGPU::BI__builtin_amdgcn_ldexph:
8270     return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_ldexp);
8271   case AMDGPU::BI__builtin_amdgcn_frexp_mant:
8272   case AMDGPU::BI__builtin_amdgcn_frexp_mantf:
8273   case AMDGPU::BI__builtin_amdgcn_frexp_manth:
8274     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_frexp_mant);
8275   case AMDGPU::BI__builtin_amdgcn_frexp_exp:
8276   case AMDGPU::BI__builtin_amdgcn_frexp_expf: {
8277     Value *Src0 = EmitScalarExpr(E->getArg(0));
8278     Value *F = CGM.getIntrinsic(Intrinsic::amdgcn_frexp_exp,
8279                                 { Builder.getInt32Ty(), Src0->getType() });
8280     return Builder.CreateCall(F, Src0);
8281   }
8282   case AMDGPU::BI__builtin_amdgcn_frexp_exph: {
8283     Value *Src0 = EmitScalarExpr(E->getArg(0));
8284     Value *F = CGM.getIntrinsic(Intrinsic::amdgcn_frexp_exp,
8285                                 { Builder.getInt16Ty(), Src0->getType() });
8286     return Builder.CreateCall(F, Src0);
8287   }
8288   case AMDGPU::BI__builtin_amdgcn_fract:
8289   case AMDGPU::BI__builtin_amdgcn_fractf:
8290   case AMDGPU::BI__builtin_amdgcn_fracth:
8291     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_fract);
8292   case AMDGPU::BI__builtin_amdgcn_lerp:
8293     return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_lerp);
8294   case AMDGPU::BI__builtin_amdgcn_uicmp:
8295   case AMDGPU::BI__builtin_amdgcn_uicmpl:
8296   case AMDGPU::BI__builtin_amdgcn_sicmp:
8297   case AMDGPU::BI__builtin_amdgcn_sicmpl:
8298     return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_icmp);
8299   case AMDGPU::BI__builtin_amdgcn_fcmp:
8300   case AMDGPU::BI__builtin_amdgcn_fcmpf:
8301     return emitTernaryBuiltin(*this, E, Intrinsic::amdgcn_fcmp);
8302   case AMDGPU::BI__builtin_amdgcn_class:
8303   case AMDGPU::BI__builtin_amdgcn_classf:
8304   case AMDGPU::BI__builtin_amdgcn_classh:
8305     return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_class);
8306
8307   case AMDGPU::BI__builtin_amdgcn_read_exec: {
8308     CallInst *CI = cast<CallInst>(
8309       EmitSpecialRegisterBuiltin(*this, E, Int64Ty, Int64Ty, true, "exec"));
8310     CI->setConvergent();
8311     return CI;
8312   }
8313
8314   // amdgcn workitem
8315   case AMDGPU::BI__builtin_amdgcn_workitem_id_x:
8316     return emitRangedBuiltin(*this, Intrinsic::amdgcn_workitem_id_x, 0, 1024);
8317   case AMDGPU::BI__builtin_amdgcn_workitem_id_y:
8318     return emitRangedBuiltin(*this, Intrinsic::amdgcn_workitem_id_y, 0, 1024);
8319   case AMDGPU::BI__builtin_amdgcn_workitem_id_z:
8320     return emitRangedBuiltin(*this, Intrinsic::amdgcn_workitem_id_z, 0, 1024);
8321
8322   // r600 intrinsics
8323   case AMDGPU::BI__builtin_r600_recipsqrt_ieee:
8324   case AMDGPU::BI__builtin_r600_recipsqrt_ieeef:
8325     return emitUnaryBuiltin(*this, E, Intrinsic::r600_recipsqrt_ieee);
8326   case AMDGPU::BI__builtin_r600_read_tidig_x:
8327     return emitRangedBuiltin(*this, Intrinsic::r600_read_tidig_x, 0, 1024);
8328   case AMDGPU::BI__builtin_r600_read_tidig_y:
8329     return emitRangedBuiltin(*this, Intrinsic::r600_read_tidig_y, 0, 1024);
8330   case AMDGPU::BI__builtin_r600_read_tidig_z:
8331     return emitRangedBuiltin(*this, Intrinsic::r600_read_tidig_z, 0, 1024);
8332   default:
8333     return nullptr;
8334   }
8335 }
8336
8337 /// Handle a SystemZ function in which the final argument is a pointer
8338 /// to an int that receives the post-instruction CC value.  At the LLVM level
8339 /// this is represented as a function that returns a {result, cc} pair.
8340 static Value *EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF,
8341                                          unsigned IntrinsicID,
8342                                          const CallExpr *E) {
8343   unsigned NumArgs = E->getNumArgs() - 1;
8344   SmallVector<Value *, 8> Args(NumArgs);
8345   for (unsigned I = 0; I < NumArgs; ++I)
8346     Args[I] = CGF.EmitScalarExpr(E->getArg(I));
8347   Address CCPtr = CGF.EmitPointerWithAlignment(E->getArg(NumArgs));
8348   Value *F = CGF.CGM.getIntrinsic(IntrinsicID);
8349   Value *Call = CGF.Builder.CreateCall(F, Args);
8350   Value *CC = CGF.Builder.CreateExtractValue(Call, 1);
8351   CGF.Builder.CreateStore(CC, CCPtr);
8352   return CGF.Builder.CreateExtractValue(Call, 0);
8353 }
8354
8355 Value *CodeGenFunction::EmitSystemZBuiltinExpr(unsigned BuiltinID,
8356                                                const CallExpr *E) {
8357   switch (BuiltinID) {
8358   case SystemZ::BI__builtin_tbegin: {
8359     Value *TDB = EmitScalarExpr(E->getArg(0));
8360     Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c);
8361     Value *F = CGM.getIntrinsic(Intrinsic::s390_tbegin);
8362     return Builder.CreateCall(F, {TDB, Control});
8363   }
8364   case SystemZ::BI__builtin_tbegin_nofloat: {
8365     Value *TDB = EmitScalarExpr(E->getArg(0));
8366     Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c);
8367     Value *F = CGM.getIntrinsic(Intrinsic::s390_tbegin_nofloat);
8368     return Builder.CreateCall(F, {TDB, Control});
8369   }
8370   case SystemZ::BI__builtin_tbeginc: {
8371     Value *TDB = llvm::ConstantPointerNull::get(Int8PtrTy);
8372     Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff08);
8373     Value *F = CGM.getIntrinsic(Intrinsic::s390_tbeginc);
8374     return Builder.CreateCall(F, {TDB, Control});
8375   }
8376   case SystemZ::BI__builtin_tabort: {
8377     Value *Data = EmitScalarExpr(E->getArg(0));
8378     Value *F = CGM.getIntrinsic(Intrinsic::s390_tabort);
8379     return Builder.CreateCall(F, Builder.CreateSExt(Data, Int64Ty, "tabort"));
8380   }
8381   case SystemZ::BI__builtin_non_tx_store: {
8382     Value *Address = EmitScalarExpr(E->getArg(0));
8383     Value *Data = EmitScalarExpr(E->getArg(1));
8384     Value *F = CGM.getIntrinsic(Intrinsic::s390_ntstg);
8385     return Builder.CreateCall(F, {Data, Address});
8386   }
8387
8388   // Vector builtins.  Note that most vector builtins are mapped automatically
8389   // to target-specific LLVM intrinsics.  The ones handled specially here can
8390   // be represented via standard LLVM IR, which is preferable to enable common
8391   // LLVM optimizations.
8392
8393   case SystemZ::BI__builtin_s390_vpopctb:
8394   case SystemZ::BI__builtin_s390_vpopcth:
8395   case SystemZ::BI__builtin_s390_vpopctf:
8396   case SystemZ::BI__builtin_s390_vpopctg: {
8397     llvm::Type *ResultType = ConvertType(E->getType());
8398     Value *X = EmitScalarExpr(E->getArg(0));
8399     Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
8400     return Builder.CreateCall(F, X);
8401   }
8402
8403   case SystemZ::BI__builtin_s390_vclzb:
8404   case SystemZ::BI__builtin_s390_vclzh:
8405   case SystemZ::BI__builtin_s390_vclzf:
8406   case SystemZ::BI__builtin_s390_vclzg: {
8407     llvm::Type *ResultType = ConvertType(E->getType());
8408     Value *X = EmitScalarExpr(E->getArg(0));
8409     Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false);
8410     Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType);
8411     return Builder.CreateCall(F, {X, Undef});
8412   }
8413
8414   case SystemZ::BI__builtin_s390_vctzb:
8415   case SystemZ::BI__builtin_s390_vctzh:
8416   case SystemZ::BI__builtin_s390_vctzf:
8417   case SystemZ::BI__builtin_s390_vctzg: {
8418     llvm::Type *ResultType = ConvertType(E->getType());
8419     Value *X = EmitScalarExpr(E->getArg(0));
8420     Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false);
8421     Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType);
8422     return Builder.CreateCall(F, {X, Undef});
8423   }
8424
8425   case SystemZ::BI__builtin_s390_vfsqdb: {
8426     llvm::Type *ResultType = ConvertType(E->getType());
8427     Value *X = EmitScalarExpr(E->getArg(0));
8428     Function *F = CGM.getIntrinsic(Intrinsic::sqrt, ResultType);
8429     return Builder.CreateCall(F, X);
8430   }
8431   case SystemZ::BI__builtin_s390_vfmadb: {
8432     llvm::Type *ResultType = ConvertType(E->getType());
8433     Value *X = EmitScalarExpr(E->getArg(0));
8434     Value *Y = EmitScalarExpr(E->getArg(1));
8435     Value *Z = EmitScalarExpr(E->getArg(2));
8436     Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
8437     return Builder.CreateCall(F, {X, Y, Z});
8438   }
8439   case SystemZ::BI__builtin_s390_vfmsdb: {
8440     llvm::Type *ResultType = ConvertType(E->getType());
8441     Value *X = EmitScalarExpr(E->getArg(0));
8442     Value *Y = EmitScalarExpr(E->getArg(1));
8443     Value *Z = EmitScalarExpr(E->getArg(2));
8444     Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
8445     Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
8446     return Builder.CreateCall(F, {X, Y, Builder.CreateFSub(Zero, Z, "sub")});
8447   }
8448   case SystemZ::BI__builtin_s390_vflpdb: {
8449     llvm::Type *ResultType = ConvertType(E->getType());
8450     Value *X = EmitScalarExpr(E->getArg(0));
8451     Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType);
8452     return Builder.CreateCall(F, X);
8453   }
8454   case SystemZ::BI__builtin_s390_vflndb: {
8455     llvm::Type *ResultType = ConvertType(E->getType());
8456     Value *X = EmitScalarExpr(E->getArg(0));
8457     Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
8458     Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType);
8459     return Builder.CreateFSub(Zero, Builder.CreateCall(F, X), "sub");
8460   }
8461   case SystemZ::BI__builtin_s390_vfidb: {
8462     llvm::Type *ResultType = ConvertType(E->getType());
8463     Value *X = EmitScalarExpr(E->getArg(0));
8464     // Constant-fold the M4 and M5 mask arguments.
8465     llvm::APSInt M4, M5;
8466     bool IsConstM4 = E->getArg(1)->isIntegerConstantExpr(M4, getContext());
8467     bool IsConstM5 = E->getArg(2)->isIntegerConstantExpr(M5, getContext());
8468     assert(IsConstM4 && IsConstM5 && "Constant arg isn't actually constant?");
8469     (void)IsConstM4; (void)IsConstM5;
8470     // Check whether this instance of vfidb can be represented via a LLVM
8471     // standard intrinsic.  We only support some combinations of M4 and M5.
8472     Intrinsic::ID ID = Intrinsic::not_intrinsic;
8473     switch (M4.getZExtValue()) {
8474     default: break;
8475     case 0:  // IEEE-inexact exception allowed
8476       switch (M5.getZExtValue()) {
8477       default: break;
8478       case 0: ID = Intrinsic::rint; break;
8479       }
8480       break;
8481     case 4:  // IEEE-inexact exception suppressed
8482       switch (M5.getZExtValue()) {
8483       default: break;
8484       case 0: ID = Intrinsic::nearbyint; break;
8485       case 1: ID = Intrinsic::round; break;
8486       case 5: ID = Intrinsic::trunc; break;
8487       case 6: ID = Intrinsic::ceil; break;
8488       case 7: ID = Intrinsic::floor; break;
8489       }
8490       break;
8491     }
8492     if (ID != Intrinsic::not_intrinsic) {
8493       Function *F = CGM.getIntrinsic(ID, ResultType);
8494       return Builder.CreateCall(F, X);
8495     }
8496     Function *F = CGM.getIntrinsic(Intrinsic::s390_vfidb);
8497     Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4);
8498     Value *M5Value = llvm::ConstantInt::get(getLLVMContext(), M5);
8499     return Builder.CreateCall(F, {X, M4Value, M5Value});
8500   }
8501
8502   // Vector intrisincs that output the post-instruction CC value.
8503
8504 #define INTRINSIC_WITH_CC(NAME) \
8505     case SystemZ::BI__builtin_##NAME: \
8506       return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E)
8507
8508   INTRINSIC_WITH_CC(s390_vpkshs);
8509   INTRINSIC_WITH_CC(s390_vpksfs);
8510   INTRINSIC_WITH_CC(s390_vpksgs);
8511
8512   INTRINSIC_WITH_CC(s390_vpklshs);
8513   INTRINSIC_WITH_CC(s390_vpklsfs);
8514   INTRINSIC_WITH_CC(s390_vpklsgs);
8515
8516   INTRINSIC_WITH_CC(s390_vceqbs);
8517   INTRINSIC_WITH_CC(s390_vceqhs);
8518   INTRINSIC_WITH_CC(s390_vceqfs);
8519   INTRINSIC_WITH_CC(s390_vceqgs);
8520
8521   INTRINSIC_WITH_CC(s390_vchbs);
8522   INTRINSIC_WITH_CC(s390_vchhs);
8523   INTRINSIC_WITH_CC(s390_vchfs);
8524   INTRINSIC_WITH_CC(s390_vchgs);
8525
8526   INTRINSIC_WITH_CC(s390_vchlbs);
8527   INTRINSIC_WITH_CC(s390_vchlhs);
8528   INTRINSIC_WITH_CC(s390_vchlfs);
8529   INTRINSIC_WITH_CC(s390_vchlgs);
8530
8531   INTRINSIC_WITH_CC(s390_vfaebs);
8532   INTRINSIC_WITH_CC(s390_vfaehs);
8533   INTRINSIC_WITH_CC(s390_vfaefs);
8534
8535   INTRINSIC_WITH_CC(s390_vfaezbs);
8536   INTRINSIC_WITH_CC(s390_vfaezhs);
8537   INTRINSIC_WITH_CC(s390_vfaezfs);
8538
8539   INTRINSIC_WITH_CC(s390_vfeebs);
8540   INTRINSIC_WITH_CC(s390_vfeehs);
8541   INTRINSIC_WITH_CC(s390_vfeefs);
8542
8543   INTRINSIC_WITH_CC(s390_vfeezbs);
8544   INTRINSIC_WITH_CC(s390_vfeezhs);
8545   INTRINSIC_WITH_CC(s390_vfeezfs);
8546
8547   INTRINSIC_WITH_CC(s390_vfenebs);
8548   INTRINSIC_WITH_CC(s390_vfenehs);
8549   INTRINSIC_WITH_CC(s390_vfenefs);
8550
8551   INTRINSIC_WITH_CC(s390_vfenezbs);
8552   INTRINSIC_WITH_CC(s390_vfenezhs);
8553   INTRINSIC_WITH_CC(s390_vfenezfs);
8554
8555   INTRINSIC_WITH_CC(s390_vistrbs);
8556   INTRINSIC_WITH_CC(s390_vistrhs);
8557   INTRINSIC_WITH_CC(s390_vistrfs);
8558
8559   INTRINSIC_WITH_CC(s390_vstrcbs);
8560   INTRINSIC_WITH_CC(s390_vstrchs);
8561   INTRINSIC_WITH_CC(s390_vstrcfs);
8562
8563   INTRINSIC_WITH_CC(s390_vstrczbs);
8564   INTRINSIC_WITH_CC(s390_vstrczhs);
8565   INTRINSIC_WITH_CC(s390_vstrczfs);
8566
8567   INTRINSIC_WITH_CC(s390_vfcedbs);
8568   INTRINSIC_WITH_CC(s390_vfchdbs);
8569   INTRINSIC_WITH_CC(s390_vfchedbs);
8570
8571   INTRINSIC_WITH_CC(s390_vftcidb);
8572
8573 #undef INTRINSIC_WITH_CC
8574
8575   default:
8576     return nullptr;
8577   }
8578 }
8579
8580 Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
8581                                              const CallExpr *E) {
8582   auto MakeLdg = [&](unsigned IntrinsicID) {
8583     Value *Ptr = EmitScalarExpr(E->getArg(0));
8584     AlignmentSource AlignSource;
8585     clang::CharUnits Align =
8586         getNaturalPointeeTypeAlignment(E->getArg(0)->getType(), &AlignSource);
8587     return Builder.CreateCall(
8588         CGM.getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(),
8589                                        Ptr->getType()}),
8590         {Ptr, ConstantInt::get(Builder.getInt32Ty(), Align.getQuantity())});
8591   };
8592   auto MakeScopedAtomic = [&](unsigned IntrinsicID) {
8593     Value *Ptr = EmitScalarExpr(E->getArg(0));
8594     return Builder.CreateCall(
8595         CGM.getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(),
8596                                        Ptr->getType()}),
8597         {Ptr, EmitScalarExpr(E->getArg(1))});
8598   };
8599   switch (BuiltinID) {
8600   case NVPTX::BI__nvvm_atom_add_gen_i:
8601   case NVPTX::BI__nvvm_atom_add_gen_l:
8602   case NVPTX::BI__nvvm_atom_add_gen_ll:
8603     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Add, E);
8604
8605   case NVPTX::BI__nvvm_atom_sub_gen_i:
8606   case NVPTX::BI__nvvm_atom_sub_gen_l:
8607   case NVPTX::BI__nvvm_atom_sub_gen_ll:
8608     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Sub, E);
8609
8610   case NVPTX::BI__nvvm_atom_and_gen_i:
8611   case NVPTX::BI__nvvm_atom_and_gen_l:
8612   case NVPTX::BI__nvvm_atom_and_gen_ll:
8613     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::And, E);
8614
8615   case NVPTX::BI__nvvm_atom_or_gen_i:
8616   case NVPTX::BI__nvvm_atom_or_gen_l:
8617   case NVPTX::BI__nvvm_atom_or_gen_ll:
8618     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Or, E);
8619
8620   case NVPTX::BI__nvvm_atom_xor_gen_i:
8621   case NVPTX::BI__nvvm_atom_xor_gen_l:
8622   case NVPTX::BI__nvvm_atom_xor_gen_ll:
8623     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Xor, E);
8624
8625   case NVPTX::BI__nvvm_atom_xchg_gen_i:
8626   case NVPTX::BI__nvvm_atom_xchg_gen_l:
8627   case NVPTX::BI__nvvm_atom_xchg_gen_ll:
8628     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Xchg, E);
8629
8630   case NVPTX::BI__nvvm_atom_max_gen_i:
8631   case NVPTX::BI__nvvm_atom_max_gen_l:
8632   case NVPTX::BI__nvvm_atom_max_gen_ll:
8633     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Max, E);
8634
8635   case NVPTX::BI__nvvm_atom_max_gen_ui:
8636   case NVPTX::BI__nvvm_atom_max_gen_ul:
8637   case NVPTX::BI__nvvm_atom_max_gen_ull:
8638     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UMax, E);
8639
8640   case NVPTX::BI__nvvm_atom_min_gen_i:
8641   case NVPTX::BI__nvvm_atom_min_gen_l:
8642   case NVPTX::BI__nvvm_atom_min_gen_ll:
8643     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::Min, E);
8644
8645   case NVPTX::BI__nvvm_atom_min_gen_ui:
8646   case NVPTX::BI__nvvm_atom_min_gen_ul:
8647   case NVPTX::BI__nvvm_atom_min_gen_ull:
8648     return MakeBinaryAtomicValue(*this, llvm::AtomicRMWInst::UMin, E);
8649
8650   case NVPTX::BI__nvvm_atom_cas_gen_i:
8651   case NVPTX::BI__nvvm_atom_cas_gen_l:
8652   case NVPTX::BI__nvvm_atom_cas_gen_ll:
8653     // __nvvm_atom_cas_gen_* should return the old value rather than the
8654     // success flag.
8655     return MakeAtomicCmpXchgValue(*this, E, /*ReturnBool=*/false);
8656
8657   case NVPTX::BI__nvvm_atom_add_gen_f: {
8658     Value *Ptr = EmitScalarExpr(E->getArg(0));
8659     Value *Val = EmitScalarExpr(E->getArg(1));
8660     // atomicrmw only deals with integer arguments so we need to use
8661     // LLVM's nvvm_atomic_load_add_f32 intrinsic for that.
8662     Value *FnALAF32 =
8663         CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_add_f32, Ptr->getType());
8664     return Builder.CreateCall(FnALAF32, {Ptr, Val});
8665   }
8666
8667   case NVPTX::BI__nvvm_atom_inc_gen_ui: {
8668     Value *Ptr = EmitScalarExpr(E->getArg(0));
8669     Value *Val = EmitScalarExpr(E->getArg(1));
8670     Value *FnALI32 =
8671         CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_inc_32, Ptr->getType());
8672     return Builder.CreateCall(FnALI32, {Ptr, Val});
8673   }
8674
8675   case NVPTX::BI__nvvm_atom_dec_gen_ui: {
8676     Value *Ptr = EmitScalarExpr(E->getArg(0));
8677     Value *Val = EmitScalarExpr(E->getArg(1));
8678     Value *FnALD32 =
8679         CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_dec_32, Ptr->getType());
8680     return Builder.CreateCall(FnALD32, {Ptr, Val});
8681   }
8682
8683   case NVPTX::BI__nvvm_ldg_c:
8684   case NVPTX::BI__nvvm_ldg_c2:
8685   case NVPTX::BI__nvvm_ldg_c4:
8686   case NVPTX::BI__nvvm_ldg_s:
8687   case NVPTX::BI__nvvm_ldg_s2:
8688   case NVPTX::BI__nvvm_ldg_s4:
8689   case NVPTX::BI__nvvm_ldg_i:
8690   case NVPTX::BI__nvvm_ldg_i2:
8691   case NVPTX::BI__nvvm_ldg_i4:
8692   case NVPTX::BI__nvvm_ldg_l:
8693   case NVPTX::BI__nvvm_ldg_ll:
8694   case NVPTX::BI__nvvm_ldg_ll2:
8695   case NVPTX::BI__nvvm_ldg_uc:
8696   case NVPTX::BI__nvvm_ldg_uc2:
8697   case NVPTX::BI__nvvm_ldg_uc4:
8698   case NVPTX::BI__nvvm_ldg_us:
8699   case NVPTX::BI__nvvm_ldg_us2:
8700   case NVPTX::BI__nvvm_ldg_us4:
8701   case NVPTX::BI__nvvm_ldg_ui:
8702   case NVPTX::BI__nvvm_ldg_ui2:
8703   case NVPTX::BI__nvvm_ldg_ui4:
8704   case NVPTX::BI__nvvm_ldg_ul:
8705   case NVPTX::BI__nvvm_ldg_ull:
8706   case NVPTX::BI__nvvm_ldg_ull2:
8707     // PTX Interoperability section 2.2: "For a vector with an even number of
8708     // elements, its alignment is set to number of elements times the alignment
8709     // of its member: n*alignof(t)."
8710     return MakeLdg(Intrinsic::nvvm_ldg_global_i);
8711   case NVPTX::BI__nvvm_ldg_f:
8712   case NVPTX::BI__nvvm_ldg_f2:
8713   case NVPTX::BI__nvvm_ldg_f4:
8714   case NVPTX::BI__nvvm_ldg_d:
8715   case NVPTX::BI__nvvm_ldg_d2:
8716     return MakeLdg(Intrinsic::nvvm_ldg_global_f);
8717
8718   case NVPTX::BI__nvvm_atom_cta_add_gen_i:
8719   case NVPTX::BI__nvvm_atom_cta_add_gen_l:
8720   case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
8721     return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_cta);
8722   case NVPTX::BI__nvvm_atom_sys_add_gen_i:
8723   case NVPTX::BI__nvvm_atom_sys_add_gen_l:
8724   case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
8725     return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_sys);
8726   case NVPTX::BI__nvvm_atom_cta_add_gen_f:
8727   case NVPTX::BI__nvvm_atom_cta_add_gen_d:
8728     return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_cta);
8729   case NVPTX::BI__nvvm_atom_sys_add_gen_f:
8730   case NVPTX::BI__nvvm_atom_sys_add_gen_d:
8731     return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_sys);
8732   case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
8733   case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
8734   case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
8735     return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_cta);
8736   case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
8737   case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
8738   case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
8739     return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_sys);
8740   case NVPTX::BI__nvvm_atom_cta_max_gen_i:
8741   case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
8742   case NVPTX::BI__nvvm_atom_cta_max_gen_l:
8743   case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
8744   case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
8745   case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
8746     return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_cta);
8747   case NVPTX::BI__nvvm_atom_sys_max_gen_i:
8748   case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
8749   case NVPTX::BI__nvvm_atom_sys_max_gen_l:
8750   case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
8751   case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
8752   case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
8753     return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_sys);
8754   case NVPTX::BI__nvvm_atom_cta_min_gen_i:
8755   case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
8756   case NVPTX::BI__nvvm_atom_cta_min_gen_l:
8757   case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
8758   case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
8759   case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
8760     return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_cta);
8761   case NVPTX::BI__nvvm_atom_sys_min_gen_i:
8762   case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
8763   case NVPTX::BI__nvvm_atom_sys_min_gen_l:
8764   case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
8765   case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
8766   case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
8767     return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_sys);
8768   case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
8769     return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_cta);
8770   case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
8771     return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_cta);
8772   case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
8773     return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_sys);
8774   case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
8775     return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_sys);
8776   case NVPTX::BI__nvvm_atom_cta_and_gen_i:
8777   case NVPTX::BI__nvvm_atom_cta_and_gen_l:
8778   case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
8779     return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_cta);
8780   case NVPTX::BI__nvvm_atom_sys_and_gen_i:
8781   case NVPTX::BI__nvvm_atom_sys_and_gen_l:
8782   case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
8783     return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_sys);
8784   case NVPTX::BI__nvvm_atom_cta_or_gen_i:
8785   case NVPTX::BI__nvvm_atom_cta_or_gen_l:
8786   case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
8787     return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_cta);
8788   case NVPTX::BI__nvvm_atom_sys_or_gen_i:
8789   case NVPTX::BI__nvvm_atom_sys_or_gen_l:
8790   case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
8791     return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_sys);
8792   case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
8793   case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
8794   case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
8795     return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_cta);
8796   case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
8797   case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
8798   case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
8799     return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_sys);
8800   case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
8801   case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
8802   case NVPTX::BI__nvvm_atom_cta_cas_gen_ll: {
8803     Value *Ptr = EmitScalarExpr(E->getArg(0));
8804     return Builder.CreateCall(
8805         CGM.getIntrinsic(
8806             Intrinsic::nvvm_atomic_cas_gen_i_cta,
8807             {Ptr->getType()->getPointerElementType(), Ptr->getType()}),
8808         {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
8809   }
8810   case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
8811   case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
8812   case NVPTX::BI__nvvm_atom_sys_cas_gen_ll: {
8813     Value *Ptr = EmitScalarExpr(E->getArg(0));
8814     return Builder.CreateCall(
8815         CGM.getIntrinsic(
8816             Intrinsic::nvvm_atomic_cas_gen_i_sys,
8817             {Ptr->getType()->getPointerElementType(), Ptr->getType()}),
8818         {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
8819   }
8820   default:
8821     return nullptr;
8822   }
8823 }
8824
8825 Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
8826                                                    const CallExpr *E) {
8827   switch (BuiltinID) {
8828   case WebAssembly::BI__builtin_wasm_current_memory: {
8829     llvm::Type *ResultType = ConvertType(E->getType());
8830     Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_current_memory, ResultType);
8831     return Builder.CreateCall(Callee);
8832   }
8833   case WebAssembly::BI__builtin_wasm_grow_memory: {
8834     Value *X = EmitScalarExpr(E->getArg(0));
8835     Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_grow_memory, X->getType());
8836     return Builder.CreateCall(Callee, X);
8837   }
8838
8839   default:
8840     return nullptr;
8841   }
8842 }